Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/1.6.x' into feat-bulk-operations

Browse Source
This commit is contained in:
Jake Barnby
2025-04-17 17:09:51 +12:00
parent 31cf1266e1 189f2e05e0
commit cd72d70432
33 changed files with 513 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1
View File
@@ -15,6 +15,7 @@ _APP_SYSTEM_TEAM_EMAIL=team@appwrite.io
_APP_EMAIL_SECURITY=security@appwrite.io
_APP_EMAIL_CERTIFICATES=certificates@appwrite.io
_APP_SYSTEM_RESPONSE_FORMAT=
_APP_CUSTOM_DOMAIN_DENY_LIST=
_APP_OPTIONS_ABUSE=disabled
_APP_OPTIONS_ROUTER_PROTECTION=disabled
_APP_OPTIONS_FORCE_HTTPS=disabled
Dockerfile
-1
View File
@@ -86,7 +86,6 @@ RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/worker-migrations && \
chmod +x /usr/local/bin/worker-webhooks && \
chmod +x /usr/local/bin/worker-stats-usage && \
chmod +x /usr/local/bin/worker-stats-usage-dump && \
chmod +x /usr/local/bin/stats-resources && \
chmod +x /usr/local/bin/worker-stats-resources
app/cli.php
+24 -6
View File
@@ -219,6 +219,13 @@ CLI::setResource('queueForCertificates', function (Publisher $publisher) {
}, ['publisher']);
CLI::setResource('logError', function (Registry $register) {
return function (Throwable $error, string $namespace, string $action) use ($register) {
Console::error('[Error] Timestamp: ' . date('c', time()));
Console::error('[Error] Type: ' . get_class($error));
Console::error('[Error] Message: ' . $error->getMessage());
Console::error('[Error] File: ' . $error->getFile());
Console::error('[Error] Line: ' . $error->getLine());
Console::error('[Error] Trace: ' . $error->getTraceAsString());
$logger = $register->get('logger');
if ($logger) {
@@ -237,6 +244,7 @@ CLI::setResource('logError', function (Registry $register) {
$log->addExtra('file', $error->getFile());
$log->addExtra('line', $error->getLine());
$log->addExtra('trace', $error->getTraceAsString());
$log->addExtra('detailedTrace', $error->getTrace());
$log->setAction($action);
@@ -250,24 +258,34 @@ CLI::setResource('logError', function (Registry $register) {
Console::error('Error pushing log: ' . $th->getMessage());
}
}
Console::warning("Failed: {$error->getMessage()}");
Console::warning($error->getTraceAsString());
};
}, ['register']);
CLI::setResource('executor', fn () => new Executor(fn (string $projectId, string $deploymentId) => System::getEnv('_APP_EXECUTOR_HOST')));
$platform = new Appwrite();
$platform->init(Service::TYPE_TASK);
$args = $platform->getEnv('argv');
if (!isset($args[0])) {
Console::error('Missing task name');
Console::exit(1);
}
\array_shift($args);
$taskName = $args[0];
$platform->init(Service::TYPE_TASK);
$cli = $platform->getCli();
$cli
->error()
->inject('error')
->action(function (Throwable $error) {
Console::error($error->getMessage());
->inject('logError')
->action(function (Throwable $error, callable $logError) use ($taskName) {
call_user_func_array($logError, [
$error,
'Task',
$taskName,
]);
});
$cli->run();
app/config/variables.php
+9
View File
@@ -79,6 +79,15 @@ return [
'question' => 'Enter your Appwrite hostname',
'filter' => ''
],
[
'name' => '_APP_CUSTOM_DOMAIN_DENY_LIST',
'description' => 'List of reserved or prohibited domains when configuring custom domains.',
'introduction' => '',
'default' => 'example.com,test.com,app.example.com',
'required' => false,
'question' => '',
'filter' => ''
],
[
'name' => '_APP_DOMAIN_FUNCTIONS',
'description' => 'A domain to use for function preview URLs. Setting to empty turns off function preview URLs.',
app/controllers/api/databases.php
+2 -11
View File
@@ -535,7 +535,6 @@ App::post('/v1/databases')
}
$queueForEvents->setParam('databaseId', $database->getId());
$queueForStatsUsage->addMetric(str_replace(['{databaseInternalId}'], [$database->getInternalId()], METRIC_DATABASE_ID_STORAGE), 1); // per database
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -831,9 +830,6 @@ App::delete('/v1/databases/:databaseId')
->setParam('databaseId', $database->getId())
->setPayload($response->output($database, Response::MODEL_DATABASE));
$queueForStatsUsage
->addMetric(METRIC_DATABASES_STORAGE, 1); // Global, deletion forces full recalculation
$response->noContent();
});
@@ -2733,9 +2729,6 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/attributes/:key
->setContext('database', $db)
->setPayload($response->output($attribute, $model));
$queueForStatsUsage
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$db->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
$response->noContent();
});
@@ -3460,8 +3453,7 @@ App::post('/v1/databases/:databaseId/collections/:collectionId/documents')
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, max($operations, 1))
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations)
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), $operations); // per collection
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
@@ -4402,8 +4394,7 @@ App::delete('/v1/databases/:databaseId/collections/:collectionId/documents/:docu
$queueForStatsUsage
->addMetric(METRIC_DATABASES_OPERATIONS_WRITES, 1)
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1)
->addMetric(str_replace(['{databaseInternalId}', '{collectionInternalId}'], [$database->getInternalId(), $collection->getInternalId()], METRIC_DATABASE_ID_COLLECTION_ID_STORAGE), 1); // per collection
->addMetric(str_replace('{databaseInternalId}', $database->getInternalId(), METRIC_DATABASE_ID_OPERATIONS_WRITES), 1); // per collection
$response->addHeader('X-Debug-Operations', 1);
app/controllers/api/health.php
-33
View File
@@ -756,38 +756,6 @@ App::get('/v1/health/queue/stats-usage')
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/queue/stats-usage-dump')
->desc('Get usage dump queue')
->groups(['api', 'health'])
->label('scope', 'health.read')
->label('sdk', new Method(
auth: [AuthType::KEY],
namespace: 'health',
name: 'getQueueStatsUsageDump',
description: '/docs/references/health/get-queue-stats-usage-dump.md',
responses: [
new SDKResponse(
code: Response::STATUS_CODE_OK,
model: Response::MODEL_HEALTH_QUEUE,
)
],
contentType: ContentType::JSON
))
->param('threshold', 5000, new Integer(true), 'Queue size threshold. When hit (equal or higher), endpoint returns server error. Default value is 5000.', true)
->inject('publisher')
->inject('response')
->action(function (int|string $threshold, Publisher $publisher, Response $response) {
$threshold = \intval($threshold);
$size = $publisher->getQueueSize(new Queue(Event::STATS_USAGE_DUMP_QUEUE_NAME));
if ($size >= $threshold) {
throw new Exception(Exception::HEALTH_QUEUE_SIZE_EXCEEDED, "Queue size threshold hit. Current size is {$size} and threshold is {$threshold}.");
}
$response->dynamic(new Document([ 'size' => $size ]), Response::MODEL_HEALTH_QUEUE);
});
App::get('/v1/health/storage/local')
->desc('Get local storage')
->groups(['api', 'health'])
@@ -954,7 +922,6 @@ App::get('/v1/health/queue/failed/:name')
Event::FUNCTIONS_QUEUE_NAME,
Event::STATS_RESOURCES_QUEUE_NAME,
Event::STATS_USAGE_QUEUE_NAME,
Event::STATS_USAGE_DUMP_QUEUE_NAME,
Event::WEBHOOK_QUEUE_NAME,
Event::CERTIFICATES_QUEUE_NAME,
Event::BUILDS_QUEUE_NAME,
app/controllers/api/migrations.php
+1
View File
@@ -408,6 +408,7 @@ App::get('/v1/migrations/appwrite/report')
->inject('project')
->inject('user')
->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) {
$appwrite = new Appwrite($projectID, $endpoint, $key);
try {
app/controllers/api/projects.php
+11 -3
View File
@@ -138,6 +138,14 @@ App::post('/v1/projects')
$databases = Config::getParam('pools-database', []);
if ($region !== 'default') {
$databaseKeys = System::getEnv('_APP_DATABASE_KEYS', '');
$keys = explode(',', $databaseKeys);
$databases = array_filter($keys, function ($value) use ($region) {
return str_contains($value, $region);
});
}
$databaseOverride = System::getEnv('_APP_DATABASE_OVERRIDE');
$index = \array_search($databaseOverride, $databases);
if ($index !== false) {
@@ -205,17 +213,17 @@ App::post('/v1/projects')
$dsn = new DSN('mysql://' . $dsn);
}
$adapter = $pools->get($dsn->getHost())->pop()->getResource();
$dbForProject = new Database($adapter, $cache);
$sharedTables = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES', ''));
$sharedTablesV1 = \explode(',', System::getEnv('_APP_DATABASE_SHARED_TABLES_V1', ''));
$projectTables = !\in_array($dsn->getHost(), $sharedTables);
$sharedTablesV1 = \in_array($dsn->getHost(), $sharedTablesV1);
$sharedTablesV2 = !$projectTables && !$sharedTablesV1;
$sharedTables = $sharedTablesV1 || $sharedTablesV2;
if (!$sharedTablesV2) {
$adapter = $pools->get($dsn->getHost())->pop()->getResource();
$dbForProject = new Database($adapter, $cache);
if ($sharedTables) {
$dbForProject
->setSharedTables(true)
app/controllers/api/proxy.php
+14 -3
View File
@@ -55,14 +55,25 @@ App::post('/v1/proxy/rules')
->inject('dbForPlatform')
->inject('dbForProject')
->action(function (string $domain, string $resourceType, string $resourceId, Response $response, Document $project, Certificate $queueForCertificates, Event $queueForEvents, Database $dbForPlatform, Database $dbForProject) {
$mainDomain = System::getEnv('_APP_DOMAIN', '');
if ($domain === $mainDomain) {
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'You cannot assign your main domain to specific resource. Please use subdomain or a different domain.');
}
$functionsDomain = System::getEnv('_APP_DOMAIN_FUNCTIONS', '');
if ($functionsDomain != '' && str_ends_with($domain, $functionsDomain)) {
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'You cannot assign your functions domain or it\'s subdomain to specific resource. Please use different domain.');
$functionsDomain = System::getEnv('_APP_DOMAIN_FUNCTIONS');
$denyListDomains = System::getEnv('_APP_CUSTOM_DOMAIN_DENY_LIST');
if (!empty($denyListDomains)) {
$functionsDomain .= ',' . $denyListDomains;
}
$deniedDomains = array_map('trim', explode(',', $functionsDomain));
foreach ($deniedDomains as $deniedDomain) {
if (str_ends_with($domain, $deniedDomain)) {
throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, 'You cannot assign your functions domain or its subdomain to a specific resource. Please use a different domain.');
}
}
if ($domain === 'localhost' || $domain === APP_HOSTNAME_INTERNAL) {
app/controllers/general.php
+10 -2
View File
@@ -72,11 +72,19 @@ function router(App $utopia, Database $dbForPlatform, callable $getProjectDB, Sw
}
if ($route->isEmpty()) {
if ($host === System::getEnv('_APP_DOMAIN_FUNCTIONS', '')) {
$appDomainFunctionsFallback = System::getEnv('_APP_DOMAIN_FUNCTIONS_FALLBACK', '');
$appDomainFunctions = System::getEnv('_APP_DOMAIN_FUNCTIONS', '');
if (!empty($appDomainFunctionsFallback) && \str_ends_with($host, $appDomainFunctionsFallback)) {
$appDomainFunctions = $appDomainFunctionsFallback;
}
if ($host === $appDomainFunctions) {
throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain cannot be used for security reasons. Please use any subdomain instead.');
}
if (\str_ends_with($host, System::getEnv('_APP_DOMAIN_FUNCTIONS', ''))) {
if (\str_ends_with($host, $appDomainFunctions)) {
throw new AppwriteException(AppwriteException::GENERAL_ACCESS_FORBIDDEN, 'This domain is not connected to any Appwrite resource yet. Please configure custom domain or function domain to allow this request.');
}
app/controllers/shared/api.php
+26 -24
View File
@@ -245,34 +245,36 @@ App::init()
subject: 'keys'
);
if ($dbKey) {
$accessedAt = $dbKey->getAttribute('accessedAt', '');
if (!$dbKey) {
throw new Exception(Exception::USER_UNAUTHORIZED);
}
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) {
$dbKey->setAttribute('accessedAt', DateTime::now());
$accessedAt = $dbKey->getAttribute('accessedAt', '');
if (DateTime::formatTz(DateTime::addSeconds(new \DateTime(), -APP_KEY_ACCESS)) > $accessedAt) {
$dbKey->setAttribute('accessedAt', DateTime::now());
$dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
}
$sdkValidator = new WhiteList($servers, true);
$sdk = $request->getHeader('x-sdk-name', 'UNKNOWN');
if ($sdkValidator->isValid($sdk)) {
$sdks = $dbKey->getAttribute('sdks', []);
if (!in_array($sdk, $sdks)) {
$sdks[] = $sdk;
$dbKey->setAttribute('sdks', $sdks);
/** Update access time as well */
$dbKey->setAttribute('accessedAt', Datetime::now());
$dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
}
$sdkValidator = new WhiteList($servers, true);
$sdk = $request->getHeader('x-sdk-name', 'UNKNOWN');
if ($sdkValidator->isValid($sdk)) {
$sdks = $dbKey->getAttribute('sdks', []);
if (!in_array($sdk, $sdks)) {
$sdks[] = $sdk;
$dbKey->setAttribute('sdks', $sdks);
/** Update access time as well */
$dbKey->setAttribute('accessedAt', Datetime::now());
$dbForPlatform->updateDocument('keys', $dbKey->getId(), $dbKey);
$dbForPlatform->purgeCachedDocument('projects', $project->getId());
}
}
$queueForAudits->setUser($user);
}
$queueForAudits->setUser($user);
}
} // Admin User Authentication
elseif (($project->getId() === 'console' && !$team->isEmpty() && !$user->isEmpty()) || ($project->getId() !== 'console' && !$user->isEmpty() && $mode === APP_MODE_ADMIN)) {
@@ -545,7 +547,7 @@ App::init()
$data = $cache->load($key, $timestamp);
if (!empty($data) && !$cacheLog->isEmpty()) {
$parts = explode('/', $cacheLog->getAttribute('resourceType'));
$parts = explode('/', $cacheLog->getAttribute('resourceType', ''));
$type = $parts[0] ?? null;
if ($type === 'bucket' && (!$isImageTransformation || !$isDisabled)) {
app/views/install/compose.phtml
+1 -29
View File
@@ -168,7 +168,7 @@ $image = $this->getParam('image', '');
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: <?php echo $organization; ?>/console:5.2.53
image: <?php echo $organization; ?>/console:5.2.58
restart: unless-stopped
networks:
- appwrite
@@ -744,34 +744,6 @@ $image = $this->getParam('image', '');
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-worker-stats-usage-dump:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: worker-stats-usage-dump
<<: *x-logging
container_name: appwrite-worker-stats-usage-dump
restart: unless-stopped
networks:
- appwrite
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
appwrite-task-scheduler-functions:
image: <?php echo $organization; ?>/<?php echo $image; ?>:<?php echo $version."\n"; ?>
entrypoint: schedule-functions
app/worker.php
-5
View File
@@ -15,7 +15,6 @@ use Appwrite\Event\Messaging;
use Appwrite\Event\Migration;
use Appwrite\Event\Realtime;
use Appwrite\Event\StatsUsage;
use Appwrite\Event\StatsUsageDump;
use Appwrite\Event\Webhook;
use Appwrite\Platform\Appwrite;
use Executor\Executor;
@@ -279,10 +278,6 @@ Server::setResource('queueForStatsUsage', function (Publisher $publisher) {
return new StatsUsage($publisher);
}, ['publisher']);
Server::setResource('queueForStatsUsageDump', function (Publisher $publisher) {
return new StatsUsageDump($publisher);
}, ['publisher']);
Server::setResource('queueForDatabase', function (Publisher $publisher) {
return new EventDatabase($publisher);
}, ['publisher']);
bin/worker-stats-usage-dump
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/worker.php stats-usage-dump $@
composer.json
+1 -1
View File
@@ -51,7 +51,7 @@
"utopia-php/cache": "0.12.*",
"utopia-php/cli": "0.15.*",
"utopia-php/config": "0.2.*",
"utopia-php/database": "0.64.*",
"utopia-php/database": "0.65.*",
"utopia-php/domains": "0.5.*",
"utopia-php/dsn": "0.2.1",
"utopia-php/framework": "0.33.*",
composer.lock
Generated
+36 -36
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "6a54c8bc4f9f14cd3883f55880864630",
"content-hash": "51ff891ef6cee8a3f8c4e5187b7fd479",
"packages": [
{
"name": "adhocore/jwt",
@@ -1365,16 +1365,16 @@
},
{
"name": "open-telemetry/sdk",
"version": "1.2.3",
"version": "1.2.4",
"source": {
"type": "git",
"url": "https://github.com/opentelemetry-php/sdk.git",
"reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc"
"reference": "47fcb66ae5328c5a799195247b1dce551d85873e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/0e7804c176c4b09d95b7985400aa38ce544cb7fc",
"reference": "0e7804c176c4b09d95b7985400aa38ce544cb7fc",
"url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/47fcb66ae5328c5a799195247b1dce551d85873e",
"reference": "47fcb66ae5328c5a799195247b1dce551d85873e",
"shasum": ""
},
"require": {
@@ -1451,7 +1451,7 @@
"issues": "https://github.com/open-telemetry/opentelemetry-php/issues",
"source": "https://github.com/open-telemetry/opentelemetry-php"
},
"time": "2025-04-08T09:55:41+00:00"
"time": "2025-04-15T07:02:07+00:00"
},
{
"name": "open-telemetry/sem-conv",
@@ -3351,16 +3351,16 @@
},
{
"name": "utopia-php/cli",
"version": "0.15.1",
"version": "0.15.2",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/cli.git",
"reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65"
"reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/d69bbe51a6a94dc4e5bcdd542b5938038b985a65",
"reference": "d69bbe51a6a94dc4e5bcdd542b5938038b985a65",
"url": "https://api.github.com/repos/utopia-php/cli/zipball/da00ff6b8b29a826a1794002ae43442cdf3a0f5f",
"reference": "da00ff6b8b29a826a1794002ae43442cdf3a0f5f",
"shasum": ""
},
"require": {
@@ -3394,9 +3394,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/cli/issues",
"source": "https://github.com/utopia-php/cli/tree/0.15.1"
"source": "https://github.com/utopia-php/cli/tree/0.15.2"
},
"time": "2024-10-04T13:55:36+00:00"
"time": "2025-04-15T10:08:48+00:00"
},
{
"name": "utopia-php/compression",
@@ -3497,16 +3497,16 @@
},
{
"name": "utopia-php/database",
"version": "0.64.1",
"version": "0.65.0",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/database.git",
"reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b"
"reference": "e589efdc5da1216523a758e8af358866d4fb563f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/database/zipball/6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b",
"reference": "6530a8a6d3c1fe92d0f9a92f0f05eda698d92e0b",
"url": "https://api.github.com/repos/utopia-php/database/zipball/e589efdc5da1216523a758e8af358866d4fb563f",
"reference": "e589efdc5da1216523a758e8af358866d4fb563f",
"shasum": ""
},
"require": {
@@ -3547,9 +3547,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/database/issues",
"source": "https://github.com/utopia-php/database/tree/0.64.1"
"source": "https://github.com/utopia-php/database/tree/0.65.0"
},
"time": "2025-04-02T00:35:29+00:00"
"time": "2025-04-14T07:39:01+00:00"
},
{
"name": "utopia-php/domains",
@@ -3660,16 +3660,16 @@
},
{
"name": "utopia-php/fetch",
"version": "0.4.0",
"version": "0.4.1",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/fetch.git",
"reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e"
"reference": "65095dac14037db0c822fb5e209e5bd3187a0303"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/fetch/zipball/46e791ff6a95864517750b9df6bbf4a17e3c9c4e",
"reference": "46e791ff6a95864517750b9df6bbf4a17e3c9c4e",
"url": "https://api.github.com/repos/utopia-php/fetch/zipball/65095dac14037db0c822fb5e209e5bd3187a0303",
"reference": "65095dac14037db0c822fb5e209e5bd3187a0303",
"shasum": ""
},
"require": {
@@ -3693,9 +3693,9 @@
"description": "A simple library that provides an interface for making HTTP Requests.",
"support": {
"issues": "https://github.com/utopia-php/fetch/issues",
"source": "https://github.com/utopia-php/fetch/tree/0.4.0"
"source": "https://github.com/utopia-php/fetch/tree/0.4.1"
},
"time": "2025-03-11T21:06:56+00:00"
"time": "2025-04-14T07:34:27+00:00"
},
{
"name": "utopia-php/framework",
@@ -3951,16 +3951,16 @@
},
{
"name": "utopia-php/migration",
"version": "0.8.5",
"version": "0.8.6",
"source": {
"type": "git",
"url": "https://github.com/utopia-php/migration.git",
"reference": "0dd95b148c581579ec05d2abbbdc13c2b4702331"
"reference": "84163e16edc0b2e64c34ad7b7c4cc5f05d762daf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/0dd95b148c581579ec05d2abbbdc13c2b4702331",
"reference": "0dd95b148c581579ec05d2abbbdc13c2b4702331",
"url": "https://api.github.com/repos/utopia-php/migration/zipball/84163e16edc0b2e64c34ad7b7c4cc5f05d762daf",
"reference": "84163e16edc0b2e64c34ad7b7c4cc5f05d762daf",
"shasum": ""
},
"require": {
@@ -4001,9 +4001,9 @@
],
"support": {
"issues": "https://github.com/utopia-php/migration/issues",
"source": "https://github.com/utopia-php/migration/tree/0.8.5"
"source": "https://github.com/utopia-php/migration/tree/0.8.6"
},
"time": "2025-04-09T05:21:09+00:00"
"time": "2025-04-14T08:22:09+00:00"
},
{
"name": "utopia-php/orchestration",
@@ -4767,16 +4767,16 @@
"packages-dev": [
{
"name": "appwrite/sdk-generator",
"version": "0.40.11",
"version": "0.40.12",
"source": {
"type": "git",
"url": "https://github.com/appwrite/sdk-generator.git",
"reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027"
"reference": "182ec17848f81b78c336379bac94ff92b7a73365"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/0ec5f4a60c15e33e208bc3444ba6148b1d0f0027",
"reference": "0ec5f4a60c15e33e208bc3444ba6148b1d0f0027",
"url": "https://api.github.com/repos/appwrite/sdk-generator/zipball/182ec17848f81b78c336379bac94ff92b7a73365",
"reference": "182ec17848f81b78c336379bac94ff92b7a73365",
"shasum": ""
},
"require": {
@@ -4812,9 +4812,9 @@
"description": "Appwrite PHP library for generating API SDKs for multiple programming languages and platforms",
"support": {
"issues": "https://github.com/appwrite/sdk-generator/issues",
"source": "https://github.com/appwrite/sdk-generator/tree/0.40.11"
"source": "https://github.com/appwrite/sdk-generator/tree/0.40.12"
},
"time": "2025-03-26T10:53:16+00:00"
"time": "2025-04-02T23:36:11+00:00"
},
{
"name": "doctrine/annotations",
docker-compose.yml
+2 -33
View File
@@ -198,13 +198,14 @@ services:
- _APP_DATABASE_SHARED_TABLES_V1
- _APP_DATABASE_SHARED_NAMESPACE
- _APP_FUNCTIONS_CREATION_ABUSE_LIMIT
- _APP_CUSTOM_DOMAIN_DENY_LIST
extra_hosts:
- "host.docker.internal:host-gateway"
appwrite-console:
<<: *x-logging
container_name: appwrite-console
image: appwrite/console:5.2.53
image: appwrite/console:5.2.58
restart: unless-stopped
networks:
- appwrite
@@ -827,38 +828,6 @@ services:
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_DATABASE_SHARED_TABLES
appwrite-worker-stats-usage-dump:
entrypoint: worker-stats-usage-dump
<<: *x-logging
container_name: appwrite-worker-stats-usage-dump
image: appwrite-dev
networks:
- appwrite
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
- _APP_DB_PORT
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_USAGE_STATS
- _APP_LOGGING_CONFIG
- _APP_USAGE_AGGREGATION_INTERVAL
- _APP_DATABASE_SHARED_TABLES
- _APP_STATS_USAGE_DUAL_WRITING_DBS
appwrite-task-scheduler-functions:
entrypoint: schedule-functions
<<: *x-logging
src/Appwrite/Event/Event.php
+2 -11
View File
@@ -24,23 +24,12 @@ class Event
public const FUNCTIONS_QUEUE_NAME = 'v1-functions';
public const FUNCTIONS_CLASS_NAME = 'FunctionsV1';
/** remove */
public const USAGE_QUEUE_NAME = 'v1-usage';
public const USAGE_CLASS_NAME = 'UsageV1';
public const USAGE_DUMP_QUEUE_NAME = 'v1-usage-dump';
public const USAGE_DUMP_CLASS_NAME = 'UsageDumpV1';
/** /remove */
public const STATS_RESOURCES_QUEUE_NAME = 'v1-stats-resources';
public const STATS_RESOURCES_CLASS_NAME = 'StatsResourcesV1';
public const STATS_USAGE_QUEUE_NAME = 'v1-stats-usage';
public const STATS_USAGE_CLASS_NAME = 'StatsUsageV1';
public const STATS_USAGE_DUMP_QUEUE_NAME = 'v1-stats-usage-dump';
public const STATS_USAGE_DUMP_CLASS_NAME = 'StatsUsageDumpV1';
public const WEBHOOK_QUEUE_NAME = 'v1-webhooks';
public const WEBHOOK_CLASS_NAME = 'WebhooksV1';
@@ -351,6 +340,7 @@ class Event
*/
public function trigger(): string|bool
{
if ($this->paused) {
return false;
}
@@ -360,6 +350,7 @@ class Event
// Merge the base payload with any trimmed values
$payload = array_merge($this->preparePayload(), $this->trimPayload());
return $this->publisher->enqueue($queue, $payload);
}
src/Appwrite/Event/StatsUsageDump.php
-44
View File
@@ -1,44 +0,0 @@
<?php
namespace Appwrite\Event;
use Utopia\Queue\Publisher;
class StatsUsageDump extends Event
{
protected array $stats;
public function __construct(protected Publisher $publisher)
{
parent::__construct($publisher);
$this
->setQueue(Event::STATS_USAGE_DUMP_QUEUE_NAME)
->setClass(Event::STATS_USAGE_DUMP_CLASS_NAME);
}
/**
* Add Stats.
*
* @param array $stats
* @return self
*/
public function setStats(array $stats): self
{
$this->stats = $stats;
return $this;
}
/**
* Prepare the payload for the usage dump event.
*
* @return array
*/
protected function preparePayload(): array
{
return [
'stats' => $this->stats,
];
}
}
src/Appwrite/Migration/Version/V21.php
+16 -1
View File
@@ -82,6 +82,14 @@ class V21 extends Migration
Console::warning("'type' from {$id}: {$th->getMessage()}");
}
break;
case 'migrations':
// Create destination attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'destination');
} catch (Throwable $th) {
Console::warning("'destination' from {$id}: {$th->getMessage()}");
}
break;
case 'schedules':
// Create data attribute
try {
@@ -91,7 +99,14 @@ class V21 extends Migration
}
break;
case 'databases':
// Create originalId attribute
try {
$this->createAttributeFromCollection($this->projectDB, $id, 'originalId');
} catch (Throwable $th) {
Console::warning("'originalId' from {$id}: {$th->getMessage()}");
}
break;
case 'functions':
// Create scopes attribute
try {
src/Appwrite/Platform/Services/Workers.php
-2
View File
@@ -13,7 +13,6 @@ use Appwrite\Platform\Workers\Messaging;
use Appwrite\Platform\Workers\Migrations;
use Appwrite\Platform\Workers\StatsResources;
use Appwrite\Platform\Workers\StatsUsage;
use Appwrite\Platform\Workers\StatsUsageDump;
use Appwrite\Platform\Workers\Webhooks;
use Utopia\Platform\Service;
@@ -32,7 +31,6 @@ class Workers extends Service
->addAction(Mails::getName(), new Mails())
->addAction(Messaging::getName(), new Messaging())
->addAction(Webhooks::getName(), new Webhooks())
->addAction(StatsUsageDump::getName(), new StatsUsageDump())
->addAction(StatsUsage::getName(), new StatsUsage())
->addAction(Migrations::getName(), new Migrations())
->addAction(StatsResources::getName(), new StatsResources())
src/Appwrite/Platform/Tasks/Maintenance.php
+14 -9
View File
@@ -47,15 +47,20 @@ class Maintenance extends Action
Console::info("[{$time}] Notifying workers with maintenance tasks every {$interval} seconds");
$dbForPlatform->foreach('projects', function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes
->setType(DELETE_TYPE_MAINTENANCE)
->setProject($project)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
}, [
Query::limit(100),
]);
$dbForPlatform->foreach(
'projects',
function (Document $project) use ($queueForDeletes, $usageStatsRetentionHourly) {
$queueForDeletes
->setType(DELETE_TYPE_MAINTENANCE)
->setProject($project)
->setUsageRetentionHourlyDateTime(DateTime::addSeconds(new \DateTime(), -1 * $usageStatsRetentionHourly))
->trigger();
},
[
Query::equal('region', [System::getEnv('_APP_REGION', 'default')]),
Query::limit(100),
]
);
$queueForDeletes
->setType(DELETE_TYPE_MAINTENANCE)
src/Appwrite/Platform/Tasks/StatsResources.php
+2 -1
View File
@@ -67,7 +67,8 @@ class StatsResources extends Action
* For each project that were accessed in last 24 hours
*/
$this->foreachDocument($this->dbForPlatform, 'projects', [
Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours))
Query::greaterThanEqual('accessedAt', DateTime::format($last24Hours)),
Query::equal('region', [System::getEnv('_APP_REGION', 'default')])
], function ($project) use ($queue) {
$queue
->setProject($project)
src/Appwrite/Platform/Workers/Databases.php
+6 -9
View File
@@ -563,22 +563,19 @@ class Databases extends Action
$start = \microtime(true);
try {
$documents = $database->deleteDocuments($collectionId, $queries);
$count = $database->deleteDocuments(
$collectionId,
$queries,
Database::DELETE_BATCH_SIZE,
$callback
);
} catch (\Throwable $th) {
$tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collectionId} {$tenant} :{$th->getMessage()}");
return;
}
if (\is_callable($callback)) {
foreach ($documents as $document) {
$callback($document);
}
}
$end = \microtime(true);
$count = \count($documents);
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
}
src/Appwrite/Platform/Workers/Deletes.php
+19 -22
View File
@@ -494,21 +494,22 @@ class Deletes extends Action
}
/**
* @param Database $dbForPlatform
* @param Document $document
* @return void
* @throws Authorization
* @throws DatabaseException
* @throws Conflict
* @throws Restricted
* @throws Structure
* @throws Exception
*/
private function deleteProjectsByTeam(Database $dbForPlatform, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
* @param Database $dbForPlatform
* @param Document $document
* @return void
* @throws Authorization
* @throws DatabaseException
* @throws Conflict
* @throws Restricted
* @throws Structure
* @throws Exception
*/
protected function deleteProjectsByTeam(Database $dbForPlatform, callable $getProjectDB, CertificatesAdapter $certificates, Document $document): void
{
$projects = $dbForPlatform->find('projects', [
Query::equal('teamInternalId', [$document->getInternalId()])
Query::equal('teamInternalId', [$document->getInternalId()]),
Query::equal('region', [System::getEnv('_APP_REGION', 'default')])
]);
foreach ($projects as $project) {
@@ -1049,24 +1050,20 @@ class Deletes extends Action
/**
* deleteDocuments uses a cursor, we need to add a unique order by field or use default
*/
try {
$documents = $database->deleteDocuments($collection, $queries);
$count = $database->deleteDocuments(
$collection,
$queries,
Database::DELETE_BATCH_SIZE,
$callback
);
} catch (Throwable $th) {
$tenant = $database->getSharedTables() ? 'Tenant:'.$database->getTenant() : '';
Console::error("Failed to delete documents for collection:{$database->getNamespace()}_{$collection} {$tenant} :{$th->getMessage()}");
return;
}
if (\is_callable($callback)) {
foreach ($documents as $document) {
$callback($document);
}
}
$end = \microtime(true);
$count = \count($documents);
Console::info("Deleted {$count} documents by group in " . ($end - $start) . " seconds");
}
src/Appwrite/Platform/Workers/StatsResources.php
-1
View File
@@ -70,7 +70,6 @@ class StatsResources extends Action
}
if (empty($project->getAttribute('database'))) {
var_dump($payload);
return;
}
src/Appwrite/Platform/Workers/StatsUsage.php
+208 -10
View File
@@ -2,17 +2,22 @@
namespace Appwrite\Platform\Workers;
use Appwrite\Event\StatsUsageDump;
use Exception;
use Throwable;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
use Utopia\Queue\Message;
use Utopia\Registry\Registry;
use Utopia\System\System;
class StatsUsage extends Action
{
/**
* In memory per project metrics calculation
*/
private array $stats = [];
private int $lastTriggeredTime = 0;
private int $keys = 0;
@@ -20,6 +25,77 @@ class StatsUsage extends Action
private const BATCH_SIZE_DEVELOPMENT = 1;
private const BATCH_SIZE_PRODUCTION = 10_000;
/**
* Stats for batch write separated per project
* @var array
*/
private array $projects = [];
/**
* Array of stat documents to batch write to logsDB
* @var array
*/
private array $statDocuments = [];
protected Registry $register;
/**
* Metrics to skip writing to logsDB
* As these metrics are calculated separately
* by logs DB
* @var array
*/
protected array $skipBaseMetrics = [
METRIC_DATABASES => true,
METRIC_BUCKETS => true,
METRIC_USERS => true,
METRIC_FUNCTIONS => true,
METRIC_TEAMS => true,
METRIC_MESSAGES => true,
METRIC_MAU => true,
METRIC_WEBHOOKS => true,
METRIC_PLATFORMS => true,
METRIC_PROVIDERS => true,
METRIC_TOPICS => true,
METRIC_KEYS => true,
METRIC_FILES => true,
METRIC_FILES_STORAGE => true,
METRIC_DEPLOYMENTS_STORAGE => true,
METRIC_BUILDS_STORAGE => true,
METRIC_DEPLOYMENTS => true,
METRIC_BUILDS => true,
METRIC_COLLECTIONS => true,
METRIC_DOCUMENTS => true,
METRIC_DATABASES_STORAGE => true,
];
/**
* Skip metrics associated with parent IDs
* these need to be checked individually with `str_ends_with`
*/
protected array $skipParentIdMetrics = [
'.files',
'.files.storage',
'.collections',
'.documents',
'.deployments',
'.deployments.storage',
'.builds',
'.builds.storage',
'.databases.storage'
];
/**
* @var callable(): Database
*/
protected mixed $getLogsDB;
protected array $periods = [
'1h' => 'Y-m-d H:00',
'1d' => 'Y-m-d 00:00',
'inf' => '0000-00-00 00:00'
];
public static function getName(): string
{
return 'stats-usage';
@@ -41,7 +117,8 @@ class StatsUsage extends Action
->desc('Stats usage worker')
->inject('message')
->inject('getProjectDB')
->inject('queueForStatsUsageDump')
->inject('getLogsDB')
->inject('register')
->callback([$this, 'action']);
$this->lastTriggeredTime = time();
@@ -49,14 +126,17 @@ class StatsUsage extends Action
/**
* @param Message $message
* @param callable $getProjectDB
* @param StatsUsageDump $queueForStatsUsageDump
* @param callable(): Database $getProjectDB
* @param callable(): Database $getLogsDB
* @param Registry $register
* @return void
* @throws \Utopia\Database\Exception
* @throws Exception
*/
public function action(Message $message, callable $getProjectDB, StatsUsageDump $queueForStatsUsageDump): void
public function action(Message $message, callable $getProjectDB, callable $getLogsDB, Registry $register): void
{
$this->getLogsDB = $getLogsDB;
$this->register = $register;
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
@@ -98,9 +178,7 @@ class StatsUsage extends Action
) {
Console::warning('[' . DateTime::now() . '] Aggregated ' . $this->keys . ' keys');
$queueForStatsUsageDump
->setStats($this->stats)
->trigger();
$this->commitToDB($getProjectDB);
$this->stats = [];
$this->keys = 0;
@@ -114,7 +192,7 @@ class StatsUsage extends Action
* @param Document $project
* @param Document $document
* @param array $metrics
* @param callable $getProjectDB
* @param callable(): Database $getProjectDB
* @return void
*/
private function reduce(Document $project, Document $document, array &$metrics, callable $getProjectDB): void
@@ -246,8 +324,128 @@ class StatsUsage extends Action
default:
break;
}
} catch (\Throwable $e) {
} catch (Throwable $e) {
console::error("[reducer] " . " {DateTime::now()} " . " {$project->getInternalId()} " . " {$e->getMessage()}");
}
}
/**
* Commit stats to DB
* @param callable(): Database $getProjectDB
* @return void
*/
public function commitToDb(callable $getProjectDB): void
{
foreach ($this->stats as $stats) {
$project = $stats['project'] ?? new Document([]);
$numberOfKeys = !empty($stats['keys']) ? count($stats['keys']) : 0;
$receivedAt = $stats['receivedAt'] ?? null;
if ($numberOfKeys === 0) {
continue;
}
console::log('['.DateTime::now().'] Id: '.$project->getId(). ' InternalId: '.$project->getInternalId(). ' Db: '.$project->getAttribute('database').' ReceivedAt: '.$receivedAt. ' Keys: '.$numberOfKeys);
try {
foreach ($stats['keys'] ?? [] as $key => $value) {
if ($value == 0) {
continue;
}
foreach ($this->periods as $period => $format) {
$time = null;
if ($period !== 'inf') {
$time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time());
}
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$this->projects[$project->getInternalId()]['project'] = new Document([
'$id' => $project->getId(),
'$internalId' => $project->getInternalId(),
'database' => $project->getAttribute('database'),
]);
$this->projects[$project->getInternalId()]['stats'][] = $document;
$this->prepareForLogsDB($project, $document);
}
}
} catch (Exception $e) {
console::error('[' . DateTime::now() . '] project [' . $project->getInternalId() . '] database [' . $project['database'] . '] ' . ' ' . $e->getMessage());
}
}
foreach ($this->projects as $internalId => $projectStats) {
if (empty($internalId)) {
continue;
}
try {
$dbForProject = $getProjectDB($projectStats['project']);
Console::log('Processing batch with ' . count($projectStats['stats']) . ' stats');
$dbForProject->createOrUpdateDocumentsWithIncrease('stats', 'value', $projectStats['stats']);
Console::success('Batch successfully written to DB');
unset($this->projects[$internalId]);
} catch (Throwable $e) {
Console::error('Error processing stats: ' . $e->getMessage());
}
}
$this->writeToLogsDB();
}
protected function prepareForLogsDB(Document $project, Document $stat)
{
if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') {
return;
}
if (array_key_exists($stat->getAttribute('metric'), $this->skipBaseMetrics)) {
return;
}
foreach ($this->skipParentIdMetrics as $skipMetric) {
if (str_ends_with($stat->getAttribute('metric'), $skipMetric)) {
return;
}
}
$documentClone = clone $stat;
$documentClone->setAttribute('$tenant', (int) $project->getInternalId());
$this->statDocuments[] = $documentClone;
}
protected function writeToLogsDB(): void
{
if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') {
Console::log('Dual Writing is disabled. Skipping...');
return;
}
$dbForLogs = call_user_func($this->getLogsDB);
$dbForLogs
->setTenant(null)
->setTenantPerDocument(true);
try {
Console::log('Processing batch with ' . count($this->statDocuments) . ' stats');
$dbForLogs->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
$this->statDocuments
);
Console::success('Usage logs pushed to Logs DB');
} catch (Throwable $th) {
Console::error($th->getMessage());
}
$this->register->get('pools')->get('logs')->reclaim();
}
}
src/Appwrite/Platform/Workers/StatsUsageDump.php
+4 -158
View File
@@ -4,7 +4,6 @@ namespace Appwrite\Platform\Workers;
use Appwrite\Extend\Exception;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Platform\Action;
@@ -12,6 +11,9 @@ use Utopia\Queue\Message;
use Utopia\Registry\Registry;
use Utopia\System\System;
/**
* TODO remove later
*/
class StatsUsageDump extends Action
{
public const METRIC_COLLECTION_LEVEL_STORAGE = 4;
@@ -135,11 +137,6 @@ class StatsUsageDump extends Action
}
if (str_contains($key, METRIC_DATABASES_STORAGE)) {
try {
$this->handleDatabaseStorage($key, $dbForProject, $project, $receivedAt);
} catch (\Exception $e) {
console::error('[' . DateTime::now() . '] failed to calculate database storage for key [' . $key . '] ' . $e->getMessage());
}
continue;
}
@@ -160,7 +157,7 @@ class StatsUsageDump extends Action
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$documentClone = clone $document;
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
@@ -177,157 +174,6 @@ class StatsUsageDump extends Action
}
}
private function handleDatabaseStorage(string $key, Database $dbForProject, Document $project, string $receivedAt): void
{
$data = explode('.', $key);
$start = microtime(true);
$updateMetric = function (Database $dbForProject, Document $project, int $value, string $key, string $period, string|null $time) use ($receivedAt) {
$id = \md5("{$time}_{$period}_{$key}");
$document = new Document([
'$id' => $id,
'period' => $period,
'time' => $time,
'metric' => $key,
'value' => $value,
'region' => System::getEnv('_APP_REGION', 'default'),
]);
$documentClone = new Document($document->getArrayCopy());
$dbForProject->createOrUpdateDocumentsWithIncrease(
'stats',
'value',
[$document]
);
$this->writeToLogsDB($project, $documentClone);
};
foreach ($this->periods as $period => $format) {
$time = null;
if ($period !== 'inf') {
$time = !empty($receivedAt) ? (new \DateTime($receivedAt))->format($format) : date($format, time());
}
$id = \md5("{$time}_{$period}_{$key}");
$value = 0;
$previousValue = 0;
try {
$previousValue = ($dbForProject->getDocument('stats', $id))->getAttribute('value', 0);
} catch (\Exception $e) {
// No previous value
}
switch (count($data)) {
// Collection Level
case self::METRIC_COLLECTION_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Collection Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collectionInternalId = $data[1];
try {
$value = $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collectionInternalId);
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
// Compare with previous value
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Collection
$updateMetric($dbForProject, $project, $diff, $key, $period, $time);
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Database Level
case self::METRIC_DATABASE_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Database Level Storage Calculation [' . $key . ']');
$databaseInternalId = $data[0];
$collections = [];
try {
$collections = $dbForProject->find('database_' . $databaseInternalId);
} catch (\Exception $e) {
// Database not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $databaseInternalId . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
$diff = $value - $previousValue;
if ($diff === 0) {
break;
}
// Update Database
$databaseKey = str_replace(['{databaseInternalId}'], [$data[0]], METRIC_DATABASE_ID_STORAGE);
$updateMetric($dbForProject, $project, $diff, $databaseKey, $period, $time);
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
// Project Level
case self::METRIC_PROJECT_LEVEL_STORAGE:
Console::log('[' . DateTime::now() . '] Project Level Storage Calculation [' . $key . ']');
// Get all project databases
$databases = $dbForProject->find('database');
// Recalculate all databases
foreach ($databases as $database) {
$collections = $dbForProject->find('database_' . $database->getInternalId());
foreach ($collections as $collection) {
try {
$value += $dbForProject->getSizeOfCollection('database_' . $database->getInternalId() . '_collection_' . $collection->getInternalId());
} catch (\Exception $e) {
// Collection not found
if ($e->getMessage() !== 'Collection not found') {
throw $e;
}
}
}
}
$diff = $value - $previousValue;
// Update Project
$projectKey = METRIC_DATABASES_STORAGE;
$updateMetric($dbForProject, $project, $diff, $projectKey, $period, $time);
break;
}
}
$end = microtime(true);
console::log('[' . DateTime::now() . '] DB Storage Calculation [' . $key . '] took ' . (($end - $start) * 1000) . ' milliseconds');
}
protected function writeToLogsDB(Document $project, Document $document): void
{
if (System::getEnv('_APP_STATS_USAGE_DUAL_WRITING', 'disabled') === 'disabled') {
src/Appwrite/Platform/Workers/Webhooks.php
+2
View File
@@ -54,6 +54,8 @@ class Webhooks extends Action
$this->errors = [];
$payload = $message->getPayload() ?? [];
if (empty($payload)) {
throw new Exception('Missing payload');
}
tests/e2e/General/UsageTest.php
+3 -253
View File
@@ -6,7 +6,6 @@ use Appwrite\Functions\Specification;
use Appwrite\Tests\Retry;
use CURLFile;
use DateTime;
use PHPUnit\Framework\ExpectationFailedException;
use Tests\E2E\Client;
use Tests\E2E\Scopes\ProjectCustom;
use Tests\E2E\Scopes\Scope;
@@ -148,7 +147,7 @@ class UsageTest extends Scope
);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(31, count($response['body']));
$this->assertGreaterThanOrEqual(31, count($response['body']));
$this->validateDates($response['body']['network']);
$this->validateDates($response['body']['requests']);
$this->validateDates($response['body']['users']);
@@ -327,7 +326,7 @@ class UsageTest extends Scope
]
);
$this->assertEquals(31, count($response['body']));
$this->assertGreaterThanOrEqual(31, count($response['body']));
$this->assertEquals(1, count($response['body']['requests']));
$this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']);
$this->validateDates($response['body']['requests']);
@@ -548,7 +547,7 @@ class UsageTest extends Scope
]
);
$this->assertEquals(31, count($response['body']));
$this->assertGreaterThanOrEqual(31, count($response['body']));
$this->assertEquals(1, count($response['body']['requests']));
$this->assertEquals(1, count($response['body']['network']));
$this->assertEquals($requestsTotal, $response['body']['requests'][array_key_last($response['body']['requests'])]['value']);
@@ -593,255 +592,6 @@ class UsageTest extends Scope
return $data;
}
public function testDatabaseStoragePrepare(): array
{
$response = $this->client->call(
Client::METHOD_POST,
'/databases',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()),
[
'databaseId' => 'unique()',
'name' => 'dbStorageStats',
]
);
$this->assertNotEmpty($response['body']['$id']);
$databaseId = $response['body']['$id'];
$response = $this->client->call(
Client::METHOD_POST,
'/databases/' . $databaseId . '/collections',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()),
[
'collectionId' => 'unique()',
'name' => 'collectionStorageStats',
'documentSecurity' => false,
'permissions' => [
Permission::read(Role::any()),
Permission::create(Role::any()),
Permission::update(Role::any()),
Permission::delete(Role::any()),
],
]
);
$this->assertNotEmpty($response['body']['$id']);
$collectionId = $response['body']['$id'];
$response = $this->client->call(
Client::METHOD_POST,
'/databases/' . $databaseId . '/collections/' . $collectionId . '/attributes' . '/string',
array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id']
], $this->getHeaders()),
[
'key' => 'data',
'size' => 100000,
'required' => true,
]
);
return [
'databaseId' => $databaseId,
'collectionId' => $collectionId,
];
}
// /** @depends testDatabaseStoragePrepare */
// #[Retry(count: 1)]
// public function testDatabaseStorageStatsCreateDocument(array $data): array
// {
// $databaseId = $data['databaseId'];
// $collectionId = $data['collectionId'];
// $originalProjectMetrics = $this->client->call(
// Client::METHOD_GET,
// '/project/usage',
// $this->getConsoleHeaders(),
// [
// 'period' => '1d',
// 'startDate' => self::getToday(),
// 'endDate' => self::getTomorrow(),
// ]
// );
// $this->assertEquals(200, $originalProjectMetrics['headers']['status-code']);
// $this->assertArrayHasKey('databasesStorageTotal', $originalProjectMetrics['body']);
// $originalProjectMetrics = $originalProjectMetrics['body'];
// $originalDatabaseMetrics = $this->client->call(
// Client::METHOD_GET,
// '/databases/' . $databaseId . '/usage?range=30d',
// $this->getConsoleHeaders()
// );
// $this->assertEquals(200, $originalDatabaseMetrics['headers']['status-code']);
// $this->assertArrayHasKey('storageTotal', $originalDatabaseMetrics['body']);
// $originalDatabaseMetrics = $originalDatabaseMetrics['body'];
// // Create documents
// for ($i = 0; $i < 100; $i++) {
// $response = $this->client->call(
// Client::METHOD_POST,
// '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents',
// array_merge([
// 'content-type' => 'application/json',
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()),
// [
// 'documentId' => 'unique()',
// 'data' => ['data' => str_repeat('a', 10000)],
// ]
// );
// $this->assertEquals(201, $response['headers']['status-code']);
// }
// for ($i = 0; $i < 3; $i++) {
// try {
// $newProjectMetrics = $this->client->call(
// Client::METHOD_GET,
// '/project/usage',
// $this->getConsoleHeaders(),
// [
// 'period' => '1d',
// 'startDate' => self::getToday(),
// 'endDate' => self::getTomorrow(),
// ]
// );
// $this->assertEquals(200, $newProjectMetrics['headers']['status-code']);
// $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']);
// $this->assertGreaterThan($originalProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']);
// $newProjectMetrics = $newProjectMetrics['body'];
// $newDatabaseMetrics = $this->client->call(
// Client::METHOD_GET,
// '/databases/' . $databaseId . '/usage?range=30d',
// $this->getConsoleHeaders()
// );
// $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']);
// $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']);
// $this->assertGreaterThan($originalDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']);
// $newDatabaseMetrics = $newDatabaseMetrics['body'];
// return [
// 'databaseId' => $databaseId,
// 'collectionId' => $collectionId,
// 'currentProjectMetrics' => $newProjectMetrics,
// 'currentDatabaseMetrics' => $newDatabaseMetrics,
// ];
// } catch (ExpectationFailedException $e) {
// if ($i === 2) {
// throw $e;
// }
// continue;
// }
// }
// }
// /** @depends testDatabaseStorageStatsCreateDocument */
// #[Retry(count: 1)]
// public function testDatabaseStorageStatsDeleteDocument(array $data): array
// {
// $databaseId = $data['databaseId'];
// $collectionId = $data['collectionId'];
// $currentProjectMetrics = $data['currentProjectMetrics'];
// $currentDatabaseMetrics = $data['currentDatabaseMetrics'];
// $documents = $this->client->call(
// Client::METHOD_GET,
// '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents',
// array_merge([
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders()),
// [
// 'queries' => [
// Query::limit(50)->toString()
// ]
// ]
// );
// foreach ($documents['body']['documents'] as $document) {
// $response = $this->client->call(
// Client::METHOD_DELETE,
// '/databases/' . $databaseId . '/collections/' . $collectionId . '/documents/' . $document['$id'],
// array_merge([
// 'x-appwrite-project' => $this->getProject()['$id']
// ], $this->getHeaders())
// );
// $this->assertEquals(204, $response['headers']['status-code']);
// }
// for ($i = 0; $i < 3; $i++) {
// try {
// $newProjectMetrics = $this->client->call(
// Client::METHOD_GET,
// '/project/usage',
// $this->getConsoleHeaders(),
// [
// 'period' => '1d',
// 'startDate' => self::getToday(),
// 'endDate' => self::getTomorrow(),
// ]
// );
// $this->assertEquals(200, $newProjectMetrics['headers']['status-code']);
// $this->assertArrayHasKey('databasesStorageTotal', $newProjectMetrics['body']);
// $this->assertLessThan($currentProjectMetrics['databasesStorageTotal'], $newProjectMetrics['body']['databasesStorageTotal']);
// $newProjectMetrics = $newProjectMetrics['body'];
// $newDatabaseMetrics = $this->client->call(
// Client::METHOD_GET,
// '/databases/' . $databaseId . '/usage?range=30d',
// $this->getConsoleHeaders()
// );
// $this->assertEquals(200, $newDatabaseMetrics['headers']['status-code']);
// $this->assertArrayHasKey('storageTotal', $newDatabaseMetrics['body']);
// $this->assertLessThan($currentDatabaseMetrics['storageTotal'], $newDatabaseMetrics['body']['storageTotal']);
// $newDatabaseMetrics = $newDatabaseMetrics['body'];
// return [
// 'databaseId' => $databaseId,
// 'collectionId' => $collectionId,
// 'currentProjectMetrics' => $newProjectMetrics,
// 'currentDatabaseMetrics' => $newDatabaseMetrics,
// ];
// } catch (ExpectationFailedException $e) {
// if ($i === 2) {
// throw $e;
// }
// continue;
// }
// }
// $newProjectMetrics = $this->client->call(
// Client::METHOD_GET,
// '/project/usage',
// $this->getConsoleHeaders(),
// [
// 'period' => '1d',
// 'startDate' => self::getToday(),
// 'endDate' => self::getTomorrow(),
// ]
// );
// }
/** @depends testDatabaseStats */
public function testPrepareFunctionsStats(array $data): array
{
tests/e2e/Services/Databases/DatabasesBase.php
+6
View File
@@ -90,6 +90,12 @@ trait DatabasesBase
*/
public function testConsoleProject(array $data)
{
if ($this->getSide() === 'server') {
// Server side can't get past the invalid key check anyway
$this->expectNotToPerformAssertions();
return;
}
$response = $this->client->call(
Client::METHOD_GET,
'/databases/console/collections/' . $data['moviesId'] . '/documents',
tests/e2e/Services/Health/HealthCustomServerTest.php
-24
View File
@@ -541,28 +541,4 @@ class HealthCustomServerTest extends Scope
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
public function testStatsUsageDumpSuccess()
{
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertIsInt($response['body']['size']);
$this->assertLessThan(100, $response['body']['size']);
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_GET, '/health/queue/stats-usage-dump?threshold=0', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), []);
$this->assertEquals(503, $response['headers']['status-code']);
}
}
tests/e2e/Services/Projects/ProjectsConsoleClientTest.php
+93 -15
View File
@@ -2786,32 +2786,35 @@ class ProjectsConsoleClientTest extends Scope
*/
public function testValidateProjectKey($data): void
{
$id = $data['projectId'] ?? '';
$projectId = $data['projectId'] ?? '';
$teamId = $data['teamId'] ?? '';
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([
// Expiring key
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Key Test',
'scopes' => ['health.read'],
'scopes' => ['users.write'],
'expire' => DateTime::addSeconds(new \DateTime(), 3600),
]);
$response = $this->client->call(Client::METHOD_GET, '/health', [
$response = $this->client->call(Client::METHOD_POST, '/users', [
'content-type' => 'application/json',
'x-appwrite-project' => $id,
'x-appwrite-project' => $projectId,
'x-appwrite-key' => $response['body']['secret']
], []);
], [
'userId' => ID::unique(),
]);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(201, $response['headers']['status-code']);
/**
* Test for SUCCESS
*/
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([
// No expiry
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
@@ -2822,7 +2825,7 @@ class ProjectsConsoleClientTest extends Scope
$response = $this->client->call(Client::METHOD_GET, '/health', [
'content-type' => 'application/json',
'x-appwrite-project' => $id,
'x-appwrite-project' => $projectId,
'x-appwrite-key' => $response['body']['secret']
], []);
@@ -2831,7 +2834,9 @@ class ProjectsConsoleClientTest extends Scope
/**
* Test for FAILURE
*/
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $id . '/keys', array_merge([
// Expired key
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
@@ -2842,9 +2847,82 @@ class ProjectsConsoleClientTest extends Scope
$response = $this->client->call(Client::METHOD_GET, '/health', [
'content-type' => 'application/json',
'x-appwrite-project' => $id,
'x-appwrite-project' => $projectId,
'x-appwrite-key' => $response['body']['secret']
], []);
]);
$this->assertEquals(401, $response['headers']['status-code']);
// Invalid key
$bucket = $this->client->call(Client::METHOD_POST, '/storage/buckets', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'x-appwrite-mode' => 'admin',
], $this->getHeaders()), [
'bucketId' => ID::unique(),
'name' => 'Test Bucket',
]);
$this->assertEquals(201, $bucket['headers']['status-code']);
$this->assertNotEmpty($bucket['body']['$id']);
$bucketId = $bucket['body']['$id'];
$response = $this->client->call(Client::METHOD_GET, "/storage/buckets/{$bucketId}/files", [
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'x-appwrite-key' => 'invalid-key'
]);
$this->assertEquals(401, $response['headers']['status-code']);
// Invalid scopes
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $projectId . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Key Test',
'scopes' => ['teams.read'],
'expire' => DateTime::addSeconds(new \DateTime(), 3600),
]);
$response = $this->client->call(Client::METHOD_GET, '/users', [
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'x-appwrite-key' => $response['body']['secret']
]);
$this->assertEquals(401, $response['headers']['status-code']);
// Invalid key from different project
$response = $this->client->call(Client::METHOD_POST, '/projects', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'projectId' => ID::unique(),
'name' => 'Project Test 2',
'teamId' => $teamId,
'region' => System::getEnv('_APP_REGION', 'default')
]);
$this->assertEquals(201, $response['headers']['status-code']);
$project2Id = $response['body']['$id'];
$response = $this->client->call(Client::METHOD_POST, '/projects/' . $project2Id . '/keys', array_merge([
'content-type' => 'application/json',
'x-appwrite-project' => $this->getProject()['$id'],
], $this->getHeaders()), [
'name' => 'Key Test',
'scopes' => ['health.read'],
'expire' => DateTime::addSeconds(new \DateTime(), 3600),
]);
$response = $this->client->call(Client::METHOD_GET, '/health', [
'content-type' => 'application/json',
'x-appwrite-project' => $projectId,
'x-appwrite-key' => $response['body']['secret']
]);
$this->assertEquals(401, $response['headers']['status-code']);
}