Appwrite/appwrite
1
0
Fork 0
mirror of https://github.com/appwrite/appwrite.git synced 2026-05-26 13:51:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

usage sync

Browse Source
This commit is contained in:
shimon
2023-10-25 08:20:19 +03:00
parent 66c57c9a4f
commit 788ae05ad3
11 changed files with 709 additions and 842 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/calc-users-stats
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php calc-users-stats $@
bin/clear-card-cache
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php clear-card-cache $@
bin/patch-delete-project-collections
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-project-collections $@
bin/patch-delete-schedule-updated-at-attribute
-3
View File
@@ -1,3 +0,0 @@
#!/bin/sh
php /usr/src/code/app/cli.php patch-delete-schedule-updated-at-attribute $@
composer.lock
Generated
+701 -240
View File
File diff suppressed because it is too large Load Diff
docker-compose.yml
+8 -52
View File
@@ -83,10 +83,7 @@ services:
- ./docs:/usr/src/code/docs
- ./public:/usr/src/code/public
- ./src:/usr/src/code/src
- ./dev:/usr/local/dev
- ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform
- ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework
- ./dev:/usr/src/code/dev
depends_on:
- mariadb
- redis
@@ -210,7 +207,6 @@ services:
- traefik.http.routers.appwrite_realtime_wss.rule=PathPrefix(`/v1/realtime`)
- traefik.http.routers.appwrite_realtime_wss.service=appwrite_realtime
- traefik.http.routers.appwrite_realtime_wss.tls=true
- traefik.http.routers.appwrite_realtime_wss.tls.certresolver=dns
networks:
- appwrite
volumes:
@@ -248,10 +244,6 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform
- ./vendor/utopia-php/queue:/usr/src/code/vendor/utopia-php/queue
- ./vendor/utopia-php/pools:/usr/src/code/vendor/utopia-php/pools
depends_on:
- redis
- mariadb
@@ -281,7 +273,6 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform
depends_on:
- redis
- mariadb
@@ -316,7 +307,6 @@ services:
- appwrite-certificates:/storage/certificates:rw
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/storage:/usr/src/code/vendor/utopia-php/storage
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
@@ -366,14 +356,11 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/database:/usr/src/code/vendor/utopia-php/database
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKERS_NUM=1
- _APP_QUEUE_NAME=database_db_main
- _APP_WORKER_PER_CORE
- _APP_OPENSSL_KEY_V1
- _APP_REDIS_HOST
@@ -387,6 +374,8 @@ services:
- _APP_DB_PASS
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
- _APP_WORKERS_NUM
- _APP_QUEUE_NAME
appwrite-worker-builds:
entrypoint: worker-builds
@@ -400,8 +389,6 @@ services:
- appwrite-builds:/storage/builds:rw
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform
- ./vendor/utopia-php/queue:/usr/src/code/vendor/utopia-php/queue
depends_on:
- redis
- mariadb
@@ -631,8 +618,6 @@ services:
volumes:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/pools:/usr/src/code/vendor/utopia-php/pools
depends_on:
- redis
environment:
@@ -651,10 +636,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_MAINTENANCE_INTERVAL
- _APP_MAINTENANCE_RETENTION_EXECUTION
- _APP_MAINTENANCE_RETENTION_CACHE
@@ -674,15 +655,14 @@ services:
- ./app:/usr/src/code/app
- ./src:/usr/src/code/src
- ./vendor/utopia-php/platform:/usr/src/code/vendor/utopia-php/platform
- ./vendor/utopia-php/framework:/usr/src/code/vendor/utopia-php/framework
- ./vendor/utopia-php/queue:/usr/src/code/vendor/utopia-php/queue
depends_on:
- redis
- mariadb
environment:
- _APP_ENV
- _APP_WORKER_PER_CORE
- _APP_CONNECTIONS_MAX
- _APP_POOL_CLIENTS
- _APP_OPENSSL_KEY_V1
- _APP_DB_HOST
@@ -693,14 +673,10 @@ services:
- _APP_REDIS_HOST
- _APP_REDIS_PORT
- _APP_REDIS_USER
- _APP_REDIS_PASS
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_REDIS_PASST
- _APP_USAGE_STATS
- DOCKERHUB_PULL_USERNAME
- DOCKERHUB_PULL_PASSWORD
- _APP_LOGGING_PROVIDER
- _APP_LOGGING_CONFIG
appwrite-schedule:
entrypoint: schedule
@@ -728,11 +704,6 @@ services:
- _APP_DB_SCHEMA
- _APP_DB_USER
- _APP_DB_PASS
- _APP_CONNECTIONS_DB_PROJECT
- _APP_CONNECTIONS_DB_CONSOLE
- _APP_CONNECTIONS_CACHE
- _APP_CONNECTIONS_QUEUE
- _APP_REGION
appwrite-assistant:
container_name: appwrite-assistant
@@ -747,7 +718,7 @@ services:
hostname: appwrite-executor
<<: *x-logging
stop_signal: SIGINT
image: openruntimes/executor:0.4.3
image: openruntimes/executor:0.4.5
restart: unless-stopped
networks:
- appwrite
@@ -877,7 +848,6 @@ services:
# MailCatcher - An SMTP server. Catches all system emails and displays them in a nice UI.
# RequestCatcher - An HTTP server. Catches all system https calls and displays them using a simple HTTP API. Used to debug & tests webhooks and HTTP tasks
# RedisCommander - A nice UI for exploring Redis data
# Resque - A nice UI for exploring Redis pub/sub, view the different queues workloads, pending and failed tasks
# Webgrind - A nice UI for exploring and debugging code-level stuff
maildev: # used mainly for dev tests
@@ -917,20 +887,6 @@ services:
# - REDIS_HOSTS=redis
# ports:
# - "8081:8081"
# resque:
# image: appwrite/resque-web:1.1.0
# restart: unless-stopped
# networks:
# - appwrite
# ports:
# - "5678:5678"
# environment:
# - RESQUE_WEB_HOST=redis
# - RESQUE_WEB_PORT=6379
# - RESQUE_WEB_HTTP_BASIC_AUTH_USER=user
# - RESQUE_WEB_HTTP_BASIC_AUTH_PASSWORD=password
# webgrind:
# image: 'jokkedk/webgrind:latest'
# volumes:
src/Appwrite/Platform/Tasks/CalcUsersStats.php
-176
View File
@@ -1,176 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Exception;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use League\Csv\Writer;
use PHPMailer\PHPMailer\PHPMailer;
use Utopia\Pools\Group;
use Utopia\Registry\Registry;
class CalcUsersStats extends Action
{
private array $columns = [
'Project ID',
'Project Name',
'Team ID',
'Team name',
'Users'
];
protected string $directory = '/usr/local';
protected string $path;
protected string $date;
public static function getName(): string
{
return 'calc-users-stats';
}
public function __construct()
{
$this
->desc('Get stats for projects')
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->inject('register')
->callback(function (Group $pools, Cache $cache, Database $dbForConsole, Registry $register) {
$this->action($pools, $cache, $dbForConsole, $register);
});
}
public function action(Group $pools, Cache $cache, Database $dbForConsole, Registry $register): void
{
//docker compose exec -t appwrite calc-users-stats
Console::title('Cloud Users calculation V1');
Console::success(APP_NAME . ' cloud Users calculation has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** CSV stuff */
$this->date = date('Y-m-d');
$this->path = "{$this->directory}/users_stats_{$this->date}.csv";
$csv = Writer::createFromPath($this->path, 'w');
$csv->insertOne($this->columns);
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 30;
$sum = 30;
$offset = 0;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Getting stats for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase('appwrite');
$dbForProject->setNamespace('_' . $project->getInternalId());
/** Get Project ID */
$stats['Project ID'] = $project->getId();
/** Get Project Name */
$stats['Project Name'] = $project->getAttribute('name');
/** Get Team Name and Id */
$teamId = $project->getAttribute('teamId', null);
$teamName = null;
if ($teamId) {
$team = $dbForConsole->getDocument('teams', $teamId);
$teamName = $team->getAttribute('name');
}
$stats['Team ID'] = $teamId;
$stats['Team name'] = $teamName;
/** Get Total Users */
$stats['users'] = $dbForProject->count('users', []);
$csv->insertOne(array_values($stats));
} catch (\Throwable $th) {
Console::error('Failed to update project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
/** @var PHPMailer $mail */
$mail = $register->get('smtp');
$mail->clearAddresses();
$mail->clearAllRecipients();
$mail->clearReplyTos();
$mail->clearAttachments();
$mail->clearBCCs();
$mail->clearCCs();
try {
/** Addresses */
$mail->setFrom(App::getEnv('_APP_SYSTEM_EMAIL_ADDRESS', APP_EMAIL_TEAM), 'Appwrite Cloud Hamster');
$recipients = explode(',', App::getEnv('_APP_USERS_STATS_RECIPIENTS', ''));
foreach ($recipients as $recipient) {
$mail->addAddress($recipient);
}
/** Attachments */
$mail->addAttachment($this->path);
/** Content */
$mail->Subject = "Cloud Report for {$this->date}";
$mail->Body = "Please find the daily cloud report atttached";
$mail->send();
Console::success('Email has been sent!');
} catch (Exception $e) {
Console::error("Message could not be sent. Mailer Error: {$mail->ErrorInfo}");
}
}
}
src/Appwrite/Platform/Tasks/ClearCardCache.php
-62
View File
@@ -1,62 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Cache\Adapter\Filesystem;
use Utopia\Cache\Cache;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Database\Validator\UID;
class ClearCardCache extends Action
{
public static function getName(): string
{
return 'clear-card-cache';
}
public function __construct()
{
$this
->desc('Deletes card cache for specific user')
->param('userId', '', new UID(), 'User UID.', false)
->inject('dbForConsole')
->callback(fn (string $userId, Database $dbForConsole) => $this->action($userId, $dbForConsole));
}
public function action(string $userId, Database $dbForConsole): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('ClearCardCache V1');
Console::success(APP_NAME . ' ClearCardCache v1 has started');
$resources = ['card/' . $userId, 'card-back/' . $userId, 'card-og/' . $userId];
$caches = Authorization::skip(fn () => $dbForConsole->find('cache', [
Query::equal('resource', $resources),
Query::limit(100)
]));
$count = \count($caches);
Console::info("Going to delete {$count} cache records in 10 seconds...");
\sleep(10);
foreach ($caches as $cache) {
$key = $cache->getId();
$cacheFolder = new Cache(
new Filesystem(APP_STORAGE_CACHE . DIRECTORY_SEPARATOR . 'app-console')
);
$cacheFolder->purge($key);
Authorization::skip(fn () => $dbForConsole->deleteDocument('cache', $cache->getId()));
}
Console::success(APP_NAME . ' ClearCardCache v1 has finished');
}
}
src/Appwrite/Platform/Tasks/PatchCreateMissingSchedules.php
-97
View File
@@ -1,97 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\DateTime;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Helpers\ID;
use Utopia\Database\Validator\Authorization;
class PatchCreateMissingSchedules extends Action
{
public static function getName(): string
{
return 'patch-create-missing-schedules';
}
public function __construct()
{
$this
->desc('Ensure every function has a schedule')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Database $dbForConsole, callable $getProjectDB) => $this->action($dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchCreateMissingSchedules V1');
Console::success(APP_NAME . ' PatchCreateMissingSchedules v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
$functionCursor = null;
while (true) {
$functionsQueries = [Query::limit($limit)];
if ($functionCursor !== null) {
$functionsQueries[] = Query::cursorAfter($functionCursor);
}
$functions = $dbForProject->find('functions', $functionsQueries);
if (count($functions) === 0) {
break;
}
foreach ($functions as $function) {
$scheduleId = $function->getAttribute('scheduleId');
$schedule = $dbForConsole->getDocument('schedules', $scheduleId);
if ($schedule->isEmpty()) {
$functionId = $function->getId();
$schedule = $dbForConsole->createDocument('schedules', new Document([
'$id' => ID::custom($scheduleId),
'region' => $project->getAttribute('region', 'default'),
'resourceType' => 'function',
'resourceId' => $functionId,
'resourceUpdatedAt' => DateTime::now(),
'projectId' => $project->getId(),
'schedule' => $function->getAttribute('schedule'),
'active' => !empty($function->getAttribute('schedule')) && !empty($function->getAttribute('deployment')),
]));
Console::success('Recreated schedule for function ' . $functionId);
}
}
$functionCursor = $functions[array_key_last($functions)];
}
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}
src/Appwrite/Platform/Tasks/PatchDeleteProjectCollections.php
-129
View File
@@ -1,129 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\App;
use Utopia\Platform\Action;
use Utopia\Cache\Cache;
use Utopia\CLI\Console;
use Utopia\Database\Database;
use Utopia\Database\Query;
use Utopia\Pools\Group;
use Utopia\Validator\Numeric;
class PatchDeleteProjectCollections extends Action
{
private array $names = [
'webhooks',
'platforms',
'schedules',
'projects',
'domains',
'certificates',
'keys',
'realtime',
];
public static function getName(): string
{
return 'patch-delete-project-collections';
}
public function __construct()
{
$this
->desc('Delete unnecessary project collections')
->param('offset', 0, new Numeric(), 'Resume deletion from param pos', true)
->inject('pools')
->inject('cache')
->inject('dbForConsole')
->callback(function (int $offset, Group $pools, Cache $cache, Database $dbForConsole) {
$this->action($offset, $pools, $cache, $dbForConsole);
});
}
public function action(int $offset, Group $pools, Cache $cache, Database $dbForConsole): void
{
//docker compose exec -t appwrite patch-delete-project-collections
Console::title('Delete project collections V1');
Console::success(APP_NAME . ' delete project collections has started');
/* Initialise new Utopia app */
$app = new App('UTC');
$console = $app->getResource('console');
/** Database connections */
$totalProjects = $dbForConsole->count('projects');
Console::success("Found a total of: {$totalProjects} projects");
$projects = [$console];
$count = 0;
$limit = 50;
$sum = 50;
$offset = $offset;
while (!empty($projects)) {
foreach ($projects as $project) {
/**
* Skip user projects with id 'console'
*/
if ($project->getId() === 'console') {
continue;
}
Console::info("Deleting collections for {$project->getId()}");
try {
$db = $project->getAttribute('database');
$adapter = $pools
->get($db)
->pop()
->getResource();
$dbForProject = new Database($adapter, $cache);
$dbForProject->setDefaultDatabase(App::getEnv('_APP_DB_SCHEMA', 'appwrite'));
$dbForProject->setNamespace('_' . $project->getInternalId());
foreach ($this->names as $name) {
if (empty($name)) {
continue;
}
if ($dbForProject->exists(App::getEnv('_APP_DB_SCHEMA', 'appwrite'), $name)) {
if ($dbForProject->deleteCollection($name)) {
Console::log('Deleted ' . $name);
} else {
Console::error('Failed to delete ' . $name);
}
}
}
} catch (\Throwable $th) {
Console::error('Failed on project ("' . $project->getId() . '") version with error: ' . $th->getMessage());
} finally {
$pools
->get($db)
->reclaim();
}
}
$sum = \count($projects);
$projects = $dbForConsole->find('projects', [
Query::limit($limit),
Query::offset($offset),
]);
if (!empty($projects)) {
Console::log('Querying..... offset=' . $offset . ' , limit=' . $limit . ', count=' . $count);
}
$offset = $offset + $limit;
$count = $count + $sum;
}
Console::log('Iterated through ' . $count - 1 . '/' . $totalProjects . ' projects...');
$pools
->get('console')
->reclaim();
}
}
src/Appwrite/Platform/Tasks/PatchDeleteScheduleUpdatedAtAttribute.php
-74
View File
@@ -1,74 +0,0 @@
<?php
namespace Appwrite\Platform\Tasks;
use Utopia\Platform\Action;
use Utopia\CLI\Console;
use Utopia\Database\Query;
use Utopia\Database\Database;
use Utopia\Database\Validator\Authorization;
use Utopia\Pools\Group;
class PatchDeleteScheduleUpdatedAtAttribute extends Action
{
public static function getName(): string
{
return 'patch-delete-schedule-updated-at-attribute';
}
public function __construct()
{
$this
->desc('Ensure function collections do not have scheduleUpdatedAt attribute')
->inject('pools')
->inject('dbForConsole')
->inject('getProjectDB')
->callback(fn (Group $pools, Database $dbForConsole, callable $getProjectDB) => $this->action($pools, $dbForConsole, $getProjectDB));
}
/**
* Iterate over every function on every project to make sure there is a schedule. If not, recreate the schedule.
*/
public function action(Group $pools, Database $dbForConsole, callable $getProjectDB): void
{
Authorization::disable();
Authorization::setDefaultStatus(false);
Console::title('PatchDeleteScheduleUpdatedAtAttribute V1');
Console::success(APP_NAME . ' PatchDeleteScheduleUpdatedAtAttribute v1 has started');
$limit = 100;
$projectCursor = null;
while (true) {
$projectsQueries = [Query::limit($limit)];
if ($projectCursor !== null) {
$projectsQueries[] = Query::cursorAfter($projectCursor);
}
$projects = $dbForConsole->find('projects', $projectsQueries);
if (count($projects) === 0) {
break;
}
foreach ($projects as $project) {
Console::log("Checking Project " . $project->getAttribute('name') . " (" . $project->getId() . ")");
$dbForProject = $getProjectDB($project);
try {
/**
* Delete 'scheduleUpdatedAt' attribute
*/
$dbForProject->deleteAttribute('functions', 'scheduleUpdatedAt');
$dbForProject->deleteCachedCollection('functions');
Console::success("'scheduleUpdatedAt' deleted.");
} catch (\Throwable $th) {
Console::warning("'scheduleUpdatedAt' errored: {$th->getMessage()}");
}
$pools->reclaim();
}
$projectCursor = $projects[array_key_last($projects)];
}
}
}