Bug fixes for proxy rule service

This commit is contained in:
Matej Baco
2023-03-10 07:42:52 +00:00
parent 9486611a68
commit 04c7db9f7b
11 changed files with 79 additions and 245 deletions
+4 -4
View File
@@ -1,4 +1,4 @@
_APP_ENV=development
_APP_ENV=production
_APP_LOCALE=en
_APP_WORKER_PER_CORE=10
_APP_CONSOLE_WHITELIST_ROOT=disabled
@@ -13,10 +13,10 @@ _APP_SYSTEM_RESPONSE_FORMAT=
_APP_OPTIONS_ABUSE=disabled
_APP_OPTIONS_FORCE_HTTPS=disabled
_APP_OPENSSL_KEY_V1=your-secret-key
_APP_DOMAIN=localhost
_APP_DOMAIN_FUNCTIONS=functions.localhost
_APP_DOMAIN=g4main.matejbaco.eu
_APP_DOMAIN_FUNCTIONS=functions.g4main.matejbaco.eu
_APP_DOMAIN_DO_TOKEN=
_APP_DOMAIN_TARGET=localhost
_APP_DOMAIN_TARGET=g4main.matejbaco.eu
_APP_REDIS_HOST=redis
_APP_REDIS_PORT=6379
_APP_REDIS_PASS=
+46 -203
View File
@@ -20,162 +20,16 @@ WORKDIR /usr/local/src/console
ARG VITE_GA_PROJECT
ARG VITE_CONSOLE_MODE
ARG VITE_APPWRITE_GROWTH_ENDPOINT=https://growth.appwrite.io/v1
ENV VITE_GA_PROJECT=$VITE_GA_PROJECT
ENV VITE_CONSOLE_MODE=$VITE_CONSOLE_MODE
ENV VITE_APPWRITE_GROWTH_ENDPOINT=$VITE_APPWRITE_GROWTH_ENDPOINT
RUN npm ci
RUN npm run build
FROM php:8.0.18-cli-alpine3.15 as compile
ARG DEBUG=false
ENV DEBUG=$DEBUG
ENV PHP_REDIS_VERSION=5.3.7 \
PHP_MONGODB_VERSION=1.13.0 \
PHP_SWOOLE_VERSION=v4.8.10 \
PHP_IMAGICK_VERSION=3.7.0 \
PHP_YAML_VERSION=2.2.2 \
PHP_MAXMINDDB_VERSION=v1.11.0 \
PHP_MEMCACHED_VERSION=v3.2.0 \
PHP_ZSTD_VERSION="4504e4186e79b197cfcb75d4d09aa47ef7d92fe9 "
RUN \
apk add --no-cache --virtual .deps \
make \
automake \
autoconf \
gcc \
g++ \
git \
zlib-dev \
brotli-dev \
openssl-dev \
yaml-dev \
imagemagick \
imagemagick-dev \
libmaxminddb-dev \
libmemcached-dev \
zstd-dev
RUN docker-php-ext-install sockets
FROM compile AS redis
RUN \
# Redis Extension
git clone --depth 1 --branch $PHP_REDIS_VERSION https://github.com/phpredis/phpredis.git && \
cd phpredis && \
phpize && \
./configure && \
make && make install
## Swoole Extension
FROM compile AS swoole
RUN \
git clone --depth 1 --branch $PHP_SWOOLE_VERSION https://github.com/swoole/swoole-src.git && \
cd swoole-src && \
phpize && \
./configure --enable-sockets --enable-http2 --enable-openssl && \
make && make install && \
cd ..
## Swoole Debugger setup
RUN if [ "$DEBUG" == "true" ]; then \
cd /tmp && \
apk add boost-dev && \
git clone --depth 1 https://github.com/swoole/yasd && \
cd yasd && \
phpize && \
./configure && \
make && make install && \
cd ..;\
fi
## Imagick Extension
FROM compile AS imagick
RUN \
git clone --depth 1 --branch $PHP_IMAGICK_VERSION https://github.com/imagick/imagick && \
cd imagick && \
phpize && \
./configure && \
make && make install
## YAML Extension
FROM compile AS yaml
RUN \
git clone --depth 1 --branch $PHP_YAML_VERSION https://github.com/php/pecl-file_formats-yaml && \
cd pecl-file_formats-yaml && \
phpize && \
./configure && \
make && make install
## Maxminddb extension
FROM compile AS maxmind
RUN \
git clone --depth 1 --branch $PHP_MAXMINDDB_VERSION https://github.com/maxmind/MaxMind-DB-Reader-php.git && \
cd MaxMind-DB-Reader-php && \
cd ext && \
phpize && \
./configure && \
make && make install
# Mongodb Extension
FROM compile as mongodb
RUN \
git clone --depth 1 --branch $PHP_MONGODB_VERSION https://github.com/mongodb/mongo-php-driver.git && \
cd mongo-php-driver && \
git submodule update --init && \
phpize && \
./configure && \
make && make install
# Memcached Extension
FROM compile as memcached
RUN \
git clone --depth 1 --branch $PHP_MEMCACHED_VERSION https://github.com/php-memcached-dev/php-memcached.git && \
cd php-memcached && \
phpize && \
./configure && \
make && make install
# Zstd Compression
FROM compile as zstd
RUN git clone --recursive -n https://github.com/kjdev/php-ext-zstd.git \
&& cd php-ext-zstd \
&& git checkout $PHP_ZSTD_VERSION \
&& phpize \
&& ./configure --with-libzstd \
&& make && make install
# Rust Extensions Compile Image
FROM php:8.0.18-cli as rust_compile
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH=/root/.cargo/bin:$PATH
RUN apt-get update && apt-get install musl-tools build-essential clang-11 git -y
RUN rustup target add $(uname -m)-unknown-linux-musl
# Install ZigBuild for easier cross-compilation
RUN curl https://ziglang.org/builds/zig-linux-$(uname -m)-0.10.0-dev.2674+d980c6a38.tar.xz --output /tmp/zig.tar.xz
RUN tar -xf /tmp/zig.tar.xz -C /tmp/ && cp -r /tmp/zig-linux-$(uname -m)-0.10.0-dev.2674+d980c6a38 /tmp/zig/
ENV PATH=/tmp/zig:$PATH
RUN cargo install cargo-zigbuild
ENV RUSTFLAGS="-C target-feature=-crt-static"
FROM rust_compile as scrypt
WORKDIR /usr/local/lib/php/extensions/
RUN \
git clone --depth 1 https://github.com/appwrite/php-scrypt.git && \
cd php-scrypt && \
cargo zigbuild --workspace --all-targets --target $(uname -m)-unknown-linux-musl --release && \
mv target/$(uname -m)-unknown-linux-musl/release/libphp_scrypt.so target/libphp_scrypt.so
FROM php:8.0.18-cli-alpine3.15 as final
FROM appwrite/base:0.1.0 as final
LABEL maintainer="team@appwrite.io"
@@ -189,12 +43,18 @@ ENV DOCKER_COMPOSE_VERSION=v2.5.0
ENV _APP_SERVER=swoole \
_APP_ENV=production \
_APP_LOCALE=en \
_APP_WORKER_PER_CORE= \
_APP_DOMAIN=localhost \
_APP_DOMAIN_TARGET=localhost \
_APP_DOMAIN_FUNCTIONS=disabled \
_APP_HOME=https://appwrite.io \
_APP_EDITION=community \
_APP_CONSOLE_WHITELIST_ROOT=enabled \
_APP_CONSOLE_WHITELIST_EMAILS= \
_APP_CONSOLE_WHITELIST_IPS= \
_APP_SYSTEM_EMAIL_NAME= \
_APP_SYSTEM_EMAIL_ADDRESS= \
_APP_SYSTEM_RESPONSE_FORMAT= \
_APP_SYSTEM_SECURITY_EMAIL_ADDRESS= \
_APP_OPTIONS_ABUSE=enabled \
_APP_OPTIONS_FORCE_HTTPS=disabled \
_APP_OPENSSL_KEY_V1=your-secret-key \
@@ -202,6 +62,27 @@ ENV _APP_SERVER=swoole \
_APP_STORAGE_ANTIVIRUS=enabled \
_APP_STORAGE_ANTIVIRUS_HOST=clamav \
_APP_STORAGE_ANTIVIRUS_PORT=3310 \
_APP_STORAGE_DEVICE=Local \
_APP_STORAGE_S3_ACCESS_KEY= \
_APP_STORAGE_S3_SECRET= \
_APP_STORAGE_S3_REGION= \
_APP_STORAGE_S3_BUCKET= \
_APP_STORAGE_DO_SPACES_ACCESS_KEY= \
_APP_STORAGE_DO_SPACES_SECRET= \
_APP_STORAGE_DO_SPACES_REGION= \
_APP_STORAGE_DO_SPACES_BUCKET= \
_APP_STORAGE_BACKBLAZE_ACCESS_KEY= \
_APP_STORAGE_BACKBLAZE_SECRET= \
_APP_STORAGE_BACKBLAZE_REGION= \
_APP_STORAGE_BACKBLAZE_BUCKET= \
_APP_STORAGE_LINODE_ACCESS_KEY= \
_APP_STORAGE_LINODE_SECRET= \
_APP_STORAGE_LINODE_REGION= \
_APP_STORAGE_LINODE_BUCKET= \
_APP_STORAGE_WASABI_ACCESS_KEY= \
_APP_STORAGE_WASABI_SECRET= \
_APP_STORAGE_WASABI_REGION= \
_APP_STORAGE_WASABI_BUCKET= \
_APP_REDIS_HOST=redis \
_APP_REDIS_PORT=6379 \
_APP_DB_HOST=mariadb \
@@ -213,55 +94,35 @@ ENV _APP_SERVER=swoole \
_APP_INFLUXDB_PORT=8086 \
_APP_STATSD_HOST=telegraf \
_APP_STATSD_PORT=8125 \
_APP_SMTP_HOST= \
_APP_SMTP_PORT= \
_APP_SMTP_SECURE= \
_APP_SMTP_USERNAME= \
_APP_SMTP_PASSWORD= \
_APP_SMS_PROVIDER= \
_APP_SMS_FROM= \
_APP_FUNCTIONS_SIZE_LIMIT=30000000 \
_APP_FUNCTIONS_TIMEOUT=900 \
_APP_FUNCTIONS_CONTAINERS=10 \
_APP_FUNCTIONS_CPUS=1 \
_APP_FUNCTIONS_MEMORY=128 \
_APP_FUNCTIONS_MEMORY_SWAP=128 \
_APP_EXECUTOR_SECRET=a-random-secret \
_APP_EXECUTOR_HOST=http://exc1/v1 \
_APP_EXECUTOR_HOST=http://appwrite-executor/v1 \
_APP_EXECUTOR_RUNTIME_NETWORK=appwrite_runtimes \
_APP_SETUP=self-hosted \
_APP_VERSION=$VERSION \
_APP_USAGE_STATS=enabled \
_APP_USAGE_AGGREGATION_INTERVAL=30 \
# 14 Days = 1209600 s
_APP_MAINTENANCE_RETENTION_EXECUTION=1209600 \
_APP_MAINTENANCE_RETENTION_AUDIT=1209600 \
# 1 Day = 86400 s
_APP_MAINTENANCE_RETENTION_ABUSE=86400 \
_APP_MAINTENANCE_RETENTION_USAGE_HOURLY=8640000 \
_APP_MAINTENANCE_INTERVAL=86400
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN \
apk update \
&& apk add --no-cache --virtual .deps \
make \
automake \
autoconf \
gcc \
g++ \
curl-dev \
&& apk add --no-cache \
libstdc++ \
certbot \
rsync \
brotli-dev \
yaml-dev \
imagemagick \
imagemagick-dev \
libmaxminddb-dev \
certbot \
docker-cli \
libgomp \
&& docker-php-ext-install sockets opcache pdo_mysql \
&& apk del .deps \
&& rm -rf /var/cache/apk/*
RUN \
mkdir -p $DOCKER_CONFIG/cli-plugins \
&& ARCH=$(uname -m) && if [ $ARCH == "armv7l" ]; then ARCH="armv7"; fi \
&& curl -SL https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-linux-$ARCH -o $DOCKER_CONFIG/cli-plugins/docker-compose \
&& chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
_APP_MAINTENANCE_INTERVAL=86400 \
_APP_LOGGING_PROVIDER= \
_APP_LOGGING_CONFIG=
RUN \
if [ "$DEBUG" == "true" ]; then \
@@ -272,15 +133,6 @@ WORKDIR /usr/src/code
COPY --from=composer /usr/local/src/vendor /usr/src/code/vendor
COPY --from=node /usr/local/src/console/build /usr/src/code/console
COPY --from=swoole /usr/local/lib/php/extensions/no-debug-non-zts-20200930/swoole.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yasd.so* /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=redis /usr/local/lib/php/extensions/no-debug-non-zts-20200930/redis.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=imagick /usr/local/lib/php/extensions/no-debug-non-zts-20200930/imagick.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=yaml /usr/local/lib/php/extensions/no-debug-non-zts-20200930/yaml.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=maxmind /usr/local/lib/php/extensions/no-debug-non-zts-20200930/maxminddb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=mongodb /usr/local/lib/php/extensions/no-debug-non-zts-20200930/mongodb.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=memcached /usr/local/lib/php/extensions/no-debug-non-zts-20200930/memcached.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=scrypt /usr/local/lib/php/extensions/php-scrypt/target/libphp_scrypt.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
COPY --from=zstd /usr/local/lib/php/extensions/no-debug-non-zts-20200930/zstd.so /usr/local/lib/php/extensions/no-debug-non-zts-20200930/
# Add Source Code
COPY ./app /usr/src/code/app
@@ -304,9 +156,7 @@ RUN mkdir -p /storage/uploads && \
# Executables
RUN chmod +x /usr/local/bin/doctor && \
chmod +x /usr/local/bin/patch-create-missing-schedules && \
chmod +x /usr/local/bin/maintenance && \
chmod +x /usr/local/bin/volume-sync && \
chmod +x /usr/local/bin/usage && \
chmod +x /usr/local/bin/install && \
chmod +x /usr/local/bin/migrate && \
@@ -331,13 +181,6 @@ RUN chmod +x /usr/local/bin/doctor && \
RUN mkdir -p /etc/letsencrypt/live/ && chmod -Rf 755 /etc/letsencrypt/live/
# Enable Extensions
RUN echo extension=swoole.so >> /usr/local/etc/php/conf.d/swoole.ini
RUN echo extension=redis.so >> /usr/local/etc/php/conf.d/redis.ini
RUN echo extension=imagick.so >> /usr/local/etc/php/conf.d/imagick.ini
RUN echo extension=yaml.so >> /usr/local/etc/php/conf.d/yaml.ini
RUN echo extension=maxminddb.so >> /usr/local/etc/php/conf.d/maxminddb.ini
RUN echo extension=libphp_scrypt.so >> /usr/local/etc/php/conf.d/libphp_scrypt.ini
RUN echo extension=zstd.so >> /usr/local/etc/php/conf.d/zstd.ini
RUN if [ "$DEBUG" == "true" ]; then printf "zend_extension=yasd \nyasd.debug_mode=remote \nyasd.init_file=/usr/local/dev/yasd_init.php \nyasd.remote_port=9005 \nyasd.log_level=-1" >> /usr/local/etc/php/conf.d/yasd.ini; fi
RUN if [ "$DEBUG" == "true" ]; then echo "opcache.enable=0" >> /usr/local/etc/php/conf.d/appwrite.ini; fi
+1 -1
View File
@@ -551,7 +551,7 @@ return [
],
Exception::RULE_VERIFICATION_FAILED => [
'name' => Exception::RULE_VERIFICATION_FAILED,
'description' => 'Rule verification for it\'s domain has failed.',
'description' => 'Domain verification for failed. Please check your DNS records.',
'code' => 401,
],
Exception::WEBHOOK_NOT_FOUND => [
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+6 -5
View File
@@ -4,7 +4,6 @@ use Appwrite\Event\Certificate;
use Appwrite\Event\Delete;
use Appwrite\Event\Event;
use Appwrite\Extend\Exception;
use Appwrite\ID;
use Appwrite\Network\Validator\CNAME;
use Appwrite\Network\Validator\Domain as DomainValidator;
use Appwrite\Network\Validator\URL;
@@ -15,6 +14,7 @@ use Utopia\Database\Database;
use Utopia\Database\Document;
use Utopia\Database\Query;
use Utopia\Database\Validator\UID;
use Utopia\Database\ID;
use Utopia\Domains\Domain;
use Utopia\Validator\Text;
use Utopia\Validator\WhiteList;
@@ -96,14 +96,14 @@ App::post('/v1/proxy/rules')
'redirect' => $redirect,
'status' => $status,
'certificateId' => '',
'search' => implode(' ', [ $domain, $ruleId, $resourceId, $resourceType, $redirect ]),
'search' => implode(' ', [ $domain->get(), $ruleId, $resourceId, $resourceType, $redirect ]),
]));
$eventsInstance->setParam('ruleId', $rule->getId());
$response
->setStatusCode(Response::STATUS_CODE_CREATED)
->dynamic($rule, Response::MODEL_RULE);
->dynamic($rule, Response::MODEL_PROXY_RULE);
});
@@ -171,7 +171,7 @@ App::get('/v1/proxy/rules/:ruleId')
->inject('project')
->inject('dbForConsole')
->action(function (string $ruleId, Response $response, Document $project, Database $dbForConsole) {
$rule = $dbForConsole->getDocument('rule', $ruleId);
$rule = $dbForConsole->getDocument('rules', $ruleId);
if ($rule->isEmpty() || $rule->getAttribute('projectInternalId') !== $project->getInternalId()) {
throw new Exception(Exception::RULE_NOT_FOUND);
@@ -252,8 +252,9 @@ App::patch('/v1/proxy/rules/:ruleId/verification')
}
$validator = new CNAME($target->get()); // Verify Domain with DNS records
$domain = new Domain($rule->getAttribute('domain', ''));
if (!$validator->isValid($rule->getAttribute('domain', ''))) {
if (!$validator->isValid($domain->get())) {
throw new Exception(Exception::RULE_VERIFICATION_FAILED);
}
+5
View File
@@ -55,6 +55,11 @@ function router(Database $dbForConsole, string $host, SwooleRequest $swooleReque
throw new AppwriteException(AppwriteException::ROUTER_UNKNOWN_HOST);
}
// Un-verified ignores Appwrite Router. Behaves as Appwrite API. Needed for certificate generation (well-known route)
if($route->getAttribute('status') !== 'verified') {
return false;
}
$type = $route->getAttribute('resourceType');
if($type === 'function') {
+1
View File
@@ -152,6 +152,7 @@ const DELETE_TYPE_ABUSE = 'abuse';
const DELETE_TYPE_USAGE = 'usage';
const DELETE_TYPE_REALTIME = 'realtime';
const DELETE_TYPE_BUCKETS = 'buckets';
const DELETE_TYPE_RULES = 'rules';
const DELETE_TYPE_SESSIONS = 'sessions';
const DELETE_TYPE_CACHE_BY_TIMESTAMP = 'cacheByTimeStamp';
const DELETE_TYPE_CACHE_BY_RESOURCE = 'cacheByResource';
+2
View File
@@ -414,6 +414,8 @@ class CertificatesV1 extends Worker
$rule->setAttribute('certificateId', $certificateId);
$rule->setAttribute('status', $success ? 'verified' : 'failed');
$this->dbForConsole->updateDocument('rules', $rule->getId(), $rule);
// TODO: Event (realtime+webhook+function)
}
}
}
+11 -30
View File
@@ -64,6 +64,8 @@ class DeletesV1 extends Worker
break;
case DELETE_TYPE_BUCKETS:
$this->deleteBucket($document, $project);
case DELETE_TYPE_RULES:
$this->deleteRule($document, $project);
break;
default:
Console::error('No lazy delete operation available for document of type: ' . $document->getCollection());
@@ -681,45 +683,19 @@ class DeletesV1 extends Worker
Console::info("Listed {$count} document by group in " . ($executionEnd - $executionStart) . " seconds");
}
// TODO: Update to new routes delete action
/**
* @param Document $document certificates document
* @param Document $document rule document
* @param Document $project project document
*/
protected function deleteCertificates(Document $document): void
protected function deleteRule(Document $document, Document $project): void
{
$consoleDB = $this->getConsoleDB();
// If domain has certificate generated
if (isset($document['certificateId'])) {
$domainUsingCertificate = $consoleDB->findOne('rules', [
Query::equal('certificateId', [$document['certificateId']])
]);
if (!$domainUsingCertificate) {
$mainDomain = App::getEnv('_APP_DOMAIN_TARGET', '');
if ($mainDomain === $document->getAttribute('domain')) {
$domainUsingCertificate = $mainDomain;
}
}
// If certificate is still used by some domain, mark we can't delete.
// Current domain should not be found, because we only have copy. Original domain is already deleted from database.
if ($domainUsingCertificate) {
Console::warning("Skipping certificate deletion, because a domain is still using it.");
return;
}
}
$domain = $document->getAttribute('domain');
$directory = APP_STORAGE_CERTIFICATES . '/' . $domain;
$checkTraversal = realpath($directory) === $directory;
if ($domain && $checkTraversal && is_dir($directory)) {
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
$consoleDB->deleteDocument('certificates', $document['certificateId']);
}
if ($checkTraversal && is_dir($directory)) {
// Delete files, so Traefik is aware of change
array_map('unlink', glob($directory . '/*.*'));
rmdir($directory);
@@ -727,6 +703,11 @@ class DeletesV1 extends Worker
} else {
Console::info("No certificate files found for {$domain}");
}
// Delete certificate document, so Appwrite is aware of change
if (isset($document['certificateId'])) {
$consoleDB->deleteDocument('certificates', $document['certificateId']);
}
}
protected function deleteBucket(Document $document, Document $project)
@@ -332,6 +332,7 @@ class Swagger2 extends Format
case 'Appwrite\Utopia\Database\Validator\Queries\Executions':
case 'Appwrite\Utopia\Database\Validator\Queries\Files':
case 'Appwrite\Utopia\Database\Validator\Queries\Functions':
case 'Appwrite\Utopia\Database\Validator\Queries\Rules':
case 'Appwrite\Utopia\Database\Validator\Queries\Memberships':
case 'Appwrite\Utopia\Database\Validator\Queries\Projects':
case 'Appwrite\Utopia\Database\Validator\Queries\Teams':