diff --git a/.gitignore b/.gitignore index 50f75e8..cc5d76c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ backups/ # Production deployment configs (contain server IPs and secrets) caddy-server/ authelia-server/ +caddy/Caddyfile caddy/Caddyfile.production # macOS diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..cf4fea3 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,77 @@ +# ============================================================================= +# GitLab CI — matrix-2 deploy.sh integration tests +# +# Runs test_deploy.sh inside a Docker-in-Docker environment. +# Both test scenarios execute sequentially in a single job (~15-20 min). +# +# Requirements on the GitLab runner: +# - Docker executor with privileged mode enabled (for dind) +# - OR shell executor with Docker + docker compose v2 already installed +# ============================================================================= + +stages: + - test + +variables: + # Docker-in-Docker TLS settings + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: "1" + DOCKER_CERT_PATH: "/certs/client" + +# ── Full integration test (config generation + live endpoint checks) ────────── +deploy-integration: + stage: test + image: docker:25-cli + services: + - name: docker:25-dind + alias: docker + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - apk add --no-cache bash openssl curl + - docker info # smoke-test dind connection + script: + - chmod +x test_deploy.sh + - bash test_deploy.sh + after_script: + # Capture container logs on failure for easier debugging + - > + docker compose --project-directory . + -f compose-variants/docker-compose.local.yml + logs --no-color 2>&1 | tail -300 > ci-container-logs.txt || true + timeout: 25 minutes + artifacts: + when: on_failure + paths: + - ci-container-logs.txt + expire_in: 1 week + rules: + - if: '$CI_COMMIT_BRANCH' # all branch pushes + - if: '$CI_MERGE_REQUEST_IID' # all merge requests + +# ── Config-only test (fast path — no endpoint checks, still needs Docker) ───── +# +# Useful for quick feedback on config-generation changes without waiting for +# full service startup. Docker is still required because deploy.sh runs +# `docker run matrixdotorg/synapse:latest generate` to create homeserver.yaml. +deploy-config-only: + stage: test + image: docker:25-cli + services: + - name: docker:25-dind + alias: docker + variables: + DOCKER_TLS_CERTDIR: "/certs" + variables: + SKIP_INTEGRATION: "true" + before_script: + - apk add --no-cache bash openssl curl + - docker info + script: + - chmod +x test_deploy.sh + - bash test_deploy.sh + timeout: 12 minutes + rules: + - if: '$CI_COMMIT_BRANCH' + - if: '$CI_MERGE_REQUEST_IID' diff --git a/caddy/Caddyfile b/caddy/Caddyfile deleted file mode 100644 index 01072cd..0000000 --- a/caddy/Caddyfile +++ /dev/null @@ -1,289 +0,0 @@ -# Local Development Caddyfile for Matrix Stack -# Uses self-signed certificates for local HTTPS testing -# Auto-generated by deploy.sh — do not edit manually - -{ - # Use local CA for self-signed certificates - local_certs - # Enable admin API - admin 0.0.0.0:2019 -} - -# ========================= -# Matrix Homeserver (Synapse) -# ========================= -matrix.example.test:443 { - # TLS with self-signed cert - tls internal - - # Well-known client endpoint - # IMPORTANT: The respond body must stay on a single line. If you edit this file manually - # and your editor wraps the JSON, Caddy will refuse to start with "invalid control character". - @wk path /.well-known/matrix/client - handle @wk { - header Content-Type application/json - header Access-Control-Allow-Origin "*" - respond `{"m.homeserver":{"base_url":"https://matrix.example.test"},"m.authentication":{"issuer":"https://auth.example.test/"},"org.matrix.msc4143.rtc_foci":[{"type":"livekit","livekit_service_url":"https://rtc.example.test/livekit/jwt"}]}` 200 - } - - # Well-known server endpoint (federation) - @wk_server path /.well-known/matrix/server - handle @wk_server { - header Content-Type application/json - respond `{"m.server":"matrix.example.test:443"}` 200 - } - - # Rendezvous endpoints for QR code login (MSC4108) - @rendezvous path_regexp rendezvous ^/_matrix/client/(unstable|v1)/org\.matrix\.(msc3886|msc4108)/rendezvous.*$ - handle @rendezvous { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept, If-Match, If-None-Match" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - encode { - } - reverse_proxy synapse:8008 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # Client versions endpoint with CORS - @versions path /_matrix/client/versions - handle @versions { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy synapse:8008 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # CORS preflight for auth metadata - @auth_preflight { - method OPTIONS - path /_matrix/client/unstable/org.matrix.msc2965/auth_metadata - } - handle @auth_preflight { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Access-Control-Max-Age "86400" - respond 204 - } - - # CORS preflight for all Matrix API - @preflight { - method OPTIONS - path_regexp matrix ^/_matrix/.*$ - } - handle @preflight { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Access-Control-Max-Age "86400" - respond 204 - } - - # MAS compat endpoints (login/logout/refresh) with CORS - @compat path \ - /_matrix/client/v3/login* \ - /_matrix/client/v3/logout* \ - /_matrix/client/v3/refresh* \ - /_matrix/client/r0/login* \ - /_matrix/client/r0/logout* \ - /_matrix/client/r0/refresh* - handle @compat { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy mas:8080 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # Everything else under /_matrix → Synapse with CORS - @matrix_rest path_regexp matrix ^/_matrix/.*$ - handle @matrix_rest { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy synapse:8008 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # Default: everything else → Synapse - handle { - reverse_proxy synapse:8008 - } -} - -# ========================= -# Matrix Authentication Service (MAS) -# ========================= -auth.example.test:443 { - tls internal - - # OIDC Discovery - @disco path /.well-known/openid-configuration - handle @disco { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # Dynamic Client Registration: CORS preflight - @reg_opts { - method OPTIONS - path /oauth2/registration - } - handle @reg_opts { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "POST, OPTIONS" - header ?Access-Control-Allow-Headers "*" - respond 204 - } - - # Dynamic Client Registration (POST) - @reg path /oauth2/registration - route @reg { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "POST, OPTIONS" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # JWKS preflight - @jwks_opts { - method OPTIONS - path /oauth2/keys.json - } - handle @jwks_opts { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - respond 204 - } - - # Map keys.json → /oauth2/jwks (MAS naming) - @jwksjson path /oauth2/keys.json - route @jwksjson { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - uri replace /oauth2/keys.json /oauth2/jwks - reverse_proxy mas:8080 - } - - # Generic OAuth2 endpoints - @oauth path /oauth2/* - route @oauth { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS, POST" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # Account portal - handle_path /account/* { - reverse_proxy mas:8080 - } - - # Authelia endpoints (proxy to authelia) - handle_path /authelia/* { - reverse_proxy authelia:9091 - } - - # Fallback: everything else to MAS - handle { - reverse_proxy mas:8080 - } - - # Add CORS on error responses - handle_errors { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Headers "*" - header ?Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - } -} - -# ========================= -# Authelia SSO -# ========================= -authelia.example.test:443 { - tls internal - - reverse_proxy authelia:9091 -} - -# ========================= -# Element Web Client -# ========================= -element.example.test:443 { - tls internal - - # Serve config.json with proper settings - # IMPORTANT: respond body must stay on a single line — see well-known note above. - @cfg path /config.json - handle @cfg { - header Content-Type application/json - header Cache-Control no-store - respond `{"default_server_config":{"m.homeserver":{"base_url":"https://matrix.example.test","server_name":"matrix.example.test"}},"default_server_name":"matrix.example.test","disable_custom_urls":false,"disable_guests":true,"features":{"feature_oidc_aware_navigation":true,"feature_element_call_video_rooms":true},"element_call":{"url":"https://call.element.io","participant_limit":8,"brand":"Element Call"}}` 200 - } - - # Everything else to Element container - handle { - reverse_proxy element:80 - } -} - -# ========================= -# Element Admin -# ========================= -admin.example.test:443 { - tls internal - - handle { - reverse_proxy element-admin:8080 - } -} - -# ========================= -# Element Call (LiveKit) -# ========================= -rtc.example.test:443 { - tls internal - - handle_path /livekit/jwt* { - reverse_proxy lk-jwt-service:8080 - } - - handle_path /livekit/sfu* { - reverse_proxy livekit:7880 - } -} - -# ========================= -# Element Call Frontend -# ========================= -call.example.test:443 { - tls internal - - reverse_proxy element-call:8080 -} diff --git a/caddy/Caddyfile.production b/caddy/Caddyfile.production deleted file mode 100644 index 1738fd7..0000000 --- a/caddy/Caddyfile.production +++ /dev/null @@ -1,267 +0,0 @@ -# Production Caddyfile for Matrix Stack -# Uses Let's Encrypt for automatic HTTPS certificates -# IMPORTANT: Replace {$DOMAIN} with your actual domain during deployment - -{ - # Production: automatic HTTPS with Let's Encrypt - # local_certs is NOT used in production - - # Enable admin API for troubleshooting - admin 0.0.0.0:2019 - - # Email for Let's Encrypt notifications (set via environment) - email {$ACME_EMAIL} -} - -# ========================= -# Matrix Homeserver (Synapse) -# ========================= -matrix.{$DOMAIN}:443 { - # Automatic HTTPS with Let's Encrypt - # No tls directive needed - Caddy handles it automatically - - # Well-known client endpoint - @wk path /.well-known/matrix/client - handle @wk { - header Content-Type application/json - respond `{"m.homeserver":{"base_url":"https://matrix.{$DOMAIN}"},"m.authentication":{"issuer":"https://auth.{$DOMAIN}/"}}` 200 - } - - # Well-known server endpoint (federation) - @wk_server path /.well-known/matrix/server - handle @wk_server { - header Content-Type application/json - respond `{"m.server":"matrix.{$DOMAIN}:443"}` 200 - } - - # Client versions endpoint with CORS - strip backend headers to prevent duplicates - @versions path /_matrix/client/versions - handle @versions { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy synapse:8008 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # CORS preflight for auth metadata - @auth_preflight { - method OPTIONS - path /_matrix/client/unstable/org.matrix.msc2965/auth_metadata - } - handle @auth_preflight { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Access-Control-Max-Age "86400" - respond 204 - } - - # CORS preflight for all Matrix API - @preflight { - method OPTIONS - path_regexp matrix ^/_matrix/.*$ - } - handle @preflight { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Access-Control-Max-Age "86400" - respond 204 - } - - # MAS compat endpoints (login/logout/refresh) with CORS - @compat path \ - /_matrix/client/v3/login* \ - /_matrix/client/v3/logout* \ - /_matrix/client/v3/refresh* \ - /_matrix/client/r0/login* \ - /_matrix/client/r0/logout* \ - /_matrix/client/r0/refresh* - handle @compat { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy mas:8080 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # Everything else under /_matrix → Synapse with CORS - @matrix_rest path_regexp matrix ^/_matrix/.*$ - handle @matrix_rest { - header Access-Control-Allow-Origin "*" - header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - header Access-Control-Allow-Headers "Authorization, Content-Type, Accept" - header Vary "Origin, Access-Control-Request-Method, Access-Control-Request-Headers" - reverse_proxy synapse:8008 { - header_down -Access-Control-Allow-Origin - header_down -Access-Control-Allow-Methods - header_down -Access-Control-Allow-Headers - header_down -Vary - } - } - - # Federation endpoint (port 8448) - # In production, this should be accessible on port 8448 OR via .well-known - handle /_matrix/federation/* { - reverse_proxy synapse:8008 - } - - # Default: everything else → Synapse - handle { - reverse_proxy synapse:8008 - } -} - -# ========================= -# Matrix Authentication Service (MAS) -# ========================= -auth.{$DOMAIN}:443 { - # Automatic HTTPS with Let's Encrypt - - # OIDC Discovery - @disco path /.well-known/openid-configuration - handle @disco { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # Dynamic Client Registration: CORS preflight - @reg_opts { - method OPTIONS - path /oauth2/registration - } - handle @reg_opts { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "POST, OPTIONS" - header ?Access-Control-Allow-Headers "*" - respond 204 - } - - # Dynamic Client Registration (POST) - @reg path /oauth2/registration - route @reg { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "POST, OPTIONS" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # JWKS preflight - @jwks_opts { - method OPTIONS - path /oauth2/keys.json - } - handle @jwks_opts { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - respond 204 - } - - # Map keys.json → /oauth2/jwks (MAS naming) - @jwksjson path /oauth2/keys.json - route @jwksjson { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS" - header ?Access-Control-Allow-Headers "*" - uri replace /oauth2/keys.json /oauth2/jwks - reverse_proxy mas:8080 - } - - # Generic OAuth2 endpoints - @oauth path /oauth2/* - route @oauth { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Methods "GET, OPTIONS, POST" - header ?Access-Control-Allow-Headers "*" - reverse_proxy mas:8080 - } - - # Account portal - handle_path /account/* { - reverse_proxy mas:8080 - } - - # Authelia endpoints (proxy to authelia if using Authelia profile) - handle_path /authelia/* { - reverse_proxy authelia:9091 - } - - # Fallback: everything else to MAS - handle { - reverse_proxy mas:8080 - } - - # Add CORS on error responses - handle_errors { - header ?Access-Control-Allow-Origin "*" - header ?Access-Control-Allow-Headers "*" - header ?Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" - } -} - -# ========================= -# Authelia SSO (Optional) -# ========================= -authelia.{$DOMAIN}:443 { - # Automatic HTTPS with Let's Encrypt - reverse_proxy authelia:9091 -} - -# ========================= -# Element Web Client -# ========================= -element.{$DOMAIN}:443 { - # Automatic HTTPS with Let's Encrypt - - # Serve config.json with proper settings - @cfg path /config.json - handle @cfg { - header Content-Type application/json - header Cache-Control no-store - respond `{ - "default_server_config": { - "m.homeserver": { - "base_url": "https://matrix.{$DOMAIN}", - "server_name": "matrix.{$DOMAIN}" - } - }, - "default_server_name": "matrix.{$DOMAIN}", - "disable_custom_urls": false, - "disable_guests": true, - "features": { - "feature_oidc_aware_navigation": true - } - }` 200 - } - - # Everything else to Element container - handle { - reverse_proxy element:80 - } -} - -# ========================= -# Element Admin -# ========================= -admin.{$DOMAIN}:443 { - # Automatic HTTPS with Let's Encrypt - - # Proxy to Element Admin container - handle { - reverse_proxy element-admin:80 - } -} diff --git a/deploy.sh b/deploy.sh index 0bc0af6..14f3efb 100755 --- a/deploy.sh +++ b/deploy.sh @@ -281,6 +281,18 @@ if [[ "$DEPLOYMENT_MODE" == "local" ]]; then RTC_DOMAIN="rtc.example.test" CALL_DOMAIN="call.example.test" + # Matrix server name (MXID identity domain) + echo "" + echo -e "${CYAN}Matrix User ID format:${NC}" + echo -e " [1] Short: @user:${DOMAIN_BASE} ← recommended" + echo -e " [2] Subdomain: @user:${MATRIX_DOMAIN}" + read -p "Choose [1/2, default: 1]: " _sn_choice + if [[ "$_sn_choice" == "2" ]]; then + SERVER_NAME="${MATRIX_DOMAIN}" + else + SERVER_NAME="${DOMAIN_BASE}" + fi + echo -e "${CYAN}Local Testing Configuration:${NC}" echo -e " Matrix API: https://${MATRIX_DOMAIN}" echo -e " Element Web: https://${ELEMENT_DOMAIN}" @@ -292,6 +304,9 @@ if [[ "$DEPLOYMENT_MODE" == "local" ]]; then fi echo "" HOSTS_DOMAINS="${MATRIX_DOMAIN} ${ELEMENT_DOMAIN} ${AUTH_DOMAIN} ${AUTHELIA_DOMAIN}" + if [[ "$SERVER_NAME" != "$MATRIX_DOMAIN" ]]; then + HOSTS_DOMAINS="${SERVER_NAME} ${HOSTS_DOMAINS}" + fi if [[ "$USE_ELEMENT_CALL" == true ]]; then HOSTS_DOMAINS="${HOSTS_DOMAINS} ${RTC_DOMAIN} ${CALL_DOMAIN}" fi @@ -348,6 +363,18 @@ else CALL_DOMAIN="${CALL_SUBDOMAIN}.${DOMAIN_BASE}" fi + # Matrix server name (MXID identity domain) + echo "" + echo -e "${CYAN}Matrix User ID format:${NC}" + echo -e " [1] Short: @user:${DOMAIN_BASE} ← recommended" + echo -e " [2] Subdomain: @user:${MATRIX_DOMAIN}" + read -p "Choose [1/2, default: 1]: " _sn_choice + if [[ "$_sn_choice" == "2" ]]; then + SERVER_NAME="${MATRIX_DOMAIN}" + else + SERVER_NAME="${DOMAIN_BASE}" + fi + echo "" echo -e "${CYAN}Backend Server Addresses (for Caddyfile):${NC}" echo -e " ${YELLOW}Enter IP addresses or hostnames${NC}" @@ -368,6 +395,7 @@ else echo "" echo -e "${GREEN}✓${NC} Configuration Summary:" echo -e " Base Domain: ${DOMAIN_BASE}" + echo -e " Server Name: ${SERVER_NAME} (@user:${SERVER_NAME})" echo -e " Matrix: https://${MATRIX_DOMAIN}" echo -e " Element: https://${ELEMENT_DOMAIN}" echo -e " MAS: https://${AUTH_DOMAIN}" @@ -441,7 +469,7 @@ ELEMENT_DOMAIN=${ELEMENT_DOMAIN} ADMIN_DOMAIN=${ADMIN_DOMAIN} AUTH_DOMAIN=${AUTH_DOMAIN} AUTHELIA_DOMAIN=${AUTHELIA_DOMAIN} -SERVER_NAME=${MATRIX_DOMAIN} +SERVER_NAME=${SERVER_NAME} # PostgreSQL POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -765,7 +793,7 @@ upstream_oauth2: set_email_verification: always matrix: - homeserver: '${MATRIX_DOMAIN}' + homeserver: '${SERVER_NAME}' endpoint: 'http://synapse:8008' secret: '${SYNAPSE_SHARED_SECRET}' @@ -777,7 +805,7 @@ else cat >> mas/config/config.yaml << EOF matrix: - homeserver: '${MATRIX_DOMAIN}' + homeserver: '${SERVER_NAME}' endpoint: 'http://synapse:8008' secret: '${SYNAPSE_SHARED_SECRET}' @@ -859,7 +887,7 @@ cat > element/config/config.json << EOF "default_server_config": { "m.homeserver": { "base_url": "https://${MATRIX_DOMAIN}", - "server_name": "${MATRIX_DOMAIN}" + "server_name": "${SERVER_NAME}" } }, "brand": "Element", @@ -880,7 +908,7 @@ cat > element/config/config.json << EOF "features": { "feature_oidc_aware_navigation": true${ELEMENT_CALL_FEATURES} }, - "default_server_name": "${MATRIX_DOMAIN}", + "default_server_name": "${SERVER_NAME}", "disable_custom_urls": false, "disable_guests": true${ELEMENT_CALL_BLOCK} } @@ -913,11 +941,13 @@ echo -e "${BLUE}[12/13] Generating Synapse configuration...${NC}" # Generate homeserver.yaml if it doesn't exist if [ ! -f "synapse/data/homeserver.yaml" ]; then print_info "Generating new homeserver.yaml..." - $DOCKER_CMD run -it --rm \ + $DOCKER_CMD run --rm \ -v $(pwd)/synapse/data:/data \ - -e SYNAPSE_SERVER_NAME=${MATRIX_DOMAIN} \ + -e SYNAPSE_SERVER_NAME=${SERVER_NAME} \ -e SYNAPSE_REPORT_STATS=no \ matrixdotorg/synapse:latest generate + # Fix ownership: synapse runs as uid 991 inside Docker; reclaim files for current user + sudo chown -R "$(id -u):$(id -g)" synapse/data/ print_status "Synapse configuration generated" else print_info "Existing homeserver.yaml found - preserving custom configurations" @@ -995,6 +1025,8 @@ rc_delayed_event_mgmt: EOF fi +# Restore ownership to Synapse uid so the container can read/write its own data +sudo chown -R 991:991 synapse/data/ print_status "Database configuration updated with current credentials" echo "" @@ -1005,10 +1037,10 @@ if [[ "$DEPLOYMENT_MODE" == "local" ]]; then # Pre-build JSON blobs for the local Caddyfile (single-line, no literal \n) if [[ "$USE_ELEMENT_CALL" == true ]]; then LOCAL_WELLKNOWN_JSON="{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\"},\"m.authentication\":{\"issuer\":\"https://${AUTH_DOMAIN}/\"},\"org.matrix.msc4143.rtc_foci\":[{\"type\":\"livekit\",\"livekit_service_url\":\"https://${RTC_DOMAIN}/livekit/jwt\"}]}" - LOCAL_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${MATRIX_DOMAIN}\"}},\"default_server_name\":\"${MATRIX_DOMAIN}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true,\"feature_element_call_video_rooms\":true},\"element_call\":{\"url\":\"https://${CALL_DOMAIN}\",\"participant_limit\":8,\"brand\":\"Element Call\"}}" + LOCAL_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${SERVER_NAME}\"}},\"default_server_name\":\"${SERVER_NAME}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true,\"feature_element_call_video_rooms\":true},\"element_call\":{\"url\":\"https://${CALL_DOMAIN}\",\"participant_limit\":8,\"brand\":\"Element Call\"}}" else LOCAL_WELLKNOWN_JSON="{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\"},\"m.authentication\":{\"issuer\":\"https://${AUTH_DOMAIN}/\"}}" - LOCAL_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${MATRIX_DOMAIN}\"}},\"default_server_name\":\"${MATRIX_DOMAIN}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true}}" + LOCAL_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${SERVER_NAME}\"}},\"default_server_name\":\"${SERVER_NAME}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true}}" fi cat > caddy/Caddyfile << 'CADDYEOF' @@ -1280,6 +1312,32 @@ ${ADMIN_DOMAIN}:443 { } EOF + # Append identity domain well-known block if SERVER_NAME differs from MATRIX_DOMAIN + if [[ "$SERVER_NAME" != "$MATRIX_DOMAIN" ]]; then + cat >> caddy/Caddyfile << EOF + +# ========================= +# Identity Domain (well-known delegation) +# ========================= +${SERVER_NAME}:443 { + tls internal + + @wk path /.well-known/matrix/client + handle @wk { + header Content-Type application/json + header Access-Control-Allow-Origin "*" + respond \`${LOCAL_WELLKNOWN_JSON}\` 200 + } + + @wk_server path /.well-known/matrix/server + handle @wk_server { + header Content-Type application/json + respond \`{"m.server":"${MATRIX_DOMAIN}:443"}\` 200 + } +} +EOF + fi + # Append Element Call blocks if enabled if [[ "$USE_ELEMENT_CALL" == true ]]; then cat >> caddy/Caddyfile << EOF @@ -1454,10 +1512,10 @@ if [[ "$DEPLOYMENT_MODE" == "production" ]]; then # Pre-build conditional JSON blobs for the Caddyfile if [[ "$USE_ELEMENT_CALL" == true ]]; then PROD_WELLKNOWN_JSON="{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\"},\"m.authentication\":{\"issuer\":\"https://${AUTH_DOMAIN}/\"},\"org.matrix.msc4143.rtc_foci\":[{\"type\":\"livekit\",\"livekit_service_url\":\"https://${RTC_DOMAIN}/livekit/jwt\"}]}" - PROD_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${MATRIX_DOMAIN}\"}},\"default_server_name\":\"${MATRIX_DOMAIN}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true,\"feature_element_call_video_rooms\":true},\"element_call\":{\"url\":\"https://${CALL_DOMAIN}\",\"participant_limit\":8,\"brand\":\"Element Call\"}}" + PROD_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${SERVER_NAME}\"}},\"default_server_name\":\"${SERVER_NAME}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true,\"feature_element_call_video_rooms\":true},\"element_call\":{\"url\":\"https://${CALL_DOMAIN}\",\"participant_limit\":8,\"brand\":\"Element Call\"}}" else PROD_WELLKNOWN_JSON="{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\"},\"m.authentication\":{\"issuer\":\"https://${AUTH_DOMAIN}/\"}}" - PROD_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${MATRIX_DOMAIN}\"}},\"default_server_name\":\"${MATRIX_DOMAIN}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true}}" + PROD_ELEMENT_CFG_JSON="{\"default_server_config\":{\"m.homeserver\":{\"base_url\":\"https://${MATRIX_DOMAIN}\",\"server_name\":\"${SERVER_NAME}\"}},\"default_server_name\":\"${SERVER_NAME}\",\"disable_custom_urls\":false,\"disable_guests\":true,\"features\":{\"feature_oidc_aware_navigation\":true}}" fi cat > caddy/Caddyfile.production << EOF @@ -1612,6 +1670,29 @@ ${ADMIN_DOMAIN} { } EOF + # Append identity domain well-known block if SERVER_NAME differs from MATRIX_DOMAIN + if [[ "$SERVER_NAME" != "$MATRIX_DOMAIN" ]]; then + cat >> caddy/Caddyfile.production << EOF + +# ========================= +# Identity Domain (well-known delegation) +# ========================= +${SERVER_NAME} { + @wk path /.well-known/matrix/client + handle @wk { + header Content-Type application/json + respond \`${PROD_WELLKNOWN_JSON}\` 200 + } + + @wk_server path /.well-known/matrix/server + handle @wk_server { + header Content-Type application/json + respond \`{"m.server":"${MATRIX_DOMAIN}:443"}\` 200 + } +} +EOF + fi + # Append Element Call blocks to production Caddyfile if enabled if [[ "$USE_ELEMENT_CALL" == true ]]; then cat >> caddy/Caddyfile.production << EOF diff --git a/test_deploy.sh b/test_deploy.sh new file mode 100755 index 0000000..703f054 --- /dev/null +++ b/test_deploy.sh @@ -0,0 +1,366 @@ +#!/bin/bash +# ============================================================================= +# test_deploy.sh — Integration test suite for deploy.sh +# +# Scenarios: +# A) TLD identity: SERVER_NAME=example.test (@user:example.test) +# B) Subdomain identity: SERVER_NAME=matrix.example.test (@user:matrix.example.test) +# +# Each scenario: +# 1. Runs deploy.sh with pre-set stdin +# 2. Validates all generated config files +# 3. Hits live endpoints via curl (Caddy:443 → 127.0.0.1) +# 4. Tears down the stack and cleans up +# +# Usage: +# ./test_deploy.sh # full suite (config + endpoints) +# SKIP_INTEGRATION=true ./test_deploy.sh # config-file checks only (no endpoint tests) +# +# Requires: docker, docker compose v2, bash ≥ 4, openssl, curl +# ============================================================================= + +set -euo pipefail + +# ─── Colors ────────────────────────────────────────────────────────────────── +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m' +BLUE='\033[0;34m'; CYAN='\033[0;36m'; MAGENTA='\033[0;35m' +BOLD='\033[1m'; NC='\033[0m' + +# ─── Config ─────────────────────────────────────────────────────────────────── +SKIP_INTEGRATION="${SKIP_INTEGRATION:-false}" +COMPOSE_FILE="compose-variants/docker-compose.local.yml" +COMPOSE_CMD="sudo docker compose --project-directory ." + +# ─── Counters ───────────────────────────────────────────────────────────────── +TESTS_PASSED=0 +TESTS_FAILED=0 + +# ─── Output helpers ─────────────────────────────────────────────────────────── +pass() { echo -e " ${GREEN}✓${NC} $1"; TESTS_PASSED=$((TESTS_PASSED + 1)); } +fail() { echo -e " ${RED}✗${NC} $1"; TESTS_FAILED=$((TESTS_FAILED + 1)); } +info() { echo -e " ${BLUE}ℹ${NC} $1"; } +warn() { echo -e " ${YELLOW}⚠${NC} $1"; } +header() { echo -e "\n${BOLD}${CYAN}── $1 ──${NC}"; } +section(){ echo -e "\n${BOLD}${MAGENTA}════ $1 ════${NC}"; } + +# ─── Sudo shim for root CI environments that lack sudo ─────────────────────── +setup_sudo_shim() { + if ! command -v sudo &>/dev/null; then + local d; d=$(mktemp -d) + printf '#!/bin/sh\nexec "$@"\n' > "$d/sudo" + chmod +x "$d/sudo" + export PATH="$d:$PATH" + info "Created sudo passthrough shim (running as root)" + fi +} + +# ─── Prerequisites ──────────────────────────────────────────────────────────── +check_prereqs() { + header "Prerequisites" + local ok=true + + for cmd in bash openssl curl; do + command -v "$cmd" &>/dev/null \ + && pass "$cmd available" \ + || { fail "$cmd not found"; ok=false; } + done + + sudo docker ps &>/dev/null \ + && pass "Docker daemon reachable" \ + || { fail "Docker not accessible (sudo docker ps failed)"; ok=false; } + + sudo docker compose version &>/dev/null \ + && pass "docker compose v2 available" \ + || { fail "docker compose not available"; ok=false; } + + [[ -f deploy.sh ]] \ + || { fail "deploy.sh not found — run from repo root"; ok=false; } + + [[ "$ok" == "true" ]] || { echo -e "\n${RED}Prerequisites failed. Aborting.${NC}"; exit 1; } +} + +# ─── Stop stack and wipe all data volumes ───────────────────────────────────── +teardown_stack() { + info "Stopping Docker stack and removing volumes..." + $COMPOSE_CMD -f "$COMPOSE_FILE" down -v --remove-orphans 2>/dev/null || true + sudo rm -rf postgres/data mas/data mas/certs caddy/data caddy/config 2>/dev/null || true + # Wipe synapse/data fully so no leftover signing keys or log configs + # confuse the next scenario's `docker run ... generate` step + sudo rm -rf synapse/data 2>/dev/null || true + mkdir -p synapse/data +} + +# ─── Remove all generated config files ──────────────────────────────────────── +cleanup_configs() { + info "Removing generated configs..." + rm -f .env mas-signing.key authelia_private.pem + rm -f caddy/Caddyfile caddy/Caddyfile.production + rm -f livekit/livekit.yaml + # These may be root-owned from docker run or previous deploys + sudo rm -f mas/config/config.yaml 2>/dev/null || true + sudo rm -f element/config/config.json 2>/dev/null || true + sudo rm -f authelia/config/configuration.yml authelia/config/users_database.yml 2>/dev/null || true + sudo rm -f synapse/data/homeserver.yaml synapse/data/homeserver.yaml.bak 2>/dev/null || true +} + +# ─── Assertions ─────────────────────────────────────────────────────────────── +assert_file() { + local file="$1" label="$2" + [[ -f "$file" ]] && pass "$label" || fail "$label (missing: $file)" +} + +assert_contains() { + local file="$1" pattern="$2" label="$3" + if grep -qF "$pattern" "$file" 2>/dev/null; then + pass "$label" + else + fail "$label [expected '${pattern}' in ${file}]" + fi +} + +assert_not_contains() { + local file="$1" pattern="$2" label="$3" + if grep -qF "$pattern" "$file" 2>/dev/null; then + fail "$label [unexpected '${pattern}' found in ${file}]" + else + pass "$label" + fi +} + +# Regex variant — use when the value may be quoted/unquoted (grep -E) +assert_matches() { + local file="$1" pattern="$2" label="$3" + if grep -qE "$pattern" "$file" 2>/dev/null; then + pass "$label" + else + fail "$label [expected /$pattern/ in ${file}]" + fi +} + +# ─── Config-file assertions (no Docker needed) ──────────────────────────────── +assert_configs() { + local server_name="$1" + local matrix_domain="matrix.example.test" + + header "Config assertions (SERVER_NAME=${server_name})" + + # .env + assert_file ".env" ".env generated" + assert_contains ".env" "SERVER_NAME=${server_name}" ".env → SERVER_NAME" + assert_contains ".env" "MATRIX_DOMAIN=${matrix_domain}" ".env → MATRIX_DOMAIN" + + # MAS config + assert_file "mas/config/config.yaml" "mas/config/config.yaml generated" + assert_contains "mas/config/config.yaml" \ + "homeserver: '${server_name}'" "MAS → homeserver" + + # Element Web config (heredoc format has spaces: `"key": "value"`) + assert_file "element/config/config.json" "element/config/config.json generated" + assert_contains "element/config/config.json" \ + "\"server_name\": \"${server_name}\"" "Element → server_name" + assert_contains "element/config/config.json" \ + "\"default_server_name\": \"${server_name}\"" "Element → default_server_name" + assert_contains "element/config/config.json" \ + "\"base_url\": \"https://${matrix_domain}\"" "Element → base_url stays matrix domain" + + # Synapse homeserver.yaml + assert_file "synapse/data/homeserver.yaml" "synapse/data/homeserver.yaml generated" + # Synapse may quote the value: `server_name: "example.test"` or `server_name: example.test` + assert_matches "synapse/data/homeserver.yaml" \ + "^server_name: \"?${server_name//./\\.}\"?" "Synapse → server_name" + + # Caddyfile (JSON blobs are compact, no spaces around ':') + assert_file "caddy/Caddyfile" "caddy/Caddyfile generated" + assert_contains "caddy/Caddyfile" \ + "\"server_name\":\"${server_name}\"" "Caddyfile JSON → server_name" + assert_contains "caddy/Caddyfile" \ + "\"default_server_name\":\"${server_name}\"" "Caddyfile JSON → default_server_name" + assert_contains "caddy/Caddyfile" \ + "\"base_url\":\"https://${matrix_domain}\"" "Caddyfile JSON → base_url stays matrix domain" + + if [[ "$server_name" != "$matrix_domain" ]]; then + # TLD mode: identity domain block must be present + assert_contains "caddy/Caddyfile" \ + "# Identity Domain (well-known delegation)" "Caddyfile → identity domain block present" + assert_contains "caddy/Caddyfile" \ + "${server_name}:443 {" "Caddyfile → ${server_name}:443 block" + assert_contains "caddy/Caddyfile" \ + "\"m.server\":\"${matrix_domain}:443\"" "Caddyfile → m.server delegates to matrix domain" + else + # Subdomain mode: no identity domain block + assert_not_contains "caddy/Caddyfile" \ + "# Identity Domain (well-known delegation)" "Caddyfile → no identity block in subdomain mode" + fi +} + +# ─── Curl an HTTPS endpoint, routing *.example.test → 127.0.0.1 ───────────── +curl_local() { + local domain="$1" path="$2" + local -a args=(-sf --max-time 15 --resolve "${domain}:443:127.0.0.1") + if [[ -f mas/certs/caddy-ca.crt ]]; then + args+=(--cacert mas/certs/caddy-ca.crt) + else + args+=(-k) + fi + curl "${args[@]}" "https://${domain}${path}" 2>/dev/null || true +} + +# ─── Copy Caddy CA to MAS and restart MAS ──────────────────────────────────── +# deploy.sh only waits 5s for the PKI file — not reliable. Use Caddy's admin +# API (localhost:2019) instead, which returns the CA cert as soon as Caddy is up. +setup_mas_ca() { + local ca_dst="mas/certs/caddy-ca.crt" + local waited=0 + info "Fetching Caddy CA via admin API (up to 60s)..." + while (( waited < 60 )); do + local resp; resp=$(curl -sf --max-time 5 http://localhost:2019/pki/ca/local 2>/dev/null || echo "") + if echo "$resp" | grep -q '"root_certificate"'; then + # Extract PEM from JSON: value uses literal \n; strip key/quotes, unescape newlines + echo "$resp" \ + | grep -o '"root_certificate":"[^"]*"' \ + | sed 's/"root_certificate":"//; s/"$//' \ + | sed 's/\\n/\n/g' \ + | sudo tee "$ca_dst" > /dev/null + info "Caddy CA fetched → restarting MAS..." + $COMPOSE_CMD -f "$COMPOSE_FILE" restart mas 2>/dev/null || true + sleep 10 + return 0 + fi + sleep 3; waited=$((waited + 3)) + done + warn "Caddy admin API did not return CA after 60s — MAS OIDC test will fail" +} + +# ─── Live endpoint assertions ───────────────────────────────────────────────── +assert_endpoints() { + local server_name="$1" + local matrix_domain="matrix.example.test" + + header "Endpoint tests (SERVER_NAME=${server_name})" + info "Allowing 20s for full service initialization..." + sleep 20 + + setup_mas_ca + + # Synapse /health + local health; health=$(curl_local "$matrix_domain" "/health") + [[ "$health" == "OK" ]] \ + && pass "Synapse /health → OK" \ + || fail "Synapse /health (got: '${health:-no response}')" + + # .well-known/matrix/client on matrix domain + local wk; wk=$(curl_local "$matrix_domain" "/.well-known/matrix/client") + echo "$wk" | grep -q '"m.homeserver"' \ + && pass "${matrix_domain} → .well-known/matrix/client responds" \ + || fail "${matrix_domain} → .well-known/matrix/client (got: '${wk:-no response}')" + echo "$wk" | grep -q "https://${matrix_domain}" \ + && pass ".well-known base_url = https://${matrix_domain}" \ + || fail ".well-known base_url wrong (got: '${wk:-}')" + + if [[ "$server_name" != "$matrix_domain" ]]; then + # TLD mode: identity domain must serve well-known + + local wk_id; wk_id=$(curl_local "$server_name" "/.well-known/matrix/client") + echo "$wk_id" | grep -q '"m.homeserver"' \ + && pass "${server_name} → .well-known/matrix/client responds" \ + || fail "${server_name} → .well-known/matrix/client (got: '${wk_id:-no response}')" + + local wk_srv; wk_srv=$(curl_local "$server_name" "/.well-known/matrix/server") + echo "$wk_srv" | grep -q '"m.server"' \ + && pass "${server_name} → .well-known/matrix/server responds" \ + || fail "${server_name} → .well-known/matrix/server (got: '${wk_srv:-no response}')" + echo "$wk_srv" | grep -q "$matrix_domain" \ + && pass ".well-known/matrix/server delegates to ${matrix_domain}" \ + || fail ".well-known/matrix/server missing ${matrix_domain} (got: '${wk_srv:-}')" + fi + + # MAS OIDC discovery + local oidc; oidc=$(curl_local "auth.example.test" "/.well-known/openid-configuration") + echo "$oidc" | grep -q '"issuer"' \ + && pass "MAS OIDC discovery responds" \ + || fail "MAS OIDC discovery (got: '${oidc:-no response}')" + + # Element Web + local elem; elem=$(curl_local "element.example.test" "/") + echo "$elem" | grep -qi "element" \ + && pass "Element Web root serves HTML" \ + || fail "Element Web root (no Element content in response)" +} + +# ─── Run one full scenario ──────────────────────────────────────────────────── +run_scenario() { + local name="$1" + local sn_choice="$2" # "1" = TLD, "2" = subdomain + local expected_sn="$3" + + section "$name" + teardown_stack + cleanup_configs + + info "Running deploy.sh (piped stdin)" + + # Stdin answers in prompt order: + # [1] Deployment type: 1 (local) + # [2] Include Authelia? n + # [3] Enable Element Call? n + # [4] Custom Docker registry prefix: (empty → default) + # [5] Use hardened images? n + # [6] SERVER_NAME choice: $sn_choice (1=TLD, 2=subdomain) + # [7] Press Enter to continue: (empty) + printf '%s\n' "1" "n" "n" "" "n" "$sn_choice" "" \ + | bash deploy.sh + + assert_configs "$expected_sn" + + if [[ "$SKIP_INTEGRATION" == "true" ]]; then + warn "Skipping endpoint tests (SKIP_INTEGRATION=true)" + else + assert_endpoints "$expected_sn" + fi +} + +# ─── Summary ────────────────────────────────────────────────────────────────── +print_summary() { + echo "" + echo -e "${BOLD}${CYAN}══════════════════════════════════════${NC}" + echo -e "${BOLD} Results: ${GREEN}${TESTS_PASSED} passed${NC}${BOLD}, ${RED}${TESTS_FAILED} failed${NC}" + echo -e "${BOLD}${CYAN}══════════════════════════════════════${NC}" + echo "" + if (( TESTS_FAILED > 0 )); then + echo -e "${RED}✗ Test suite FAILED${NC}" + exit 1 + else + echo -e "${GREEN}✓ All tests PASSED${NC}" + fi +} + +# ─── Cleanup on exit (INT, TERM, or normal exit) ────────────────────────────── +cleanup_on_exit() { + echo "" + section "Cleanup" + teardown_stack + cleanup_configs + info "Done." +} +trap cleanup_on_exit EXIT + +# ─── Main ───────────────────────────────────────────────────────────────────── +cd "$(dirname "$(realpath "$0")")" # ensure we're in the repo root +setup_sudo_shim +check_prereqs + +# Scenario A — TLD identity: @user:example.test +run_scenario \ + "A · TLD identity (@user:example.test)" \ + "1" \ + "example.test" + +# Scenario B — Subdomain identity: @user:matrix.example.test +run_scenario \ + "B · Subdomain identity (@user:matrix.example.test)" \ + "2" \ + "matrix.example.test" + +trap - EXIT +cleanup_on_exit +print_summary