From 63b2a1fb7fff0c7c22c9105585e8b95a26f37be7 Mon Sep 17 00:00:00 2001
From: Chirag Aggarwal <chiragaggarwal5k@gmail.com>
Date: Tue, 21 Apr 2026 17:07:05 +0530
Subject: [PATCH] Harden benchmark baseline reporting

---
 .github/workflows/ci.yml  | 41 ++++++++++++++++++++++++++++-----------
 tests/benchmarks/http.php |  6 +++---
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 23aa1f21de..403160a8aa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -683,10 +683,13 @@ jobs:
           docker tag ${{ env.IMAGE }} ${{ env.IMAGE }}:after
 
       - name: Prepare benchmark before
+        id: benchmark_before_prepare
+        continue-on-error: true
         run: |
           git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
           git worktree add --detach /tmp/appwrite-benchmark-before ${{ github.event.pull_request.base.sha }}
           docker build \
+            --cache-from ${{ env.IMAGE }}:after \
             --target development \
             --build-arg DEBUG=false \
             --build-arg TESTING=true \
@@ -695,6 +698,9 @@ jobs:
             /tmp/appwrite-benchmark-before
 
       - name: Start before Appwrite
+        id: benchmark_before_start
+        if: steps.benchmark_before_prepare.outcome == 'success'
+        continue-on-error: true
         working-directory: /tmp/appwrite-benchmark-before
         run: |
           docker tag ${{ env.IMAGE }}:before ${{ env.IMAGE }}
@@ -702,13 +708,15 @@ jobs:
           docker compose up -d --wait --no-build
 
       - name: Benchmark before
+        if: steps.benchmark_before_start.outcome == 'success'
+        continue-on-error: true
         run: |
           rm -f benchmark-before-summary.json benchmark-after-summary.json benchmark-before.txt benchmark.txt
           docker run --rm -i --network host --user "$(id -u):$(id -g)" -v "$PWD:/scripts" -w /scripts \
             -e APPWRITE_ENDPOINT=http://localhost/v1 \
             -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \
             -e APPWRITE_BENCHMARK_ITERATIONS=1 \
-            -e APPWRITE_BENCHMARK_VUS=1 \
+            -e APPWRITE_BENCHMARK_RUNS=1 \
             -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-before-summary.json \
             ${{ env.IMAGE }}:before php tests/benchmarks/http.php | tee benchmark-before.txt
 
@@ -717,7 +725,7 @@ jobs:
         run: |
           if [ -d /tmp/appwrite-benchmark-before ]; then
             cd /tmp/appwrite-benchmark-before
-            docker compose down -v
+            docker compose down -v || true
           fi
 
       - name: Start after Appwrite
@@ -732,7 +740,7 @@ jobs:
             -e APPWRITE_ENDPOINT=http://localhost/v1 \
             -e APPWRITE_MAILDEV_ENDPOINT=http://localhost:9503/email \
             -e APPWRITE_BENCHMARK_ITERATIONS=1 \
-            -e APPWRITE_BENCHMARK_VUS=1 \
+            -e APPWRITE_BENCHMARK_RUNS=1 \
             -e APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH=benchmark-before-summary.json \
             -e APPWRITE_BENCHMARK_SUMMARY_PATH=benchmark-after-summary.json \
             ${{ env.IMAGE }}:after php tests/benchmarks/http.php | tee benchmark.txt
@@ -746,29 +754,37 @@ jobs:
           docker run --rm -i -v "$PWD:/scripts" -w /scripts ${{ env.IMAGE }}:after php <<'PHP' > benchmark-comment.txt
           <?php
 
-          function read_summary(string $path): array
+          function fail_summary(string $message, bool $required): void
+          {
+              fwrite(STDERR, $message . "\n");
+              if ($required) {
+                  exit(1);
+              }
+          }
+
+          function read_summary(string $path, bool $required = true): ?array
           {
               if (!is_file($path)) {
-                  fwrite(STDERR, "Missing benchmark summary: {$path}\n");
-                  exit(1);
+                  fail_summary("Missing benchmark summary: {$path}", $required);
+                  return null;
               }
 
               try {
                   $summary = json_decode(file_get_contents($path), true, 512, JSON_THROW_ON_ERROR);
               } catch (JsonException $error) {
-                  fwrite(STDERR, "Invalid benchmark summary {$path}: {$error->getMessage()}\n");
-                  exit(1);
+                  fail_summary("Invalid benchmark summary {$path}: {$error->getMessage()}", $required);
+                  return null;
               }
 
               if (!is_array($summary)) {
-                  fwrite(STDERR, "Invalid benchmark summary {$path}: expected JSON object\n");
-                  exit(1);
+                  fail_summary("Invalid benchmark summary {$path}: expected JSON object", $required);
+                  return null;
               }
 
               return $summary;
           }
 
-          $before = read_summary('benchmark-before-summary.json');
+          $before = read_summary('benchmark-before-summary.json', false);
           $after = read_summary('benchmark-after-summary.json');
 
           function metric_value(?array $data, string $metric, string $stat): mixed
@@ -829,6 +845,9 @@ jobs:
           echo "<!-- appwrite-benchmark-results -->\n";
           echo "## :sparkles: Benchmark results\n\n";
           echo 'Comparing `${{ github.event.pull_request.base.ref }}` (before) to `${{ github.event.pull_request.head.ref }}` (after).' . "\n\n";
+          if ($before === null) {
+              echo "> Before benchmark did not complete; showing current branch metrics only.\n\n";
+          }
           echo "| Metric | Before | After | Delta |\n";
           echo "| --- | ---: | ---: | ---: |\n";
           echo implode("\n", $rows) . "\n\n";
diff --git a/tests/benchmarks/http.php b/tests/benchmarks/http.php
index 1c28ef33ea..c44677c4ec 100644
--- a/tests/benchmarks/http.php
+++ b/tests/benchmarks/http.php
@@ -260,7 +260,7 @@ final class HttpBenchmark
     private int $mailTimeoutMs;
     private int $workerTimeoutMs;
     private int $iterations;
-    private int $vus;
+    private int $runs;
     private string $summaryPath;
     private ?array $previousSummary;
 
@@ -276,7 +276,7 @@ final class HttpBenchmark
         $this->mailTimeoutMs = (int) $this->env('APPWRITE_MAIL_TIMEOUT_MS', '20000');
         $this->workerTimeoutMs = (int) $this->env('APPWRITE_WORKER_TIMEOUT_MS', '60000');
         $this->iterations = max(1, (int) $this->env('APPWRITE_BENCHMARK_ITERATIONS', '1'));
-        $this->vus = max(1, (int) $this->env('APPWRITE_BENCHMARK_VUS', '1'));
+        $this->runs = max(1, (int) $this->env('APPWRITE_BENCHMARK_RUNS', $this->env('APPWRITE_BENCHMARK_VUS', '1')));
         $this->summaryPath = $this->env('APPWRITE_BENCHMARK_SUMMARY_PATH', 'tests/benchmarks/http-summary.json');
         $this->previousSummary = $this->loadPreviousSummary($this->env('APPWRITE_BENCHMARK_PREVIOUS_SUMMARY_PATH', $this->summaryPath));
     }
@@ -289,7 +289,7 @@ final class HttpBenchmark
         try {
             $context = $this->setup();
 
-            for ($i = 0; $i < $this->iterations * $this->vus; $i++) {
+            for ($i = 0; $i < $this->iterations * $this->runs; $i++) {
                 try {
                     $this->curatedFlows($context);
                 } catch (Throwable $error) {