From bbeeb356ab4f740588fd9b810d70ae0cf3fa6369 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 26 Aug 2022 21:21:08 +1200
Subject: [PATCH] Allow flaky tests to be automatically retried with an
 attribute

---
 tests/e2e/Scopes/Scope.php                    |  3 +
 .../Functions/FunctionsCustomServerTest.php   |  2 +
 tests/e2e/Services/Webhooks/WebhooksBase.php  |  2 +
 tests/extensions/FlakyTest.php                | 16 +++++
 tests/extensions/Retryable.php                | 70 +++++++++++++++++++
 5 files changed, 93 insertions(+)
 create mode 100644 tests/extensions/FlakyTest.php
 create mode 100644 tests/extensions/Retryable.php

diff --git a/tests/e2e/Scopes/Scope.php b/tests/e2e/Scopes/Scope.php
index a4d65636a6..673ca8173b 100644
--- a/tests/e2e/Scopes/Scope.php
+++ b/tests/e2e/Scopes/Scope.php
@@ -2,11 +2,14 @@
 
 namespace Tests\E2E\Scopes;
 
+use Appwrite\Tests\Retryable;
 use Tests\E2E\Client;
 use PHPUnit\Framework\TestCase;
 
 abstract class Scope extends TestCase
 {
+    use Retryable;
+
     /**
      * @var Client
      */
diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php
index 444611e07a..6f4d182d56 100644
--- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php
+++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php
@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\Services\Functions;
 
+use Appwrite\Tests\FlakyTest;
 use CURLFile;
 use Tests\E2E\Client;
 use Tests\E2E\Scopes\ProjectCustom;
@@ -1206,6 +1207,7 @@ class FunctionsCustomServerTest extends Scope
         $this->assertEquals(204, $response['headers']['status-code']);
     }
 
+    #[FlakyTest(retries: 1)]
     public function testCreateCustomRubyExecution()
     {
         $name = 'ruby-3.1';
diff --git a/tests/e2e/Services/Webhooks/WebhooksBase.php b/tests/e2e/Services/Webhooks/WebhooksBase.php
index c1647b6e87..03854978c1 100644
--- a/tests/e2e/Services/Webhooks/WebhooksBase.php
+++ b/tests/e2e/Services/Webhooks/WebhooksBase.php
@@ -2,6 +2,7 @@
 
 namespace Tests\E2E\Services\Webhooks;
 
+use Appwrite\Tests\FlakyTest;
 use CURLFile;
 use Tests\E2E\Client;
 
@@ -296,6 +297,7 @@ trait WebhooksBase
     /**
      * @depends testCreateCollection
      */
+    #[FlakyTest(retries: 1)]
     public function testDeleteDocument(array $data): array
     {
         $actorsId = $data['actorsId'];
diff --git a/tests/extensions/FlakyTest.php b/tests/extensions/FlakyTest.php
new file mode 100644
index 0000000000..7f930094f4
--- /dev/null
+++ b/tests/extensions/FlakyTest.php
@@ -0,0 +1,16 @@
+<?php
+
+namespace Appwrite\Tests;
+
+/**
+ * Allows test methods to be retried if they fail.
+ *
+ * Requires that the test class extends {@see TestCase} and has trait {@see Retryable}.
+ */
+#[\Attribute(\Attribute::TARGET_METHOD)]
+class FlakyTest
+{
+    public function __construct(protected int $retries = 1)
+    {
+    }
+}
diff --git a/tests/extensions/Retryable.php b/tests/extensions/Retryable.php
new file mode 100644
index 0000000000..c7fc8e22d2
--- /dev/null
+++ b/tests/extensions/Retryable.php
@@ -0,0 +1,70 @@
+<?php
+
+namespace Appwrite\Tests;
+
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Allows test methods annotated with {@see FlakyTest} to be retried.
+ */
+trait Retryable
+{
+    /**
+     * Custom runBare, hides and defers to PHPUnit {@see TestCase} runBare function,
+     * accounting for any retries configured by the {@see FlakyTest} annotation.
+     *
+     * @return void
+     * @throws \ReflectionException
+     * @throws \Throwable
+     */
+    public function runBare(): void
+    {
+        $retries = $this->getNumberOfRetries();
+        $ex = null;
+        for ($i = 0; $i <= $retries; ++$i) {
+            try {
+                parent::runBare();
+                return;
+            } catch (\Throwable | \Exception $ex) {
+                // Swallow the exception until we have exhausted our retries.
+                if ($i !== $retries) {
+                    echo 'Flaky test failed, retrying...' . PHP_EOL;
+                }
+            }
+        }
+        if ($ex) {
+            throw $ex;
+        }
+    }
+
+    /**
+     * @return int
+     * @throws \ReflectionException
+     */
+    private function getNumberOfRetries(): int
+    {
+        $root = new \ReflectionClass($this);
+        $case = $this->getTestCaseRoot($root);
+        $name = $case->getProperty('name');
+        $name->setAccessible(true);
+        $name = $name->getValue($this);
+        $method = $root->getMethod($name);
+        $attributes = $method->getAttributes(FlakyTest::class);
+        $attribute = $attributes[0] ?? null;
+        $args = $attribute?->getArguments();
+        $retries = $args['retries'] ?? 0;
+        return \max(0, $retries);
+    }
+
+    /**
+     * @param \ReflectionClass $reflection
+     * @return \ReflectionClass
+     */
+    private function getTestCaseRoot(\ReflectionClass $reflection): \ReflectionClass
+    {
+        if ($reflection->getName() === TestCase::class) {
+            return $reflection;
+        }
+        return $this->getTestCaseRoot($reflection->getParentClass());
+    }
+}