Implement solution for ShadowTree commmit exhaustion using recursive locks (behind a flag) (#52795)

Summary:
Pull Request resolved: https://github.com/facebook/react-native/pull/52795

Changelog: [internal]

This is another attempt to fix https://github.com/facebook/react-native/issues/51870, inspired by https://github.com/facebook/react-native/pull/52314 but gated behind a feature flag until we've tested it carefully.

Reviewed By: sammy-SC

Differential Revision: D78817100

fbshipit-source-id: 45e6cae019b212528f2b2e74b9f52fe43d07f537

packages/react-native/ReactCommon/react/renderer/mounting/ShadowTree.cpp

@@ -22,6 +22,10 @@
 
 namespace facebook::react {
 
+namespace {
+const int MAX_COMMIT_ATTEMPTS_BEFORE_LOCKING = 3;
+} // namespace
+
 using CommitStatus = ShadowTree::CommitStatus;
 using CommitMode = ShadowTree::CommitMode;
 
@@ -210,7 +214,8 @@ void ShadowTree::setCommitMode(CommitMode commitMode) const {
   auto revision = ShadowTreeRevision{};
 
   {
-    std::unique_lock lock(commitMutex_);
+    ShadowTree::UniqueLock lock = uniqueCommitLock();
+
     if (commitMode_ == commitMode) {
       return;
     }
@@ -227,7 +232,7 @@ void ShadowTree::setCommitMode(CommitMode commitMode) const {
 }
 
 CommitMode ShadowTree::getCommitMode() const {
-  std::shared_lock lock(commitMutex_);
+  SharedLock lock = sharedCommitLock();
   return commitMode_;
 }
 
@@ -241,17 +246,32 @@ CommitStatus ShadowTree::commit(
     const CommitOptions& commitOptions) const {
   [[maybe_unused]] int attempts = 0;
 
-  while (true) {
-    attempts++;
-
-    auto status = tryCommit(transaction, commitOptions);
-    if (status != CommitStatus::Failed) {
-      return status;
+  if (ReactNativeFeatureFlags::preventShadowTreeCommitExhaustion()) {
+    while (attempts < MAX_COMMIT_ATTEMPTS_BEFORE_LOCKING) {
+      auto status = tryCommit(transaction, commitOptions);
+      if (status != CommitStatus::Failed) {
+        return status;
+      }
+      attempts++;
     }
 
-    // After multiple attempts, we failed to commit the transaction.
-    // Something internally went terribly wrong.
-    react_native_assert(attempts < 1024);
+    {
+      std::unique_lock lock(commitMutexRecursive_);
+      return tryCommit(transaction, commitOptions);
+    }
+  } else {
+    while (true) {
+      attempts++;
+
+      auto status = tryCommit(transaction, commitOptions);
+      if (status != CommitStatus::Failed) {
+        return status;
+      }
+
+      // After multiple attempts, we failed to commit the transaction.
+      // Something internally went terribly wrong.
+      react_native_assert(attempts < 1024);
+    }
   }
 }
 
@@ -269,7 +289,7 @@ CommitStatus ShadowTree::tryCommit(
 
   {
     // Reading `currentRevision_` in shared manner.
-    std::shared_lock lock(commitMutex_);
+    SharedLock lock = sharedCommitLock();
     commitMode = commitMode_;
     oldRevision = currentRevision_;
   }
@@ -310,7 +330,7 @@ CommitStatus ShadowTree::tryCommit(
 
   {
     // Updating `currentRevision_` in unique manner if it hasn't changed.
-    std::unique_lock lock(commitMutex_);
+    UniqueLock lock = uniqueCommitLock();
 
     if (currentRevision_.number != oldRevision.number) {
       return CommitStatus::Failed;
@@ -349,7 +369,7 @@ CommitStatus ShadowTree::tryCommit(
 }
 
 ShadowTreeRevision ShadowTree::getCurrentRevision() const {
-  std::shared_lock lock(commitMutex_);
+  SharedLock lock = sharedCommitLock();
   return currentRevision_;
 }
 
@@ -397,4 +417,20 @@ void ShadowTree::notifyDelegatesOfUpdates() const {
   delegate_.shadowTreeDidFinishTransaction(mountingCoordinator_, true);
 }
 
+inline ShadowTree::UniqueLock ShadowTree::uniqueCommitLock() const {
+  if (ReactNativeFeatureFlags::preventShadowTreeCommitExhaustion()) {
+    return std::unique_lock{commitMutexRecursive_};
+  } else {
+    return std::unique_lock{commitMutex_};
+  }
+}
+
+inline ShadowTree::SharedLock ShadowTree::sharedCommitLock() const {
+  if (ReactNativeFeatureFlags::preventShadowTreeCommitExhaustion()) {
+    return std::unique_lock{commitMutexRecursive_};
+  } else {
+    return std::shared_lock{commitMutex_};
+  }
+}
+
 } // namespace facebook::react

packages/react-native/ReactCommon/react/renderer/mounting/ShadowTree.h

@@ -8,6 +8,8 @@
 #pragma once
 
 #include <memory>
+#include <mutex>
+#include <shared_mutex>
 
 #include <react/renderer/components/root/RootShadowNode.h>
 #include <react/renderer/core/LayoutConstraints.h>
@@ -150,10 +152,21 @@ class ShadowTree final {
   const SurfaceId surfaceId_;
   const ShadowTreeDelegate& delegate_;
   mutable std::shared_mutex commitMutex_;
+  mutable std::recursive_mutex commitMutexRecursive_;
   mutable CommitMode commitMode_{
       CommitMode::Normal}; // Protected by `commitMutex_`.
   mutable ShadowTreeRevision currentRevision_; // Protected by `commitMutex_`.
   std::shared_ptr<const MountingCoordinator> mountingCoordinator_;
+
+  using UniqueLock = std::variant<
+      std::unique_lock<std::shared_mutex>,
+      std::unique_lock<std::recursive_mutex>>;
+  using SharedLock = std::variant<
+      std::shared_lock<std::shared_mutex>,
+      std::unique_lock<std::recursive_mutex>>;
+
+  inline UniqueLock uniqueCommitLock() const;
+  inline SharedLock sharedCommitLock() const;
 };
 
 } // namespace facebook::react

packages/react-native/scripts/featureflags/ReactNativeFeatureFlags.config.js

@@ -821,6 +821,17 @@ const definitions: FeatureFlagDefinitions = {
       },
       ossReleaseStage: 'none',
     },
+    preventShadowTreeCommitExhaustion: {
+      defaultValue: false,
+      metadata: {
+        dateAdded: '2025-07-23',
+        description:
+          'Enables a new mechanism in ShadowTree to prevent problems caused by multiple threads trying to commit concurrently. If a thread tries to commit a few times unsuccessfully, it will acquire a lock and try again.',
+        expectedReleaseValue: true,
+        purpose: 'experimentation',
+      },
+      ossReleaseStage: 'experimental',
+    },
   },
 };
 

packages/react-native/src/private/featureflags/ReactNativeFeatureFlags.js

@@ -4,7 +4,7 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  *
- * @generated SignedSource<<b75fccb46a36b07c692d890f0659f9a3>>
+ * @generated SignedSource<<bd2acd901f794a0d6e13712a34aaac5d>>
  * @flow strict
  * @noformat
  */
@@ -44,6 +44,7 @@ export type ReactNativeFeatureFlagsJsOnly = $ReadOnly<{
   shouldUseRemoveClippedSubviewsAsDefaultOnIOS: Getter<boolean>,
   shouldUseSetNativePropsInFabric: Getter<boolean>,
   utilizeTokensInIntersectionObserver: Getter<boolean>,
+  preventShadowTreeCommitExhaustion: Getter<boolean>,
 }>;
 
 export type ReactNativeFeatureFlagsJsOnlyOverrides = OverridesFor<ReactNativeFeatureFlagsJsOnly>;
@@ -191,6 +192,11 @@ export const shouldUseSetNativePropsInFabric: Getter<boolean> = createJavaScript
  */
 export const utilizeTokensInIntersectionObserver: Getter<boolean> = createJavaScriptFlagGetter('utilizeTokensInIntersectionObserver', true);
 
+/**
+ * Enables a new mechanism in ShadowTree to prevent problems caused by multiple threads trying to commit concurrently. If a thread tries to commit a few times unsuccessfully, it will acquire a lock and try again.
+ */
+export const preventShadowTreeCommitExhaustion: Getter<boolean> = createJavaScriptFlagGetter('preventShadowTreeCommitExhaustion', false);
+
 /**
  * Common flag for testing. Do NOT modify.
  */