Rewrite accessibilityOrder with virtual view hierarchy (#51692)

Summary: Pull Request resolved: https://github.com/facebook/react-native/pull/51692 The original algorithm for accessibilityOrder on Android had unexpected bugs. For some reason `.traversalAfter()` and `traversalBefore()` have unexpected behaviors when dealing with ancestor/descendant relationships getting more and more unexpected the further apart they are. So we are ditching that approach entirely. Now we have the view with accessibilityOrder create a virtual view hierarchy. We create a virtual node for each child that is in the order, and set the virtual node's position to be the same as the View it is trying to represent. We then also populate that node with the same stuff we populate regular ax nodes with the `populateAccessibilityNodeInfo()` function and the content description of the view it is backing so we get matching descriptions with what would otherwise be the normal announcement. **We have no way to exhaustively check every accessibility use case so we'll have to fine tune this as bugs come up, I'm expecting there to not be many issues since we populate the node the exact same way we populate every other node but anything that happens before React Native handles the node might miss some things** Changelog: [Internal] Reviewed By: joevilches Differential Revision: D74766296 fbshipit-source-id: 5e77c17bed1644bc5fbf5c1e19c3c6908cc1e3e9
2025-11-01 09:14:26 +00:00 · 2025-06-04 14:31:21 -07:00
parent c27a8804a6
commit 997b7c99f6
4 changed files with 210 additions and 133 deletions
@@ -292,7 +292,6 @@ public abstract class BaseViewManager<T extends View, C extends LayoutShadowNode
    if (view.getTag(R.id.accessibility_order_parent) != null) {
      ViewGroup accessibilityParent = (ViewGroup) view.getTag(R.id.accessibility_order_parent);

-      ReactAxOrderHelper.unsetAccessibilityOrder(accessibilityParent);
      accessibilityParent.setTag(R.id.accessibility_order_dirty, true);

      accessibilityParent.notifySubtreeAccessibilityStateChanged(
@@ -334,12 +333,11 @@ public abstract class BaseViewManager<T extends View, C extends LayoutShadowNode
                  view.setTag(R.id.accessibility_order_dirty, true);
                }
              });
-    }

-    ReactAxOrderHelper.unsetAccessibilityOrder(view);
-    ((ViewGroup) view)
-        .notifySubtreeAccessibilityStateChanged(
-            view, view, AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
+      ((ViewGroup) view)
+          .notifySubtreeAccessibilityStateChanged(
+              view, view, AccessibilityEvent.CONTENT_CHANGE_TYPE_SUBTREE);
+    }
  }

  @ReactProp(name = ViewProps.ACCESSIBILITY_LABELLED_BY)
@@ -17,6 +17,7 @@ import android.view.View;
 import android.view.ViewGroup;
 import android.view.accessibility.AccessibilityEvent;
 import android.widget.EditText;
+import android.widget.TextView;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.core.view.ViewCompat;
@@ -41,8 +42,11 @@ import com.facebook.react.uimanager.common.ViewUtil;
 import com.facebook.react.uimanager.events.Event;
 import com.facebook.react.uimanager.events.EventDispatcher;
 import com.facebook.react.uimanager.util.ReactFindViewUtil;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;

 /**
 * Utility class that handles the addition of a "role" for accessibility to either a View or
@@ -65,6 +69,7 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {
  private static final String STATE_CHECKED = "checked";

  private final View mView;
+  private List<View> mAxOrderViews;
  private Handler mHandler;
  private final HashMap<Integer, String> mAccessibilityActionsMap;

@@ -108,7 +113,6 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {
    if (!ViewCompat.hasAccessibilityDelegate(view)
        && (view.getTag(R.id.accessibility_role) != null
            || view.getTag(R.id.accessibility_order) != null
-            || view.getTag(R.id.accessibility_order_parent) != null
            || view.getTag(R.id.accessibility_state) != null
            || view.getTag(R.id.accessibility_actions) != null
            || view.getTag(R.id.react_test_id) != null
@@ -134,22 +138,7 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {
    return mView;
  }

-  @Override
-  public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfoCompat info) {
-    super.onInitializeAccessibilityNodeInfo(host, info);
-
-    if (host.getTag(R.id.accessibility_order_dirty) != null) {
-      boolean isAxOrderDirty = (boolean) host.getTag(R.id.accessibility_order_dirty);
-      if (isAxOrderDirty) {
-        ReactAxOrderHelper.setCustomAccessibilityFocusOrder(host);
-        host.setTag(R.id.accessibility_order_dirty, false);
-      }
-    }
-
-    if (host.getTag(R.id.accessibility_order_flow_to) != null) {
-      ReactAxOrderHelper.applyFlowToTraversal(host, info);
-    }
-
+  private void populateAccessibilityNodeInfo(View host, AccessibilityNodeInfoCompat info) {
    if (host.getTag(R.id.accessibility_state_expanded) != null) {
      final boolean accessibilityStateExpanded =
          (boolean) host.getTag(R.id.accessibility_state_expanded);
@@ -266,6 +255,34 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {
    }
  }

+  @Override
+  public void onInitializeAccessibilityNodeInfo(View host, AccessibilityNodeInfoCompat info) {
+    // If we set an accessibility order then all the focusing logic should go through our custom
+    // virtual view tree hierarchy and ignore the default path
+    ReadableArray axOrderIds = (ReadableArray) mView.getTag(R.id.accessibility_order);
+    if (axOrderIds != null && axOrderIds.size() != 0) {
+
+      Boolean isAxOrderDirty = (Boolean) mView.getTag(R.id.accessibility_order_dirty);
+      if (isAxOrderDirty != null && isAxOrderDirty) {
+        List<String> axOrderIdsList = new ArrayList<>();
+        Set<String> axOrderSet = new HashSet<>();
+        for (int i = 0; i < axOrderIds.size(); i++) {
+          String id = axOrderIds.getString(i);
+          if (id != null) {
+            axOrderIdsList.add(id);
+            axOrderSet.add(id);
+          }
+        }
+
+        mAxOrderViews = ReactAxOrderHelper.processAxOrderTree(mView, axOrderIdsList, axOrderSet);
+      }
+      return;
+    }
+
+    super.onInitializeAccessibilityNodeInfo(host, info);
+    populateAccessibilityNodeInfo(host, info);
+  }
+
  @Override
  public void onInitializeAccessibilityEvent(View host, AccessibilityEvent event) {
    super.onInitializeAccessibilityEvent(host, event);
@@ -436,17 +453,65 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {

  @Override
  protected int getVirtualViewAt(float x, float y) {
-    return INVALID_ID;
+    if (mAxOrderViews == null) {
+      return HOST_ID;
+    }
+
+    int closestViewId = HOST_ID;
+    int smallestArea = Integer.MAX_VALUE;
+
+    for (int i = 0; i < mAxOrderViews.size(); i++) {
+      Rect bounds = ReactAxOrderHelper.getVirtualViewBounds(mView, mAxOrderViews.get(i));
+      if (bounds.contains((int) x, (int) y)) {
+        int area = bounds.width() * bounds.height();
+        if (area < smallestArea) {
+          smallestArea = area;
+          closestViewId = i;
+        }
+      }
+    }
+
+    return closestViewId;
  }

  @Override
-  protected void getVisibleVirtualViews(List<Integer> virtualViewIds) {}
+  protected void getVisibleVirtualViews(List<Integer> virtualViewIds) {
+    if (mAxOrderViews != null && !mAxOrderViews.isEmpty()) {
+      for (int i = 0; i < mAxOrderViews.size(); i++) {
+        virtualViewIds.add(i);
+      }
+    }
+  }

  @Override
  protected void onPopulateNodeForVirtualView(
      int virtualViewId, @NonNull AccessibilityNodeInfoCompat node) {
-    node.setContentDescription("");
-    node.setBoundsInParent(new Rect(0, 0, 1, 1));
+    if (mView.getTag(R.id.accessibility_order) != null) {
+      if (mAxOrderViews.size() <= virtualViewId) {
+        node.setContentDescription("");
+        node.setBoundsInParent(new Rect(0, 0, 1, 1));
+        return;
+      }
+
+      View virtualView = mAxOrderViews.get(virtualViewId);
+
+      node.setContentDescription("");
+      if (virtualView == mView) {
+        if (mView.getContentDescription() != null) {
+          node.setContentDescription(mView.getContentDescription());
+        }
+
+        if (mView instanceof TextView && ((TextView) mView).getText() != null) {
+          node.setText(((TextView) mView).getText());
+        }
+
+        populateAccessibilityNodeInfo(mView, node);
+        node.setBoundsInParent(new Rect(0, 0, mView.getWidth(), mView.getHeight()));
+      } else {
+        node.setBoundsInParent(ReactAxOrderHelper.getVirtualViewBounds(mView, virtualView));
+      }
+      node.addChild(virtualView);
+    }
  }

  @Override
@@ -457,6 +522,10 @@ public class ReactAccessibilityDelegate extends ExploreByTouchHelper {

  @Override
  public @Nullable AccessibilityNodeProviderCompat getAccessibilityNodeProvider(View host) {
+    if (mView.getTag(R.id.accessibility_order) != null) {
+      return super.getAccessibilityNodeProvider(host);
+    }
+
    return null;
  }

@@ -7,69 +7,14 @@

 package com.facebook.react.uimanager

+import android.graphics.Rect
 import android.view.View
 import android.view.ViewGroup
 import android.widget.TextView
-import androidx.core.view.accessibility.AccessibilityNodeInfoCompat
 import com.facebook.react.R
 import com.facebook.react.bridge.ReadableArray

 private object ReactAxOrderHelper {
-  @JvmStatic
-  public fun setCustomAccessibilityFocusOrder(host: View) {
-
-    val axOrderIds = host.getTag(R.id.accessibility_order) as ReadableArray?
-
-    if (axOrderIds == null || axOrderIds.size() == 0) {
-      return
-    }
-
-    val axOrderIdsList = mutableListOf<String>()
-    val axOrderSet: MutableSet<String> = HashSet()
-    for (i in 0 until axOrderIds.size()) {
-      val id = axOrderIds.getString(i)
-      if (id != null) {
-        axOrderIdsList.add(id)
-        axOrderSet.add(axOrderIdsList[i])
-      }
-    }
-
-    val axOrderViews = processAxOrderTree(host, axOrderIdsList, axOrderSet).filterNotNull()
-
-    // Set up traversal order between views
-    for (i in 0 until axOrderViews.size - 1) {
-      val currentView = axOrderViews[i]
-      val flowToView = axOrderViews[i + 1]
-
-      currentView.setTag(R.id.accessibility_order_flow_to, flowToView)
-    }
-  }
-
-  @JvmStatic
-  public fun applyFlowToTraversal(host: View, info: AccessibilityNodeInfoCompat) {
-    val flowTo = host.getTag(R.id.accessibility_order_flow_to) as View?
-
-    if (flowTo != null) {
-      info.setTraversalBefore(flowTo)
-    }
-  }
-
-  @JvmStatic
-  public fun unsetAccessibilityOrder(view: View) {
-    view.setTag(R.id.accessibility_order_flow_to, null)
-    // Restore original accessibility importance if it was saved
-    val originalImportance = view.getTag(R.id.original_important_for_ax) as Int?
-    if (originalImportance != null) {
-      view.importantForAccessibility = originalImportance
-    }
-
-    if (view is ViewGroup) {
-      for (i in 0 until view.childCount) {
-        unsetAccessibilityOrder(view.getChildAt(i))
-      }
-    }
-  }
-
  /**
   * Processes the View tree that begins at the View with AccessibilityOrder set
   *
@@ -84,63 +29,134 @@ private object ReactAxOrderHelper {
   *
   * @return an array of views following the accessibility order
   */
-  private fun processAxOrderTree(
+  @JvmStatic
+  public fun processAxOrderTree(
      root: View,
-      axOrderIds: List<String?>,
-      axOrderSet: Set<String?>
-  ): Array<View?> {
-    val axOrderViews = arrayOfNulls<View?>(axOrderIds.size)
+      axOrderIds: MutableList<String?>,
+      axOrderSet: MutableSet<String?>
+  ): List<View> {
+    val axOrderViews = Array(axOrderIds.size) { mutableListOf<View?>() }.toMutableList()

-    fun traverseAndDisableAxFromExcludedViews(
-        view: View,
-        parent: View,
-        hasCooptingAncestor: Boolean
-    ) {
+    fun traverseAndBuildAxOrder(parent: View, view: View, containerId: String?) {
      val nativeId = view.getTag(R.id.view_tag_native_id) as String?

-      val isIncluded = nativeId != null && axOrderSet.contains(nativeId)
+      val isContained = (containerId != null && axOrderSet.contains(containerId))
+      val isIncluded = (nativeId != null && axOrderSet.contains(nativeId))

-      if (nativeId != null) {
-        view.setTag(R.id.accessibility_order_parent, parent)
-        ReactAccessibilityDelegate.setDelegate(
-            view, view.isFocusable, view.importantForAccessibility)
+      val isNestedAxOrder = view.getTag(R.id.accessibility_order) != null && view != parent
+
+      if (isIncluded && view.isFocusable) {
+        axOrderViews[axOrderIds.indexOf(nativeId)].add(view)
+        if (parent != view) {
+          view.setTag(R.id.accessibility_order_parent, parent)
+        }
+      } else if (isContained && view.isFocusable) {
+        axOrderViews[axOrderIds.indexOf(containerId)].add(view)
+        if (parent != view) {
+          view.setTag(R.id.accessibility_order_parent, parent)
+        }
      }

-      // There is a strange bug with TalkBack where if a focused view has nothing to announce, and
-      // there are no accessibility node's below it, then it will run OCR model on its bounds and
-      // announce any text it sees. Also, if there is a TextView below the View backing the node,
-      // it will announce that text too. This can happen frequently with our implementation here
-      // we change importantForAccessibility for TextView's that are not included in
-      // accessibilityOrder thus kicking off the logic described. To avoid this double announcement
-      // we just do not set importantForAccessibility on TextView's in the case where someone is
-      // coopting them. We will not focus the text since they got coopted.
-      if (isIncluded) {
-        axOrderViews[axOrderIds.indexOf(nativeId)] = view
-      } else if (!(view is TextView && hasCooptingAncestor)) {
-        // Save original state before disabling
-        view.setTag(R.id.original_important_for_ax, view.importantForAccessibility)
+      if (isNestedAxOrder) {
+        val nestedOrder = view.getTag(R.id.accessibility_order) as ReadableArray
+        for (i in 0 until nestedOrder.size()) {
+          val id = nestedOrder.getString(i)
+          if (id != null) {
+            val insertIdx = axOrderIds.indexOf(nativeId) + 1
+            if (insertIdx < axOrderIds.size) {
+              axOrderIds.add(axOrderIds.indexOf(nativeId) + 1 + i, id)
+              axOrderViews.add(axOrderIds.indexOf(nativeId) + 1 + i, mutableListOf())
+            } else {
+              axOrderIds.add(id)
+              axOrderViews.add(mutableListOf())
+            }
+
+            axOrderSet.add(id)
+          }
+        }
+      }
+
+      if (!isIncluded && !isContained && parent != view && view !is TextView) {
        view.importantForAccessibility = View.IMPORTANT_FOR_ACCESSIBILITY_NO
      }

-      val wantsToCoopt = isIncluded && view.contentDescription.isNullOrEmpty()
-
+      // Don't traverse the children of a nested accessibility order
      if (view is ViewGroup) {
-        // Continue to try to disable children if this view is not included and is
-        // focusable. This view being focusable means it's an element, and not a container which
-        // means its presence doesn't imply all its children should be focusable. And if its not
-        // included we still want to attempt to disable the children of the container
-        if (!isIncluded || view.isFocusable()) {
-          for (i in 0 until view.childCount) {
-            traverseAndDisableAxFromExcludedViews(view.getChildAt(i), parent, wantsToCoopt)
+        val axChildren: ArrayList<View> = getAxChildren(view)
+
+        // If the View is a "container" (Not focusable but is included in the order) We add all its
+        // children to the order.
+        if (containerId != null) {
+          for (i in 0 until axChildren.size) {
+            traverseAndBuildAxOrder(parent, axChildren[i], containerId)
+          }
+        } else if (!view.isFocusable && isIncluded) {
+          for (i in 0 until axChildren.size) {
+            traverseAndBuildAxOrder(parent, axChildren[i], nativeId)
+          }
+        } else {
+          for (i in 0 until axChildren.size) {
+            traverseAndBuildAxOrder(parent, axChildren[i], null)
          }
        }
      }
    }

-    // Technically we don't know if there is a coopting ancestor here, as it could be above the root
-    // but those cases should be fairly rare
-    traverseAndDisableAxFromExcludedViews(root, root, false)
+    traverseAndBuildAxOrder(
+        root,
+        root,
+        null,
+    )

-    return axOrderViews
+    val result = mutableListOf<View>()
+    for (viewList in axOrderViews) {
+      for (view in viewList) {
+        if (view != null) {
+          result.add(view)
+        }
+      }
+    }
+
+    root.setTag(R.id.accessibility_order_dirty, false)
+
+    return result
+  }
+
+  @JvmStatic
+  public fun getVirtualViewBounds(host: View, virtualView: View): Rect {
+    var currentView: View = virtualView
+    val viewBoundsInParent =
+        Rect(virtualView.left, virtualView.top, virtualView.right, virtualView.bottom)
+    while (currentView.parent != host && currentView != host) {
+      val parent = currentView.parent as View
+      viewBoundsInParent.top += parent.top
+      viewBoundsInParent.bottom += parent.top
+      viewBoundsInParent.left += parent.left
+      viewBoundsInParent.right += parent.left
+      currentView = parent
+    }
+
+    return viewBoundsInParent
+  }
+
+  private fun getAxChildren(host: ViewGroup): ArrayList<View> {
+    val axChildren: ArrayList<View> = ArrayList()
+
+    // When host has an accessibilityNodeProvider it means the order is not default so ViewGroup's
+    // method bails, in this case we should just add the children to the node in the edge case where
+    // we need to coopt at accessibilityOrderParent level
+    if (host.accessibilityNodeProvider != null) {
+      for (i in 0 until host.childCount) {
+        val child = host.getChildAt(i)
+        if (child != null) {
+          axChildren.add(child)
+        }
+      }
+    } else {
+      // This extracts the children of host sorted in accessibility order, this is by layout
+      // top to bottom, left to right
+      host.addChildrenForAccessibility(axChildren)
+    }
+    return axChildren
  }
 }
@@ -9,17 +9,11 @@
  <!-- tag is used to store the accessibleElements tag -->
  <item type="id" name="accessibility_order"/>

-  <!-- tag is used to store the current state of the accessibility order tree-->
-  <item type="id" name="accessibility_order_dirty"/>
-
-  <!-- tag is used to store the accessibleElements parent View tag -->
+  <!-- tag is used to store whether the view is coopted by an accessibilityOrder-->
  <item type="id" name="accessibility_order_parent"/>

-  <!-- tag is used to store the next focusable View's id -->
-  <item type="id" name="accessibility_order_flow_to"/>
-
-  <!-- tag is used to store the original important for accessibility value-->
-  <item type="id" name="original_important_for_ax"/>
+  <!-- tag is used to store the current state of the accessibility order tree-->
+  <item type="id" name="accessibility_order_dirty"/>

  <!-- tag is used to store the nativeID tag -->
  <item type="id" name="view_tag_instance_handle"/>