[compiler runtime] repro: infinite render with useMemoCache + render phase updates (#30849)

Repro for an infinite render bug we found when testing internally. See
equivalent codesandbox repro
[here](https://codesandbox.io/p/sandbox/epic-euclid-mr7lm3).

When render phase updates cause a re-render, useMemoCache arrays for the
fiber are
[cleared](https://github.com/facebook/react/blob/main/packages/react-reconciler/src/ReactFiberHooks.js#L819)
and [recreated on every
retry](https://github.com/facebook/react/blob/main/packages/react-reconciler/src/ReactFiberHooks.js#L1223)
while hook state is preserved.

This pattern (queuing re-renders on the current fiber during render) is
perfectly valid. I believe this is a bug as React compiler currently
replaces `useMemo`s with `useMemoCache` calls and inlined instructions,
taking care to preserve existing memoization dependencies. This should
be the identity transform, but runtime implementation differences mean
that uncompiled code behaves as expected (no infinite render) while
compiled code fails to render.

packages/react-reconciler/src/__tests__/useMemoCache-test.js

@@ -13,8 +13,10 @@ let ReactNoop;
 let Scheduler;
 let act;
 let assertLog;
+let useMemo;
 let useState;
 let useMemoCache;
+let waitForThrow;
 let MemoCacheSentinel;
 let ErrorBoundary;
 
@@ -27,8 +29,10 @@ describe('useMemoCache()', () => {
     Scheduler = require('scheduler');
     act = require('internal-test-utils').act;
     assertLog = require('internal-test-utils').assertLog;
+    useMemo = React.useMemo;
     useMemoCache = require('react/compiler-runtime').c;
     useState = React.useState;
+    waitForThrow = require('internal-test-utils').waitForThrow;
     MemoCacheSentinel = Symbol.for('react.memo_cache_sentinel');
 
     class _ErrorBoundary extends React.Component {
@@ -645,4 +649,59 @@ describe('useMemoCache()', () => {
       </>,
     );
   });
+
+  // @gate enableUseMemoCacheHook
+  it('(repro) infinite renders when used with setState during render', async () => {
+    // Output of react compiler on `useUserMemo`
+    function useCompilerMemo(value) {
+      let arr;
+      const $ = useMemoCache(2);
+      if ($[0] !== value) {
+        arr = [value];
+        $[0] = value;
+        $[1] = arr;
+      } else {
+        arr = $[1];
+      }
+      return arr;
+    }
+
+    // Baseline / source code
+    function useUserMemo(value) {
+      return useMemo(() => [value], [value]);
+    }
+
+    function makeComponent(hook) {
+      return function Component({value}) {
+        const state = hook(value);
+        const [prevState, setPrevState] = useState(null);
+        if (state !== prevState) {
+          setPrevState(state);
+        }
+        return <div>{state.join(',')}</div>;
+      };
+    }
+
+    /**
+     * Test case: note that the initial render never completes
+     */
+    let root = ReactNoop.createRoot();
+    const IncorrectInfiniteComponent = makeComponent(useCompilerMemo);
+    root.render(<IncorrectInfiniteComponent value={2} />);
+    await waitForThrow(
+      'Too many re-renders. React limits the number of renders to prevent ' +
+        'an infinite loop.',
+    );
+
+    /**
+     * Baseline test: initial render is expected to complete after a retry
+     * (triggered by the setState)
+     */
+    root = ReactNoop.createRoot();
+    const CorrectComponent = makeComponent(useUserMemo);
+    await act(() => {
+      root.render(<CorrectComponent value={2} />);
+    });
+    expect(root).toMatchRenderedOutput(<div>2</div>);
+  });
 });