Add memory allocation failure testing framework

Introduce ADD_MFAIL_TEST for exhaustive testing of allocation failure
handling in individual functions. The framework repeatedly calls the
test function, each time failing one allocation later within the
section bracketed by mfail_start() and mfail_end(), verifying that
every failure path returns 0 without crashing or leaking.

Custom allocators are installed once at startup via
CRYPTO_set_mem_functions(). When not armed, they pass through to
malloc/realloc/free. Installation can be disabled by setting
OPENSSL_TEST_MFAIL_DISABLE for tests that need the default allocator
(e.g. those using OPENSSL_MALLOC_FAILURES).

Additional environment variables control test execution:
OPENSSL_TEST_MFAIL_SKIP_ALL, OPENSSL_TEST_MFAIL_SKIP_SLOW,
OPENSSL_TEST_MFAIL_POINT, and OPENSSL_TEST_MFAIL_START.

Reviewed-by: Saša Nedvědický <sashan@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.foundation>
MergeDate: Thu Apr 23 20:23:34 2026
(Merged from https://github.com/openssl/openssl/pull/30871)

CHANGES.md

@@ -31,6 +31,10 @@ OpenSSL Releases
 
 ### Changes between 4.0 and 4.1 [xx XXX xxxx]
 
+ * Added test framework for testing function memory allocation failures.
+
+   *Jakub Zelenka*
+
  * Improved DTLS handshake robustness under UDP reordering by buffering and
    replaying early ChangeCipherSpec (CCS) records at the expected state.
 

test/README.md

@@ -185,6 +185,35 @@ To run the tests using the order defined by the random seed `42`:
 
     $ make OPENSSL_TEST_RAND_ORDER=42 test
 
+Memory Allocation Failure Tests
+-------------------------------
+
+Some tests use the `ADD_MFAIL_TEST` framework to exhaustively verify that
+functions handle every possible allocation failure gracefully. These tests
+run repeatedly, failing one allocation later each iteration, and can be
+controlled with the following environment variables:
+
+    OPENSSL_TEST_MFAIL_DISABLE=1    Disable mfail custom allocator installation.
+
+    OPENSSL_TEST_MFAIL_SKIP_ALL=1   Skip all mfail tests.
+
+    OPENSSL_TEST_MFAIL_SKIP_SLOW=1  Skip only slow mfail tests
+                                    (registered with ADD_MFAIL_SLOW_TEST).
+
+    OPENSSL_TEST_MFAIL_POINT=N      Run only failure point N (0-indexed),
+                                    useful for debugging a specific failure.
+
+    OPENSSL_TEST_MFAIL_START=N      Start iteration from point N, skipping
+                                    earlier points that are already fixed.
+
+For example, to debug a failure at allocation point 42:
+
+    $ OPENSSL_TEST_MFAIL_POINT=42 ./test/crltest -test test_crl_diff_mfail
+
+Or to skip already-fixed points and collect remaining failures:
+
+    $ OPENSSL_TEST_MFAIL_START=13 make TESTS=test_crl test
+
 Running Tests under Valgrind
 ----------------------------
 

test/build.info

@@ -32,7 +32,7 @@ IF[{- !$disabled{tests} -}]
           testutil/test_cleanup.c testutil/main.c testutil/testutil_init.c \
           testutil/options.c testutil/test_options.c testutil/provider.c \
           testutil/apps_shims.c testutil/random.c testutil/helper.c \
-          testutil/compare.c $LIBAPPSSRC
+          testutil/compare.c testutil/mfail.c $LIBAPPSSRC
   INCLUDE[libtestutil.a]=../include ../apps/include ..
   DEPEND[libtestutil.a]=../libcrypto
 

test/crltest.c

@@ -855,6 +855,31 @@ static int test_crl_idp_malformed2(void)
     return test;
 }
 
+static int test_crl_diff_mfail(void)
+{
+    X509_CRL *base_crl = NULL, *newer_crl = NULL, *delta = NULL;
+    int ret = 0;
+
+    base_crl = CRL_from_strings(kBasicCRL);
+    newer_crl = CRL_from_strings(kRevokedCRL);
+    if (!TEST_ptr(base_crl) || !TEST_ptr(newer_crl))
+        goto err;
+
+    MFAIL_start();
+    delta = X509_CRL_diff(base_crl, newer_crl, NULL, NULL, 0);
+    MFAIL_end();
+
+    if (delta == NULL)
+        goto err;
+
+    ret = 1;
+err:
+    X509_CRL_free(delta);
+    X509_CRL_free(base_crl);
+    X509_CRL_free(newer_crl);
+    return ret;
+}
+
 int setup_tests(void)
 {
     if (!TEST_ptr(test_root = X509_from_strings(kCRLTestRoot))
@@ -878,6 +903,7 @@ int setup_tests(void)
     ADD_TEST(test_unknown_critical_crl1);
     ADD_TEST(test_unknown_critical_crl2);
     ADD_ALL_TESTS(test_reuse_crl, 6);
+    ADD_MFAIL_TEST(test_crl_diff_mfail);
 
     return 1;
 }

test/recipes/02-test_mem_alloc.t

@@ -14,6 +14,7 @@ plan skip_all => "This test should not be run under valgrind"
     if ( defined $ENV{OSSL_USE_VALGRIND} );
 
 {
+    local $ENV{"OPENSSL_TEST_MFAIL_DISABLE"} = 1;
     local $ENV{"ASAN_OPTIONS"} = "allocator_may_return_null=true";
     local $ENV{"MSAN_OPTIONS"} = "allocator_may_return_null=true";
 

test/recipes/02-test_mem_alloc_custom_fns.t

@@ -14,6 +14,7 @@ plan skip_all => "This test should not be run under valgrind"
     if ( defined $ENV{OSSL_USE_VALGRIND} );
 
 {
+    local $ENV{"OPENSSL_TEST_MFAIL_DISABLE"} = 1;
     local $ENV{"ASAN_OPTIONS"} = "allocator_may_return_null=true";
     local $ENV{"MSAN_OPTIONS"} = "allocator_may_return_null=true";
 

test/recipes/90-test_memfail.t

@@ -68,6 +68,8 @@ plan skip_all => "could not get malloc counts (one or more count runs failed or
 #
 plan tests => $total_malloccount;
 
+$ENV{OPENSSL_TEST_MFAIL_DISABLE} = "1";
+
 sub run_memfail_test {
     my $skipcount = $_[0];
     my @mallocseq = (1..$_[1]);

test/testutil.h

@@ -57,6 +57,21 @@
  */
 #define ADD_ALL_TESTS(test_function, num) \
     add_all_tests(#test_function, test_function, num, 1)
+
+/*
+ * Memory failure exhaustive test. Runs test_fn repeatedly, each time
+ * injecting an allocation failure one step later. When a failure is
+ * injected, asserts test_fn returns 0. When no failure is injected
+ * (all allocation points exhausted), asserts test_fn returns 1 and stops.
+ *
+ * The slow variant is for marking the slow test that can be skipped using
+ * environment variable.
+ *
+ * test_fn has no parameters and returns 1 on success, 0 on failure.
+ */
+#define ADD_MFAIL_TEST(test_fn) add_mfail_test(#test_fn, test_fn, 0)
+#define ADD_MFAIL_SLOW_TEST(test_fn) add_mfail_test(#test_fn, test_fn, 1)
+
 /*
  * A variant of the same without TAP output.
  */
@@ -227,6 +242,20 @@ int test_arg_libctx(OSSL_LIB_CTX **libctx, OSSL_PROVIDER **default_null_prov,
 void add_test(const char *test_case_name, int (*test_fn)(void));
 void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num,
     int subtest);
+void add_mfail_test(const char *test_case_name, int (*test_fn)(void), int slow);
+
+/*
+ * Start the memory allocation failure counter.
+ */
+void mfail_start(void);
+
+/*
+ * Stop the memory allocation failure counter.
+ */
+void mfail_end(void);
+
+#define MFAIL_start mfail_start
+#define MFAIL_end mfail_end
 
 /*
  * Declarations for user defined functions.

test/testutil/driver.c

@@ -33,7 +33,9 @@ typedef struct test_info {
     int num;
 
     /* flags */
-    int subtest : 1;
+    unsigned int subtest : 1;
+    unsigned int mfail : 1;
+    unsigned int mfail_slow : 1;
 } TEST_INFO;
 
 static TEST_INFO all_tests[1024];
@@ -44,6 +46,7 @@ static int single_iter = -1;
 static int level = 0;
 static int seed = 0;
 static int rand_order = 0;
+static int mfail_added = 0;
 
 /*
  * A parameterised test runs a loop of test cases.
@@ -79,6 +82,19 @@ void add_all_tests(const char *test_case_name, int (*test_fn)(int idx),
         num_test_cases += num;
 }
 
+void add_mfail_test(const char *test_case_name, int (*test_fn)(void), int slow)
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].test_fn = test_fn;
+    all_tests[num_tests].num = -1;
+    all_tests[num_tests].mfail = 1;
+    all_tests[num_tests].mfail_slow = slow ? 1 : 0;
+    ++num_tests;
+    ++num_test_cases;
+    mfail_added = 1;
+}
+
 static int gcd(int a, int b)
 {
     while (b != 0) {
@@ -305,6 +321,9 @@ int run_tests(const char *test_prog_name)
 
     test_flush_tapout();
 
+    if (mfail_added)
+        mfail_init();
+
     for (i = 0; i < num_tests; i++)
         permute[i] = i;
     if (rand_order != 0)
@@ -333,7 +352,14 @@ int run_tests(const char *test_prog_name)
         } else if (all_tests[i].num == -1) {
             set_test_title(all_tests[i].test_case_name);
             ERR_clear_error();
-            verdict = all_tests[i].test_fn();
+            if (all_tests[i].mfail)
+                if (mfail_should_skip(all_tests[i].mfail_slow))
+                    verdict = TEST_skip("mfail test skipped");
+                else
+                    verdict = mfail_run_test(all_tests[i].test_case_name,
+                        all_tests[i].test_fn);
+            else
+                verdict = all_tests[i].test_fn();
             finalize(verdict != 0);
             test_verdict(verdict, "%d - %s", test_case_count + 1, test_title);
             if (verdict == 0)

test/testutil/main.c

@@ -31,6 +31,8 @@ int main(int argc, char *argv[])
     int setup_res;
     int gi_ret;
 
+    mfail_install();
+
     gi_ret = global_init();
 
     test_open_streams();

test/testutil/mfail.c

@@ -0,0 +1,196 @@
+/*
+ * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+#include "tu_local.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <openssl/crypto.h>
+
+static int mfail_fail_after = -1;
+static int mfail_alloc_count = 0;
+static int mfail_triggered = 0;
+static int mfail_counting = 0;
+static int mfail_do_skip_all = 0;
+static int mfail_do_skip_slow = 0;
+static int mfail_single_point = -1;
+static int mfail_start_point = 0;
+static int mfail_installed = 0;
+
+static int should_fail(void)
+{
+    if (mfail_fail_after < 0 || !mfail_counting || mfail_triggered)
+        return 0;
+    if (mfail_alloc_count++ == mfail_fail_after) {
+        mfail_triggered = 1;
+        return 1;
+    }
+    return 0;
+}
+
+static void *mfail_malloc(size_t num, const char *file, int line)
+{
+    if (num == 0)
+        return NULL;
+    if (should_fail())
+        return NULL;
+    return malloc(num);
+}
+
+static void *mfail_realloc(void *addr, size_t num, const char *file, int line)
+{
+    if (addr == NULL)
+        return mfail_malloc(num, file, line);
+    if (num == 0) {
+        free(addr);
+        return NULL;
+    }
+    if (should_fail())
+        return NULL;
+    return realloc(addr, num);
+}
+
+static void mfail_free(void *addr, const char *file, int line)
+{
+    free(addr);
+}
+
+static int env_is_true(const char *name)
+{
+    const char *val = getenv(name);
+
+    return val != NULL && *val != '\0' && strcmp(val, "0") != 0;
+}
+
+void mfail_install(void)
+{
+    if (env_is_true("OPENSSL_TEST_MFAIL_DISABLE"))
+        return;
+    if (!CRYPTO_set_mem_functions(mfail_malloc, mfail_realloc, mfail_free))
+        return;
+    mfail_installed = 1;
+}
+
+void mfail_start(void)
+{
+    mfail_alloc_count = 0;
+    mfail_counting = 1;
+}
+
+void mfail_end(void)
+{
+    mfail_counting = 0;
+}
+
+static void mfail_arm(int fail_after)
+{
+    mfail_fail_after = fail_after;
+    mfail_alloc_count = 0;
+    mfail_triggered = 0;
+    mfail_counting = 0;
+}
+
+static void mfail_disarm(void)
+{
+    mfail_fail_after = -1;
+    mfail_alloc_count = 0;
+    mfail_triggered = 0;
+    mfail_counting = 0;
+}
+
+static double elapsed_secs(clock_t start)
+{
+    return (double)(clock() - start) / CLOCKS_PER_SEC;
+}
+
+void mfail_init(void)
+{
+    const char *env;
+
+    mfail_do_skip_all = env_is_true("OPENSSL_TEST_MFAIL_SKIP_ALL");
+    mfail_do_skip_slow = env_is_true("OPENSSL_TEST_MFAIL_SKIP_SLOW");
+
+    env = getenv("OPENSSL_TEST_MFAIL_POINT");
+    if (env != NULL && *env != '\0')
+        mfail_single_point = atoi(env);
+
+    env = getenv("OPENSSL_TEST_MFAIL_START");
+    if (env != NULL && *env != '\0')
+        mfail_start_point = atoi(env);
+}
+
+int mfail_should_skip(int slow)
+{
+    if (!mfail_installed)
+        return 1;
+    return mfail_do_skip_all || (slow && mfail_do_skip_slow);
+}
+
+int mfail_run_test(const char *test_case_name, int (*test_fn)(void))
+{
+    int alloc_point, ret = 1;
+    clock_t start;
+
+    start = clock();
+
+    if (mfail_single_point >= 0) {
+        int rv, triggered;
+
+        ERR_clear_error();
+        mfail_arm(mfail_single_point);
+        rv = test_fn();
+        triggered = mfail_triggered;
+        mfail_disarm();
+
+        if (!triggered) {
+            TEST_info("mfail test '%s': point %d is beyond the last "
+                      "allocation point, test %s",
+                test_case_name, mfail_single_point,
+                rv == 1 ? "succeeded" : "failed");
+        } else if (!TEST_int_eq(rv, 0)) {
+            TEST_error("mfail test '%s': allocation failure at point %d "
+                       "not handled",
+                test_case_name, mfail_single_point);
+            ret = 0;
+        }
+    } else {
+        for (alloc_point = mfail_start_point;; alloc_point++) {
+            int rv, triggered;
+
+            ERR_clear_error();
+            mfail_arm(alloc_point);
+            rv = test_fn();
+            triggered = mfail_triggered;
+            mfail_disarm();
+
+            if (!triggered) {
+                if (!TEST_int_eq(rv, 1)) {
+                    TEST_error("mfail test '%s': no injection but test failed",
+                        test_case_name);
+                    ret = 0;
+                }
+                break;
+            }
+
+            if (!TEST_int_eq(rv, 0)) {
+                TEST_error("mfail test '%s': allocation failure at point %d "
+                           "not handled",
+                    test_case_name, alloc_point);
+                ret = 0;
+            }
+        }
+        TEST_info("mfail test '%s': points %d..%d, %d iterations, %.6f seconds",
+            test_case_name, mfail_start_point, alloc_point,
+            alloc_point - mfail_start_point + 1, elapsed_secs(start));
+    }
+
+    return ret;
+}

test/testutil/tu_local.h

@@ -49,6 +49,11 @@ void test_fail_memory_message(const char *prefix, const char *file,
 __owur int setup_test_framework(int argc, char *argv[]);
 __owur int pulldown_test_framework(int ret);
 
+void mfail_install(void);
+void mfail_init(void);
+int mfail_should_skip(int slow);
+int mfail_run_test(const char *test_case_name, int (*test_fn)(void));
+
 __owur int run_tests(const char *test_prog_name);
 void set_test_title(const char *title);