x509: add EXFLAG_DUPLICATE and cheap O(1) extension duplicate check

In ossl_x509v3_cache_extensions(), introduce EXFLAG_DUPLICATE flag to
signal duplicate X.509 extensions. Add O(1) duplicate detection
using a bitset with minimal stack memory footprint, in compliance with
RFC 5280 Section 4.2.

Fixes #26325

Co-authored-by: Tomáš Mráz <tm@t8m.info>
Co-authored-by: David von Oheimb <DDvO@users.noreply.github.com>

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Simo Sorce <simo@redhat.com>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Neil Horman <nhorman@openssl.org>
MergeDate: Tue Mar 17 13:43:13 2026
(Merged from https://github.com/openssl/openssl/pull/30233)

include/openssl/x509_vfy.h.in

@@ -337,6 +337,8 @@ void X509_STORE_CTX_set_depth(X509_STORE_CTX *ctx, int depth);
 #define X509_V_ERR_EMPTY_AUTHORITY_KEY_IDENTIFIER 102
 #define X509_V_ERR_AKID_ISSUER_SERIAL_NOT_PAIRED 103
 
+#define X509_V_ERR_DUPLICATE_EXTENSION 104
+
 /* Certificate verify flags */
 #ifndef OPENSSL_NO_DEPRECATED_1_1_0
 #define X509_V_FLAG_CB_ISSUER_CHECK 0x0 /* Deprecated */

include/openssl/x509v3.h.in

@@ -455,6 +455,9 @@ struct ISSUING_DIST_POINT_st {
 #define EXFLAG_SKID_CRITICAL 0x40000
 #define EXFLAG_SAN_CRITICAL 0x80000
 
+/* A certificate MUST NOT include more than one instance of an extension. */
+#define EXFLAG_DUPLICATE 0x200000
+
 /* https://datatracker.ietf.org/doc/html/rfc5280#section-4.2.1.3 */
 #define KU_DIGITAL_SIGNATURE X509v3_KU_DIGITAL_SIGNATURE
 #define KU_NON_REPUDIATION X509v3_KU_NON_REPUDIATION