Implement named character entity replacement

https://html.spec.whatwg.org/multipage/syntax.html#tokenizing-character-references

HTMLKit/HTMLInputStreamReader.h

@@ -32,6 +32,7 @@
 - (BOOL)consumeString:(NSString *)string caseSensitive:(BOOL)caseSensitive;
 - (NSString *)consumeCharactersUpToCharactersInString:(NSString *)characters;
 - (NSString *)consumeCharactersUpToString:(NSString *)string;
+- (NSString *)consumeAlphanumericCharacters;
 
 - (void)markCurrentLocation;
 - (void)rewindToMarkedLocation;

HTMLKit/HTMLInputStreamReader.m

@@ -170,6 +170,16 @@
 	return consumed;
 }
 
+- (NSString *)consumeAlphanumericCharacters
+{
+	NSCharacterSet *set = [NSCharacterSet alphanumericCharacterSet];
+	NSString *consumed = nil;
+
+	[_scanner scanCharactersFromSet:set intoString:&consumed];
+	_location = _scanner.scanLocation;
+	return consumed;
+}
+
 - (void)unconsumeCurrentInputCharacter
 {
 	_location -= _consume;

HTMLKit/HTMLTokenizer.m

@@ -11,6 +11,7 @@
 #import "HTMLToken.h"
 #import "HTMLTokenizerStates.h"
 #import "HTMLTokenizerCharacters.h"
+#import "HTMLTokenizerEntities.h"
 
 @interface HTMLTokenizer ()
 {
@@ -182,10 +183,11 @@
 
 #pragma mark - Consume Character Reference
 
-- (NSString *)attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)additionalAllowedCharacter
+- (NSString *)attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)additional
+																   inAttribute:(BOOL)inAttribute
 {
 	UTF32Char character = [_inputStreamReader nextInputCharacter];
-	if (additionalAllowedCharacter != (UTF32Char)EOF && character == additionalAllowedCharacter) {
+	if (additional != (UTF32Char)EOF && character == additional) {
 		return nil;
 	}
 
@@ -206,8 +208,10 @@
 			return numberReference;
 		}
 		default:
-#warning Implement named entity replacement
-			return nil;
+		{
+			NSString *namedEntity = [self attemptToConsumeNamedCharacterReferenceInAttribute:inAttribute];
+			return namedEntity;
+		}
 	}
 }
 
@@ -256,6 +260,62 @@
 	return StringFromUTF32Char(character);
 }
 
+- (NSString *)attemptToConsumeNamedCharacterReferenceInAttribute:(BOOL)inAttribute
+{
+	[_inputStreamReader markCurrentLocation];
+
+	NSString *entityName = nil;
+
+	UTF32Char inputCharacter = [_inputStreamReader currentInputCharacter];
+	NSArray *names = [HTMLTokenizerEntities entityNames];
+	NSMutableString *name = [NSMutableString stringWithString:StringFromUTF32Char(inputCharacter)];
+
+	while (YES) {
+		NSPredicate *predicate = [NSPredicate predicateWithFormat:@"SELF BEGINSWITH %@", name];
+		names = [names filteredArrayUsingPredicate:predicate];
+		if (names.count == 0) break;
+
+		inputCharacter = [_inputStreamReader consumeNextInputCharacter];
+		[name appendString:StringFromUTF32Char(inputCharacter)];
+
+		if ([names containsObject:name]) {
+			entityName = [name copy];
+			if ([entityName hasSuffix:@";"]) {
+				break;
+			}
+		}
+	}
+
+	if (entityName == nil) {
+		if ([_inputStreamReader consumeAlphanumericCharacters] != nil) {
+			if ([_inputStreamReader consumeString:@";" caseSensitive:NO]) {
+				[self emitParseError:@"Undefined named entity with semicolon found"];
+			}
+		}
+		[_inputStreamReader rewindToMarkedLocation];
+		return nil;
+	}
+
+	NSString *replacement = [HTMLTokenizerEntities replacementForNamedCharacterEntity:entityName];
+
+	if (inAttribute && [entityName hasSuffix:@";"] == NO) {
+		UTF32Char nextCharacter = [_inputStreamReader nextInputCharacter];
+		if (nextCharacter == EQUALS_SIGN || isAlphanumeric(nextCharacter)) {
+			[_inputStreamReader rewindToMarkedLocation];
+			if (nextCharacter == EQUALS_SIGN) {
+				[self emitParseError:@"Named entity in attribute ending with equal-sign"];
+			}
+			return nil;
+		}
+	}
+
+	if ([entityName hasSuffix:@";"] == NO) {
+		[self emitParseError:@"Named entity without semicolon"];
+	}
+
+	return replacement;
+}
+
 #pragma mark - States
 
 - (void)HTMLTokenizerStateData
@@ -285,7 +345,8 @@
 {
 	[self switchToState:HTMLTokenizerStateData];
 
-	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter];
+	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter
+																							  inAttribute:NO];
 	if (characterReference == nil) {
 		[self emitCharacterToken:AMPERSAND];
 	} else {
@@ -320,7 +381,8 @@
 {
 	[self switchToState:HTMLTokenizerStateRCDATA];
 
-	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)EOF];
+	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)EOF
+																							  inAttribute:NO];
 	if (characterReference == nil) {
 		[self emitCharacterToken:AMPERSAND];
 	} else {
@@ -1368,7 +1430,8 @@
 
 - (void)HTMLTokenizerStateCharacterReferenceInAttributeValue
 {
-	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter];
+	NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter
+																							  inAttribute:YES];
 	if (characterReference == nil) {
 		[_currentAttributeValue appendString:StringFromUniChar(AMPERSAND)];
 	} else {

HTMLKit/HTMLTokenizerCharacters.h

@@ -137,6 +137,13 @@ NS_INLINE BOOL isHexDigit(UTF32Char character)
 			(character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_F));
 }
 
+NS_INLINE BOOL isAlphanumeric(UTF32Char character)
+{
+	return ((character >= DIGIT_ZERO && character <= DIGIT_NINE) ||
+			(character >= LATIN_CAPITAL_LETTER_A && character <= LATIN_CAPITAL_LETTER_Z) ||
+			(character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_Z));
+}
+
 NS_INLINE BOOL isValidNumericRange(UTF32Char character)
 {
 	return ((character >= 0xD800 && character <= 0xDFFF) ||

HTMLKit/HTMLTokenizerEntities.h

@@ -10,6 +10,7 @@
 
 @interface HTMLTokenizerEntities : NSObject
 
++ (NSArray *)entityNames;
 + (NSString *)replacementForNamedCharacterEntity:(NSString *)entity;
 
 @end

HTMLKit/HTMLTokenizerEntities.m

@@ -2241,34 +2241,38 @@
 	NAMED_CHARACTER_REFERENCE( "zwj;", "\U0000200d" ) \
 	NAMED_CHARACTER_REFERENCE( "zwnj;", "\U0000200c" )
 
-static CFStringRef NamedReferencesNames[] = {
-#define NAMED_CHARACTER_REFERENCE( name, value ) CFSTR(name),
-	NAMED_CHARACTER_REFERENCES
-#undef 	NAMED_CHARACTER_REFERENCE
-};
-
-static CFStringRef NamedReferencesValues[] = {
-#define NAMED_CHARACTER_REFERENCE( name, value ) CFSTR(value),
-	NAMED_CHARACTER_REFERENCES
-#undef 	NAMED_CHARACTER_REFERENCE
-};
-
-static CFDictionaryRef _entities;
+static NSDictionary *_entities;
 
 @implementation HTMLTokenizerEntities
 
 + (void)initialize
 {
 	if (self == [HTMLTokenizerEntities class]) {
-		CFIndex numValues = sizeof(NamedReferencesNames) / sizeof(NamedReferencesNames[0]);
-		_entities = CFDictionaryCreate(NULL, (const void **) &NamedReferencesNames, (const void **) &NamedReferencesValues, numValues,
-									   &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
+
+		NSArray * NamedReferencesNames = @[
+#define NAMED_CHARACTER_REFERENCE( name, value ) @(name),
+			NAMED_CHARACTER_REFERENCES
+#undef 	NAMED_CHARACTER_REFERENCE
+		];
+
+		NSArray * NamedReferencesValues = @[
+#define NAMED_CHARACTER_REFERENCE( name, value ) @(value),
+			NAMED_CHARACTER_REFERENCES
+#undef 	NAMED_CHARACTER_REFERENCE
+		];
+
+		_entities = [[NSDictionary alloc] initWithObjects:NamedReferencesValues forKeys:NamedReferencesNames];
 	}
 }
 
++ (NSArray *)entityNames
+{
+	return [_entities allKeys];
+}
+
 + (NSString *)replacementForNamedCharacterEntity:(NSString *)entity
 {
-	NSString *replacement = (NSString *) CFDictionaryGetValue(_entities, (__bridge const void *)(entity));
+	NSString *replacement = [_entities objectForKey:entity];
 	return replacement;
 }