From 968a6aff7998eb0d7ca4443c5880b253195c740e Mon Sep 17 00:00:00 2001 From: iska Date: Thu, 23 Oct 2014 20:07:30 +0200 Subject: [PATCH] Implement named character entity replacement https://html.spec.whatwg.org/multipage/syntax.html#tokenizing-character-references --- HTMLKit/HTMLInputStreamReader.h | 1 + HTMLKit/HTMLInputStreamReader.m | 10 ++++ HTMLKit/HTMLTokenizer.m | 77 ++++++++++++++++++++++++++++--- HTMLKit/HTMLTokenizerCharacters.h | 7 +++ HTMLKit/HTMLTokenizerEntities.h | 1 + HTMLKit/HTMLTokenizerEntities.m | 38 ++++++++------- 6 files changed, 110 insertions(+), 24 deletions(-) diff --git a/HTMLKit/HTMLInputStreamReader.h b/HTMLKit/HTMLInputStreamReader.h index f102ba4..0c4d90b 100644 --- a/HTMLKit/HTMLInputStreamReader.h +++ b/HTMLKit/HTMLInputStreamReader.h @@ -32,6 +32,7 @@ - (BOOL)consumeString:(NSString *)string caseSensitive:(BOOL)caseSensitive; - (NSString *)consumeCharactersUpToCharactersInString:(NSString *)characters; - (NSString *)consumeCharactersUpToString:(NSString *)string; +- (NSString *)consumeAlphanumericCharacters; - (void)markCurrentLocation; - (void)rewindToMarkedLocation; diff --git a/HTMLKit/HTMLInputStreamReader.m b/HTMLKit/HTMLInputStreamReader.m index c91f029..8e91cb3 100644 --- a/HTMLKit/HTMLInputStreamReader.m +++ b/HTMLKit/HTMLInputStreamReader.m @@ -170,6 +170,16 @@ return consumed; } +- (NSString *)consumeAlphanumericCharacters +{ + NSCharacterSet *set = [NSCharacterSet alphanumericCharacterSet]; + NSString *consumed = nil; + + [_scanner scanCharactersFromSet:set intoString:&consumed]; + _location = _scanner.scanLocation; + return consumed; +} + - (void)unconsumeCurrentInputCharacter { _location -= _consume; diff --git a/HTMLKit/HTMLTokenizer.m b/HTMLKit/HTMLTokenizer.m index dad65e9..56e5b39 100644 --- a/HTMLKit/HTMLTokenizer.m +++ b/HTMLKit/HTMLTokenizer.m @@ -11,6 +11,7 @@ #import "HTMLToken.h" #import "HTMLTokenizerStates.h" #import "HTMLTokenizerCharacters.h" +#import "HTMLTokenizerEntities.h" @interface HTMLTokenizer () { @@ -182,10 +183,11 @@ #pragma mark - Consume Character Reference -- (NSString *)attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)additionalAllowedCharacter +- (NSString *)attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)additional + inAttribute:(BOOL)inAttribute { UTF32Char character = [_inputStreamReader nextInputCharacter]; - if (additionalAllowedCharacter != (UTF32Char)EOF && character == additionalAllowedCharacter) { + if (additional != (UTF32Char)EOF && character == additional) { return nil; } @@ -206,8 +208,10 @@ return numberReference; } default: -#warning Implement named entity replacement - return nil; + { + NSString *namedEntity = [self attemptToConsumeNamedCharacterReferenceInAttribute:inAttribute]; + return namedEntity; + } } } @@ -256,6 +260,62 @@ return StringFromUTF32Char(character); } +- (NSString *)attemptToConsumeNamedCharacterReferenceInAttribute:(BOOL)inAttribute +{ + [_inputStreamReader markCurrentLocation]; + + NSString *entityName = nil; + + UTF32Char inputCharacter = [_inputStreamReader currentInputCharacter]; + NSArray *names = [HTMLTokenizerEntities entityNames]; + NSMutableString *name = [NSMutableString stringWithString:StringFromUTF32Char(inputCharacter)]; + + while (YES) { + NSPredicate *predicate = [NSPredicate predicateWithFormat:@"SELF BEGINSWITH %@", name]; + names = [names filteredArrayUsingPredicate:predicate]; + if (names.count == 0) break; + + inputCharacter = [_inputStreamReader consumeNextInputCharacter]; + [name appendString:StringFromUTF32Char(inputCharacter)]; + + if ([names containsObject:name]) { + entityName = [name copy]; + if ([entityName hasSuffix:@";"]) { + break; + } + } + } + + if (entityName == nil) { + if ([_inputStreamReader consumeAlphanumericCharacters] != nil) { + if ([_inputStreamReader consumeString:@";" caseSensitive:NO]) { + [self emitParseError:@"Undefined named entity with semicolon found"]; + } + } + [_inputStreamReader rewindToMarkedLocation]; + return nil; + } + + NSString *replacement = [HTMLTokenizerEntities replacementForNamedCharacterEntity:entityName]; + + if (inAttribute && [entityName hasSuffix:@";"] == NO) { + UTF32Char nextCharacter = [_inputStreamReader nextInputCharacter]; + if (nextCharacter == EQUALS_SIGN || isAlphanumeric(nextCharacter)) { + [_inputStreamReader rewindToMarkedLocation]; + if (nextCharacter == EQUALS_SIGN) { + [self emitParseError:@"Named entity in attribute ending with equal-sign"]; + } + return nil; + } + } + + if ([entityName hasSuffix:@";"] == NO) { + [self emitParseError:@"Named entity without semicolon"]; + } + + return replacement; +} + #pragma mark - States - (void)HTMLTokenizerStateData @@ -285,7 +345,8 @@ { [self switchToState:HTMLTokenizerStateData]; - NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter]; + NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter + inAttribute:NO]; if (characterReference == nil) { [self emitCharacterToken:AMPERSAND]; } else { @@ -320,7 +381,8 @@ { [self switchToState:HTMLTokenizerStateRCDATA]; - NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)EOF]; + NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:(UTF32Char)EOF + inAttribute:NO]; if (characterReference == nil) { [self emitCharacterToken:AMPERSAND]; } else { @@ -1368,7 +1430,8 @@ - (void)HTMLTokenizerStateCharacterReferenceInAttributeValue { - NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter]; + NSString *characterReference = [self attemptToConsumeCharachterReferenceWithAddtionalAllowedCharacter:_additionalAllowedCharacter + inAttribute:YES]; if (characterReference == nil) { [_currentAttributeValue appendString:StringFromUniChar(AMPERSAND)]; } else { diff --git a/HTMLKit/HTMLTokenizerCharacters.h b/HTMLKit/HTMLTokenizerCharacters.h index 6e7c574..a35c293 100644 --- a/HTMLKit/HTMLTokenizerCharacters.h +++ b/HTMLKit/HTMLTokenizerCharacters.h @@ -137,6 +137,13 @@ NS_INLINE BOOL isHexDigit(UTF32Char character) (character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_F)); } +NS_INLINE BOOL isAlphanumeric(UTF32Char character) +{ + return ((character >= DIGIT_ZERO && character <= DIGIT_NINE) || + (character >= LATIN_CAPITAL_LETTER_A && character <= LATIN_CAPITAL_LETTER_Z) || + (character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_Z)); +} + NS_INLINE BOOL isValidNumericRange(UTF32Char character) { return ((character >= 0xD800 && character <= 0xDFFF) || diff --git a/HTMLKit/HTMLTokenizerEntities.h b/HTMLKit/HTMLTokenizerEntities.h index ca37aeb..49cee29 100644 --- a/HTMLKit/HTMLTokenizerEntities.h +++ b/HTMLKit/HTMLTokenizerEntities.h @@ -10,6 +10,7 @@ @interface HTMLTokenizerEntities : NSObject ++ (NSArray *)entityNames; + (NSString *)replacementForNamedCharacterEntity:(NSString *)entity; @end diff --git a/HTMLKit/HTMLTokenizerEntities.m b/HTMLKit/HTMLTokenizerEntities.m index 474d7e6..cdddce3 100644 --- a/HTMLKit/HTMLTokenizerEntities.m +++ b/HTMLKit/HTMLTokenizerEntities.m @@ -2241,34 +2241,38 @@ NAMED_CHARACTER_REFERENCE( "zwj;", "\U0000200d" ) \ NAMED_CHARACTER_REFERENCE( "zwnj;", "\U0000200c" ) -static CFStringRef NamedReferencesNames[] = { -#define NAMED_CHARACTER_REFERENCE( name, value ) CFSTR(name), - NAMED_CHARACTER_REFERENCES -#undef NAMED_CHARACTER_REFERENCE -}; - -static CFStringRef NamedReferencesValues[] = { -#define NAMED_CHARACTER_REFERENCE( name, value ) CFSTR(value), - NAMED_CHARACTER_REFERENCES -#undef NAMED_CHARACTER_REFERENCE -}; - -static CFDictionaryRef _entities; +static NSDictionary *_entities; @implementation HTMLTokenizerEntities + (void)initialize { if (self == [HTMLTokenizerEntities class]) { - CFIndex numValues = sizeof(NamedReferencesNames) / sizeof(NamedReferencesNames[0]); - _entities = CFDictionaryCreate(NULL, (const void **) &NamedReferencesNames, (const void **) &NamedReferencesValues, numValues, - &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); + + NSArray * NamedReferencesNames = @[ +#define NAMED_CHARACTER_REFERENCE( name, value ) @(name), + NAMED_CHARACTER_REFERENCES +#undef NAMED_CHARACTER_REFERENCE + ]; + + NSArray * NamedReferencesValues = @[ +#define NAMED_CHARACTER_REFERENCE( name, value ) @(value), + NAMED_CHARACTER_REFERENCES +#undef NAMED_CHARACTER_REFERENCE + ]; + + _entities = [[NSDictionary alloc] initWithObjects:NamedReferencesValues forKeys:NamedReferencesNames]; } } ++ (NSArray *)entityNames +{ + return [_entities allKeys]; +} + + (NSString *)replacementForNamedCharacterEntity:(NSString *)entity { - NSString *replacement = (NSString *) CFDictionaryGetValue(_entities, (__bridge const void *)(entity)); + NSString *replacement = [_entities objectForKey:entity]; return replacement; }