GitHub/HTMLKit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update tokenizer implementation as per spec as of 2017.09.09

Browse Source 
- Use new initial states in tests according to:
https://github.com/html5lib/html5lib-tests/pull/101

- Implement tokenization errors introduced in:
https://github.com/whatwg/html/pull/2701
https://github.com/html5lib/html5lib-tests/pull/92
This commit is contained in:
iska
2017-09-10 02:23:31 +02:00
parent d4ff2e3869
commit 4a41b89ecf
8 changed files with 921 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/HTMLParseErrorToken.m
+14
View File
@@ -33,6 +33,20 @@
return self;
}
- (BOOL)isEqual:(id)other
{
if ([other isKindOfClass:[self class]]) {
HTMLParseErrorToken *token = (HTMLParseErrorToken *)other;
return bothNilOrEqual(self.code, token.code);
}
return NO;
}
- (NSUInteger)hash
{
return self.code.hash + self.code.hash;
}
- (NSString *)description
{
return [NSString stringWithFormat:@"<%@: %p Code='%@' Details='%@' Location='%lu'>", self.class, self, _code, _details, (unsigned long)_location];
Sources/HTMLTokenizer.m
+812 -643
View File
File diff suppressed because it is too large Load Diff
Sources/include/HTMLTokenizer.h
+14
View File
@@ -16,6 +16,13 @@
@class HTMLParser;
/**
Typedef for the parse error callback block.
@param token The parse error token.
*/
typedef void (^ HTMLTokenizerParseErrorCallback)(HTMLParseErrorToken *token);
/**
* HTML Tokenizer
* https://html.spec.whatwg.org/multipage/syntax.html#tokenization
@@ -39,6 +46,13 @@
*/
@property (nonatomic, weak) HTMLParser *parser;
/**
An error callback block, which gets called when encountering parse errors while tokenizing the stream
Parse error tokens are dropped if the callback is `nil`.
*/
@property (nonatomic, copy) HTMLTokenizerParseErrorCallback parseErrorCallback;
/**
Initializes a new Tokenizer with the given string.
Sources/include/HTMLTokenizerCharacters.h
+28 -10
View File
@@ -23,11 +23,16 @@
CHAR( DIGIT_NINE, 0x0039 ) \
CHAR( LATIN_CAPITAL_LETTER_A, 0x0041 ) \
CHAR( LATIN_CAPITAL_LETTER_F, 0x0046 ) \
CHAR( LATIN_CAPITAL_LETTER_P, 0x0050 ) \
CHAR( LATIN_CAPITAL_LETTER_S, 0x0053 ) \
CHAR( LATIN_CAPITAL_LETTER_X, 0x0058 ) \
CHAR( LATIN_CAPITAL_LETTER_Z, 0x005A ) \
CHAR( RIGHT_SQUARE_BRACKET, 0x005D ) \
CHAR( GRAVE_ACCENT, 0x0060 ) \
CHAR( LATIN_SMALL_LETTER_A, 0x0061 ) \
CHAR( LATIN_SMALL_LETTER_F, 0x0066 ) \
CHAR( LATIN_SMALL_LETTER_P, 0x0070 ) \
CHAR( LATIN_SMALL_LETTER_S, 0x0073 ) \
CHAR( LATIN_SMALL_LETTER_X, 0x0078 ) \
CHAR( LATIN_SMALL_LETTER_Z, 0x007A ) \
CHAR( HYPHEN_MINUS, 0x002D ) \
@@ -82,13 +87,17 @@ NUMERIC_REPLACEMENT_CHARACTERS
#undef CHAR
};
NS_INLINE BOOL isControlOrUndefinedCharacter(UTF32Char character)
NS_INLINE BOOL isControlCharacter(unsigned long long character)
{
return ((character >= 0x0001 && character <= 0x0008) ||
(character >= 0x000D && character <= 0x001F) ||
(character >= 0x007F && character <= 0x009F) ||
(character >= 0xFDD0 && character <= 0xFDEF) ||
character == 0x000B ||
(character >= 0x000E && character <= 0x001F) ||
(character >= 0x007F && character <= 0x009F));
}
NS_INLINE BOOL isNoncharacter(unsigned long long character)
{
return ((character >= 0xFDD0 && character <= 0xFDEF) ||
character == 0xFFFE ||
character == 0xFFFF ||
character == 0x1FFFE ||
@@ -137,6 +146,18 @@ NS_INLINE BOOL isHexDigit(UTF32Char character)
(character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_F));
}
NS_INLINE BOOL isUpperHexDigit(UTF32Char character)
{
return ((character >= LATIN_CAPITAL_LETTER_A && character <= LATIN_CAPITAL_LETTER_F) ||
(character >= DIGIT_ZERO && character <= DIGIT_NINE));
}
NS_INLINE BOOL isLowerHexDigit(UTF32Char character)
{
return ((character >= LATIN_SMALL_LETTER_A && character <= LATIN_SMALL_LETTER_F) ||
(character >= DIGIT_ZERO && character <= DIGIT_NINE));
}
NS_INLINE BOOL isAlphanumeric(UTF32Char character)
{
return ((character >= DIGIT_ZERO && character <= DIGIT_NINE) ||
@@ -151,17 +172,14 @@ NS_INLINE BOOL isStringAlphanumeric(NSString *string)
return ([string rangeOfCharacterFromSet:set].location == NSNotFound);
}
NS_INLINE BOOL isInvalidNumericRange(unsigned long long numeric)
NS_INLINE BOOL isSurrogate(unsigned long long character)
{
return ((numeric >= 0xD800 && numeric <= 0xDFFF) ||
numeric > 0x10FFFF);
return (character >= 0xD800 && character <= 0xDFFF);
}
NS_INLINE unichar NumericReplacementCharacter(UTF32Char character)
{
if (character == NULL_CHAR) {
return REPLACEMENT_CHAR;
} else if (character >= 0x0080 && character <= 0x009F) {
if (character >= 0x0080 && character <= 0x009F) {
return NumericReplacementTable[character - 0x0080];
} else {
return NULL_CHAR;
Sources/include/HTMLTokenizerStates.h
+16 -3
View File
@@ -12,9 +12,7 @@
#define TOKENIZER_STATES \
STATE_ENTRY( HTMLTokenizerStateData, = 0) \
STATE_ENTRY( HTMLTokenizerStateCharacterReferenceInData, ) \
STATE_ENTRY( HTMLTokenizerStateRCDATA, ) \
STATE_ENTRY( HTMLTokenizerStateCharacterReferenceInRCDATA, ) \
STATE_ENTRY( HTMLTokenizerStateRAWTEXT, ) \
STATE_ENTRY( HTMLTokenizerStateScriptData, ) \
STATE_ENTRY( HTMLTokenizerStatePLAINTEXT, ) \
@@ -59,6 +57,10 @@
STATE_ENTRY( HTMLTokenizerStateCommentStart, ) \
STATE_ENTRY( HTMLTokenizerStateCommentStartDash, ) \
STATE_ENTRY( HTMLTokenizerStateComment, ) \
STATE_ENTRY( HTMLTokenizerStateCommentLessThanSign, ) \
STATE_ENTRY( HTMLTokenizerStateCommentLessThanSignBang, ) \
STATE_ENTRY( HTMLTokenizerStateCommentLessThanSignBangDash, ) \
STATE_ENTRY( HTMLTokenizerStateCommentLessThanSignBangDashDash, ) \
STATE_ENTRY( HTMLTokenizerStateCommentEndDash, ) \
STATE_ENTRY( HTMLTokenizerStateCommentEnd, ) \
STATE_ENTRY( HTMLTokenizerStateCommentEndBang, ) \
@@ -78,7 +80,18 @@
STATE_ENTRY( HTMLTokenizerStateDOCTYPESystemIdentifierSingleQuoted, ) \
STATE_ENTRY( HTMLTokenizerStateAfterDOCTYPESystemIdentifier, ) \
STATE_ENTRY( HTMLTokenizerStateBogusDOCTYPE, ) \
STATE_ENTRY( HTMLTokenizerStateCDATASection, )
STATE_ENTRY( HTMLTokenizerStateCDATASection, ) \
STATE_ENTRY( HTMLTokenizerStateCDATASectionBracket, ) \
STATE_ENTRY( HTMLTokenizerStateCDATASectionEnd, ) \
STATE_ENTRY( HTMLTokenizerStateCharacterReference, ) \
STATE_ENTRY( HTMLTokenizerStateNamedCharacterReference, ) \
STATE_ENTRY( HTMLTokenizerStateAmbiguousAmpersand, ) \
STATE_ENTRY( HTMLTokenizerStateNumericCharacterReference, ) \
STATE_ENTRY( HTMLTokenizerStateHexadecimalCharacterReferenceStart, ) \
STATE_ENTRY( HTMLTokenizerStateDecimalCharacterReferenceStart, ) \
STATE_ENTRY( HTMLTokenizerStateHexadecimalCharacterReference, ) \
STATE_ENTRY( HTMLTokenizerStateDecimalCharacterReference, ) \
STATE_ENTRY( HTMLTokenizerStateNumericCharacterReferenceEnd, )
typedef NS_ENUM(NSUInteger, HTMLTokenizerState)
{
Tests/HTMLKitTests/HTML5LibTokenizerTest.h
+1 -1
View File
@@ -14,9 +14,9 @@
@property (nonatomic, copy) NSString *title;
@property (nonatomic, copy) NSString *input;
@property (nonatomic, strong) NSArray *output;
@property (nonatomic, strong) NSArray *errors;
@property (nonatomic, strong) NSArray *initialStates;
@property (nonatomic, copy) NSString *lastStartTag;
@property (nonatomic, assign) BOOL ignoreErrorOrder;
+ (NSDictionary *)loadHTML5LibTokenizerTests;
Tests/HTMLKitTests/HTML5LibTokenizerTest.m
+17 -13
View File
@@ -70,16 +70,16 @@ static NSString * const Tokenizer = @"tokenizer";
{
BOOL doubleEscaped = [test[@"doubleEscaped"] boolValue];
// Test Title
// Test Title
self.title = test[@"description"];
// Test Input
// Test Input
self.input = test[@"input"];
if (doubleEscaped) {
self.input = [self processDoubleEscaped:self.input];
}
// Test Output
// Test Output
NSMutableArray *tokens = [NSMutableArray array];
NSArray *outputs = test[@"output"];
for (NSArray *output in outputs) {
@@ -89,7 +89,7 @@ static NSString * const Tokenizer = @"tokenizer";
[tokens addObject:[HTMLEOFToken token]];
self.output = tokens;
// Test Initial States
// Test Initial States
NSMutableArray *initialStates = [NSMutableArray array];
NSArray *states = test[@"initialStates"];
@@ -101,6 +101,10 @@ static NSString * const Tokenizer = @"tokenizer";
state = HTMLTokenizerStateRCDATA;
} else if ([name isEqualToString:@"RAWTEXT state"]) {
state = HTMLTokenizerStateRAWTEXT;
} else if ([name isEqualToString:@"Script data state"]) {
state = HTMLTokenizerStateScriptData;
} else if ([name isEqualToString:@"CDATA section state"]) {
state = HTMLTokenizerStateCDATASection;
}
[initialStates addObject:@(state)];
}
@@ -110,19 +114,21 @@ static NSString * const Tokenizer = @"tokenizer";
self.initialStates = initialStates;
// Test Last Start Tag
// Test Last Start Tag
self.lastStartTag = test[@"lastStartTag"];
// Ignore Error Order
self.ignoreErrorOrder = [test[@"ignoreErrorOrder"] boolValue];
// Test errors
NSArray *errors = test[@"errors"];
NSMutableArray *errorTokens = [NSMutableArray new];
for (NSDictionary *error in errors) {
HTMLParseErrorToken *token = [[HTMLParseErrorToken alloc] initWithCode:error[@"code"] details:nil location:0];
[errorTokens addObject:token];
}
self.errors = errorTokens;
}
- (HTMLToken *)processOutputToken:(id)output doubleEscaped:(BOOL)doubleEscaped
{
if ([output isKindOfClass:[NSString class]] && [output isEqualToString:@"ParseError"]) {
return [HTMLParseErrorToken new];
}
NSString *type = [output firstObject];
NSString *data = nil;
@@ -146,8 +152,6 @@ static NSString * const Tokenizer = @"tokenizer";
return token;
} else if ([type isEqualToString:@"EndTag"]) {
return [[HTMLEndTagToken alloc] initWithTagName:data];
} else if ([type isEqualToString:@"ParseError"]) {
return [HTMLParseErrorToken new];
} else if ([type isEqualToString:@"StartTag"]) {
HTMLStartTagToken *token = [[HTMLStartTagToken alloc] initWithTagName:data];
NSDictionary *attributes = output[2];
Tests/HTMLKitTests/HTMLKitTokenizerTests.m
+19 -8
View File
@@ -81,20 +81,31 @@
for (NSNumber *state in test.initialStates) {
HTMLTokenizer *tokenizer = [[HTMLTokenizer alloc] initWithString:test.input];
[tokenizer setValue:test.lastStartTag forKey:@"_lastStartTagName"];
[tokenizer setValue:test.lastStartTag forKey:@"_lastStartTagName"];
tokenizer.state = [state integerValue];
NSArray *expectedTokens = test.output;
NSArray *tokens = tokenizer.allObjects;
NSArray *expectedErrors = test.errors;
NSMutableArray *actualErrors = [NSMutableArray new];
tokenizer.parseErrorCallback = ^(HTMLParseErrorToken *token) {
[actualErrors addObject:token];
};
NSString *message = [NSString stringWithFormat:@"HTML5Lib test in file: \'%@\' Title: '%@'\nInput: '%@'\nExpected:\n%@\nActual:\n%@\n",
NSArray *expectedTokens = test.output;
NSArray *actualTokens = tokenizer.allObjects;
NSString *message = [NSString stringWithFormat:@"\nTest file: \'%@\'\nTitle: '%@'\nInput: '%@'\n",
test.testFile,
test.title,
test.input,
expectedTokens,
tokens];
XCTAssertEqualObjects(tokens, expectedTokens, @"%@", message);
test.input];
XCTAssertEqualObjects(actualTokens, expectedTokens, @"%@", message);
message = [NSString stringWithFormat:@"\nTest file: \'%@\'\nTitle: '%@'\nInput: '%@'\n",
test.testFile,
test.title,
test.input];
XCTAssertEqualObjects(actualErrors, expectedErrors, @"%@", message);
}
}
}