Add implementation for HTML Tokenization

- The Token classes are all defined in one single header right now and do not include an implmentation.
- The Parser class is currently only a stub

HTML Tokenization Spec: https://html.spec.whatwg.org/multipage/syntax.html#tokenization
This commit is contained in:
iska
2014-10-04 23:30:32 +02:00
parent c5b60e2977
commit 17f2f32350
4 changed files with 1745 additions and 60 deletions
+8 -2
View File
@@ -14,8 +14,10 @@
CHAR( CARRIAGE_RETURN, 0x000D ) \
CHAR( SPACE, 0x0020 ) \
CHAR( EXCLAMATION_MARK, 0x0021 ) \
CHAR( QUOTATION_MARK, 0x0022 ) \
CHAR( NUMBER_SIGN, 0x0023 ) \
CHAR( AMPERSAND, 0x0026 ) \
CHAR( APOSTROPHE, 0x0027 ) \
CHAR( SOLIDUS, 0x002F ) \
CHAR( DIGIT_ZERO, 0x0030 ) \
CHAR( DIGIT_NINE, 0x0039 ) \
@@ -23,12 +25,16 @@
CHAR( LATIN_CAPITAL_LETTER_F, 0x0046 ) \
CHAR( LATIN_CAPITAL_LETTER_X, 0x0058 ) \
CHAR( LATIN_CAPITAL_LETTER_Z, 0x005A ) \
CHAR( GRAVE_ACCENT, 0x0060 ) \
CHAR( LATIN_SMALL_LETTER_A, 0x0061 ) \
CHAR( LATIN_SMALL_LETTER_F, 0x0066 ) \
CHAR( LATIN_SMALL_LETTER_X, 0x0078 ) \
CHAR( LATIN_SMALL_LETTER_Z, 0x007A ) \
CHAR( HYPHEN_MINUS, 0x002D ) \
CHAR( SEMICOLON, 0x003B ) \
CHAR( LESS_THAN_SIGN, 0x003C ) \
CHAR( EQUALS_SIGN, 0x003D ) \
CHAR( GREATER_THAN_SIGN, 0x003E ) \
CHAR( QUESTION_MARK, 0x003F ) \
CHAR( REPLACEMENT_CHAR, 0xFFFD )
@@ -93,8 +99,8 @@ NS_INLINE BOOL isHexDigit(UTF32Char character)
NS_INLINE BOOL isValidNumericRange(UTF32Char character)
{
return ((character >= 0xD800 && character <= 0xDFFF)
|| character > 0x10FFFF);
return ((character >= 0xD800 && character <= 0xDFFF) ||
character > 0x10FFFF);
}
NS_INLINE unichar NumericReplacementCharacter(UTF32Char character)