diff --git a/Sources/Tokenizer.swift b/Sources/Tokenizer.swift index 383785e7..2e76dd7d 100644 --- a/Sources/Tokenizer.swift +++ b/Sources/Tokenizer.swift @@ -542,6 +542,7 @@ extension Collection where Element == Token { extension UnicodeScalar { var isDigit: Bool { isdigit(Int32(value)) > 0 } var isHexDigit: Bool { isxdigit(Int32(value)) > 0 } + var isLinebreak: Bool { "\n\r\u{000B}\u{000C}".unicodeScalars.contains(self) } var isSpace: Bool { switch value { case 0x0009, 0x0011, 0x0012, 0x0020, @@ -554,7 +555,7 @@ extension UnicodeScalar { } var isSpaceOrLinebreak: Bool { - isSpace || "\n\r\u{000B}\u{000C}".unicodeScalars.contains(self) + isSpace || isLinebreak } } @@ -769,7 +770,7 @@ private extension UnicodeScalarView { } let start = self if readString("\"\"") { - if first != "#" { + if first?.isLinebreak ?? true { return .startOfScope("\"\"\"") } self = start diff --git a/Tests/TokenizerTests.swift b/Tests/TokenizerTests.swift index 78470398..10702002 100644 --- a/Tests/TokenizerTests.swift +++ b/Tests/TokenizerTests.swift @@ -610,6 +610,16 @@ class TokenizerTests: XCTestCase { XCTAssertEqual(tokenize(input), output) } + func testRawStringContainingJustTwoUnescapedQuotes() { + let input = "#\"\"\"\"#" + let output: [Token] = [ + .startOfScope("#\""), + .stringBody("\"\""), + .endOfScope("\"#"), + ] + XCTAssertEqual(tokenize(input), output) + } + func testRawStringContainingUnhashedBackslash() { let input = "#\"\\\"#" let output: [Token] = [