Parse Swift 6.4 :: module selectors as a single infix operator token (#2395)

Co-authored-by: calda <1811727+calda@users.noreply.github.com>

Sources/Tokenizer.swift

@@ -989,6 +989,12 @@ extension UnicodeScalarView {
             }
         }
 
+        // `::` is always parsed as a single operator.
+        // `:` is not a valid operator head, so it must be handled specially here.
+        if readString("::") {
+            return .operator("::", .none)
+        }
+
         var start = self
         if var tail = readCharacter(where: isHead) {
             switch tail {
@@ -1687,7 +1693,7 @@ public func tokenize(_ source: String) -> [Token] {
         let prevToken: Token = tokens[i - 1]
         let type: OperatorType
         switch string {
-        case ":", "=", "->":
+        case ":", "::", "=", "->":
             type = .infix
         case ".":
             var _type = OperatorType.prefix

Tests/TokenizerTests.swift

@@ -2402,6 +2402,39 @@ final class TokenizerTests: XCTestCase {
         XCTAssertEqual(tokenize(input), output)
     }
 
+    func testDoubleColonOperator() {
+        let input = "Module::Type"
+        let output: [Token] = [
+            .identifier("Module"),
+            .operator("::", .infix),
+            .identifier("Type"),
+        ]
+        XCTAssertEqual(tokenize(input), output)
+    }
+
+    func testDoubleColonOperatorWithSpaces() {
+        let input = "Module :: Type"
+        let output: [Token] = [
+            .identifier("Module"),
+            .space(" "),
+            .operator("::", .infix),
+            .space(" "),
+            .identifier("Type"),
+        ]
+        XCTAssertEqual(tokenize(input), output)
+    }
+
+    func testDoubleColonOperatorWithLeadingNewline() {
+        let input = "Module\n::Type"
+        let output: [Token] = [
+            .identifier("Module"),
+            .linebreak("\n", 1),
+            .operator("::", .infix),
+            .identifier("Type"),
+        ]
+        XCTAssertEqual(tokenize(input), output)
+    }
+
     // MARK: chevrons (might be operators or generics)
 
     func testLessThanGreaterThan() {