#include "SyntaxHighlighter.h" #include "UnicodeEscapes.h" #include #include #include #include #include #include #include #include #include namespace { struct Token { std::string type; std::string value; bool operator==(const Token&) const = default; }; struct Inspection { std::string text; std::vector tokens; size_t depth = 0; }; void Inspect( const TokenList& list, Inspection& result, std::string_view type = {}, size_t depth = 0) { result.depth = std::max(result.depth, depth); for (const auto& node : list) { if (node.isSyntax()) { const auto& syntax = static_cast(node); Inspect(syntax.children(), result, syntax.type(), depth + 1); } else { const auto text = static_cast(node).value(); result.text.append(text); if (!type.empty()) { result.tokens.push_back({ std::string(type), std::string(text) }); } } } } bool CheckTokens( SyntaxHighlighter& highlighter, const std::string& language, const std::string& text, const std::vector& expected) { const auto tokens = highlighter.tokenize(text, language); auto result = Inspection(); Inspect(tokens, result); if (result.text == text && result.tokens == expected) { return true; } std::cerr << "Unexpected tokens for " << language << ": " << text << '\n'; for (const auto& token : result.tokens) { std::cerr << " " << token.type << ": " << token.value << '\n'; } return false; } // The tokens of a "---" and "+++" pair naming the same file, "a/" and "b/", // whose language is . std::vector Files(const std::string& stem, const std::string& name) { return { { "coord", "--- " }, { "embedded-language-name", "a/" + stem }, { "embedded-language-name", name }, { "coord", "\n" }, { "coord", "+++ " }, { "embedded-language-name", "b/" + stem }, { "embedded-language-name", name }, }; } std::vector Concat(std::vector tokens, const std::vector& more) { tokens.insert(tokens.end(), more.begin(), more.end()); return tokens; } // CRLF line ends are read as LF ones are: the same tokens, each holding the same text but for // the carriage returns. bool CheckLineEnds( SyntaxHighlighter& highlighter, const std::string& language, const std::string& text) { auto crlf = std::string(); for (const char c : text) { if (c == '\n') { crlf += '\r'; } crlf += c; } auto lf = Inspection(); Inspect(highlighter.tokenize(text, language), lf); auto result = Inspection(); Inspect(highlighter.tokenize(crlf, language), result); auto stripped = std::vector(); for (auto token : result.tokens) { std::erase(token.value, '\r'); if (!token.value.empty()) { stripped.push_back(token); } } if (result.text == crlf && stripped == lf.tokens) { return true; } std::cerr << "CRLF tokens differ from LF ones for " << language << ": " << text << '\n'; return false; } void CollectAliases(const TokenList& list, const std::string& type, std::vector& result) { for (const auto& node : list) { if (!node.isSyntax()) { continue; } const auto& syntax = static_cast(node); if (syntax.type() == type && !syntax.alias().empty()) { auto inspection = Inspection(); Inspect(syntax.children(), inspection); result.push_back({ syntax.alias(), inspection.text }); } CollectAliases(syntax.children(), type, result); } } // The aliases of the tokens of `type`, each with the text it covers. bool CheckAliases( SyntaxHighlighter& highlighter, const std::string& language, const std::string& text, const std::string& type, const std::vector& expected) { auto result = std::vector(); CollectAliases(highlighter.tokenize(text, language), type, result); if (result == expected) { return true; } std::cerr << "Unexpected aliases of " << type << " for " << language << ": " << text << '\n'; for (const auto& token : result) { std::cerr << " " << token.type << ": " << token.value << '\n'; } return false; } bool CheckPattern( const std::string& pattern, const std::string& text, bool expected) { const auto regex = boost::regex( libprisma::widenUnicodeClasses(pattern), boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); if (boost::regex_match(text, regex) == expected) { return true; } std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; return false; } } int main(int argc, char** argv) { if (argc != 2) { std::cerr << "usage: check-unicode \n"; return 2; } auto input = std::ifstream(argv[1], std::ios::binary); if (!input) { std::cerr << "Cannot read grammar table\n"; return 2; } const auto grammars = std::string(std::istreambuf_iterator(input), {}); auto highlighter = SyntaxHighlighter(grammars); auto failed = 0; for (const auto& quoted : std::vector{ "\u201cone\u201d", "\u201ca\u2014b\u201d", "\u201ca\u201d\u201db\u201d", "\"one\"", }) { failed += !CheckTokens( highlighter, "visual-basic", quoted + " + 42 + " + quoted, { { "string", quoted }, { "operator", "+" }, { "number", "42" }, { "operator", "+" }, { "string", quoted }, }); } for (const auto& quoted : std::vector{ "\u2018foo\u2019", "\u2018a\u2014b\u2019", "`foo'", }) { failed += !CheckTokens( highlighter, "stata", quoted + " + 42 + " + quoted, { { "variable", quoted }, { "operator", "+" }, { "number", "42" }, { "operator", "+" }, { "variable", quoted }, }); } failed += !CheckPattern(R"([^\u2019]+)", "a\u2019b", false); failed += !CheckPattern(R"([^\u2019]+)", "a\u2014\U0001F600b", true); failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u201db", false); failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\"b", false); failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u2014b\n", true); failed += !CheckPattern(R"([^\u2019]+)", "", false); failed += !CheckPattern(R"([^\u2019]{2})", "ab", true); failed += !CheckPattern(R"([^\u2019]{2})", "a\u2019", false); failed += !CheckPattern(R"([^^\u2019]+)", "a^b", false); failed += !CheckPattern(R"([^^\u2019]+)", "abc", true); failed += !CheckPattern(R"([\u201c\u201d]+)", "\u201c\u201d", true); failed += !CheckPattern(R"([\uD800-\uDBFF][\uDC00-\uDFFF])", "\U0001F600", true); const auto text = std::string("#iFclude "); const auto tokens = highlighter.tokenize(text, "brightscript"); auto result = Inspection(); Inspect(tokens, result); if (result.text != text || result.depth > 32) { std::cerr << "BrightScript recursion guard failed\n"; ++failed; } failed += !CheckTokens( highlighter, "brightscript", "print 42", { { "keyword", "print" }, { "number", "42" }, }); // The language the file names, or the one asked for whatever they name: as Python, "/*" // would be two operators rather than a comment. One asked for that is no language leaves // the file names to say. const auto hunk = std::string( "@@ -1,4 +1,4 @@ class Foo\n" " /* one\n" " two */\n" "-int x;\n" "+int y;\n"); const auto code = std::vector{ { "coord", "@@ -1,4 +1,4 @@ " }, { "keyword", "class" }, { "embedded-language", " " }, { "class-name", "Foo" }, { "prefix", " " }, { "comment", "/* one\n" }, { "prefix", " " }, { "comment", " two */" }, { "embedded-language", "\n" }, { "prefix", "-" }, { "line", "int x;\n" }, { "prefix", "+" }, { "line", "int y;\n" }, }; for (const auto& [language, extension] : std::vector>{ { "diff", "cs" }, { "diff-cs", "py" }, { "diff-vue", "cs" }, }) { const auto patch = "--- a/Foo." + extension + "\n+++ b/Foo." + extension + "\n" + hunk; failed += !CheckTokens(highlighter, language, patch, Concat(Files("Foo.", extension), code)); failed += !CheckLineEnds(highlighter, language, patch); } failed += !CheckTokens( highlighter, "diff", hunk, { { "coord", "@@ -1,4 +1,4 @@ " }, { "embedded-language", "class Foo" }, { "prefix", " " }, { "embedded-language", "/* one\n" }, { "prefix", " " }, { "embedded-language", " two */\n" }, { "prefix", "-" }, { "line", "int x;\n" }, { "prefix", "+" }, { "line", "int y;\n" }, }); // The files are coloured as their lines. Alone, "---" is a removed line, here a SQL comment, // and names no file. const auto sql = std::string( "--- a/q.sql\n" "+++ b/q.sql\n" "@@ -1,3 +1,2 @@\n" " select 1;\n" "--- see helper.py\n" " select 2;\n"); failed += !CheckAliases( highlighter, "diff", sql, "coord", { { "deleted", "--- a/q.sql" }, { "inserted", "+++ b/q.sql" }, { "deleted", "--- see helper.py" }, }); failed += !CheckTokens( highlighter, "diff", sql, Concat(Files("q.", "sql"), { { "coord", "@@ -1,3 +1,2 @@" }, { "prefix", " " }, { "keyword", "select" }, { "embedded-language", " " }, { "number", "1" }, { "punctuation", ";" }, { "embedded-language", "\n" }, { "coord", "--- see helper.py" }, { "prefix", " " }, { "keyword", "select" }, { "embedded-language", " " }, { "number", "2" }, { "punctuation", ";" }, { "embedded-language", "\n" }, })); // Each file's code is in the language its name says, up to the next file: a quoted name // still says it, and one that says nothing ends the language before. A deleted file's // "+++ /dev/null" leaves its old name standing. const auto files = std::string( "--- a/x.py\n" "+++ b/x.py\n" "@@ -1 +1 @@\n" " x = 1\n" "--- \"a/\\303\\266.cs\"\n" "+++ \"b/\\303\\266.cs\"\n" "@@ -1 +1 @@\n" " /* c */\n" "--- a/foo.c~\n" "+++ b/foo.c~\n" "@@ -1 +1 @@\n" " /* c */\n" "--- a/Foo.RS\t2024-01-01 00:00:00\n" "+++ /dev/null\n" "@@ -1 +0,0 @@ fn main()\n" " let x = 1;\n" "--- a/Makefile\n" "+++ b/Makefile\n" "@@ -1 +1 @@\n" " all:\n"); failed += !CheckTokens( highlighter, "diff", files, { { "coord", "--- " }, { "embedded-language-name", "a/x." }, { "embedded-language-name", "py" }, { "coord", "\n" }, { "coord", "+++ " }, { "embedded-language-name", "b/x." }, { "embedded-language-name", "py" }, { "coord", "@@ -1 +1 @@" }, { "prefix", " " }, { "embedded-language", "x " }, { "operator", "=" }, { "embedded-language", " " }, { "number", "1" }, { "embedded-language", "\n" }, { "coord", "--- " }, { "embedded-language-name", "\"a/\\303\\266." }, { "embedded-language-name", "cs" }, { "embedded-language-name", "\"" }, { "coord", "\n" }, { "coord", "+++ " }, { "embedded-language-name", "\"b/\\303\\266." }, { "embedded-language-name", "cs" }, { "embedded-language-name", "\"" }, { "coord", "@@ -1 +1 @@" }, { "prefix", " " }, { "comment", "/* c */" }, { "embedded-language", "\n" }, { "coord", "--- " }, { "embedded-language-name", "a/foo.c~" }, { "coord", "\n" }, { "coord", "+++ " }, { "embedded-language-name", "b/foo.c~" }, { "coord", "@@ -1 +1 @@" }, { "prefix", " " }, { "embedded-language", "/* c */\n" }, { "coord", "--- " }, { "embedded-language-name", "a/Foo." }, { "embedded-language-name", "RS" }, { "coord", "\t2024-01-01 00:00:00" }, { "coord", "\n" }, { "coord", "+++ /dev/null" }, { "coord", "@@ -1 +0,0 @@ " }, { "keyword", "fn" }, { "embedded-language", " " }, { "function-definition", "main" }, { "punctuation", "(" }, { "punctuation", ")" }, { "prefix", " " }, { "keyword", "let" }, { "embedded-language", " x " }, { "operator", "=" }, { "embedded-language", " " }, { "number", "1" }, { "punctuation", ";" }, { "embedded-language", "\n" }, { "coord", "--- " }, { "embedded-language-name", "a/" }, { "embedded-language-name", "Makefile" }, { "coord", "\n" }, { "coord", "+++ " }, { "embedded-language-name", "b/" }, { "embedded-language-name", "Makefile" }, { "coord", "@@ -1 +1 @@" }, { "prefix", " " }, { "target", "all" }, { "punctuation", ":" }, { "embedded-language", "\n" }, }); // A file extension that is no language name reads as itself, as an unknown name does. for (const auto& [language, name] : std::vector>{ { "rs", "rs" }, { "rust", "Rust" }, { "nope", "nope" }, }) { if (highlighter.languageName(language) != name) { std::cerr << "Unexpected name for " << language << ": " << highlighter.languageName(language) << '\n'; ++failed; } } auto dataweave = std::string(); for (int i = 0; i < 2033; ++i) { dataweave += "\"/ "; } const auto start = std::chrono::steady_clock::now(); failed += !CheckTokens(highlighter, "dataweave", dataweave, {}); if (std::chrono::steady_clock::now() - start > std::chrono::seconds(2)) { std::cerr << "DataWeave highlighting exceeded the work limit\n"; ++failed; } failed += !CheckTokens( highlighter, "dataweave", "var answer = 42", { { "keyword", "var" }, { "operator", "=" }, { "number", "42" }, }); std::cout << failed << " Unicode highlighting checks failed\n"; return failed ? 1 : 0; }