diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml index 3e7a4a2..677d970 100644 --- a/.github/workflows/check-grammars.yml +++ b/.github/workflows/check-grammars.yml @@ -47,6 +47,13 @@ jobs: g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt + - name: Check Unicode highlighting and recursion + run: | + g++ -std=c++20 -O0 -g -Ilibprisma -o tools/check-unicode \ + tools/check-unicode.cpp libprisma/Highlight.cpp libprisma/LanguageTree.cpp \ + libprisma/SyntaxHighlighter.cpp libprisma/TokenList.cpp -lboost_regex + ./tools/check-unicode libprisma/grammars.dat + - name: Compare what the two engines matched run: | if ! diff -u boost.txt java.txt > engines.diff; then diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h index d72c450..6b8b6fc 100644 --- a/libprisma/UnicodeEscapes.h +++ b/libprisma/UnicodeEscapes.h @@ -254,9 +254,7 @@ namespace libprisma } } - // a negated class drops its wide members: they exclude characters that cannot occur - // as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones - if (characters.negated || wide.empty()) + if (wide.empty()) { out += '['; if (characters.negated) @@ -268,10 +266,15 @@ namespace libprisma return characters.end - begin; } - out += "(?:"; + // Exclude whole UTF-8 sequences without rejecting bytes shared by other characters. + out += characters.negated ? "(?:(?!" : "(?:"; if (!narrow.empty()) { out += '['; + if (narrow.front() == '^') + { + out += '\\'; + } out += narrow; out += ']'; out += '|'; @@ -284,7 +287,7 @@ namespace libprisma } appendUtf8(out, wide[i]); } - out += ')'; + out += characters.negated ? ")[\\s\\S])" : ")"; return characters.end - begin; } diff --git a/tools/check-unicode.cpp b/tools/check-unicode.cpp new file mode 100644 index 0000000..80caada --- /dev/null +++ b/tools/check-unicode.cpp @@ -0,0 +1,183 @@ +#include "SyntaxHighlighter.h" +#include "UnicodeEscapes.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +struct Token +{ + std::string type; + std::string value; + + bool operator==(const Token&) const = default; +}; + +struct Inspection +{ + std::string text; + std::vector tokens; + size_t depth = 0; +}; + +void Inspect( + const TokenList& list, + Inspection& result, + std::string_view type = {}, + size_t depth = 0) +{ + result.depth = std::max(result.depth, depth); + for (const auto& node : list) + { + if (node.isSyntax()) + { + const auto& syntax = static_cast(node); + Inspect(syntax.children(), result, syntax.type(), depth + 1); + } + else + { + const auto text = static_cast(node).value(); + result.text.append(text); + if (!type.empty()) + { + result.tokens.push_back({ std::string(type), std::string(text) }); + } + } + } +} + +bool CheckTokens( + SyntaxHighlighter& highlighter, + const std::string& language, + const std::string& text, + const std::vector& expected) +{ + const auto tokens = highlighter.tokenize(text, language); + auto result = Inspection(); + Inspect(tokens, result); + if (result.text == text && result.tokens == expected) + { + return true; + } + std::cerr << "Unexpected tokens for " << language << ": " << text << '\n'; + for (const auto& token : result.tokens) + { + std::cerr << " " << token.type << ": " << token.value << '\n'; + } + return false; +} + +bool CheckPattern( + const std::string& pattern, + const std::string& text, + bool expected) +{ + const auto regex = boost::regex( + libprisma::widenUnicodeClasses(pattern), + boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m); + if (boost::regex_match(text, regex) == expected) + { + return true; + } + std::cerr << "Unexpected match for " << pattern << ": " << text << '\n'; + return false; +} + +} + +int main(int argc, char** argv) +{ + if (argc != 2) + { + std::cerr << "usage: check-unicode \n"; + return 2; + } + auto input = std::ifstream(argv[1], std::ios::binary); + if (!input) + { + std::cerr << "Cannot read grammar table\n"; + return 2; + } + const auto grammars = std::string(std::istreambuf_iterator(input), {}); + auto highlighter = SyntaxHighlighter(grammars); + auto failed = 0; + + for (const auto& quoted : std::vector{ + "\u201cone\u201d", + "\u201ca\u2014b\u201d", + "\u201ca\u201d\u201db\u201d", + "\"one\"", + }) + { + failed += !CheckTokens( + highlighter, + "visual-basic", + quoted + " + 42 + " + quoted, + { + { "string", quoted }, + { "operator", "+" }, + { "number", "42" }, + { "operator", "+" }, + { "string", quoted }, + }); + } + for (const auto& quoted : std::vector{ + "\u2018foo\u2019", + "\u2018a\u2014b\u2019", + "`foo'", + }) + { + failed += !CheckTokens( + highlighter, + "stata", + quoted + " + 42 + " + quoted, + { + { "variable", quoted }, + { "operator", "+" }, + { "number", "42" }, + { "operator", "+" }, + { "variable", quoted }, + }); + } + + failed += !CheckPattern(R"([^\u2019]+)", "a\u2019b", false); + failed += !CheckPattern(R"([^\u2019]+)", "a\u2014\U0001F600b", true); + failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u201db", false); + failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\"b", false); + failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u2014b\n", true); + failed += !CheckPattern(R"([^\u2019]+)", "", false); + failed += !CheckPattern(R"([^\u2019]{2})", "ab", true); + failed += !CheckPattern(R"([^\u2019]{2})", "a\u2019", false); + failed += !CheckPattern(R"([^^\u2019]+)", "a^b", false); + failed += !CheckPattern(R"([^^\u2019]+)", "abc", true); + failed += !CheckPattern(R"([\u201c\u201d]+)", "\u201c\u201d", true); + failed += !CheckPattern(R"([\uD800-\uDBFF][\uDC00-\uDFFF])", "\U0001F600", true); + + const auto text = std::string("#iFclude "); + const auto tokens = highlighter.tokenize(text, "brightscript"); + auto result = Inspection(); + Inspect(tokens, result); + if (result.text != text || result.depth > 32) + { + std::cerr << "BrightScript recursion guard failed\n"; + ++failed; + } + failed += !CheckTokens( + highlighter, + "brightscript", + "print 42", + { + { "keyword", "print" }, + { "number", "42" }, + }); + + std::cout << failed << " Unicode highlighting checks failed\n"; + return failed ? 1 : 0; +}