Fix highlight regressions.

This commit is contained in:
John Preston committed 2026-09-29 16:55:32 +04:00
1 parent ca59e6434e
commit 31a5d6fbbe
3 files changed
+198 -5

No files matched your search

+7
View File
@@ -47,6 +47,13 @@ jobs:
g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
- name: Check Unicode highlighting and recursion
run: |
g++ -std=c++20 -O0 -g -Ilibprisma -o tools/check-unicode \
tools/check-unicode.cpp libprisma/Highlight.cpp libprisma/LanguageTree.cpp \
libprisma/SyntaxHighlighter.cpp libprisma/TokenList.cpp -lboost_regex
./tools/check-unicode libprisma/grammars.dat
- name: Compare what the two engines matched
run: |
if ! diff -u boost.txt java.txt > engines.diff; then
+8 -5
View File
@@ -254,9 +254,7 @@ namespace libprisma
}
}
// a negated class drops its wide members: they exclude characters that cannot occur
// as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones
if (characters.negated || wide.empty())
if (wide.empty())
{
out += '[';
if (characters.negated)
@@ -268,10 +266,15 @@ namespace libprisma
return characters.end - begin;
}
out += "(?:";
// Exclude whole UTF-8 sequences without rejecting bytes shared by other characters.
out += characters.negated ? "(?:(?!" : "(?:";
if (!narrow.empty())
{
out += '[';
if (narrow.front() == '^')
{
out += '\\';
}
out += narrow;
out += ']';
out += '|';
@@ -284,7 +287,7 @@ namespace libprisma
}
appendUtf8(out, wide[i]);
}
out += ')';
out += characters.negated ? ")[\\s\\S])" : ")";
return characters.end - begin;
}
+183
View File
@@ -0,0 +1,183 @@
#include "SyntaxHighlighter.h"
#include "UnicodeEscapes.h"
#include <boost/regex.hpp>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <string>
#include <vector>
namespace {
struct Token
{
std::string type;
std::string value;
bool operator==(const Token&) const = default;
};
struct Inspection
{
std::string text;
std::vector<Token> tokens;
size_t depth = 0;
};
void Inspect(
const TokenList& list,
Inspection& result,
std::string_view type = {},
size_t depth = 0)
{
result.depth = std::max(result.depth, depth);
for (const auto& node : list)
{
if (node.isSyntax())
{
const auto& syntax = static_cast<const Syntax&>(node);
Inspect(syntax.children(), result, syntax.type(), depth + 1);
}
else
{
const auto text = static_cast<const Text&>(node).value();
result.text.append(text);
if (!type.empty())
{
result.tokens.push_back({ std::string(type), std::string(text) });
}
}
}
}
bool CheckTokens(
SyntaxHighlighter& highlighter,
const std::string& language,
const std::string& text,
const std::vector<Token>& expected)
{
const auto tokens = highlighter.tokenize(text, language);
auto result = Inspection();
Inspect(tokens, result);
if (result.text == text && result.tokens == expected)
{
return true;
}
std::cerr << "Unexpected tokens for " << language << ": " << text << '\n';
for (const auto& token : result.tokens)
{
std::cerr << " " << token.type << ": " << token.value << '\n';
}
return false;
}
bool CheckPattern(
const std::string& pattern,
const std::string& text,
bool expected)
{
const auto regex = boost::regex(
libprisma::widenUnicodeClasses(pattern),
boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m);
if (boost::regex_match(text, regex) == expected)
{
return true;
}
std::cerr << "Unexpected match for " << pattern << ": " << text << '\n';
return false;
}
}
int main(int argc, char** argv)
{
if (argc != 2)
{
std::cerr << "usage: check-unicode <grammars.dat>\n";
return 2;
}
auto input = std::ifstream(argv[1], std::ios::binary);
if (!input)
{
std::cerr << "Cannot read grammar table\n";
return 2;
}
const auto grammars = std::string(std::istreambuf_iterator<char>(input), {});
auto highlighter = SyntaxHighlighter(grammars);
auto failed = 0;
for (const auto& quoted : std::vector<std::string>{
"\u201cone\u201d",
"\u201ca\u2014b\u201d",
"\u201ca\u201d\u201db\u201d",
"\"one\"",
})
{
failed += !CheckTokens(
highlighter,
"visual-basic",
quoted + " + 42 + " + quoted,
{
{ "string", quoted },
{ "operator", "+" },
{ "number", "42" },
{ "operator", "+" },
{ "string", quoted },
});
}
for (const auto& quoted : std::vector<std::string>{
"\u2018foo\u2019",
"\u2018a\u2014b\u2019",
"`foo'",
})
{
failed += !CheckTokens(
highlighter,
"stata",
quoted + " + 42 + " + quoted,
{
{ "variable", quoted },
{ "operator", "+" },
{ "number", "42" },
{ "operator", "+" },
{ "variable", quoted },
});
}
failed += !CheckPattern(R"([^\u2019]+)", "a\u2019b", false);
failed += !CheckPattern(R"([^\u2019]+)", "a\u2014\U0001F600b", true);
failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u201db", false);
failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\"b", false);
failed += !CheckPattern(R"([^"\u201c\u201d]+)", "a\u2014b\n", true);
failed += !CheckPattern(R"([^\u2019]+)", "", false);
failed += !CheckPattern(R"([^\u2019]{2})", "ab", true);
failed += !CheckPattern(R"([^\u2019]{2})", "a\u2019", false);
failed += !CheckPattern(R"([^^\u2019]+)", "a^b", false);
failed += !CheckPattern(R"([^^\u2019]+)", "abc", true);
failed += !CheckPattern(R"([\u201c\u201d]+)", "\u201c\u201d", true);
failed += !CheckPattern(R"([\uD800-\uDBFF][\uDC00-\uDFFF])", "\U0001F600", true);
const auto text = std::string("#iFclude <a>");
const auto tokens = highlighter.tokenize(text, "brightscript");
auto result = Inspection();
Inspect(tokens, result);
if (result.text != text || result.depth > 32)
{
std::cerr << "BrightScript recursion guard failed\n";
++failed;
}
failed += !CheckTokens(
highlighter,
"brightscript",
"print 42",
{
{ "keyword", "print" },
{ "number", "42" },
});
std::cout << failed << " Unicode highlighting checks failed\n";
return failed ? 1 : 0;
}