mirror of
https://github.com/TelegramMessenger/libprisma.git
synced 2026-11-18 20:23:45 +00:00
Check every pattern against both regex engines on a pull request
This commit is contained in:
1 parent
d007354e40
commit
2f9ca450d2
5 files changed
+453
No files matched your search
@@ -0,0 +1,58 @@
|
||||
name: Check grammars
|
||||
|
||||
# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex,
|
||||
# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it
|
||||
# to that, rather than waiting for a language to quietly stop highlighting.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- components/**
|
||||
- generate.js
|
||||
- libprisma/**
|
||||
- tools/**
|
||||
- package.json
|
||||
- package-lock.json
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
patterns:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
# the committed table is regenerated on master, so a pull request is checked against what
|
||||
# its own components and generate.js produce
|
||||
- run: npm ci --ignore-scripts
|
||||
- run: npm start
|
||||
|
||||
- name: Escapes every engine agrees on
|
||||
run: node tools/check-escapes.js libprisma/grammars.dat
|
||||
|
||||
- name: Compile every pattern with java.util.regex
|
||||
run: |
|
||||
javac -d tools tools/PatternProbe.java
|
||||
java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt
|
||||
|
||||
- name: Compile every pattern with Boost
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libboost-regex-dev
|
||||
g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
|
||||
./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
|
||||
|
||||
- name: Compare what the two engines matched
|
||||
run: |
|
||||
if ! diff -u boost.txt java.txt > engines.diff; then
|
||||
echo "The two engines disagree about what these patterns match."
|
||||
echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt."
|
||||
head -40 engines.diff
|
||||
exit 1
|
||||
fi
|
||||
echo "Boost and java.util.regex agree on every pattern."
|
||||
@@ -0,0 +1,163 @@
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
|
||||
* matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
|
||||
* the one thing it promises: that a pattern means the same to every client that reads it.
|
||||
*
|
||||
* The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
|
||||
* anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
|
||||
* looking for.
|
||||
*/
|
||||
public class PatternProbe {
|
||||
|
||||
/** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
|
||||
private static final class Entry {
|
||||
String source;
|
||||
int flags;
|
||||
}
|
||||
|
||||
private static final class Reader {
|
||||
private final byte[] bytes;
|
||||
private int offset;
|
||||
|
||||
Reader(byte[] bytes) {
|
||||
this.bytes = bytes;
|
||||
}
|
||||
|
||||
int uint8() {
|
||||
return bytes[offset++] & 0xFF;
|
||||
}
|
||||
|
||||
int uint16() {
|
||||
return uint8() | (uint8() << 8);
|
||||
}
|
||||
|
||||
String string() {
|
||||
int length = uint8();
|
||||
if (length >= 254) {
|
||||
length = uint8() | (uint8() << 8) | (uint8() << 16);
|
||||
}
|
||||
String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
|
||||
offset += length;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Entry> read(String path) throws IOException {
|
||||
Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
|
||||
List<Entry> entries = new ArrayList<>();
|
||||
|
||||
for (int count = reader.uint16(); count > 0; count--) {
|
||||
String item = reader.string();
|
||||
int begin = item.indexOf('/');
|
||||
int end = item.lastIndexOf('/');
|
||||
if (begin < 0 || end <= begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String options = item.substring(end + 1);
|
||||
int aliasBegin = options.indexOf(',');
|
||||
|
||||
Entry entry = new Entry();
|
||||
entry.source = item.substring(begin + 1, end);
|
||||
for (int i = 0; i < aliasBegin; ++i) {
|
||||
switch (options.charAt(i)) {
|
||||
case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
|
||||
case 'm': entry.flags |= Pattern.MULTILINE; break;
|
||||
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
|
||||
}
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
|
||||
* line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
|
||||
* A Java client has to do this; it cannot go in the table, where \z would read as a z.
|
||||
*/
|
||||
private static String endOfInput(String source) {
|
||||
if (source.indexOf('$') < 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
StringBuilder result = new StringBuilder(source.length());
|
||||
boolean inClass = false;
|
||||
for (int i = 0; i < source.length(); ++i) {
|
||||
char c = source.charAt(i);
|
||||
if (c == '\\' && i + 1 < source.length()) {
|
||||
result.append(c).append(source.charAt(++i));
|
||||
} else if (c == '[') {
|
||||
inClass = true;
|
||||
result.append(c);
|
||||
} else if (c == ']') {
|
||||
inClass = false;
|
||||
result.append(c);
|
||||
} else if (c == '$' && !inClass) {
|
||||
result.append("\\z");
|
||||
} else {
|
||||
result.append(c);
|
||||
}
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
List<Entry> entries = read(args[0]);
|
||||
String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
|
||||
.replace(String.valueOf((char) 13), "");
|
||||
|
||||
StringBuilder out = new StringBuilder();
|
||||
int failed = 0;
|
||||
|
||||
for (int i = 0; i < entries.size(); ++i) {
|
||||
Entry entry = entries.get(i);
|
||||
String source = entry.source;
|
||||
if ((entry.flags & Pattern.MULTILINE) == 0) {
|
||||
source = endOfInput(source);
|
||||
}
|
||||
|
||||
Pattern pattern;
|
||||
try {
|
||||
pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
|
||||
} catch (PatternSyntaxException e) {
|
||||
if (failed++ < 20) {
|
||||
System.err.println("pattern " + i + ": " + e.getDescription());
|
||||
System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length())));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out.append(i);
|
||||
java.util.regex.Matcher matcher = pattern.matcher(corpus);
|
||||
int at = 0;
|
||||
while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
|
||||
out.append(' ').append(matcher.start()).append(':').append(matcher.end());
|
||||
at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
|
||||
if (at > corpus.length()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.append('\n');
|
||||
}
|
||||
|
||||
Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
|
||||
System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
|
||||
|
||||
if (failed > 0) {
|
||||
System.err.println(failed + " patterns java.util.regex will not take. "
|
||||
+ "Teach normalizeEscapes in generate.js to rewrite them.");
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Every escape in grammars.dat has to mean the same thing to every engine that reads it.
|
||||
// A pattern that fails to compile is caught by the probes beside this script; this one is for
|
||||
// the escapes that compile everywhere and quietly mean something else, which is the worse
|
||||
// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline
|
||||
// included, by java.util.regex, which made ini values run over the end of the line.
|
||||
const fs = require('fs')
|
||||
|
||||
// what generate.js is expected to emit, once normalizeEscapes has run over it
|
||||
const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split(''))
|
||||
|
||||
const REASONS = {
|
||||
v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B',
|
||||
h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere',
|
||||
H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere',
|
||||
V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere',
|
||||
R: 'any line break in java.util.regex, a literal R elsewhere',
|
||||
X: 'a grapheme cluster in java.util.regex, a literal X elsewhere',
|
||||
A: 'start of input in java.util.regex, a literal A elsewhere',
|
||||
Z: 'end of input in java.util.regex, a literal Z elsewhere',
|
||||
z: 'end of input in java.util.regex, a literal z elsewhere',
|
||||
G: 'end of the previous match in java.util.regex, a literal G elsewhere',
|
||||
Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere',
|
||||
E: 'ends one in java.util.regex, a literal E elsewhere',
|
||||
p: 'a unicode property in both, but the property names differ',
|
||||
N: 'a named character in java.util.regex',
|
||||
0: 'starts an octal escape in java.util.regex - emit \\x00',
|
||||
}
|
||||
|
||||
function readPatterns(path) {
|
||||
const buffer = fs.readFileSync(path)
|
||||
let offset = 0
|
||||
|
||||
const uint8 = () => buffer[offset++]
|
||||
const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value }
|
||||
const string = () => {
|
||||
let length = uint8()
|
||||
if (length >= 254) {
|
||||
length = uint8() | (uint8() << 8) | (uint8() << 16)
|
||||
}
|
||||
const value = buffer.toString('latin1', offset, offset + length)
|
||||
offset += length
|
||||
return value
|
||||
}
|
||||
|
||||
const patterns = []
|
||||
for (let count = uint16(); count > 0; count--) {
|
||||
patterns.push(string())
|
||||
}
|
||||
return patterns
|
||||
}
|
||||
|
||||
const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat')
|
||||
const problems = []
|
||||
|
||||
for (const item of patterns) {
|
||||
const end = item.lastIndexOf('/')
|
||||
const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end)
|
||||
|
||||
for (let i = 0; i < source.length; i++) {
|
||||
if (source[i] !== '\\') {
|
||||
continue
|
||||
}
|
||||
|
||||
const escaped = source[++i]
|
||||
if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) {
|
||||
problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set'])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`${patterns.length} patterns checked`)
|
||||
|
||||
if (problems.length === 0) {
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const seen = new Set()
|
||||
for (const [source, escape, reason] of problems) {
|
||||
if (seen.has(escape)) {
|
||||
continue
|
||||
}
|
||||
seen.add(escape)
|
||||
console.error(`\n${escape} is ${reason}`)
|
||||
console.error(` e.g. ${source.slice(0, 120)}`)
|
||||
}
|
||||
console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`)
|
||||
process.exit(1)
|
||||
@@ -0,0 +1,13 @@
|
||||
class Foo<T> where T : IBar { public int x = 1; } /* c */ // line
|
||||
<!DOCTYPE html><a href="x" [attr]='y'>{{ tpl }}</a>
|
||||
def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py
|
||||
let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3;
|
||||
SELECT * FROM t WHERE a IN (1,2) -- sql
|
||||
#include <stdio.h>
|
||||
key = value ; ini
|
||||
[section]
|
||||
- item:
|
||||
nested: [1, 2]
|
||||
0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y
|
||||
tab and trailing spaces
|
||||
#!/bin/sh
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down
|
||||
// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and
|
||||
// the two files are diffed, which is what holds the table to meaning one thing everywhere.
|
||||
//
|
||||
// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so
|
||||
// anything above U+007F diverges on offsets alone and would drown out the escape bugs.
|
||||
#include <boost/regex.hpp>
|
||||
|
||||
#include "../libprisma/UnicodeEscapes.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string readAll(const char* path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
return std::string((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
struct Reader {
|
||||
const std::string& bytes;
|
||||
size_t offset = 0;
|
||||
|
||||
uint8_t uint8() {
|
||||
return offset < bytes.size() ? static_cast<uint8_t>(bytes[offset++]) : 0;
|
||||
}
|
||||
|
||||
uint16_t uint16() {
|
||||
uint16_t low = uint8();
|
||||
return static_cast<uint16_t>(low | (uint8() << 8));
|
||||
}
|
||||
|
||||
std::string string() {
|
||||
size_t length = uint8();
|
||||
if (length >= 254) {
|
||||
size_t a = uint8(), b = uint8(), c = uint8();
|
||||
length = a | (b << 8) | (c << 16);
|
||||
}
|
||||
std::string value = bytes.substr(offset, length);
|
||||
offset += length;
|
||||
return value;
|
||||
}
|
||||
};
|
||||
|
||||
struct Entry {
|
||||
std::string source;
|
||||
boost::regex_constants::syntax_option_type flags;
|
||||
};
|
||||
|
||||
// "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads
|
||||
std::vector<Entry> read(const std::string& table) {
|
||||
Reader reader{ table };
|
||||
std::vector<Entry> entries;
|
||||
|
||||
for (uint16_t count = reader.uint16(); count > 0; --count) {
|
||||
const std::string item = reader.string();
|
||||
const size_t begin = item.find_first_of('/');
|
||||
const size_t end = item.find_last_of('/');
|
||||
if (begin == std::string::npos || end <= begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string options = item.substr(end + 1);
|
||||
const size_t aliasBegin = options.find_first_of(',');
|
||||
|
||||
Entry entry;
|
||||
entry.source = item.substr(begin + 1, end - (begin + 1));
|
||||
entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m;
|
||||
|
||||
for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) {
|
||||
switch (options[i]) {
|
||||
case 'i': entry.flags |= boost::regex_constants::icase; break;
|
||||
case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break;
|
||||
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
|
||||
}
|
||||
}
|
||||
|
||||
entries.push_back(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 4) {
|
||||
fprintf(stderr, "usage: probe <grammars.dat> <corpus> <out>\n");
|
||||
return 2;
|
||||
}
|
||||
|
||||
const std::vector<Entry> entries = read(readAll(argv[1]));
|
||||
|
||||
std::string corpus = readAll(argv[2]);
|
||||
corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end());
|
||||
|
||||
std::ofstream out(argv[3], std::ios::binary);
|
||||
int failed = 0;
|
||||
|
||||
for (size_t i = 0; i < entries.size(); ++i) {
|
||||
boost::regex re;
|
||||
try {
|
||||
re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags);
|
||||
} catch (const std::exception& e) {
|
||||
if (failed++ < 20) {
|
||||
fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out << i;
|
||||
auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re,
|
||||
boost::regex_constants::match_not_dot_newline);
|
||||
for (auto end = boost::sregex_iterator(); it != end; ++it) {
|
||||
out << ' ' << it->position() << ':' << (it->position() + it->length());
|
||||
}
|
||||
out << '\n';
|
||||
}
|
||||
|
||||
printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size());
|
||||
|
||||
if (failed > 0) {
|
||||
fprintf(stderr, "%d patterns Boost will not take.\n", failed);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user