mirror of
https://github.com/TelegramMessenger/libprisma.git
synced 2026-11-18 20:23:45 +00:00
Check every pattern against both regex engines on a pull request
This commit is contained in:
1 parent
d007354e40
commit
2f9ca450d2
5 files changed
+453
No files matched your search
@@ -0,0 +1,163 @@
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
|
||||
* matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
|
||||
* the one thing it promises: that a pattern means the same to every client that reads it.
|
||||
*
|
||||
* The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
|
||||
* anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
|
||||
* looking for.
|
||||
*/
|
||||
public class PatternProbe {
|
||||
|
||||
/** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
|
||||
private static final class Entry {
|
||||
String source;
|
||||
int flags;
|
||||
}
|
||||
|
||||
private static final class Reader {
|
||||
private final byte[] bytes;
|
||||
private int offset;
|
||||
|
||||
Reader(byte[] bytes) {
|
||||
this.bytes = bytes;
|
||||
}
|
||||
|
||||
int uint8() {
|
||||
return bytes[offset++] & 0xFF;
|
||||
}
|
||||
|
||||
int uint16() {
|
||||
return uint8() | (uint8() << 8);
|
||||
}
|
||||
|
||||
String string() {
|
||||
int length = uint8();
|
||||
if (length >= 254) {
|
||||
length = uint8() | (uint8() << 8) | (uint8() << 16);
|
||||
}
|
||||
String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
|
||||
offset += length;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Entry> read(String path) throws IOException {
|
||||
Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
|
||||
List<Entry> entries = new ArrayList<>();
|
||||
|
||||
for (int count = reader.uint16(); count > 0; count--) {
|
||||
String item = reader.string();
|
||||
int begin = item.indexOf('/');
|
||||
int end = item.lastIndexOf('/');
|
||||
if (begin < 0 || end <= begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String options = item.substring(end + 1);
|
||||
int aliasBegin = options.indexOf(',');
|
||||
|
||||
Entry entry = new Entry();
|
||||
entry.source = item.substring(begin + 1, end);
|
||||
for (int i = 0; i < aliasBegin; ++i) {
|
||||
switch (options.charAt(i)) {
|
||||
case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
|
||||
case 'm': entry.flags |= Pattern.MULTILINE; break;
|
||||
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
|
||||
}
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
|
||||
* line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
|
||||
* A Java client has to do this; it cannot go in the table, where \z would read as a z.
|
||||
*/
|
||||
private static String endOfInput(String source) {
|
||||
if (source.indexOf('$') < 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
StringBuilder result = new StringBuilder(source.length());
|
||||
boolean inClass = false;
|
||||
for (int i = 0; i < source.length(); ++i) {
|
||||
char c = source.charAt(i);
|
||||
if (c == '\\' && i + 1 < source.length()) {
|
||||
result.append(c).append(source.charAt(++i));
|
||||
} else if (c == '[') {
|
||||
inClass = true;
|
||||
result.append(c);
|
||||
} else if (c == ']') {
|
||||
inClass = false;
|
||||
result.append(c);
|
||||
} else if (c == '$' && !inClass) {
|
||||
result.append("\\z");
|
||||
} else {
|
||||
result.append(c);
|
||||
}
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
List<Entry> entries = read(args[0]);
|
||||
String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
|
||||
.replace(String.valueOf((char) 13), "");
|
||||
|
||||
StringBuilder out = new StringBuilder();
|
||||
int failed = 0;
|
||||
|
||||
for (int i = 0; i < entries.size(); ++i) {
|
||||
Entry entry = entries.get(i);
|
||||
String source = entry.source;
|
||||
if ((entry.flags & Pattern.MULTILINE) == 0) {
|
||||
source = endOfInput(source);
|
||||
}
|
||||
|
||||
Pattern pattern;
|
||||
try {
|
||||
pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
|
||||
} catch (PatternSyntaxException e) {
|
||||
if (failed++ < 20) {
|
||||
System.err.println("pattern " + i + ": " + e.getDescription());
|
||||
System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length())));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out.append(i);
|
||||
java.util.regex.Matcher matcher = pattern.matcher(corpus);
|
||||
int at = 0;
|
||||
while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
|
||||
out.append(' ').append(matcher.start()).append(':').append(matcher.end());
|
||||
at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
|
||||
if (at > corpus.length()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.append('\n');
|
||||
}
|
||||
|
||||
Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
|
||||
System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
|
||||
|
||||
if (failed > 0) {
|
||||
System.err.println(failed + " patterns java.util.regex will not take. "
|
||||
+ "Teach normalizeEscapes in generate.js to rewrite them.");
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user