Merge remote-tracking branch 'upstream/master' into HEAD

This commit is contained in:
John Preston committed 2026-09-29 16:30:43 +04:00
commit ca59e6434e
25 files changed
+1599 -49

No files matched your search

+1
View File
@@ -62,3 +62,4 @@
#*.rtf diff=astextplain
#*.RTF diff=astextplain
libprisma/grammars.json eol=lf
libprisma/grammars.dat linguist-generated
+19 -9
View File
@@ -1,27 +1,37 @@
name: Update grammars
on: [ workflow_dispatch ]
on:
workflow_dispatch:
push:
branches: [ master ]
# grammars.dat itself is deliberately not listed: regenerating it must not trigger another run.
paths:
- components/**
- generate.js
- package.json
- package-lock.json
permissions:
contents: write
pull-requests: write
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18.x]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
- uses: actions/checkout@v5
- uses: actions/setup-node@v5
with:
node-version: 18
node-version: 22
- run: npm install
- run: npm start
- name: Upload a Build Artifact
uses: actions/upload-artifact@v3.1.3
uses: actions/upload-artifact@v7
with:
name: grammars.dat
path: ./libprisma/grammars.dat
- name: Create Pull Request
uses: peter-evans/create-pull-request@v5
uses: peter-evans/create-pull-request@v8
with:
commit-message: Update grammars
title: Update grammars
+22
View File
@@ -0,0 +1,22 @@
name: Check generated files
# The path filter is the whole test: the job only exists when the pull request
# touches grammars.dat, so reaching it is already the failure.
on:
pull_request:
paths:
- libprisma/grammars.dat
permissions: {}
jobs:
grammars:
runs-on: ubuntu-latest
# A pull request opened with GITHUB_TOKEN cannot trigger this workflow at all,
# so the guard only matters if "Update grammars" is ever moved to a PAT.
if: github.event.pull_request.user.login != 'github-actions[bot]'
steps:
- name: Reject changes to the generated grammar table
run: |
echo "::error file=libprisma/grammars.dat::libprisma/grammars.dat is generated by generate.js and is regenerated on master by the Update grammars workflow. Please drop it from this pull request and keep only the components/ and generate.js changes."
exit 1
+58
View File
@@ -0,0 +1,58 @@
name: Check grammars
# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex,
# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it
# to that, rather than waiting for a language to quietly stop highlighting.
on:
pull_request:
paths:
- components/**
- generate.js
- libprisma/**
- tools/**
- package.json
- package-lock.json
workflow_dispatch:
permissions: {}
jobs:
patterns:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
with:
persist-credentials: false
- uses: actions/setup-node@v5
with:
node-version: 22
# the committed table is regenerated on master, so a pull request is checked against what
# its own components and generate.js produce
- run: npm ci --ignore-scripts
- run: npm start
- name: Escapes every engine agrees on
run: node tools/check-escapes.js libprisma/grammars.dat
- name: Compile every pattern with java.util.regex
run: |
javac -d tools tools/PatternProbe.java
java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt
- name: Compile every pattern with Boost
run: |
sudo apt-get update
sudo apt-get install -y libboost-regex-dev
g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
- name: Compare what the two engines matched
run: |
if ! diff -u boost.txt java.txt > engines.diff; then
echo "The two engines disagree about what these patterns match."
echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt."
head -40 engines.diff
exit 1
fi
echo "Boost and java.util.regex agree on every pattern."
+5
View File
@@ -129,10 +129,12 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`F#`|`fsharp`|
|`Factor`|`factor`|
|`False`|`false`|
|`Fift`|`fift`|
|`Firestore security rules`|`firestore-security-rules`|
|`Flow`|`flow`|
|`Fortran`|`fortran`|
|`FreeMarker Template Language`|`ftl`|
|`FunC`|`func`|
|`GameMaker Language`|`gml`,`gamemakerlanguage`|
|`GAP (CAS)`|`gap`|
|`G-code`|`gcode`|
@@ -290,6 +292,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`SuperCollider`|`supercollider`,`sclang`|
|`Swift`|`swift`|
|`Systemd configuration file`|`systemd`|
|`Tact`|`tact`|
|`T4 templating`|`t4-templating`|
|`T4 Text Templates (C#)`|`t4-cs`,`t4`|
|`VB.Net`|`vbnet`|
@@ -299,6 +302,8 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`Template Toolkit 2`|`tt2`|
|`TOML`|`toml`|
|`Tremor`|`tremor`,`tremor`,`trickle`,`troy`|
|`Type Language`|`tl`|
|`Type Language - Binary`|`tlb`|
|`TypoScript`|`typoscript`,`tsconfig`|
|`UnrealScript`|`unrealscript`,`unrealscript`,`uscript`,`uc`|
|`UO Razor Script`|`uorazor`|
+127
View File
@@ -0,0 +1,127 @@
/**
* @file Prism.js definition for Fift
* @link https://docs.ton.org/develop/fift/overview
* @author Nikita Sobolev (https://github.com/sobolevn)
* @license MIT
*/
Prism.languages.fift = {
'symbol': [
/[xX]\{[0-9a-fA-F_]*\}/,
/[bB]\{[01]*\}/,
/'\S+/,
],
'string': /"([^"\r\n\\]|\\.)*"/,
'comment': [
{
pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
lookbehind: true,
greedy: true,
},
{
pattern: /\/\/.*/,
lookbehind: true,
greedy: true,
},
],
'operator': [
// Full list can be found in
// Appendix A. List of Fift words
// Ordered the same way source code does this.
// Except: shorter words must follow
// longer ones with the same base part.
// Example: `#` comes after `#s`
/#>/, /#s/, /\$#/, /#/,
/\$\+/, /\$,/, /\$\d/, /\$=/, /\$(?=\()/,
/\$>smca/, /\$>s/,
/\$@\+/, /\$@\?\+/, /\$@\?/, /\$@/,
/\$cmp/, /\$len/, /\$pos/, /\$reverse/,
/%1<</,
/\('\)/, /\(-trailing\)/, /\(\.\)/, /\(atom\)/, /\(b\.\)/,
/\(compile\)/, /\(create\)/, /\(def\?\)/, /\(dump\)/, /\(execute\)/,
/\(forget\)/, /\(number\)/, /\(x\.\)/, /\(\{\)/, /\(\}\)/,
/\*\/cmod/, /\*\/c/, /\*\/mod/, /\*\/rmod/, /\*\/r/, /\*\//,
/\*>>c/, /\*>>r/, /\*>>/, /\*mod/, /\*/,
/\+!/, /\+/, /,/, /-!/, /-/, /!/,
/-1<</, /\._/, /\.dump/, /\.l/, /\.sl/, /\.s/, /\.tc/,
/\//, /\/\*/, /\/cmod/, /\/c/, /\/mod/, /\/rmod/, /\/r/,
/0!/, /0<=/, /0<>/, /0</, /0=/, /0>=/, /0>/,
/1\+!/, /1\+/, /1-!/, /1-/, /1<</, /1<<1-/,
/2\*/, /2\+/, /2-/, /2\//, /2=:/,
/::_/, /::/, /:_/, /=:/, /:/,
/<#/, /<<\/c/, /<<\/r/, /<=/, /<>/, /<b/, /<s/, /<<\//,
/<</, /</, /=/,
/>=/, />>c/, />>r/, />>/, />/,
/\?dup/,
/@'/, /@/,
/B\+/, /B,/, /B=/, /B>Li@\+/, /B>Li@/, /B>Lu@\+/, /B>Lu@/, /B>boc/,
/B>file/, /B>i@\+/, /B>i@/, /B>u@\+/, /B>u@/,
/B@\?\+/, /B@\+/, /B@\?/, /B@/,
/Bcmp/, /BhashB/, /Bhashu/, /Bhash/, /Blen/, /Bx\./,
/B\|/, /Li>B/, /Lu>B/,
/\[\]/, /\[compile\]/, /\[/, /\]/,
/atom\?/,
/b\+/, /b\._/, /b\./,
/b>idict!\+/, /b>idict!/, /b>sdict!\+/, /b>sdict!/,
/b>udict!\+/, /b>udict!/,
/b>/, /boc\+>B/, /boc>B/,
/csr\./,
/def\?/,
/empty\?/, /eq\?/,
/file-exists\?/, /file>B/,
/i,/, /i>B/, /i@\+/, /i@\?\+/, /i@\?/, /i@/,
/idict!\+/, /idict!/, /idict-/, /idict@-/, /idict@/,
/null!/, /null\?/,
/pfxdict!\+/, /pfxdict!/, /pfxdict@/, /priv>pub/,
/ref@\+/, /ref@/, /ref@\?\+/, /ref@\?/,
/s,/, /s>c/, /s>/,
/sdict!\+/, /sdict!/, /sdict-/, /sdict@-/, /sdict@/,
/smca>\$/, /sr,/,
/tuple\?/,
/u,/, /u>B/, /u@\+/, /u@\?\+/, /u@\?/,
/udict!\+/, /udict!/, /udict-/, /udict@-/, /udict@/,
/undef\?/,
/x\._/, /x\./,
/\|\+/, /\|/, /\|_/,
/\?\./, /'/,
// Should be the last:
/\./,
],
'keyword': /\b(?:-roll|-rot|-trailing|-trailing0|2constant|2drop|2dup|2over|2swap|abort|abs|allot|and|anon|atom|bbitrefs|bbits|bl|box|brefs|brembitrefs|brembits|bremrefs|bye|caddr|cadr|car|cddr|cdr|char|chr|cmp|cond|cons|constant|count|cr|create|depth|dictmap|dictmerge|dictnew|does|drop|dup|ed25519_chksign|ed25519_sign|ed25519_sign_uint|emit|exch|exch2|execute|explode|find|first|fits|forget|gasrunvm|gasrunvmcode|gasrunvmctx|gasrunvmdict|halt|hash|hashB|hashu|hold|hole|if|ifnot|include|list|max|min|minmax|mod|negate|newkeypair|nil|nip|nop|not|now|null|or|over|pair|pick|quit|remaining|reverse|roll|rot|runvm|runvmcode|runvmctx|runvmdict|sbitrefs|sbits|second|sgn|shash|sign|single|skipspc|space|srefs|swap|ten|third|times|triple|tuck|tuple|type|ufits|uncons|unpair|unsingle|until|untriple|untuple|variable|while|word|words|xor)\b/,
'boolean': /\b(?:false|true)\b/,
'number': [
/(0[xX][0-9a-fA-F]+)/,
/(0[bB][01]+)/,
/(-?\d+(\/-?\d+)?)/,
],
'variable': /[\w$-]+/,
'punctuation': /[\[\{\}\],\(\)]/,
};
+148
View File
@@ -0,0 +1,148 @@
/**
* @file Prism.js definition for FunC
* @link https://docs.ton.org/develop/func/overview
* @version 0.4.4
* @author Novus Nota (https://github.com/novusnota)
* @author Nikita Sobolev (https://github.com/sobolevn)
* @license MIT
*/
(function(Prism) {
/** @type {Prism.GrammarValue} */
var number = /-?\b(?:\d+|0x[\da-fA-F]+)\b(?![^\.~,;\[\]\(\)\s])/;
/** @type {Prism.GrammarValue} */
var string = [
{ // multi-line
pattern: /"""[\s\S]*?"""[Hhcusa]?/,
greedy: true,
inside: {
// string type
'symbol': {
pattern: /("""[\s\S]*?""")[Hhcusa]/,
lookbehind: true,
greedy: true,
}
},
},
{ // single-line
pattern: /"[^\n"]*"[Hhcusa]?/,
greedy: true,
inside: {
// string type
'symbol': {
pattern: /("[^\n"]*")[Hhcusa]/,
lookbehind: true,
greedy: true,
}
},
},
];
/** @type {RegExp} */
var operator_body = /(?:!=|\?|:|%=|%|&=|&|\*=|\*|\+=|\+|->|-=|-|\/%|\/=|\/|<=>|<<=|<<|<=|<|==|=|>>=|>>|>=|>|\^>>=|\^>>|\^=|\^\/=|\^\/|\^%=|\^%|\^|\|=|\||~>>=|~>>|~\/=|~\/|~%|~)/;
// An operator only counts as one when whitespace follows it, as identifiers may contain these characters.
/** @type {Prism.GrammarValue} */
var operator = new RegExp(operator_body.source + /(?=\s)/.source);
/** @type {RegExp[]} */
var var_identifier = [
// quoted
/`[^`\n]+`/,
// plain
/[^\.~,;\[\]\(\)\s]+/
];
/** Prism.js definition for FunC */
Prism.languages.func = {
'comment': [
{ // single-line
pattern: /;;.*/,
lookbehind: true,
greedy: true,
},
{ // multi-line, not nested (TODO: nesting)
// . isn't used, because it only applies to the single line
pattern: /\{-[\s\S]*?(?:-\}|$)/,
lookbehind: true,
greedy: true,
},
{
// unused variable identifier
pattern: /\b_\b(?![^\.~,;\[\]\(\)\s])/
},
],
// Custom token for #pragma's
'pragma': {
pattern: /#pragma(.*);/,
inside: {
'keyword': /(?:#pragma|test-version-set|not-version|version|allow-post-modification|compute-asm-ltr)\b/,
'number': /(?:\d+)(?:.\d+)?(?:.\d+)?/,
'operator': /=|\^|<=|>=|<|>/,
'string': string,
'punctuation': /;/,
}
},
// Custom token for #include's
'include': {
pattern: /#include(.*);/,
inside: {
'keyword': /#include\b/,
'string': string,
'punctuation': /;/,
}
},
// builtin types
'builtin': /\b(?:int|cell|slice|builder|cont|tuple|type)\b/,
// builtin constants
'boolean': /\b(?:false|true|nil|Nil)\b/,
'keyword': /\b(?:forall|extern|global|asm|impure|inline_ref|inline|auto_apply|method_id|operator|infixl|infixr|infix|const|return|var|repeat|do|while|until|try|catch|ifnot|if|then|elseifnot|elseif|else)\b/,
'number': number,
'string': string,
'operator': operator,
'punctuation': [/[\.,;\(\)\[\]]/, /[\{\}](?![^\.~,;\[\]\(\)\s])/],
// Function and method names in declarations, definitions and calls
'function': [
{ // quoted
pattern: new RegExp(var_identifier[0].source + /(?=\s*\()/.source),
greedy: true,
},
{ // bitwise not operator
pattern: /~_/,
greedy: true,
},
{ // remaining operators
// operator_body, not operator: its trailing lookahead would forbid the closing _
pattern: new RegExp(/\^?_/.source + operator_body.source + /_/.source),
greedy: true,
},
{ // plain function or method name
pattern: new RegExp(/[\.~]?/.source + var_identifier[1].source + /(?=\s*\()/.source),
greedy: true,
},
{ // builtin functions and methods
pattern: /\b(?:divmod|moddiv|muldiv|muldivr|muldivc|muldivmod|null\?|throw|throw_if|throw_unless|throw_arg|throw_arg_if|throw_arg_unless|load_int|load_uint|preload_int|preload_uint|store_int|store_uint|load_bits|preload_bits|int_at|cell_at|slice_at|tuple_at|at|touch|touch2|run_method0|run_method1|run_method2|run_method3|~divmod|~moddiv|~store_int|~store_uint|~touch|~touch2|~dump|~stdump)\b/,
greedy: true,
},
],
// Parametric polymorphism
'class-name': /[A-Z][^\.~,;\[\]\(\)\s]*/,
// All the rest of identifiers
'variable': var_identifier.map(x => { return { pattern: x, greedy: true } }),
};
// Adding a fc alias
Prism.languages.fc = Prism.languages.func;
}(Prism));
+171
View File
@@ -0,0 +1,171 @@
/**
* @file Prism.js definition for Tact
* @link https://tact-lang.org
* @version 1.6.0
* @author Novus Nota (https://github.com/novusnota)
* @license MIT
*/
(function(Prism) {
Prism.languages.tact = {
// reserved keywords
'keyword': [
{
pattern: /\b(?:abstract|asm|as|catch|codeOf|const|contract(?!:)|do|else|extend|extends|foreach|fun|get|if|in|import|initOf|inline|let|message(?!:)|mutates|native|override|primitive|public|repeat|return|self|struct(?!:)|trait(?!:)|try|until|virtual|while|with)\b/,
},
{ // reserved function names
pattern: /\b(?:bounced|external|init|receive)\b(?=\()/
},
],
// built-in types
'builtin': [
{
pattern: /\b(?:Address|Bool|Builder|Cell|Int|Slice|String|StringBuilder)\b/,
},
{ // keyword after as, see: https://prismjs.com/extending.html#object-notation
pattern: /(\bas\s+)(?:coins|varu?int(?:32|16)|remaining|bytes(?:32|64)|int257|u?int(?:25[0-6]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?))\b/,
lookbehind: true,
greedy: true,
},
],
// SCREAMING_SNAKE_CASE for null values and names of constants
'constant': [
{
pattern: /\bnull\b/,
},
{
pattern: /\b[A-Z][A-Z0-9_]*\b/,
},
],
// UpperCamelCase for names of contracts, traits, structs, messages
'class-name': {
pattern: /\b[A-Z]\w*\b/,
},
// mappings to FunC
'attribute': [
{ // functions
pattern: /@name/,
inside: {
'function': /.+/,
},
},
{ // contract interfaces
pattern: /@interface/,
inside: {
'function': /.+/,
}
}
],
'function': {
pattern: /\b\w+(?=\()/,
},
'boolean': {
pattern: /\b(?:false|true)\b/,
},
'number': [
{ // hexadecimal, case-insensitive /i
pattern: /\b0x[0-9a-f](?:_?[0-9a-f])*\b/i,
},
{ // octal, case-insensitive /i
pattern: /\b0o[0-7](?:_?[0-7])*\b/i,
},
{ // binary, case-insensitive /i
pattern: /\b0b[01](?:_?[01])*\b/i,
},
{ // decimal integers, starting with 0
pattern: /\b0\d*\b/,
},
{ // other decimal integers
pattern: /\b[1-9](?:_?\d)*\b/,
},
],
'string': undefined,
'punctuation': {
pattern: /[{}[\]();,.:?]/,
},
'comment': [
{ // single-line
pattern: /(^|[^\\:])\/\/.*/,
lookbehind: true,
greedy: true,
},
{ // multi-line
pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
lookbehind: true,
greedy: true,
},
{
// unused variable identifier
pattern: /\b_\b/
}
],
'operator': {
'pattern': /![!=]?|->|[+\-*/%=]=?|[<>]=|<<?=?|>>?=?|~|\|\|=|\|[\|=]?|&&=|&[&=]?|\^=?/,
},
'variable': {
'pattern': /\b[a-zA-Z_]\w*\b/,
},
};
// strings, made this way to not collide with other entities
Prism.languages.insertBefore('tact', 'string', {
'string-literal': {
pattern: /(?:(")(?:\\.|(?!\1)[^\\\r\n])*\1(?!\1))/,
greedy: true,
inside: {
'regex': [
{ // \\ \" \n \r \t \v \b \f
pattern: /\\[\\"nrtvbf]/,
},
{ // hexEscape, \x00 through \xFF
pattern: /\\x[0-9a-fA-F]{2}/,
},
{ // unicodeEscape, \u0000 through \uFFFF
pattern: /\\u[0-9a-fA-F]{4}/,
},
{ // unicodeCodePoint, \u{0} through \u{FFFFFF}
pattern: /\\u\{[0-9a-fA-F]{1,6}\}/,
},
],
'string': {
pattern: /[\s\S]+/,
},
},
},
});
// map and bounced message generic type modifiers
Prism.languages.insertBefore('tact', 'keyword', {
'generics': {
pattern: /(?:\b(?:bounced|map)\b<[^\\\r\n]*>)/,
greedy: true,
inside: {
'builtin': [
...Prism.languages['tact']['builtin'],
{
pattern: /\b(?:bounced(?=<)|map(?=<))\b/
},
],
'class-name': Prism.languages['tact']['class-name'],
'punctuation': {
pattern: /[<>(),.?]/,
},
'keyword': {
pattern: /\bas\b/,
},
},
},
});
}(Prism));
+44
View File
@@ -0,0 +1,44 @@
/**
* @file Prism.js definition for TL-B
* @link https://docs.ton.org/develop/data-formats/tl-b-language
* @author Nikita Sobolev (https://github.com/sobolevn)
* @license MIT
*/
Prism.languages.tlb = {
'builtin': /\b(?:Bool|Both|Cell|Either|Maybe|Type|Unit|bits256|bits512|int16|int32|int64|int8|uint15|uint16|uint32|uint63|uint64|uint8)\b/,
'keyword': /\b(?:BoolFalse|BoolTrue|False|Null|True)\b/,
'comment': {
pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,
greedy: true
},
'comment-multiline': [
{
pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
lookbehind: true,
greedy: true,
alias: 'comment'
},
{
pattern: /(^|[^\\:])\/\/.*/,
lookbehind: true,
greedy: true,
alias: 'comment'
}
],
'symbol': [
/#[0-9a-f]*_?/,
/\$[01]*_?/,
/\b(?:##|#<|#<=)\b/,
],
'variable': /[a-zA-Z_]\w*/,
'operator': [
/\+/, /-/, /\*/, /\//,
/!=/, /==/, /=/,
/\?/, /~/, /\./, /\^/,
/<=/, />=/, /</, />/,
],
'number': /\d+/,
'punctuation': /[;\(\):\[\]\{\}]/,
};
+71
View File
@@ -0,0 +1,71 @@
/**
* @file Prism.js definition for Tolk
* @link https://docs.ton.org/tolk/overview
* @version 1.4.0
* @author Danil Ovchinnikov (https://github.com/Danil42Russia)
* @license MIT
*/
(function(Prism) {
Prism.languages.tolk = {
'comment': {
pattern: /\/\/.*/,
greedy: true,
},
'block-comment': {
pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
greedy: true,
},
'string': [
{
pattern: /"""(?:\\[\s\S]|(?!""")[\s\S])*(?:"""|$)/,
greedy: true,
},
{
pattern: /"(?:\\.|[^"\\\r\n])*(?:"|$)/,
greedy: true,
},
],
'attr-name': /@[a-zA-Z0-9_.]+/,
'number': [
/\b0x[0-9A-Fa-f_]+\b/,
/\b0b[01_]+\b/,
/\b\d[\d_]*\b/,
],
'boolean': /\b(?:false|null|true)\b/,
'constant': /\b[A-Z][A-Z0-9_]*_[A-Z0-9_]*\b/,
'keyword': [
/!is\b/,
/\b(?:as|asm|assert|break|builtin|catch|const|continue|contract|do|else|enum|fun|get|global|if|import|is|lazy|match|mutate|private|readonly|repeat|return|self|struct|throw|tolk|try|type|val|var|while)\b/,
],
'function': [
/`[^`\r\n]+`(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
/\b[a-zA-Z_$][a-zA-Z0-9_$]*\b(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
],
'builtin': [
/\b(?:Cell|address|any_address|array|blockchain|bool|builder|cell|coins|continuation|contract|debug|dict|lisp_list|map|never|random|reflect|slice|string|tuple|unknown|void)\b/,
/\b(?:int\d*|uint\d+|varint\d+|varuint\d+|bits\d+|bytes\d+)\b/,
],
'class-name': /\b[A-Z][a-zA-Z0-9_$]*\b/,
'property': {
pattern: /(\.)(?:`[^`\r\n]+`|[a-zA-Z_$][a-zA-Z0-9_$]*)/,
lookbehind: true,
},
'operator': /<<=|>>=|<=>|~>>|\^>>|==|!=|<=|>=|<<|>>|&&|\|\||~\/|\^\/|\+=|-=|\*=|\/=|%=|&=|\|=|\^=|->|=>|\+\+|--|\?\?|[+\-*\/%?=<>!&|^~]/,
'punctuation': /[.,;:(){}\[\]]/,
'symbol': /`[^`\r\n]+`/,
};
}(Prism));
+100 -21
View File
@@ -1,5 +1,9 @@
const fs = require('fs')
const isEqual = require('lodash.isequal')
const Prism = require('prismjs')
const components = require('prismjs/components.js')
global.Prism = Prism
const SCRIPTS = {}
const include = function (src) {
@@ -7,12 +11,6 @@ const include = function (src) {
(1, eval)(src.toString())
}
async function loadScript(src) {
const script = await fetch(src)
const text = await script.text()
include(text)
}
async function loadLanguages(lngs) {
if (lngs) {
lngs = Array.isArray(lngs) ? lngs : [lngs];
@@ -39,7 +37,7 @@ async function loadLanguage(lng) {
console.log(`${langNumber} | Loading ${lng}`);
// TODO: version should probably not be hardcoded
await loadScript(`https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-${lng}.min.js`)
require(`prismjs/components/prism-${lng}.js`)
}
}
@@ -52,6 +50,81 @@ function loadLocalLanguage(path, code, title, alias) {
}
}
// Spell out the constructs whose meaning differs between regex engines, so one table can
// serve all of them. java.util.regex reads a [ inside a character class as the start of a
// nested class, requires a { outside one to open a quantifier, reads \0 as the start of an
// octal escape, and reads \v as the whole vertical whitespace class rather than as U+000B.
// All four rewrites are no-ops for ECMAScript and for Boost, which is what libprisma
// compiles the patterns with.
const QUANTIFIER = /^\{\d+(?:,\d*)?\}/
function normalizeEscapes(pattern) {
let result = ''
let inClass = false
for (let i = 0; i < pattern.length; i++) {
const c = pattern[i]
if (c === '\\') {
if (pattern[i + 1] === '0' && !/[0-7]/.test(pattern[i + 2] || '')) {
result += '\\x00'
} else if (pattern[i + 1] === 'v') {
result += '\\x0B'
} else {
result += pattern.substr(i, 2)
}
i++
continue
}
if (inClass) {
if (c === '[') {
result += '\\['
continue
}
// && is set intersection in java.util.regex
if (c === '&' && pattern[i + 1] === '&') {
result += '\\&\\&'
i++
continue
}
if (c === ']') {
inClass = false
}
} else if (c === '[') {
inClass = true
} else if (c === '{' && !QUANTIFIER.test(pattern.substr(i))) {
result += '\\{'
continue
}
result += c
}
return result
}
// A character written literally in a grammar - BQN's •, APL's ⍵ - has to reach the readers as
// an escape rather than as itself: they disagree about what the bytes of a pattern mean, and
// \uXXXX is the one spelling all of them read as a code point. An astral character is two code
// units here, so it becomes a surrogate pair, which UnicodeEscapes.h puts back together.
function escapeNonAscii(pattern) {
if (!/[^\x00-\x7F]/.test(pattern)) {
return pattern
}
let result = ''
for (let i = 0; i < pattern.length; i++) {
const code = pattern.charCodeAt(i)
result += code > 0x7F
? '\\u' + code.toString(16).padStart(4, '0')
: pattern[i]
}
return result
}
function unique(a, fn) {
if (a.length === 0 || a.length === 1) {
return a;
@@ -137,7 +210,7 @@ async function generate() {
pattern = pattern.replaceAll("|[])", ")");
pattern = pattern.replaceAll(":[]", ":");
return pattern
return escapeNonAscii(normalizeEscapes(pattern))
}
for (var token in copy) {
@@ -215,14 +288,17 @@ async function generate() {
"sparql" // requires turtle
]
await loadScript("https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-core.min.js")
await loadScript("https://prismjs.com/components.js")
await loadLanguages(Object.keys(components.languages))
console.log(`\nLoaded all ${langNumber} languages`)
console.log("Processing...")
// Manually add local definitions
loadLocalLanguage('./components/prism-tl.js', 'typelanguage', 'TypeLanguage', 'tl')
loadLocalLanguage('./components/prism-tlb.js', 'tlb', 'TypeLanguage-Binary', 'tlb')
loadLocalLanguage('./components/prism-fift.js', 'fift', 'Fift', 'fift')
loadLocalLanguage('./components/prism-func.js', 'func', 'FunC', ['func', 'fc'])
loadLocalLanguage('./components/prism-tact.js', 'tact', 'Tact', 'tact')
loadLocalLanguage('./components/prism-tolk.js', 'tolk', 'Tolk', 'tolk')
Object.keys(Prism.languages).forEach(lng => {
if (unsupported.includes(lng) || !components.languages[lng]) {
@@ -338,11 +414,11 @@ async function generate() {
for (const lng of alias) {
allLanguages[lng] = allGrammars.indexOf(find)
if (components.languages[name].aliasTitles) {
languageNames[lng] = components.languages[name].aliasTitles[lng]
} else {
languageNames[lng] = components.languages[name].title
}
// aliasTitles only names the aliases whose title differs from the language's,
// so an alias missing from it takes the language title rather than no title:
// an empty one drops the alias out of SyntaxHighlighter::languages().
const aliasTitles = components.languages[name].aliasTitles
languageNames[lng] = (aliasTitles && aliasTitles[lng]) || components.languages[name].title
}
}
})
@@ -358,15 +434,18 @@ async function generate() {
const writeUint16 = i => chunks.push(new Uint16Array([i]))
const writeUint8 = i => chunks.push(new Uint8Array([i]))
const writeString = str => {
if (str.length < 253) {
writeUint8(str.length)
// one byte per code unit truncated anything above U+00FF; the length is the byte
// count, which is what the readers advance by
const bytes = new TextEncoder().encode(str)
if (bytes.length < 253) {
writeUint8(bytes.length)
} else {
writeUint8(254 & 0xFF)
writeUint8(str.length & 0xFF)
writeUint8((str.length >> 8) & 0xFF)
writeUint8((str.length >> 16) & 0xFF)
writeUint8(bytes.length & 0xFF)
writeUint8((bytes.length >> 8) & 0xFF)
writeUint8((bytes.length >> 16) & 0xFF)
}
chunks.push(new Uint8Array(str.split('').map(char => char.charCodeAt(0))))
chunks.push(bytes)
}
// Patterns
+5
View File
@@ -31,6 +31,11 @@ const Pattern* PatternPtr::get() const
return m_tree->resolvePattern(m_path);
}
std::shared_ptr<Pattern> PatternRaw::realize()
{
return std::make_shared<Pattern>(m_regex, m_flags, m_lookbehind, m_greedy, m_alias, m_inside);
}
const Grammar* Pattern::inside() const
{
if (m_inside)
+26 -8
View File
@@ -29,17 +29,35 @@ private:
size_t m_path;
};
class PatternRaw
{
public:
PatternRaw(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr<GrammarPtr> inside)
: m_regex(std::string{pattern})
, m_flags(flags)
, m_lookbehind(lookbehind)
, m_greedy(greedy)
, m_alias(alias)
, m_inside(inside)
{
}
std::shared_ptr<Pattern> realize();
private:
std::string m_regex;
boost::regex_constants::syntax_option_type m_flags;
bool m_lookbehind;
bool m_greedy;
std::string m_alias;
std::shared_ptr<GrammarPtr> m_inside;
};
class Pattern
{
public:
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, GrammarPtr inside)
: Pattern(pattern, flags, lookbehind, greedy, alias, std::make_shared<GrammarPtr>(inside))
{
}
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind = false, bool greedy = false, std::string alias = "", std::shared_ptr<GrammarPtr> inside = nullptr)
: m_regex(boost::regex(std::string{pattern}, flags | boost::regex_constants::optimize))
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr<GrammarPtr> inside)
: m_regex(boost::regex(std::string{ pattern }, flags | boost::regex_constants::optimize))
, m_lookbehind(lookbehind)
, m_greedy(greedy)
, m_alias(alias)
+15 -3
View File
@@ -1,6 +1,7 @@
#include "LanguageTree.h"
#include "TokenList.h"
#include "UnicodeEscapes.h"
#include <boost/regex.hpp>
void LanguageTree::load(const std::string& content)
@@ -96,6 +97,8 @@ void LanguageTree::parsePatterns(Buffer &buffer)
{
uint16_t count = freadUint16(buffer);
m_patterns.resize(count);
for (int i = 0; i < count; ++i)
{
std::string item = freadString(buffer);
@@ -163,11 +166,11 @@ void LanguageTree::parsePatterns(Buffer &buffer)
if (inside != std::string::npos)
{
m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }, std::make_shared<GrammarPtr>(shared_from_this(), inside)));
m_patternsRaw.push_back(std::make_shared<PatternRaw>(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, std::make_shared<GrammarPtr>(shared_from_this(), inside)));
}
else
{
m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }));
m_patternsRaw.push_back(std::make_shared<PatternRaw>(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, nullptr));
}
}
}
@@ -175,7 +178,16 @@ void LanguageTree::parsePatterns(Buffer &buffer)
const Pattern* LanguageTree::resolvePattern(size_t path)
{
return m_patterns[path].get();
std::shared_ptr<Pattern> &parsed = m_patterns[path];
if (parsed == nullptr)
{
parsed = m_patternsRaw[path]->realize();
m_patternsRaw[path] = nullptr;
m_patterns[path] = parsed;
}
return parsed.get();
}
const Grammar* LanguageTree::resolveGrammar(size_t path)
+12
View File
@@ -38,6 +38,17 @@ public:
return keys;
}
std::string languageName(const std::string& language) const
{
const auto& find = m_languages.find(language);
if (find != m_languages.end())
{
return find->second.first;
}
return language;
}
const Grammar* find(const std::string& key) const
{
const auto& value = m_languages.find(key);
@@ -57,4 +68,5 @@ private:
std::map<std::string, std::pair<std::string, size_t>> m_languages;
std::vector<std::shared_ptr<Grammar>> m_grammars;
std::vector<std::shared_ptr<Pattern>> m_patterns;
std::vector<std::shared_ptr<PatternRaw>> m_patternsRaw;
};
+11 -6
View File
@@ -24,6 +24,11 @@ std::map<std::string, std::string> SyntaxHighlighter::languages() const
return m_tree->keys();
}
std::string SyntaxHighlighter::languageName(const std::string& language) const
{
return m_tree->languageName(language);
}
TokenList SyntaxHighlighter::tokenize(std::string_view text, const Grammar* grammar)
{
// Grammars may include themselves through "inside" (for example
@@ -62,9 +67,9 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
return;
}
const auto& pattern = *j;
const auto& inside = pattern->inside();
const bool greedy = pattern->greedy();
const auto& pattern = *j->get();
const auto& inside = pattern.inside();
const bool greedy = pattern.greedy();
size_t pos = startPos;
@@ -99,7 +104,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
if (greedy)
{
match = pattern->match(matchSuccess, matchIndex, text);
match = pattern.match(matchSuccess, matchIndex, text);
if (!matchSuccess || matchIndex >= text.length())
{
break;
@@ -143,7 +148,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
else
{
matchIndex = 0;
match = pattern->match(matchSuccess, matchIndex, str);
match = pattern.match(matchSuccess, matchIndex, str);
if (!matchSuccess)
{
continue;
@@ -183,7 +188,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
currentNode = tokenList.addAfter(removeFrom, token.name(),
std::move(tokenEntries),
pattern->alias(),
pattern.alias(),
match.size());
if (after.size())
+2
View File
@@ -25,6 +25,8 @@ public:
std::map<std::string, std::string> languages() const;
std::string languageName(const std::string& language) const;
private:
TokenList tokenize(std::string_view text, const Grammar* grammar);
void matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch);
+354
View File
@@ -0,0 +1,354 @@
#pragma once
#include <string>
#include <string_view>
#include <vector>
namespace libprisma
{
// Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one
// simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and
// where one sits inside a character class - whose members have to be a single byte each -
// lift it out into an alternation. This belongs here rather than in grammars.dat, which
// keeps the \u spelling for the clients that match over UTF-16 and read it correctly.
inline bool isHexDigit(char c)
{
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
inline int hexValue(char c)
{
return c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10;
}
// the code point of the \uXXXX escape at begin, or -1 if there is not one there
inline int unicodeEscapeAt(std::string_view pattern, size_t begin)
{
if (begin + 6 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'u')
{
return -1;
}
int value = 0;
for (size_t i = begin + 2; i < begin + 6; ++i)
{
if (!isHexDigit(pattern[i]))
{
return -1;
}
value = value * 16 + hexValue(pattern[i]);
}
return value;
}
inline void appendByte(std::string& out, unsigned char byte)
{
static const char digits[] = "0123456789ABCDEF";
out += "\\x";
out += digits[byte >> 4];
out += digits[byte & 0x0F];
}
inline void appendUtf8(std::string& out, int codePoint)
{
if (codePoint < 0x80)
{
appendByte(out, static_cast<unsigned char>(codePoint));
}
else if (codePoint < 0x800)
{
appendByte(out, static_cast<unsigned char>(0xC0 | (codePoint >> 6)));
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
}
else if (codePoint < 0x10000)
{
appendByte(out, static_cast<unsigned char>(0xE0 | (codePoint >> 12)));
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 6) & 0x3F)));
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
}
else
{
appendByte(out, static_cast<unsigned char>(0xF0 | (codePoint >> 18)));
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 12) & 0x3F)));
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 6) & 0x3F)));
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
}
}
// The code point at begin, joining a surrogate pair into the one character it spells, with
// the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it
// to UTF-8 itself; a pair it would expand to two three byte sequences instead.
inline int codePointAt(std::string_view pattern, size_t begin, size_t& length)
{
const int value = unicodeEscapeAt(pattern, begin);
length = value < 0 ? 0 : 6;
if (value >= 0xD800 && value <= 0xDBFF)
{
const int low = unicodeEscapeAt(pattern, begin + 6);
if (low >= 0xDC00 && low <= 0xDFFF)
{
length = 12;
return 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00);
}
}
return value;
}
struct ClassMember
{
std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z"
int low = -1; // code point, for a member that is a lone \u escape
int high = -1; // and the other end, when the member is a range of them
};
struct CharacterClass
{
size_t end = 0; // index just past the closing ]
bool negated = false;
bool wide = false; // holds a code point that cannot be a single byte
std::vector<ClassMember> members;
};
// one class member starting at begin, or 0 if the class ends there
inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member)
{
auto unit = [&](size_t at, int& codePoint) -> size_t {
size_t length = 0;
codePoint = codePointAt(pattern, at, length);
if (codePoint >= 0)
{
return length;
}
return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1;
};
int low = -1;
size_t length = unit(begin, low);
size_t at = begin + length;
// a - b, where a trailing - is a literal rather than the start of a range
if (at + 1 < pattern.size() && pattern[at] == '-' && pattern[at + 1] != ']')
{
int high = -1;
size_t tail = unit(at + 1, high);
member.low = low;
member.high = high;
member.text = pattern.substr(begin, length + 1 + tail);
return length + 1 + tail;
}
member.low = low;
member.text = pattern.substr(begin, length);
return length;
}
inline bool readClass(std::string_view pattern, size_t begin, CharacterClass& result)
{
size_t at = begin + 1;
if (at < pattern.size() && pattern[at] == '^')
{
result.negated = true;
++at;
}
while (at < pattern.size() && pattern[at] != ']')
{
ClassMember member;
size_t length = readMember(pattern, at, member);
if (length == 0)
{
return false;
}
if (member.low > 0x7F || member.high > 0x7F)
{
result.wide = true;
}
result.members.push_back(member);
at += length;
}
if (at >= pattern.size())
{
return false;
}
result.end = at + 1;
return true;
}
inline bool isRange(const CharacterClass& characters, int low, int high)
{
return characters.members.size() == 1
&& characters.members[0].low == low
&& characters.members[0].high == high;
}
// [\uD800-\uDBFF][\uDC00-\uDFFF] and [\uD800-\uDFFF]{2} are how a JavaScript regex spells
// one astral character, as its two UTF-16 halves. In UTF-8 that is one four byte sequence.
inline size_t appendSurrogatePair(std::string& out, std::string_view pattern, size_t begin)
{
CharacterClass lead;
if (!readClass(pattern, begin, lead) || lead.negated)
{
return 0;
}
size_t end = 0;
if (isRange(lead, 0xD800, 0xDBFF))
{
CharacterClass trail;
if (!readClass(pattern, lead.end, trail) || trail.negated || !isRange(trail, 0xDC00, 0xDFFF))
{
return 0;
}
end = trail.end;
}
else if (isRange(lead, 0xD800, 0xDFFF) && pattern.substr(lead.end, 3) == "{2}")
{
end = lead.end + 3;
}
else
{
return 0;
}
out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})";
return end - begin;
}
// the class at begin, with any member that cannot be a single byte moved out of it
inline size_t appendClass(std::string& out, std::string_view pattern, size_t begin)
{
CharacterClass characters;
if (!readClass(pattern, begin, characters) || !characters.wide)
{
return 0;
}
std::vector<int> wide;
std::string narrow;
for (const auto& member : characters.members)
{
if (member.low <= 0x7F && member.high <= 0x7F)
{
narrow += member.text;
}
else if (member.high < 0)
{
wide.push_back(member.low);
}
else if (!characters.negated)
{
// a range of code points that is not the surrogate idiom; leave it be rather
// than expand what could be the whole of Unicode
return 0;
}
}
// a negated class drops its wide members: they exclude characters that cannot occur
// as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones
if (characters.negated || wide.empty())
{
out += '[';
if (characters.negated)
{
out += '^';
}
out += narrow;
out += ']';
return characters.end - begin;
}
out += "(?:";
if (!narrow.empty())
{
out += '[';
out += narrow;
out += ']';
out += '|';
}
for (size_t i = 0; i < wide.size(); ++i)
{
if (i > 0)
{
out += '|';
}
appendUtf8(out, wide[i]);
}
out += ')';
return characters.end - begin;
}
inline std::string widenUnicodeClasses(std::string_view pattern)
{
if (pattern.find("\\u") == std::string_view::npos)
{
return std::string{ pattern };
}
std::string out;
out.reserve(pattern.size());
for (size_t i = 0; i < pattern.size(); )
{
if (pattern[i] == '\\')
{
size_t length = 0;
const int codePoint = codePointAt(pattern, i, length);
if (codePoint >= 0)
{
// grouped, because a quantifier after it has to apply to the whole
// character and not to the last byte of its encoding
const bool multiByte = codePoint > 0x7F;
if (multiByte)
{
out += "(?:";
}
appendUtf8(out, codePoint);
if (multiByte)
{
out += ')';
}
i += length;
continue;
}
out += pattern.substr(i, 2);
i += 2;
continue;
}
if (pattern[i] == '[')
{
if (size_t consumed = appendSurrogatePair(out, pattern, i))
{
i += consumed;
continue;
}
if (size_t consumed = appendClass(out, pattern, i))
{
i += consumed;
continue;
}
}
out += pattern[i];
++i;
}
return out;
}
}
BIN
View File
Binary file not shown.
+11 -1
View File
@@ -9,13 +9,23 @@
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"lodash.isequal": "^4.5.0"
"lodash.isequal": "^4.5.0",
"prismjs": "1.29.0"
}
},
"node_modules/lodash.isequal": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
"integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="
},
"node_modules/prismjs": {
"version": "1.29.0",
"resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
"integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==",
"license": "MIT",
"engines": {
"node": ">=6"
}
}
}
}
+2 -1
View File
@@ -17,6 +17,7 @@
},
"homepage": "https://github.com/TelegramMessenger/libprisma#readme",
"dependencies": {
"lodash.isequal": "^4.5.0"
"lodash.isequal": "^4.5.0",
"prismjs": "1.29.0"
}
}
+163
View File
@@ -0,0 +1,163 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
/**
* Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
* matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
* the one thing it promises: that a pattern means the same to every client that reads it.
*
* The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
* anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
* looking for.
*/
public class PatternProbe {
/** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
private static final class Entry {
String source;
int flags;
}
private static final class Reader {
private final byte[] bytes;
private int offset;
Reader(byte[] bytes) {
this.bytes = bytes;
}
int uint8() {
return bytes[offset++] & 0xFF;
}
int uint16() {
return uint8() | (uint8() << 8);
}
String string() {
int length = uint8();
if (length >= 254) {
length = uint8() | (uint8() << 8) | (uint8() << 16);
}
String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
offset += length;
return value;
}
}
private static List<Entry> read(String path) throws IOException {
Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
List<Entry> entries = new ArrayList<>();
for (int count = reader.uint16(); count > 0; count--) {
String item = reader.string();
int begin = item.indexOf('/');
int end = item.lastIndexOf('/');
if (begin < 0 || end <= begin) {
continue;
}
String options = item.substring(end + 1);
int aliasBegin = options.indexOf(',');
Entry entry = new Entry();
entry.source = item.substring(begin + 1, end);
for (int i = 0; i < aliasBegin; ++i) {
switch (options.charAt(i)) {
case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
case 'm': entry.flags |= Pattern.MULTILINE; break;
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
}
}
entries.add(entry);
}
return entries;
}
/**
* Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
* line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
* A Java client has to do this; it cannot go in the table, where \z would read as a z.
*/
private static String endOfInput(String source) {
if (source.indexOf('$') < 0) {
return source;
}
StringBuilder result = new StringBuilder(source.length());
boolean inClass = false;
for (int i = 0; i < source.length(); ++i) {
char c = source.charAt(i);
if (c == '\\' && i + 1 < source.length()) {
result.append(c).append(source.charAt(++i));
} else if (c == '[') {
inClass = true;
result.append(c);
} else if (c == ']') {
inClass = false;
result.append(c);
} else if (c == '$' && !inClass) {
result.append("\\z");
} else {
result.append(c);
}
}
return result.toString();
}
public static void main(String[] args) throws IOException {
List<Entry> entries = read(args[0]);
String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
.replace(String.valueOf((char) 13), "");
StringBuilder out = new StringBuilder();
int failed = 0;
for (int i = 0; i < entries.size(); ++i) {
Entry entry = entries.get(i);
String source = entry.source;
if ((entry.flags & Pattern.MULTILINE) == 0) {
source = endOfInput(source);
}
Pattern pattern;
try {
pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
} catch (PatternSyntaxException e) {
if (failed++ < 20) {
System.err.println("pattern " + i + ": " + e.getDescription());
System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length())));
}
continue;
}
out.append(i);
java.util.regex.Matcher matcher = pattern.matcher(corpus);
int at = 0;
while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
out.append(' ').append(matcher.start()).append(':').append(matcher.end());
at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
if (at > corpus.length()) {
break;
}
}
out.append('\n');
}
Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
if (failed > 0) {
System.err.println(failed + " patterns java.util.regex will not take. "
+ "Teach normalizeEscapes in generate.js to rewrite them.");
System.exit(1);
}
}
}
+87
View File
@@ -0,0 +1,87 @@
// Every escape in grammars.dat has to mean the same thing to every engine that reads it.
// A pattern that fails to compile is caught by the probes beside this script; this one is for
// the escapes that compile everywhere and quietly mean something else, which is the worse
// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline
// included, by java.util.regex, which made ini values run over the end of the line.
const fs = require('fs')
// what generate.js is expected to emit, once normalizeEscapes has run over it
const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split(''))
const REASONS = {
v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B',
h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere',
H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere',
V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere',
R: 'any line break in java.util.regex, a literal R elsewhere',
X: 'a grapheme cluster in java.util.regex, a literal X elsewhere',
A: 'start of input in java.util.regex, a literal A elsewhere',
Z: 'end of input in java.util.regex, a literal Z elsewhere',
z: 'end of input in java.util.regex, a literal z elsewhere',
G: 'end of the previous match in java.util.regex, a literal G elsewhere',
Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere',
E: 'ends one in java.util.regex, a literal E elsewhere',
p: 'a unicode property in both, but the property names differ',
N: 'a named character in java.util.regex',
0: 'starts an octal escape in java.util.regex - emit \\x00',
}
function readPatterns(path) {
const buffer = fs.readFileSync(path)
let offset = 0
const uint8 = () => buffer[offset++]
const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value }
const string = () => {
let length = uint8()
if (length >= 254) {
length = uint8() | (uint8() << 8) | (uint8() << 16)
}
const value = buffer.toString('latin1', offset, offset + length)
offset += length
return value
}
const patterns = []
for (let count = uint16(); count > 0; count--) {
patterns.push(string())
}
return patterns
}
const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat')
const problems = []
for (const item of patterns) {
const end = item.lastIndexOf('/')
const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end)
for (let i = 0; i < source.length; i++) {
if (source[i] !== '\\') {
continue
}
const escaped = source[++i]
if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) {
problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set'])
}
}
}
console.log(`${patterns.length} patterns checked`)
if (problems.length === 0) {
process.exit(0)
}
const seen = new Set()
for (const [source, escape, reason] of problems) {
if (seen.has(escape)) {
continue
}
seen.add(escape)
console.error(`\n${escape} is ${reason}`)
console.error(` e.g. ${source.slice(0, 120)}`)
}
console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`)
process.exit(1)
+13
View File
@@ -0,0 +1,13 @@
class Foo<T> where T : IBar { public int x = 1; } /* c */ // line
<!DOCTYPE html><a href="x" [attr]='y'>{{ tpl }}</a>
def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py
let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3;
SELECT * FROM t WHERE a IN (1,2) -- sql
#include <stdio.h>
key = value ; ini
[section]
- item:
nested: [1, 2]
0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y
tab and trailing spaces
#!/bin/sh
+132
View File
@@ -0,0 +1,132 @@
// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down
// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and
// the two files are diffed, which is what holds the table to meaning one thing everywhere.
//
// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so
// anything above U+007F diverges on offsets alone and would drown out the escape bugs.
#include <boost/regex.hpp>
#include "../libprisma/UnicodeEscapes.h"
#include <algorithm>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <string>
#include <vector>
namespace {
std::string readAll(const char* path) {
std::ifstream in(path, std::ios::binary);
return std::string((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
}
struct Reader {
const std::string& bytes;
size_t offset = 0;
uint8_t uint8() {
return offset < bytes.size() ? static_cast<uint8_t>(bytes[offset++]) : 0;
}
uint16_t uint16() {
uint16_t low = uint8();
return static_cast<uint16_t>(low | (uint8() << 8));
}
std::string string() {
size_t length = uint8();
if (length >= 254) {
size_t a = uint8(), b = uint8(), c = uint8();
length = a | (b << 8) | (c << 16);
}
std::string value = bytes.substr(offset, length);
offset += length;
return value;
}
};
struct Entry {
std::string source;
boost::regex_constants::syntax_option_type flags;
};
// "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads
std::vector<Entry> read(const std::string& table) {
Reader reader{ table };
std::vector<Entry> entries;
for (uint16_t count = reader.uint16(); count > 0; --count) {
const std::string item = reader.string();
const size_t begin = item.find_first_of('/');
const size_t end = item.find_last_of('/');
if (begin == std::string::npos || end <= begin) {
continue;
}
const std::string options = item.substr(end + 1);
const size_t aliasBegin = options.find_first_of(',');
Entry entry;
entry.source = item.substr(begin + 1, end - (begin + 1));
entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m;
for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) {
switch (options[i]) {
case 'i': entry.flags |= boost::regex_constants::icase; break;
case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break;
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
}
}
entries.push_back(entry);
}
return entries;
}
}
int main(int argc, char** argv) {
if (argc < 4) {
fprintf(stderr, "usage: probe <grammars.dat> <corpus> <out>\n");
return 2;
}
const std::vector<Entry> entries = read(readAll(argv[1]));
std::string corpus = readAll(argv[2]);
corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end());
std::ofstream out(argv[3], std::ios::binary);
int failed = 0;
for (size_t i = 0; i < entries.size(); ++i) {
boost::regex re;
try {
re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags);
} catch (const std::exception& e) {
if (failed++ < 20) {
fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str());
}
continue;
}
out << i;
auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re,
boost::regex_constants::match_not_dot_newline);
for (auto end = boost::sregex_iterator(); it != end; ++it) {
out << ' ' << it->position() << ':' << (it->position() + it->length());
}
out << '\n';
}
printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size());
if (failed > 0) {
fprintf(stderr, "%d patterns Boost will not take.\n", failed);
return 1;
}
return 0;
}