mirror of
https://github.com/TelegramMessenger/libprisma.git
synced 2026-11-18 20:23:45 +00:00
Merge remote-tracking branch 'upstream/master' into HEAD
This commit is contained in:
commit
ca59e6434e
25 files changed
+1599
-49
No files matched your search
@@ -62,3 +62,4 @@
|
||||
#*.rtf diff=astextplain
|
||||
#*.RTF diff=astextplain
|
||||
libprisma/grammars.json eol=lf
|
||||
libprisma/grammars.dat linguist-generated
|
||||
@@ -1,27 +1,37 @@
|
||||
name: Update grammars
|
||||
|
||||
on: [ workflow_dispatch ]
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ master ]
|
||||
# grammars.dat itself is deliberately not listed: regenerating it must not trigger another run.
|
||||
paths:
|
||||
- components/**
|
||||
- generate.js
|
||||
- package.json
|
||||
- package-lock.json
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [18.x]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 18
|
||||
node-version: 22
|
||||
- run: npm install
|
||||
- run: npm start
|
||||
- name: Upload a Build Artifact
|
||||
uses: actions/upload-artifact@v3.1.3
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: grammars.dat
|
||||
path: ./libprisma/grammars.dat
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v5
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
commit-message: Update grammars
|
||||
title: Update grammars
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Check generated files
|
||||
|
||||
# The path filter is the whole test: the job only exists when the pull request
|
||||
# touches grammars.dat, so reaching it is already the failure.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- libprisma/grammars.dat
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
grammars:
|
||||
runs-on: ubuntu-latest
|
||||
# A pull request opened with GITHUB_TOKEN cannot trigger this workflow at all,
|
||||
# so the guard only matters if "Update grammars" is ever moved to a PAT.
|
||||
if: github.event.pull_request.user.login != 'github-actions[bot]'
|
||||
steps:
|
||||
- name: Reject changes to the generated grammar table
|
||||
run: |
|
||||
echo "::error file=libprisma/grammars.dat::libprisma/grammars.dat is generated by generate.js and is regenerated on master by the Update grammars workflow. Please drop it from this pull request and keep only the components/ and generate.js changes."
|
||||
exit 1
|
||||
@@ -0,0 +1,58 @@
|
||||
name: Check grammars
|
||||
|
||||
# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex,
|
||||
# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it
|
||||
# to that, rather than waiting for a language to quietly stop highlighting.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- components/**
|
||||
- generate.js
|
||||
- libprisma/**
|
||||
- tools/**
|
||||
- package.json
|
||||
- package-lock.json
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
patterns:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
# the committed table is regenerated on master, so a pull request is checked against what
|
||||
# its own components and generate.js produce
|
||||
- run: npm ci --ignore-scripts
|
||||
- run: npm start
|
||||
|
||||
- name: Escapes every engine agrees on
|
||||
run: node tools/check-escapes.js libprisma/grammars.dat
|
||||
|
||||
- name: Compile every pattern with java.util.regex
|
||||
run: |
|
||||
javac -d tools tools/PatternProbe.java
|
||||
java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt
|
||||
|
||||
- name: Compile every pattern with Boost
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libboost-regex-dev
|
||||
g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
|
||||
./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
|
||||
|
||||
- name: Compare what the two engines matched
|
||||
run: |
|
||||
if ! diff -u boost.txt java.txt > engines.diff; then
|
||||
echo "The two engines disagree about what these patterns match."
|
||||
echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt."
|
||||
head -40 engines.diff
|
||||
exit 1
|
||||
fi
|
||||
echo "Boost and java.util.regex agree on every pattern."
|
||||
@@ -129,10 +129,12 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|
||||
|`F#`|`fsharp`|
|
||||
|`Factor`|`factor`|
|
||||
|`False`|`false`|
|
||||
|`Fift`|`fift`|
|
||||
|`Firestore security rules`|`firestore-security-rules`|
|
||||
|`Flow`|`flow`|
|
||||
|`Fortran`|`fortran`|
|
||||
|`FreeMarker Template Language`|`ftl`|
|
||||
|`FunC`|`func`|
|
||||
|`GameMaker Language`|`gml`,`gamemakerlanguage`|
|
||||
|`GAP (CAS)`|`gap`|
|
||||
|`G-code`|`gcode`|
|
||||
@@ -290,6 +292,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|
||||
|`SuperCollider`|`supercollider`,`sclang`|
|
||||
|`Swift`|`swift`|
|
||||
|`Systemd configuration file`|`systemd`|
|
||||
|`Tact`|`tact`|
|
||||
|`T4 templating`|`t4-templating`|
|
||||
|`T4 Text Templates (C#)`|`t4-cs`,`t4`|
|
||||
|`VB.Net`|`vbnet`|
|
||||
@@ -299,6 +302,8 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|
||||
|`Template Toolkit 2`|`tt2`|
|
||||
|`TOML`|`toml`|
|
||||
|`Tremor`|`tremor`,`tremor`,`trickle`,`troy`|
|
||||
|`Type Language`|`tl`|
|
||||
|`Type Language - Binary`|`tlb`|
|
||||
|`TypoScript`|`typoscript`,`tsconfig`|
|
||||
|`UnrealScript`|`unrealscript`,`unrealscript`,`uscript`,`uc`|
|
||||
|`UO Razor Script`|`uorazor`|
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* @file Prism.js definition for Fift
|
||||
* @link https://docs.ton.org/develop/fift/overview
|
||||
* @author Nikita Sobolev (https://github.com/sobolevn)
|
||||
* @license MIT
|
||||
*/
|
||||
Prism.languages.fift = {
|
||||
'symbol': [
|
||||
/[xX]\{[0-9a-fA-F_]*\}/,
|
||||
/[bB]\{[01]*\}/,
|
||||
/'\S+/,
|
||||
],
|
||||
'string': /"([^"\r\n\\]|\\.)*"/,
|
||||
|
||||
'comment': [
|
||||
{
|
||||
pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
{
|
||||
pattern: /\/\/.*/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
],
|
||||
|
||||
'operator': [
|
||||
// Full list can be found in
|
||||
// Appendix A. List of Fift words
|
||||
// Ordered the same way source code does this.
|
||||
// Except: shorter words must follow
|
||||
// longer ones with the same base part.
|
||||
// Example: `#` comes after `#s`
|
||||
/#>/, /#s/, /\$#/, /#/,
|
||||
|
||||
/\$\+/, /\$,/, /\$\d/, /\$=/, /\$(?=\()/,
|
||||
/\$>smca/, /\$>s/,
|
||||
/\$@\+/, /\$@\?\+/, /\$@\?/, /\$@/,
|
||||
/\$cmp/, /\$len/, /\$pos/, /\$reverse/,
|
||||
|
||||
/%1<</,
|
||||
|
||||
/\('\)/, /\(-trailing\)/, /\(\.\)/, /\(atom\)/, /\(b\.\)/,
|
||||
/\(compile\)/, /\(create\)/, /\(def\?\)/, /\(dump\)/, /\(execute\)/,
|
||||
/\(forget\)/, /\(number\)/, /\(x\.\)/, /\(\{\)/, /\(\}\)/,
|
||||
|
||||
/\*\/cmod/, /\*\/c/, /\*\/mod/, /\*\/rmod/, /\*\/r/, /\*\//,
|
||||
/\*>>c/, /\*>>r/, /\*>>/, /\*mod/, /\*/,
|
||||
|
||||
/\+!/, /\+/, /,/, /-!/, /-/, /!/,
|
||||
/-1<</, /\._/, /\.dump/, /\.l/, /\.sl/, /\.s/, /\.tc/,
|
||||
/\//, /\/\*/, /\/cmod/, /\/c/, /\/mod/, /\/rmod/, /\/r/,
|
||||
/0!/, /0<=/, /0<>/, /0</, /0=/, /0>=/, /0>/,
|
||||
/1\+!/, /1\+/, /1-!/, /1-/, /1<</, /1<<1-/,
|
||||
/2\*/, /2\+/, /2-/, /2\//, /2=:/,
|
||||
|
||||
/::_/, /::/, /:_/, /=:/, /:/,
|
||||
|
||||
/<#/, /<<\/c/, /<<\/r/, /<=/, /<>/, /<b/, /<s/, /<<\//,
|
||||
/<</, /</, /=/,
|
||||
|
||||
/>=/, />>c/, />>r/, />>/, />/,
|
||||
|
||||
/\?dup/,
|
||||
|
||||
/@'/, /@/,
|
||||
|
||||
/B\+/, /B,/, /B=/, /B>Li@\+/, /B>Li@/, /B>Lu@\+/, /B>Lu@/, /B>boc/,
|
||||
/B>file/, /B>i@\+/, /B>i@/, /B>u@\+/, /B>u@/,
|
||||
/B@\?\+/, /B@\+/, /B@\?/, /B@/,
|
||||
/Bcmp/, /BhashB/, /Bhashu/, /Bhash/, /Blen/, /Bx\./,
|
||||
/B\|/, /Li>B/, /Lu>B/,
|
||||
|
||||
/\[\]/, /\[compile\]/, /\[/, /\]/,
|
||||
|
||||
/atom\?/,
|
||||
|
||||
/b\+/, /b\._/, /b\./,
|
||||
/b>idict!\+/, /b>idict!/, /b>sdict!\+/, /b>sdict!/,
|
||||
/b>udict!\+/, /b>udict!/,
|
||||
/b>/, /boc\+>B/, /boc>B/,
|
||||
|
||||
/csr\./,
|
||||
/def\?/,
|
||||
/empty\?/, /eq\?/,
|
||||
/file-exists\?/, /file>B/,
|
||||
|
||||
/i,/, /i>B/, /i@\+/, /i@\?\+/, /i@\?/, /i@/,
|
||||
/idict!\+/, /idict!/, /idict-/, /idict@-/, /idict@/,
|
||||
|
||||
/null!/, /null\?/,
|
||||
/pfxdict!\+/, /pfxdict!/, /pfxdict@/, /priv>pub/,
|
||||
/ref@\+/, /ref@/, /ref@\?\+/, /ref@\?/,
|
||||
|
||||
/s,/, /s>c/, /s>/,
|
||||
/sdict!\+/, /sdict!/, /sdict-/, /sdict@-/, /sdict@/,
|
||||
/smca>\$/, /sr,/,
|
||||
|
||||
/tuple\?/,
|
||||
|
||||
/u,/, /u>B/, /u@\+/, /u@\?\+/, /u@\?/,
|
||||
/udict!\+/, /udict!/, /udict-/, /udict@-/, /udict@/,
|
||||
/undef\?/,
|
||||
|
||||
/x\._/, /x\./,
|
||||
|
||||
/\|\+/, /\|/, /\|_/,
|
||||
|
||||
/\?\./, /'/,
|
||||
|
||||
// Should be the last:
|
||||
/\./,
|
||||
],
|
||||
|
||||
'keyword': /\b(?:-roll|-rot|-trailing|-trailing0|2constant|2drop|2dup|2over|2swap|abort|abs|allot|and|anon|atom|bbitrefs|bbits|bl|box|brefs|brembitrefs|brembits|bremrefs|bye|caddr|cadr|car|cddr|cdr|char|chr|cmp|cond|cons|constant|count|cr|create|depth|dictmap|dictmerge|dictnew|does|drop|dup|ed25519_chksign|ed25519_sign|ed25519_sign_uint|emit|exch|exch2|execute|explode|find|first|fits|forget|gasrunvm|gasrunvmcode|gasrunvmctx|gasrunvmdict|halt|hash|hashB|hashu|hold|hole|if|ifnot|include|list|max|min|minmax|mod|negate|newkeypair|nil|nip|nop|not|now|null|or|over|pair|pick|quit|remaining|reverse|roll|rot|runvm|runvmcode|runvmctx|runvmdict|sbitrefs|sbits|second|sgn|shash|sign|single|skipspc|space|srefs|swap|ten|third|times|triple|tuck|tuple|type|ufits|uncons|unpair|unsingle|until|untriple|untuple|variable|while|word|words|xor)\b/,
|
||||
'boolean': /\b(?:false|true)\b/,
|
||||
|
||||
'number': [
|
||||
/(0[xX][0-9a-fA-F]+)/,
|
||||
/(0[bB][01]+)/,
|
||||
/(-?\d+(\/-?\d+)?)/,
|
||||
],
|
||||
'variable': /[\w$-]+/,
|
||||
|
||||
'punctuation': /[\[\{\}\],\(\)]/,
|
||||
};
|
||||
@@ -0,0 +1,148 @@
|
||||
/**
|
||||
* @file Prism.js definition for FunC
|
||||
* @link https://docs.ton.org/develop/func/overview
|
||||
* @version 0.4.4
|
||||
* @author Novus Nota (https://github.com/novusnota)
|
||||
* @author Nikita Sobolev (https://github.com/sobolevn)
|
||||
* @license MIT
|
||||
*/
|
||||
(function(Prism) {
|
||||
/** @type {Prism.GrammarValue} */
|
||||
var number = /-?\b(?:\d+|0x[\da-fA-F]+)\b(?![^\.~,;\[\]\(\)\s])/;
|
||||
|
||||
/** @type {Prism.GrammarValue} */
|
||||
var string = [
|
||||
{ // multi-line
|
||||
pattern: /"""[\s\S]*?"""[Hhcusa]?/,
|
||||
greedy: true,
|
||||
inside: {
|
||||
// string type
|
||||
'symbol': {
|
||||
pattern: /("""[\s\S]*?""")[Hhcusa]/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
}
|
||||
},
|
||||
},
|
||||
{ // single-line
|
||||
pattern: /"[^\n"]*"[Hhcusa]?/,
|
||||
greedy: true,
|
||||
inside: {
|
||||
// string type
|
||||
'symbol': {
|
||||
pattern: /("[^\n"]*")[Hhcusa]/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/** @type {RegExp} */
|
||||
var operator_body = /(?:!=|\?|:|%=|%|&=|&|\*=|\*|\+=|\+|->|-=|-|\/%|\/=|\/|<=>|<<=|<<|<=|<|==|=|>>=|>>|>=|>|\^>>=|\^>>|\^=|\^\/=|\^\/|\^%=|\^%|\^|\|=|\||~>>=|~>>|~\/=|~\/|~%|~)/;
|
||||
|
||||
// An operator only counts as one when whitespace follows it, as identifiers may contain these characters.
|
||||
/** @type {Prism.GrammarValue} */
|
||||
var operator = new RegExp(operator_body.source + /(?=\s)/.source);
|
||||
|
||||
/** @type {RegExp[]} */
|
||||
var var_identifier = [
|
||||
// quoted
|
||||
/`[^`\n]+`/,
|
||||
// plain
|
||||
/[^\.~,;\[\]\(\)\s]+/
|
||||
];
|
||||
|
||||
/** Prism.js definition for FunC */
|
||||
Prism.languages.func = {
|
||||
'comment': [
|
||||
{ // single-line
|
||||
pattern: /;;.*/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
{ // multi-line, not nested (TODO: nesting)
|
||||
// . isn't used, because it only applies to the single line
|
||||
pattern: /\{-[\s\S]*?(?:-\}|$)/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
{
|
||||
// unused variable identifier
|
||||
pattern: /\b_\b(?![^\.~,;\[\]\(\)\s])/
|
||||
},
|
||||
],
|
||||
|
||||
// Custom token for #pragma's
|
||||
'pragma': {
|
||||
pattern: /#pragma(.*);/,
|
||||
inside: {
|
||||
'keyword': /(?:#pragma|test-version-set|not-version|version|allow-post-modification|compute-asm-ltr)\b/,
|
||||
'number': /(?:\d+)(?:.\d+)?(?:.\d+)?/,
|
||||
'operator': /=|\^|<=|>=|<|>/,
|
||||
'string': string,
|
||||
'punctuation': /;/,
|
||||
}
|
||||
},
|
||||
|
||||
// Custom token for #include's
|
||||
'include': {
|
||||
pattern: /#include(.*);/,
|
||||
inside: {
|
||||
'keyword': /#include\b/,
|
||||
'string': string,
|
||||
'punctuation': /;/,
|
||||
}
|
||||
},
|
||||
|
||||
// builtin types
|
||||
'builtin': /\b(?:int|cell|slice|builder|cont|tuple|type)\b/,
|
||||
|
||||
// builtin constants
|
||||
'boolean': /\b(?:false|true|nil|Nil)\b/,
|
||||
|
||||
'keyword': /\b(?:forall|extern|global|asm|impure|inline_ref|inline|auto_apply|method_id|operator|infixl|infixr|infix|const|return|var|repeat|do|while|until|try|catch|ifnot|if|then|elseifnot|elseif|else)\b/,
|
||||
|
||||
'number': number,
|
||||
|
||||
'string': string,
|
||||
|
||||
'operator': operator,
|
||||
|
||||
'punctuation': [/[\.,;\(\)\[\]]/, /[\{\}](?![^\.~,;\[\]\(\)\s])/],
|
||||
|
||||
// Function and method names in declarations, definitions and calls
|
||||
'function': [
|
||||
{ // quoted
|
||||
pattern: new RegExp(var_identifier[0].source + /(?=\s*\()/.source),
|
||||
greedy: true,
|
||||
},
|
||||
{ // bitwise not operator
|
||||
pattern: /~_/,
|
||||
greedy: true,
|
||||
},
|
||||
{ // remaining operators
|
||||
// operator_body, not operator: its trailing lookahead would forbid the closing _
|
||||
pattern: new RegExp(/\^?_/.source + operator_body.source + /_/.source),
|
||||
greedy: true,
|
||||
},
|
||||
{ // plain function or method name
|
||||
pattern: new RegExp(/[\.~]?/.source + var_identifier[1].source + /(?=\s*\()/.source),
|
||||
greedy: true,
|
||||
},
|
||||
{ // builtin functions and methods
|
||||
pattern: /\b(?:divmod|moddiv|muldiv|muldivr|muldivc|muldivmod|null\?|throw|throw_if|throw_unless|throw_arg|throw_arg_if|throw_arg_unless|load_int|load_uint|preload_int|preload_uint|store_int|store_uint|load_bits|preload_bits|int_at|cell_at|slice_at|tuple_at|at|touch|touch2|run_method0|run_method1|run_method2|run_method3|~divmod|~moddiv|~store_int|~store_uint|~touch|~touch2|~dump|~stdump)\b/,
|
||||
greedy: true,
|
||||
},
|
||||
],
|
||||
|
||||
// Parametric polymorphism
|
||||
'class-name': /[A-Z][^\.~,;\[\]\(\)\s]*/,
|
||||
|
||||
// All the rest of identifiers
|
||||
'variable': var_identifier.map(x => { return { pattern: x, greedy: true } }),
|
||||
};
|
||||
|
||||
// Adding a fc alias
|
||||
Prism.languages.fc = Prism.languages.func;
|
||||
}(Prism));
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* @file Prism.js definition for Tact
|
||||
* @link https://tact-lang.org
|
||||
* @version 1.6.0
|
||||
* @author Novus Nota (https://github.com/novusnota)
|
||||
* @license MIT
|
||||
*/
|
||||
(function(Prism) {
|
||||
Prism.languages.tact = {
|
||||
// reserved keywords
|
||||
'keyword': [
|
||||
{
|
||||
pattern: /\b(?:abstract|asm|as|catch|codeOf|const|contract(?!:)|do|else|extend|extends|foreach|fun|get|if|in|import|initOf|inline|let|message(?!:)|mutates|native|override|primitive|public|repeat|return|self|struct(?!:)|trait(?!:)|try|until|virtual|while|with)\b/,
|
||||
},
|
||||
{ // reserved function names
|
||||
pattern: /\b(?:bounced|external|init|receive)\b(?=\()/
|
||||
},
|
||||
],
|
||||
|
||||
// built-in types
|
||||
'builtin': [
|
||||
{
|
||||
pattern: /\b(?:Address|Bool|Builder|Cell|Int|Slice|String|StringBuilder)\b/,
|
||||
},
|
||||
{ // keyword after as, see: https://prismjs.com/extending.html#object-notation
|
||||
pattern: /(\bas\s+)(?:coins|varu?int(?:32|16)|remaining|bytes(?:32|64)|int257|u?int(?:25[0-6]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?))\b/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
],
|
||||
|
||||
// SCREAMING_SNAKE_CASE for null values and names of constants
|
||||
'constant': [
|
||||
{
|
||||
pattern: /\bnull\b/,
|
||||
},
|
||||
{
|
||||
pattern: /\b[A-Z][A-Z0-9_]*\b/,
|
||||
},
|
||||
],
|
||||
|
||||
// UpperCamelCase for names of contracts, traits, structs, messages
|
||||
'class-name': {
|
||||
pattern: /\b[A-Z]\w*\b/,
|
||||
},
|
||||
|
||||
// mappings to FunC
|
||||
'attribute': [
|
||||
{ // functions
|
||||
pattern: /@name/,
|
||||
inside: {
|
||||
'function': /.+/,
|
||||
},
|
||||
},
|
||||
{ // contract interfaces
|
||||
pattern: /@interface/,
|
||||
inside: {
|
||||
'function': /.+/,
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
'function': {
|
||||
pattern: /\b\w+(?=\()/,
|
||||
},
|
||||
|
||||
'boolean': {
|
||||
pattern: /\b(?:false|true)\b/,
|
||||
},
|
||||
|
||||
'number': [
|
||||
{ // hexadecimal, case-insensitive /i
|
||||
pattern: /\b0x[0-9a-f](?:_?[0-9a-f])*\b/i,
|
||||
},
|
||||
{ // octal, case-insensitive /i
|
||||
pattern: /\b0o[0-7](?:_?[0-7])*\b/i,
|
||||
},
|
||||
{ // binary, case-insensitive /i
|
||||
pattern: /\b0b[01](?:_?[01])*\b/i,
|
||||
},
|
||||
{ // decimal integers, starting with 0
|
||||
pattern: /\b0\d*\b/,
|
||||
},
|
||||
{ // other decimal integers
|
||||
pattern: /\b[1-9](?:_?\d)*\b/,
|
||||
},
|
||||
],
|
||||
|
||||
'string': undefined,
|
||||
|
||||
'punctuation': {
|
||||
pattern: /[{}[\]();,.:?]/,
|
||||
},
|
||||
|
||||
'comment': [
|
||||
{ // single-line
|
||||
pattern: /(^|[^\\:])\/\/.*/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
{ // multi-line
|
||||
pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
},
|
||||
{
|
||||
// unused variable identifier
|
||||
pattern: /\b_\b/
|
||||
}
|
||||
],
|
||||
|
||||
'operator': {
|
||||
'pattern': /![!=]?|->|[+\-*/%=]=?|[<>]=|<<?=?|>>?=?|~|\|\|=|\|[\|=]?|&&=|&[&=]?|\^=?/,
|
||||
},
|
||||
|
||||
'variable': {
|
||||
'pattern': /\b[a-zA-Z_]\w*\b/,
|
||||
},
|
||||
|
||||
};
|
||||
|
||||
// strings, made this way to not collide with other entities
|
||||
Prism.languages.insertBefore('tact', 'string', {
|
||||
'string-literal': {
|
||||
pattern: /(?:(")(?:\\.|(?!\1)[^\\\r\n])*\1(?!\1))/,
|
||||
greedy: true,
|
||||
inside: {
|
||||
'regex': [
|
||||
{ // \\ \" \n \r \t \v \b \f
|
||||
pattern: /\\[\\"nrtvbf]/,
|
||||
},
|
||||
{ // hexEscape, \x00 through \xFF
|
||||
pattern: /\\x[0-9a-fA-F]{2}/,
|
||||
},
|
||||
{ // unicodeEscape, \u0000 through \uFFFF
|
||||
pattern: /\\u[0-9a-fA-F]{4}/,
|
||||
},
|
||||
{ // unicodeCodePoint, \u{0} through \u{FFFFFF}
|
||||
pattern: /\\u\{[0-9a-fA-F]{1,6}\}/,
|
||||
},
|
||||
],
|
||||
'string': {
|
||||
pattern: /[\s\S]+/,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// map and bounced message generic type modifiers
|
||||
Prism.languages.insertBefore('tact', 'keyword', {
|
||||
'generics': {
|
||||
pattern: /(?:\b(?:bounced|map)\b<[^\\\r\n]*>)/,
|
||||
greedy: true,
|
||||
inside: {
|
||||
'builtin': [
|
||||
...Prism.languages['tact']['builtin'],
|
||||
{
|
||||
pattern: /\b(?:bounced(?=<)|map(?=<))\b/
|
||||
},
|
||||
],
|
||||
'class-name': Prism.languages['tact']['class-name'],
|
||||
'punctuation': {
|
||||
pattern: /[<>(),.?]/,
|
||||
},
|
||||
'keyword': {
|
||||
pattern: /\bas\b/,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}(Prism));
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* @file Prism.js definition for TL-B
|
||||
* @link https://docs.ton.org/develop/data-formats/tl-b-language
|
||||
* @author Nikita Sobolev (https://github.com/sobolevn)
|
||||
* @license MIT
|
||||
*/
|
||||
Prism.languages.tlb = {
|
||||
'builtin': /\b(?:Bool|Both|Cell|Either|Maybe|Type|Unit|bits256|bits512|int16|int32|int64|int8|uint15|uint16|uint32|uint63|uint64|uint8)\b/,
|
||||
'keyword': /\b(?:BoolFalse|BoolTrue|False|Null|True)\b/,
|
||||
|
||||
'comment': {
|
||||
pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,
|
||||
greedy: true
|
||||
},
|
||||
'comment-multiline': [
|
||||
{
|
||||
pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
alias: 'comment'
|
||||
},
|
||||
{
|
||||
pattern: /(^|[^\\:])\/\/.*/,
|
||||
lookbehind: true,
|
||||
greedy: true,
|
||||
alias: 'comment'
|
||||
}
|
||||
],
|
||||
|
||||
'symbol': [
|
||||
/#[0-9a-f]*_?/,
|
||||
/\$[01]*_?/,
|
||||
/\b(?:##|#<|#<=)\b/,
|
||||
],
|
||||
'variable': /[a-zA-Z_]\w*/,
|
||||
'operator': [
|
||||
/\+/, /-/, /\*/, /\//,
|
||||
/!=/, /==/, /=/,
|
||||
/\?/, /~/, /\./, /\^/,
|
||||
/<=/, />=/, /</, />/,
|
||||
],
|
||||
'number': /\d+/,
|
||||
'punctuation': /[;\(\):\[\]\{\}]/,
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* @file Prism.js definition for Tolk
|
||||
* @link https://docs.ton.org/tolk/overview
|
||||
* @version 1.4.0
|
||||
* @author Danil Ovchinnikov (https://github.com/Danil42Russia)
|
||||
* @license MIT
|
||||
*/
|
||||
(function(Prism) {
|
||||
Prism.languages.tolk = {
|
||||
'comment': {
|
||||
pattern: /\/\/.*/,
|
||||
greedy: true,
|
||||
},
|
||||
|
||||
'block-comment': {
|
||||
pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
|
||||
greedy: true,
|
||||
},
|
||||
|
||||
'string': [
|
||||
{
|
||||
pattern: /"""(?:\\[\s\S]|(?!""")[\s\S])*(?:"""|$)/,
|
||||
greedy: true,
|
||||
},
|
||||
{
|
||||
pattern: /"(?:\\.|[^"\\\r\n])*(?:"|$)/,
|
||||
greedy: true,
|
||||
},
|
||||
],
|
||||
|
||||
'attr-name': /@[a-zA-Z0-9_.]+/,
|
||||
|
||||
'number': [
|
||||
/\b0x[0-9A-Fa-f_]+\b/,
|
||||
/\b0b[01_]+\b/,
|
||||
/\b\d[\d_]*\b/,
|
||||
],
|
||||
|
||||
'boolean': /\b(?:false|null|true)\b/,
|
||||
|
||||
'constant': /\b[A-Z][A-Z0-9_]*_[A-Z0-9_]*\b/,
|
||||
|
||||
'keyword': [
|
||||
/!is\b/,
|
||||
/\b(?:as|asm|assert|break|builtin|catch|const|continue|contract|do|else|enum|fun|get|global|if|import|is|lazy|match|mutate|private|readonly|repeat|return|self|struct|throw|tolk|try|type|val|var|while)\b/,
|
||||
],
|
||||
|
||||
'function': [
|
||||
/`[^`\r\n]+`(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
|
||||
/\b[a-zA-Z_$][a-zA-Z0-9_$]*\b(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
|
||||
],
|
||||
|
||||
'builtin': [
|
||||
/\b(?:Cell|address|any_address|array|blockchain|bool|builder|cell|coins|continuation|contract|debug|dict|lisp_list|map|never|random|reflect|slice|string|tuple|unknown|void)\b/,
|
||||
/\b(?:int\d*|uint\d+|varint\d+|varuint\d+|bits\d+|bytes\d+)\b/,
|
||||
],
|
||||
|
||||
'class-name': /\b[A-Z][a-zA-Z0-9_$]*\b/,
|
||||
|
||||
'property': {
|
||||
pattern: /(\.)(?:`[^`\r\n]+`|[a-zA-Z_$][a-zA-Z0-9_$]*)/,
|
||||
lookbehind: true,
|
||||
},
|
||||
|
||||
'operator': /<<=|>>=|<=>|~>>|\^>>|==|!=|<=|>=|<<|>>|&&|\|\||~\/|\^\/|\+=|-=|\*=|\/=|%=|&=|\|=|\^=|->|=>|\+\+|--|\?\?|[+\-*\/%?=<>!&|^~]/,
|
||||
|
||||
'punctuation': /[.,;:(){}\[\]]/,
|
||||
|
||||
'symbol': /`[^`\r\n]+`/,
|
||||
};
|
||||
}(Prism));
|
||||
+100
-21
@@ -1,5 +1,9 @@
|
||||
const fs = require('fs')
|
||||
const isEqual = require('lodash.isequal')
|
||||
const Prism = require('prismjs')
|
||||
const components = require('prismjs/components.js')
|
||||
|
||||
global.Prism = Prism
|
||||
|
||||
const SCRIPTS = {}
|
||||
const include = function (src) {
|
||||
@@ -7,12 +11,6 @@ const include = function (src) {
|
||||
(1, eval)(src.toString())
|
||||
}
|
||||
|
||||
async function loadScript(src) {
|
||||
const script = await fetch(src)
|
||||
const text = await script.text()
|
||||
include(text)
|
||||
}
|
||||
|
||||
async function loadLanguages(lngs) {
|
||||
if (lngs) {
|
||||
lngs = Array.isArray(lngs) ? lngs : [lngs];
|
||||
@@ -39,7 +37,7 @@ async function loadLanguage(lng) {
|
||||
console.log(`${langNumber} | Loading ${lng}`);
|
||||
// TODO: version should probably not be hardcoded
|
||||
|
||||
await loadScript(`https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-${lng}.min.js`)
|
||||
require(`prismjs/components/prism-${lng}.js`)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +50,81 @@ function loadLocalLanguage(path, code, title, alias) {
|
||||
}
|
||||
}
|
||||
|
||||
// Spell out the constructs whose meaning differs between regex engines, so one table can
|
||||
// serve all of them. java.util.regex reads a [ inside a character class as the start of a
|
||||
// nested class, requires a { outside one to open a quantifier, reads \0 as the start of an
|
||||
// octal escape, and reads \v as the whole vertical whitespace class rather than as U+000B.
|
||||
// All four rewrites are no-ops for ECMAScript and for Boost, which is what libprisma
|
||||
// compiles the patterns with.
|
||||
const QUANTIFIER = /^\{\d+(?:,\d*)?\}/
|
||||
|
||||
function normalizeEscapes(pattern) {
|
||||
let result = ''
|
||||
let inClass = false
|
||||
|
||||
for (let i = 0; i < pattern.length; i++) {
|
||||
const c = pattern[i]
|
||||
|
||||
if (c === '\\') {
|
||||
if (pattern[i + 1] === '0' && !/[0-7]/.test(pattern[i + 2] || '')) {
|
||||
result += '\\x00'
|
||||
} else if (pattern[i + 1] === 'v') {
|
||||
result += '\\x0B'
|
||||
} else {
|
||||
result += pattern.substr(i, 2)
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if (inClass) {
|
||||
if (c === '[') {
|
||||
result += '\\['
|
||||
continue
|
||||
}
|
||||
// && is set intersection in java.util.regex
|
||||
if (c === '&' && pattern[i + 1] === '&') {
|
||||
result += '\\&\\&'
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (c === ']') {
|
||||
inClass = false
|
||||
}
|
||||
} else if (c === '[') {
|
||||
inClass = true
|
||||
} else if (c === '{' && !QUANTIFIER.test(pattern.substr(i))) {
|
||||
result += '\\{'
|
||||
continue
|
||||
}
|
||||
|
||||
result += c
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// A character written literally in a grammar - BQN's •, APL's ⍵ - has to reach the readers as
|
||||
// an escape rather than as itself: they disagree about what the bytes of a pattern mean, and
|
||||
// \uXXXX is the one spelling all of them read as a code point. An astral character is two code
|
||||
// units here, so it becomes a surrogate pair, which UnicodeEscapes.h puts back together.
|
||||
function escapeNonAscii(pattern) {
|
||||
if (!/[^\x00-\x7F]/.test(pattern)) {
|
||||
return pattern
|
||||
}
|
||||
|
||||
let result = ''
|
||||
|
||||
for (let i = 0; i < pattern.length; i++) {
|
||||
const code = pattern.charCodeAt(i)
|
||||
result += code > 0x7F
|
||||
? '\\u' + code.toString(16).padStart(4, '0')
|
||||
: pattern[i]
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function unique(a, fn) {
|
||||
if (a.length === 0 || a.length === 1) {
|
||||
return a;
|
||||
@@ -137,7 +210,7 @@ async function generate() {
|
||||
pattern = pattern.replaceAll("|[])", ")");
|
||||
pattern = pattern.replaceAll(":[]", ":");
|
||||
|
||||
return pattern
|
||||
return escapeNonAscii(normalizeEscapes(pattern))
|
||||
}
|
||||
|
||||
for (var token in copy) {
|
||||
@@ -215,14 +288,17 @@ async function generate() {
|
||||
"sparql" // requires turtle
|
||||
]
|
||||
|
||||
await loadScript("https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-core.min.js")
|
||||
await loadScript("https://prismjs.com/components.js")
|
||||
await loadLanguages(Object.keys(components.languages))
|
||||
console.log(`\nLoaded all ${langNumber} languages`)
|
||||
console.log("Processing...")
|
||||
|
||||
// Manually add local definitions
|
||||
loadLocalLanguage('./components/prism-tl.js', 'typelanguage', 'TypeLanguage', 'tl')
|
||||
loadLocalLanguage('./components/prism-tlb.js', 'tlb', 'TypeLanguage-Binary', 'tlb')
|
||||
loadLocalLanguage('./components/prism-fift.js', 'fift', 'Fift', 'fift')
|
||||
loadLocalLanguage('./components/prism-func.js', 'func', 'FunC', ['func', 'fc'])
|
||||
loadLocalLanguage('./components/prism-tact.js', 'tact', 'Tact', 'tact')
|
||||
loadLocalLanguage('./components/prism-tolk.js', 'tolk', 'Tolk', 'tolk')
|
||||
|
||||
Object.keys(Prism.languages).forEach(lng => {
|
||||
if (unsupported.includes(lng) || !components.languages[lng]) {
|
||||
@@ -338,11 +414,11 @@ async function generate() {
|
||||
for (const lng of alias) {
|
||||
allLanguages[lng] = allGrammars.indexOf(find)
|
||||
|
||||
if (components.languages[name].aliasTitles) {
|
||||
languageNames[lng] = components.languages[name].aliasTitles[lng]
|
||||
} else {
|
||||
languageNames[lng] = components.languages[name].title
|
||||
}
|
||||
// aliasTitles only names the aliases whose title differs from the language's,
|
||||
// so an alias missing from it takes the language title rather than no title:
|
||||
// an empty one drops the alias out of SyntaxHighlighter::languages().
|
||||
const aliasTitles = components.languages[name].aliasTitles
|
||||
languageNames[lng] = (aliasTitles && aliasTitles[lng]) || components.languages[name].title
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -358,15 +434,18 @@ async function generate() {
|
||||
const writeUint16 = i => chunks.push(new Uint16Array([i]))
|
||||
const writeUint8 = i => chunks.push(new Uint8Array([i]))
|
||||
const writeString = str => {
|
||||
if (str.length < 253) {
|
||||
writeUint8(str.length)
|
||||
// one byte per code unit truncated anything above U+00FF; the length is the byte
|
||||
// count, which is what the readers advance by
|
||||
const bytes = new TextEncoder().encode(str)
|
||||
if (bytes.length < 253) {
|
||||
writeUint8(bytes.length)
|
||||
} else {
|
||||
writeUint8(254 & 0xFF)
|
||||
writeUint8(str.length & 0xFF)
|
||||
writeUint8((str.length >> 8) & 0xFF)
|
||||
writeUint8((str.length >> 16) & 0xFF)
|
||||
writeUint8(bytes.length & 0xFF)
|
||||
writeUint8((bytes.length >> 8) & 0xFF)
|
||||
writeUint8((bytes.length >> 16) & 0xFF)
|
||||
}
|
||||
chunks.push(new Uint8Array(str.split('').map(char => char.charCodeAt(0))))
|
||||
chunks.push(bytes)
|
||||
}
|
||||
|
||||
// Patterns
|
||||
|
||||
@@ -31,6 +31,11 @@ const Pattern* PatternPtr::get() const
|
||||
return m_tree->resolvePattern(m_path);
|
||||
}
|
||||
|
||||
std::shared_ptr<Pattern> PatternRaw::realize()
|
||||
{
|
||||
return std::make_shared<Pattern>(m_regex, m_flags, m_lookbehind, m_greedy, m_alias, m_inside);
|
||||
}
|
||||
|
||||
const Grammar* Pattern::inside() const
|
||||
{
|
||||
if (m_inside)
|
||||
|
||||
+26
-8
@@ -29,17 +29,35 @@ private:
|
||||
size_t m_path;
|
||||
};
|
||||
|
||||
class PatternRaw
|
||||
{
|
||||
public:
|
||||
PatternRaw(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr<GrammarPtr> inside)
|
||||
: m_regex(std::string{pattern})
|
||||
, m_flags(flags)
|
||||
, m_lookbehind(lookbehind)
|
||||
, m_greedy(greedy)
|
||||
, m_alias(alias)
|
||||
, m_inside(inside)
|
||||
{
|
||||
}
|
||||
|
||||
std::shared_ptr<Pattern> realize();
|
||||
|
||||
private:
|
||||
std::string m_regex;
|
||||
boost::regex_constants::syntax_option_type m_flags;
|
||||
bool m_lookbehind;
|
||||
bool m_greedy;
|
||||
std::string m_alias;
|
||||
std::shared_ptr<GrammarPtr> m_inside;
|
||||
};
|
||||
|
||||
class Pattern
|
||||
{
|
||||
public:
|
||||
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, GrammarPtr inside)
|
||||
: Pattern(pattern, flags, lookbehind, greedy, alias, std::make_shared<GrammarPtr>(inside))
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind = false, bool greedy = false, std::string alias = "", std::shared_ptr<GrammarPtr> inside = nullptr)
|
||||
: m_regex(boost::regex(std::string{pattern}, flags | boost::regex_constants::optimize))
|
||||
Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr<GrammarPtr> inside)
|
||||
: m_regex(boost::regex(std::string{ pattern }, flags | boost::regex_constants::optimize))
|
||||
, m_lookbehind(lookbehind)
|
||||
, m_greedy(greedy)
|
||||
, m_alias(alias)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "LanguageTree.h"
|
||||
|
||||
#include "TokenList.h"
|
||||
#include "UnicodeEscapes.h"
|
||||
#include <boost/regex.hpp>
|
||||
|
||||
void LanguageTree::load(const std::string& content)
|
||||
@@ -96,6 +97,8 @@ void LanguageTree::parsePatterns(Buffer &buffer)
|
||||
{
|
||||
uint16_t count = freadUint16(buffer);
|
||||
|
||||
m_patterns.resize(count);
|
||||
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
std::string item = freadString(buffer);
|
||||
@@ -163,11 +166,11 @@ void LanguageTree::parsePatterns(Buffer &buffer)
|
||||
|
||||
if (inside != std::string::npos)
|
||||
{
|
||||
m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }, std::make_shared<GrammarPtr>(shared_from_this(), inside)));
|
||||
m_patternsRaw.push_back(std::make_shared<PatternRaw>(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, std::make_shared<GrammarPtr>(shared_from_this(), inside)));
|
||||
}
|
||||
else
|
||||
{
|
||||
m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }));
|
||||
m_patternsRaw.push_back(std::make_shared<PatternRaw>(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, nullptr));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,7 +178,16 @@ void LanguageTree::parsePatterns(Buffer &buffer)
|
||||
|
||||
const Pattern* LanguageTree::resolvePattern(size_t path)
|
||||
{
|
||||
return m_patterns[path].get();
|
||||
std::shared_ptr<Pattern> &parsed = m_patterns[path];
|
||||
if (parsed == nullptr)
|
||||
{
|
||||
parsed = m_patternsRaw[path]->realize();
|
||||
|
||||
m_patternsRaw[path] = nullptr;
|
||||
m_patterns[path] = parsed;
|
||||
}
|
||||
|
||||
return parsed.get();
|
||||
}
|
||||
|
||||
const Grammar* LanguageTree::resolveGrammar(size_t path)
|
||||
|
||||
@@ -38,6 +38,17 @@ public:
|
||||
return keys;
|
||||
}
|
||||
|
||||
std::string languageName(const std::string& language) const
|
||||
{
|
||||
const auto& find = m_languages.find(language);
|
||||
if (find != m_languages.end())
|
||||
{
|
||||
return find->second.first;
|
||||
}
|
||||
|
||||
return language;
|
||||
}
|
||||
|
||||
const Grammar* find(const std::string& key) const
|
||||
{
|
||||
const auto& value = m_languages.find(key);
|
||||
@@ -57,4 +68,5 @@ private:
|
||||
std::map<std::string, std::pair<std::string, size_t>> m_languages;
|
||||
std::vector<std::shared_ptr<Grammar>> m_grammars;
|
||||
std::vector<std::shared_ptr<Pattern>> m_patterns;
|
||||
std::vector<std::shared_ptr<PatternRaw>> m_patternsRaw;
|
||||
};
|
||||
@@ -24,6 +24,11 @@ std::map<std::string, std::string> SyntaxHighlighter::languages() const
|
||||
return m_tree->keys();
|
||||
}
|
||||
|
||||
std::string SyntaxHighlighter::languageName(const std::string& language) const
|
||||
{
|
||||
return m_tree->languageName(language);
|
||||
}
|
||||
|
||||
TokenList SyntaxHighlighter::tokenize(std::string_view text, const Grammar* grammar)
|
||||
{
|
||||
// Grammars may include themselves through "inside" (for example
|
||||
@@ -62,9 +67,9 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
|
||||
return;
|
||||
}
|
||||
|
||||
const auto& pattern = *j;
|
||||
const auto& inside = pattern->inside();
|
||||
const bool greedy = pattern->greedy();
|
||||
const auto& pattern = *j->get();
|
||||
const auto& inside = pattern.inside();
|
||||
const bool greedy = pattern.greedy();
|
||||
|
||||
size_t pos = startPos;
|
||||
|
||||
@@ -99,7 +104,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
|
||||
|
||||
if (greedy)
|
||||
{
|
||||
match = pattern->match(matchSuccess, matchIndex, text);
|
||||
match = pattern.match(matchSuccess, matchIndex, text);
|
||||
if (!matchSuccess || matchIndex >= text.length())
|
||||
{
|
||||
break;
|
||||
@@ -143,7 +148,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
|
||||
else
|
||||
{
|
||||
matchIndex = 0;
|
||||
match = pattern->match(matchSuccess, matchIndex, str);
|
||||
match = pattern.match(matchSuccess, matchIndex, str);
|
||||
if (!matchSuccess)
|
||||
{
|
||||
continue;
|
||||
@@ -183,7 +188,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
|
||||
|
||||
currentNode = tokenList.addAfter(removeFrom, token.name(),
|
||||
std::move(tokenEntries),
|
||||
pattern->alias(),
|
||||
pattern.alias(),
|
||||
match.size());
|
||||
|
||||
if (after.size())
|
||||
|
||||
@@ -25,6 +25,8 @@ public:
|
||||
|
||||
std::map<std::string, std::string> languages() const;
|
||||
|
||||
std::string languageName(const std::string& language) const;
|
||||
|
||||
private:
|
||||
TokenList tokenize(std::string_view text, const Grammar* grammar);
|
||||
void matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch);
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
namespace libprisma
|
||||
{
|
||||
// Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one
|
||||
// simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and
|
||||
// where one sits inside a character class - whose members have to be a single byte each -
|
||||
// lift it out into an alternation. This belongs here rather than in grammars.dat, which
|
||||
// keeps the \u spelling for the clients that match over UTF-16 and read it correctly.
|
||||
|
||||
inline bool isHexDigit(char c)
|
||||
{
|
||||
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
|
||||
}
|
||||
|
||||
inline int hexValue(char c)
|
||||
{
|
||||
return c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10;
|
||||
}
|
||||
|
||||
// the code point of the \uXXXX escape at begin, or -1 if there is not one there
|
||||
inline int unicodeEscapeAt(std::string_view pattern, size_t begin)
|
||||
{
|
||||
if (begin + 6 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'u')
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
int value = 0;
|
||||
for (size_t i = begin + 2; i < begin + 6; ++i)
|
||||
{
|
||||
if (!isHexDigit(pattern[i]))
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
value = value * 16 + hexValue(pattern[i]);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
inline void appendByte(std::string& out, unsigned char byte)
|
||||
{
|
||||
static const char digits[] = "0123456789ABCDEF";
|
||||
|
||||
out += "\\x";
|
||||
out += digits[byte >> 4];
|
||||
out += digits[byte & 0x0F];
|
||||
}
|
||||
|
||||
inline void appendUtf8(std::string& out, int codePoint)
|
||||
{
|
||||
if (codePoint < 0x80)
|
||||
{
|
||||
appendByte(out, static_cast<unsigned char>(codePoint));
|
||||
}
|
||||
else if (codePoint < 0x800)
|
||||
{
|
||||
appendByte(out, static_cast<unsigned char>(0xC0 | (codePoint >> 6)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
|
||||
}
|
||||
else if (codePoint < 0x10000)
|
||||
{
|
||||
appendByte(out, static_cast<unsigned char>(0xE0 | (codePoint >> 12)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 6) & 0x3F)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
|
||||
}
|
||||
else
|
||||
{
|
||||
appendByte(out, static_cast<unsigned char>(0xF0 | (codePoint >> 18)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 12) & 0x3F)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | ((codePoint >> 6) & 0x3F)));
|
||||
appendByte(out, static_cast<unsigned char>(0x80 | (codePoint & 0x3F)));
|
||||
}
|
||||
}
|
||||
|
||||
// The code point at begin, joining a surrogate pair into the one character it spells, with
|
||||
// the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it
|
||||
// to UTF-8 itself; a pair it would expand to two three byte sequences instead.
|
||||
inline int codePointAt(std::string_view pattern, size_t begin, size_t& length)
|
||||
{
|
||||
const int value = unicodeEscapeAt(pattern, begin);
|
||||
length = value < 0 ? 0 : 6;
|
||||
|
||||
if (value >= 0xD800 && value <= 0xDBFF)
|
||||
{
|
||||
const int low = unicodeEscapeAt(pattern, begin + 6);
|
||||
if (low >= 0xDC00 && low <= 0xDFFF)
|
||||
{
|
||||
length = 12;
|
||||
return 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00);
|
||||
}
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
struct ClassMember
|
||||
{
|
||||
std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z"
|
||||
int low = -1; // code point, for a member that is a lone \u escape
|
||||
int high = -1; // and the other end, when the member is a range of them
|
||||
};
|
||||
|
||||
struct CharacterClass
|
||||
{
|
||||
size_t end = 0; // index just past the closing ]
|
||||
bool negated = false;
|
||||
bool wide = false; // holds a code point that cannot be a single byte
|
||||
std::vector<ClassMember> members;
|
||||
};
|
||||
|
||||
// one class member starting at begin, or 0 if the class ends there
|
||||
inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member)
|
||||
{
|
||||
auto unit = [&](size_t at, int& codePoint) -> size_t {
|
||||
size_t length = 0;
|
||||
codePoint = codePointAt(pattern, at, length);
|
||||
if (codePoint >= 0)
|
||||
{
|
||||
return length;
|
||||
}
|
||||
return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1;
|
||||
};
|
||||
|
||||
int low = -1;
|
||||
size_t length = unit(begin, low);
|
||||
size_t at = begin + length;
|
||||
|
||||
// a - b, where a trailing - is a literal rather than the start of a range
|
||||
if (at + 1 < pattern.size() && pattern[at] == '-' && pattern[at + 1] != ']')
|
||||
{
|
||||
int high = -1;
|
||||
size_t tail = unit(at + 1, high);
|
||||
member.low = low;
|
||||
member.high = high;
|
||||
member.text = pattern.substr(begin, length + 1 + tail);
|
||||
return length + 1 + tail;
|
||||
}
|
||||
|
||||
member.low = low;
|
||||
member.text = pattern.substr(begin, length);
|
||||
return length;
|
||||
}
|
||||
|
||||
inline bool readClass(std::string_view pattern, size_t begin, CharacterClass& result)
|
||||
{
|
||||
size_t at = begin + 1;
|
||||
if (at < pattern.size() && pattern[at] == '^')
|
||||
{
|
||||
result.negated = true;
|
||||
++at;
|
||||
}
|
||||
|
||||
while (at < pattern.size() && pattern[at] != ']')
|
||||
{
|
||||
ClassMember member;
|
||||
size_t length = readMember(pattern, at, member);
|
||||
if (length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (member.low > 0x7F || member.high > 0x7F)
|
||||
{
|
||||
result.wide = true;
|
||||
}
|
||||
|
||||
result.members.push_back(member);
|
||||
at += length;
|
||||
}
|
||||
|
||||
if (at >= pattern.size())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
result.end = at + 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
inline bool isRange(const CharacterClass& characters, int low, int high)
|
||||
{
|
||||
return characters.members.size() == 1
|
||||
&& characters.members[0].low == low
|
||||
&& characters.members[0].high == high;
|
||||
}
|
||||
|
||||
// [\uD800-\uDBFF][\uDC00-\uDFFF] and [\uD800-\uDFFF]{2} are how a JavaScript regex spells
|
||||
// one astral character, as its two UTF-16 halves. In UTF-8 that is one four byte sequence.
|
||||
inline size_t appendSurrogatePair(std::string& out, std::string_view pattern, size_t begin)
|
||||
{
|
||||
CharacterClass lead;
|
||||
if (!readClass(pattern, begin, lead) || lead.negated)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t end = 0;
|
||||
if (isRange(lead, 0xD800, 0xDBFF))
|
||||
{
|
||||
CharacterClass trail;
|
||||
if (!readClass(pattern, lead.end, trail) || trail.negated || !isRange(trail, 0xDC00, 0xDFFF))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
end = trail.end;
|
||||
}
|
||||
else if (isRange(lead, 0xD800, 0xDFFF) && pattern.substr(lead.end, 3) == "{2}")
|
||||
{
|
||||
end = lead.end + 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})";
|
||||
return end - begin;
|
||||
}
|
||||
|
||||
// the class at begin, with any member that cannot be a single byte moved out of it
|
||||
inline size_t appendClass(std::string& out, std::string_view pattern, size_t begin)
|
||||
{
|
||||
CharacterClass characters;
|
||||
if (!readClass(pattern, begin, characters) || !characters.wide)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::vector<int> wide;
|
||||
std::string narrow;
|
||||
|
||||
for (const auto& member : characters.members)
|
||||
{
|
||||
if (member.low <= 0x7F && member.high <= 0x7F)
|
||||
{
|
||||
narrow += member.text;
|
||||
}
|
||||
else if (member.high < 0)
|
||||
{
|
||||
wide.push_back(member.low);
|
||||
}
|
||||
else if (!characters.negated)
|
||||
{
|
||||
// a range of code points that is not the surrogate idiom; leave it be rather
|
||||
// than expand what could be the whole of Unicode
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// a negated class drops its wide members: they exclude characters that cannot occur
|
||||
// as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones
|
||||
if (characters.negated || wide.empty())
|
||||
{
|
||||
out += '[';
|
||||
if (characters.negated)
|
||||
{
|
||||
out += '^';
|
||||
}
|
||||
out += narrow;
|
||||
out += ']';
|
||||
return characters.end - begin;
|
||||
}
|
||||
|
||||
out += "(?:";
|
||||
if (!narrow.empty())
|
||||
{
|
||||
out += '[';
|
||||
out += narrow;
|
||||
out += ']';
|
||||
out += '|';
|
||||
}
|
||||
for (size_t i = 0; i < wide.size(); ++i)
|
||||
{
|
||||
if (i > 0)
|
||||
{
|
||||
out += '|';
|
||||
}
|
||||
appendUtf8(out, wide[i]);
|
||||
}
|
||||
out += ')';
|
||||
|
||||
return characters.end - begin;
|
||||
}
|
||||
|
||||
inline std::string widenUnicodeClasses(std::string_view pattern)
|
||||
{
|
||||
if (pattern.find("\\u") == std::string_view::npos)
|
||||
{
|
||||
return std::string{ pattern };
|
||||
}
|
||||
|
||||
std::string out;
|
||||
out.reserve(pattern.size());
|
||||
|
||||
for (size_t i = 0; i < pattern.size(); )
|
||||
{
|
||||
if (pattern[i] == '\\')
|
||||
{
|
||||
size_t length = 0;
|
||||
const int codePoint = codePointAt(pattern, i, length);
|
||||
|
||||
if (codePoint >= 0)
|
||||
{
|
||||
// grouped, because a quantifier after it has to apply to the whole
|
||||
// character and not to the last byte of its encoding
|
||||
const bool multiByte = codePoint > 0x7F;
|
||||
if (multiByte)
|
||||
{
|
||||
out += "(?:";
|
||||
}
|
||||
appendUtf8(out, codePoint);
|
||||
if (multiByte)
|
||||
{
|
||||
out += ')';
|
||||
}
|
||||
|
||||
i += length;
|
||||
continue;
|
||||
}
|
||||
|
||||
out += pattern.substr(i, 2);
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pattern[i] == '[')
|
||||
{
|
||||
if (size_t consumed = appendSurrogatePair(out, pattern, i))
|
||||
{
|
||||
i += consumed;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (size_t consumed = appendClass(out, pattern, i))
|
||||
{
|
||||
i += consumed;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
out += pattern[i];
|
||||
++i;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
}
|
||||
Generated
BIN
Binary file not shown.
Generated
+11
-1
@@ -9,13 +9,23 @@
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"lodash.isequal": "^4.5.0"
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"prismjs": "1.29.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.isequal": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
|
||||
"integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="
|
||||
},
|
||||
"node_modules/prismjs": {
|
||||
"version": "1.29.0",
|
||||
"resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
|
||||
"integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -17,6 +17,7 @@
|
||||
},
|
||||
"homepage": "https://github.com/TelegramMessenger/libprisma#readme",
|
||||
"dependencies": {
|
||||
"lodash.isequal": "^4.5.0"
|
||||
"lodash.isequal": "^4.5.0",
|
||||
"prismjs": "1.29.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
|
||||
* matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
|
||||
* the one thing it promises: that a pattern means the same to every client that reads it.
|
||||
*
|
||||
* The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
|
||||
* anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
|
||||
* looking for.
|
||||
*/
|
||||
public class PatternProbe {
|
||||
|
||||
/** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
|
||||
private static final class Entry {
|
||||
String source;
|
||||
int flags;
|
||||
}
|
||||
|
||||
private static final class Reader {
|
||||
private final byte[] bytes;
|
||||
private int offset;
|
||||
|
||||
Reader(byte[] bytes) {
|
||||
this.bytes = bytes;
|
||||
}
|
||||
|
||||
int uint8() {
|
||||
return bytes[offset++] & 0xFF;
|
||||
}
|
||||
|
||||
int uint16() {
|
||||
return uint8() | (uint8() << 8);
|
||||
}
|
||||
|
||||
String string() {
|
||||
int length = uint8();
|
||||
if (length >= 254) {
|
||||
length = uint8() | (uint8() << 8) | (uint8() << 16);
|
||||
}
|
||||
String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
|
||||
offset += length;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Entry> read(String path) throws IOException {
|
||||
Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
|
||||
List<Entry> entries = new ArrayList<>();
|
||||
|
||||
for (int count = reader.uint16(); count > 0; count--) {
|
||||
String item = reader.string();
|
||||
int begin = item.indexOf('/');
|
||||
int end = item.lastIndexOf('/');
|
||||
if (begin < 0 || end <= begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String options = item.substring(end + 1);
|
||||
int aliasBegin = options.indexOf(',');
|
||||
|
||||
Entry entry = new Entry();
|
||||
entry.source = item.substring(begin + 1, end);
|
||||
for (int i = 0; i < aliasBegin; ++i) {
|
||||
switch (options.charAt(i)) {
|
||||
case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
|
||||
case 'm': entry.flags |= Pattern.MULTILINE; break;
|
||||
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
|
||||
}
|
||||
}
|
||||
entries.add(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
|
||||
* line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
|
||||
* A Java client has to do this; it cannot go in the table, where \z would read as a z.
|
||||
*/
|
||||
private static String endOfInput(String source) {
|
||||
if (source.indexOf('$') < 0) {
|
||||
return source;
|
||||
}
|
||||
|
||||
StringBuilder result = new StringBuilder(source.length());
|
||||
boolean inClass = false;
|
||||
for (int i = 0; i < source.length(); ++i) {
|
||||
char c = source.charAt(i);
|
||||
if (c == '\\' && i + 1 < source.length()) {
|
||||
result.append(c).append(source.charAt(++i));
|
||||
} else if (c == '[') {
|
||||
inClass = true;
|
||||
result.append(c);
|
||||
} else if (c == ']') {
|
||||
inClass = false;
|
||||
result.append(c);
|
||||
} else if (c == '$' && !inClass) {
|
||||
result.append("\\z");
|
||||
} else {
|
||||
result.append(c);
|
||||
}
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
List<Entry> entries = read(args[0]);
|
||||
String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
|
||||
.replace(String.valueOf((char) 13), "");
|
||||
|
||||
StringBuilder out = new StringBuilder();
|
||||
int failed = 0;
|
||||
|
||||
for (int i = 0; i < entries.size(); ++i) {
|
||||
Entry entry = entries.get(i);
|
||||
String source = entry.source;
|
||||
if ((entry.flags & Pattern.MULTILINE) == 0) {
|
||||
source = endOfInput(source);
|
||||
}
|
||||
|
||||
Pattern pattern;
|
||||
try {
|
||||
pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
|
||||
} catch (PatternSyntaxException e) {
|
||||
if (failed++ < 20) {
|
||||
System.err.println("pattern " + i + ": " + e.getDescription());
|
||||
System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length())));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out.append(i);
|
||||
java.util.regex.Matcher matcher = pattern.matcher(corpus);
|
||||
int at = 0;
|
||||
while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
|
||||
out.append(' ').append(matcher.start()).append(':').append(matcher.end());
|
||||
at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
|
||||
if (at > corpus.length()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out.append('\n');
|
||||
}
|
||||
|
||||
Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
|
||||
System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
|
||||
|
||||
if (failed > 0) {
|
||||
System.err.println(failed + " patterns java.util.regex will not take. "
|
||||
+ "Teach normalizeEscapes in generate.js to rewrite them.");
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Every escape in grammars.dat has to mean the same thing to every engine that reads it.
|
||||
// A pattern that fails to compile is caught by the probes beside this script; this one is for
|
||||
// the escapes that compile everywhere and quietly mean something else, which is the worse
|
||||
// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline
|
||||
// included, by java.util.regex, which made ini values run over the end of the line.
|
||||
const fs = require('fs')
|
||||
|
||||
// what generate.js is expected to emit, once normalizeEscapes has run over it
|
||||
const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split(''))
|
||||
|
||||
const REASONS = {
|
||||
v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B',
|
||||
h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere',
|
||||
H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere',
|
||||
V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere',
|
||||
R: 'any line break in java.util.regex, a literal R elsewhere',
|
||||
X: 'a grapheme cluster in java.util.regex, a literal X elsewhere',
|
||||
A: 'start of input in java.util.regex, a literal A elsewhere',
|
||||
Z: 'end of input in java.util.regex, a literal Z elsewhere',
|
||||
z: 'end of input in java.util.regex, a literal z elsewhere',
|
||||
G: 'end of the previous match in java.util.regex, a literal G elsewhere',
|
||||
Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere',
|
||||
E: 'ends one in java.util.regex, a literal E elsewhere',
|
||||
p: 'a unicode property in both, but the property names differ',
|
||||
N: 'a named character in java.util.regex',
|
||||
0: 'starts an octal escape in java.util.regex - emit \\x00',
|
||||
}
|
||||
|
||||
function readPatterns(path) {
|
||||
const buffer = fs.readFileSync(path)
|
||||
let offset = 0
|
||||
|
||||
const uint8 = () => buffer[offset++]
|
||||
const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value }
|
||||
const string = () => {
|
||||
let length = uint8()
|
||||
if (length >= 254) {
|
||||
length = uint8() | (uint8() << 8) | (uint8() << 16)
|
||||
}
|
||||
const value = buffer.toString('latin1', offset, offset + length)
|
||||
offset += length
|
||||
return value
|
||||
}
|
||||
|
||||
const patterns = []
|
||||
for (let count = uint16(); count > 0; count--) {
|
||||
patterns.push(string())
|
||||
}
|
||||
return patterns
|
||||
}
|
||||
|
||||
const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat')
|
||||
const problems = []
|
||||
|
||||
for (const item of patterns) {
|
||||
const end = item.lastIndexOf('/')
|
||||
const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end)
|
||||
|
||||
for (let i = 0; i < source.length; i++) {
|
||||
if (source[i] !== '\\') {
|
||||
continue
|
||||
}
|
||||
|
||||
const escaped = source[++i]
|
||||
if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) {
|
||||
problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set'])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`${patterns.length} patterns checked`)
|
||||
|
||||
if (problems.length === 0) {
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const seen = new Set()
|
||||
for (const [source, escape, reason] of problems) {
|
||||
if (seen.has(escape)) {
|
||||
continue
|
||||
}
|
||||
seen.add(escape)
|
||||
console.error(`\n${escape} is ${reason}`)
|
||||
console.error(` e.g. ${source.slice(0, 120)}`)
|
||||
}
|
||||
console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`)
|
||||
process.exit(1)
|
||||
@@ -0,0 +1,13 @@
|
||||
class Foo<T> where T : IBar { public int x = 1; } /* c */ // line
|
||||
<!DOCTYPE html><a href="x" [attr]='y'>{{ tpl }}</a>
|
||||
def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py
|
||||
let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3;
|
||||
SELECT * FROM t WHERE a IN (1,2) -- sql
|
||||
#include <stdio.h>
|
||||
key = value ; ini
|
||||
[section]
|
||||
- item:
|
||||
nested: [1, 2]
|
||||
0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y
|
||||
tab and trailing spaces
|
||||
#!/bin/sh
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down
|
||||
// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and
|
||||
// the two files are diffed, which is what holds the table to meaning one thing everywhere.
|
||||
//
|
||||
// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so
|
||||
// anything above U+007F diverges on offsets alone and would drown out the escape bugs.
|
||||
#include <boost/regex.hpp>
|
||||
|
||||
#include "../libprisma/UnicodeEscapes.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string readAll(const char* path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
return std::string((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
struct Reader {
|
||||
const std::string& bytes;
|
||||
size_t offset = 0;
|
||||
|
||||
uint8_t uint8() {
|
||||
return offset < bytes.size() ? static_cast<uint8_t>(bytes[offset++]) : 0;
|
||||
}
|
||||
|
||||
uint16_t uint16() {
|
||||
uint16_t low = uint8();
|
||||
return static_cast<uint16_t>(low | (uint8() << 8));
|
||||
}
|
||||
|
||||
std::string string() {
|
||||
size_t length = uint8();
|
||||
if (length >= 254) {
|
||||
size_t a = uint8(), b = uint8(), c = uint8();
|
||||
length = a | (b << 8) | (c << 16);
|
||||
}
|
||||
std::string value = bytes.substr(offset, length);
|
||||
offset += length;
|
||||
return value;
|
||||
}
|
||||
};
|
||||
|
||||
struct Entry {
|
||||
std::string source;
|
||||
boost::regex_constants::syntax_option_type flags;
|
||||
};
|
||||
|
||||
// "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads
|
||||
std::vector<Entry> read(const std::string& table) {
|
||||
Reader reader{ table };
|
||||
std::vector<Entry> entries;
|
||||
|
||||
for (uint16_t count = reader.uint16(); count > 0; --count) {
|
||||
const std::string item = reader.string();
|
||||
const size_t begin = item.find_first_of('/');
|
||||
const size_t end = item.find_last_of('/');
|
||||
if (begin == std::string::npos || end <= begin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string options = item.substr(end + 1);
|
||||
const size_t aliasBegin = options.find_first_of(',');
|
||||
|
||||
Entry entry;
|
||||
entry.source = item.substr(begin + 1, end - (begin + 1));
|
||||
entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m;
|
||||
|
||||
for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) {
|
||||
switch (options[i]) {
|
||||
case 'i': entry.flags |= boost::regex_constants::icase; break;
|
||||
case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break;
|
||||
default: break; // l and y are libprisma's, for Prism lookbehind and greedy
|
||||
}
|
||||
}
|
||||
|
||||
entries.push_back(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
if (argc < 4) {
|
||||
fprintf(stderr, "usage: probe <grammars.dat> <corpus> <out>\n");
|
||||
return 2;
|
||||
}
|
||||
|
||||
const std::vector<Entry> entries = read(readAll(argv[1]));
|
||||
|
||||
std::string corpus = readAll(argv[2]);
|
||||
corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end());
|
||||
|
||||
std::ofstream out(argv[3], std::ios::binary);
|
||||
int failed = 0;
|
||||
|
||||
for (size_t i = 0; i < entries.size(); ++i) {
|
||||
boost::regex re;
|
||||
try {
|
||||
re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags);
|
||||
} catch (const std::exception& e) {
|
||||
if (failed++ < 20) {
|
||||
fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out << i;
|
||||
auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re,
|
||||
boost::regex_constants::match_not_dot_newline);
|
||||
for (auto end = boost::sregex_iterator(); it != end; ++it) {
|
||||
out << ' ' << it->position() << ':' << (it->position() + it->length());
|
||||
}
|
||||
out << '\n';
|
||||
}
|
||||
|
||||
printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size());
|
||||
|
||||
if (failed > 0) {
|
||||
fprintf(stderr, "%d patterns Boost will not take.\n", failed);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user