diff --git a/.gitattributes b/.gitattributes index 63081ae..3b5a7e3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -62,3 +62,4 @@ #*.rtf diff=astextplain #*.RTF diff=astextplain libprisma/grammars.json eol=lf +libprisma/grammars.dat linguist-generated diff --git a/.github/workflows/build-grammars.yml b/.github/workflows/build-grammars.yml index ec56456..e3784a9 100644 --- a/.github/workflows/build-grammars.yml +++ b/.github/workflows/build-grammars.yml @@ -1,27 +1,37 @@ name: Update grammars -on: [ workflow_dispatch ] +on: + workflow_dispatch: + push: + branches: [ master ] + # grammars.dat itself is deliberately not listed: regenerating it must not trigger another run. + paths: + - components/** + - generate.js + - package.json + - package-lock.json + +permissions: + contents: write + pull-requests: write jobs: build: runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 with: - node-version: 18 + node-version: 22 - run: npm install - run: npm start - name: Upload a Build Artifact - uses: actions/upload-artifact@v3.1.3 + uses: actions/upload-artifact@v7 with: name: grammars.dat path: ./libprisma/grammars.dat - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v8 with: commit-message: Update grammars title: Update grammars diff --git a/.github/workflows/check-generated.yml b/.github/workflows/check-generated.yml new file mode 100644 index 0000000..9e2dad0 --- /dev/null +++ b/.github/workflows/check-generated.yml @@ -0,0 +1,22 @@ +name: Check generated files + +# The path filter is the whole test: the job only exists when the pull request +# touches grammars.dat, so reaching it is already the failure. +on: + pull_request: + paths: + - libprisma/grammars.dat + +permissions: {} + +jobs: + grammars: + runs-on: ubuntu-latest + # A pull request opened with GITHUB_TOKEN cannot trigger this workflow at all, + # so the guard only matters if "Update grammars" is ever moved to a PAT. + if: github.event.pull_request.user.login != 'github-actions[bot]' + steps: + - name: Reject changes to the generated grammar table + run: | + echo "::error file=libprisma/grammars.dat::libprisma/grammars.dat is generated by generate.js and is regenerated on master by the Update grammars workflow. Please drop it from this pull request and keep only the components/ and generate.js changes." + exit 1 diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml new file mode 100644 index 0000000..3e7a4a2 --- /dev/null +++ b/.github/workflows/check-grammars.yml @@ -0,0 +1,58 @@ +name: Check grammars + +# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex, +# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it +# to that, rather than waiting for a language to quietly stop highlighting. +on: + pull_request: + paths: + - components/** + - generate.js + - libprisma/** + - tools/** + - package.json + - package-lock.json + workflow_dispatch: + +permissions: {} + +jobs: + patterns: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: actions/setup-node@v5 + with: + node-version: 22 + + # the committed table is regenerated on master, so a pull request is checked against what + # its own components and generate.js produce + - run: npm ci --ignore-scripts + - run: npm start + + - name: Escapes every engine agrees on + run: node tools/check-escapes.js libprisma/grammars.dat + + - name: Compile every pattern with java.util.regex + run: | + javac -d tools tools/PatternProbe.java + java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt + + - name: Compile every pattern with Boost + run: | + sudo apt-get update + sudo apt-get install -y libboost-regex-dev + g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex + ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt + + - name: Compare what the two engines matched + run: | + if ! diff -u boost.txt java.txt > engines.diff; then + echo "The two engines disagree about what these patterns match." + echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt." + head -40 engines.diff + exit 1 + fi + echo "Boost and java.util.regex agree on every pattern." diff --git a/README.md b/README.md index f5e307f..8fb403c 100644 --- a/README.md +++ b/README.md @@ -129,10 +129,12 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`. |`F#`|`fsharp`| |`Factor`|`factor`| |`False`|`false`| +|`Fift`|`fift`| |`Firestore security rules`|`firestore-security-rules`| |`Flow`|`flow`| |`Fortran`|`fortran`| |`FreeMarker Template Language`|`ftl`| +|`FunC`|`func`| |`GameMaker Language`|`gml`,`gamemakerlanguage`| |`GAP (CAS)`|`gap`| |`G-code`|`gcode`| @@ -290,6 +292,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`. |`SuperCollider`|`supercollider`,`sclang`| |`Swift`|`swift`| |`Systemd configuration file`|`systemd`| +|`Tact`|`tact`| |`T4 templating`|`t4-templating`| |`T4 Text Templates (C#)`|`t4-cs`,`t4`| |`VB.Net`|`vbnet`| @@ -299,6 +302,8 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`. |`Template Toolkit 2`|`tt2`| |`TOML`|`toml`| |`Tremor`|`tremor`,`tremor`,`trickle`,`troy`| +|`Type Language`|`tl`| +|`Type Language - Binary`|`tlb`| |`TypoScript`|`typoscript`,`tsconfig`| |`UnrealScript`|`unrealscript`,`unrealscript`,`uscript`,`uc`| |`UO Razor Script`|`uorazor`| diff --git a/components/prism-fift.js b/components/prism-fift.js new file mode 100644 index 0000000..44f3376 --- /dev/null +++ b/components/prism-fift.js @@ -0,0 +1,127 @@ +/** + * @file Prism.js definition for Fift + * @link https://docs.ton.org/develop/fift/overview + * @author Nikita Sobolev (https://github.com/sobolevn) + * @license MIT + */ +Prism.languages.fift = { + 'symbol': [ + /[xX]\{[0-9a-fA-F_]*\}/, + /[bB]\{[01]*\}/, + /'\S+/, + ], + 'string': /"([^"\r\n\\]|\\.)*"/, + + 'comment': [ + { + pattern: /\/\*[\s\S]*?(?:\*\/|$)/, + lookbehind: true, + greedy: true, + }, + { + pattern: /\/\/.*/, + lookbehind: true, + greedy: true, + }, + ], + + 'operator': [ + // Full list can be found in + // Appendix A. List of Fift words + // Ordered the same way source code does this. + // Except: shorter words must follow + // longer ones with the same base part. + // Example: `#` comes after `#s` + /#>/, /#s/, /\$#/, /#/, + + /\$\+/, /\$,/, /\$\d/, /\$=/, /\$(?=\()/, + /\$>smca/, /\$>s/, + /\$@\+/, /\$@\?\+/, /\$@\?/, /\$@/, + /\$cmp/, /\$len/, /\$pos/, /\$reverse/, + + /%1<>c/, /\*>>r/, /\*>>/, /\*mod/, /\*/, + + /\+!/, /\+/, /,/, /-!/, /-/, /!/, + /-1</, /0=/, /0>/, + /1\+!/, /1\+/, /1-!/, /1-/, /1</, /=/, />>c/, />>r/, />>/, />/, + + /\?dup/, + + /@'/, /@/, + + /B\+/, /B,/, /B=/, /B>Li@\+/, /B>Li@/, /B>Lu@\+/, /B>Lu@/, /B>boc/, + /B>file/, /B>i@\+/, /B>i@/, /B>u@\+/, /B>u@/, + /B@\?\+/, /B@\+/, /B@\?/, /B@/, + /Bcmp/, /BhashB/, /Bhashu/, /Bhash/, /Blen/, /Bx\./, + /B\|/, /Li>B/, /Lu>B/, + + /\[\]/, /\[compile\]/, /\[/, /\]/, + + /atom\?/, + + /b\+/, /b\._/, /b\./, + /b>idict!\+/, /b>idict!/, /b>sdict!\+/, /b>sdict!/, + /b>udict!\+/, /b>udict!/, + /b>/, /boc\+>B/, /boc>B/, + + /csr\./, + /def\?/, + /empty\?/, /eq\?/, + /file-exists\?/, /file>B/, + + /i,/, /i>B/, /i@\+/, /i@\?\+/, /i@\?/, /i@/, + /idict!\+/, /idict!/, /idict-/, /idict@-/, /idict@/, + + /null!/, /null\?/, + /pfxdict!\+/, /pfxdict!/, /pfxdict@/, /priv>pub/, + /ref@\+/, /ref@/, /ref@\?\+/, /ref@\?/, + + /s,/, /s>c/, /s>/, + /sdict!\+/, /sdict!/, /sdict-/, /sdict@-/, /sdict@/, + /smca>\$/, /sr,/, + + /tuple\?/, + + /u,/, /u>B/, /u@\+/, /u@\?\+/, /u@\?/, + /udict!\+/, /udict!/, /udict-/, /udict@-/, /udict@/, + /undef\?/, + + /x\._/, /x\./, + + /\|\+/, /\|/, /\|_/, + + /\?\./, /'/, + + // Should be the last: + /\./, + ], + + 'keyword': /\b(?:-roll|-rot|-trailing|-trailing0|2constant|2drop|2dup|2over|2swap|abort|abs|allot|and|anon|atom|bbitrefs|bbits|bl|box|brefs|brembitrefs|brembits|bremrefs|bye|caddr|cadr|car|cddr|cdr|char|chr|cmp|cond|cons|constant|count|cr|create|depth|dictmap|dictmerge|dictnew|does|drop|dup|ed25519_chksign|ed25519_sign|ed25519_sign_uint|emit|exch|exch2|execute|explode|find|first|fits|forget|gasrunvm|gasrunvmcode|gasrunvmctx|gasrunvmdict|halt|hash|hashB|hashu|hold|hole|if|ifnot|include|list|max|min|minmax|mod|negate|newkeypair|nil|nip|nop|not|now|null|or|over|pair|pick|quit|remaining|reverse|roll|rot|runvm|runvmcode|runvmctx|runvmdict|sbitrefs|sbits|second|sgn|shash|sign|single|skipspc|space|srefs|swap|ten|third|times|triple|tuck|tuple|type|ufits|uncons|unpair|unsingle|until|untriple|untuple|variable|while|word|words|xor)\b/, + 'boolean': /\b(?:false|true)\b/, + + 'number': [ + /(0[xX][0-9a-fA-F]+)/, + /(0[bB][01]+)/, + /(-?\d+(\/-?\d+)?)/, + ], + 'variable': /[\w$-]+/, + + 'punctuation': /[\[\{\}\],\(\)]/, +}; \ No newline at end of file diff --git a/components/prism-func.js b/components/prism-func.js new file mode 100644 index 0000000..4dd2dc8 --- /dev/null +++ b/components/prism-func.js @@ -0,0 +1,148 @@ +/** + * @file Prism.js definition for FunC + * @link https://docs.ton.org/develop/func/overview + * @version 0.4.4 + * @author Novus Nota (https://github.com/novusnota) + * @author Nikita Sobolev (https://github.com/sobolevn) + * @license MIT + */ +(function(Prism) { + /** @type {Prism.GrammarValue} */ + var number = /-?\b(?:\d+|0x[\da-fA-F]+)\b(?![^\.~,;\[\]\(\)\s])/; + + /** @type {Prism.GrammarValue} */ + var string = [ + { // multi-line + pattern: /"""[\s\S]*?"""[Hhcusa]?/, + greedy: true, + inside: { + // string type + 'symbol': { + pattern: /("""[\s\S]*?""")[Hhcusa]/, + lookbehind: true, + greedy: true, + } + }, + }, + { // single-line + pattern: /"[^\n"]*"[Hhcusa]?/, + greedy: true, + inside: { + // string type + 'symbol': { + pattern: /("[^\n"]*")[Hhcusa]/, + lookbehind: true, + greedy: true, + } + }, + }, + ]; + + /** @type {RegExp} */ + var operator_body = /(?:!=|\?|:|%=|%|&=|&|\*=|\*|\+=|\+|->|-=|-|\/%|\/=|\/|<=>|<<=|<<|<=|<|==|=|>>=|>>|>=|>|\^>>=|\^>>|\^=|\^\/=|\^\/|\^%=|\^%|\^|\|=|\||~>>=|~>>|~\/=|~\/|~%|~)/; + + // An operator only counts as one when whitespace follows it, as identifiers may contain these characters. + /** @type {Prism.GrammarValue} */ + var operator = new RegExp(operator_body.source + /(?=\s)/.source); + + /** @type {RegExp[]} */ + var var_identifier = [ + // quoted + /`[^`\n]+`/, + // plain + /[^\.~,;\[\]\(\)\s]+/ + ]; + + /** Prism.js definition for FunC */ + Prism.languages.func = { + 'comment': [ + { // single-line + pattern: /;;.*/, + lookbehind: true, + greedy: true, + }, + { // multi-line, not nested (TODO: nesting) + // . isn't used, because it only applies to the single line + pattern: /\{-[\s\S]*?(?:-\}|$)/, + lookbehind: true, + greedy: true, + }, + { + // unused variable identifier + pattern: /\b_\b(?![^\.~,;\[\]\(\)\s])/ + }, + ], + + // Custom token for #pragma's + 'pragma': { + pattern: /#pragma(.*);/, + inside: { + 'keyword': /(?:#pragma|test-version-set|not-version|version|allow-post-modification|compute-asm-ltr)\b/, + 'number': /(?:\d+)(?:.\d+)?(?:.\d+)?/, + 'operator': /=|\^|<=|>=|<|>/, + 'string': string, + 'punctuation': /;/, + } + }, + + // Custom token for #include's + 'include': { + pattern: /#include(.*);/, + inside: { + 'keyword': /#include\b/, + 'string': string, + 'punctuation': /;/, + } + }, + + // builtin types + 'builtin': /\b(?:int|cell|slice|builder|cont|tuple|type)\b/, + + // builtin constants + 'boolean': /\b(?:false|true|nil|Nil)\b/, + + 'keyword': /\b(?:forall|extern|global|asm|impure|inline_ref|inline|auto_apply|method_id|operator|infixl|infixr|infix|const|return|var|repeat|do|while|until|try|catch|ifnot|if|then|elseifnot|elseif|else)\b/, + + 'number': number, + + 'string': string, + + 'operator': operator, + + 'punctuation': [/[\.,;\(\)\[\]]/, /[\{\}](?![^\.~,;\[\]\(\)\s])/], + + // Function and method names in declarations, definitions and calls + 'function': [ + { // quoted + pattern: new RegExp(var_identifier[0].source + /(?=\s*\()/.source), + greedy: true, + }, + { // bitwise not operator + pattern: /~_/, + greedy: true, + }, + { // remaining operators + // operator_body, not operator: its trailing lookahead would forbid the closing _ + pattern: new RegExp(/\^?_/.source + operator_body.source + /_/.source), + greedy: true, + }, + { // plain function or method name + pattern: new RegExp(/[\.~]?/.source + var_identifier[1].source + /(?=\s*\()/.source), + greedy: true, + }, + { // builtin functions and methods + pattern: /\b(?:divmod|moddiv|muldiv|muldivr|muldivc|muldivmod|null\?|throw|throw_if|throw_unless|throw_arg|throw_arg_if|throw_arg_unless|load_int|load_uint|preload_int|preload_uint|store_int|store_uint|load_bits|preload_bits|int_at|cell_at|slice_at|tuple_at|at|touch|touch2|run_method0|run_method1|run_method2|run_method3|~divmod|~moddiv|~store_int|~store_uint|~touch|~touch2|~dump|~stdump)\b/, + greedy: true, + }, + ], + + // Parametric polymorphism + 'class-name': /[A-Z][^\.~,;\[\]\(\)\s]*/, + + // All the rest of identifiers + 'variable': var_identifier.map(x => { return { pattern: x, greedy: true } }), + }; + + // Adding a fc alias + Prism.languages.fc = Prism.languages.func; +}(Prism)); diff --git a/components/prism-tact.js b/components/prism-tact.js new file mode 100644 index 0000000..ab21fcf --- /dev/null +++ b/components/prism-tact.js @@ -0,0 +1,171 @@ +/** + * @file Prism.js definition for Tact + * @link https://tact-lang.org + * @version 1.6.0 + * @author Novus Nota (https://github.com/novusnota) + * @license MIT + */ +(function(Prism) { + Prism.languages.tact = { + // reserved keywords + 'keyword': [ + { + pattern: /\b(?:abstract|asm|as|catch|codeOf|const|contract(?!:)|do|else|extend|extends|foreach|fun|get|if|in|import|initOf|inline|let|message(?!:)|mutates|native|override|primitive|public|repeat|return|self|struct(?!:)|trait(?!:)|try|until|virtual|while|with)\b/, + }, + { // reserved function names + pattern: /\b(?:bounced|external|init|receive)\b(?=\()/ + }, + ], + + // built-in types + 'builtin': [ + { + pattern: /\b(?:Address|Bool|Builder|Cell|Int|Slice|String|StringBuilder)\b/, + }, + { // keyword after as, see: https://prismjs.com/extending.html#object-notation + pattern: /(\bas\s+)(?:coins|varu?int(?:32|16)|remaining|bytes(?:32|64)|int257|u?int(?:25[0-6]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?))\b/, + lookbehind: true, + greedy: true, + }, + ], + + // SCREAMING_SNAKE_CASE for null values and names of constants + 'constant': [ + { + pattern: /\bnull\b/, + }, + { + pattern: /\b[A-Z][A-Z0-9_]*\b/, + }, + ], + + // UpperCamelCase for names of contracts, traits, structs, messages + 'class-name': { + pattern: /\b[A-Z]\w*\b/, + }, + + // mappings to FunC + 'attribute': [ + { // functions + pattern: /@name/, + inside: { + 'function': /.+/, + }, + }, + { // contract interfaces + pattern: /@interface/, + inside: { + 'function': /.+/, + } + } + ], + + 'function': { + pattern: /\b\w+(?=\()/, + }, + + 'boolean': { + pattern: /\b(?:false|true)\b/, + }, + + 'number': [ + { // hexadecimal, case-insensitive /i + pattern: /\b0x[0-9a-f](?:_?[0-9a-f])*\b/i, + }, + { // octal, case-insensitive /i + pattern: /\b0o[0-7](?:_?[0-7])*\b/i, + }, + { // binary, case-insensitive /i + pattern: /\b0b[01](?:_?[01])*\b/i, + }, + { // decimal integers, starting with 0 + pattern: /\b0\d*\b/, + }, + { // other decimal integers + pattern: /\b[1-9](?:_?\d)*\b/, + }, + ], + + 'string': undefined, + + 'punctuation': { + pattern: /[{}[\]();,.:?]/, + }, + + 'comment': [ + { // single-line + pattern: /(^|[^\\:])\/\/.*/, + lookbehind: true, + greedy: true, + }, + { // multi-line + pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/, + lookbehind: true, + greedy: true, + }, + { + // unused variable identifier + pattern: /\b_\b/ + } + ], + + 'operator': { + 'pattern': /![!=]?|->|[+\-*/%=]=?|[<>]=|<>?=?|~|\|\|=|\|[\|=]?|&&=|&[&=]?|\^=?/, + }, + + 'variable': { + 'pattern': /\b[a-zA-Z_]\w*\b/, + }, + + }; + + // strings, made this way to not collide with other entities + Prism.languages.insertBefore('tact', 'string', { + 'string-literal': { + pattern: /(?:(")(?:\\.|(?!\1)[^\\\r\n])*\1(?!\1))/, + greedy: true, + inside: { + 'regex': [ + { // \\ \" \n \r \t \v \b \f + pattern: /\\[\\"nrtvbf]/, + }, + { // hexEscape, \x00 through \xFF + pattern: /\\x[0-9a-fA-F]{2}/, + }, + { // unicodeEscape, \u0000 through \uFFFF + pattern: /\\u[0-9a-fA-F]{4}/, + }, + { // unicodeCodePoint, \u{0} through \u{FFFFFF} + pattern: /\\u\{[0-9a-fA-F]{1,6}\}/, + }, + ], + 'string': { + pattern: /[\s\S]+/, + }, + }, + }, + }); + + // map and bounced message generic type modifiers + Prism.languages.insertBefore('tact', 'keyword', { + 'generics': { + pattern: /(?:\b(?:bounced|map)\b<[^\\\r\n]*>)/, + greedy: true, + inside: { + 'builtin': [ + ...Prism.languages['tact']['builtin'], + { + pattern: /\b(?:bounced(?=<)|map(?=<))\b/ + }, + ], + 'class-name': Prism.languages['tact']['class-name'], + 'punctuation': { + pattern: /[<>(),.?]/, + }, + 'keyword': { + pattern: /\bas\b/, + }, + }, + }, + }); +}(Prism)); diff --git a/components/prism-tlb.js b/components/prism-tlb.js new file mode 100644 index 0000000..9878908 --- /dev/null +++ b/components/prism-tlb.js @@ -0,0 +1,44 @@ +/** + * @file Prism.js definition for TL-B + * @link https://docs.ton.org/develop/data-formats/tl-b-language + * @author Nikita Sobolev (https://github.com/sobolevn) + * @license MIT + */ +Prism.languages.tlb = { + 'builtin': /\b(?:Bool|Both|Cell|Either|Maybe|Type|Unit|bits256|bits512|int16|int32|int64|int8|uint15|uint16|uint32|uint63|uint64|uint8)\b/, + 'keyword': /\b(?:BoolFalse|BoolTrue|False|Null|True)\b/, + + 'comment': { + pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/, + greedy: true + }, + 'comment-multiline': [ + { + pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/, + lookbehind: true, + greedy: true, + alias: 'comment' + }, + { + pattern: /(^|[^\\:])\/\/.*/, + lookbehind: true, + greedy: true, + alias: 'comment' + } + ], + + 'symbol': [ + /#[0-9a-f]*_?/, + /\$[01]*_?/, + /\b(?:##|#<|#<=)\b/, + ], + 'variable': /[a-zA-Z_]\w*/, + 'operator': [ + /\+/, /-/, /\*/, /\//, + /!=/, /==/, /=/, + /\?/, /~/, /\./, /\^/, + /<=/, />=/, //, + ], + 'number': /\d+/, + 'punctuation': /[;\(\):\[\]\{\}]/, +}; \ No newline at end of file diff --git a/components/prism-tolk.js b/components/prism-tolk.js new file mode 100644 index 0000000..db4dbca --- /dev/null +++ b/components/prism-tolk.js @@ -0,0 +1,71 @@ +/** + * @file Prism.js definition for Tolk + * @link https://docs.ton.org/tolk/overview + * @version 1.4.0 + * @author Danil Ovchinnikov (https://github.com/Danil42Russia) + * @license MIT + */ +(function(Prism) { + Prism.languages.tolk = { + 'comment': { + pattern: /\/\/.*/, + greedy: true, + }, + + 'block-comment': { + pattern: /\/\*[\s\S]*?(?:\*\/|$)/, + greedy: true, + }, + + 'string': [ + { + pattern: /"""(?:\\[\s\S]|(?!""")[\s\S])*(?:"""|$)/, + greedy: true, + }, + { + pattern: /"(?:\\.|[^"\\\r\n])*(?:"|$)/, + greedy: true, + }, + ], + + 'attr-name': /@[a-zA-Z0-9_.]+/, + + 'number': [ + /\b0x[0-9A-Fa-f_]+\b/, + /\b0b[01_]+\b/, + /\b\d[\d_]*\b/, + ], + + 'boolean': /\b(?:false|null|true)\b/, + + 'constant': /\b[A-Z][A-Z0-9_]*_[A-Z0-9_]*\b/, + + 'keyword': [ + /!is\b/, + /\b(?:as|asm|assert|break|builtin|catch|const|continue|contract|do|else|enum|fun|get|global|if|import|is|lazy|match|mutate|private|readonly|repeat|return|self|struct|throw|tolk|try|type|val|var|while)\b/, + ], + + 'function': [ + /`[^`\r\n]+`(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/, + /\b[a-zA-Z_$][a-zA-Z0-9_$]*\b(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/, + ], + + 'builtin': [ + /\b(?:Cell|address|any_address|array|blockchain|bool|builder|cell|coins|continuation|contract|debug|dict|lisp_list|map|never|random|reflect|slice|string|tuple|unknown|void)\b/, + /\b(?:int\d*|uint\d+|varint\d+|varuint\d+|bits\d+|bytes\d+)\b/, + ], + + 'class-name': /\b[A-Z][a-zA-Z0-9_$]*\b/, + + 'property': { + pattern: /(\.)(?:`[^`\r\n]+`|[a-zA-Z_$][a-zA-Z0-9_$]*)/, + lookbehind: true, + }, + + 'operator': /<<=|>>=|<=>|~>>|\^>>|==|!=|<=|>=|<<|>>|&&|\|\||~\/|\^\/|\+=|-=|\*=|\/=|%=|&=|\|=|\^=|->|=>|\+\+|--|\?\?|[+\-*\/%?=<>!&|^~]/, + + 'punctuation': /[.,;:(){}\[\]]/, + + 'symbol': /`[^`\r\n]+`/, + }; +}(Prism)); diff --git a/generate.js b/generate.js index df26e06..c322550 100644 --- a/generate.js +++ b/generate.js @@ -1,5 +1,9 @@ const fs = require('fs') const isEqual = require('lodash.isequal') +const Prism = require('prismjs') +const components = require('prismjs/components.js') + +global.Prism = Prism const SCRIPTS = {} const include = function (src) { @@ -7,12 +11,6 @@ const include = function (src) { (1, eval)(src.toString()) } -async function loadScript(src) { - const script = await fetch(src) - const text = await script.text() - include(text) -} - async function loadLanguages(lngs) { if (lngs) { lngs = Array.isArray(lngs) ? lngs : [lngs]; @@ -39,7 +37,7 @@ async function loadLanguage(lng) { console.log(`${langNumber} | Loading ${lng}`); // TODO: version should probably not be hardcoded - await loadScript(`https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-${lng}.min.js`) + require(`prismjs/components/prism-${lng}.js`) } } @@ -52,6 +50,81 @@ function loadLocalLanguage(path, code, title, alias) { } } +// Spell out the constructs whose meaning differs between regex engines, so one table can +// serve all of them. java.util.regex reads a [ inside a character class as the start of a +// nested class, requires a { outside one to open a quantifier, reads \0 as the start of an +// octal escape, and reads \v as the whole vertical whitespace class rather than as U+000B. +// All four rewrites are no-ops for ECMAScript and for Boost, which is what libprisma +// compiles the patterns with. +const QUANTIFIER = /^\{\d+(?:,\d*)?\}/ + +function normalizeEscapes(pattern) { + let result = '' + let inClass = false + + for (let i = 0; i < pattern.length; i++) { + const c = pattern[i] + + if (c === '\\') { + if (pattern[i + 1] === '0' && !/[0-7]/.test(pattern[i + 2] || '')) { + result += '\\x00' + } else if (pattern[i + 1] === 'v') { + result += '\\x0B' + } else { + result += pattern.substr(i, 2) + } + i++ + continue + } + + if (inClass) { + if (c === '[') { + result += '\\[' + continue + } + // && is set intersection in java.util.regex + if (c === '&' && pattern[i + 1] === '&') { + result += '\\&\\&' + i++ + continue + } + if (c === ']') { + inClass = false + } + } else if (c === '[') { + inClass = true + } else if (c === '{' && !QUANTIFIER.test(pattern.substr(i))) { + result += '\\{' + continue + } + + result += c + } + + return result +} + +// A character written literally in a grammar - BQN's •, APL's ⍵ - has to reach the readers as +// an escape rather than as itself: they disagree about what the bytes of a pattern mean, and +// \uXXXX is the one spelling all of them read as a code point. An astral character is two code +// units here, so it becomes a surrogate pair, which UnicodeEscapes.h puts back together. +function escapeNonAscii(pattern) { + if (!/[^\x00-\x7F]/.test(pattern)) { + return pattern + } + + let result = '' + + for (let i = 0; i < pattern.length; i++) { + const code = pattern.charCodeAt(i) + result += code > 0x7F + ? '\\u' + code.toString(16).padStart(4, '0') + : pattern[i] + } + + return result +} + function unique(a, fn) { if (a.length === 0 || a.length === 1) { return a; @@ -137,7 +210,7 @@ async function generate() { pattern = pattern.replaceAll("|[])", ")"); pattern = pattern.replaceAll(":[]", ":"); - return pattern + return escapeNonAscii(normalizeEscapes(pattern)) } for (var token in copy) { @@ -215,14 +288,17 @@ async function generate() { "sparql" // requires turtle ] - await loadScript("https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-core.min.js") - await loadScript("https://prismjs.com/components.js") await loadLanguages(Object.keys(components.languages)) console.log(`\nLoaded all ${langNumber} languages`) console.log("Processing...") // Manually add local definitions loadLocalLanguage('./components/prism-tl.js', 'typelanguage', 'TypeLanguage', 'tl') + loadLocalLanguage('./components/prism-tlb.js', 'tlb', 'TypeLanguage-Binary', 'tlb') + loadLocalLanguage('./components/prism-fift.js', 'fift', 'Fift', 'fift') + loadLocalLanguage('./components/prism-func.js', 'func', 'FunC', ['func', 'fc']) + loadLocalLanguage('./components/prism-tact.js', 'tact', 'Tact', 'tact') + loadLocalLanguage('./components/prism-tolk.js', 'tolk', 'Tolk', 'tolk') Object.keys(Prism.languages).forEach(lng => { if (unsupported.includes(lng) || !components.languages[lng]) { @@ -338,11 +414,11 @@ async function generate() { for (const lng of alias) { allLanguages[lng] = allGrammars.indexOf(find) - if (components.languages[name].aliasTitles) { - languageNames[lng] = components.languages[name].aliasTitles[lng] - } else { - languageNames[lng] = components.languages[name].title - } + // aliasTitles only names the aliases whose title differs from the language's, + // so an alias missing from it takes the language title rather than no title: + // an empty one drops the alias out of SyntaxHighlighter::languages(). + const aliasTitles = components.languages[name].aliasTitles + languageNames[lng] = (aliasTitles && aliasTitles[lng]) || components.languages[name].title } } }) @@ -358,15 +434,18 @@ async function generate() { const writeUint16 = i => chunks.push(new Uint16Array([i])) const writeUint8 = i => chunks.push(new Uint8Array([i])) const writeString = str => { - if (str.length < 253) { - writeUint8(str.length) + // one byte per code unit truncated anything above U+00FF; the length is the byte + // count, which is what the readers advance by + const bytes = new TextEncoder().encode(str) + if (bytes.length < 253) { + writeUint8(bytes.length) } else { writeUint8(254 & 0xFF) - writeUint8(str.length & 0xFF) - writeUint8((str.length >> 8) & 0xFF) - writeUint8((str.length >> 16) & 0xFF) + writeUint8(bytes.length & 0xFF) + writeUint8((bytes.length >> 8) & 0xFF) + writeUint8((bytes.length >> 16) & 0xFF) } - chunks.push(new Uint8Array(str.split('').map(char => char.charCodeAt(0)))) + chunks.push(bytes) } // Patterns diff --git a/libprisma/Highlight.cpp b/libprisma/Highlight.cpp index 8d55452..6e45235 100644 --- a/libprisma/Highlight.cpp +++ b/libprisma/Highlight.cpp @@ -31,6 +31,11 @@ const Pattern* PatternPtr::get() const return m_tree->resolvePattern(m_path); } +std::shared_ptr PatternRaw::realize() +{ + return std::make_shared(m_regex, m_flags, m_lookbehind, m_greedy, m_alias, m_inside); +} + const Grammar* Pattern::inside() const { if (m_inside) diff --git a/libprisma/Highlight.h b/libprisma/Highlight.h index a1130e0..6248700 100644 --- a/libprisma/Highlight.h +++ b/libprisma/Highlight.h @@ -29,17 +29,35 @@ private: size_t m_path; }; +class PatternRaw +{ +public: + PatternRaw(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr inside) + : m_regex(std::string{pattern}) + , m_flags(flags) + , m_lookbehind(lookbehind) + , m_greedy(greedy) + , m_alias(alias) + , m_inside(inside) + { + } + + std::shared_ptr realize(); + +private: + std::string m_regex; + boost::regex_constants::syntax_option_type m_flags; + bool m_lookbehind; + bool m_greedy; + std::string m_alias; + std::shared_ptr m_inside; +}; + class Pattern { public: - Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, GrammarPtr inside) - : Pattern(pattern, flags, lookbehind, greedy, alias, std::make_shared(inside)) - { - - } - - Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind = false, bool greedy = false, std::string alias = "", std::shared_ptr inside = nullptr) - : m_regex(boost::regex(std::string{pattern}, flags | boost::regex_constants::optimize)) + Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr inside) + : m_regex(boost::regex(std::string{ pattern }, flags | boost::regex_constants::optimize)) , m_lookbehind(lookbehind) , m_greedy(greedy) , m_alias(alias) diff --git a/libprisma/LanguageTree.cpp b/libprisma/LanguageTree.cpp index 1518bcb..01c9902 100644 --- a/libprisma/LanguageTree.cpp +++ b/libprisma/LanguageTree.cpp @@ -1,6 +1,7 @@ #include "LanguageTree.h" #include "TokenList.h" +#include "UnicodeEscapes.h" #include void LanguageTree::load(const std::string& content) @@ -96,6 +97,8 @@ void LanguageTree::parsePatterns(Buffer &buffer) { uint16_t count = freadUint16(buffer); + m_patterns.resize(count); + for (int i = 0; i < count; ++i) { std::string item = freadString(buffer); @@ -163,11 +166,11 @@ void LanguageTree::parsePatterns(Buffer &buffer) if (inside != std::string::npos) { - m_patterns.push_back(std::make_shared(pattern, flags, lookbehind, greedy, std::string{ alias }, std::make_shared(shared_from_this(), inside))); + m_patternsRaw.push_back(std::make_shared(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, std::make_shared(shared_from_this(), inside))); } else { - m_patterns.push_back(std::make_shared(pattern, flags, lookbehind, greedy, std::string{ alias })); + m_patternsRaw.push_back(std::make_shared(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, nullptr)); } } } @@ -175,7 +178,16 @@ void LanguageTree::parsePatterns(Buffer &buffer) const Pattern* LanguageTree::resolvePattern(size_t path) { - return m_patterns[path].get(); + std::shared_ptr &parsed = m_patterns[path]; + if (parsed == nullptr) + { + parsed = m_patternsRaw[path]->realize(); + + m_patternsRaw[path] = nullptr; + m_patterns[path] = parsed; + } + + return parsed.get(); } const Grammar* LanguageTree::resolveGrammar(size_t path) diff --git a/libprisma/LanguageTree.h b/libprisma/LanguageTree.h index 422a2ff..46b613b 100644 --- a/libprisma/LanguageTree.h +++ b/libprisma/LanguageTree.h @@ -38,6 +38,17 @@ public: return keys; } + std::string languageName(const std::string& language) const + { + const auto& find = m_languages.find(language); + if (find != m_languages.end()) + { + return find->second.first; + } + + return language; + } + const Grammar* find(const std::string& key) const { const auto& value = m_languages.find(key); @@ -57,4 +68,5 @@ private: std::map> m_languages; std::vector> m_grammars; std::vector> m_patterns; + std::vector> m_patternsRaw; }; \ No newline at end of file diff --git a/libprisma/SyntaxHighlighter.cpp b/libprisma/SyntaxHighlighter.cpp index 69c158b..2298335 100644 --- a/libprisma/SyntaxHighlighter.cpp +++ b/libprisma/SyntaxHighlighter.cpp @@ -24,6 +24,11 @@ std::map SyntaxHighlighter::languages() const return m_tree->keys(); } +std::string SyntaxHighlighter::languageName(const std::string& language) const +{ + return m_tree->languageName(language); +} + TokenList SyntaxHighlighter::tokenize(std::string_view text, const Grammar* grammar) { // Grammars may include themselves through "inside" (for example @@ -62,9 +67,9 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList return; } - const auto& pattern = *j; - const auto& inside = pattern->inside(); - const bool greedy = pattern->greedy(); + const auto& pattern = *j->get(); + const auto& inside = pattern.inside(); + const bool greedy = pattern.greedy(); size_t pos = startPos; @@ -99,7 +104,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList if (greedy) { - match = pattern->match(matchSuccess, matchIndex, text); + match = pattern.match(matchSuccess, matchIndex, text); if (!matchSuccess || matchIndex >= text.length()) { break; @@ -143,7 +148,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList else { matchIndex = 0; - match = pattern->match(matchSuccess, matchIndex, str); + match = pattern.match(matchSuccess, matchIndex, str); if (!matchSuccess) { continue; @@ -183,7 +188,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList currentNode = tokenList.addAfter(removeFrom, token.name(), std::move(tokenEntries), - pattern->alias(), + pattern.alias(), match.size()); if (after.size()) diff --git a/libprisma/SyntaxHighlighter.h b/libprisma/SyntaxHighlighter.h index a4c981f..8e435bf 100644 --- a/libprisma/SyntaxHighlighter.h +++ b/libprisma/SyntaxHighlighter.h @@ -25,6 +25,8 @@ public: std::map languages() const; + std::string languageName(const std::string& language) const; + private: TokenList tokenize(std::string_view text, const Grammar* grammar); void matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch); diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h new file mode 100644 index 0000000..d72c450 --- /dev/null +++ b/libprisma/UnicodeEscapes.h @@ -0,0 +1,354 @@ +#pragma once + +#include +#include +#include + +namespace libprisma +{ + // Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one + // simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and + // where one sits inside a character class - whose members have to be a single byte each - + // lift it out into an alternation. This belongs here rather than in grammars.dat, which + // keeps the \u spelling for the clients that match over UTF-16 and read it correctly. + + inline bool isHexDigit(char c) + { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + } + + inline int hexValue(char c) + { + return c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10; + } + + // the code point of the \uXXXX escape at begin, or -1 if there is not one there + inline int unicodeEscapeAt(std::string_view pattern, size_t begin) + { + if (begin + 6 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'u') + { + return -1; + } + + int value = 0; + for (size_t i = begin + 2; i < begin + 6; ++i) + { + if (!isHexDigit(pattern[i])) + { + return -1; + } + + value = value * 16 + hexValue(pattern[i]); + } + + return value; + } + + inline void appendByte(std::string& out, unsigned char byte) + { + static const char digits[] = "0123456789ABCDEF"; + + out += "\\x"; + out += digits[byte >> 4]; + out += digits[byte & 0x0F]; + } + + inline void appendUtf8(std::string& out, int codePoint) + { + if (codePoint < 0x80) + { + appendByte(out, static_cast(codePoint)); + } + else if (codePoint < 0x800) + { + appendByte(out, static_cast(0xC0 | (codePoint >> 6))); + appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + } + else if (codePoint < 0x10000) + { + appendByte(out, static_cast(0xE0 | (codePoint >> 12))); + appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); + appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + } + else + { + appendByte(out, static_cast(0xF0 | (codePoint >> 18))); + appendByte(out, static_cast(0x80 | ((codePoint >> 12) & 0x3F))); + appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F))); + appendByte(out, static_cast(0x80 | (codePoint & 0x3F))); + } + } + + // The code point at begin, joining a surrogate pair into the one character it spells, with + // the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it + // to UTF-8 itself; a pair it would expand to two three byte sequences instead. + inline int codePointAt(std::string_view pattern, size_t begin, size_t& length) + { + const int value = unicodeEscapeAt(pattern, begin); + length = value < 0 ? 0 : 6; + + if (value >= 0xD800 && value <= 0xDBFF) + { + const int low = unicodeEscapeAt(pattern, begin + 6); + if (low >= 0xDC00 && low <= 0xDFFF) + { + length = 12; + return 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00); + } + } + + return value; + } + + struct ClassMember + { + std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z" + int low = -1; // code point, for a member that is a lone \u escape + int high = -1; // and the other end, when the member is a range of them + }; + + struct CharacterClass + { + size_t end = 0; // index just past the closing ] + bool negated = false; + bool wide = false; // holds a code point that cannot be a single byte + std::vector members; + }; + + // one class member starting at begin, or 0 if the class ends there + inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member) + { + auto unit = [&](size_t at, int& codePoint) -> size_t { + size_t length = 0; + codePoint = codePointAt(pattern, at, length); + if (codePoint >= 0) + { + return length; + } + return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1; + }; + + int low = -1; + size_t length = unit(begin, low); + size_t at = begin + length; + + // a - b, where a trailing - is a literal rather than the start of a range + if (at + 1 < pattern.size() && pattern[at] == '-' && pattern[at + 1] != ']') + { + int high = -1; + size_t tail = unit(at + 1, high); + member.low = low; + member.high = high; + member.text = pattern.substr(begin, length + 1 + tail); + return length + 1 + tail; + } + + member.low = low; + member.text = pattern.substr(begin, length); + return length; + } + + inline bool readClass(std::string_view pattern, size_t begin, CharacterClass& result) + { + size_t at = begin + 1; + if (at < pattern.size() && pattern[at] == '^') + { + result.negated = true; + ++at; + } + + while (at < pattern.size() && pattern[at] != ']') + { + ClassMember member; + size_t length = readMember(pattern, at, member); + if (length == 0) + { + return false; + } + + if (member.low > 0x7F || member.high > 0x7F) + { + result.wide = true; + } + + result.members.push_back(member); + at += length; + } + + if (at >= pattern.size()) + { + return false; + } + + result.end = at + 1; + return true; + } + + inline bool isRange(const CharacterClass& characters, int low, int high) + { + return characters.members.size() == 1 + && characters.members[0].low == low + && characters.members[0].high == high; + } + + // [\uD800-\uDBFF][\uDC00-\uDFFF] and [\uD800-\uDFFF]{2} are how a JavaScript regex spells + // one astral character, as its two UTF-16 halves. In UTF-8 that is one four byte sequence. + inline size_t appendSurrogatePair(std::string& out, std::string_view pattern, size_t begin) + { + CharacterClass lead; + if (!readClass(pattern, begin, lead) || lead.negated) + { + return 0; + } + + size_t end = 0; + if (isRange(lead, 0xD800, 0xDBFF)) + { + CharacterClass trail; + if (!readClass(pattern, lead.end, trail) || trail.negated || !isRange(trail, 0xDC00, 0xDFFF)) + { + return 0; + } + end = trail.end; + } + else if (isRange(lead, 0xD800, 0xDFFF) && pattern.substr(lead.end, 3) == "{2}") + { + end = lead.end + 3; + } + else + { + return 0; + } + + out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})"; + return end - begin; + } + + // the class at begin, with any member that cannot be a single byte moved out of it + inline size_t appendClass(std::string& out, std::string_view pattern, size_t begin) + { + CharacterClass characters; + if (!readClass(pattern, begin, characters) || !characters.wide) + { + return 0; + } + + std::vector wide; + std::string narrow; + + for (const auto& member : characters.members) + { + if (member.low <= 0x7F && member.high <= 0x7F) + { + narrow += member.text; + } + else if (member.high < 0) + { + wide.push_back(member.low); + } + else if (!characters.negated) + { + // a range of code points that is not the surrogate idiom; leave it be rather + // than expand what could be the whole of Unicode + return 0; + } + } + + // a negated class drops its wide members: they exclude characters that cannot occur + // as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones + if (characters.negated || wide.empty()) + { + out += '['; + if (characters.negated) + { + out += '^'; + } + out += narrow; + out += ']'; + return characters.end - begin; + } + + out += "(?:"; + if (!narrow.empty()) + { + out += '['; + out += narrow; + out += ']'; + out += '|'; + } + for (size_t i = 0; i < wide.size(); ++i) + { + if (i > 0) + { + out += '|'; + } + appendUtf8(out, wide[i]); + } + out += ')'; + + return characters.end - begin; + } + + inline std::string widenUnicodeClasses(std::string_view pattern) + { + if (pattern.find("\\u") == std::string_view::npos) + { + return std::string{ pattern }; + } + + std::string out; + out.reserve(pattern.size()); + + for (size_t i = 0; i < pattern.size(); ) + { + if (pattern[i] == '\\') + { + size_t length = 0; + const int codePoint = codePointAt(pattern, i, length); + + if (codePoint >= 0) + { + // grouped, because a quantifier after it has to apply to the whole + // character and not to the last byte of its encoding + const bool multiByte = codePoint > 0x7F; + if (multiByte) + { + out += "(?:"; + } + appendUtf8(out, codePoint); + if (multiByte) + { + out += ')'; + } + + i += length; + continue; + } + + out += pattern.substr(i, 2); + i += 2; + continue; + } + + if (pattern[i] == '[') + { + if (size_t consumed = appendSurrogatePair(out, pattern, i)) + { + i += consumed; + continue; + } + + if (size_t consumed = appendClass(out, pattern, i)) + { + i += consumed; + continue; + } + } + + out += pattern[i]; + ++i; + } + + return out; + } +} diff --git a/libprisma/grammars.dat b/libprisma/grammars.dat index 532ed27..783d168 100644 Binary files a/libprisma/grammars.dat and b/libprisma/grammars.dat differ diff --git a/package-lock.json b/package-lock.json index 1f9a3ba..c10dd45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,23 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "lodash.isequal": "^4.5.0" + "lodash.isequal": "^4.5.0", + "prismjs": "1.29.0" } }, "node_modules/lodash.isequal": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==" + }, + "node_modules/prismjs": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", + "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } } } } diff --git a/package.json b/package.json index 828a9ac..094cf10 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ }, "homepage": "https://github.com/TelegramMessenger/libprisma#readme", "dependencies": { - "lodash.isequal": "^4.5.0" + "lodash.isequal": "^4.5.0", + "prismjs": "1.29.0" } } diff --git a/tools/PatternProbe.java b/tools/PatternProbe.java new file mode 100644 index 0000000..92296a6 --- /dev/null +++ b/tools/PatternProbe.java @@ -0,0 +1,163 @@ +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one + * matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to + * the one thing it promises: that a pattern means the same to every client that reads it. + * + * The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so + * anything above U+007F diverges on offsets alone and would drown out the escape bugs this is + * looking for. + */ +public class PatternProbe { + + /** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */ + private static final class Entry { + String source; + int flags; + } + + private static final class Reader { + private final byte[] bytes; + private int offset; + + Reader(byte[] bytes) { + this.bytes = bytes; + } + + int uint8() { + return bytes[offset++] & 0xFF; + } + + int uint16() { + return uint8() | (uint8() << 8); + } + + String string() { + int length = uint8(); + if (length >= 254) { + length = uint8() | (uint8() << 8) | (uint8() << 16); + } + String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1); + offset += length; + return value; + } + } + + private static List read(String path) throws IOException { + Reader reader = new Reader(Files.readAllBytes(Paths.get(path))); + List entries = new ArrayList<>(); + + for (int count = reader.uint16(); count > 0; count--) { + String item = reader.string(); + int begin = item.indexOf('/'); + int end = item.lastIndexOf('/'); + if (begin < 0 || end <= begin) { + continue; + } + + String options = item.substring(end + 1); + int aliasBegin = options.indexOf(','); + + Entry entry = new Entry(); + entry.source = item.substring(begin + 1, end); + for (int i = 0; i < aliasBegin; ++i) { + switch (options.charAt(i)) { + case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break; + case 'm': entry.flags |= Pattern.MULTILINE; break; + default: break; // l and y are libprisma's, for Prism lookbehind and greedy + } + } + entries.add(entry); + } + + return entries; + } + + /** + * Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a + * line terminator that ends it" in java.util.regex. \z is the Java spelling of the first. + * A Java client has to do this; it cannot go in the table, where \z would read as a z. + */ + private static String endOfInput(String source) { + if (source.indexOf('$') < 0) { + return source; + } + + StringBuilder result = new StringBuilder(source.length()); + boolean inClass = false; + for (int i = 0; i < source.length(); ++i) { + char c = source.charAt(i); + if (c == '\\' && i + 1 < source.length()) { + result.append(c).append(source.charAt(++i)); + } else if (c == '[') { + inClass = true; + result.append(c); + } else if (c == ']') { + inClass = false; + result.append(c); + } else if (c == '$' && !inClass) { + result.append("\\z"); + } else { + result.append(c); + } + } + return result.toString(); + } + + public static void main(String[] args) throws IOException { + List entries = read(args[0]); + String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1) + .replace(String.valueOf((char) 13), ""); + + StringBuilder out = new StringBuilder(); + int failed = 0; + + for (int i = 0; i < entries.size(); ++i) { + Entry entry = entries.get(i); + String source = entry.source; + if ((entry.flags & Pattern.MULTILINE) == 0) { + source = endOfInput(source); + } + + Pattern pattern; + try { + pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES); + } catch (PatternSyntaxException e) { + if (failed++ < 20) { + System.err.println("pattern " + i + ": " + e.getDescription()); + System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length()))); + } + continue; + } + + out.append(i); + java.util.regex.Matcher matcher = pattern.matcher(corpus); + int at = 0; + while (matcher.find(at) && out.length() < Integer.MAX_VALUE) { + out.append(' ').append(matcher.start()).append(':').append(matcher.end()); + at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1; + if (at > corpus.length()) { + break; + } + } + out.append('\n'); + } + + Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII)); + System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns"); + + if (failed > 0) { + System.err.println(failed + " patterns java.util.regex will not take. " + + "Teach normalizeEscapes in generate.js to rewrite them."); + System.exit(1); + } + } +} diff --git a/tools/check-escapes.js b/tools/check-escapes.js new file mode 100644 index 0000000..a7d4afe --- /dev/null +++ b/tools/check-escapes.js @@ -0,0 +1,87 @@ +// Every escape in grammars.dat has to mean the same thing to every engine that reads it. +// A pattern that fails to compile is caught by the probes beside this script; this one is for +// the escapes that compile everywhere and quietly mean something else, which is the worse +// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline +// included, by java.util.regex, which made ini values run over the end of the line. +const fs = require('fs') + +// what generate.js is expected to emit, once normalizeEscapes has run over it +const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split('')) + +const REASONS = { + v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B', + h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere', + H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere', + V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere', + R: 'any line break in java.util.regex, a literal R elsewhere', + X: 'a grapheme cluster in java.util.regex, a literal X elsewhere', + A: 'start of input in java.util.regex, a literal A elsewhere', + Z: 'end of input in java.util.regex, a literal Z elsewhere', + z: 'end of input in java.util.regex, a literal z elsewhere', + G: 'end of the previous match in java.util.regex, a literal G elsewhere', + Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere', + E: 'ends one in java.util.regex, a literal E elsewhere', + p: 'a unicode property in both, but the property names differ', + N: 'a named character in java.util.regex', + 0: 'starts an octal escape in java.util.regex - emit \\x00', +} + +function readPatterns(path) { + const buffer = fs.readFileSync(path) + let offset = 0 + + const uint8 = () => buffer[offset++] + const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value } + const string = () => { + let length = uint8() + if (length >= 254) { + length = uint8() | (uint8() << 8) | (uint8() << 16) + } + const value = buffer.toString('latin1', offset, offset + length) + offset += length + return value + } + + const patterns = [] + for (let count = uint16(); count > 0; count--) { + patterns.push(string()) + } + return patterns +} + +const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat') +const problems = [] + +for (const item of patterns) { + const end = item.lastIndexOf('/') + const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end) + + for (let i = 0; i < source.length; i++) { + if (source[i] !== '\\') { + continue + } + + const escaped = source[++i] + if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) { + problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set']) + } + } +} + +console.log(`${patterns.length} patterns checked`) + +if (problems.length === 0) { + process.exit(0) +} + +const seen = new Set() +for (const [source, escape, reason] of problems) { + if (seen.has(escape)) { + continue + } + seen.add(escape) + console.error(`\n${escape} is ${reason}`) + console.error(` e.g. ${source.slice(0, 120)}`) +} +console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`) +process.exit(1) diff --git a/tools/corpus.txt b/tools/corpus.txt new file mode 100644 index 0000000..2086075 --- /dev/null +++ b/tools/corpus.txt @@ -0,0 +1,13 @@ +class Foo where T : IBar { public int x = 1; } /* c */ // line +{{ tpl }} +def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py +let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3; +SELECT * FROM t WHERE a IN (1,2) -- sql +#include +key = value ; ini +[section] +- item: + nested: [1, 2] +0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y + tab and trailing spaces +#!/bin/sh diff --git a/tools/probe.cpp b/tools/probe.cpp new file mode 100644 index 0000000..4881df4 --- /dev/null +++ b/tools/probe.cpp @@ -0,0 +1,132 @@ +// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down +// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and +// the two files are diffed, which is what holds the table to meaning one thing everywhere. +// +// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so +// anything above U+007F diverges on offsets alone and would drown out the escape bugs. +#include + +#include "../libprisma/UnicodeEscapes.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + + std::string readAll(const char* path) { + std::ifstream in(path, std::ios::binary); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + } + + struct Reader { + const std::string& bytes; + size_t offset = 0; + + uint8_t uint8() { + return offset < bytes.size() ? static_cast(bytes[offset++]) : 0; + } + + uint16_t uint16() { + uint16_t low = uint8(); + return static_cast(low | (uint8() << 8)); + } + + std::string string() { + size_t length = uint8(); + if (length >= 254) { + size_t a = uint8(), b = uint8(), c = uint8(); + length = a | (b << 8) | (c << 16); + } + std::string value = bytes.substr(offset, length); + offset += length; + return value; + } + }; + + struct Entry { + std::string source; + boost::regex_constants::syntax_option_type flags; + }; + + // "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads + std::vector read(const std::string& table) { + Reader reader{ table }; + std::vector entries; + + for (uint16_t count = reader.uint16(); count > 0; --count) { + const std::string item = reader.string(); + const size_t begin = item.find_first_of('/'); + const size_t end = item.find_last_of('/'); + if (begin == std::string::npos || end <= begin) { + continue; + } + + const std::string options = item.substr(end + 1); + const size_t aliasBegin = options.find_first_of(','); + + Entry entry; + entry.source = item.substr(begin + 1, end - (begin + 1)); + entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m; + + for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) { + switch (options[i]) { + case 'i': entry.flags |= boost::regex_constants::icase; break; + case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break; + default: break; // l and y are libprisma's, for Prism lookbehind and greedy + } + } + + entries.push_back(entry); + } + + return entries; + } +} + +int main(int argc, char** argv) { + if (argc < 4) { + fprintf(stderr, "usage: probe \n"); + return 2; + } + + const std::vector entries = read(readAll(argv[1])); + + std::string corpus = readAll(argv[2]); + corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end()); + + std::ofstream out(argv[3], std::ios::binary); + int failed = 0; + + for (size_t i = 0; i < entries.size(); ++i) { + boost::regex re; + try { + re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags); + } catch (const std::exception& e) { + if (failed++ < 20) { + fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str()); + } + continue; + } + + out << i; + auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re, + boost::regex_constants::match_not_dot_newline); + for (auto end = boost::sregex_iterator(); it != end; ++it) { + out << ' ' << it->position() << ':' << (it->position() + it->length()); + } + out << '\n'; + } + + printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size()); + + if (failed > 0) { + fprintf(stderr, "%d patterns Boost will not take.\n", failed); + return 1; + } + return 0; +}