diff --git a/.gitattributes b/.gitattributes
index 63081ae..3b5a7e3 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -62,3 +62,4 @@
#*.rtf diff=astextplain
#*.RTF diff=astextplain
libprisma/grammars.json eol=lf
+libprisma/grammars.dat linguist-generated
diff --git a/.github/workflows/build-grammars.yml b/.github/workflows/build-grammars.yml
index ec56456..e3784a9 100644
--- a/.github/workflows/build-grammars.yml
+++ b/.github/workflows/build-grammars.yml
@@ -1,27 +1,37 @@
name: Update grammars
-on: [ workflow_dispatch ]
+on:
+ workflow_dispatch:
+ push:
+ branches: [ master ]
+ # grammars.dat itself is deliberately not listed: regenerating it must not trigger another run.
+ paths:
+ - components/**
+ - generate.js
+ - package.json
+ - package-lock.json
+
+permissions:
+ contents: write
+ pull-requests: write
jobs:
build:
runs-on: ubuntu-latest
- strategy:
- matrix:
- node-version: [18.x]
steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-node@v3
+ - uses: actions/checkout@v5
+ - uses: actions/setup-node@v5
with:
- node-version: 18
+ node-version: 22
- run: npm install
- run: npm start
- name: Upload a Build Artifact
- uses: actions/upload-artifact@v3.1.3
+ uses: actions/upload-artifact@v7
with:
name: grammars.dat
path: ./libprisma/grammars.dat
- name: Create Pull Request
- uses: peter-evans/create-pull-request@v5
+ uses: peter-evans/create-pull-request@v8
with:
commit-message: Update grammars
title: Update grammars
diff --git a/.github/workflows/check-generated.yml b/.github/workflows/check-generated.yml
new file mode 100644
index 0000000..9e2dad0
--- /dev/null
+++ b/.github/workflows/check-generated.yml
@@ -0,0 +1,22 @@
+name: Check generated files
+
+# The path filter is the whole test: the job only exists when the pull request
+# touches grammars.dat, so reaching it is already the failure.
+on:
+ pull_request:
+ paths:
+ - libprisma/grammars.dat
+
+permissions: {}
+
+jobs:
+ grammars:
+ runs-on: ubuntu-latest
+ # A pull request opened with GITHUB_TOKEN cannot trigger this workflow at all,
+ # so the guard only matters if "Update grammars" is ever moved to a PAT.
+ if: github.event.pull_request.user.login != 'github-actions[bot]'
+ steps:
+ - name: Reject changes to the generated grammar table
+ run: |
+ echo "::error file=libprisma/grammars.dat::libprisma/grammars.dat is generated by generate.js and is regenerated on master by the Update grammars workflow. Please drop it from this pull request and keep only the components/ and generate.js changes."
+ exit 1
diff --git a/.github/workflows/check-grammars.yml b/.github/workflows/check-grammars.yml
new file mode 100644
index 0000000..3e7a4a2
--- /dev/null
+++ b/.github/workflows/check-grammars.yml
@@ -0,0 +1,58 @@
+name: Check grammars
+
+# grammars.dat is read by libprisma with Boost and by the Android client with java.util.regex,
+# so every pattern in it has to mean the same thing to both. Regenerate the table and hold it
+# to that, rather than waiting for a language to quietly stop highlighting.
+on:
+ pull_request:
+ paths:
+ - components/**
+ - generate.js
+ - libprisma/**
+ - tools/**
+ - package.json
+ - package-lock.json
+ workflow_dispatch:
+
+permissions: {}
+
+jobs:
+ patterns:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v5
+ with:
+ persist-credentials: false
+ - uses: actions/setup-node@v5
+ with:
+ node-version: 22
+
+ # the committed table is regenerated on master, so a pull request is checked against what
+ # its own components and generate.js produce
+ - run: npm ci --ignore-scripts
+ - run: npm start
+
+ - name: Escapes every engine agrees on
+ run: node tools/check-escapes.js libprisma/grammars.dat
+
+ - name: Compile every pattern with java.util.regex
+ run: |
+ javac -d tools tools/PatternProbe.java
+ java -cp tools PatternProbe libprisma/grammars.dat tools/corpus.txt java.txt
+
+ - name: Compile every pattern with Boost
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y libboost-regex-dev
+ g++ -std=c++17 -O2 -o tools/probe tools/probe.cpp -lboost_regex
+ ./tools/probe libprisma/grammars.dat tools/corpus.txt boost.txt
+
+ - name: Compare what the two engines matched
+ run: |
+ if ! diff -u boost.txt java.txt > engines.diff; then
+ echo "The two engines disagree about what these patterns match."
+ echo "Each line is a pattern index followed by its start:end offsets in tools/corpus.txt."
+ head -40 engines.diff
+ exit 1
+ fi
+ echo "Boost and java.util.regex agree on every pattern."
diff --git a/README.md b/README.md
index f5e307f..8fb403c 100644
--- a/README.md
+++ b/README.md
@@ -129,10 +129,12 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`F#`|`fsharp`|
|`Factor`|`factor`|
|`False`|`false`|
+|`Fift`|`fift`|
|`Firestore security rules`|`firestore-security-rules`|
|`Flow`|`flow`|
|`Fortran`|`fortran`|
|`FreeMarker Template Language`|`ftl`|
+|`FunC`|`func`|
|`GameMaker Language`|`gml`,`gamemakerlanguage`|
|`GAP (CAS)`|`gap`|
|`G-code`|`gcode`|
@@ -290,6 +292,7 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`SuperCollider`|`supercollider`,`sclang`|
|`Swift`|`swift`|
|`Systemd configuration file`|`systemd`|
+|`Tact`|`tact`|
|`T4 templating`|`t4-templating`|
|`T4 Text Templates (C#)`|`t4-cs`,`t4`|
|`VB.Net`|`vbnet`|
@@ -299,6 +302,8 @@ The new version of `grammars.dat` will appear in the `libprisma/grammars.dat`.
|`Template Toolkit 2`|`tt2`|
|`TOML`|`toml`|
|`Tremor`|`tremor`,`tremor`,`trickle`,`troy`|
+|`Type Language`|`tl`|
+|`Type Language - Binary`|`tlb`|
|`TypoScript`|`typoscript`,`tsconfig`|
|`UnrealScript`|`unrealscript`,`unrealscript`,`uscript`,`uc`|
|`UO Razor Script`|`uorazor`|
diff --git a/components/prism-fift.js b/components/prism-fift.js
new file mode 100644
index 0000000..44f3376
--- /dev/null
+++ b/components/prism-fift.js
@@ -0,0 +1,127 @@
+/**
+ * @file Prism.js definition for Fift
+ * @link https://docs.ton.org/develop/fift/overview
+ * @author Nikita Sobolev (https://github.com/sobolevn)
+ * @license MIT
+ */
+Prism.languages.fift = {
+ 'symbol': [
+ /[xX]\{[0-9a-fA-F_]*\}/,
+ /[bB]\{[01]*\}/,
+ /'\S+/,
+ ],
+ 'string': /"([^"\r\n\\]|\\.)*"/,
+
+ 'comment': [
+ {
+ pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
+ lookbehind: true,
+ greedy: true,
+ },
+ {
+ pattern: /\/\/.*/,
+ lookbehind: true,
+ greedy: true,
+ },
+ ],
+
+ 'operator': [
+ // Full list can be found in
+ // Appendix A. List of Fift words
+ // Ordered the same way source code does this.
+ // Except: shorter words must follow
+ // longer ones with the same base part.
+ // Example: `#` comes after `#s`
+ /#>/, /#s/, /\$#/, /#/,
+
+ /\$\+/, /\$,/, /\$\d/, /\$=/, /\$(?=\()/,
+ /\$>smca/, /\$>s/,
+ /\$@\+/, /\$@\?\+/, /\$@\?/, /\$@/,
+ /\$cmp/, /\$len/, /\$pos/, /\$reverse/,
+
+ /%1<,
+
+ /\('\)/, /\(-trailing\)/, /\(\.\)/, /\(atom\)/, /\(b\.\)/,
+ /\(compile\)/, /\(create\)/, /\(def\?\)/, /\(dump\)/, /\(execute\)/,
+ /\(forget\)/, /\(number\)/, /\(x\.\)/, /\(\{\)/, /\(\}\)/,
+
+ /\*\/cmod/, /\*\/c/, /\*\/mod/, /\*\/rmod/, /\*\/r/, /\*\//,
+ /\*>>c/, /\*>>r/, /\*>>/, /\*mod/, /\*/,
+
+ /\+!/, /\+/, /,/, /-!/, /-/, /!/,
+ /-1<, /\._/, /\.dump/, /\.l/, /\.sl/, /\.s/, /\.tc/,
+ /\//, /\/\*/, /\/cmod/, /\/c/, /\/mod/, /\/rmod/, /\/r/,
+ /0!/, /0<=/, /0<>/, /0, /0=/, /0>=/, /0>/,
+ /1\+!/, /1\+/, /1-!/, /1-/, /1<, /1<<1-/,
+ /2\*/, /2\+/, /2-/, /2\//, /2=:/,
+
+ /::_/, /::/, /:_/, /=:/, /:/,
+
+ /<#/, /<<\/c/, /<<\/r/, /<=/, /<>/, /=/, />>c/, />>r/, />>/, />/,
+
+ /\?dup/,
+
+ /@'/, /@/,
+
+ /B\+/, /B,/, /B=/, /B>Li@\+/, /B>Li@/, /B>Lu@\+/, /B>Lu@/, /B>boc/,
+ /B>file/, /B>i@\+/, /B>i@/, /B>u@\+/, /B>u@/,
+ /B@\?\+/, /B@\+/, /B@\?/, /B@/,
+ /Bcmp/, /BhashB/, /Bhashu/, /Bhash/, /Blen/, /Bx\./,
+ /B\|/, /Li>B/, /Lu>B/,
+
+ /\[\]/, /\[compile\]/, /\[/, /\]/,
+
+ /atom\?/,
+
+ /b\+/, /b\._/, /b\./,
+ /b>idict!\+/, /b>idict!/, /b>sdict!\+/, /b>sdict!/,
+ /b>udict!\+/, /b>udict!/,
+ /b>/, /boc\+>B/, /boc>B/,
+
+ /csr\./,
+ /def\?/,
+ /empty\?/, /eq\?/,
+ /file-exists\?/, /file>B/,
+
+ /i,/, /i>B/, /i@\+/, /i@\?\+/, /i@\?/, /i@/,
+ /idict!\+/, /idict!/, /idict-/, /idict@-/, /idict@/,
+
+ /null!/, /null\?/,
+ /pfxdict!\+/, /pfxdict!/, /pfxdict@/, /priv>pub/,
+ /ref@\+/, /ref@/, /ref@\?\+/, /ref@\?/,
+
+ /s,/, /s>c/, /s>/,
+ /sdict!\+/, /sdict!/, /sdict-/, /sdict@-/, /sdict@/,
+ /smca>\$/, /sr,/,
+
+ /tuple\?/,
+
+ /u,/, /u>B/, /u@\+/, /u@\?\+/, /u@\?/,
+ /udict!\+/, /udict!/, /udict-/, /udict@-/, /udict@/,
+ /undef\?/,
+
+ /x\._/, /x\./,
+
+ /\|\+/, /\|/, /\|_/,
+
+ /\?\./, /'/,
+
+ // Should be the last:
+ /\./,
+ ],
+
+ 'keyword': /\b(?:-roll|-rot|-trailing|-trailing0|2constant|2drop|2dup|2over|2swap|abort|abs|allot|and|anon|atom|bbitrefs|bbits|bl|box|brefs|brembitrefs|brembits|bremrefs|bye|caddr|cadr|car|cddr|cdr|char|chr|cmp|cond|cons|constant|count|cr|create|depth|dictmap|dictmerge|dictnew|does|drop|dup|ed25519_chksign|ed25519_sign|ed25519_sign_uint|emit|exch|exch2|execute|explode|find|first|fits|forget|gasrunvm|gasrunvmcode|gasrunvmctx|gasrunvmdict|halt|hash|hashB|hashu|hold|hole|if|ifnot|include|list|max|min|minmax|mod|negate|newkeypair|nil|nip|nop|not|now|null|or|over|pair|pick|quit|remaining|reverse|roll|rot|runvm|runvmcode|runvmctx|runvmdict|sbitrefs|sbits|second|sgn|shash|sign|single|skipspc|space|srefs|swap|ten|third|times|triple|tuck|tuple|type|ufits|uncons|unpair|unsingle|until|untriple|untuple|variable|while|word|words|xor)\b/,
+ 'boolean': /\b(?:false|true)\b/,
+
+ 'number': [
+ /(0[xX][0-9a-fA-F]+)/,
+ /(0[bB][01]+)/,
+ /(-?\d+(\/-?\d+)?)/,
+ ],
+ 'variable': /[\w$-]+/,
+
+ 'punctuation': /[\[\{\}\],\(\)]/,
+};
\ No newline at end of file
diff --git a/components/prism-func.js b/components/prism-func.js
new file mode 100644
index 0000000..4dd2dc8
--- /dev/null
+++ b/components/prism-func.js
@@ -0,0 +1,148 @@
+/**
+ * @file Prism.js definition for FunC
+ * @link https://docs.ton.org/develop/func/overview
+ * @version 0.4.4
+ * @author Novus Nota (https://github.com/novusnota)
+ * @author Nikita Sobolev (https://github.com/sobolevn)
+ * @license MIT
+ */
+(function(Prism) {
+ /** @type {Prism.GrammarValue} */
+ var number = /-?\b(?:\d+|0x[\da-fA-F]+)\b(?![^\.~,;\[\]\(\)\s])/;
+
+ /** @type {Prism.GrammarValue} */
+ var string = [
+ { // multi-line
+ pattern: /"""[\s\S]*?"""[Hhcusa]?/,
+ greedy: true,
+ inside: {
+ // string type
+ 'symbol': {
+ pattern: /("""[\s\S]*?""")[Hhcusa]/,
+ lookbehind: true,
+ greedy: true,
+ }
+ },
+ },
+ { // single-line
+ pattern: /"[^\n"]*"[Hhcusa]?/,
+ greedy: true,
+ inside: {
+ // string type
+ 'symbol': {
+ pattern: /("[^\n"]*")[Hhcusa]/,
+ lookbehind: true,
+ greedy: true,
+ }
+ },
+ },
+ ];
+
+ /** @type {RegExp} */
+ var operator_body = /(?:!=|\?|:|%=|%|&=|&|\*=|\*|\+=|\+|->|-=|-|\/%|\/=|\/|<=>|<<=|<<|<=|<|==|=|>>=|>>|>=|>|\^>>=|\^>>|\^=|\^\/=|\^\/|\^%=|\^%|\^|\|=|\||~>>=|~>>|~\/=|~\/|~%|~)/;
+
+ // An operator only counts as one when whitespace follows it, as identifiers may contain these characters.
+ /** @type {Prism.GrammarValue} */
+ var operator = new RegExp(operator_body.source + /(?=\s)/.source);
+
+ /** @type {RegExp[]} */
+ var var_identifier = [
+ // quoted
+ /`[^`\n]+`/,
+ // plain
+ /[^\.~,;\[\]\(\)\s]+/
+ ];
+
+ /** Prism.js definition for FunC */
+ Prism.languages.func = {
+ 'comment': [
+ { // single-line
+ pattern: /;;.*/,
+ lookbehind: true,
+ greedy: true,
+ },
+ { // multi-line, not nested (TODO: nesting)
+ // . isn't used, because it only applies to the single line
+ pattern: /\{-[\s\S]*?(?:-\}|$)/,
+ lookbehind: true,
+ greedy: true,
+ },
+ {
+ // unused variable identifier
+ pattern: /\b_\b(?![^\.~,;\[\]\(\)\s])/
+ },
+ ],
+
+ // Custom token for #pragma's
+ 'pragma': {
+ pattern: /#pragma(.*);/,
+ inside: {
+ 'keyword': /(?:#pragma|test-version-set|not-version|version|allow-post-modification|compute-asm-ltr)\b/,
+ 'number': /(?:\d+)(?:.\d+)?(?:.\d+)?/,
+ 'operator': /=|\^|<=|>=|<|>/,
+ 'string': string,
+ 'punctuation': /;/,
+ }
+ },
+
+ // Custom token for #include's
+ 'include': {
+ pattern: /#include(.*);/,
+ inside: {
+ 'keyword': /#include\b/,
+ 'string': string,
+ 'punctuation': /;/,
+ }
+ },
+
+ // builtin types
+ 'builtin': /\b(?:int|cell|slice|builder|cont|tuple|type)\b/,
+
+ // builtin constants
+ 'boolean': /\b(?:false|true|nil|Nil)\b/,
+
+ 'keyword': /\b(?:forall|extern|global|asm|impure|inline_ref|inline|auto_apply|method_id|operator|infixl|infixr|infix|const|return|var|repeat|do|while|until|try|catch|ifnot|if|then|elseifnot|elseif|else)\b/,
+
+ 'number': number,
+
+ 'string': string,
+
+ 'operator': operator,
+
+ 'punctuation': [/[\.,;\(\)\[\]]/, /[\{\}](?![^\.~,;\[\]\(\)\s])/],
+
+ // Function and method names in declarations, definitions and calls
+ 'function': [
+ { // quoted
+ pattern: new RegExp(var_identifier[0].source + /(?=\s*\()/.source),
+ greedy: true,
+ },
+ { // bitwise not operator
+ pattern: /~_/,
+ greedy: true,
+ },
+ { // remaining operators
+ // operator_body, not operator: its trailing lookahead would forbid the closing _
+ pattern: new RegExp(/\^?_/.source + operator_body.source + /_/.source),
+ greedy: true,
+ },
+ { // plain function or method name
+ pattern: new RegExp(/[\.~]?/.source + var_identifier[1].source + /(?=\s*\()/.source),
+ greedy: true,
+ },
+ { // builtin functions and methods
+ pattern: /\b(?:divmod|moddiv|muldiv|muldivr|muldivc|muldivmod|null\?|throw|throw_if|throw_unless|throw_arg|throw_arg_if|throw_arg_unless|load_int|load_uint|preload_int|preload_uint|store_int|store_uint|load_bits|preload_bits|int_at|cell_at|slice_at|tuple_at|at|touch|touch2|run_method0|run_method1|run_method2|run_method3|~divmod|~moddiv|~store_int|~store_uint|~touch|~touch2|~dump|~stdump)\b/,
+ greedy: true,
+ },
+ ],
+
+ // Parametric polymorphism
+ 'class-name': /[A-Z][^\.~,;\[\]\(\)\s]*/,
+
+ // All the rest of identifiers
+ 'variable': var_identifier.map(x => { return { pattern: x, greedy: true } }),
+ };
+
+ // Adding a fc alias
+ Prism.languages.fc = Prism.languages.func;
+}(Prism));
diff --git a/components/prism-tact.js b/components/prism-tact.js
new file mode 100644
index 0000000..ab21fcf
--- /dev/null
+++ b/components/prism-tact.js
@@ -0,0 +1,171 @@
+/**
+ * @file Prism.js definition for Tact
+ * @link https://tact-lang.org
+ * @version 1.6.0
+ * @author Novus Nota (https://github.com/novusnota)
+ * @license MIT
+ */
+(function(Prism) {
+ Prism.languages.tact = {
+ // reserved keywords
+ 'keyword': [
+ {
+ pattern: /\b(?:abstract|asm|as|catch|codeOf|const|contract(?!:)|do|else|extend|extends|foreach|fun|get|if|in|import|initOf|inline|let|message(?!:)|mutates|native|override|primitive|public|repeat|return|self|struct(?!:)|trait(?!:)|try|until|virtual|while|with)\b/,
+ },
+ { // reserved function names
+ pattern: /\b(?:bounced|external|init|receive)\b(?=\()/
+ },
+ ],
+
+ // built-in types
+ 'builtin': [
+ {
+ pattern: /\b(?:Address|Bool|Builder|Cell|Int|Slice|String|StringBuilder)\b/,
+ },
+ { // keyword after as, see: https://prismjs.com/extending.html#object-notation
+ pattern: /(\bas\s+)(?:coins|varu?int(?:32|16)|remaining|bytes(?:32|64)|int257|u?int(?:25[0-6]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?))\b/,
+ lookbehind: true,
+ greedy: true,
+ },
+ ],
+
+ // SCREAMING_SNAKE_CASE for null values and names of constants
+ 'constant': [
+ {
+ pattern: /\bnull\b/,
+ },
+ {
+ pattern: /\b[A-Z][A-Z0-9_]*\b/,
+ },
+ ],
+
+ // UpperCamelCase for names of contracts, traits, structs, messages
+ 'class-name': {
+ pattern: /\b[A-Z]\w*\b/,
+ },
+
+ // mappings to FunC
+ 'attribute': [
+ { // functions
+ pattern: /@name/,
+ inside: {
+ 'function': /.+/,
+ },
+ },
+ { // contract interfaces
+ pattern: /@interface/,
+ inside: {
+ 'function': /.+/,
+ }
+ }
+ ],
+
+ 'function': {
+ pattern: /\b\w+(?=\()/,
+ },
+
+ 'boolean': {
+ pattern: /\b(?:false|true)\b/,
+ },
+
+ 'number': [
+ { // hexadecimal, case-insensitive /i
+ pattern: /\b0x[0-9a-f](?:_?[0-9a-f])*\b/i,
+ },
+ { // octal, case-insensitive /i
+ pattern: /\b0o[0-7](?:_?[0-7])*\b/i,
+ },
+ { // binary, case-insensitive /i
+ pattern: /\b0b[01](?:_?[01])*\b/i,
+ },
+ { // decimal integers, starting with 0
+ pattern: /\b0\d*\b/,
+ },
+ { // other decimal integers
+ pattern: /\b[1-9](?:_?\d)*\b/,
+ },
+ ],
+
+ 'string': undefined,
+
+ 'punctuation': {
+ pattern: /[{}[\]();,.:?]/,
+ },
+
+ 'comment': [
+ { // single-line
+ pattern: /(^|[^\\:])\/\/.*/,
+ lookbehind: true,
+ greedy: true,
+ },
+ { // multi-line
+ pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
+ lookbehind: true,
+ greedy: true,
+ },
+ {
+ // unused variable identifier
+ pattern: /\b_\b/
+ }
+ ],
+
+ 'operator': {
+ 'pattern': /![!=]?|->|[+\-*/%=]=?|[<>]=|<=?|>>?=?|~|\|\|=|\|[\|=]?|&&=|&[&=]?|\^=?/,
+ },
+
+ 'variable': {
+ 'pattern': /\b[a-zA-Z_]\w*\b/,
+ },
+
+ };
+
+ // strings, made this way to not collide with other entities
+ Prism.languages.insertBefore('tact', 'string', {
+ 'string-literal': {
+ pattern: /(?:(")(?:\\.|(?!\1)[^\\\r\n])*\1(?!\1))/,
+ greedy: true,
+ inside: {
+ 'regex': [
+ { // \\ \" \n \r \t \v \b \f
+ pattern: /\\[\\"nrtvbf]/,
+ },
+ { // hexEscape, \x00 through \xFF
+ pattern: /\\x[0-9a-fA-F]{2}/,
+ },
+ { // unicodeEscape, \u0000 through \uFFFF
+ pattern: /\\u[0-9a-fA-F]{4}/,
+ },
+ { // unicodeCodePoint, \u{0} through \u{FFFFFF}
+ pattern: /\\u\{[0-9a-fA-F]{1,6}\}/,
+ },
+ ],
+ 'string': {
+ pattern: /[\s\S]+/,
+ },
+ },
+ },
+ });
+
+ // map and bounced message generic type modifiers
+ Prism.languages.insertBefore('tact', 'keyword', {
+ 'generics': {
+ pattern: /(?:\b(?:bounced|map)\b<[^\\\r\n]*>)/,
+ greedy: true,
+ inside: {
+ 'builtin': [
+ ...Prism.languages['tact']['builtin'],
+ {
+ pattern: /\b(?:bounced(?=<)|map(?=<))\b/
+ },
+ ],
+ 'class-name': Prism.languages['tact']['class-name'],
+ 'punctuation': {
+ pattern: /[<>(),.?]/,
+ },
+ 'keyword': {
+ pattern: /\bas\b/,
+ },
+ },
+ },
+ });
+}(Prism));
diff --git a/components/prism-tlb.js b/components/prism-tlb.js
new file mode 100644
index 0000000..9878908
--- /dev/null
+++ b/components/prism-tlb.js
@@ -0,0 +1,44 @@
+/**
+ * @file Prism.js definition for TL-B
+ * @link https://docs.ton.org/develop/data-formats/tl-b-language
+ * @author Nikita Sobolev (https://github.com/sobolevn)
+ * @license MIT
+ */
+Prism.languages.tlb = {
+ 'builtin': /\b(?:Bool|Both|Cell|Either|Maybe|Type|Unit|bits256|bits512|int16|int32|int64|int8|uint15|uint16|uint32|uint63|uint64|uint8)\b/,
+ 'keyword': /\b(?:BoolFalse|BoolTrue|False|Null|True)\b/,
+
+ 'comment': {
+ pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,
+ greedy: true
+ },
+ 'comment-multiline': [
+ {
+ pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,
+ lookbehind: true,
+ greedy: true,
+ alias: 'comment'
+ },
+ {
+ pattern: /(^|[^\\:])\/\/.*/,
+ lookbehind: true,
+ greedy: true,
+ alias: 'comment'
+ }
+ ],
+
+ 'symbol': [
+ /#[0-9a-f]*_?/,
+ /\$[01]*_?/,
+ /\b(?:##|#<|#<=)\b/,
+ ],
+ 'variable': /[a-zA-Z_]\w*/,
+ 'operator': [
+ /\+/, /-/, /\*/, /\//,
+ /!=/, /==/, /=/,
+ /\?/, /~/, /\./, /\^/,
+ /<=/, />=/, /, />/,
+ ],
+ 'number': /\d+/,
+ 'punctuation': /[;\(\):\[\]\{\}]/,
+};
\ No newline at end of file
diff --git a/components/prism-tolk.js b/components/prism-tolk.js
new file mode 100644
index 0000000..db4dbca
--- /dev/null
+++ b/components/prism-tolk.js
@@ -0,0 +1,71 @@
+/**
+ * @file Prism.js definition for Tolk
+ * @link https://docs.ton.org/tolk/overview
+ * @version 1.4.0
+ * @author Danil Ovchinnikov (https://github.com/Danil42Russia)
+ * @license MIT
+ */
+(function(Prism) {
+ Prism.languages.tolk = {
+ 'comment': {
+ pattern: /\/\/.*/,
+ greedy: true,
+ },
+
+ 'block-comment': {
+ pattern: /\/\*[\s\S]*?(?:\*\/|$)/,
+ greedy: true,
+ },
+
+ 'string': [
+ {
+ pattern: /"""(?:\\[\s\S]|(?!""")[\s\S])*(?:"""|$)/,
+ greedy: true,
+ },
+ {
+ pattern: /"(?:\\.|[^"\\\r\n])*(?:"|$)/,
+ greedy: true,
+ },
+ ],
+
+ 'attr-name': /@[a-zA-Z0-9_.]+/,
+
+ 'number': [
+ /\b0x[0-9A-Fa-f_]+\b/,
+ /\b0b[01_]+\b/,
+ /\b\d[\d_]*\b/,
+ ],
+
+ 'boolean': /\b(?:false|null|true)\b/,
+
+ 'constant': /\b[A-Z][A-Z0-9_]*_[A-Z0-9_]*\b/,
+
+ 'keyword': [
+ /!is\b/,
+ /\b(?:as|asm|assert|break|builtin|catch|const|continue|contract|do|else|enum|fun|get|global|if|import|is|lazy|match|mutate|private|readonly|repeat|return|self|struct|throw|tolk|try|type|val|var|while)\b/,
+ ],
+
+ 'function': [
+ /`[^`\r\n]+`(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
+ /\b[a-zA-Z_$][a-zA-Z0-9_$]*\b(?=\s*(?:<[^(){};\r\n]*>\s*)?\()/,
+ ],
+
+ 'builtin': [
+ /\b(?:Cell|address|any_address|array|blockchain|bool|builder|cell|coins|continuation|contract|debug|dict|lisp_list|map|never|random|reflect|slice|string|tuple|unknown|void)\b/,
+ /\b(?:int\d*|uint\d+|varint\d+|varuint\d+|bits\d+|bytes\d+)\b/,
+ ],
+
+ 'class-name': /\b[A-Z][a-zA-Z0-9_$]*\b/,
+
+ 'property': {
+ pattern: /(\.)(?:`[^`\r\n]+`|[a-zA-Z_$][a-zA-Z0-9_$]*)/,
+ lookbehind: true,
+ },
+
+ 'operator': /<<=|>>=|<=>|~>>|\^>>|==|!=|<=|>=|<<|>>|&&|\|\||~\/|\^\/|\+=|-=|\*=|\/=|%=|&=|\|=|\^=|->|=>|\+\+|--|\?\?|[+\-*\/%?=<>!&|^~]/,
+
+ 'punctuation': /[.,;:(){}\[\]]/,
+
+ 'symbol': /`[^`\r\n]+`/,
+ };
+}(Prism));
diff --git a/generate.js b/generate.js
index df26e06..c322550 100644
--- a/generate.js
+++ b/generate.js
@@ -1,5 +1,9 @@
const fs = require('fs')
const isEqual = require('lodash.isequal')
+const Prism = require('prismjs')
+const components = require('prismjs/components.js')
+
+global.Prism = Prism
const SCRIPTS = {}
const include = function (src) {
@@ -7,12 +11,6 @@ const include = function (src) {
(1, eval)(src.toString())
}
-async function loadScript(src) {
- const script = await fetch(src)
- const text = await script.text()
- include(text)
-}
-
async function loadLanguages(lngs) {
if (lngs) {
lngs = Array.isArray(lngs) ? lngs : [lngs];
@@ -39,7 +37,7 @@ async function loadLanguage(lng) {
console.log(`${langNumber} | Loading ${lng}`);
// TODO: version should probably not be hardcoded
- await loadScript(`https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-${lng}.min.js`)
+ require(`prismjs/components/prism-${lng}.js`)
}
}
@@ -52,6 +50,81 @@ function loadLocalLanguage(path, code, title, alias) {
}
}
+// Spell out the constructs whose meaning differs between regex engines, so one table can
+// serve all of them. java.util.regex reads a [ inside a character class as the start of a
+// nested class, requires a { outside one to open a quantifier, reads \0 as the start of an
+// octal escape, and reads \v as the whole vertical whitespace class rather than as U+000B.
+// All four rewrites are no-ops for ECMAScript and for Boost, which is what libprisma
+// compiles the patterns with.
+const QUANTIFIER = /^\{\d+(?:,\d*)?\}/
+
+function normalizeEscapes(pattern) {
+ let result = ''
+ let inClass = false
+
+ for (let i = 0; i < pattern.length; i++) {
+ const c = pattern[i]
+
+ if (c === '\\') {
+ if (pattern[i + 1] === '0' && !/[0-7]/.test(pattern[i + 2] || '')) {
+ result += '\\x00'
+ } else if (pattern[i + 1] === 'v') {
+ result += '\\x0B'
+ } else {
+ result += pattern.substr(i, 2)
+ }
+ i++
+ continue
+ }
+
+ if (inClass) {
+ if (c === '[') {
+ result += '\\['
+ continue
+ }
+ // && is set intersection in java.util.regex
+ if (c === '&' && pattern[i + 1] === '&') {
+ result += '\\&\\&'
+ i++
+ continue
+ }
+ if (c === ']') {
+ inClass = false
+ }
+ } else if (c === '[') {
+ inClass = true
+ } else if (c === '{' && !QUANTIFIER.test(pattern.substr(i))) {
+ result += '\\{'
+ continue
+ }
+
+ result += c
+ }
+
+ return result
+}
+
+// A character written literally in a grammar - BQN's •, APL's ⍵ - has to reach the readers as
+// an escape rather than as itself: they disagree about what the bytes of a pattern mean, and
+// \uXXXX is the one spelling all of them read as a code point. An astral character is two code
+// units here, so it becomes a surrogate pair, which UnicodeEscapes.h puts back together.
+function escapeNonAscii(pattern) {
+ if (!/[^\x00-\x7F]/.test(pattern)) {
+ return pattern
+ }
+
+ let result = ''
+
+ for (let i = 0; i < pattern.length; i++) {
+ const code = pattern.charCodeAt(i)
+ result += code > 0x7F
+ ? '\\u' + code.toString(16).padStart(4, '0')
+ : pattern[i]
+ }
+
+ return result
+}
+
function unique(a, fn) {
if (a.length === 0 || a.length === 1) {
return a;
@@ -137,7 +210,7 @@ async function generate() {
pattern = pattern.replaceAll("|[])", ")");
pattern = pattern.replaceAll(":[]", ":");
- return pattern
+ return escapeNonAscii(normalizeEscapes(pattern))
}
for (var token in copy) {
@@ -215,14 +288,17 @@ async function generate() {
"sparql" // requires turtle
]
- await loadScript("https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/prism-core.min.js")
- await loadScript("https://prismjs.com/components.js")
await loadLanguages(Object.keys(components.languages))
console.log(`\nLoaded all ${langNumber} languages`)
console.log("Processing...")
// Manually add local definitions
loadLocalLanguage('./components/prism-tl.js', 'typelanguage', 'TypeLanguage', 'tl')
+ loadLocalLanguage('./components/prism-tlb.js', 'tlb', 'TypeLanguage-Binary', 'tlb')
+ loadLocalLanguage('./components/prism-fift.js', 'fift', 'Fift', 'fift')
+ loadLocalLanguage('./components/prism-func.js', 'func', 'FunC', ['func', 'fc'])
+ loadLocalLanguage('./components/prism-tact.js', 'tact', 'Tact', 'tact')
+ loadLocalLanguage('./components/prism-tolk.js', 'tolk', 'Tolk', 'tolk')
Object.keys(Prism.languages).forEach(lng => {
if (unsupported.includes(lng) || !components.languages[lng]) {
@@ -338,11 +414,11 @@ async function generate() {
for (const lng of alias) {
allLanguages[lng] = allGrammars.indexOf(find)
- if (components.languages[name].aliasTitles) {
- languageNames[lng] = components.languages[name].aliasTitles[lng]
- } else {
- languageNames[lng] = components.languages[name].title
- }
+ // aliasTitles only names the aliases whose title differs from the language's,
+ // so an alias missing from it takes the language title rather than no title:
+ // an empty one drops the alias out of SyntaxHighlighter::languages().
+ const aliasTitles = components.languages[name].aliasTitles
+ languageNames[lng] = (aliasTitles && aliasTitles[lng]) || components.languages[name].title
}
}
})
@@ -358,15 +434,18 @@ async function generate() {
const writeUint16 = i => chunks.push(new Uint16Array([i]))
const writeUint8 = i => chunks.push(new Uint8Array([i]))
const writeString = str => {
- if (str.length < 253) {
- writeUint8(str.length)
+ // one byte per code unit truncated anything above U+00FF; the length is the byte
+ // count, which is what the readers advance by
+ const bytes = new TextEncoder().encode(str)
+ if (bytes.length < 253) {
+ writeUint8(bytes.length)
} else {
writeUint8(254 & 0xFF)
- writeUint8(str.length & 0xFF)
- writeUint8((str.length >> 8) & 0xFF)
- writeUint8((str.length >> 16) & 0xFF)
+ writeUint8(bytes.length & 0xFF)
+ writeUint8((bytes.length >> 8) & 0xFF)
+ writeUint8((bytes.length >> 16) & 0xFF)
}
- chunks.push(new Uint8Array(str.split('').map(char => char.charCodeAt(0))))
+ chunks.push(bytes)
}
// Patterns
diff --git a/libprisma/Highlight.cpp b/libprisma/Highlight.cpp
index 8d55452..6e45235 100644
--- a/libprisma/Highlight.cpp
+++ b/libprisma/Highlight.cpp
@@ -31,6 +31,11 @@ const Pattern* PatternPtr::get() const
return m_tree->resolvePattern(m_path);
}
+std::shared_ptr PatternRaw::realize()
+{
+ return std::make_shared(m_regex, m_flags, m_lookbehind, m_greedy, m_alias, m_inside);
+}
+
const Grammar* Pattern::inside() const
{
if (m_inside)
diff --git a/libprisma/Highlight.h b/libprisma/Highlight.h
index a1130e0..6248700 100644
--- a/libprisma/Highlight.h
+++ b/libprisma/Highlight.h
@@ -29,17 +29,35 @@ private:
size_t m_path;
};
+class PatternRaw
+{
+public:
+ PatternRaw(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr inside)
+ : m_regex(std::string{pattern})
+ , m_flags(flags)
+ , m_lookbehind(lookbehind)
+ , m_greedy(greedy)
+ , m_alias(alias)
+ , m_inside(inside)
+ {
+ }
+
+ std::shared_ptr realize();
+
+private:
+ std::string m_regex;
+ boost::regex_constants::syntax_option_type m_flags;
+ bool m_lookbehind;
+ bool m_greedy;
+ std::string m_alias;
+ std::shared_ptr m_inside;
+};
+
class Pattern
{
public:
- Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, GrammarPtr inside)
- : Pattern(pattern, flags, lookbehind, greedy, alias, std::make_shared(inside))
- {
-
- }
-
- Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind = false, bool greedy = false, std::string alias = "", std::shared_ptr inside = nullptr)
- : m_regex(boost::regex(std::string{pattern}, flags | boost::regex_constants::optimize))
+ Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, std::shared_ptr inside)
+ : m_regex(boost::regex(std::string{ pattern }, flags | boost::regex_constants::optimize))
, m_lookbehind(lookbehind)
, m_greedy(greedy)
, m_alias(alias)
diff --git a/libprisma/LanguageTree.cpp b/libprisma/LanguageTree.cpp
index 1518bcb..01c9902 100644
--- a/libprisma/LanguageTree.cpp
+++ b/libprisma/LanguageTree.cpp
@@ -1,6 +1,7 @@
#include "LanguageTree.h"
#include "TokenList.h"
+#include "UnicodeEscapes.h"
#include
void LanguageTree::load(const std::string& content)
@@ -96,6 +97,8 @@ void LanguageTree::parsePatterns(Buffer &buffer)
{
uint16_t count = freadUint16(buffer);
+ m_patterns.resize(count);
+
for (int i = 0; i < count; ++i)
{
std::string item = freadString(buffer);
@@ -163,11 +166,11 @@ void LanguageTree::parsePatterns(Buffer &buffer)
if (inside != std::string::npos)
{
- m_patterns.push_back(std::make_shared(pattern, flags, lookbehind, greedy, std::string{ alias }, std::make_shared(shared_from_this(), inside)));
+ m_patternsRaw.push_back(std::make_shared(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, std::make_shared(shared_from_this(), inside)));
}
else
{
- m_patterns.push_back(std::make_shared(pattern, flags, lookbehind, greedy, std::string{ alias }));
+ m_patternsRaw.push_back(std::make_shared(libprisma::widenUnicodeClasses(pattern), flags, lookbehind, greedy, std::string{ alias }, nullptr));
}
}
}
@@ -175,7 +178,16 @@ void LanguageTree::parsePatterns(Buffer &buffer)
const Pattern* LanguageTree::resolvePattern(size_t path)
{
- return m_patterns[path].get();
+ std::shared_ptr &parsed = m_patterns[path];
+ if (parsed == nullptr)
+ {
+ parsed = m_patternsRaw[path]->realize();
+
+ m_patternsRaw[path] = nullptr;
+ m_patterns[path] = parsed;
+ }
+
+ return parsed.get();
}
const Grammar* LanguageTree::resolveGrammar(size_t path)
diff --git a/libprisma/LanguageTree.h b/libprisma/LanguageTree.h
index 422a2ff..46b613b 100644
--- a/libprisma/LanguageTree.h
+++ b/libprisma/LanguageTree.h
@@ -38,6 +38,17 @@ public:
return keys;
}
+ std::string languageName(const std::string& language) const
+ {
+ const auto& find = m_languages.find(language);
+ if (find != m_languages.end())
+ {
+ return find->second.first;
+ }
+
+ return language;
+ }
+
const Grammar* find(const std::string& key) const
{
const auto& value = m_languages.find(key);
@@ -57,4 +68,5 @@ private:
std::map> m_languages;
std::vector> m_grammars;
std::vector> m_patterns;
+ std::vector> m_patternsRaw;
};
\ No newline at end of file
diff --git a/libprisma/SyntaxHighlighter.cpp b/libprisma/SyntaxHighlighter.cpp
index 69c158b..2298335 100644
--- a/libprisma/SyntaxHighlighter.cpp
+++ b/libprisma/SyntaxHighlighter.cpp
@@ -24,6 +24,11 @@ std::map SyntaxHighlighter::languages() const
return m_tree->keys();
}
+std::string SyntaxHighlighter::languageName(const std::string& language) const
+{
+ return m_tree->languageName(language);
+}
+
TokenList SyntaxHighlighter::tokenize(std::string_view text, const Grammar* grammar)
{
// Grammars may include themselves through "inside" (for example
@@ -62,9 +67,9 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
return;
}
- const auto& pattern = *j;
- const auto& inside = pattern->inside();
- const bool greedy = pattern->greedy();
+ const auto& pattern = *j->get();
+ const auto& inside = pattern.inside();
+ const bool greedy = pattern.greedy();
size_t pos = startPos;
@@ -99,7 +104,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
if (greedy)
{
- match = pattern->match(matchSuccess, matchIndex, text);
+ match = pattern.match(matchSuccess, matchIndex, text);
if (!matchSuccess || matchIndex >= text.length())
{
break;
@@ -143,7 +148,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
else
{
matchIndex = 0;
- match = pattern->match(matchSuccess, matchIndex, str);
+ match = pattern.match(matchSuccess, matchIndex, str);
if (!matchSuccess)
{
continue;
@@ -183,7 +188,7 @@ void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList
currentNode = tokenList.addAfter(removeFrom, token.name(),
std::move(tokenEntries),
- pattern->alias(),
+ pattern.alias(),
match.size());
if (after.size())
diff --git a/libprisma/SyntaxHighlighter.h b/libprisma/SyntaxHighlighter.h
index a4c981f..8e435bf 100644
--- a/libprisma/SyntaxHighlighter.h
+++ b/libprisma/SyntaxHighlighter.h
@@ -25,6 +25,8 @@ public:
std::map languages() const;
+ std::string languageName(const std::string& language) const;
+
private:
TokenList tokenize(std::string_view text, const Grammar* grammar);
void matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch);
diff --git a/libprisma/UnicodeEscapes.h b/libprisma/UnicodeEscapes.h
new file mode 100644
index 0000000..d72c450
--- /dev/null
+++ b/libprisma/UnicodeEscapes.h
@@ -0,0 +1,354 @@
+#pragma once
+
+#include
+#include
+#include
+
+namespace libprisma
+{
+ // Boost matches over bytes and does not read a \uXXXX escape at all: a pattern holding one
+ // simply never matches. Spell every escape as the bytes of its UTF-8 encoding instead, and
+ // where one sits inside a character class - whose members have to be a single byte each -
+ // lift it out into an alternation. This belongs here rather than in grammars.dat, which
+ // keeps the \u spelling for the clients that match over UTF-16 and read it correctly.
+
+ inline bool isHexDigit(char c)
+ {
+ return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+ }
+
+ inline int hexValue(char c)
+ {
+ return c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10;
+ }
+
+ // the code point of the \uXXXX escape at begin, or -1 if there is not one there
+ inline int unicodeEscapeAt(std::string_view pattern, size_t begin)
+ {
+ if (begin + 6 > pattern.size() || pattern[begin] != '\\' || pattern[begin + 1] != 'u')
+ {
+ return -1;
+ }
+
+ int value = 0;
+ for (size_t i = begin + 2; i < begin + 6; ++i)
+ {
+ if (!isHexDigit(pattern[i]))
+ {
+ return -1;
+ }
+
+ value = value * 16 + hexValue(pattern[i]);
+ }
+
+ return value;
+ }
+
+ inline void appendByte(std::string& out, unsigned char byte)
+ {
+ static const char digits[] = "0123456789ABCDEF";
+
+ out += "\\x";
+ out += digits[byte >> 4];
+ out += digits[byte & 0x0F];
+ }
+
+ inline void appendUtf8(std::string& out, int codePoint)
+ {
+ if (codePoint < 0x80)
+ {
+ appendByte(out, static_cast(codePoint));
+ }
+ else if (codePoint < 0x800)
+ {
+ appendByte(out, static_cast(0xC0 | (codePoint >> 6)));
+ appendByte(out, static_cast(0x80 | (codePoint & 0x3F)));
+ }
+ else if (codePoint < 0x10000)
+ {
+ appendByte(out, static_cast(0xE0 | (codePoint >> 12)));
+ appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F)));
+ appendByte(out, static_cast(0x80 | (codePoint & 0x3F)));
+ }
+ else
+ {
+ appendByte(out, static_cast(0xF0 | (codePoint >> 18)));
+ appendByte(out, static_cast(0x80 | ((codePoint >> 12) & 0x3F)));
+ appendByte(out, static_cast(0x80 | ((codePoint >> 6) & 0x3F)));
+ appendByte(out, static_cast(0x80 | (codePoint & 0x3F)));
+ }
+ }
+
+ // The code point at begin, joining a surrogate pair into the one character it spells, with
+ // the characters it occupies in length. A lone \uXXXX is left to Boost, which expands it
+ // to UTF-8 itself; a pair it would expand to two three byte sequences instead.
+ inline int codePointAt(std::string_view pattern, size_t begin, size_t& length)
+ {
+ const int value = unicodeEscapeAt(pattern, begin);
+ length = value < 0 ? 0 : 6;
+
+ if (value >= 0xD800 && value <= 0xDBFF)
+ {
+ const int low = unicodeEscapeAt(pattern, begin + 6);
+ if (low >= 0xDC00 && low <= 0xDFFF)
+ {
+ length = 12;
+ return 0x10000 + ((value - 0xD800) << 10) + (low - 0xDC00);
+ }
+ }
+
+ return value;
+ }
+
+ struct ClassMember
+ {
+ std::string_view text; // as spelled in the pattern, "a", "\\n", "\\u2227", "a-z"
+ int low = -1; // code point, for a member that is a lone \u escape
+ int high = -1; // and the other end, when the member is a range of them
+ };
+
+ struct CharacterClass
+ {
+ size_t end = 0; // index just past the closing ]
+ bool negated = false;
+ bool wide = false; // holds a code point that cannot be a single byte
+ std::vector members;
+ };
+
+ // one class member starting at begin, or 0 if the class ends there
+ inline size_t readMember(std::string_view pattern, size_t begin, ClassMember& member)
+ {
+ auto unit = [&](size_t at, int& codePoint) -> size_t {
+ size_t length = 0;
+ codePoint = codePointAt(pattern, at, length);
+ if (codePoint >= 0)
+ {
+ return length;
+ }
+ return pattern[at] == '\\' && at + 1 < pattern.size() ? 2 : 1;
+ };
+
+ int low = -1;
+ size_t length = unit(begin, low);
+ size_t at = begin + length;
+
+ // a - b, where a trailing - is a literal rather than the start of a range
+ if (at + 1 < pattern.size() && pattern[at] == '-' && pattern[at + 1] != ']')
+ {
+ int high = -1;
+ size_t tail = unit(at + 1, high);
+ member.low = low;
+ member.high = high;
+ member.text = pattern.substr(begin, length + 1 + tail);
+ return length + 1 + tail;
+ }
+
+ member.low = low;
+ member.text = pattern.substr(begin, length);
+ return length;
+ }
+
+ inline bool readClass(std::string_view pattern, size_t begin, CharacterClass& result)
+ {
+ size_t at = begin + 1;
+ if (at < pattern.size() && pattern[at] == '^')
+ {
+ result.negated = true;
+ ++at;
+ }
+
+ while (at < pattern.size() && pattern[at] != ']')
+ {
+ ClassMember member;
+ size_t length = readMember(pattern, at, member);
+ if (length == 0)
+ {
+ return false;
+ }
+
+ if (member.low > 0x7F || member.high > 0x7F)
+ {
+ result.wide = true;
+ }
+
+ result.members.push_back(member);
+ at += length;
+ }
+
+ if (at >= pattern.size())
+ {
+ return false;
+ }
+
+ result.end = at + 1;
+ return true;
+ }
+
+ inline bool isRange(const CharacterClass& characters, int low, int high)
+ {
+ return characters.members.size() == 1
+ && characters.members[0].low == low
+ && characters.members[0].high == high;
+ }
+
+ // [\uD800-\uDBFF][\uDC00-\uDFFF] and [\uD800-\uDFFF]{2} are how a JavaScript regex spells
+ // one astral character, as its two UTF-16 halves. In UTF-8 that is one four byte sequence.
+ inline size_t appendSurrogatePair(std::string& out, std::string_view pattern, size_t begin)
+ {
+ CharacterClass lead;
+ if (!readClass(pattern, begin, lead) || lead.negated)
+ {
+ return 0;
+ }
+
+ size_t end = 0;
+ if (isRange(lead, 0xD800, 0xDBFF))
+ {
+ CharacterClass trail;
+ if (!readClass(pattern, lead.end, trail) || trail.negated || !isRange(trail, 0xDC00, 0xDFFF))
+ {
+ return 0;
+ }
+ end = trail.end;
+ }
+ else if (isRange(lead, 0xD800, 0xDFFF) && pattern.substr(lead.end, 3) == "{2}")
+ {
+ end = lead.end + 3;
+ }
+ else
+ {
+ return 0;
+ }
+
+ out += "(?:[\\xF0-\\xF4][\\x80-\\xBF]{3})";
+ return end - begin;
+ }
+
+ // the class at begin, with any member that cannot be a single byte moved out of it
+ inline size_t appendClass(std::string& out, std::string_view pattern, size_t begin)
+ {
+ CharacterClass characters;
+ if (!readClass(pattern, begin, characters) || !characters.wide)
+ {
+ return 0;
+ }
+
+ std::vector wide;
+ std::string narrow;
+
+ for (const auto& member : characters.members)
+ {
+ if (member.low <= 0x7F && member.high <= 0x7F)
+ {
+ narrow += member.text;
+ }
+ else if (member.high < 0)
+ {
+ wide.push_back(member.low);
+ }
+ else if (!characters.negated)
+ {
+ // a range of code points that is not the surrogate idiom; leave it be rather
+ // than expand what could be the whole of Unicode
+ return 0;
+ }
+ }
+
+ // a negated class drops its wide members: they exclude characters that cannot occur
+ // as bytes in valid UTF-8 anyway, and as bytes they would exclude unrelated ones
+ if (characters.negated || wide.empty())
+ {
+ out += '[';
+ if (characters.negated)
+ {
+ out += '^';
+ }
+ out += narrow;
+ out += ']';
+ return characters.end - begin;
+ }
+
+ out += "(?:";
+ if (!narrow.empty())
+ {
+ out += '[';
+ out += narrow;
+ out += ']';
+ out += '|';
+ }
+ for (size_t i = 0; i < wide.size(); ++i)
+ {
+ if (i > 0)
+ {
+ out += '|';
+ }
+ appendUtf8(out, wide[i]);
+ }
+ out += ')';
+
+ return characters.end - begin;
+ }
+
+ inline std::string widenUnicodeClasses(std::string_view pattern)
+ {
+ if (pattern.find("\\u") == std::string_view::npos)
+ {
+ return std::string{ pattern };
+ }
+
+ std::string out;
+ out.reserve(pattern.size());
+
+ for (size_t i = 0; i < pattern.size(); )
+ {
+ if (pattern[i] == '\\')
+ {
+ size_t length = 0;
+ const int codePoint = codePointAt(pattern, i, length);
+
+ if (codePoint >= 0)
+ {
+ // grouped, because a quantifier after it has to apply to the whole
+ // character and not to the last byte of its encoding
+ const bool multiByte = codePoint > 0x7F;
+ if (multiByte)
+ {
+ out += "(?:";
+ }
+ appendUtf8(out, codePoint);
+ if (multiByte)
+ {
+ out += ')';
+ }
+
+ i += length;
+ continue;
+ }
+
+ out += pattern.substr(i, 2);
+ i += 2;
+ continue;
+ }
+
+ if (pattern[i] == '[')
+ {
+ if (size_t consumed = appendSurrogatePair(out, pattern, i))
+ {
+ i += consumed;
+ continue;
+ }
+
+ if (size_t consumed = appendClass(out, pattern, i))
+ {
+ i += consumed;
+ continue;
+ }
+ }
+
+ out += pattern[i];
+ ++i;
+ }
+
+ return out;
+ }
+}
diff --git a/libprisma/grammars.dat b/libprisma/grammars.dat
index 532ed27..783d168 100644
Binary files a/libprisma/grammars.dat and b/libprisma/grammars.dat differ
diff --git a/package-lock.json b/package-lock.json
index 1f9a3ba..c10dd45 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9,13 +9,23 @@
"version": "1.0.0",
"license": "MIT",
"dependencies": {
- "lodash.isequal": "^4.5.0"
+ "lodash.isequal": "^4.5.0",
+ "prismjs": "1.29.0"
}
},
"node_modules/lodash.isequal": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz",
"integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ=="
+ },
+ "node_modules/prismjs": {
+ "version": "1.29.0",
+ "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",
+ "integrity": "sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=6"
+ }
}
}
}
diff --git a/package.json b/package.json
index 828a9ac..094cf10 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,7 @@
},
"homepage": "https://github.com/TelegramMessenger/libprisma#readme",
"dependencies": {
- "lodash.isequal": "^4.5.0"
+ "lodash.isequal": "^4.5.0",
+ "prismjs": "1.29.0"
}
}
diff --git a/tools/PatternProbe.java b/tools/PatternProbe.java
new file mode 100644
index 0000000..92296a6
--- /dev/null
+++ b/tools/PatternProbe.java
@@ -0,0 +1,163 @@
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * Compiles every pattern of a grammars.dat with java.util.regex and writes down what each one
+ * matches in a corpus, for probe.cpp to be diffed against. Between them they hold the table to
+ * the one thing it promises: that a pattern means the same to every client that reads it.
+ *
+ * The corpus is deliberately ASCII. Boost matches bytes and java.util.regex matches chars, so
+ * anything above U+007F diverges on offsets alone and would drown out the escape bugs this is
+ * looking for.
+ */
+public class PatternProbe {
+
+ /** A pattern record, "/source/flags,alias,inside", as LanguageTree.cpp reads it. */
+ private static final class Entry {
+ String source;
+ int flags;
+ }
+
+ private static final class Reader {
+ private final byte[] bytes;
+ private int offset;
+
+ Reader(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ int uint8() {
+ return bytes[offset++] & 0xFF;
+ }
+
+ int uint16() {
+ return uint8() | (uint8() << 8);
+ }
+
+ String string() {
+ int length = uint8();
+ if (length >= 254) {
+ length = uint8() | (uint8() << 8) | (uint8() << 16);
+ }
+ String value = new String(bytes, offset, length, StandardCharsets.ISO_8859_1);
+ offset += length;
+ return value;
+ }
+ }
+
+ private static List read(String path) throws IOException {
+ Reader reader = new Reader(Files.readAllBytes(Paths.get(path)));
+ List entries = new ArrayList<>();
+
+ for (int count = reader.uint16(); count > 0; count--) {
+ String item = reader.string();
+ int begin = item.indexOf('/');
+ int end = item.lastIndexOf('/');
+ if (begin < 0 || end <= begin) {
+ continue;
+ }
+
+ String options = item.substring(end + 1);
+ int aliasBegin = options.indexOf(',');
+
+ Entry entry = new Entry();
+ entry.source = item.substring(begin + 1, end);
+ for (int i = 0; i < aliasBegin; ++i) {
+ switch (options.charAt(i)) {
+ case 'i': entry.flags |= Pattern.CASE_INSENSITIVE; break;
+ case 'm': entry.flags |= Pattern.MULTILINE; break;
+ default: break; // l and y are libprisma's, for Prism lookbehind and greedy
+ }
+ }
+ entries.add(entry);
+ }
+
+ return entries;
+ }
+
+ /**
+ * Outside MULTILINE, $ means end of input in ECMAScript but "end of input, or before a
+ * line terminator that ends it" in java.util.regex. \z is the Java spelling of the first.
+ * A Java client has to do this; it cannot go in the table, where \z would read as a z.
+ */
+ private static String endOfInput(String source) {
+ if (source.indexOf('$') < 0) {
+ return source;
+ }
+
+ StringBuilder result = new StringBuilder(source.length());
+ boolean inClass = false;
+ for (int i = 0; i < source.length(); ++i) {
+ char c = source.charAt(i);
+ if (c == '\\' && i + 1 < source.length()) {
+ result.append(c).append(source.charAt(++i));
+ } else if (c == '[') {
+ inClass = true;
+ result.append(c);
+ } else if (c == ']') {
+ inClass = false;
+ result.append(c);
+ } else if (c == '$' && !inClass) {
+ result.append("\\z");
+ } else {
+ result.append(c);
+ }
+ }
+ return result.toString();
+ }
+
+ public static void main(String[] args) throws IOException {
+ List entries = read(args[0]);
+ String corpus = new String(Files.readAllBytes(Paths.get(args[1])), StandardCharsets.ISO_8859_1)
+ .replace(String.valueOf((char) 13), "");
+
+ StringBuilder out = new StringBuilder();
+ int failed = 0;
+
+ for (int i = 0; i < entries.size(); ++i) {
+ Entry entry = entries.get(i);
+ String source = entry.source;
+ if ((entry.flags & Pattern.MULTILINE) == 0) {
+ source = endOfInput(source);
+ }
+
+ Pattern pattern;
+ try {
+ pattern = Pattern.compile(source, entry.flags | Pattern.UNIX_LINES);
+ } catch (PatternSyntaxException e) {
+ if (failed++ < 20) {
+ System.err.println("pattern " + i + ": " + e.getDescription());
+ System.err.println(" " + entry.source.substring(0, Math.min(140, entry.source.length())));
+ }
+ continue;
+ }
+
+ out.append(i);
+ java.util.regex.Matcher matcher = pattern.matcher(corpus);
+ int at = 0;
+ while (matcher.find(at) && out.length() < Integer.MAX_VALUE) {
+ out.append(' ').append(matcher.start()).append(':').append(matcher.end());
+ at = matcher.end() > matcher.start() ? matcher.end() : matcher.start() + 1;
+ if (at > corpus.length()) {
+ break;
+ }
+ }
+ out.append('\n');
+ }
+
+ Files.write(Paths.get(args[2]), out.toString().getBytes(StandardCharsets.US_ASCII));
+ System.out.println("compiled " + (entries.size() - failed) + " of " + entries.size() + " patterns");
+
+ if (failed > 0) {
+ System.err.println(failed + " patterns java.util.regex will not take. "
+ + "Teach normalizeEscapes in generate.js to rewrite them.");
+ System.exit(1);
+ }
+ }
+}
diff --git a/tools/check-escapes.js b/tools/check-escapes.js
new file mode 100644
index 0000000..a7d4afe
--- /dev/null
+++ b/tools/check-escapes.js
@@ -0,0 +1,87 @@
+// Every escape in grammars.dat has to mean the same thing to every engine that reads it.
+// A pattern that fails to compile is caught by the probes beside this script; this one is for
+// the escapes that compile everywhere and quietly mean something else, which is the worse
+// failure - \v was read as U+000B by Boost and as the whole vertical whitespace class, newline
+// included, by java.util.regex, which made ini values run over the end of the line.
+const fs = require('fs')
+
+// what generate.js is expected to emit, once normalizeEscapes has run over it
+const ALLOWED = new Set('BSWbdfnrstuwx123456789'.split(''))
+
+const REASONS = {
+ v: 'the vertical whitespace class in java.util.regex, U+000B elsewhere - emit \\x0B',
+ h: 'the horizontal whitespace class in java.util.regex, a literal h elsewhere',
+ H: 'non-horizontal-whitespace in java.util.regex, a literal H elsewhere',
+ V: 'non-vertical-whitespace in java.util.regex, a literal V elsewhere',
+ R: 'any line break in java.util.regex, a literal R elsewhere',
+ X: 'a grapheme cluster in java.util.regex, a literal X elsewhere',
+ A: 'start of input in java.util.regex, a literal A elsewhere',
+ Z: 'end of input in java.util.regex, a literal Z elsewhere',
+ z: 'end of input in java.util.regex, a literal z elsewhere',
+ G: 'end of the previous match in java.util.regex, a literal G elsewhere',
+ Q: 'starts a quoted run in java.util.regex, a literal Q elsewhere',
+ E: 'ends one in java.util.regex, a literal E elsewhere',
+ p: 'a unicode property in both, but the property names differ',
+ N: 'a named character in java.util.regex',
+ 0: 'starts an octal escape in java.util.regex - emit \\x00',
+}
+
+function readPatterns(path) {
+ const buffer = fs.readFileSync(path)
+ let offset = 0
+
+ const uint8 = () => buffer[offset++]
+ const uint16 = () => { const value = buffer.readUInt16LE(offset); offset += 2; return value }
+ const string = () => {
+ let length = uint8()
+ if (length >= 254) {
+ length = uint8() | (uint8() << 8) | (uint8() << 16)
+ }
+ const value = buffer.toString('latin1', offset, offset + length)
+ offset += length
+ return value
+ }
+
+ const patterns = []
+ for (let count = uint16(); count > 0; count--) {
+ patterns.push(string())
+ }
+ return patterns
+}
+
+const patterns = readPatterns(process.argv[2] || 'libprisma/grammars.dat')
+const problems = []
+
+for (const item of patterns) {
+ const end = item.lastIndexOf('/')
+ const source = item.slice(item.indexOf('/') + 1, end < 0 ? undefined : end)
+
+ for (let i = 0; i < source.length; i++) {
+ if (source[i] !== '\\') {
+ continue
+ }
+
+ const escaped = source[++i]
+ if (/[a-zA-Z0-9]/.test(escaped) && !ALLOWED.has(escaped)) {
+ problems.push([source, '\\' + escaped, REASONS[escaped] || 'not in the allowed set'])
+ }
+ }
+}
+
+console.log(`${patterns.length} patterns checked`)
+
+if (problems.length === 0) {
+ process.exit(0)
+}
+
+const seen = new Set()
+for (const [source, escape, reason] of problems) {
+ if (seen.has(escape)) {
+ continue
+ }
+ seen.add(escape)
+ console.error(`\n${escape} is ${reason}`)
+ console.error(` e.g. ${source.slice(0, 120)}`)
+}
+console.error(`\n${problems.length} occurrences. Teach normalizeEscapes to rewrite them, or add the escape to ALLOWED once every engine is known to agree on it.`)
+process.exit(1)
diff --git a/tools/corpus.txt b/tools/corpus.txt
new file mode 100644
index 0000000..2086075
--- /dev/null
+++ b/tools/corpus.txt
@@ -0,0 +1,13 @@
+class Foo where T : IBar { public int x = 1; } /* c */ // line
+{{ tpl }}
+def f(a, b=[1,2], *args, **kw): return {'k': a[0]} # py
+let re = /ab[c\]]+/g; const s = `t${x}`; arr[0] ?? 3;
+SELECT * FROM t WHERE a IN (1,2) -- sql
+#include
+key = value ; ini
+[section]
+- item:
+ nested: [1, 2]
+0x1F 0b1010 1.5e-3 'c' "str" :sym @attr $var %h &&x ||y
+ tab and trailing spaces
+#!/bin/sh
diff --git a/tools/probe.cpp b/tools/probe.cpp
new file mode 100644
index 0000000..4881df4
--- /dev/null
+++ b/tools/probe.cpp
@@ -0,0 +1,132 @@
+// Compiles every pattern of a grammars.dat with Boost, the way libprisma does, and writes down
+// what each one matches in a corpus. PatternProbe.java does the same with java.util.regex and
+// the two files are diffed, which is what holds the table to meaning one thing everywhere.
+//
+// The corpus is deliberately ASCII: Boost matches bytes where java.util.regex matches chars, so
+// anything above U+007F diverges on offsets alone and would drown out the escape bugs.
+#include
+
+#include "../libprisma/UnicodeEscapes.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace {
+
+ std::string readAll(const char* path) {
+ std::ifstream in(path, std::ios::binary);
+ return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator());
+ }
+
+ struct Reader {
+ const std::string& bytes;
+ size_t offset = 0;
+
+ uint8_t uint8() {
+ return offset < bytes.size() ? static_cast(bytes[offset++]) : 0;
+ }
+
+ uint16_t uint16() {
+ uint16_t low = uint8();
+ return static_cast(low | (uint8() << 8));
+ }
+
+ std::string string() {
+ size_t length = uint8();
+ if (length >= 254) {
+ size_t a = uint8(), b = uint8(), c = uint8();
+ length = a | (b << 8) | (c << 16);
+ }
+ std::string value = bytes.substr(offset, length);
+ offset += length;
+ return value;
+ }
+ };
+
+ struct Entry {
+ std::string source;
+ boost::regex_constants::syntax_option_type flags;
+ };
+
+ // "/source/flags,alias,inside", the record LanguageTree::parsePatterns reads
+ std::vector read(const std::string& table) {
+ Reader reader{ table };
+ std::vector entries;
+
+ for (uint16_t count = reader.uint16(); count > 0; --count) {
+ const std::string item = reader.string();
+ const size_t begin = item.find_first_of('/');
+ const size_t end = item.find_last_of('/');
+ if (begin == std::string::npos || end <= begin) {
+ continue;
+ }
+
+ const std::string options = item.substr(end + 1);
+ const size_t aliasBegin = options.find_first_of(',');
+
+ Entry entry;
+ entry.source = item.substr(begin + 1, end - (begin + 1));
+ entry.flags = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m;
+
+ for (size_t i = 0; i < aliasBegin && i < options.size(); ++i) {
+ switch (options[i]) {
+ case 'i': entry.flags |= boost::regex_constants::icase; break;
+ case 'm': entry.flags &= ~boost::regex_constants::no_mod_m; break;
+ default: break; // l and y are libprisma's, for Prism lookbehind and greedy
+ }
+ }
+
+ entries.push_back(entry);
+ }
+
+ return entries;
+ }
+}
+
+int main(int argc, char** argv) {
+ if (argc < 4) {
+ fprintf(stderr, "usage: probe \n");
+ return 2;
+ }
+
+ const std::vector entries = read(readAll(argv[1]));
+
+ std::string corpus = readAll(argv[2]);
+ corpus.erase(std::remove(corpus.begin(), corpus.end(), '\r'), corpus.end());
+
+ std::ofstream out(argv[3], std::ios::binary);
+ int failed = 0;
+
+ for (size_t i = 0; i < entries.size(); ++i) {
+ boost::regex re;
+ try {
+ re.assign(libprisma::widenUnicodeClasses(entries[i].source), entries[i].flags);
+ } catch (const std::exception& e) {
+ if (failed++ < 20) {
+ fprintf(stderr, "pattern %zu: %s\n %.140s\n", i, e.what(), entries[i].source.c_str());
+ }
+ continue;
+ }
+
+ out << i;
+ auto it = boost::sregex_iterator(corpus.begin(), corpus.end(), re,
+ boost::regex_constants::match_not_dot_newline);
+ for (auto end = boost::sregex_iterator(); it != end; ++it) {
+ out << ' ' << it->position() << ':' << (it->position() + it->length());
+ }
+ out << '\n';
+ }
+
+ printf("compiled %zu of %zu patterns\n", entries.size() - failed, entries.size());
+
+ if (failed > 0) {
+ fprintf(stderr, "%d patterns Boost will not take.\n", failed);
+ return 1;
+ }
+ return 0;
+}