Skip to content

Commit 51e3ecc

Browse files
RunDevelopmentRunDevelopment
authored
Crystal: Improved tokenization (#3194)
1 parent e137035 commit 51e3ecc

11 files changed

Lines changed: 759 additions & 66 deletions

‎components/prism-crystal.js‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,57 @@
11
(function (Prism) {
22
Prism.languages.crystal = Prism.languages.extend('ruby', {
3-
keyword: [
4-
/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield)\b/,
3+
'keyword': [
4+
/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|annotation|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|ifdef|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|ptr|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|undef|uninitialized|union|unless|until|when|while|with|yield)\b/,
55
{
66
pattern: /(\.\s*)(?:is_a|responds_to)\?/,
77
lookbehind: true
88
}
99
],
10-
11-
number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/
10+
'number': /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/,
11+
'operator': [
12+
/->/,
13+
Prism.languages.ruby.operator,
14+
],
15+
'punctuation': /[(){}[\].,;\\]/,
1216
});
1317

1418
Prism.languages.insertBefore('crystal', 'string-literal', {
15-
attribute: {
16-
pattern: /@\[.+?\]/,
17-
alias: 'attr-name',
19+
'attribute': {
20+
pattern: /@\[.*?\]/,
1821
inside: {
19-
delimiter: {
22+
'delimiter': {
2023
pattern: /^@\[|\]$/,
21-
alias: 'tag'
24+
alias: 'punctuation'
25+
},
26+
'attribute': {
27+
pattern: /^(\s*)\w+/,
28+
lookbehind: true,
29+
alias: 'class-name'
30+
},
31+
'args': {
32+
pattern: /\S(?:[\s\S]*\S)?/,
33+
inside: Prism.languages.crystal
2234
},
23-
rest: Prism.languages.crystal
2435
}
2536
},
26-
27-
expansion: [
28-
{
29-
pattern: /\{\{.+?\}\}/,
30-
inside: {
31-
delimiter: {
32-
pattern: /^\{\{|\}\}$/,
33-
alias: 'tag'
34-
},
35-
rest: Prism.languages.crystal
36-
}
37-
},
38-
{
39-
pattern: /\{%.+?%\}/,
40-
inside: {
41-
delimiter: {
42-
pattern: /^\{%|%\}$/,
43-
alias: 'tag'
44-
},
45-
rest: Prism.languages.crystal
37+
'expansion': {
38+
pattern: /\{(?:\{.*?\}|%.*?%)\}/,
39+
inside: {
40+
'content': {
41+
pattern: /^(\{.)[\s\S]+(?=.\}$)/,
42+
lookbehind: true,
43+
inside: Prism.languages.crystal
44+
},
45+
'delimiter': {
46+
pattern: /^\{[\{%]|[\}%]\}$/,
47+
alias: 'operator'
4648
}
4749
}
48-
]
50+
},
51+
'char': {
52+
pattern: /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/,
53+
greedy: true
54+
}
4955
});
5056

5157
}(Prism));

‎components/prism-crystal.min.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/crystal/attribute_feature.test‎

Lines changed: 32 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,47 +8,53 @@
88
[
99
["attribute", [
1010
["delimiter", "@["],
11-
"AlwaysInline",
11+
["attribute", "AlwaysInline"],
1212
["delimiter", "]"]
1313
]],
1414
["attribute", [
1515
["delimiter", "@["],
16-
"CallConvention",
17-
["punctuation", "("],
18-
["string-literal", [
19-
["string", "\"X86_StdCall\""]
16+
["attribute", "CallConvention"],
17+
["args", [
18+
["punctuation", "("],
19+
["string-literal", [
20+
["string", "\"X86_StdCall\""]
21+
]],
22+
["punctuation", ")"]
2023
]],
21-
["punctuation", ")"],
2224
["delimiter", "]"]
2325
]],
2426
["attribute", [
2527
["delimiter", "@["],
26-
"MyAnnotation",
27-
["punctuation", "("],
28-
["symbol", "key"],
29-
["operator", ":"],
30-
["string-literal", [
31-
["string", "\"value\""]
28+
["attribute", "MyAnnotation"],
29+
["args", [
30+
["punctuation", "("],
31+
["symbol", "key"],
32+
["operator", ":"],
33+
["string-literal", [
34+
["string", "\"value\""]
35+
]],
36+
["punctuation", ","],
37+
["symbol", "value"],
38+
["operator", ":"],
39+
["number", "123"],
40+
["punctuation", ")"]
3241
]],
33-
["punctuation", ","],
34-
["symbol", "value"],
35-
["operator", ":"],
36-
["number", "123"],
37-
["punctuation", ")"],
3842
["delimiter", "]"]
3943
]],
4044
["attribute", [
4145
["delimiter", "@["],
42-
"MyAnnotation",
43-
["punctuation", "("],
44-
["string-literal", [
45-
["string", "\"foo\""]
46+
["attribute", "MyAnnotation"],
47+
["args", [
48+
["punctuation", "("],
49+
["string-literal", [
50+
["string", "\"foo\""]
51+
]],
52+
["punctuation", ","],
53+
["number", "123"],
54+
["punctuation", ","],
55+
["boolean", "false"],
56+
["punctuation", ")"]
4657
]],
47-
["punctuation", ","],
48-
["number", "123"],
49-
["punctuation", ","],
50-
["boolean", "false"],
51-
["punctuation", ")"],
5258
["delimiter", "]"]
5359
]]
5460
]

‎tests/languages/crystal/char_feature.test‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
'a'
2+
'z'
3+
'0'
4+
'_'
5+
'あ'
6+
'\'' # single quote
7+
'\\' # backslash
8+
'\a' # alert
9+
'\b' # backspace
10+
'\e' # escape
11+
'\f' # form feed
12+
'\n' # newline
13+
'\r' # carriage return
14+
'\t' # tab
15+
'\v' # vertical tab
16+
'\0' # null character
17+
'\uFFFF' # hexadecimal unicode character
18+
'\u{10FFFF}' # hexadecimal unicode character
19+
20+
----------------------------------------------------
21+
22+
[
23+
["char", "'a'"],
24+
["char", "'z'"],
25+
["char", "'0'"],
26+
["char", "'_'"],
27+
["char", "'あ'"],
28+
["char", "'\\''"], ["comment", "# single quote"],
29+
["char", "'\\\\'"], ["comment", "# backslash"],
30+
["char", "'\\a'"], ["comment", "# alert"],
31+
["char", "'\\b'"], ["comment", "# backspace"],
32+
["char", "'\\e'"], ["comment", "# escape"],
33+
["char", "'\\f'"], ["comment", "# form feed"],
34+
["char", "'\\n'"], ["comment", "# newline"],
35+
["char", "'\\r'"], ["comment", "# carriage return"],
36+
["char", "'\\t'"], ["comment", "# tab"],
37+
["char", "'\\v'"], ["comment", "# vertical tab"],
38+
["char", "'\\0'"], ["comment", "# null character"],
39+
["char", "'\\uFFFF'"], ["comment", "# hexadecimal unicode character"],
40+
["char", "'\\u{10FFFF}'"], ["comment", "# hexadecimal unicode character"]
41+
]

‎tests/languages/crystal/expansion_feature.test‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,34 @@
88
[
99
["expansion", [
1010
["delimiter", "{{"],
11-
["number", "1_u32"],
11+
["content", [
12+
["number", "1_u32"]
13+
]],
1214
["delimiter", "}}"]
1315
]],
1416
["expansion", [
1517
["delimiter", "{%"],
16-
["number", "2_u32"],
18+
["content", [
19+
["number", "2_u32"]
20+
]],
1721
["delimiter", "%}"]
1822
]],
1923
["expansion", [
2024
["delimiter", "{{"],
21-
["punctuation", "{"],
22-
["number", "3_u32"],
23-
["punctuation", "}"],
25+
["content", [
26+
["punctuation", "{"],
27+
["number", "3_u32"],
28+
["punctuation", "}"]
29+
]],
2430
["delimiter", "}}"]
2531
]],
2632
["expansion", [
2733
["delimiter", "{%"],
28-
["operator", "%"],
29-
["number", "4_u32"],
30-
["operator", "%"],
34+
["content", [
35+
["operator", "%"],
36+
["number", "4_u32"],
37+
["operator", "%"]
38+
]],
3139
["delimiter", "%}"]
3240
]]
3341
]

‎tests/languages/crystal/keyword_feature.test‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
.responds_to?
33
abstract
44
alias
5+
annotation;
56
as
67
asm
78
begin
@@ -19,6 +20,7 @@ extend
1920
for
2021
fun
2122
if
23+
ifdef
2224
include
2325
instance_sizeof
2426
lib
@@ -30,6 +32,7 @@ out
3032
pointerof
3133
private
3234
protected
35+
ptr
3336
require
3437
rescue
3538
return
@@ -41,6 +44,7 @@ super
4144
then
4245
type
4346
typeof
47+
undef
4448
uninitialized
4549
union
4650
unless
@@ -62,6 +66,7 @@ __LINE__
6266
["punctuation", "."], ["keyword", "responds_to?"],
6367
["keyword", "abstract"],
6468
["keyword", "alias"],
69+
["keyword", "annotation"], ["punctuation", ";"],
6570
["keyword", "as"],
6671
["keyword", "asm"],
6772
["keyword", "begin"],
@@ -79,6 +84,7 @@ __LINE__
7984
["keyword", "for"],
8085
["keyword", "fun"],
8186
["keyword", "if"],
87+
["keyword", "ifdef"],
8288
["keyword", "include"],
8389
["keyword", "instance_sizeof"],
8490
["keyword", "lib"],
@@ -90,6 +96,7 @@ __LINE__
9096
["keyword", "pointerof"],
9197
["keyword", "private"],
9298
["keyword", "protected"],
99+
["keyword", "ptr"],
93100
["keyword", "require"],
94101
["keyword", "rescue"],
95102
["keyword", "return"],
@@ -101,6 +108,7 @@ __LINE__
101108
["keyword", "then"],
102109
["keyword", "type"],
103110
["keyword", "typeof"],
111+
["keyword", "undef"],
104112
["keyword", "uninitialized"],
105113
["keyword", "union"],
106114
["keyword", "unless"],

0 commit comments

Comments
 (0)