floss/tree-sitter-org
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat!: match elements in list items, remove (itemtext)

Browse source 
BREAKING CHANGE: (itemtext) removed, typically in favor of (paragraph)
as the listitem should now be parsed via (body) logic.
This commit is contained in:
Emilia Simmons 2022-03-28 15:41:02 -04:00
parent 7f9a7e51e4
commit 9a595e51c1
8 changed files with 60410 additions and 38518 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

277
corpus/basic.tst
View file

@ -482,21 +482,7 @@ List.1a - Basic: dash [-]
(list
(listitem
(bullet)
(itemtext
(expr))))))
================================================================================
List.1b - Basic: plus [+]
================================================================================
+ a
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -510,7 +496,7 @@ List.1c - Basic: star [*]
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -524,7 +510,7 @@ List.1d - Basic: count dot [1.]
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -538,7 +524,7 @@ List.1e - Basic: count paren [1)]
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -552,7 +538,7 @@ List.1f - Basic: letter dot [a.]
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -566,51 +552,7 @@ List.1g - Basic: letter paren [a)]
(list
(listitem
(bullet)
(itemtext
(expr))))))
================================================================================
List.1h - Basic: checkbox
================================================================================
- [ ] a
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(checkbox)
(itemtext
(expr))))))
================================================================================
List.1i - Basic: checkbox no text
================================================================================
- [ ]
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(checkbox)))))
================================================================================
List.1j - Basic: description
================================================================================
- a :: description
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(description
(expr))
(itemtext
(paragraph
(expr))))))
================================================================================
@ -627,11 +569,11 @@ List.2a - two items
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr)))
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -648,11 +590,11 @@ List.2d - two items
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr)))
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -670,11 +612,11 @@ List.2b - two items
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr)))
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -693,12 +635,12 @@ List.2c - two lists
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -718,59 +660,19 @@ List.3a - sublist
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr)
(expr)
(list
(listitem
(bullet)
(itemtext
(expr))))
(expr)))
(listitem
(bullet)
(itemtext
(expr))))))
================================================================================
List.3b - sublist with description and checkbox
================================================================================
- a :: description
a
- b :: description
- [ ] b
a
- a
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(description
(expr))
(itemtext
(expr)
(expr)
(list
(listitem
(bullet)
(description
(expr))
(itemtext
(expr)))
(listitem
(bullet)
(checkbox)
(itemtext
(expr))))
(list
(listitem
(bullet)
(paragraph
(expr))))
(paragraph
(expr)))
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -787,7 +689,7 @@ List.4a - multiline item
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr)
(expr))))))
@ -806,88 +708,9 @@ List.4b - multiline item
(list
(listitem
(bullet)
(itemtext
(expr)
(expr))))))
================================================================================
List.4c - multiline item checkbox
================================================================================
- [ ] a
b
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(checkbox)
(itemtext
(expr)
(expr))))))
================================================================================
List.4d - multiline item checkbox
================================================================================
- [ ] a
b
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(checkbox)
(itemtext
(expr)
(expr))))))
================================================================================
List.4e - multiline item description
================================================================================
- a :: description
b
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(description
(paragraph
(expr))
(itemtext
(expr)
(expr))))))
================================================================================
List.4f - multiline item description
================================================================================
- a :: description
b
--------------------------------------------------------------------------------
(document
(body
(list
(listitem
(bullet)
(description
(expr))
(itemtext
(expr)
(paragraph
(expr))))))
================================================================================
@ -905,14 +728,14 @@ List.5 - dedent
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))
(paragraph
(expr))
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -933,18 +756,18 @@ b
(list
(listitem
(bullet)
(itemtext
(expr)
(list
(listitem
(bullet)
(itemtext
(expr)))))))
(paragraph
(expr))
(list
(listitem
(bullet)
(paragraph
(expr))))))
(paragraph
(expr))))
================================================================================
List.7a - changing
List.7a - change bullet
================================================================================
- a
@ -957,12 +780,12 @@ List.7a - changing
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -988,7 +811,7 @@ List.8b - Whitespace after text
(list
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -1110,7 +933,7 @@ Directive.6a - List
(expr)))
(listitem
(bullet)
(itemtext
(paragraph
(expr))))))
================================================================================
@ -1133,17 +956,17 @@ Directive.6b - Sublist
(expr)))
(listitem
(bullet)
(itemtext
(expr)
(list
(directive
(expr)
(value
(expr)))
(listitem
(bullet)
(itemtext
(expr)))))))))
(paragraph
(expr))
(list
(directive
(expr)
(value
(expr)))
(listitem
(bullet)
(paragraph
(expr))))))))
================================================================================
Directive.7 - Directive unrelated to section

43
grammar.js
View file

@ -22,6 +22,7 @@ org_grammar = {
$._eol,
$._ts_contents,
$._directive_list,
$._body_contents,
],
precedences: _ => [
@ -44,7 +45,7 @@ org_grammar = {
[$.expr, $.drawer],
// headline 'entry_token1' ':' • '<' …
[$.entry, $.expr]
[$.entry, $.expr],
],
@ -53,7 +54,9 @@ org_grammar = {
document: $ => seq(optional($.body), repeat($.section)),
// Set up to prevent lexing conflicts of having two paragraphs in a row
body: $ => choice(
body: $ => $._body_contents,
_body_contents: $ => choice(
repeat1($._nl),
seq(repeat($._nl), $._multis),
seq(
@ -229,38 +232,24 @@ org_grammar = {
list: $ => seq(
optional($._directive_list),
$._liststart, // captures indent length and bullet type
repeat(seq($.listitem, $._listitemend, repeat1($._nl))),
repeat(seq($.listitem, $._listitemend, repeat($._nl))),
seq($.listitem, $._listend)
),
listitem: $ => seq(
$.bullet,
optional(field('checkbox', $.checkbox)),
optional(seq(alias(optional($._expr_line), $.description), '::')),
optional($.itemtext),
choice(
$._eof,
alias($.body, "item_body"),
)
),
checkbox: _ => seq(
token(prec('non-immediate', '[')), // ]
field(
'status',
choice(
token.immediate(' '),
token.immediate('-'),
token.immediate('x'),
token.immediate('X'),
token.immediate(/./),
)),
token(prec('special', ']')), // [.] could be an (expr)
),
itemtext: $ => seq(
$._expr_line,
repeat(seq(
$._nl,
optional($._nl),
choice($._expr_line, $.list)
)),
listitem: $ => seq(
field('bullet', $.bullet),
choice(
$._eof,
field('contents', $._body_contents),
),
),
table: $ => prec.right(seq(

12
queries/highlights.scm
View file

@ -19,8 +19,8 @@
; Match cookies in a headline or listitem. If you want the numbers
; differently highlighted from the borders, add a capture name to "num".
([ (item) (itemtext) ] (expr "[" "num"? @OrgCookieNum "/" "num"? @OrgCookieNum "]" ) @OrgProgressCookie (#match? @OrgProgressCookie "^\[\d*/\d*\]$"))
([ (item) (itemtext) ] (expr "[" "num"? @OrgCookieNum "%" "]" ) @OrgPercentCookie (#match? @OrgPercentCookie "^\[\d*%\]$"))
; ([ (item) (itemtext) ] (expr "[" "num"? @OrgCookieNum "/" "num"? @OrgCookieNum "]" ) @OrgProgressCookie (#match? @OrgProgressCookie "^\[\d*/\d*\]$"))
; ([ (item) (itemtext) ] (expr "[" "num"? @OrgCookieNum "%" "]" ) @OrgPercentCookie (#match? @OrgPercentCookie "^\[\d*%\]$"))
(tag_list (tag) @OrgTag) @OrgTagList
@ -59,12 +59,12 @@
; Can match different styles with a (#match?) or (#eq?) predicate if desired
(bullet) @OrgListBullet
(listitem (description) @OrgListDescription)
; Get different colors for different statuses as follows
(checkbox) @OrgCheckbox
(checkbox status: [ "x" "X" ] @OrgCheckDone)
(checkbox status: "-" @OrgCheckInProgress)
(listitem . (bullet) . (paragraph . (expr "[" "str" @OrgCheckDone "]") @OrgCheckbox (#match? @OrgCheckbox "^\[[xX]\]$")))
(listitem . (bullet) . (paragraph . (expr "[" "-" @OrgCheckInProgress "]") @OrgCheckbox (#eq? @OrgCheckbox "[-]")))
(listitem . (bullet) . (paragraph . (expr "[") @OrgCheckbox.left (#eq? @OrgCheckbox.left "[") . (expr "]") @OrgCheckbox.right (#eq? @OrgCheckbox.right "]")))
; (listitem . (bullet) . (paragraph (expr ":" ":") @OrgListDescriptionSeparator (#eq? @OrgListDescriptionSeparator "::"))) -- matches multiple, requires a special search.
; If you want the ruler one color and the separators a different color,
; something like this would do it:

16
queries/markup.scm
View file

@ -13,22 +13,6 @@
(expr "+" @strikethrough.start "+" @strikethrough.end)
])
; listitem text
(itemtext [
((expr "*" @bold.start) (expr "*" @bold.end))
(expr "*" @bold.start "*" @bold.end)
((expr "~" @code.start) (expr "~" @code.end))
(expr "~" @code.start "~" @code.end)
((expr "/" @italic.start) (expr "/" @italic.end))
(expr "/" @italic.start "/" @italic.end)
((expr "_" @underline.start) (expr "_" @underline.end))
(expr "_" @underline.start "_" @underline.end)
((expr "=" @verbatim.start) (expr "=" @verbatim.end))
(expr "=" @verbatim.start "=" @verbatim.end)
((expr "+" @strikethrough.start) (expr "+" @strikethrough.end))
(expr "+" @strikethrough.start "+" @strikethrough.end)
])
; headline item
(item [
((expr "*" @bold.start) (expr "*" @bold.end))

193
src/grammar.json
View file

@ -26,6 +26,10 @@
]
},
"body": {
"type": "SYMBOL",
"name": "_body_contents"
},
"_body_contents": {
"type": "CHOICE",
"members": [
{
@ -1306,7 +1310,7 @@
"name": "_listitemend"
},
{
"type": "REPEAT1",
"type": "REPEAT",
"content": {
"type": "SYMBOL",
"name": "_nl"
@ -1334,190 +1338,32 @@
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "bullet"
},
{
"type": "CHOICE",
"members": [
{
"type": "FIELD",
"name": "checkbox",
"content": {
"type": "SYMBOL",
"name": "checkbox"
}
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "ALIAS",
"content": {
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_expr_line"
},
{
"type": "BLANK"
}
]
},
"named": true,
"value": "description"
},
{
"type": "STRING",
"value": "::"
}
]
},
{
"type": "BLANK"
}
]
"type": "FIELD",
"name": "bullet",
"content": {
"type": "SYMBOL",
"name": "bullet"
}
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "itemtext"
"name": "_eof"
},
{
"type": "BLANK"
"type": "FIELD",
"name": "contents",
"content": {
"type": "SYMBOL",
"name": "_body_contents"
}
}
]
}
]
},
"checkbox": {
"type": "SEQ",
"members": [
{
"type": "TOKEN",
"content": {
"type": "PREC",
"value": "non-immediate",
"content": {
"type": "STRING",
"value": "["
}
}
},
{
"type": "FIELD",
"name": "status",
"content": {
"type": "CHOICE",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": " "
}
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "-"
}
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "x"
}
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "STRING",
"value": "X"
}
},
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "PATTERN",
"value": "."
}
}
]
}
},
{
"type": "TOKEN",
"content": {
"type": "PREC",
"value": "special",
"content": {
"type": "STRING",
"value": "]"
}
}
}
]
},
"itemtext": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_expr_line"
},
{
"type": "REPEAT",
"content": {
"type": "SEQ",
"members": [
{
"type": "SYMBOL",
"name": "_nl"
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_nl"
},
{
"type": "BLANK"
}
]
},
{
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "_expr_line"
},
{
"type": "SYMBOL",
"name": "list"
}
]
}
]
}
}
]
},
"table": {
"type": "PREC_RIGHT",
"value": 0,
@ -4400,7 +4246,8 @@
"_nl",
"_eol",
"_ts_contents",
"_directive_list"
"_directive_list",
"_body_contents"
],
"supertypes": []
}

149
src/node-types.json
View file

@ -119,34 +119,6 @@
}
}
},
{
"type": "checkbox",
"named": true,
"fields": {
"status": {
"multiple": false,
"required": true,
"types": [
{
"type": " ",
"named": false
},
{
"type": "-",
"named": false
},
{
"type": "X",
"named": false
},
{
"type": "x",
"named": false
}
]
}
}
},
{
"type": "comment",
"named": true,
@ -458,25 +430,6 @@
]
}
},
{
"type": "itemtext",
"named": true,
"fields": {},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "expr",
"named": true
},
{
"type": "list",
"named": true
}
]
}
},
{
"type": "latex_env",
"named": true,
@ -553,34 +506,80 @@
"type": "listitem",
"named": true,
"fields": {
"checkbox": {
"bullet": {
"multiple": false,
"required": true,
"types": [
{
"type": "bullet",
"named": true
}
]
},
"contents": {
"multiple": true,
"required": false,
"types": [
{
"type": "checkbox",
"type": "\n",
"named": false
},
{
"type": "\r",
"named": false
},
{
"type": "block",
"named": true
},
{
"type": "comment",
"named": true
},
{
"type": "directive",
"named": true
},
{
"type": "drawer",
"named": true
},
{
"type": "dynamic_block",
"named": true
},
{
"type": "fndef",
"named": true
},
{
"type": "latex_env",
"named": true
},
{
"type": "list",
"named": true
},
{
"type": "paragraph",
"named": true
},
{
"type": "table",
"named": true
}
]
},
"directive": {
"multiple": true,
"required": false,
"types": [
{
"type": "directive",
"named": true
}
]
}
},
"children": {
"multiple": true,
"required": true,
"types": [
{
"type": "bullet",
"named": true
},
{
"type": "description",
"named": true
},
{
"type": "itemtext",
"named": true
}
]
}
},
{
@ -852,10 +851,6 @@
"type": "\r",
"named": false
},
{
"type": " ",
"named": false
},
{
"type": "!",
"named": false
@ -944,10 +939,6 @@
"type": ":",
"named": false
},
{
"type": "::",
"named": false
},
{
"type": ":end:",
"named": false
@ -988,10 +979,6 @@
"type": "@",
"named": false
},
{
"type": "X",
"named": false
},
{
"type": "[",
"named": false
@ -1084,10 +1071,6 @@
"type": "time",
"named": true
},
{
"type": "x",
"named": false
},
{
"type": "{",
"named": false

98185
src/parser.c
View file

File diff suppressed because it is too large Load diff

53
src/scanner.cc
View file

@ -179,37 +179,34 @@ bool scan(TSLexer *lexer, const bool *valid_symbols) {
// Listend -> end of a line, looking for:
// 1. dedent
// 2. same indent, not a bullet
// 3. three eols
if (lexer->lookahead == '\n') {
if (valid_symbols[LISTEND] || valid_symbols[LISTITEMEND]) {
int16_t newlines = 0;
for (;;) {
if (lexer->lookahead == ' ') {
indent_length++;
} else if (lexer->lookahead == '\t') {
indent_length += 8;
} else if (lexer->lookahead == '\0') {
return dedent(lexer);
} else if (lexer->lookahead == '\n') {
if (++newlines > 2) return dedent(lexer);
indent_length = 0;
} else {
break;
}
skip(lexer);
}
if (indent_length < indent_length_stack.back()) {
return dedent(lexer);
} else if (indent_length == indent_length_stack.back()) {
if (getbullet(lexer) == bullet_stack.back()) {
lexer->result_symbol = LISTITEMEND;
return true;
}
// 3. two eols
if (valid_symbols[LISTEND] || valid_symbols[LISTITEMEND]) {
int16_t newlines = 0;
for (;;) {
if (lexer->lookahead == ' ') {
indent_length++;
} else if (lexer->lookahead == '\t') {
indent_length += 8;
} else if (lexer->lookahead == '\0') {
return dedent(lexer);
} else if (lexer->lookahead == '\n') {
if (++newlines > 1) return dedent(lexer);
indent_length = 0;
} else {
break;
}
skip(lexer);
}
if (indent_length < indent_length_stack.back()) {
return dedent(lexer);
} else if (indent_length == indent_length_stack.back()) {
if (getbullet(lexer) == bullet_stack.back()) {
lexer->result_symbol = LISTITEMEND;
return true;
}
return dedent(lexer);
}
return false;
}
// - Col=0 star