chore: update escaping rules

usememos · Sep 17, 2024 · b2451d1 · b2451d1
1 parent 3ccacc4
commit b2451d1
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/parser/tests/escaping_character_test.go b/parser/tests/escaping_character_test.go
@@ -15,11 +15,17 @@ func TestEscapingCharacterParser(t *testing.T) {
 		node ast.Node
 	}{
 		{
-			text: `\# 123`,
+			text: `\#`,
 			node: &ast.EscapingCharacter{
 				Symbol: "#",
 			},
 		},
+		{
+			text: `\' test`,
+			node: &ast.EscapingCharacter{
+				Symbol: "'",
+			},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/parser/tokenizer/tokenizer.go b/parser/tokenizer/tokenizer.go
@@ -25,6 +25,7 @@ const (
 	Pipe               TokenType = "|"
 	Colon              TokenType = ":"
 	Caret              TokenType = "^"
+	Apostrophe         TokenType = "'"
 	Backslash          TokenType = "\\"
 	Slash              TokenType = "/"
 	NewLine            TokenType = "\n"
@@ -95,6 +96,8 @@ func Tokenize(text string) []*Token {
 			tokens = append(tokens, NewToken(Colon, ":"))
 		case '^':
 			tokens = append(tokens, NewToken(Caret, "^"))
+		case '\'':
+			tokens = append(tokens, NewToken(Apostrophe, "'"))
 		case '\\':
 			tokens = append(tokens, NewToken(Backslash, `\`))
 		case '/':