Merge pull request #18 from documente/tokenize-done-at-end-of-lines

Tokenize 'done' at end of lines
documente · Dec 25, 2023 · 475c9da · 475c9da
2 parents cb026bd + ba53ebc
commit 475c9da
Showing 1 changed file with 13 additions and 1 deletion.
diff --git a/src/tokenizer.ts b/src/tokenizer.ts
@@ -20,14 +20,26 @@ export function tokenize(sentence: string): Token[] {
     throw new Error(prettyPrintError(message, sentence, { line, column }));
   };
 
+  function isAtEndOfLine() {
+    let j = i;
+
+    while (j < sentence.length) {
+      const char = sentence[j];
+
+      if (char === ' ' || char === '\t') {
+        j++;
+      } else return char === '\n';
+    }
+  }
+
   function pushToken() {
     if (currentToken === '') {
       return;
     }
 
     let kind: Token['kind'] = 'generic';
 
-    if (currentToken === 'done' && isAtStartOfLine) {
+    if (currentToken === 'done' && (isAtStartOfLine || isAtEndOfLine())) {
       kind = 'done';
     } else if (currentToken === '-' && isAtStartOfLine) {
       kind = 'bullet';