diff --git a/jac/jaclang/compiler/__init__.py b/jac/jaclang/compiler/__init__.py index d1b178b4fe..d421d765db 100644 --- a/jac/jaclang/compiler/__init__.py +++ b/jac/jaclang/compiler/__init__.py @@ -45,7 +45,11 @@ def generate_static_parser(force: bool = False) -> None: contextlib.suppress(ModuleNotFoundError) TOKEN_MAP = { - x.name: x.pattern.value + x.name: ( + x.pattern.value[2:-2] + if x.pattern.value.startswith("\\b") and x.pattern.value.endswith("\\b") + else x.pattern.value + ) for x in jac_lark.Lark_StandAlone().parser.lexer_conf.terminals } diff --git a/jac/jaclang/compiler/jac.lark b/jac/jaclang/compiler/jac.lark index 2a31e4f88a..f43e4b655e 100644 --- a/jac/jaclang/compiler/jac.lark +++ b/jac/jaclang/compiler/jac.lark @@ -476,78 +476,78 @@ builtin_type: TYP_TYPE | TYP_STRING // Lexer Tokens -TYP_STRING: "str" -TYP_INT: "int" -TYP_FLOAT: "float" -TYP_LIST: "list" -TYP_TUPLE: "tuple" -TYP_SET: "set" -TYP_DICT: "dict" -TYP_BOOL: "bool" -TYP_BYTES: "bytes" -TYP_ANY: "any" -TYP_TYPE: "type" -KW_LET: "let" -KW_ABSTRACT: "abs" -KW_CLASS: "class" -KW_OBJECT: "obj" -KW_ENUM: "enum" -KW_NODE: "node" -KW_IGNORE: "ignore" -KW_VISIT: "visit" -KW_REVISIT: "revisit" -KW_SPAWN: "spawn" -KW_WITH: "with" -KW_ENTRY: "entry" -KW_EXIT: "exit" -KW_IMPORT: "import" -KW_INCLUDE: "include" -KW_FROM: "from" -KW_AS: "as" -KW_EDGE: "edge" -KW_WALKER: "walker" -KW_ASYNC: "async" -KW_AWAIT: "await" -KW_TEST: "test" -KW_ASSERT: "assert" -KW_CHECK: "check" -KW_IF: "if" -KW_ELIF: "elif" -KW_ELSE: "else" -KW_FOR: "for" -KW_TO: "to" -KW_BY: "by" -KW_WHILE: "while" -KW_CONTINUE: "continue" -KW_BREAK: "break" -KW_DISENGAGE: "disengage" -KW_YIELD: "yield" -KW_SKIP: "skip" -KW_REPORT: "report" -KW_RETURN: "return" -KW_DELETE: "del" -KW_TRY: "try" -KW_EXCEPT: "except" -KW_FINALLY: "finally" -KW_RAISE: "raise" -KW_IN: "in" -KW_IS: "is" -KW_PRIV: "priv" -KW_PUB: "pub" -KW_PROT: "protect" -KW_HAS: "has" -KW_GLOBAL: "glob" -KW_CAN: "can" -KW_STATIC: "static" -KW_OVERRIDE: "override" -KW_MATCH: "match" -KW_CASE: "case" -KW_HERE: "here" -KW_SELF: "self" -KW_INIT: "init" -KW_POST_INIT: "postinit" -KW_SUPER: "super" -KW_ROOT: "root" +TYP_STRING: /\bstr\b/ +TYP_INT: /\bint\b/ +TYP_FLOAT: /\bfloat\b/ +TYP_LIST: /\blist\b/ +TYP_TUPLE: /\btuple\b/ +TYP_SET: /\bset\b/ +TYP_DICT: /\bdict\b/ +TYP_BOOL: /\bbool\b/ +TYP_BYTES: /\bbytes\b/ +TYP_ANY: /\bany\b/ +TYP_TYPE: /\btype\b/ +KW_LET: /\blet\b/ +KW_ABSTRACT: /\babs\b/ +KW_CLASS: /\bclass\b/ +KW_OBJECT: /\bobj\b/ +KW_ENUM: /\benum\b/ +KW_NODE: /\bnode\b/ +KW_IGNORE: /\bignore\b/ +KW_VISIT: /\bvisit\b/ +KW_REVISIT: /\brevisit\b/ +KW_SPAWN: /\bspawn\b/ +KW_WITH: /\bwith\b/ +KW_ENTRY: /\bentry\b/ +KW_EXIT: /\bexit\b/ +KW_IMPORT: /\bimport\b/ +KW_INCLUDE: /\binclude\b/ +KW_FROM: /\bfrom\b/ +KW_AS: /\bas\b/ +KW_EDGE: /\bedge\b/ +KW_WALKER: /\bwalker\b/ +KW_ASYNC: /\basync\b/ +KW_AWAIT: /\bawait\b/ +KW_TEST: /\btest\b/ +KW_ASSERT: /\bassert\b/ +KW_CHECK: /\bcheck\b/ +KW_IF: /\bif\b/ +KW_ELIF: /\belif\b/ +KW_ELSE: /\belse\b/ +KW_FOR: /\bfor\b/ +KW_TO: /\bto\b/ +KW_BY: /\bby\b/ +KW_WHILE: /\bwhile\b/ +KW_CONTINUE: /\bcontinue\b/ +KW_BREAK: /\bbreak\b/ +KW_DISENGAGE: /\bdisengage\b/ +KW_YIELD: /\byield\b/ +KW_SKIP: /\bskip\b/ +KW_REPORT: /\breport\b/ +KW_RETURN: /\breturn\b/ +KW_DELETE: /\bdel\b/ +KW_TRY: /\btry\b/ +KW_EXCEPT: /\bexcept\b/ +KW_FINALLY: /\bfinally\b/ +KW_RAISE: /\braise\b/ +KW_IN: /\bin\b/ +KW_IS: /\bis\b/ +KW_PRIV: /\bpriv\b/ +KW_PUB: /\bpub\b/ +KW_PROT: /\bprotect\b/ +KW_HAS: /\bhas\b/ +KW_GLOBAL: /\bglob\b/ +KW_CAN: /\bcan\b/ +KW_STATIC: /\bstatic\b/ +KW_OVERRIDE: /\boverride\b/ +KW_MATCH: /\bmatch\b/ +KW_CASE: /\bcase\b/ +KW_HERE: /\bhere\b/ +KW_SELF: /\bself\b/ +KW_INIT: /\binit\b/ +KW_POST_INIT: /\bpostinit\b/ +KW_SUPER: /\bsuper\b/ +KW_ROOT: /\broot\b/ FLOAT: /(\d+(\.\d*)|\.\d+)([eE][+-]?\d+)?|\d+([eE][-+]?\d+)/ DOC_STRING.1: /"""(.|\n|\r)*?"""|'''(.|\n|\r)*?'''/