From cbbca87d7b32ea955a88d91dd3cac8a78c5a074e Mon Sep 17 00:00:00 2001 From: David Hagen Date: Mon, 18 Nov 2024 21:03:40 -0500 Subject: [PATCH] Fix some tests --- examples/expressions.py | 44 ++++++++++++++++++--------------- examples/json.py | 3 ++- examples/positioned.py | 22 ++++++++--------- src/parsita/parsers/_debug.py | 22 ++++++++++++++++- src/parsita/parsers/_literal.py | 35 ++++++++++++++++++++------ src/parsita/parsers/_regex.py | 6 +++-- src/parsita/parsers/_until.py | 4 +-- src/parsita/state/_result.py | 13 +++++----- src/parsita/util.py | 7 +++--- tests/test_basic.py | 24 +++++++++--------- tests/test_metaclass_scopes.py | 26 +++++++++---------- tests/test_state.py | 16 ++++++++---- tests/test_util.py | 8 +++--- 13 files changed, 142 insertions(+), 88 deletions(-) diff --git a/examples/expressions.py b/examples/expressions.py index d1cee16..8ac9f29 100644 --- a/examples/expressions.py +++ b/examples/expressions.py @@ -1,6 +1,30 @@ +from typing import Literal, Sequence + from parsita import ParserContext, lit, opt, reg, rep +def make_term(args: tuple[float, Sequence[tuple[Literal["*", "/"], float]]]) -> float: + factor1, factors = args + result = factor1 + for op, factor in factors: + if op == "*": + result = result * factor + else: + result = result / factor + return result + + +def make_expr(args: tuple[float, Sequence[tuple[Literal["+", "-"], float]]]) -> float: + term1, terms = args + result = term1 + for op, term2 in terms: + if op == "+": + result = result + term2 + else: + result = result - term2 + return result + + class ExpressionParsers(ParserContext, whitespace=r"[ ]*"): number = reg(r"[+-]?\d+(\.\d+)?(e[+-]?\d+)?") > float @@ -8,28 +32,8 @@ class ExpressionParsers(ParserContext, whitespace=r"[ ]*"): factor = base & opt("^" >> base) > (lambda x: x[0] ** x[1][0] if x[1] else x[0]) - def make_term(args): - factor1, factors = args - result = factor1 - for op, factor in factors: - if op == "*": - result = result * factor - else: - result = result / factor - return result - term = factor & rep(lit("*", "/") & factor) > make_term - def make_expr(args): - term1, terms = args - result = term1 - for op, term2 in terms: - if op == "+": - result = result + term2 - else: - result = result - term2 - return result - expr = term & rep(lit("+", "-") & term) > make_expr diff --git a/examples/json.py b/examples/json.py index e484b06..4ccf140 100644 --- a/examples/json.py +++ b/examples/json.py @@ -13,7 +13,7 @@ class JsonStringParsers(ParserContext): line_feed = lit(r"\n") > constant("\n") carriage_return = lit(r"\r") > constant("\r") tab = lit(r"\t") > constant("\t") - uni = reg(r"\\u([0-9a-fA-F]{4})") > (lambda x: chr(int(x.group(1), 16))) + uni = reg(r"\\u[0-9a-fA-F]{4}") > (lambda x: chr(int(x[2:], 16))) escaped = ( quote @@ -61,6 +61,7 @@ class JsonParsers(ParserContext, whitespace=r"[ \t\n\r]*"): "width" : 4.0 }""", '{"text" : ""}', + r'"\u2260"', ] for string in strings: diff --git a/examples/positioned.py b/examples/positioned.py index 0477506..e924912 100644 --- a/examples/positioned.py +++ b/examples/positioned.py @@ -8,7 +8,7 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Generic +from typing import Generic, Optional from parsita import Parser, ParserContext, Reader, reg from parsita.state import Continue, Input, Output, State @@ -45,7 +45,7 @@ def __init__(self, parser: Parser[Input, PositionAware[Output]]): super().__init__() self.parser = parser - def _consume(self, state: State, reader: Reader[Input]): + def _consume(self, state: State, reader: Reader[Input]) -> Optional[Continue[Input, Output]]: start = reader.position status = self.parser.consume(state, reader) @@ -55,11 +55,11 @@ def _consume(self, state: State, reader: Reader[Input]): else: return status - def __repr__(self): + def __repr__(self) -> str: return self.name_or_nothing() + f"positioned({self.parser.name_or_repr()})" -def positioned(parser: Parser[Input, PositionAware[Output]]): +def positioned(parser: Parser[Input, PositionAware[Output]]) -> PositionedParser[Input, Output]: """Set the position on a PositionAware value. This parser matches ``parser`` and, if successful, calls ``set_position`` @@ -75,18 +75,18 @@ def positioned(parser: Parser[Input, PositionAware[Output]]): # Everything below here is an example use case @dataclass -class UnfinishedVariable(PositionAware): +class Variable: name: str - - def set_position(self, start: int, length: int): - return Variable(self.name, start, length) + start: int + length: int @dataclass -class Variable: +class UnfinishedVariable(PositionAware[Variable]): name: str - start: int - length: int + + def set_position(self, start: int, length: int) -> Variable: + return Variable(self.name, start, length) @dataclass diff --git a/src/parsita/parsers/_debug.py b/src/parsita/parsers/_debug.py index 1317da5..b3a46f0 100644 --- a/src/parsita/parsers/_debug.py +++ b/src/parsita/parsers/_debug.py @@ -1,6 +1,6 @@ __all__ = ["DebugParser", "debug"] -from typing import Callable, Generic, Optional +from typing import Any, Callable, Generic, Optional, Sequence, Union, overload from ..state import Continue, Input, Output, Reader, State from ._base import Parser, wrap_literal @@ -37,12 +37,32 @@ def __repr__(self) -> str: return self.name_or_nothing() + f"debug({self.parser.name_or_repr()})" +@overload +def debug( + parser: Sequence[Input], + *, + verbose: bool = False, + callback: Optional[Callable[[Parser[Input, Sequence[Input]], Reader[Input]], None]] = None, +) -> DebugParser[Input, Sequence[Input]]: + pass + + +@overload def debug( parser: Parser[Input, Output], *, verbose: bool = False, callback: Optional[Callable[[Parser[Input, Output], Reader[Input]], None]] = None, ) -> DebugParser[Input, Output]: + pass + + +def debug( + parser: Union[Parser[Input, Output], Sequence[Input]], + *, + verbose: bool = False, + callback: Optional[Callable[[Parser[Input, Output], Reader[Input]], None]] = None, +) -> DebugParser[Input, Any]: """Execute debugging hooks before a parser. This parser is used purely for debugging purposes. From a parsing diff --git a/src/parsita/parsers/_literal.py b/src/parsita/parsers/_literal.py index 74b80d9..1c1af32 100644 --- a/src/parsita/parsers/_literal.py +++ b/src/parsita/parsers/_literal.py @@ -1,23 +1,24 @@ __all__ = ["LiteralParser", "lit"] -from typing import Generic, Optional, Sequence +from typing import Any, Generic, Optional, Sequence, TypeVar, Union, overload from .. import options from ..state import Continue, Element, Reader, State, StringReader from ._base import Parser +# The bound should be Sequence[Element], but mypy doesn't support higher-kinded types. +Literal = TypeVar("Literal", bound=Sequence[Any], covariant=True) -class LiteralParser(Generic[Element], Parser[Element, Sequence[Element]]): - def __init__( - self, pattern: Sequence[Element], whitespace: Optional[Parser[Element, object]] = None - ): + +class LiteralParser(Generic[Element, Literal], Parser[Element, Literal]): + def __init__(self, pattern: Literal, whitespace: Optional[Parser[Element, object]] = None): super().__init__() self.pattern = pattern self.whitespace = whitespace def _consume( self, state: State, reader: Reader[Element] - ) -> Optional[Continue[Element, Sequence[Element]]]: + ) -> Optional[Continue[Element, Literal]]: if self.whitespace is not None: status = self.whitespace.consume(state, reader) reader = status.remainder # type: ignore # whitespace is infallible @@ -49,9 +50,27 @@ def __repr__(self) -> str: return self.name_or_nothing() + repr(self.pattern) +FunctionLiteral = TypeVar("FunctionLiteral", bound=Sequence[Any]) + + +@overload +def lit(literal: str, *literals: str) -> Parser[str, str]: + pass + + +@overload +def lit(literal: bytes, *literals: bytes) -> Parser[int, bytes]: + pass + + +@overload +def lit(literal: FunctionLiteral, *literals: FunctionLiteral) -> Parser[Any, FunctionLiteral]: + pass + + def lit( - literal: Sequence[Element], *literals: Sequence[Element] -) -> Parser[Element, Sequence[Element]]: + literal: Union[FunctionLiteral, str, bytes], *literals: Union[FunctionLiteral, str, bytes] +) -> Parser[Element, object]: """Match a literal sequence. This parser returns successfully if the subsequence of the parsing Element diff --git a/src/parsita/parsers/_regex.py b/src/parsita/parsers/_regex.py index 1291c75..cc0f11a 100644 --- a/src/parsita/parsers/_regex.py +++ b/src/parsita/parsers/_regex.py @@ -1,7 +1,7 @@ __all__ = ["RegexParser", "reg"] import re -from typing import Generic, Optional, TypeVar, Union, no_type_check +from typing import Any, Generic, Optional, TypeVar, Union, no_type_check from .. import options from ..state import Continue, State, StringReader @@ -10,7 +10,9 @@ StringType = TypeVar("StringType", str, bytes) -class RegexParser(Generic[StringType], Parser[StringType, StringType]): +# The Element type is str for str and int for bytes, but there is not way to +# express that in Python. +class RegexParser(Generic[StringType], Parser[Any, StringType]): def __init__( self, pattern: re.Pattern[StringType], diff --git a/src/parsita/parsers/_until.py b/src/parsita/parsers/_until.py index 6ca5d91..3aaf796 100644 --- a/src/parsita/parsers/_until.py +++ b/src/parsita/parsers/_until.py @@ -1,6 +1,6 @@ __all__ = ["UntilParser", "until"] -from typing import Any, Generic, Optional, Sequence +from typing import Any, Generic, Optional, Sequence, Union from ..state import Continue, Element, Reader, State from ._base import Parser, wrap_literal @@ -31,7 +31,7 @@ def __repr__(self) -> str: return self.name_or_nothing() + f"until({self.parser.name_or_repr()})" -def until(parser: Parser[Element, object]) -> UntilParser[Element]: +def until(parser: Union[Parser[Element, object], Sequence[Element]]) -> UntilParser[Element]: """Match everything until it matches the provided parser. This parser matches all Element until it encounters a position in the Element diff --git a/src/parsita/state/_result.py b/src/parsita/state/_result.py index 2397b82..fe8a65a 100644 --- a/src/parsita/state/_result.py +++ b/src/parsita/state/_result.py @@ -1,6 +1,6 @@ __all__ = ["Result", "Success", "Failure"] -from typing import TYPE_CHECKING, TypeVar +from typing import TypeVar from returns import result @@ -9,11 +9,10 @@ Output = TypeVar("Output") # Reexport Returns Result types +# Failure and Result fail in isinstance +# Failure is replaced by plain Failure, which works at runtime +# Result is left as is because cannot be fixed without breaking eager type annotations Result = result.Result[Output, ParseError] Success = result.Success -if TYPE_CHECKING: - # This object fails in isinstance - # Result does too, but that cannot be fixed without breaking eager type annotations - Failure = result.Failure[ParseError] -else: - Failure = result.Failure +Failure: type[result.Failure[ParseError]] = result.Failure[ParseError] +Failure = result.Failure diff --git a/src/parsita/util.py b/src/parsita/util.py index de12f22..f6c69f8 100644 --- a/src/parsita/util.py +++ b/src/parsita/util.py @@ -2,7 +2,7 @@ __all__ = ["constant", "splat", "unsplat"] -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Any, Callable, Sequence if TYPE_CHECKING: # ParamSpec was introduced in Python 3.10 @@ -31,7 +31,8 @@ def constanted(*args: P.args, **kwargs: P.kwargs) -> A: return constanted -def splat(f: Callable[[Unpack[Ts]], A], /) -> Callable[[tuple[Unpack[Ts]]], A]: +# This signature cannot be expressed narrowly because SequenceParser does not return a tuple +def splat(f: Callable[[Unpack[Ts]], A], /) -> Callable[[Sequence[Any]], A]: """Convert a function of multiple arguments into a function of a single iterable argument. Args: @@ -50,7 +51,7 @@ def splat(f: Callable[[Unpack[Ts]], A], /) -> Callable[[tuple[Unpack[Ts]]], A]: $ g([1, 2, 3]) # 6 """ - def splatted(args: tuple[Unpack[Ts]], /) -> A: + def splatted(args: Sequence[Any], /) -> A: return f(*args) return splatted diff --git a/tests/test_basic.py b/tests/test_basic.py index 80d2edb..d7fb37d 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,3 +1,5 @@ +from typing import Union + import pytest from parsita import ( @@ -424,14 +426,14 @@ class TestParsers(ParserContext): def test_conversion(): + def make_twentyone(x: tuple[float, float]) -> float: + return x[0] * 10 + x[1] + class TestParsers(ParserContext): one = lit("1") > int two = lit("2") > int twelve = one & two > (lambda x: x[0] * 10 + x[1]) - def make_twentyone(x): - return x[0] * 10 + x[1] - twentyone = two & one > make_twentyone assert TestParsers.one.parse("1") == Success(1) @@ -442,6 +444,14 @@ def make_twentyone(x): def test_recursion(): + def make_expr(x: tuple[float, Union[tuple[()], tuple[float]]]) -> float: + digits1, maybe_expr = x + if maybe_expr: + digits2 = maybe_expr[0] + return digits1 + digits2 + else: + return digits1 + class TestParsers(ParserContext): one = lit("1") > float six = lit("6") > float @@ -449,14 +459,6 @@ class TestParsers(ParserContext): numbers = eleven | one | six - def make_expr(x): - digits1, maybe_expr = x - if maybe_expr: - digits2 = maybe_expr[0] - return digits1 + digits2 - else: - return digits1 - expr = numbers & opt("+" >> expr) > make_expr assert TestParsers.expr.parse("11") == Success(11) diff --git a/tests/test_metaclass_scopes.py b/tests/test_metaclass_scopes.py index 0ba0185..0a260d8 100644 --- a/tests/test_metaclass_scopes.py +++ b/tests/test_metaclass_scopes.py @@ -3,7 +3,7 @@ from parsita import ParserContext, Success, lit -def convert(value: str): +def convert(value: str) -> str: return "global" @@ -20,7 +20,7 @@ def test_global_class_global_function(): class GlobalLocal(ParserContext): - def convert(value: str): + def convert(value: str) -> str: return "local" x = lit("x") > convert @@ -35,7 +35,7 @@ def test_global_class_local_function(): class GlobalInner(ParserContext): - def convert(value: str): + def convert(value: str) -> str: return "local" x = lit("x") > convert @@ -65,7 +65,7 @@ class Inner(ParserContext): def test_local_class_local_function(): class LocalLocal(ParserContext): - def convert(value: str): + def convert(value: str) -> str: return "local" x = lit("x") > convert @@ -79,13 +79,13 @@ class Inner(ParserContext): def test_inner_class_inner_function(): class LocalLocal(ParserContext): - def convert(value: str): + def convert(value: str) -> str: return "local" x = lit("x") > convert class Inner(ParserContext): - def convert(value: str): + def convert(value: str) -> str: return "nested" x = lit("x") > convert @@ -95,7 +95,7 @@ def convert(value: str): def test_nested_class_global_function(): - def nested(): + def nested() -> type[ParserContext]: class LocalLocal(ParserContext): x = lit("x") > convert @@ -110,7 +110,7 @@ class Inner(ParserContext): def factory(): - def convert(value: str): + def convert(value: str) -> str: return "local" class LocalLocal(ParserContext): @@ -123,7 +123,7 @@ class Inner(ParserContext): def test_factory_class_local_function(): - def convert(value: str): + def convert(value: str) -> str: return "caller" returned_class = factory() @@ -133,10 +133,10 @@ def convert(value: str): def test_nested_class_nonlocal_function(): - def convert(value: str): + def convert(value: str) -> str: return "nonlocal" - def nested(): + def nested() -> type[ParserContext]: class LocalLocal(ParserContext): x = lit("x") > convert @@ -152,10 +152,10 @@ class Inner(ParserContext): def test_nested_class_local_function(): - def convert(value: str): + def convert(value: str) -> str: return "nonlocal" - def nested(): + def nested() -> type[ParserContext]: def convert(value: str): return "local" diff --git a/tests/test_state.py b/tests/test_state.py index e3419c8..9d25211 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -9,11 +9,13 @@ from parsita.state import Continue, State -def test_state_creation(): +def test_sequence_reader_creation(): read = SequenceReader([1, 2, 3]) assert read.first == 1 assert read.rest.first == 2 + +def test_string_reader_creation(): read = StringReader("a b") assert read.first == "a" assert read.rest.first == " " @@ -38,7 +40,7 @@ def test_parse_error_str_string_reader(): @pytest.mark.parametrize("source", ["a a", "a a\n"]) -def test_parse_error_str_string_reader_end_of_source(source): +def test_parse_error_str_string_reader_end_of_source(source: str): err = ParseError(StringReader(source, 4), ["'b'"]) assert str(err) == "Expected 'b' but found end of source\nLine 1, character 4\n\na a\n ^" @@ -52,6 +54,7 @@ def test_register_failure_first(): state = State() state.register_failure("foo", StringReader("bar baz", 0)) assert state.expected == ["foo"] + assert isinstance(state.farthest, Reader) assert state.farthest.position == 0 @@ -59,6 +62,7 @@ def test_register_failure_at_middle(): state = State() state.register_failure("foo", StringReader("bar baz", 4)) assert state.expected == ["foo"] + assert isinstance(state.farthest, Reader) assert state.farthest.position == 4 @@ -67,6 +71,7 @@ def test_register_failure_latest(): state.register_failure("foo", StringReader("bar baz", 0)) state.register_failure("egg", StringReader("bar baz", 4)) assert state.expected == ["egg"] + assert isinstance(state.farthest, Reader) assert state.farthest.position == 4 @@ -75,6 +80,7 @@ def test_register_failure_tied(): state.register_failure("foo", StringReader("bar baz", 4)) state.register_failure("egg", StringReader("bar baz", 4)) assert state.expected == ["foo", "egg"] + assert isinstance(state.farthest, Reader) assert state.farthest.position == 4 @@ -91,8 +97,8 @@ def test_isinstance(): def test_isinstance_result(): success = Success(1) failure = Failure(ParseError(StringReader("bar baz", 4), ["foo"])) - assert isinstance(success, Result) - assert isinstance(failure, Result) + assert isinstance(success, Result) # type: ignore + assert isinstance(failure, Result) # type: ignore def test_result_annotation(): @@ -120,7 +126,7 @@ def test_reader_drop(): # drop, so this test is for that in case someone extends Reader. @dataclass(frozen=True) - class BytesReader(Reader): + class BytesReader(Reader[int]): source: bytes position: int = 0 diff --git a/tests/test_util.py b/tests/test_util.py index 9c3c536..bbb3d17 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -10,21 +10,21 @@ def test_constant(): def test_splat(): - def f(a, b, c): + def f(a: int, b: int, c: int) -> int: return a + b + c assert f(1, 2, 3) == 6 g = splat(f) - args = [1, 2, 3] + args = (1, 2, 3) assert g(args) == 6 def test_unsplat(): - def f(a): + def f(a: tuple[int, int, int]) -> int: return a[0] + a[1] + a[2] - args = [1, 2, 3] + args = (1, 2, 3) assert f(args) == 6 g = unsplat(f)