diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py index 3417dedd1c..a6e5126d42 100644 --- a/pydantic/_internal/_generate_schema.py +++ b/pydantic/_internal/_generate_schema.py @@ -1967,12 +1967,25 @@ def _computed_field_schema( def _annotated_schema(self, annotated_type: Any) -> core_schema.CoreSchema: """Generate schema for an Annotated type, e.g. `Annotated[int, Field(...)]` or `Annotated[int, Gt(0)]`.""" FieldInfo = import_cached_field_info() - - source_type, *annotations = self._get_args_resolving_forward_refs( - annotated_type, - required=True, - ) - schema = self._apply_annotations(source_type, annotations) + # Ideally, we should delegate all this to `_typing_extra.unpack_annotated`, e.g.: + # `typ, annotations = _typing_extra.unpack_annotated(annotated_type); schema = self.apply_annotations(...)` + # if it was able to use a `NsResolver`. But because `unpack_annotated` is also used + # when constructing `FieldInfo` instances (where we don't have access to a `NsResolver`), + # the implementation of the function does *not* resolve forward annotations. This could + # be solved by calling `unpack_annotated` directly inside `collect_model_fields`. + # For now, we at least resolve the annotated type if it is a forward ref, but note that + # unexpected results will happen if you have something like `Annotated[Alias, ...]` and + # `Alias` is a PEP 695 type alias containing forward references. + typ, *annotations = get_args(annotated_type) + if isinstance(typ, str): + typ = _typing_extra._make_forward_ref(typ) + if isinstance(typ, ForwardRef): + typ = self._resolve_forward_ref(typ) + + typ, sub_annotations = _typing_extra.unpack_annotated(typ) + annotations = sub_annotations + annotations + + schema = self._apply_annotations(typ, annotations) # put the default validator last so that TypeAdapter.get_default_value() works # even if there are function validators involved for annotation in annotations: diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py index 6681bedb59..c5381f22db 100644 --- a/pydantic/_internal/_typing_extra.py +++ b/pydantic/_internal/_typing_extra.py @@ -128,6 +128,89 @@ def annotated_type(tp: Any, /) -> Any | None: return get_args(tp)[0] if is_annotated(tp) else None +def unpack_annotated(annotation: Any, /) -> tuple[Any, list[Any]]: + """Unpack the annotation if it is wrapped with the `Annotated` type qualifier. + + This function also unpacks PEP 695 type aliases if necessary (and also generic + aliases with a PEP 695 type alias origin). However, it does *not* try to evaluate + forward references, so users should make sure the type alias' `__value__` does not + contain unresolvable forward references. + + Example: + ```python {test="skip" lint="skip"} + from typing import Annotated + + type InnerList[T] = Annotated[list[T], 'meta_1'] + type MyList[T] = Annotated[InnerList[T], 'meta_2'] + type MyIntList = MyList[int] + + _unpack_annotated(MyList) + #> (list[T], ['meta_1', 'meta_2']) + _unpack_annotated(MyList[int]) + #> (list[int], ['meta_1', 'meta_2']) + _unpack_annotated(MyIntList) + #> (list[int], ['meta_1', 'meta_2']) + ``` + + Returns: + A two-tuple, the first element is the annotated type and the second element + is a list containing the annotated metadata. If the annotation wasn't + wrapped with `Annotated` in the first place, it is returned as is and the + metadata list is empty. + """ + if is_annotated(annotation): + typ, *metadata = typing_extensions.get_args(annotation) + # The annotated type might be a PEP 695 type alias, so we need to recursively + # unpack it. Note that we could make an optimization here: the following next + # call to `_unpack_annotated` could omit the `is_annotated` check, because Python + # already flattens `Annotated[Annotated[, ...], ...]` forms. However, we would + # need to "re-enable" the check for further recursive calls. + typ, sub_meta = unpack_annotated(typ) + metadata = sub_meta + metadata + return typ, metadata + elif is_type_alias_type(annotation): + try: + value = annotation.__value__ + except NameError: + # The type alias value contains an unresolvable reference. Note that even if it + # resolves successfully, it might contain string annotations, and because of design + # limitations we don't evaluate the type (we don't have access to a `NsResolver` instance). + pass + else: + typ, metadata = unpack_annotated(value) + if metadata: + # Having metadata means the type alias' `__value__` was an `Annotated` form + # (or, recursively, a type alias to an `Annotated` form). It is important to + # check for this as we don't want to unpack "normal" type aliases (e.g. `type MyInt = int`). + return typ, metadata + return annotation, [] + elif is_generic_alias(annotation): + # When parametrized, a PEP 695 type alias becomes a generic alias + # (e.g. with `type MyList[T] = Annotated[list[T], ...]`, `MyList[int]` + # is a generic alias). + origin = typing_extensions.get_origin(annotation) + if is_type_alias_type(origin): + try: + value = origin.__value__ + except NameError: + pass + else: + # Circular import (note that these two functions should probably be defined in `_typing_extra`): + from ._generics import get_standard_typevars_map, replace_types + + # While Python already handles type variable replacement for simple `Annotated` forms, + # we need to manually apply the same logic for PEP 695 type aliases: + # - With `MyList = Annotated[list[T], ...]`, `MyList[int] == Annotated[list[int], ...]` + # - With `type MyList = Annotated[list[T], ...]`, `MyList[int].__value__ == Annotated[list[T], ...]`. + value = replace_types(value, get_standard_typevars_map(annotation)) + typ, metadata = unpack_annotated(value) + if metadata: + return typ, metadata + return annotation, [] + + return annotation, [] + + def is_unpack(tp: Any, /) -> bool: """Return whether the provided argument is a `Unpack` special form. @@ -221,6 +304,8 @@ def is_paramspec(tp: Any, /) -> bool: if sys.version_info >= (3, 12): _TYPE_ALIAS_TYPES = (*_TYPE_ALIAS_TYPES, typing.TypeAliasType) +_IS_PY310 = sys.version_info[:2] == (3, 10) + def is_type_alias_type(tp: Any, /) -> TypeIs[typing_extensions.TypeAliasType]: """Return whether the provided argument is an instance of `TypeAliasType`. @@ -234,7 +319,14 @@ def is_type_alias_type(tp: Any, /) -> TypeIs[typing_extensions.TypeAliasType]: #> True ``` """ - return isinstance(tp, _TYPE_ALIAS_TYPES) + if _IS_PY310: + # Parametrized PEP 695 type aliases are instances of `types.GenericAlias` in typing_extensions>=4.13.0. + # On Python 3.10, with `Alias[int]` being such an instance of `GenericAlias`, + # `isinstance(Alias[int], TypeAliasType)` returns `True`. + # See https://github.com/python/cpython/issues/89828. + return type(tp) is not types.GenericAlias and isinstance(tp, _TYPE_ALIAS_TYPES) + else: + return isinstance(tp, _TYPE_ALIAS_TYPES) def is_classvar(tp: Any, /) -> bool: diff --git a/pydantic/fields.py b/pydantic/fields.py index 79c6253105..34ae090935 100644 --- a/pydantic/fields.py +++ b/pydantic/fields.py @@ -49,6 +49,7 @@ class _FromFieldInfoInputs(typing_extensions.TypedDict, total=False): """This class exists solely to add type checking for the `**kwargs` in `FieldInfo.from_field`.""" + # TODO PEP 747: use TypeForm: annotation: type[Any] | None default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None alias: str | None @@ -207,7 +208,7 @@ def __init__(self, **kwargs: Unpack[_FieldInfoInputs]) -> None: """ self._attributes_set = {k: v for k, v in kwargs.items() if v is not _Unset} kwargs = {k: _DefaultValues.get(k) if v is _Unset else v for k, v in kwargs.items()} # type: ignore - self.annotation, annotation_metadata = self._extract_metadata(kwargs.get('annotation')) + self.annotation = kwargs.get('annotation') self.evaluated = False default = kwargs.pop('default', PydanticUndefined) @@ -247,7 +248,7 @@ def __init__(self, **kwargs: Unpack[_FieldInfoInputs]) -> None: self.init_var = kwargs.pop('init_var', None) self.kw_only = kwargs.pop('kw_only', None) - self.metadata = self._collect_metadata(kwargs) + annotation_metadata # type: ignore + self.metadata = self._collect_metadata(kwargs) # type: ignore @staticmethod def from_field(default: Any = PydanticUndefined, **kwargs: Unpack[_FromFieldInfoInputs]) -> FieldInfo: @@ -310,34 +311,51 @@ class MyModel(pydantic.BaseModel): Returns: An instance of the field metadata. """ - final = False - if _typing_extra.is_finalvar(annotation): - final = True - if annotation is not typing_extensions.Final: + # 1. Check if the annotation is the `Final` type qualifier: + final = _typing_extra.is_finalvar(annotation) + if final: + if _typing_extra.is_generic_alias(annotation): + # 1.1. The annotation is a parametrized `Final`, e.g. `Final[int]`. + # In this case, `annotation` will be `int`: annotation = typing_extensions.get_args(annotation)[0] - - if _typing_extra.is_annotated(annotation): - first_arg, *extra_args = typing_extensions.get_args(annotation) - if _typing_extra.is_finalvar(first_arg): - final = True - field_info_annotations = [a for a in extra_args if isinstance(a, FieldInfo)] - field_info = FieldInfo.merge_field_infos(*field_info_annotations, annotation=first_arg) + else: + # 1.2. The annotation is a bare `Final`. Use `Any` as a type annotation: + return FieldInfo(annotation=Any, frozen=True) # pyright: ignore[reportArgumentType] (PEP 747) + + # 2. Check if the annotation is an `Annotated` form. + # In this case, `annotation` will be the annotated type: + annotation, metadata = _typing_extra.unpack_annotated(annotation) + + # 3. If we have metadata, `annotation` was the annotated type: + if metadata: + # 3.1. Check if the annotated type is the `Final` type qualifier. + # (i.e. `Annotated[Final[...], ...]`). Note that we only do + # so if `final` isn't `True` already, because we don't want to + # support the invalid `Final[Annotated[Final, ...]]` form. + if not final: + final = _typing_extra.is_finalvar(annotation) + if final and _typing_extra.is_generic_alias(annotation): + annotation = typing_extensions.get_args(annotation)[0] + + field_info_annotations = [a for a in metadata if isinstance(a, FieldInfo)] + field_info = FieldInfo.merge_field_infos(*field_info_annotations, annotation=annotation) if field_info: new_field_info = copy(field_info) - new_field_info.annotation = first_arg + new_field_info.annotation = annotation new_field_info.frozen = final or field_info.frozen - metadata: list[Any] = [] - for a in extra_args: + field_metadata: list[Any] = [] + for a in metadata: if _typing_extra.is_deprecated_instance(a): new_field_info.deprecated = a.message elif not isinstance(a, FieldInfo): - metadata.append(a) + field_metadata.append(a) else: - metadata.extend(a.metadata) - new_field_info.metadata = metadata + field_metadata.extend(a.metadata) + new_field_info.metadata = field_metadata return new_field_info - return FieldInfo(annotation=annotation, frozen=final or None) # pyright: ignore[reportArgumentType] + # 4. We don't have metadata: + return FieldInfo(annotation=annotation, frozen=final or None) # pyright: ignore[reportArgumentType] (PEP 747) @staticmethod def from_annotated_attribute(annotation: type[Any], default: Any) -> FieldInfo: @@ -367,16 +385,16 @@ class MyModel(pydantic.BaseModel): if annotation is default: raise PydanticUserError( 'Error when building FieldInfo from annotated attribute. ' - "Make sure you don't have any field name clashing with a type annotation ", + "Make sure you don't have any field name clashing with a type annotation.", code='unevaluable-type-annotation', ) final = _typing_extra.is_finalvar(annotation) - if final and annotation is not typing_extensions.Final: + if final and _typing_extra.is_generic_alias(annotation): annotation = typing_extensions.get_args(annotation)[0] if isinstance(default, FieldInfo): - default.annotation, annotation_metadata = FieldInfo._extract_metadata(annotation) # pyright: ignore[reportArgumentType] + default.annotation, annotation_metadata = _typing_extra.unpack_annotated(annotation) default.metadata += annotation_metadata default = default.merge_field_infos( *[x for x in annotation_metadata if isinstance(x, FieldInfo)], default, annotation=default.annotation @@ -394,7 +412,7 @@ class MyModel(pydantic.BaseModel): annotation = annotation.type pydantic_field = FieldInfo._from_dataclass_field(default) - pydantic_field.annotation, annotation_metadata = FieldInfo._extract_metadata(annotation) # pyright: ignore[reportArgumentType] + pydantic_field.annotation, annotation_metadata = _typing_extra.unpack_annotated(annotation) pydantic_field.metadata += annotation_metadata pydantic_field = pydantic_field.merge_field_infos( *[x for x in annotation_metadata if isinstance(x, FieldInfo)], @@ -407,19 +425,20 @@ class MyModel(pydantic.BaseModel): pydantic_field.kw_only = getattr(default, 'kw_only', None) return pydantic_field - if _typing_extra.is_annotated(annotation): - first_arg, *extra_args = typing_extensions.get_args(annotation) - field_infos = [a for a in extra_args if isinstance(a, FieldInfo)] - field_info = FieldInfo.merge_field_infos(*field_infos, annotation=first_arg, default=default) - metadata: list[Any] = [] - for a in extra_args: + annotation, metadata = _typing_extra.unpack_annotated(annotation) + + if metadata: + field_infos = [a for a in metadata if isinstance(a, FieldInfo)] + field_info = FieldInfo.merge_field_infos(*field_infos, annotation=annotation, default=default) + field_metadata: list[Any] = [] + for a in metadata: if _typing_extra.is_deprecated_instance(a): field_info.deprecated = a.message elif not isinstance(a, FieldInfo): - metadata.append(a) + field_metadata.append(a) else: - metadata.extend(a.metadata) - field_info.metadata = metadata + field_metadata.extend(a.metadata) + field_info.metadata = field_metadata return field_info return FieldInfo(annotation=annotation, default=default, frozen=final or None) # pyright: ignore[reportArgumentType] @@ -516,23 +535,6 @@ def _from_dataclass_field(dc_field: DataclassField[Any]) -> FieldInfo: dc_field_metadata = {k: v for k, v in dc_field.metadata.items() if k in _FIELD_ARG_NAMES} return Field(default=default, default_factory=default_factory, repr=dc_field.repr, **dc_field_metadata) # pyright: ignore[reportCallIssue] - @staticmethod - def _extract_metadata(annotation: type[Any] | None) -> tuple[type[Any] | None, list[Any]]: - """Tries to extract metadata/constraints from an annotation if it uses `Annotated`. - - Args: - annotation: The type hint annotation for which metadata has to be extracted. - - Returns: - A tuple containing the extracted metadata type and the list of extra arguments. - """ - if annotation is not None: - if _typing_extra.is_annotated(annotation): - first_arg, *extra_args = typing_extensions.get_args(annotation) - return first_arg, list(extra_args) - - return annotation, [] - @staticmethod def _collect_metadata(kwargs: dict[str, Any]) -> list[Any]: """Collect annotations from kwargs. diff --git a/tests/test_type_alias_type.py b/tests/test_type_alias_type.py index dd3c311185..0eba0b7949 100644 --- a/tests/test_type_alias_type.py +++ b/tests/test_type_alias_type.py @@ -314,28 +314,14 @@ def test_recursive_generic_type_alias_annotated_defs() -> None: } -@pytest.mark.xfail(reason='description is currently dropped') -def test_field() -> None: - SomeAlias = TypeAliasType('SomeAlias', Annotated[int, Field(description='number')]) - - ta = TypeAdapter(Annotated[SomeAlias, Field(title='abc')]) - - # insert_assert(ta.json_schema()) - assert ta.json_schema() == { - '$defs': {'SomeAlias': {'type': 'integer', 'description': 'number'}}, - '$ref': '#/$defs/SomeAlias', - 'title': 'abc', - } - - def test_nested_generic_type_alias_type() -> None: class MyModel(BaseModel): field_1: MyList[bool] field_2: MyList[str] - model = MyModel(field_1=[True], field_2=['abc']) + MyModel(field_1=[True], field_2=['abc']) - assert model.model_json_schema() == { + assert MyModel.model_json_schema() == { '$defs': { 'MyList_bool_': {'items': {'type': 'boolean'}, 'type': 'array'}, 'MyList_str_': {'items': {'type': 'string'}, 'type': 'array'}, @@ -443,3 +429,45 @@ class MyModel(BaseModel): assert exc_info.value.code == 'circular-reference-schema' assert exc_info.value.message.startswith('tests.test_type_alias_type.C') + + +## Tests related to (recursive) unpacking of annotated types, when PEP 695 type aliases are involved: + + +def test_nested_annotated_with_type_aliases() -> None: + SomeAlias = TypeAliasType('SomeAlias', Annotated[int, Field(description='number')]) + + ta = TypeAdapter(Annotated[SomeAlias, Field(title='abc')]) + + assert ta.json_schema() == {'description': 'number', 'title': 'abc', 'type': 'integer'} + + +@pytest.mark.xfail( + reason="When trying to recursively unpack the annotated form, we don't resolve " + 'forward annotations in PEP 695 type aliases (due to current limitations) ' + '(see https://github.com/pydantic/pydantic/issues/11122).', +) +def test_nested_annotated_with_type_aliases_and_forward_ref() -> None: + SomeAlias = TypeAliasType('SomeAlias', "Annotated[int, Field(description='number')]") + + ta = TypeAdapter(Annotated[SomeAlias, Field(title='abc')]) + + assert ta.json_schema() == {'description': 'number', 'title': 'abc', 'type': 'integer'} + + +def test_nested_annotated_model_field() -> None: + T = TypeVar('T') + + InnerList = TypeAliasType('InnerList', Annotated[List[T], Field(alias='alias')], type_params=(T,)) + MyList = TypeAliasType('MyList', Annotated[InnerList[T], Field(deprecated=True)], type_params=(T,)) + MyIntList = TypeAliasType('MyIntList', MyList[int]) + + class Model(BaseModel): + f1: Annotated[MyIntList, Field(json_schema_extra={'extra': 'test'})] + + f1_info = Model.model_fields['f1'] + + assert f1_info.annotation == List[int] + assert f1_info.alias == 'alias' + assert f1_info.deprecated + assert f1_info.json_schema_extra == {'extra': 'test'}