Sequence support (pydantic#428)

fix pydantic#304

* Sequence support added

* Unittests for sequence added

* Fix HISTORY

* Sequence validation simplified

* Fix type conversion for Sequence

HISTORY.rst

@@ -3,13 +3,13 @@
 History
 -------
 
-v0.21.1 (unreleased)
+v0.22 (unreleased)
 ....................
 * add ``IPv{4,6,Any}Network`` and ``IPv{4,6,Any}Interface`` types from ``ipaddress`` stdlib, #333 by @pilosus
 * add docs for ``datetime`` types, #386 by @pilosus
 * fix to schema generation in dataclass-based models, #408 by @pilosus
 * fix path in nested models #437, by @kataev
-
+* add ``Sequence`` support, #304 by @pilosus
 
 v0.21.0 (2019-03-15)
 ....................

docs/examples/ex_typing.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 from pydantic import BaseModel
 
@@ -19,6 +19,8 @@ class Model(BaseModel):
     str_or_bytes: Union[str, bytes] = None
     none_or_str: Optional[str] = None
 
+    sequence_of_ints: Sequence[int] = None
+
     compound: Dict[Union[str, bytes], List[Set[int]]] = None
 
 print(Model(simple_list=['1', '2', '3']).simple_list)  # > ['1', '2', '3']
@@ -29,3 +31,6 @@ class Model(BaseModel):
 
 print(Model(simple_tuple=[1, 2, 3, 4]).simple_tuple)  # > (1, 2, 3, 4)
 print(Model(tuple_of_different_types=[1, 2, 3, 4]).tuple_of_different_types)  # > (1, 2.0, '3', True)
+
+print(Model(sequence_of_ints=[1, 2, 3, 4]).sequence_of_ints)  # > [1, 2, 3, 4]
+print(Model(sequence_of_ints=(1, 2, 3, 4)).sequence_of_ints)  # > (1, 2, 3, 4)

pydantic/errors.py

@@ -117,6 +117,10 @@ class PyObjectError(PydanticTypeError):
     msg_template = 'ensure this value contains valid import path: {error_message}'
 
 
+class SequenceError(PydanticTypeError):
+    msg_template = 'value is not a valid sequence'
+
+
 class ListError(PydanticTypeError):
     msg_template = 'value is not a valid list'
 

pydantic/fields.py

@@ -1,12 +1,29 @@
 import warnings
 from enum import IntEnum
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Pattern, Set, Tuple, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Pattern,
+    Sequence,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 from . import errors as errors_
 from .class_validators import Validator, make_generic_validator
 from .error_wrappers import ErrorWrapper
 from .types import Json, JsonWrapper
-from .utils import AnyCallable, AnyType, Callable, ForwardRef, display_as_type, lenient_issubclass, list_like
+from .utils import AnyCallable, AnyType, Callable, ForwardRef, display_as_type, lenient_issubclass, sequence_like
 from .validators import NoneType, dict_validator, find_validators
 
 Required: Any = Ellipsis
@@ -28,6 +45,7 @@ class Shape(IntEnum):
     SET = 3
     MAPPING = 4
     TUPLE = 5
+    SEQUENCE = 6
 
 
 class Field:
@@ -152,7 +170,7 @@ def prepare(self) -> None:
         self._populate_sub_fields()
         self._populate_validators()
 
-    def _populate_sub_fields(self) -> None:
+    def _populate_sub_fields(self) -> None:  # noqa: C901 (ignore complexity)
         # typing interface is horrible, we have to do some ugly checks
         if lenient_issubclass(self.type_, JsonWrapper):
             self.type_ = self.type_.inner_type  # type: ignore
@@ -190,6 +208,9 @@ def _populate_sub_fields(self) -> None:
         elif issubclass(origin, Set):
             self.type_ = self.type_.__args__[0]  # type: ignore
             self.shape = Shape.SET
+        elif issubclass(origin, Sequence):
+            self.type_ = self.type_.__args__[0]  # type: ignore
+            self.shape = Shape.SEQUENCE
         else:
             assert issubclass(origin, Mapping)
             self.key_field = self._create_sub_type(
@@ -265,10 +286,8 @@ def validate(
         elif self.shape is Shape.TUPLE:
             v, errors = self._validate_tuple(v, values, loc, cls)
         else:
-            # list or set
-            v, errors = self._validate_list_set(v, values, loc, cls)
-            if not errors and self.shape is Shape.SET:
-                v = set(v)
+            #  sequence, list, tuple, set, generator
+            v, errors = self._validate_sequence_like(v, values, loc, cls)
 
         if not errors and self.whole_post_validators:
             v, errors = self._apply_validators(v, values, loc, cls, self.whole_post_validators)
@@ -280,11 +299,21 @@ def _validate_json(self, v: str, loc: Tuple[str, ...]) -> Tuple[Optional[Any], O
         except (ValueError, TypeError) as exc:
             return v, ErrorWrapper(exc, loc=loc, config=self.model_config)
 
-    def _validate_list_set(
+    def _validate_sequence_like(
         self, v: Any, values: Dict[str, Any], loc: 'LocType', cls: Optional[Type['BaseModel']]
     ) -> 'ValidateReturn':
-        if not list_like(v):
-            e = errors_.ListError() if self.shape is Shape.LIST else errors_.SetError()
+        """
+        Validate sequence-like containers: lists, tuples, sets and generators
+        """
+
+        if not sequence_like(v):
+            e: errors_.PydanticTypeError
+            if self.shape is Shape.LIST:
+                e = errors_.ListError()
+            elif self.shape is Shape.SET:
+                e = errors_.SetError()
+            else:
+                e = errors_.SequenceError()
             return v, ErrorWrapper(e, loc=loc, config=self.model_config)
 
         result = []
@@ -299,14 +328,25 @@ def _validate_list_set(
 
         if errors:
             return v, errors
-        else:
-            return result, None
+
+        converted: Union[List[Any], Set[Any], Tuple[Any, ...], Iterator[Any]] = result
+
+        if self.shape is Shape.SET:
+            converted = set(result)
+        elif self.shape is Shape.SEQUENCE:
+            if isinstance(v, tuple):
+                converted = tuple(result)
+            elif isinstance(v, set):
+                converted = set(result)
+            elif isinstance(v, Generator):
+                converted = iter(result)
+        return converted, None
 
     def _validate_tuple(
         self, v: Any, values: Dict[str, Any], loc: 'LocType', cls: Optional[Type['BaseModel']]
     ) -> 'ValidateReturn':
         e: Optional[Exception] = None
-        if not list_like(v):
+        if not sequence_like(v):
             e = errors_.TupleError()
         else:
             actual_length, expected_length = len(v), len(self.sub_fields)  # type: ignore

pydantic/utils.py

@@ -180,7 +180,7 @@ def clean_docstring(d: str) -> str:
     return dedent(d).strip(' \r\n\t')
 
 
-def list_like(v: AnyType) -> bool:
+def sequence_like(v: AnyType) -> bool:
     return isinstance(v, (list, tuple, set)) or inspect.isgenerator(v)
 
 

pydantic/validators.py

@@ -10,7 +10,7 @@
 
 from . import errors
 from .datetime_parse import parse_date, parse_datetime, parse_duration, parse_time
-from .utils import AnyCallable, AnyType, change_exception, display_as_type, is_callable_type, list_like
+from .utils import AnyCallable, AnyType, change_exception, display_as_type, is_callable_type, sequence_like
 
 if TYPE_CHECKING:  # pragma: no cover
     from .fields import Field
@@ -154,7 +154,7 @@ def dict_validator(v: Any) -> Dict[Any, Any]:
 def list_validator(v: Any) -> List[Any]:
     if isinstance(v, list):
         return v
-    elif list_like(v):
+    elif sequence_like(v):
         return list(v)
     else:
         raise errors.ListError()
@@ -163,7 +163,7 @@ def list_validator(v: Any) -> List[Any]:
 def tuple_validator(v: Any) -> Tuple[Any, ...]:
     if isinstance(v, tuple):
         return v
-    elif list_like(v):
+    elif sequence_like(v):
         return tuple(v)
     else:
         raise errors.TupleError()
@@ -172,7 +172,7 @@ def tuple_validator(v: Any) -> Tuple[Any, ...]:
 def set_validator(v: Any) -> Set[Any]:
     if isinstance(v, set):
         return v
-    elif list_like(v):
+    elif sequence_like(v):
         return set(v)
     else:
         raise errors.SetError()

pydantic/version.py

@@ -2,4 +2,4 @@
 
 __all__ = ['VERSION']
 
-VERSION = StrictVersion('0.21.1')
+VERSION = StrictVersion('0.22')

tests/test_types.py

@@ -5,7 +5,7 @@
 from decimal import Decimal
 from enum import Enum, IntEnum
 from pathlib import Path
-from typing import List, NewType, Pattern, Set
+from typing import Iterator, List, NewType, Pattern, Sequence, Set, Tuple
 from uuid import UUID
 
 import pytest
@@ -548,6 +548,106 @@ class Model(BaseModel):
     assert exc_info.value.errors() == [{'loc': ('v',), 'msg': 'value is not a valid set', 'type': 'type_error.set'}]
 
 
+@pytest.mark.parametrize(
+    'cls, value,result',
+    (
+        (int, [1, 2, 3], [1, 2, 3]),
+        (int, (1, 2, 3), (1, 2, 3)),
+        (float, {1.0, 2.0, 3.0}, {1.0, 2.0, 3.0}),
+        (Set[int], [{1, 2}, {3, 4}, {5, 6}], [{1, 2}, {3, 4}, {5, 6}]),
+        (Tuple[int, str], ((1, 'a'), (2, 'b'), (3, 'c')), ((1, 'a'), (2, 'b'), (3, 'c'))),
+    ),
+)
+def test_sequence_success(cls, value, result):
+    class Model(BaseModel):
+        v: Sequence[cls]
+
+    assert Model(v=value).v == result
+
+
+@pytest.mark.parametrize(
+    'cls, value,result',
+    (
+        (int, (i for i in range(3)), iter([0, 1, 2])),
+        (float, (float(i) for i in range(3)), iter([0.0, 1.0, 2.0])),
+        (str, (str(i) for i in range(3)), iter(['0', '1', '2'])),
+    ),
+)
+def test_sequence_generator_success(cls, value, result):
+    class Model(BaseModel):
+        v: Sequence[cls]
+
+    validated = Model(v=value).v
+    assert isinstance(validated, Iterator)
+    assert list(validated) == list(result)
+
+
+@pytest.mark.parametrize(
+    'cls,value,errors',
+    (
+        (
+            int,
+            (i for i in ['a', 'b', 'c']),
+            [
+                {'loc': ('v', 0), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'},
+                {'loc': ('v', 1), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'},
+                {'loc': ('v', 2), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'},
+            ],
+        ),
+        (
+            float,
+            (i for i in ['a', 'b', 'c']),
+            [
+                {'loc': ('v', 0), 'msg': 'value is not a valid float', 'type': 'type_error.float'},
+                {'loc': ('v', 1), 'msg': 'value is not a valid float', 'type': 'type_error.float'},
+                {'loc': ('v', 2), 'msg': 'value is not a valid float', 'type': 'type_error.float'},
+            ],
+        ),
+    ),
+)
+def test_sequence_generator_fails(cls, value, errors):
+    class Model(BaseModel):
+        v: Sequence[cls]
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(v=value)
+    assert exc_info.value.errors() == errors
+
+
+@pytest.mark.parametrize(
+    'cls,value,errors',
+    (
+        (int, [1, 'a', 3], [{'loc': ('v', 1), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'}]),
+        (int, (1, 2, 'a'), [{'loc': ('v', 2), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'}]),
+        (float, range(10), [{'loc': ('v',), 'msg': 'value is not a valid sequence', 'type': 'type_error.sequence'}]),
+        (float, ('a', 2.2, 3.3), [{'loc': ('v', 0), 'msg': 'value is not a valid float', 'type': 'type_error.float'}]),
+        (float, (1.1, 2.2, 'a'), [{'loc': ('v', 2), 'msg': 'value is not a valid float', 'type': 'type_error.float'}]),
+        (
+            Set[int],
+            [{1, 2}, {2, 3}, {'d'}],
+            [{'loc': ('v', 2, 0), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'}],
+        ),
+        (
+            Tuple[int, str],
+            ((1, 'a'), ('a', 'a'), (3, 'c')),
+            [{'loc': ('v', 1, 0), 'msg': 'value is not a valid integer', 'type': 'type_error.integer'}],
+        ),
+        (
+            List[int],
+            [{'a': 1, 'b': 2}, [1, 2], [2, 3]],
+            [{'loc': ('v', 0), 'msg': 'value is not a valid list', 'type': 'type_error.list'}],
+        ),
+    ),
+)
+def test_sequence_fails(cls, value, errors):
+    class Model(BaseModel):
+        v: Sequence[cls]
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(v=value)
+    assert exc_info.value.errors() == errors
+
+
 def test_int_validation():
     class Model(BaseModel):
         a: PositiveInt = None