storage: Use pydantic v1 compatibility

Use the bundled v1 of pydantic when running with v2 for now. This allows
for usage in Home Assistant until pydantic v2 is supported there.

Relates to #2261

base_versions.txt

@@ -6,7 +6,7 @@ ifaddr==0.1.7
 mediafile==0.8.1
 miniaudio==1.45
 protobuf==4.23.4
-pydantic==2.4.0
+pydantic==1.10.10
 requests==2.23.0
 srptools==0.2.0
 tabulate==0.9.0

docs/api/pyatv.storage.html

@@ -51,7 +51,7 @@ <h1 class="title">Module <code>pyatv.storage</code></h1>
 </header>
 <section id="section-intro">
 <p>Storage module.</p>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L1-L175" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L1-L174" class="git-link">Browse git</a></div>
 </section>
 <section>
 <h2 class="section-title" id="header-submodules">Sub-modules</h2>
@@ -81,7 +81,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 <p>New storage modules should generally inherit from this class and implement save and
 load according to the underlying storage mechanism.</p>
 <p>Initialize a new AbstractStorage instance.</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L42-L175" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L41-L174" class="git-link">Browse git</a></div>
 <h3>Ancestors</h3>
 <ul class="hlist">
 <li><a title="pyatv.interface.Storage" href="../../interface#pyatv.interface.Storage">Storage</a></li>
@@ -100,12 +100,12 @@ <h3>Instance variables</h3>
 <p>This property will return True if a any setting has been changed. It is reset
 when data is loaded into storage (by calling load) or manually by calling
 mark_as_solved (typically done by save).</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L54-L62" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L53-L61" class="git-link">Browse git</a></div>
 </dd>
 <dt id="pyatv.storage.AbstractStorage.storage_model"><code class="name">var <span class="ident">storage_model</span> -> <a title="pyatv.storage.StorageModel" href="#pyatv.storage.StorageModel">StorageModel</a></code></dt>
 <dd>
 <section class="desc"><p>Return storage model representation.</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L69-L72" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L68-L71" class="git-link">Browse git</a></div>
 </dd>
 </dl>
 <h3>Methods</h3>
@@ -124,7 +124,7 @@ <h3>Methods</h3>
 identitiers, DeviceIdMissingError will be raised.</p>
 <p>If settings exists for a configuration but mismatch, they will be automatically
 updated in the storage. Set ignore_update to False to not update storage.</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L89-L122" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L88-L121" class="git-link">Browse git</a></div>
 </dd>
 <dt id="pyatv.storage.AbstractStorage.mark_as_saved">
 <code class="name flex">
@@ -135,7 +135,7 @@ <h3>Methods</h3>
 <section class="desc"><p>Call after saving to indicate settings have been saved.</p>
 <p>The changed property reflects whether something has been changed in the model
 or not based on calling this method.</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L81-L87" class="git-link">Browse git</a></div>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L80-L86" class="git-link">Browse git</a></div>
 </dd>
 </dl>
 <h3>Inherited members</h3>
@@ -158,22 +158,20 @@ <h3>Inherited members</h3>
 <dd>
 <section class="desc"><p>Storage model of data that is saved or restored to underlying storage.</p>
 <p>Create a new model by parsing and validating input data from keyword arguments.</p>
-<p>Raises [<code>ValidationError</code>][pydantic_core.ValidationError] if the input data cannot be
-validated to form a valid model.</p>
-<p><code>__init__</code> uses <code>__pydantic_self__</code> instead of the more common <code>self</code> for the first arg to
-allow <code>self</code> as a field name.</p></section>
-<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L35-L39" class="git-link">Browse git</a></div>
+<p>Raises ValidationError if the input data cannot be parsed to form a valid model.</p></section>
+<div class="git-link-div"><a href="https://github.com/postlund/pyatv/blob/master/pyatv/storage/__init__.py#L34-L38" class="git-link">Browse git</a></div>
 <h3>Ancestors</h3>
 <ul class="hlist">
-<li>pydantic.main.BaseModel</li>
+<li>pydantic.v1.main.BaseModel</li>
+<li>pydantic.v1.utils.Representation</li>
 </ul>
 <h3>Class variables</h3>
 <dl>
-<dt id="pyatv.storage.StorageModel.devices"><code class="name">var <span class="ident">devices</span> -> List[<a title="pyatv.settings.Settings" href="../../settings#pyatv.settings.Settings">Settings</a>] = PydanticUndefined</code></dt>
+<dt id="pyatv.storage.StorageModel.devices"><code class="name">var <span class="ident">devices</span> -> List[<a title="pyatv.settings.Settings" href="../../settings#pyatv.settings.Settings">Settings</a>]</code></dt>
 <dd>
 <section class="desc"></section>
 </dd>
-<dt id="pyatv.storage.StorageModel.version"><code class="name">var <span class="ident">version</span> -> int = PydanticUndefined</code></dt>
+<dt id="pyatv.storage.StorageModel.version"><code class="name">var <span class="ident">version</span> -> int</code></dt>
 <dd>
 <section class="desc"></section>
 </dd>

docs/pdoc_templates/html.mako

@@ -272,7 +272,10 @@
                   if any(cls.name.startswith("enum.") for cls in mro):
                     var_value = f" = {getattr(c.obj, v.name).value}"
                   elif any(cls.name.startswith("pydantic.main.BaseModel") for cls in mro):
-                    field_info = c.obj.model_fields[v.name]
+                    if hasattr(c.obj, "__fields__"):
+                      field_info = c.obj.__fields__[v.name]
+                    else:
+                      field_info = c.obj.model_fields[v.name]
 
                     # Ignore fields that are pydantic models as they produce unnecessary output
                     if "__pydantic_serializer__" not in dir(field_info.annotation):

pyatv/interface.py

@@ -1361,15 +1361,15 @@ def apply(self, settings: Settings) -> None:
         """Apply settings to configuration."""
         for service in self.services:
             if service.protocol == Protocol.AirPlay:
-                service.apply(settings.protocols.airplay.model_dump())
+                service.apply(dict(settings.protocols.airplay))
             elif service.protocol == Protocol.Companion:
-                service.apply(settings.protocols.companion.model_dump())
+                service.apply(dict(settings.protocols.companion))
             elif service.protocol == Protocol.DMAP:
-                service.apply(settings.protocols.dmap.model_dump())
+                service.apply(dict(settings.protocols.dmap))
             elif service.protocol == Protocol.MRP:
-                service.apply(settings.protocols.mrp.model_dump())
+                service.apply(dict(settings.protocols.mrp))
             elif service.protocol == Protocol.RAOP:
-                service.apply(settings.protocols.raop.model_dump())
+                service.apply(dict(settings.protocols.raop))
 
     def __eq__(self, other) -> bool:
         """Compare instance with another instance."""

pyatv/settings.py

@@ -3,9 +3,7 @@
 import re
 from typing import Optional
 
-from pydantic import BaseModel, Field
-from pydantic.functional_validators import AfterValidator
-from typing_extensions import Annotated
+from pyatv.support.pydantic_compat import BaseModel, Field, field_validator
 
 __pdoc__ = {
     "InfoSettings.model_config": False,
@@ -34,15 +32,6 @@
 _MAC_REGEX = r"[0-9a-fA-F]{2}(:[0-9a-fA-F]{2}){5}"
 
 
-def _mac_validator(mac_addr: str) -> str:
-    assert (
-        re.match(_MAC_REGEX, mac_addr) is not None
-    ), f"{mac_addr} is not a valid MAC address"
-    return mac_addr
-
-
-MacAddress = Annotated[str, AfterValidator(_mac_validator)]
-
 DEFAULT_NAME = "pyatv"
 DEFUALT_MAC = "02:70:79:61:74:76"  # Locally administrated (02) + "pyatv" in hex
 DEFAULT_DEVICE_ID = "FF:70:79:61:74:76"  # 0xFF + "pyatv"
@@ -69,13 +58,21 @@ class InfoSettings(BaseModel, extra="ignore"):  # type: ignore[call-arg]
     """Information related settings."""
 
     name: str = DEFAULT_NAME
-    mac: MacAddress = DEFUALT_MAC
+    mac: str = DEFUALT_MAC
     model: str = DEFAULT_MODEL
     device_id: str = DEFAULT_DEVICE_ID
     os_name: str = DEFAULT_OS_NAME
     os_build: str = DEFAULT_OS_BUILD
     os_version: str = DEFAULT_OS_VERSION
 
+    @field_validator("mac")
+    @classmethod
+    def mac_validator(cls, mac: str) -> str:
+        """Validate MAC address to be correct."""
+        if re.match(_MAC_REGEX, mac) is None:
+            raise ValueError(f"{mac} is not a valid MAC address")
+        return mac
+
 
 class AirPlaySettings(BaseModel, extra="ignore"):  # type: ignore[call-arg]
     """Settings related to AirPlay."""

pyatv/storage/__init__.py

@@ -2,12 +2,11 @@
 from hashlib import sha256
 from typing import List, Sequence
 
-from pydantic import BaseModel
-
 from pyatv.const import Protocol
 from pyatv.exceptions import DeviceIdMissingError, SettingsError
 from pyatv.interface import BaseConfig, Storage
 from pyatv.settings import Settings
+from pyatv.support.pydantic_compat import BaseModel, model_copy
 
 __pdoc_dev_page__ = "/development/storage"
 
@@ -25,7 +24,7 @@ def _calculate_settings_hash(settings: List[Settings]) -> str:
     # (especially when making multiple calls) but should be reliable and good enough.
     hasher = sha256()
     for setting in settings:
-        setting_json = setting.model_dump_json(exclude_defaults=True)
+        setting_json = setting.json(exclude_defaults=True)
         hasher.update(setting_json.encode("utf-8"))
     return hasher.hexdigest()
 
@@ -145,28 +144,28 @@ def _update_settings_from_config(config: BaseConfig, settings: Settings) -> None
         for service in config.services:
             # TODO: Clean this up/make more general
             if service.protocol == Protocol.AirPlay:
-                settings.protocols.airplay = settings.protocols.airplay.model_copy(
-                    update=service.settings()
+                settings.protocols.airplay = model_copy(
+                    settings.protocols.airplay, update=service.settings()
                 )
                 settings.protocols.airplay.identifier = service.identifier
             elif service.protocol == Protocol.DMAP:
-                settings.protocols.dmap = settings.protocols.dmap.model_copy(
-                    update=service.settings()
+                settings.protocols.dmap = model_copy(
+                    settings.protocols.dmap, update=service.settings()
                 )
                 settings.protocols.dmap.identifier = service.identifier
             elif service.protocol == Protocol.Companion:
-                settings.protocols.companion = settings.protocols.companion.model_copy(
-                    update=service.settings()
+                settings.protocols.companion = model_copy(
+                    settings.protocols.companion, update=service.settings()
                 )
                 settings.protocols.companion.identifier = service.identifier
             elif service.protocol == Protocol.MRP:
-                settings.protocols.mrp = settings.protocols.mrp.model_copy(
-                    update=service.settings()
+                settings.protocols.mrp = model_copy(
+                    settings.protocols.mrp, update=service.settings()
                 )
                 settings.protocols.mrp.identifier = service.identifier
             if service.protocol == Protocol.RAOP:
-                settings.protocols.raop = settings.protocols.raop.model_copy(
-                    update=service.settings()
+                settings.protocols.raop = model_copy(
+                    settings.protocols.raop, update=service.settings()
                 )
                 settings.protocols.raop.identifier = service.identifier
 

pyatv/storage/file_storage.py

@@ -45,7 +45,7 @@ def _save_file(self) -> None:
         # If settings are empty for a device (e.e. no settings overridden or credentials
         # saved), then the output will just be an empty dict. To not pollute the output
         # with those, we do some filtering here.
-        dumped = self.storage_model.model_dump(exclude_defaults=True)
+        dumped = self.storage_model.dict(exclude_defaults=True)
         dumped["devices"] = [device for device in dumped["devices"] if device != {}]
 
         with open(self._filename, "w", encoding="utf-8") as _fh:

pyatv/support/__init__.py

@@ -9,10 +9,10 @@
 import warnings
 
 from google.protobuf.text_format import MessageToString
-from pydantic import BaseModel
 
 import pyatv
 from pyatv import exceptions
+from pyatv.support.pydantic_compat import BaseModel
 
 _PROTOBUF_LINE_LENGTH = 150
 _BINARY_LINE_LENGTH = 512
@@ -170,27 +170,26 @@ def stringify_model(model: BaseModel) -> Sequence[str]:
     It is assumed optional field does not contain other models (only basic types).
     """
 
+    def _lookup_type(current_model: BaseModel, type_path: str) -> str:
+        splitted_path = type_path.split(".", maxsplit=1)
+        value = current_model.__annotations__[splitted_path[0]]
+        if len(splitted_path) == 1:
+            if value.__dict__.get("__origin__") is Union:
+                return ", ".join(arg.__name__ for arg in value.__args__)
+            return value.__name__
+        return _lookup_type(value, splitted_path[1])
+
     def _recurse_into(
         current_model: BaseModel, prefix: str, output: List[str]
     ) -> Sequence[str]:
-        for name, field in current_model.model_fields.items():
-            if field.annotation.__dict__.get("__origin__") is Union:
-                field_types = ", ".join(
-                    arg.__name__ for arg in field.annotation.__args__
-                )
-                output.append(
-                    f"{prefix}{name} = {getattr(current_model, name)} ({field_types})"
-                )
-            elif BaseModel in field.annotation.__mro__:
+        for name, field in dict(current_model).items():
+            if hasattr(field, "__annotations__"):
                 _recurse_into(
                     getattr(current_model, name), (prefix or "") + f"{name}.", output
                 )
-            elif field.default is not None:
-                output.append(
-                    f"{prefix}{name} = "
-                    f"{getattr(current_model, name)} "
-                    f"({field.annotation.__name__})"
-                )
+            else:
+                field_type = _lookup_type(model, f"{prefix}{name}")
+                output.append(f"{prefix}{name} = {field} ({field_type})")
         return output
 
     return _recurse_into(model, "", [])
@@ -209,5 +208,5 @@ def update_model_field(
     if len(splitted_path) > 1:
         update_model_field(getattr(model, next_field), splitted_path[1], value)
     else:
-        model.model_validate({field: value})
+        model.parse_obj({field: value})
         setattr(model, field, value)

pyatv/support/pydantic_compat.py

@@ -0,0 +1,36 @@
+"""Compatibility module for pydantic.
+
+This module provides some compatibility methods to support both pydantic v1 and v2 in
+pyatv. Ideally only v2 should be supported, but due to Home Assistant being stuck at v1
+for now, backwards compatibility will be provided until that is resolved. More info in
+https://github.com/postlund/pyatv/issues/2261.
+
+The idea is that the rest of the code never imports anything directly from pydantic,
+but instead getting import from here. That makes it easy to remove these changes
+later on.
+"""
+
+from typing import Any, Mapping
+
+# pylint: disable=unused-import
+
+try:
+    from pydantic.v1 import BaseModel, Field, ValidationError  # noqa
+    from pydantic.v1 import validator as field_validator  # noqa
+except ImportError:
+    from pydantic import BaseModel, Field, ValidationError  # noqa
+    from pydantic import validator as field_validator  # noqa
+
+# pylint: enable=unused-import
+
+
+def model_copy(model: BaseModel, /, update: Mapping[str, Any]) -> BaseModel:
+    """Model copy compatible with pydantic v2.
+
+    Seems like pydantic v1 carries over keys with None values even though target model
+    doesn't have the key. Not the case with v2. This method removes keys with None
+    values.
+    """
+    return model.copy(
+        update={key: value for key, value in update.items() if value is not None}
+    )

tests/support/test_support.py

@@ -9,7 +9,6 @@
 from unittest.mock import MagicMock, patch
 
 from deepdiff import DeepDiff
-from pydantic import BaseModel, Field, ValidationError
 import pytest
 
 from pyatv import exceptions
@@ -24,6 +23,7 @@
     stringify_model,
     update_model_field,
 )
+from pyatv.support.pydantic_compat import BaseModel, Field, ValidationError
 
 
 class DummyException(Exception):

tests/test_storage_functional.py

@@ -49,7 +49,7 @@ async def test_scan_inserts_into_storage(event_loop, unicast_scan, mockfs):
     # Compare content to ensure they are exactly the same.
     storage2 = await new_storage(STORAGE_FILENAME, event_loop)
     settings2 = await storage2.get_settings(conf)
-    assert not DeepDiff(settings2.model_dump(), settings1.model_dump())
+    assert not DeepDiff(settings2.dict(), settings1.dict())
 
 
 async def test_provides_storage_to_pairing_handler(