Update to Pydantic 2.9

pyproject.toml

@@ -249,7 +249,8 @@ opensearch-py = "~1.1"
 palace-webpub-manifest-parser = "^4.0.0"
 pillow = "^10.0"
 pycryptodome = "^3.18"
-pydantic = {version = "^1.10.9", extras = ["dotenv", "email"]}
+pydantic = {version = "^2.9.2", extras = ["dotenv", "email"]}
+pydantic-settings = "^2.5.2"
 pyinstrument = "^4.6"
 PyJWT = "^2.8"
 PyLD = "2.0.4"
@@ -326,6 +327,7 @@ addopts = [
     "--strict-markers",
 ]
 filterwarnings = [
+    "error::pydantic.PydanticDeprecatedSince20",
     "error::pytest.PytestWarning",
 ]
 markers = [

src/palace/manager/api/admin/config.py

@@ -1,19 +1,21 @@
 import os
 from enum import Enum
+from typing import Literal
 from urllib.parse import urljoin
 
-from pydantic import Field
+from pydantic import AliasGenerator, Field
+from pydantic.alias_generators import to_camel
+from pydantic_settings import SettingsConfigDict
 from requests import RequestException
 
-from palace.manager.service.configuration.limited_env_override import (
-    ServiceConfigurationWithLimitedEnvOverride,
+from palace.manager.service.configuration.service_configuration import (
+    ServiceConfiguration,
 )
-from palace.manager.util.flask_util import _snake_to_camel_case
 from palace.manager.util.http import HTTP, RequestNetworkException
 from palace.manager.util.log import LoggerMixin
 
 
-class AdminClientFeatureFlags(ServiceConfigurationWithLimitedEnvOverride):
+class AdminClientFeatureFlags(ServiceConfiguration):
     # The following CAN be overridden by environment variables.
     reports_only_for_sysadmins: bool = Field(
         True,
@@ -24,38 +26,20 @@ class AdminClientFeatureFlags(ServiceConfigurationWithLimitedEnvOverride):
         description="Show QuickSight dashboards only for sysadmins.",
     )
 
-    # The following fields CANNOT be overridden by environment variables.
-    # Setting `const=True` ensures that the default value is not overridden.
-    # Add them to the one of the `environment_override_*` Config settings
-    # below to prevent them from being overridden.
-    # NB: Overriding the `env_prefix` with `env=...` here may lead to
-    #     incorrect values in warnings and exceptions, since `env_prefix`
-    #     is used to generate the full environment variable name.
-    enable_auto_list: bool = Field(
+    # The following fields CANNOT be overridden by environment variables, so
+    # their types are set to Literal[True] to prevent them from being overridden.
+    enable_auto_list: Literal[True] = Field(
         True,
-        const=True,
         description="Enable auto-list of items.",
     )
-    show_circ_events_download: bool = Field(
+    show_circ_events_download: Literal[True] = Field(
         True,
-        const=True,
         description="Show download button for Circulation Events.",
     )
-
-    class Config:
-        env_prefix = "PALACE_ADMINUI_FEATURE_"
-
-        # We use lower camel case aliases, since we're sending to the web.
-        alias_generator = _snake_to_camel_case
-
-        # Add any fields that should not be overridden by environment variables here.
-        # - environment_override_warning_fields: warnings and ignore environment
-        # - environment_override_error_fields: raise exception
-        environment_override_warning_fields: set[str] = {
-            "enable_auto_list",
-            "show_circ_events_download",
-        }
-        environment_override_error_fields: set[str] = set()
+    model_config = SettingsConfigDict(
+        env_prefix="PALACE_ADMINUI_FEATURE_",
+        alias_generator=AliasGenerator(serialization_alias=to_camel),
+    )
 
 
 class OperationalMode(str, Enum):

src/palace/manager/api/admin/controller/catalog_services.py

@@ -85,7 +85,7 @@ def process_post(self) -> Response | ProblemDetail:
             impl_cls = self.registry[protocol]
             settings_class = impl_cls.settings_class()
             validated_settings = ProcessFormData.get_settings(settings_class, form_data)
-            catalog_service.settings_dict = validated_settings.dict()
+            catalog_service.settings_dict = validated_settings.model_dump()
 
             # Update library settings
             if libraries_data:

src/palace/manager/api/admin/controller/collection_settings.py

@@ -126,7 +126,7 @@ def process_post(self) -> Response | ProblemDetail:
 
             # Update settings
             validated_settings = ProcessFormData.get_settings(settings_class, form_data)
-            integration.settings_dict = validated_settings.dict()
+            integration.settings_dict = validated_settings.model_dump()
 
             # Update library settings
             if libraries_data:

src/palace/manager/api/admin/controller/custom_lists.py

@@ -7,7 +7,6 @@
 import flask
 from flask import Response, url_for
 from flask_babel import lazy_gettext as _
-from pydantic import BaseModel
 
 from palace.manager.api.admin.controller.base import AdminPermissionsControllerMixin
 from palace.manager.api.admin.problem_details import (
@@ -37,18 +36,18 @@
 from palace.manager.sqlalchemy.model.licensing import LicensePool
 from palace.manager.sqlalchemy.model.work import Work
 from palace.manager.sqlalchemy.util import create, get_one
-from palace.manager.util.flask_util import parse_multi_dict
+from palace.manager.util.flask_util import CustomBaseModel, parse_multi_dict
 from palace.manager.util.problem_detail import ProblemDetail, ProblemDetailException
 
 
 class CustomListsController(
     CirculationManagerController, AdminPermissionsControllerMixin
 ):
-    class CustomListSharePostResponse(BaseModel):
+    class CustomListSharePostResponse(CustomBaseModel):
         successes: int = 0
         failures: int = 0
 
-    class CustomListPostRequest(BaseModel):
+    class CustomListPostRequest(CustomBaseModel):
         name: str
         id: int | None = None
         entries: list[dict] = []
@@ -98,7 +97,7 @@ def custom_lists(self) -> dict | ProblemDetail | Response | None:
             return dict(custom_lists=custom_lists)
 
         if flask.request.method == "POST":
-            list_ = self.CustomListPostRequest.parse_obj(
+            list_ = self.CustomListPostRequest.model_validate(
                 parse_multi_dict(flask.request.form)
             )
             return self._create_or_update_list(
@@ -359,7 +358,7 @@ def custom_list(self, list_id: int) -> Response | dict | ProblemDetail | None:
             )
 
         elif flask.request.method == "POST":
-            list_ = self.CustomListPostRequest.parse_obj(
+            list_ = self.CustomListPostRequest.model_validate(
                 parse_multi_dict(flask.request.form)
             )
             return self._create_or_update_list(
@@ -458,7 +457,7 @@ def share_locally_POST(
         self.log.info(f"Done sharing customlist {customlist.name}")
         return self.CustomListSharePostResponse(
             successes=len(successes), failures=len(failures)
-        ).dict()
+        ).model_dump()
 
     def share_locally_DELETE(self, customlist: CustomList) -> ProblemDetail | Response:
         """Delete the shared status of a custom list

src/palace/manager/api/admin/controller/discovery_services.py

@@ -52,7 +52,7 @@ def set_up_default_registry(self) -> None:
         settings = OpdsRegistrationService.settings_class()(
             url=OpdsRegistrationService.DEFAULT_LIBRARY_REGISTRY_URL
         )
-        default_registry.settings_dict = settings.dict()
+        default_registry.settings_dict = settings.model_dump()
 
     def process_post(self) -> Response | ProblemDetail:
         try:
@@ -62,7 +62,7 @@ def process_post(self) -> Response | ProblemDetail:
             impl_cls = self.registry[protocol]
             settings_class = impl_cls.settings_class()
             validated_settings = ProcessFormData.get_settings(settings_class, form_data)
-            service.settings_dict = validated_settings.dict()
+            service.settings_dict = validated_settings.model_dump()
 
             # Make sure that the URL of the service is unique.
             self.check_url_unique(service, validated_settings.url)

src/palace/manager/api/admin/controller/individual_admin_settings.py

@@ -3,7 +3,7 @@
 import flask
 from flask import Response
 from flask_babel import lazy_gettext as _
-from pydantic import EmailStr, parse_obj_as
+from pydantic import EmailStr, TypeAdapter
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.orm import Session
 
@@ -299,7 +299,7 @@ def validate_form_fields(self, email):
             )
 
         try:
-            parse_obj_as(EmailStr, email)
+            TypeAdapter(EmailStr).validate_python(email)
         except ValueError:
             return INVALID_EMAIL.detailed(
                 _('"%(email)s" is not a valid email address.', email=email)

src/palace/manager/api/admin/controller/integration_settings.py

@@ -399,8 +399,8 @@ def process_updated_libraries(
         Update the settings for any IntegrationLibraryConfigurations that were updated or added.
         """
         for integration, settings in libraries:
-            validated_settings = settings_class(**settings)
-            integration.settings_dict = validated_settings.dict()
+            validated_settings = settings_class.model_validate(settings)
+            integration.settings_dict = validated_settings.model_dump()
 
     def process_libraries(
         self,

src/palace/manager/api/admin/controller/library_settings.py

@@ -137,7 +137,7 @@ def process_post(self) -> Response:
 
         library.name = name
         library.short_name = short_name
-        library.settings_dict = validated_settings.dict()
+        library.settings_dict = validated_settings.model_dump()
 
         # Validate and scale logo
         self.scale_and_store_logo(library, flask.request.files.get("logo"))

src/palace/manager/api/admin/controller/metadata_services.py

@@ -66,7 +66,7 @@ def process_post(self) -> Response | ProblemDetail:
             impl_cls = self.registry[protocol]
             settings_class = impl_cls.settings_class()
             validated_settings = ProcessFormData.get_settings(settings_class, form_data)
-            metadata_service.settings_dict = validated_settings.dict()
+            metadata_service.settings_dict = validated_settings.model_dump()
 
             # Update library settings
             if libraries_data and issubclass(

src/palace/manager/api/admin/controller/patron_auth_services.py

@@ -70,7 +70,7 @@ def process_post(self) -> Response | ProblemDetail:
             impl_cls = self.registry[protocol]
             settings_class = impl_cls.settings_class()
             validated_settings = ProcessFormData.get_settings(settings_class, form_data)
-            auth_service.settings_dict = validated_settings.dict()
+            auth_service.settings_dict = validated_settings.model_dump()
 
             # Update library settings
             if libraries_data:

src/palace/manager/api/admin/controller/view.py

@@ -80,7 +80,7 @@ def __call__(self, collection, book, path=None):
                 roles=roles,
                 admin_js=admin_js,
                 admin_css=admin_css,
-                feature_flags=AdminClientConfig.admin_feature_flags().json(
+                feature_flags=AdminClientConfig.admin_feature_flags().model_dump_json(
                     by_alias=True
                 ),
             )

src/palace/manager/api/admin/form_data.py

@@ -39,23 +39,26 @@ def get_settings_dict(
         a dictionary that we can use to update the settings.
         """
         return_data: dict[str, Any] = {}
-        for field in settings_class.__fields__.values():
-            if not isinstance(field.field_info, FormFieldInfo):
-                continue
-            form_item = field.field_info.form
+        for field_name, field_info in settings_class.model_fields.items():
+            assert isinstance(
+                field_info, FormFieldInfo
+            ), f"Expected FormFieldInfo, got {field_info.__class__}"
+            form_item = field_info.form
             if form_item.type == ConfigurationFormItemType.LIST:
-                return_data[field.name] = cls._process_list(field.name, form_data)
+                return_data[field_name] = cls._process_list(field_name, form_data)
             elif form_item.type == ConfigurationFormItemType.MENU:
-                return_data[field.name] = cls._process_menu(field.name, form_data)
+                return_data[field_name] = cls._process_menu(field_name, form_data)
             else:
-                data = form_data.get(field.name)
+                data = form_data.get(field_name)
                 if data is not None:
-                    return_data[field.name] = data
+                    return_data[field_name] = data
 
         return return_data
 
     @classmethod
     def get_settings(
         cls, settings_class: type[T], form_data: ImmutableMultiDict[str, str]
     ) -> T:
-        return settings_class(**cls.get_settings_dict(settings_class, form_data))
+        return settings_class.model_validate(
+            cls.get_settings_dict(settings_class, form_data)
+        )

src/palace/manager/api/admin/model/dashboard_statistics.py

@@ -20,13 +20,13 @@ def __getitem__(self, item: str) -> Any:
     def __add__(self, other: Self) -> Self:
         """Sum each property and return new instance."""
         return self.__class__(
-            **{field: self[field] + other[field] for field in self.__fields__.keys()}
+            **{field: self[field] + other[field] for field in self.model_fields.keys()}
         )
 
     @classmethod
     def zeroed(cls) -> Self:
         """An instance of this class with all values set to zero."""
-        return cls(**{field: 0 for field in cls.__fields__.keys()})
+        return cls(**{field: 0 for field in cls.model_fields.keys()})
 
 
 class PatronStatistics(StatisticsBaseModel):

src/palace/manager/api/admin/model/quicksight.py

@@ -1,6 +1,6 @@
 from uuid import UUID
 
-from pydantic import Field, validator
+from pydantic import Field, field_validator
 
 from palace.manager.util.flask_util import CustomBaseModel, str_comma_list_validator
 
@@ -10,7 +10,8 @@ class QuicksightGenerateUrlRequest(CustomBaseModel):
         description="The list of libraries to include in the dataset, an empty list is equivalent to all the libraries the user is allowed to access."
     )
 
-    @validator("library_uuids", pre=True)
+    @field_validator("library_uuids", mode="before")
+    @classmethod
     def parse_library_uuids(cls, value) -> list[str]:
         return str_comma_list_validator(value)
 

src/palace/manager/api/authentication/basic.py

@@ -8,10 +8,11 @@
 from typing import Any, TypeVar, cast
 
 from flask import url_for
-from pydantic import PositiveInt, validator
+from pydantic import PositiveInt, model_validator
 from sqlalchemy import or_
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound
+from typing_extensions import Self
 from werkzeug.datastructures import Authorization
 
 from palace.manager.api.admin.problem_details import (
@@ -246,27 +247,24 @@ class BasicAuthProviderLibrarySettings(AuthProviderLibrarySettings):
             "This value is not used if <em>Library Identifier Restriction Type</em> "
             "is set to 'No restriction'.",
         ),
-        alias="library_identifier_restriction",
     )
 
-    @validator("library_identifier_restriction_criteria")
-    def validate_restriction_criteria(
-        cls, v: str | None, values: dict[str, Any]
-    ) -> str | None:
+    @model_validator(mode="after")
+    def validate_restriction_criteria(self) -> Self:
         """Validate the library_identifier_restriction_criteria field."""
-        if not v:
-            return v
-
-        restriction_type = values.get("library_identifier_restriction_type")
-        if restriction_type == LibraryIdentifierRestriction.REGEX:
+        restriction_criteria = self.library_identifier_restriction_criteria
+        restriction_type = self.library_identifier_restriction_type
+        if (
+            restriction_criteria
+            and restriction_type == LibraryIdentifierRestriction.REGEX
+        ):
             try:
-                re.compile(v)
+                re.compile(restriction_criteria)
             except re.error:
                 raise SettingsValidationError(
                     problem_detail=INVALID_LIBRARY_IDENTIFIER_RESTRICTION_REGULAR_EXPRESSION
                 )
-
-        return v
+        return self
 
 
 SettingsType = TypeVar("SettingsType", bound=BasicAuthProviderSettings, covariant=True)

src/palace/manager/api/axis.py

@@ -18,7 +18,7 @@
 from flask_babel import lazy_gettext as _
 from lxml import etree
 from lxml.etree import _Element, _ElementTree
-from pydantic import validator
+from pydantic import field_validator
 from requests import Response as RequestsResponse
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import ObjectDeletedError, StaleDataError
@@ -155,7 +155,8 @@ class Axis360Settings(BaseCirculationApiSettings):
         ),
     )
 
-    @validator("url")
+    @field_validator("url")
+    @classmethod
     def _validate_url(cls, v: str) -> str:
         # Validate if the url provided is valid http or a valid nickname
         valid_names = list(Axis360APIConstants.SERVER_NICKNAMES.keys())

src/palace/manager/api/config.py

@@ -107,7 +107,7 @@ def estimate_language_collections_for_library(cls, library: Library) -> None:
         """
         holdings = library.estimated_holdings_by_language()
         large, small, tiny = cls.classify_holdings(holdings)
-        settings = LibrarySettings.construct(
+        settings = LibrarySettings.model_construct(
             large_collection_languages=large,
             small_collection_languages=small,
             tiny_collection_languages=tiny,