Split post table into notifiable posts and context

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:0-3.8-bullseye
+FROM mcr.microsoft.com/devcontainers/python:0-3.8
 
 RUN apt-get install libxml2-dev libxslt-dev
 

.devcontainer/devcontainer.json

@@ -1,6 +1,8 @@
 {
   "name": "Python 3",
-  "build": { "dockerfile": "Dockerfile" },
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
   "features": {
     "ghcr.io/devcontainers-contrib/features/poetry:2": {
       "version": "latest"
@@ -12,27 +14,34 @@
       "settings": {
         "python.defaultInterpreterPath": "./.venv/bin/python",
         "python.linting.pylintPath": "./.venv/bin/pylint",
-        "python.analysis.extraPaths": [".venv/bin"],
-        "python.linting.mypyEnabled": true,
-        "python.linting.enabled": true,
-        "python.linting.mypyArgs": ["--strict", "--show-column-numbers"],
-        "python.linting.mypyPath": "./.venv/bin/mypy",
+        "python.analysis.extraPaths": [
+          ".venv/bin"
+        ],
         "python.formatting.blackPath": "./.venv/bin/black",
         "black-formatter.path": [
           "./.venv/bin/black"
         ],
-        "editor.defaultFormatter": "ms-python.black-formatter",
-        "mypy.runUsingActiveInterpreter": true
+        "mypy.runUsingActiveInterpreter": true,
+        "[python]": {
+          "editor.defaultFormatter": "ms-python.black-formatter",
+        },
+        "python.formatting.provider": "none",
+        "editor.formatOnSave": true,
+        "mypy-type-checker.args": [
+          "--strict"
+        ],
+        "mypy-type-checker.importStrategy": "fromEnvironment"
       },
       "extensions": [
         "ms-azuretools.vscode-docker",
         "ms-python.black-formatter",
-        "matangover.mypy",
         "ms-python.vscode-pylance",
         "ms-python.python",
-        "ms-python.pylint"
+        "ms-python.pylint",
+        "ms-python.mypy-type-checker",
+        "tamasfe.even-better-toml"
       ]
     }
   },
   "remoteUser": "root"
-}
+}

notifier/config/user.py

@@ -1,6 +1,7 @@
 import logging
+from operator import itemgetter
 import re
-from typing import List, Optional, Tuple, Union, cast
+from typing import List, Optional, Tuple, TypedDict, Union, cast
 
 import tomlkit
 from tomlkit.exceptions import TOMLKitError
@@ -154,7 +155,9 @@ def parse_subscriptions(
             # (Empty rows are permitted so this is valid)
             continue
         try:
-            thread_id, post_id = parse_thread_url(url)
+            thread_id, post_id = itemgetter("thread_id", "post_id")(
+                parse_thread_url(url)
+            )
         except ValueError:
             # The URL parser is extremely forgiving, so this should not
             # happen; if it somehow does, discard it
@@ -165,11 +168,18 @@ def parse_subscriptions(
     return subscriptions
 
 
-def parse_thread_url(url: str) -> Tuple[str, Union[str, None]]:
+ThreadPostIds = TypedDict(
+    "ThreadPostIds", {"thread_id": str, "post_id": Union[str, None]}
+)
+
+
+def parse_thread_url(
+    url: str,
+) -> ThreadPostIds:
     """Parses a URL to a thread ID and optionally a post ID."""
     pattern = re.compile(r"(t-[0-9]+)(?:.*#(post-[0-9]+))?")
     match = pattern.search(url)
     if not match:
         raise ValueError("Thread URL does not match expected pattern")
     thread_id, post_id = match.groups()
-    return thread_id, post_id
+    return {"thread_id": thread_id, "post_id": post_id}

notifier/database/drivers/base.py

@@ -1,16 +1,17 @@
 from abc import ABC, abstractmethod
-from typing import Any, Iterable, List, Tuple
+from typing import Any, List, Tuple
 
 from notifier.types import (
     ActivationLogDump,
     CachedUserConfig,
     ChannelLogDump,
     LogDump,
-    NewPostsInfo,
-    RawPost,
+    NotifiablePost,
+    PostInfo,
+    PostMeta,
     RawUserConfig,
     SupportedWikiConfig,
-    ThreadInfo,
+    Context,
 )
 
 
@@ -37,34 +38,15 @@ def create_tables(self) -> None:
         """Initial setup for the database."""
 
     @abstractmethod
-    def find_new_posts(self, post_ids: Iterable[str]) -> List[str]:
-        """From a list of post IDs, return those that are not already
-        present in the cache."""
+    def get_latest_post_timestamp(self) -> int:
+        """Returns the timestamp of the latest stored post."""
 
     @abstractmethod
-    def find_new_threads(self, thread_ids: Iterable[str]) -> List[str]:
-        """From a list of thread IDs, return those that are not already
-        present in the cache."""
-
-    @abstractmethod
-    def mark_thread_as_deleted(self, thread_id: str) -> None:
-        """Marks a thread as deleted, preventing its posts from appearing
-        in notifications."""
-
-    @abstractmethod
-    def mark_post_as_deleted(self, post_id: str) -> None:
-        """Marks a post as deleted, preventing it from appearing in
-        notifications. Also mark its children as deleted, recursively."""
-
-    @abstractmethod
-    def get_new_posts_for_user(
+    def get_notifiable_posts_for_user(
         self, user_id: str, timestamp_range: Tuple[int, int]
-    ) -> NewPostsInfo:
+    ) -> List[PostInfo]:
         """Get new posts for the users with the given ID made during the
-        given time range.
-
-        Returns a dict containing the thread posts and the post replies.
-        """
+        given time range."""
 
     @abstractmethod
     def get_user_configs(self, frequency: str) -> List[CachedUserConfig]:
@@ -80,13 +62,20 @@ def count_user_configs(self) -> int:
         """Count the number of subscribed users."""
 
     @abstractmethod
-    def get_notifiable_users(
-        self, frequency: str, post_lower_timestamp_limit: int
-    ) -> List[str]:
+    def get_notifiable_users(self, frequency: str) -> List[str]:
         """Get the list of IDs for users subscribed to the given channel
         frequency who have at least one notification waiting for them.
         """
 
+    @abstractmethod
+    def get_posts_to_check_for_deletion(
+        self, timestamp: int
+    ) -> List[PostMeta]:
+        """Get a list of posts to check for having potentially been deleted.
+
+        Timestamp is assumed to be time to check relative to.
+        """
+
     @abstractmethod
     def store_user_configs(
         self,
@@ -117,30 +106,42 @@ def store_user_last_notified(
     def get_supported_wikis(self) -> List[SupportedWikiConfig]:
         """Get a list of supported wikis."""
 
-    @abstractmethod
-    def count_supported_wikis(self) -> int:
-        """Count the number of supported wikis."""
-
     @abstractmethod
     def store_supported_wikis(self, wikis: List[SupportedWikiConfig]) -> None:
         """Stores a set of supported wikis in the database, overwriting any
         that are already present."""
 
     @abstractmethod
-    def store_thread(self, thread: ThreadInfo) -> None:
-        """Store a thread.
+    def store_post(self, post: NotifiablePost) -> None:
+        """Store a post."""
 
-        Doesn't matter if the thread or category is already known or not.
-        """
+    @abstractmethod
+    def store_context_forum_category(
+        self, context_forum_category: Context.ForumCategory
+    ) -> None:
+        """Store a forum category for context."""
 
     @abstractmethod
-    def store_thread_first_post(self, thread_id: str, post_id: str) -> None:
-        """Store the relationship between a thread and the first post it
-        contains."""
+    def store_context_thread(self, context_thread: Context.Thread) -> None:
+        """Store a thread for context."""
 
     @abstractmethod
-    def store_post(self, post: RawPost) -> None:
-        """Store a post."""
+    def store_context_parent_post(
+        self, context_parent_post: Context.ParentPost
+    ) -> None:
+        """Store a parent post for context."""
+
+    @abstractmethod
+    def delete_post(self, post_id: str) -> None:
+        """Delete a post."""
+
+    @abstractmethod
+    def delete_non_notifiable_posts(self) -> None:
+        """Delete posts that will not emit notifications."""
+
+    @abstractmethod
+    def delete_context_thread(self, thread_id: str) -> None:
+        """Delete posts with the given thread context."""
 
     @abstractmethod
     def store_channel_log_dump(self, log: ChannelLogDump) -> None: