Filter users before iterating for notification

.devcontainer/devcontainer.json

@@ -12,7 +12,7 @@
       "settings": {
         "python.linting.pylintPath": "/workspace/.venv/bin/pylint"
       },
-      "extensions": ["ms-azuretools.vscode-docker"]
+      "extensions": ["ms-azuretools.vscode-docker", "ms-python.black-formatter"]
     }
   },
   "remoteUser": "root"

Dockerfile

@@ -5,9 +5,6 @@ ENV PYTHONUNBUFFERED=1
 
 WORKDIR /app
 
-
-FROM base as build
-
 ENV PIP_DEFAULT_TIMEOUT=100
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 ENV PIP_NO_CACHE_DIR=1
@@ -20,26 +17,31 @@ COPY pyproject.toml poetry.lock ./
 RUN poetry config virtualenvs.in-project true
 RUN poetry install --no-interaction --no-root
 
+
+FROM base as build
+
 COPY . .
 
 RUN poetry build
 
 
 FROM base AS execute
+ENV PATH=/app/.venv/bin:$PATH
 COPY --from=build /app/dist .
 COPY --from=build /app/.venv ./.venv
 RUN ./.venv/bin/pip install *.whl
 COPY ./config ./config
-ENV PATH=/app/.venv/bin:$PATH
 ENTRYPOINT ["python", "-m", "notifier"]
 
 
-FROM execute AS test
-RUN apt-get update && apt-get install -y default-mysql-client
+FROM base AS test
 ENV PATH=/app/.venv/bin:$PATH
+RUN apt-get update && apt-get install -y default-mysql-client
 COPY conftest.py .
+COPY config/ config/
 COPY tests/ tests/
-ENTRYPOINT ["pytest", "-vx"]
+COPY notifier/ notifier/
+ENTRYPOINT ["pytest", "-vvx"]
 
 
 FROM amazon/aws-lambda-python:3.8 AS execute_lambda

README.md

@@ -50,6 +50,13 @@ To start the notifier service locally:
 poetry run python3 -m notifier path_to_config_file path_to_auth_file
 ```
 
+Or with Docker:
+
+```shell
+docker build --target execute --tag notifier:execute .
+docker run --rm notifier:execute path_to_config_file path_to_auth_file
+```
+
 The config file that my notifier instance uses is `config/config.toml`. A
 sample auth file with dummy secrets, used for CI tests, can be found at
 `config/auth.ci.toml`.

docker-compose.test.yml

@@ -7,7 +7,7 @@ services:
       context: .
       dockerfile: ./Dockerfile
       target: test
-    command: "--notifier-config config/config.toml --notifier-auth config/auth.compose.toml"
+    command: "--notifier-config config/config.toml --notifier-auth config/auth.compose.toml ${PYTEST_ARGS-}"
     links:
       - database
     depends_on:

docs/database.md

@@ -33,7 +33,7 @@ Create the MySQL Server container:
 docker create --name notifier_mysql \
   -p 3306:3306 \
   -e MYSQL_ROOT_PASSWORD=root \
-  mysql:5.6.17
+  mysql:5.7.4
 ```
 
 For an ephemeral, development-only, containerised MySQL installation I'm

lambda_function.py

@@ -46,9 +46,9 @@ def lambda_handler(event, context):
 
     logger.debug("Lambda: starting main procedure")
     main(
-        read_local_config(local_config_path),
-        read_local_auth(local_auth_path),
-        [],
+        config=read_local_config(local_config_path),
+        auth=read_local_auth(local_auth_path),
+        execute_now=[],
         force_current_time=force_current_time,
     )
     logger.info("Lambda finished")

notifier/cli.py

@@ -12,11 +12,12 @@ def cli():
     """Run main procedure as a command-line tool."""
     args = read_command_line_arguments()
     main(
-        read_local_config(args.config),
-        read_local_auth(args.auth),
-        args.execute_now,
-        args.limit_wikis,
-        args.force_initial_search_timestamp,
+        config=read_local_config(args.config),
+        auth=read_local_auth(args.auth),
+        execute_now=args.execute_now,
+        limit_wikis=args.limit_wikis,
+        force_initial_search_timestamp=args.force_initial_search_timestamp,
+        dry_run=args.dry_run,
     )
 
 
@@ -49,5 +50,12 @@ def read_command_line_arguments():
         type=int,
         help="""The lower timestamp to use when searching for posts.""",
     )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=False,
+        help="""A dry run will skip remote config acquisition, new post
+        acquisition, and will not actually send any notifications.""",
+    )
 
     return parser.parse_args()

notifier/database/drivers/base.py

@@ -86,10 +86,20 @@ def get_user_configs(self, frequency: str) -> List[CachedUserConfig]:
         """
 
     @abstractmethod
-    def store_user_configs(self, user_configs: List[RawUserConfig]) -> None:
+    def get_notifiable_users(self, frequency: str) -> List[str]:
+        """Get the list of IDs for users subscribed to the given channel
+        frequency who have at least one notification waiting for them.
+        """
+
+    @abstractmethod
+    def store_user_configs(
+        self, user_configs: List[RawUserConfig], *, overwrite_existing=True
+    ) -> None:
         """Caches user notification configurations.
 
         :param user_configs: List of configurations for all users.
+        :param overwrite_existing: Whether to overwrite the existing set of
+        user configs. Default true. If false, will append.
         """
 
     @abstractmethod

notifier/database/drivers/mysql.py

@@ -319,10 +319,23 @@ def get_user_configs(self, frequency: str) -> List[CachedUserConfig]:
             ]
         return user_configs
 
-    def store_user_configs(self, user_configs: List[RawUserConfig]) -> None:
+    def get_notifiable_users(self, frequency: str) -> List[str]:
+        user_ids = [
+            cast(str, row["user_id"])
+            for row in self.execute_named(
+                "get_user_ids_for_frequency_with_notifications",
+                {"frequency": frequency},
+            ).fetchall()
+        ]
+        return user_ids
+
+    def store_user_configs(
+        self, user_configs: List[RawUserConfig], *, overwrite_existing=True
+    ) -> None:
         with self.transaction() as cursor:
-            # Overwrite all current configs
-            self.execute_named("delete_user_configs", None, cursor)
+            if overwrite_existing:
+                # Overwrite all current configs
+                self.execute_named("delete_user_configs", None, cursor)
             for user_config in user_configs:
                 self.execute_named(
                     "store_user_config",

notifier/database/queries/get_posts_in_subscribed_threads.sql

@@ -16,15 +16,15 @@ SELECT
 FROM
   post
   INNER JOIN
-  thread ON post.thread_id = thread.id
+  thread ON thread.id = post.thread_id
   INNER JOIN
-  wiki ON thread.wiki_id = wiki.id
+  wiki ON wiki.id = thread.wiki_id
   INNER JOIN
   thread_first_post ON thread_first_post.thread_id = thread.id
   INNER JOIN
-  post AS first_post ON thread_first_post.post_id = first_post.id
+  post AS first_post_in_thread ON first_post_in_thread.id = thread_first_post.post_id
   LEFT JOIN
-  category ON thread.category_id = category.id
+  category ON category.id = thread.category_id
   LEFT JOIN
   manual_sub AS thread_sub ON (
     thread_sub.user_id = %(user_id)s
@@ -54,7 +54,7 @@ WHERE
     thread_sub.sub = 1
 
     -- Get posts in threads started by the user
-    OR first_post.user_id = %(user_id)s
+    OR first_post_in_thread.user_id = %(user_id)s
   )
 
   -- Remove posts in threads unsubscribed from

notifier/database/queries/get_replies_to_subscribed_posts.sql

@@ -20,13 +20,13 @@ SELECT
 FROM
   post
   INNER JOIN
-  thread ON post.thread_id = thread.id
+  thread ON thread.id = post.thread_id
   INNER JOIN
-  wiki ON thread.wiki_id = wiki.id
+  wiki ON wiki.id = thread.wiki_id
   INNER JOIN
-  post AS parent_post ON post.parent_post_id = parent_post.id
+  post AS parent_post ON parent_post.id = post.parent_post_id
   LEFT JOIN
-  category ON thread.category_id = category.id
+  category ON category.id = thread.category_id
   LEFT JOIN
   manual_sub AS post_sub ON (
     post_sub.user_id = %(user_id)s

notifier/database/queries/get_user_ids_for_frequency_with_notifications.sql

@@ -0,0 +1,72 @@
+SELECT
+  user_config.user_id AS user_id
+FROM
+  user_config
+  LEFT JOIN
+  user_last_notified ON user_config.user_id = user_last_notified.user_id
+WHERE
+  -- Only users on the given channel
+  user_config.frequency = %(frequency)s
+
+  -- Only users with a notification waiting for them
+  AND EXISTS (
+    SELECT NULL FROM
+      post
+      INNER JOIN
+      thread ON thread.id = post.thread_id
+      LEFT JOIN
+      post AS parent_post ON parent_post.id = post.parent_post_id
+      INNER JOIN
+      thread_first_post ON thread_first_post.thread_id = thread.id
+      INNER JOIN
+      post AS first_post_in_thread ON first_post_in_thread.id = thread_first_post.post_id
+    WHERE
+      -- Remove deleted posts
+      post.is_deleted = 0
+
+      -- Remove posts made by the user
+      AND post.user_id <> user_config.user_id
+
+      -- Only posts posted since the user was last notified
+      AND post.posted_timestamp > user_last_notified.notified_timestamp
+
+      -- Remove deleted threads
+      AND thread.is_deleted = 0
+
+      -- Only posts matching thread or post subscription criteria
+      AND (
+        -- Posts in threads started by the user
+        first_post_in_thread.user_id = user_config.user_id
+
+        -- Replies to posts made by the user
+        OR parent_post.user_id = user_config.user_id
+
+        -- Posts in threads subscribed to and replies to posts subscribed to
+        OR EXISTS (
+          SELECT NULL FROM
+            manual_sub
+          WHERE
+            manual_sub.user_id = user_config.user_id
+            AND manual_sub.thread_id = thread.id
+            AND (
+              manual_sub.post_id IS NULL  -- Threads
+              OR manual_sub.post_id = parent_post.id  -- Post replies
+            )
+            AND manual_sub.sub = 1
+        )
+      )
+
+      -- Remove posts/replies in/to threads/posts unsubscribed from
+      AND NOT EXISTS (
+        SELECT NULL FROM
+          manual_sub
+        WHERE
+          manual_sub.user_id = user_config.user_id
+          AND manual_sub.thread_id = thread.id
+          AND (
+            manual_sub.post_id IS NULL  -- Threads
+            OR manual_sub.post_id = parent_post.id  -- Post replies
+          )
+          AND manual_sub.sub = -1
+      )
+  )

notifier/emailer.py

@@ -1,12 +1,26 @@
+import logging
+from typing import cast
+
 import yagmail
 
+logger = logging.getLogger(__name__)
+
 
 class Emailer:  # pylint: disable=too-few-public-methods
     """Responsible for sending emails."""
 
-    def __init__(self, gmail_username: str, gmail_password: str):
-        self.yag = yagmail.SMTP(gmail_username, gmail_password)
+    def __init__(
+        self, gmail_username: str, gmail_password: str, *, dry_run=False
+    ):
+        self.dry_run = dry_run
+        if dry_run:
+            self.yag = cast(yagmail.SMTP, object())
+        else:
+            self.yag = yagmail.SMTP(gmail_username, gmail_password)
 
     def send(self, address: str, subject: str, body: str) -> None:
         """Send an email to an address."""
+        if self.dry_run:
+            logger.warn("Dry run: email send was rejected")
+            return
         self.yag.send(address, subject, body, prettify_html=False)