Unified search query syntax using the full-text search capabilities of the underlying DB

changelog.d/11635.feature

@@ -0,0 +1 @@
+Allow use of postgres and sqllite full-text search operators in search queries.

synapse/storage/databases/main/search.py

@@ -15,7 +15,7 @@
 import logging
 import re
 from collections import namedtuple
-from typing import TYPE_CHECKING, Collection, Iterable, List, Optional, Set
+from typing import TYPE_CHECKING, Collection, Iterable, List, Optional, Set, Tuple
 
 from synapse.api.errors import SynapseError
 from synapse.events import EventBase
@@ -389,8 +389,6 @@ async def search_msgs(self, room_ids, search_term, keys):
         """
         clauses = []
 
-        search_query = _parse_query(self.database_engine, search_term)
-
         args = []
 
         # Make sure we don't explode because the person is in too many rooms.
@@ -412,20 +410,25 @@ async def search_msgs(self, room_ids, search_term, keys):
         count_clauses = clauses
 
         if isinstance(self.database_engine, PostgresEngine):
+            search_query, tsquery_func = _parse_query_for_pgsql(
+                search_term, self.database_engine
+            )
             sql = (
-                "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) AS rank,"
+                f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank,"
                 " room_id, event_id"
                 " FROM event_search"
-                " WHERE vector @@ to_tsquery('english', ?)"
+                f" WHERE vector @@  {tsquery_func}('english', ?)"
             )
             args = [search_query, search_query] + args
 
             count_sql = (
                 "SELECT room_id, count(*) as count FROM event_search"
-                " WHERE vector @@ to_tsquery('english', ?)"
+                f" WHERE vector @@ {tsquery_func}('english', ?)"
             )
             count_args = [search_query] + count_args
         elif isinstance(self.database_engine, Sqlite3Engine):
+            search_query = _parse_query_for_sqlite(search_term)
+
             sql = (
                 "SELECT rank(matchinfo(event_search)) as rank, room_id, event_id"
                 " FROM event_search"
@@ -437,7 +440,7 @@ async def search_msgs(self, room_ids, search_term, keys):
                 "SELECT room_id, count(*) as count FROM event_search"
                 " WHERE value MATCH ?"
             )
-            count_args = [search_term] + count_args
+            count_args = [search_query] + count_args
         else:
             # This should be unreachable.
             raise Exception("Unrecognized database engine")
@@ -469,7 +472,9 @@ async def search_msgs(self, room_ids, search_term, keys):
 
         highlights = None
         if isinstance(self.database_engine, PostgresEngine):
-            highlights = await self._find_highlights_in_postgres(search_query, events)
+            highlights = await self._find_highlights_in_postgres(
+                search_query, events, tsquery_func
+            )
 
         count_sql += " GROUP BY room_id"
 
@@ -511,8 +516,6 @@ async def search_rooms(
         """
         clauses = []
 
-        search_query = _parse_query(self.database_engine, search_term)
-
         args = []
 
         # Make sure we don't explode because the person is in too many rooms.
@@ -550,20 +553,24 @@ async def search_rooms(
             args.extend([origin_server_ts, origin_server_ts, stream])
 
         if isinstance(self.database_engine, PostgresEngine):
+            search_query, tsquery_func = _parse_query_for_pgsql(
+                search_term, self.database_engine
+            )
             sql = (
-                "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
+                f"SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank,"
                 " origin_server_ts, stream_ordering, room_id, event_id"
                 " FROM event_search"
-                " WHERE vector @@ to_tsquery('english', ?) AND "
+                f" WHERE vector @@ {tsquery_func}('english', ?) AND "
             )
             args = [search_query, search_query] + args
 
             count_sql = (
                 "SELECT room_id, count(*) as count FROM event_search"
-                " WHERE vector @@ to_tsquery('english', ?) AND "
+                f" WHERE vector @@ {tsquery_func}('english', ?) AND "
             )
             count_args = [search_query] + count_args
         elif isinstance(self.database_engine, Sqlite3Engine):
+
             # We use CROSS JOIN here to ensure we use the right indexes.
             # https://sqlite.org/optoverview.html#crossjoin
             #
@@ -582,13 +589,14 @@ async def search_rooms(
                 " CROSS JOIN events USING (event_id)"
                 " WHERE "
             )
+            search_query = _parse_query_for_sqlite(search_term)
             args = [search_query] + args
 
             count_sql = (
                 "SELECT room_id, count(*) as count FROM event_search"
                 " WHERE value MATCH ? AND "
             )
-            count_args = [search_term] + count_args
+            count_args = [search_query] + count_args
         else:
             # This should be unreachable.
             raise Exception("Unrecognized database engine")
@@ -627,7 +635,9 @@ async def search_rooms(
 
         highlights = None
         if isinstance(self.database_engine, PostgresEngine):
-            highlights = await self._find_highlights_in_postgres(search_query, events)
+            highlights = await self._find_highlights_in_postgres(
+                search_query, events, tsquery_func
+            )
 
         count_sql += " GROUP BY room_id"
 
@@ -653,7 +663,7 @@ async def search_rooms(
         }
 
     async def _find_highlights_in_postgres(
-        self, search_query: str, events: List[EventBase]
+        self, search_query: str, events: List[EventBase], tsquery_func: str
     ) -> Set[str]:
         """Given a list of events and a search term, return a list of words
         that match from the content of the event.
@@ -664,6 +674,7 @@ async def _find_highlights_in_postgres(
         Args:
             search_query
             events: A list of events
+            tsquery_func: The tsquery_* function to use when making queries
 
         Returns:
             A set of strings.
@@ -696,7 +707,7 @@ def f(txn):
                 while stop_sel in value:
                     stop_sel += ">"
 
-                query = "SELECT ts_headline(?, to_tsquery('english', ?), %s)" % (
+                query = f"SELECT ts_headline(?, {tsquery_func}('english', ?), %s)" % (
                     _to_postgres_options(
                         {
                             "StartSel": start_sel,
@@ -727,20 +738,25 @@ def _to_postgres_options(options_dict):
     return "'%s'" % (",".join("%s=%s" % (k, v) for k, v in options_dict.items()),)
 
 
-def _parse_query(database_engine, search_term):
+def _parse_query_for_sqlite(search_term: str) -> str:
     """Takes a plain unicode string from the user and converts it into a form
-    that can be passed to database.
-    We use this so that we can add prefix matching, which isn't something
-    that is supported by default.
+    that can be passed to sqllite's matchinfo().
     """
+    return search_term
+
 
-    # Pull out the individual words, discarding any non-word characters.
-    results = re.findall(r"([\w\-]+)", search_term, re.UNICODE)
+def _parse_query_for_pgsql(search_term: str, engine: PostgresEngine) -> Tuple[str, str]:
+    """Selects a tsquery_* func to use and transforms the search_term into syntax appropriate for it.
+
+    Args:
+        search_term: A user supplied search query.
+        engine: The database engine.
+
+    Returns:
+        A tuple of (parsed search_term, tsquery func to use).
+    """
 
-    if isinstance(database_engine, PostgresEngine):
-        return " & ".join(result + ":*" for result in results)
-    elif isinstance(database_engine, Sqlite3Engine):
-        return " & ".join(result + "*" for result in results)
+    if engine.supports_websearch_to_tsquery:
+        return search_term, "websearch_to_tsquery"
     else:
-        # This should be unreachable.
-        raise Exception("Unrecognized database engine")
+        return search_term, "plainto_tsquery"

synapse/storage/engines/postgres.py

@@ -138,6 +138,10 @@ def supports_returning(self) -> bool:
         """Do we support the `RETURNING` clause in insert/update/delete?"""
         return True
 
+    @property
+    def supports_websearch_to_tsquery(self) -> bool:
+        return int(self.server_version.split(".")[0]) >= 11
+
     def is_deadlock(self, error):
         if isinstance(error, self.module.DatabaseError):
             # https://www.postgresql.org/docs/current/static/errcodes-appendix.html

tests/storage/test_room_search.py

@@ -12,9 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Tuple
+from unittest.case import SkipTest
+from unittest.mock import PropertyMock, patch
+
 import synapse.rest.admin
 from synapse.rest.client import login, room
+from synapse.storage.databases.main import DataStore
 from synapse.storage.engines import PostgresEngine
+from synapse.storage.engines.sqlite import Sqlite3Engine
 
 from tests.unittest import HomeserverTestCase
 
@@ -72,3 +78,128 @@ def test_null_byte(self):
         )
         if isinstance(store.database_engine, PostgresEngine):
             self.assertIn("alice", result.get("highlights"))
+
+
+class MessageSearchTest(HomeserverTestCase):
+    servlets = [
+        synapse.rest.admin.register_servlets_for_client_rest_resource,
+        login.register_servlets,
+        room.register_servlets,
+    ]
+
+    PHRASE = "the quick brown fox jumps over the lazy dog"
+
+    def setUp(self):
+        super().setUp()
+
+        # Register a user and create a room, create some messages
+        self.register_user("alice", "password")
+        self.access_token = self.login("alice", "password")
+        self.room_id = self.helper.create_room_as("alice", tok=self.access_token)
+
+        # Send the phrase as a message and check it was created
+        response = self.helper.send(self.room_id, self.PHRASE, tok=self.access_token)
+        self.assertIn("event_id", response)
+
+    def _check_test_cases(
+        self, store: DataStore, cases: list[Tuple[str, bool]]
+    ) -> None:
+        # Run all the test cases versus search_msgs
+        for query, has_results in cases:
+            result = self.get_success(
+                store.search_msgs([self.room_id], query, ["content.body"])
+            )
+            self.assertEquals(result["count"], 1 if has_results else 0, query)
+            self.assertEquals(len(result["results"]), 1 if has_results else 0, query)
+
+        # Run them again versus search_rooms
+        for query, has_results in cases:
+            result = self.get_success(
+                store.search_rooms([self.room_id], query, ["content.body"], 10)
+            )
+            self.assertEquals(result["count"], 1 if has_results else 0, query)
+            self.assertEquals(len(result["results"]), 1 if has_results else 0, query)
+
+    def test_postgres_web_search_for_phrase(self):
+        """
+        Test searching for phrases using typical web search syntax, as per postgres' websearch_to_tsquery.
+        This test is skipped unless the postgres instance supports websearch_to_tsquery.
+        """
+
+        store = self.hs.get_datastore()
+        if not isinstance(store.database_engine, PostgresEngine):
+            raise SkipTest("Test only applies when postgres is used as the database")
+
+        if not store.database_engine.supports_websearch_to_tsquery:
+            raise SkipTest(
+                "Test only applies when postgres supporting websearch_to_tsquery is used as the database"
+            )
+
+        cases = [
+            ("brown", True),
+            ("quick brown", True),
+            ("brown quick", True),
+            ('"brown quick"', False),
+            ('"jumps over"', True),
+            ('"quick fox"', False),
+            ("furphy OR fox", True),
+            ("nope OR doublenope", False),
+            ("-fox", False),
+            ("-nope", True),
+        ]
+
+        self._check_test_cases(store, cases)
+
+    def test_postgres_non_web_search_for_phrase(self):
+        """
+        Test postgres searching for phrases without using web search, which is used when websearch_to_tsquery isn't
+        supported by the current postgres version.
+        """
+
+        store = self.hs.get_datastore()
+        if not isinstance(store.database_engine, PostgresEngine):
+            raise SkipTest("Test only applies when postgres is used as the database")
+
+        cases = [
+            ("nope", False),
+            ("brown", True),
+            ("quick brown", True),
+            ("brown quick", True),
+            ("brown nope", False),
+            (
+                "furphy OR fox",
+                False,
+            ),  # syntax not supported, OR will be ignored as it'll be between &
+            ('"jumps over"', True),  # syntax not supported, quotes are ignored
+            ("-nope", False),  # syntax not supported, - will be ignored
+        ]
+
+        # Patch supports_websearch_to_tsquery to always return False to ensure we're testing the plainto_tsquery path.
+        with patch(
+            "synapse.storage.engines.postgres.PostgresEngine.supports_websearch_to_tsquery",
+            new_callable=PropertyMock,
+        ) as supports_websearch_to_tsquery:
+            supports_websearch_to_tsquery.return_value = False
+            self._check_test_cases(store, cases)
+
+    def test_sqlite_search(self):
+        """
+        Test sqlite searching for phrases.
+        """
+        store = self.hs.get_datastore()
+        if not isinstance(store.database_engine, Sqlite3Engine):
+            raise SkipTest("Test only applies when sqlite is used as the database")
+
+        cases = [
+            ("nope", False),
+            ("brown", True),
+            ("quick brown", True),
+            ("brown quick", True),
+            ("brown nope", False),
+            ("furphy OR fox", True),  # sqllite supports OR
+            ('"jumps over"', True),
+            ('"quick fox"', False),  # syntax supports quotes
+            ("fox NOT nope", True),  # sqllite supports NOT
+        ]
+
+        self._check_test_cases(store, cases)