Only show world_readable rooms in the room directory

server/lib/matrix-utils/fetch-accessible-rooms.js

@@ -0,0 +1,191 @@
+'use strict';
+
+const assert = require('assert');
+
+const urlJoin = require('url-join');
+const { DIRECTION } = require('matrix-public-archive-shared/lib/reference-values');
+const { fetchEndpointAsJson } = require('../fetch-endpoint');
+const { traceFunction } = require('../../tracing/trace-utilities');
+
+const config = require('../config');
+const matrixServerUrl = config.get('matrixServerUrl');
+assert(matrixServerUrl);
+
+// The number of requests we should make to try to fill the limit before bailing out
+const NUM_MAX_REQUESTS = 10;
+
+async function requestPublicRooms(
+  accessToken,
+  { server, searchTerm, paginationToken, limit, abortSignal } = {}
+) {
+  let qs = new URLSearchParams();
+  if (server) {
+    qs.append('server', server);
+  }
+
+  const publicRoomsEndpoint = urlJoin(
+    matrixServerUrl,
+    `_matrix/client/v3/publicRooms?${qs.toString()}`
+  );
+
+  const { data: publicRoomsRes } = await fetchEndpointAsJson(publicRoomsEndpoint, {
+    method: 'POST',
+    body: {
+      include_all_networks: true,
+      filter: {
+        generic_search_term: searchTerm,
+      },
+      since: paginationToken,
+      limit,
+    },
+    accessToken,
+    abortSignal,
+  });
+
+  return publicRoomsRes;
+}
+
+// eslint-disable-next-line complexity, max-statements
+async function fetchAccessibleRooms(
+  accessToken,
+  {
+    server,
+    searchTerm,
+    // Direction is baked into the pagination token but we're unable to decipher it from
+    // the opaque token, we also have to pass it in explicitly.
+    paginationToken,
+    direction = DIRECTION.forward,
+    limit,
+    abortSignal,
+  } = {}
+) {
+  assert(accessToken);
+  assert([DIRECTION.forward, DIRECTION.backward].includes(direction), 'direction must be [f|b]');
+
+  // Based off of the matrix.org room directory, only 42% of rooms are world_readable,
+  // which means our best bet to fill up the results to the limit is to request at least
+  // 2.4 times as many. I've doubled and rounded it up to 5 times as many so we can have
+  // less round-trips.
+  const bulkPaginationLimit = Math.ceil(5 * limit);
+
+  let accessibleRooms = [];
+
+  let firstResponse;
+  let lastResponse;
+
+  let loopToken = paginationToken;
+  let lastLoopToken;
+  let continuationIndex;
+  let currentRequestCount = 0;
+  while (
+    // Stop if we have reached the limit of rooms we want to fetch
+    accessibleRooms.length < limit &&
+    // And bail if we're already gone through a bunch of pages to try to fill the limit
+    currentRequestCount < NUM_MAX_REQUESTS &&
+    // And bail if we've reached the end of the pagination
+    // Always do the first request
+    (currentRequestCount === 0 ||
+      // If we have a next token, we can do another request
+      (currentRequestCount > 0 && loopToken))
+  ) {
+    const publicRoomsRes = await requestPublicRooms(accessToken, {
+      server,
+      searchTerm,
+      paginationToken: loopToken,
+      limit: bulkPaginationLimit,
+      abortSignal,
+    });
+    lastLoopToken = loopToken;
+    lastResponse = publicRoomsRes;
+
+    if (currentRequestCount === 0) {
+      firstResponse = publicRoomsRes;
+    }
+
+    // Get the token ready for the next loop
+    loopToken =
+      direction === DIRECTION.forward ? publicRoomsRes.next_batch : publicRoomsRes.prev_batch;
+
+    const fetchedRooms = publicRoomsRes.chunk;
+    const fetchedRoomsInDirection =
+      direction === DIRECTION.forward ? fetchedRooms : fetchedRooms.reverse();
+
+    // We only want to see world_readable rooms in the archive
+    let index = 0;
+    for (let room of fetchedRoomsInDirection) {
+      if (room.world_readable) {
+        if (direction === DIRECTION.forward) {
+          accessibleRooms.push(room);
+        } else if (direction === DIRECTION.backward) {
+          accessibleRooms.unshift(room);
+        } else {
+          throw new Error(`Invalid direction: ${direction}`);
+        }
+      }
+
+      if (accessibleRooms.length === limit && !continuationIndex) {
+        continuationIndex = index;
+      }
+
+      // Stop after we've reached the limit
+      if (accessibleRooms.length >= limit) {
+        break;
+      }
+
+      index += 1;
+    }
+
+    currentRequestCount += 1;
+  }
+
+  // Back-track to get the perfect continuation point and show exactly the limit of
+  // rooms in the grid.
+  //
+  // Alternatively, we could just not worry about and show more than the limit of rooms
+  //
+  // XXX: Since the room directory order is not stable, this is slightly flawed as the
+  // results could have shifted slightly from when we made the last request to now but
+  // we assume it's good enough.
+  let nextPaginationToken;
+  let prevPaginationToken;
+  if (continuationIndex) {
+    const publicRoomsRes = await requestPublicRooms(accessToken, {
+      server,
+      searchTerm,
+      // Start from the last request
+      paginationToken: lastLoopToken,
+      // Then only go out as far out as the continuation index (the point when we filled
+      // the limit)
+      limit: continuationIndex + 1,
+      abortSignal,
+    });
+
+    if (direction === DIRECTION.forward) {
+      prevPaginationToken = firstResponse.prev_batch;
+      nextPaginationToken = publicRoomsRes.next_batch;
+    } else if (direction === DIRECTION.backward) {
+      prevPaginationToken = publicRoomsRes.prev_batch;
+      nextPaginationToken = firstResponse.next_batch;
+    } else {
+      throw new Error(`Invalid direction: ${direction}`);
+    }
+  } else {
+    if (direction === DIRECTION.forward) {
+      prevPaginationToken = firstResponse.prev_batch;
+      nextPaginationToken = lastResponse.next_batch;
+    } else if (direction === DIRECTION.backward) {
+      prevPaginationToken = lastResponse.prev_batch;
+      nextPaginationToken = firstResponse.next_batch;
+    } else {
+      throw new Error(`Invalid direction: ${direction}`);
+    }
+  }
+
+  return {
+    rooms: accessibleRooms,
+    prevPaginationToken,
+    nextPaginationToken,
+  };
+}
+
+module.exports = traceFunction(fetchAccessibleRooms);

server/routes/room-directory-routes.js

@@ -6,10 +6,11 @@ const urlJoin = require('url-join');
 const express = require('express');
 const asyncHandler = require('../lib/express-async-handler');
 
+const { DIRECTION } = require('matrix-public-archive-shared/lib/reference-values');
 const RouteTimeoutAbortError = require('../lib/errors/route-timeout-abort-error');
 const UserClosedConnectionAbortError = require('../lib/errors/user-closed-connection-abort-error');
 const identifyRoute = require('../middleware/identify-route-middleware');
-const fetchPublicRooms = require('../lib/matrix-utils/fetch-public-rooms');
+const fetchAccessibleRooms = require('../lib/matrix-utils/fetch-accessible-rooms');
 const renderHydrogenVmRenderScriptToPageHtml = require('../hydrogen-render/render-hydrogen-vm-render-script-to-page-html');
 const setHeadersToPreloadAssets = require('../lib/set-headers-to-preload-assets');
 
@@ -33,9 +34,19 @@ router.get(
   '/',
   identifyRoute('app-room-directory-index'),
   asyncHandler(async function (req, res) {
-    const paginationToken = req.query.page;
     const searchTerm = req.query.search;
     const homeserver = req.query.homeserver;
+    const paginationToken = req.query.page;
+    const direction = req.query.dir;
+
+    // You must provide both `paginationToken` and `direction` if either is defined
+    if (paginationToken || direction) {
+      assert(
+        [DIRECTION.forward, DIRECTION.backward].includes(direction),
+        '?dir query parameter must be [f|b]'
+      );
+      assert(paginationToken, '?page query parameter must be defined if ?dir is defined');
+    }
 
     // It would be good to grab more rooms than we display in case we need
     // to filter any out but then the pagination tokens with the homeserver
@@ -48,12 +59,13 @@ router.get(
     let prevPaginationToken;
     let roomFetchError;
     try {
-      ({ rooms, nextPaginationToken, prevPaginationToken } = await fetchPublicRooms(
+      ({ rooms, nextPaginationToken, prevPaginationToken } = await fetchAccessibleRooms(
         matrixAccessToken,
         {
           server: homeserver,
           searchTerm,
           paginationToken,
+          direction,
           limit,
           abortSignal: req.abortSignal,
         }

shared/lib/url-creator.js

@@ -3,7 +3,10 @@
 const urlJoin = require('url-join');
 
 const assert = require('matrix-public-archive-shared/lib/assert');
-const { TIME_PRECISION_VALUES } = require('matrix-public-archive-shared/lib/reference-values');
+const {
+  DIRECTION,
+  TIME_PRECISION_VALUES,
+} = require('matrix-public-archive-shared/lib/reference-values');
 
 function qsToUrlPiece(qs) {
   if (qs.toString()) {
@@ -25,7 +28,16 @@ class URLCreator {
     return `https://matrix.to/#/${roomIdOrAlias}`;
   }
 
-  roomDirectoryUrl({ searchTerm, homeserver, paginationToken } = {}) {
+  roomDirectoryUrl({ searchTerm, homeserver, paginationToken, direction } = {}) {
+    // You must provide both `paginationToken` and `direction` if either is defined
+    if (paginationToken || direction) {
+      assert(
+        [DIRECTION.forward, DIRECTION.backward].includes(direction),
+        'direction must be [f|b]'
+      );
+      assert(paginationToken);
+    }
+
     let qs = new URLSearchParams();
     if (searchTerm) {
       qs.append('search', searchTerm);
@@ -36,6 +48,9 @@ class URLCreator {
     if (paginationToken) {
       qs.append('page', paginationToken);
     }
+    if (direction) {
+      qs.append('dir', direction);
+    }
 
     return `${this._basePath}${qsToUrlPiece(qs)}`;
   }

shared/viewmodels/RoomDirectoryViewModel.js

@@ -9,6 +9,7 @@ const ModalViewModel = require('matrix-public-archive-shared/viewmodels/ModalVie
 const HomeserverSelectionModalContentViewModel = require('matrix-public-archive-shared/viewmodels/HomeserverSelectionModalContentViewModel');
 const RoomCardViewModel = require('matrix-public-archive-shared/viewmodels/RoomCardViewModel');
 const checkTextForNsfw = require('matrix-public-archive-shared/lib/check-text-for-nsfw');
+const { DIRECTION } = require('../lib/reference-values');
 
 const DEFAULT_SERVER_LIST = ['matrix.org', 'gitter.im'];
 
@@ -304,6 +305,7 @@ class RoomDirectoryViewModel extends ViewModel {
         homeserver: this.homeserverSelection,
         searchTerm: this.searchTerm,
         paginationToken: this._nextPaginationToken,
+        direction: DIRECTION.forward,
       });
     }
 
@@ -316,6 +318,7 @@ class RoomDirectoryViewModel extends ViewModel {
         homeserver: this.homeserverSelection,
         searchTerm: this.searchTerm,
         paginationToken: this._prevPaginationToken,
+        direction: DIRECTION.backward,
       });
     }
 

shared/views/RoomDirectoryView.js

@@ -322,10 +322,21 @@ class RoomDirectoryView extends TemplateView {
           t.view(roomList),
           t.div({ className: 'RoomDirectoryView_paginationButtonCombo' }, [
             t.a(
-              { className: 'RoomDirectoryView_paginationButton', href: vm.prevPageUrl },
+              {
+                className: 'RoomDirectoryView_paginationButton',
+                href: vm.prevPageUrl,
+                'data-testid': 'room-directory-prev-link',
+              },
               'Previous'
             ),
-            t.a({ className: 'RoomDirectoryView_paginationButton', href: vm.nextPageUrl }, 'Next'),
+            t.a(
+              {
+                className: 'RoomDirectoryView_paginationButton',
+                href: vm.nextPageUrl,
+                'data-testid': 'room-directory-next-link',
+              },
+              'Next'
+            ),
           ]),
         ]),
         t.if(