Clean up searching for messages (#2058)

Signed-off-by: Christoph Wurst <christoph@winzerhof-wurst.at>

lib/AppInfo/BootstrapSingleton.php

@@ -26,6 +26,7 @@
 use OCA\Mail\Contracts\IAttachmentService;
 use OCA\Mail\Contracts\IAvatarService;
 use OCA\Mail\Contracts\IMailManager;
+use OCA\Mail\Contracts\IMailSearch;
 use OCA\Mail\Contracts\IMailTransmission;
 use OCA\Mail\Contracts\IUserPreferences;
 use OCA\Mail\Events\BeforeMessageDeletedEvent;
@@ -44,6 +45,7 @@
 use OCA\Mail\Service\Group\IGroupService;
 use OCA\Mail\Service\Group\NextcloudGroupService;
 use OCA\Mail\Service\MailManager;
+use OCA\Mail\Service\MailSearch;
 use OCA\Mail\Service\MailTransmission;
 use OCA\Mail\Service\UserPreferenceSevice;
 use OCP\AppFramework\IAppContainer;
@@ -92,6 +94,7 @@ private function initializeAppContainer(IAppContainer $container) {
 		$container->registerAlias(IAvatarService::class, AvatarService::class);
 		$container->registerAlias(IAttachmentService::class, AttachmentService::class);
 		$container->registerAlias(IMailManager::class, MailManager::class);
+		$container->registerAlias(IMailSearch::class, MailSearch::class);
 		$container->registerAlias(IMailTransmission::class, MailTransmission::class);
 		$container->registerAlias(IUserPreferences::class, UserPreferenceSevice::class);
 

lib/Contracts/IMailSearch.php

@@ -0,0 +1,43 @@
+<?php declare(strict_types=1);
+
+/**
+ * @copyright 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Mail\Contracts;
+
+use OCA\Mail\Account;
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\Model\IMAPMessage;
+
+interface IMailSearch {
+
+	/**
+	 * @param Account $account
+	 * @param string $mailboxName
+	 * @param string|null $filter
+	 * @param string|null $cursor
+	 *
+	 * @return IMAPMessage[]
+	 * @throws ServiceException
+	 */
+	public function findMessages(Account $account, string $mailboxName, ?string $filter, ?int $cursor): array;
+
+}

lib/Controller/MessagesController.php

@@ -33,6 +33,7 @@
 use Exception;
 use OCA\Mail\Account;
 use OCA\Mail\Contracts\IMailManager;
+use OCA\Mail\Contracts\IMailSearch;
 use OCA\Mail\Exception\ServiceException;
 use OCA\Mail\Http\AttachmentDownloadResponse;
 use OCA\Mail\Http\HtmlResponse;
@@ -52,6 +53,7 @@
 use OCP\ILogger;
 use OCP\IRequest;
 use OCP\IURLGenerator;
+use function base64_decode;
 
 class MessagesController extends Controller {
 
@@ -61,6 +63,9 @@ class MessagesController extends Controller {
 	/** @var IMailManager */
 	private $mailManager;
 
+	/** @var IMailSearch */
+	private $mailSearch;
+
 	/** @var string */
 	private $currentUserId;
 
@@ -101,6 +106,7 @@ public function __construct(string $appName,
 								IRequest $request,
 								AccountService $accountService,
 								IMailManager $mailManager,
+								IMailSearch $mailSearch,
 								string $UserId,
 								$userFolder,
 								ILogger $logger,
@@ -111,14 +117,15 @@ public function __construct(string $appName,
 		parent::__construct($appName, $request);
 
 		$this->accountService = $accountService;
+		$this->mailManager = $mailManager;
+		$this->mailSearch = $mailSearch;
 		$this->currentUserId = $UserId;
 		$this->userFolder = $userFolder;
 		$this->logger = $logger;
 		$this->l10n = $l10n;
 		$this->mimeTypeDetector = $mimeTypeDetector;
 		$this->urlGenerator = $urlGenerator;
 		$this->timeFactory = $timeFactory;
-		$this->mailManager = $mailManager;
 	}
 
 	/**
@@ -129,26 +136,26 @@ public function __construct(string $appName,
 	 * @param string $folderId
 	 * @param int $cursor
 	 * @param string $filter
+	 *
 	 * @return JSONResponse
+	 * @throws ServiceException
 	 */
 	public function index(int $accountId, string $folderId, int $cursor = null, string $filter = null): JSONResponse {
-		$mailBox = $this->getFolder($accountId, $folderId);
-
-		$this->logger->debug("loading messages of folder <$folderId>");
-
-		if ($cursor === '') {
-			$cursor = null;
+		$account = $this->getAccount($accountId);
+		if ($account === null) {
+			return new JSONResponse(null, Http::STATUS_FORBIDDEN);
 		}
-		$messages = $mailBox->getMessages($filter, $cursor);
 
-		$json = array_map(function ($j) use ($mailBox) {
-			if ($mailBox->getSpecialRole() === 'trash') {
-				$j['delete'] = (string)$this->l10n->t('Delete permanently');
-			}
-			return $j;
-		}, $messages);
+		$this->logger->debug("loading messages of folder <$folderId>");
 
-		return new JSONResponse($json);
+		return new JSONResponse(
+			$this->mailSearch->findMessages(
+				$account,
+				base64_decode($folderId),
+				$filter === '' ? null : $filter,
+				$cursor
+			)
+		);
 	}
 
 	/**
@@ -158,6 +165,7 @@ public function index(int $accountId, string $folderId, int $cursor = null, stri
 	 * @param int $accountId
 	 * @param string $folderId
 	 * @param int $id
+	 *
 	 * @return JSONResponse
 	 * @throws ServiceException
 	 */
@@ -193,6 +201,7 @@ public function show(int $accountId, string $folderId, int $id): JSONResponse {
 	 * @param int $id
 	 * @param int $destAccountId
 	 * @param string $destFolderId
+	 *
 	 * @return JSONResponse
 	 * @throws Exception
 	 */
@@ -270,6 +279,7 @@ public function getHtmlBody(int $accountId, string $folderId, int $messageId): R
 	 * @param string $folderId
 	 * @param int $messageId
 	 * @param int $attachmentId
+	 *
 	 * @return AttachmentDownloadResponse
 	 */
 	public function downloadAttachment(int $accountId, string $folderId, int $messageId,
@@ -291,6 +301,7 @@ public function downloadAttachment(int $accountId, string $folderId, int $messag
 	 * @param int $messageId
 	 * @param int $attachmentId
 	 * @param string $targetPath
+	 *
 	 * @return JSONResponse
 	 */
 	public function saveAttachment(int $accountId, string $folderId, int $messageId,
@@ -336,6 +347,7 @@ public function saveAttachment(int $accountId, string $folderId, int $messageId,
 	 * @param string $folderId
 	 * @param string $messageId
 	 * @param array $flags
+	 *
 	 * @return JSONResponse
 	 */
 	public function setFlags(int $accountId, string $folderId, int $messageId, array $flags): JSONResponse {
@@ -375,9 +387,12 @@ public function destroy(int $accountId, string $folderId, int $id): JSONResponse
 
 	/**
 	 * @param int $accountId
+	 *
 	 * @return Account|null
+	 * @todo add caching to \OCA\Mail\Service\AccountService
+	 * @deprecated use \OCA\Mail\Service\AccountService::find
 	 */
-	private function getAccount($accountId) {
+	private function getAccount($accountId): ?Account {
 		if (!array_key_exists($accountId, $this->accounts)) {
 			$this->accounts[$accountId] = $this->accountService->find(
 				$this->currentUserId,
@@ -390,6 +405,7 @@ private function getAccount($accountId) {
 	/**
 	 * @param int $accountId
 	 * @param string $folderId
+	 *
 	 * @return IMailBox
 	 */
 	private function getFolder(int $accountId, string $folderId): IMailBox {
@@ -402,6 +418,7 @@ private function getFolder(int $accountId, string $folderId): IMailBox {
 	 * @param string $folderId
 	 * @param int $messageId
 	 * @param array $attachment
+	 *
 	 * @return array
 	 */
 	private function enrichDownloadUrl(int $accountId, string $folderId, int $messageId,
@@ -441,6 +458,7 @@ private function attachmentIsImage(array $attachment): bool {
 
 	/**
 	 * @param array $attachment
+	 *
 	 * @return boolean
 	 */
 	private function attachmentIsCalendarEvent(array $attachment): bool {

lib/Db/Mailbox.php

@@ -23,8 +23,14 @@
 
 namespace OCA\Mail\Db;
 
+use Horde_Imap_Client;
 use OCA\Mail\Folder;
 use OCP\AppFramework\Db\Entity;
+use function in_array;
+use function json_decode;
+use function ltrim;
+use function rtrim;
+use function strtolower;
 
 /**
  * @method string getName()
@@ -73,10 +79,35 @@ public function toFolder(): Folder {
 			$this->delimiter
 		);
 		$folder->setSyncToken($this->getSyncToken());
-		foreach ((json_decode($this->getSpecialUse() ?? '[]', true) ?? []) as $use) {
+		foreach ($this->getSpecialUseParsed() as $use) {
 			$folder->addSpecialUse($use);
 		}
 		return $folder;
 	}
 
+	private function getSpecialUseParsed(): array {
+		return json_decode($this->getSpecialUse() ?? '[]', true) ?? [];
+	}
+
+	public function isSpecialUse(string $specialUse): bool {
+		return in_array(
+			ltrim(
+				strtolower($specialUse),
+				'\\'
+			),
+			array_map("strtolower", $this->getSpecialUseParsed()),
+			true
+		);
+	}
+
+	public function getMailbox(): string {
+		// TODO: evaluate if SPECIALUSE_FLAGGED can also be set on a real IMAP mailbox
+		// because then this could be problematic if they name also ends with /FLAGGED
+		// though, this sounds very unlikely
+		if ($this->isSpecialUse(Horde_Imap_Client::SPECIALUSE_FLAGGED)) {
+			return rtrim($this->getName(), '/FLAGGED');
+		}
+		return $this->getName();
+	}
+
 }

lib/IMAP/Search/FullScanSearchStrategy.php

@@ -0,0 +1,86 @@
+<?php declare(strict_types=1);
+
+/**
+ * @copyright 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+namespace OCA\Mail\IMAP\Search;
+
+use Horde_Imap_Client_Exception;
+use Horde_Imap_Client_Fetch_Query;
+use Horde_Imap_Client_Ids;
+use Horde_Imap_Client_Socket;
+use function array_keys;
+use function array_slice;
+use function uasort;
+
+class FullScanSearchStrategy implements ISearchStrategy {
+
+	/** @var Horde_Imap_Client_Socket */
+	private $client;
+
+	/** @var string */
+	private $mailbox;
+
+	/** @var int|null */
+	private $cursor;
+
+	public function __construct(Horde_Imap_Client_Socket $client,
+								string $mailbox,
+								?int $cursor) {
+		$this->client = $client;
+		$this->mailbox = $mailbox;
+		$this->cursor = $cursor;
+	}
+
+	/**
+	 * Scan all messages of a mailbox and filter out matching ones
+	 *
+	 * This is slow, but some IMAP server don't support the SORT capability.
+	 *
+	 * @throws Horde_Imap_Client_Exception
+	 */
+	public function getIds(int $maxResults, array $flags = []): Horde_Imap_Client_Ids {
+		$query = new Horde_Imap_Client_Fetch_Query();
+		$query->uid();
+		$query->imapDate();
+
+		$result = $this->client->fetch($this->mailbox, $query);
+		$uidMap = [];
+		foreach ($result as $r) {
+			$ts = $r->getImapDate()->getTimeStamp();
+			if ($this->cursor === null || $ts < $this->cursor) {
+				$uidMap[$r->getUid()] = $ts;
+			}
+		}
+		// sort by time
+		uasort($uidMap, function ($a, $b) {
+			return $a < $b;
+		});
+		return new Horde_Imap_Client_Ids(
+			array_slice(
+				array_keys($uidMap),
+				0,
+				$maxResults
+			)
+		);
+	}
+
+}