From 4869afc4d3e858344d7548405ce94dc432f07c93 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 28 Sep 2023 09:36:25 +0700 Subject: [PATCH] Fix 500 server error on project dashboard (#1779) Large projects were returning an HTTP 500 on the project dashboard, because PHP was running out of memory trying to return every entry for stats to be calculated client-side. This changes the stats to be calculated server-side instead. We get rid of the "number of entries with audio" stat because it's not possible to calculate in a simple Mongo query and would require serializing the entire entry database server-side, causing the very memory error we're trying to get rid of. --- .../projects/[project_code]/+page.svelte | 6 ---- .../projects/[project_code]/meta/+server.ts | 26 +++++----------- .../Languageforge/Lexicon/Dto/LexStatsDto.php | 25 +++++++++++++++ src/Api/Model/Shared/Mapper/MongoQueries.php | 31 +++++++++++++++++++ src/Api/Service/Sf.php | 3 +- 5 files changed, 65 insertions(+), 26 deletions(-) create mode 100644 src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php create mode 100644 src/Api/Model/Shared/Mapper/MongoQueries.php diff --git a/next-app/src/routes/projects/[project_code]/+page.svelte b/next-app/src/routes/projects/[project_code]/+page.svelte index 00eb8d52e4..130254988d 100644 --- a/next-app/src/routes/projects/[project_code]/+page.svelte +++ b/next-app/src/routes/projects/[project_code]/+page.svelte @@ -33,12 +33,6 @@ icon: NotesIcon, url: `/app/lexicon/${ project.id }`, }, - { - title: 'Entries with audio', - value: project.num_entries_with_audio, - icon: VoiceIcon, - url: `/app/lexicon/${ project.id }#!/editor/entry/000000?filterBy=Audio`, - }, { title: 'Entries with pictures', value: project.num_entries_with_pictures, diff --git a/next-app/src/routes/projects/[project_code]/meta/+server.ts b/next-app/src/routes/projects/[project_code]/meta/+server.ts index 35bc918f3d..c01e3a1dc9 100644 --- a/next-app/src/routes/projects/[project_code]/meta/+server.ts +++ b/next-app/src/routes/projects/[project_code]/meta/+server.ts @@ -9,8 +9,9 @@ type LegacyProjectDetails = { } type LegacyStats = { - entries: object[], - comments: Comment[], + num_entries, + num_entries_with_pictures, + num_unresolved_comments, } type Comment = { @@ -23,40 +24,27 @@ export type ProjectDetails = { name: string, num_users: number, num_entries: number, - num_entries_with_audio: number, num_entries_with_pictures: number, num_unresolved_comments?: number, } export async function fetch_project_details({ project_code, cookie }) { const { id, projectName: name, users }: LegacyProjectDetails = await sf({ name: 'set_project', args: [ project_code ], cookie }) - const { entries, comments }: LegacyStats = await sf({ name: 'lex_stats', cookie }) + const stats: LegacyStats = await sf({ name: 'lex_stats', cookie }) const details: ProjectDetails = { id, code: project_code, name, num_users: Object.keys(users).length, - num_entries: entries.length, - num_entries_with_audio: entries.filter(has_audio).length, - num_entries_with_pictures: entries.filter(has_picture).length, + num_entries: stats.num_entries, + num_entries_with_pictures: stats.num_entries_with_pictures, } const { role } = await fetch_current_user(cookie) if (can_view_comments(role)) { - const unresolved_comments = comments.filter(({ status }) => status !== 'resolved') - - details.num_unresolved_comments = unresolved_comments.length + details.num_unresolved_comments = stats.num_unresolved_comments } return details } - -function has_picture(entry: object) { - return JSON.stringify(entry).includes('"pictures":') -} - -// audio can be found in lots of places other than lexeme, ref impl used: https://github.com/sillsdev/web-languageforge/blob/develop/src/angular-app/bellows/core/offline/editor-data.service.ts#L523 -function has_audio(entry: object) { - return JSON.stringify(entry).includes('-audio":') // naming convention imposed by src/angular-app/languageforge/lexicon/settings/configuration/input-system-view.model.ts L81 -} diff --git a/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php new file mode 100644 index 0000000000..55a385797b --- /dev/null +++ b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php @@ -0,0 +1,25 @@ +databaseName()); + $num_entries = MongoQueries::countEntries($db, "lexicon"); + $num_entries_with_pictures = MongoQueries::countEntriesWithPictures($db, "lexicon"); + $num_unresolved_comments = MongoQueries::countUnresolvedComments($db, "lexiconComments"); + return [ + "num_entries" => $num_entries, + "num_entries_with_pictures" => $num_entries_with_pictures, + "num_unresolved_comments" => $num_unresolved_comments, + ]; + } +} diff --git a/src/Api/Model/Shared/Mapper/MongoQueries.php b/src/Api/Model/Shared/Mapper/MongoQueries.php new file mode 100644 index 0000000000..7f59f1d9cf --- /dev/null +++ b/src/Api/Model/Shared/Mapper/MongoQueries.php @@ -0,0 +1,31 @@ +selectCollection($collectionName); + return $coll->count(); + } + + public static function countEntriesWithPictures($db, $collectionName) + { + $coll = $db->selectCollection($collectionName); + $query = [ + "senses" => ['$exists' => true, '$ne' => []], + "senses.pictures" => ['$exists' => true, '$ne' => []], + ]; + return $coll->count($query); + } + + public static function countUnresolvedComments($db, $collectionName) + { + $coll = $db->selectCollection($collectionName); + $query = [ + "status" => ['$exists' => true, '$ne' => "resolved"], + ]; + return $coll->count($query); + } +} diff --git a/src/Api/Service/Sf.php b/src/Api/Service/Sf.php index d3ae5bc22c..97a51cdefe 100644 --- a/src/Api/Service/Sf.php +++ b/src/Api/Service/Sf.php @@ -14,6 +14,7 @@ use Api\Model\Languageforge\Lexicon\Dto\LexBaseViewDto; use Api\Model\Languageforge\Lexicon\Dto\LexDbeDto; use Api\Model\Languageforge\Lexicon\Dto\LexProjectDto; +use Api\Model\Languageforge\Lexicon\Dto\LexStatsDto; use Api\Model\Shared\Command\ProjectCommands; use Api\Model\Shared\Command\SessionCommands; use Api\Model\Shared\Command\UserCommands; @@ -518,7 +519,7 @@ public function lex_stats() $user = new UserModel($this->userId); if ($user->isMemberOfProject($this->projectId)) { - return LexDbeDto::encode($projectModel->id->asString(), $this->userId, 1); + return LexStatsDto::encode($projectModel); } throw new UserUnauthorizedException("User $this->userId is not a member of project $projectModel->projectCode");