Fix emrge errors

sillsdev · Oct 17, 2024 · ec88283 · ec88283
1 parent 5033bd3
commit ec88283
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 25 deletions.
diff --git a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs
@@ -75,30 +75,34 @@ await client.BuildStartedAsync(
 
                 try
                 {
+                    List<InsertPretranslationsRequest> pretranslationsRequests = [];
+                    ParallelCorpusPreprocessor.PreprocessCorpora(
+                        request.Corpora.Select(Map).ToList(),
+                        row => { },
+                        (row, corpus) =>
+                        {
+                            pretranslationsRequests.Add(
+                                new InsertPretranslationsRequest
+                                {
+                                    EngineId = request.EngineId,
+                                    CorpusId = corpus.Id,
+                                    TextId = row.TextId,
+                                    Refs = { row.Refs.Select(r => r.ToString()) },
+                                    Translation = row.SourceSegment
+                                }
+                            );
+                        },
+                        false
+                    );
                     using (
                         AsyncClientStreamingCall<InsertPretranslationsRequest, Empty> call =
                             client.InsertPretranslations(cancellationToken: cancellationToken)
                     )
                     {
-                        ParallelCorpusPreprocessor.PreprocessCorpora(
-                            request.Corpora.Select(Map).ToList(),
-                            row => { },
-                            async (row, corpus) =>
-                            {
-                                await call.RequestStream.WriteAsync(
-                                    new InsertPretranslationsRequest
-                                    {
-                                        EngineId = request.EngineId,
-                                        CorpusId = corpus.Id,
-                                        TextId = row.TextId,
-                                        Refs = { row.Refs.Select(r => r.ToString()) },
-                                        Translation = row.SourceSegment
-                                    },
-                                    cancellationToken
-                                );
-                            },
-                            false
-                        );
+                        foreach (InsertPretranslationsRequest request in pretranslationsRequests)
+                        {
+                            await call.RequestStream.WriteAsync(request, cancellationToken);
+                        }
                         await call.RequestStream.CompleteAsync();
                         await call;
                     }
@@ -214,14 +218,18 @@ private static SIL.ServiceToolkit.Models.MonolingualCorpus Map(MonolingualCorpus
             kvp => kvp.Value.Chapters.ToHashSet()
         );
         var trainOnTextIds = source.TrainOnTextIds.ToHashSet();
-        FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds);
+        FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll);
 
         var pretranslateChapters = source.PretranslateChapters.ToDictionary(
             kvp => kvp.Key,
             kvp => kvp.Value.Chapters.ToHashSet()
         );
         var pretranslateTextIds = source.PretranslateTextIds.ToHashSet();
-        FilterChoice pretranslateFilter = GetFilterChoice(pretranslateChapters, pretranslateTextIds);
+        FilterChoice pretranslateFilter = GetFilterChoice(
+            pretranslateChapters,
+            pretranslateTextIds,
+            source.PretranslateAll
+        );
 
         return new SIL.ServiceToolkit.Models.MonolingualCorpus
         {
@@ -254,12 +262,13 @@ private enum FilterChoice
 
     private static FilterChoice GetFilterChoice(
         IReadOnlyDictionary<string, HashSet<int>> chapters,
-        HashSet<string> textIds
+        HashSet<string> textIds,
+        bool noFilter
     )
     {
         // Only either textIds or Scripture Range will be used at a time
         // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text)
-        if (chapters is null && textIds is null)
+        if (noFilter || (chapters is null && textIds is null))
             return FilterChoice.None;
         if (chapters is null || chapters.Count == 0)
             return FilterChoice.TextIds;

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj b/src/ServiceToolkit/src/SIL.ServiceToolkit/SIL.ServiceToolkit.csproj
@@ -16,7 +16,7 @@
 	  <PackageReference Include="Hangfire.Core" Version="1.8.14" />
 	  <PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 	  <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
-	  <PackageReference Include="SIL.Machine" Version="3.2.8" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+	  <PackageReference Include="SIL.Machine" Version="3.4.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 	</ItemGroup>
 
 	<ItemGroup>

diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/ParallelCorpusPreprocessor.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Utils/ParallelCorpusPreprocessor.cs
@@ -70,7 +70,6 @@ row.Ref is not ScriptureRef sr
                             sc.Corpus.PretranslateTextIds.Except(sc.Corpus.TrainOnTextIds ?? new())
                         );
                     }
-
                     return textCorpus.Where(row =>
                         row.Ref is not ScriptureRef sr
                         || sc.Corpus.PretranslateChapters is null