diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 472e33d0..fc2a72df 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -10,7 +10,7 @@ jobs: build: name: Build runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 env: SERVAL_CLIENT_ID: ${{ secrets.SERVAL_CLIENT_ID }} diff --git a/deploy/qa-ext-values.yaml b/deploy/qa-ext-values.yaml index 96074da6..bca7463f 100644 --- a/deploy/qa-ext-values.yaml +++ b/deploy/qa-ext-values.yaml @@ -1,6 +1,6 @@ externalHost: qa.serval-api.org environment: Production -deploymentVersion: '1.6.QA3' +deploymentVersion: '1.7.QA0' alertEmail: ext-qa-serval-alerts@languagetechnology.org emailsToAlert: john_lambert@sil.org enableTls: true @@ -8,7 +8,7 @@ namespace: serval auth0Domain: dev-sillsdev.auth0.com lokiTenent: serval-tenant lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local -servalImage: ghcr.io/sillsdev/serval:1.6.3 +servalImage: ghcr.io/sillsdev/serval:1.7.0 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.6.3 ClearMLQueue: production MongoConnectionPrefix: qa_ diff --git a/deploy/qa-int-values.yaml b/deploy/qa-int-values.yaml index 21aaec25..3a520728 100644 --- a/deploy/qa-int-values.yaml +++ b/deploy/qa-int-values.yaml @@ -8,7 +8,7 @@ namespace: nlp auth0Domain: sil-appbuilder.auth0.com lokiTenent: nlp-tenant lokiUrl: http://loki-distributed-gateway.loki.svc.cluster.local -servalImage: ghcr.io/sillsdev/serval:1.6.1 +servalImage: ghcr.io/sillsdev/serval:1.7.0 ClearMLDockerImage: ghcr.io/sillsdev/machine.py:1.6.3 ClearMLQueue: lambert_24gb MongoConnectionPrefix: qa_int_ diff --git a/src/Echo/src/EchoTranslationEngine/Program.cs b/src/Echo/src/EchoTranslationEngine/Program.cs index 6c6f3768..a679dfb5 100644 --- a/src/Echo/src/EchoTranslationEngine/Program.cs +++ b/src/Echo/src/EchoTranslationEngine/Program.cs @@ -17,9 +17,6 @@ WebApplication app = builder.Build(); -// Configure the HTTP request pipeline. -app.UseHttpsRedirection(); - app.MapGrpcService(); app.MapGrpcService(); diff --git a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs index 67779bc0..254fe0af 100644 --- a/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs +++ b/src/Echo/src/EchoTranslationEngine/TranslationEngineServiceV1.cs @@ -85,7 +85,11 @@ await client.BuildStartedAsync( var sourceFiles = corpus .SourceCorpora.SelectMany(sc => sc.Files.Where(f => - (sc.PretranslateTextIds is null || sc.PretranslateTextIds.Contains(f.TextId)) + ( + sc.PretranslateAll + || sc.PretranslateTextIds is null + || sc.PretranslateTextIds.Contains(f.TextId) + ) && f.Format == FileFormat.Text ) ) @@ -93,7 +97,11 @@ await client.BuildStartedAsync( var targetFiles = corpus .TargetCorpora.SelectMany(tc => tc.Files.Where(f => - (tc.PretranslateTextIds is null || tc.PretranslateTextIds.Contains(f.TextId)) + ( + tc.PretranslateAll + || tc.PretranslateTextIds is null + || tc.PretranslateTextIds.Contains(f.TextId) + ) && f.Format == FileFormat.Text ) ) diff --git a/src/Machine/src/Serval.Machine.EngineServer/Program.cs b/src/Machine/src/Serval.Machine.EngineServer/Program.cs index e36db6c2..b03f6575 100644 --- a/src/Machine/src/Serval.Machine.EngineServer/Program.cs +++ b/src/Machine/src/Serval.Machine.EngineServer/Program.cs @@ -35,8 +35,6 @@ var app = builder.Build(); -app.UseHttpsRedirection(); - app.MapServalTranslationEngineService(); app.MapHangfireDashboard(); diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj index 97d7fb64..3091b02f 100644 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -36,9 +36,9 @@ - - - + + + diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index d9e433ce..ecd52876 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -238,22 +238,27 @@ row.Ref is not ScriptureRef sr } } } + void WriteRow(Utf8JsonWriter writer, string textId, IReadOnlyList refs, string translation) + { + writer.WriteStartObject(); + writer.WriteString("corpusId", corpus.Id); + writer.WriteString("textId", textId); + writer.WriteStartArray("refs"); + foreach (object rowRef in refs) + writer.WriteStringValue(rowRef.ToString()); + writer.WriteEndArray(); + writer.WriteString("translation", translation); + writer.WriteEndObject(); + pretranslateCount++; + } + + ITextCorpus targetCorpus = + targetCorpora.Length > 0 ? targetCorpora[0].TextCorpus : new DictionaryTextCorpus(); - foreach (Row row in AlignPretranslateCorpus(sourcePretranslateCorpora, targetCorpora[0].TextCorpus)) + foreach (Row row in AlignPretranslateCorpus(sourcePretranslateCorpora, targetCorpus)) { if (row.SourceSegment.Length > 0) - { - pretranslateWriter.WriteStartObject(); - pretranslateWriter.WriteString("corpusId", corpus.Id); - pretranslateWriter.WriteString("textId", row.TextId); - pretranslateWriter.WriteStartArray("refs"); - foreach (object rowRef in row.Refs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteString("translation", row.SourceSegment); - pretranslateWriter.WriteEndObject(); - pretranslateCount++; - } + WriteRow(pretranslateWriter, row.TextId, row.Refs, row.SourceSegment); } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs index bced613b..dfc52263 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs @@ -286,16 +286,20 @@ private static Models.MonolingualCorpus Map(Translation.V1.MonolingualCorpus sou kvp => kvp.Value.Chapters.ToHashSet() ); var trainOnTextIds = source.TrainOnTextIds.ToHashSet(); - FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds); + FilterChoice trainingFilter = GetFilterChoice(trainOnChapters, trainOnTextIds, source.TrainOnAll); var pretranslateChapters = source.PretranslateChapters.ToDictionary( kvp => kvp.Key, kvp => kvp.Value.Chapters.ToHashSet() ); var pretranslateTextIds = source.PretranslateTextIds.ToHashSet(); - FilterChoice pretranslateFilter = GetFilterChoice(pretranslateChapters, pretranslateTextIds); + FilterChoice pretranslateFilter = GetFilterChoice( + pretranslateChapters, + pretranslateTextIds, + source.PretranslateAll + ); - return new Models.MonolingualCorpus + var corpus = new Models.MonolingualCorpus { Id = source.Id, Language = source.Language, @@ -305,6 +309,7 @@ private static Models.MonolingualCorpus Map(Translation.V1.MonolingualCorpus sou PretranslateChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, PretranslateTextIds = pretranslateFilter == FilterChoice.TextIds ? pretranslateTextIds : null }; + return corpus; } private static Models.CorpusFile Map(Translation.V1.CorpusFile source) @@ -326,12 +331,13 @@ private enum FilterChoice private static FilterChoice GetFilterChoice( IReadOnlyDictionary> chapters, - HashSet textIds + HashSet textIds, + bool noFilter ) { // Only either textIds or Scripture Range will be used at a time // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (chapters is null && textIds is null) + if (noFilter || (chapters is null && textIds is null)) return FilterChoice.None; if (chapters is null || chapters.Count == 0) return FilterChoice.TextIds; diff --git a/src/Serval/src/Serval.Client/Serval.Client.csproj b/src/Serval/src/Serval.Client/Serval.Client.csproj index 0a72b611..4075c023 100644 --- a/src/Serval/src/Serval.Client/Serval.Client.csproj +++ b/src/Serval/src/Serval.Client/Serval.Client.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 1.5.0 + 1.7.0 Client classes for Serval. Serval.Client Serval diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto index 98918f0c..609a3fc0 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/engine.proto @@ -161,6 +161,8 @@ message ParallelCorpus { message MonolingualCorpus { string id = 1; string language = 2; + bool train_on_all = 3; + bool pretranslate_all = 4; map train_on_chapters = 5; map pretranslate_chapters = 6; repeated string train_on_text_ids = 7; diff --git a/src/Serval/src/Serval.Shared/Serval.Shared.csproj b/src/Serval/src/Serval.Shared/Serval.Shared.csproj index 6ea07ec4..5af835f5 100644 --- a/src/Serval/src/Serval.Shared/Serval.Shared.csproj +++ b/src/Serval/src/Serval.Shared/Serval.Shared.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/Serval/src/Serval.Translation/Services/EngineService.cs b/src/Serval/src/Serval.Translation/Services/EngineService.cs index 47c4ab9b..5b3d08ff 100644 --- a/src/Serval/src/Serval.Translation/Services/EngineService.cs +++ b/src/Serval/src/Serval.Translation/Services/EngineService.cs @@ -601,7 +601,12 @@ private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, Pre V1.MonolingualCorpus targetCorpus = new() { Language = source.TargetLanguage, Files = { source.TargetFiles.Select(Map) } }; - if (trainingCorpus != null) + if (trainingCorpus is null || (trainingCorpus.TextIds is null && trainingCorpus.ScriptureRange is null)) + { + sourceCorpus.TrainOnAll = true; + targetCorpus.TrainOnAll = true; + } + else { if (trainingCorpus.TextIds is not null && trainingCorpus.ScriptureRange is not null) { @@ -636,7 +641,15 @@ private V1.ParallelCorpus Map(Corpus source, TrainingCorpus? trainingCorpus, Pre targetCorpus.TrainOnChapters.Add(chapters); } } - if (pretranslateCorpus != null) + if ( + pretranslateCorpus is null + || (pretranslateCorpus.TextIds is null && pretranslateCorpus.ScriptureRange is null) + ) + { + sourceCorpus.PretranslateAll = true; + targetCorpus.PretranslateAll = true; + } + else { if (pretranslateCorpus.TextIds is not null && pretranslateCorpus.ScriptureRange is not null) { @@ -767,14 +780,32 @@ pretranslateFilter is not null Files = { source.Files.Select(Map) } }; - if (trainOnChapters is not null) - corpus.TrainOnChapters.Add(trainOnChapters); - if (trainingFilter?.TextIds is not null) - corpus.TrainOnTextIds.Add(trainingFilter.TextIds); - if (pretranslateChapters is not null) - corpus.PretranslateChapters.Add(pretranslateChapters); - if (pretranslateFilter?.TextIds is not null) - corpus.PretranslateTextIds.Add(pretranslateFilter.TextIds); + if (trainingFilter is null || (trainingFilter.TextIds is null && trainingFilter.ScriptureRange is null)) + { + corpus.TrainOnAll = true; + } + else + { + if (trainOnChapters is not null) + corpus.TrainOnChapters.Add(trainOnChapters); + if (trainingFilter?.TextIds is not null) + corpus.TrainOnTextIds.Add(trainingFilter.TextIds); + } + + if ( + pretranslateFilter is null + || (pretranslateFilter.TextIds is null && pretranslateFilter.ScriptureRange is null) + ) + { + corpus.PretranslateAll = true; + } + else + { + if (pretranslateChapters is not null) + corpus.PretranslateChapters.Add(pretranslateChapters); + if (pretranslateFilter?.TextIds is not null) + corpus.PretranslateTextIds.Add(pretranslateFilter.TextIds); + } return corpus; } diff --git a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs index 6d0b2df2..cdf1bcf3 100644 --- a/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs +++ b/src/Serval/test/Serval.ApiServer.IntegrationTests/TranslationEngineTests.cs @@ -1660,6 +1660,105 @@ public async Task StartBuildAsync_ParallelCorpus() Assert.That(build, Is.Not.Null); } + [Test] + public async Task StartBuildAsync_Corpus_NoFilter() + { + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); + TranslationCorpus addedCorpus = await client.AddCorpusAsync(NMT_ENGINE1_ID, TestCorpusConfig); + PretranslateCorpusConfig ptcc = + new() { CorpusId = addedCorpus.Id, SourceFilters = [new() { CorpusId = SOURCE_CORPUS_ID }] }; + TrainingCorpusConfig tcc = + new() + { + CorpusId = addedCorpus.Id, + SourceFilters = [new() { CorpusId = SOURCE_CORPUS_ID }], + TargetFilters = [new() { CorpusId = TARGET_CORPUS_ID }] + }; + ; + TranslationBuildConfig tbc = new TranslationBuildConfig + { + Pretranslate = [ptcc], + TrainOn = [tcc], + Options = """ + {"max_steps":10, + "use_key_terms":false, + "some_double":10.5, + "some_nested": {"more_nested": {"other_double":10.5}}, + "some_string":"string"} + """ + }; + TranslationBuild resultAfterStart; + Assert.ThrowsAsync(async () => + { + resultAfterStart = await client.GetCurrentBuildAsync(NMT_ENGINE1_ID); + }); + + TranslationBuild build = await client.StartBuildAsync(NMT_ENGINE1_ID, tbc); + Assert.That(build, Is.Not.Null); + Assert.That(build.TrainOn, Is.Not.Null); + Assert.That(build.TrainOn.Count, Is.EqualTo(1)); + Assert.That(build.TrainOn[0].TextIds, Is.Null); + Assert.That(build.TrainOn[0].ScriptureRange, Is.Null); + Assert.That(build.Pretranslate, Is.Not.Null); + Assert.That(build.Pretranslate.Count, Is.EqualTo(1)); + Assert.That(build.Pretranslate[0].TextIds, Is.Null); + Assert.That(build.Pretranslate[0].ScriptureRange, Is.Null); + + build = await client.GetCurrentBuildAsync(NMT_ENGINE1_ID); + Assert.That(build, Is.Not.Null); + } + + [Test] + public async Task StartBuildAsync_ParallelCorpus_NoFilter() + { + TranslationEnginesClient client = _env.CreateTranslationEnginesClient(); + TranslationParallelCorpus addedCorpus = await client.AddParallelCorpusAsync( + NMT_ENGINE1_ID, + TestParallelCorpusConfig + ); + PretranslateCorpusConfig ptcc = + new() { ParallelCorpusId = addedCorpus.Id, SourceFilters = [new() { CorpusId = SOURCE_CORPUS_ID }] }; + TrainingCorpusConfig tcc = + new() + { + ParallelCorpusId = addedCorpus.Id, + SourceFilters = [new() { CorpusId = SOURCE_CORPUS_ID }], + TargetFilters = [new() { CorpusId = TARGET_CORPUS_ID }] + }; + ; + TranslationBuildConfig tbc = new TranslationBuildConfig + { + Pretranslate = [ptcc], + TrainOn = [tcc], + Options = """ + {"max_steps":10, + "use_key_terms":false, + "some_double":10.5, + "some_nested": {"more_nested": {"other_double":10.5}}, + "some_string":"string"} + """ + }; + TranslationBuild resultAfterStart; + Assert.ThrowsAsync(async () => + { + resultAfterStart = await client.GetCurrentBuildAsync(NMT_ENGINE1_ID); + }); + + TranslationBuild build = await client.StartBuildAsync(NMT_ENGINE1_ID, tbc); + Assert.That(build, Is.Not.Null); + Assert.That(build.TrainOn, Is.Not.Null); + Assert.That(build.TrainOn.Count, Is.EqualTo(1)); + Assert.That(build.TrainOn[0].TextIds, Is.Null); + Assert.That(build.TrainOn[0].ScriptureRange, Is.Null); + Assert.That(build.Pretranslate, Is.Not.Null); + Assert.That(build.Pretranslate.Count, Is.EqualTo(1)); + Assert.That(build.Pretranslate[0].TextIds, Is.Null); + Assert.That(build.Pretranslate[0].ScriptureRange, Is.Null); + + build = await client.GetCurrentBuildAsync(NMT_ENGINE1_ID); + Assert.That(build, Is.Not.Null); + } + [Test] public async Task StartBuildAsync_ParallelCorpus_PretranslateParallelAndNormalCorpus() { diff --git a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs index e532d1f1..83d6d6a0 100644 --- a/src/Serval/test/Serval.E2ETests/ServalApiTests.cs +++ b/src/Serval/test/Serval.E2ETests/ServalApiTests.cs @@ -166,8 +166,8 @@ public async Task NmtQueueMultiple() engineIds[i] = await _helperClient.CreateNewEngineAsync("Nmt", "es", "en", $"NMT1_{i}"); string engineId = engineIds[i]; string[] books = ["MAT.txt", "1JN.txt", "2JN.txt"]; - await _helperClient.AddTextCorpusToEngineAsync(engineId, books, "es", "en", false); - await _helperClient.AddTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); + await _helperClient.AddParallelTextCorpusToEngineAsync(engineId, books, "es", "en", false); + await _helperClient.AddParallelTextCorpusToEngineAsync(engineId, ["3JN.txt"], "es", "en", true); await _helperClient.StartBuildAsync(engineId); //Ensure that tasks are enqueued roughly in order await Task.Delay(1_000); @@ -276,7 +276,7 @@ public async Task CircuitousRouteGetWordGraphAsync() Assert.That(ex.StatusCode, Is.EqualTo(409)); //Add corpus - string cId = await _helperClient.AddTextCorpusToEngineAsync( + string cId = await _helperClient.AddParallelTextCorpusToEngineAsync( smtEngineId, ["2JN.txt", "3JN.txt"], "es", @@ -288,10 +288,10 @@ public async Task CircuitousRouteGetWordGraphAsync() await _helperClient.BuildEngineAsync(smtEngineId); //Remove added corpus (shouldn't affect translation) - await _helperClient.TranslationEnginesClient.DeleteCorpusAsync(smtEngineId, cId, deleteFiles: false); + await _helperClient.TranslationEnginesClient.DeleteParallelCorpusAsync(smtEngineId, cId); // Add corpus - await _helperClient.AddTextCorpusToEngineAsync( + await _helperClient.AddParallelTextCorpusToEngineAsync( smtEngineId, ["1JN.txt", "2JN.txt", "3JN.txt"], "es", diff --git a/src/Serval/test/Serval.E2ETests/ServalClientHelper.cs b/src/Serval/test/Serval.E2ETests/ServalClientHelper.cs index ae70f6ce..d64fb15a 100644 --- a/src/Serval/test/Serval.E2ETests/ServalClientHelper.cs +++ b/src/Serval/test/Serval.E2ETests/ServalClientHelper.cs @@ -3,6 +3,7 @@ namespace Serval.E2ETests; public class ServalClientHelper : IAsyncDisposable { public DataFilesClient DataFilesClient { get; } + public CorporaClient CorporaClient { get; } public TranslationEnginesClient TranslationEnginesClient { get; } public TranslationEngineTypesClient TranslationEngineTypesClient { get; } @@ -32,6 +33,7 @@ public ServalClientHelper(string audience, string prefix = "SCE_", bool ignoreSS _httpClient.BaseAddress = new Uri(hostUrl); _httpClient.Timeout = TimeSpan.FromSeconds(60); DataFilesClient = new DataFilesClient(_httpClient); + CorporaClient = new CorporaClient(_httpClient); TranslationEnginesClient = new TranslationEnginesClient(_httpClient); TranslationEngineTypesClient = new TranslationEngineTypesClient(_httpClient); _prefix = prefix; @@ -229,6 +231,77 @@ bool pretranslate return response.Id; } + public async Task AddParallelTextCorpusToEngineAsync( + string engineId, + string[] filesToAdd, + string sourceLanguage, + string targetLanguage, + bool pretranslate + ) + { + List sourceFiles = await UploadFilesAsync(filesToAdd, FileFormat.Text, sourceLanguage); + + var targetFileConfig = new List(); + if (!pretranslate) + { + List targetFiles = await UploadFilesAsync(filesToAdd, FileFormat.Text, targetLanguage); + foreach (var item in targetFiles.Select((file, i) => new { i, file })) + { + targetFileConfig.Add(new CorpusFileConfig { FileId = item.file.Id, TextId = filesToAdd[item.i] }); + } + } + + CorpusConfig targetCorpusConfig = + new() + { + Name = "None", + Language = targetLanguage, + Files = targetFileConfig + }; + + var targetCorpus = await CorporaClient.CreateAsync(targetCorpusConfig); + + var sourceFileConfig = new List(); + + if (sourceLanguage == targetLanguage && !pretranslate) + { + // if it's the same language, and we are not pretranslating, do nothing (echo for suggestions) + // if pretranslating, we need to upload the source separately + // if different languages, we are not echoing. + } + else + { + for (int i = 0; i < sourceFiles.Count; i++) + { + sourceFileConfig.Add(new CorpusFileConfig { FileId = sourceFiles[i].Id, TextId = filesToAdd[i] }); + } + } + + CorpusConfig sourceCorpusConfig = + new() + { + Name = "None", + Language = sourceLanguage, + Files = sourceFileConfig + }; + + var sourceCorpus = await CorporaClient.CreateAsync(sourceCorpusConfig); + + TranslationParallelCorpusConfig parallelCorpusConfig = + new() { SourceCorpusIds = { sourceCorpus.Id }, TargetCorpusIds = { targetCorpus.Id } }; + + var parallelCorpus = await TranslationEnginesClient.AddParallelCorpusAsync(engineId, parallelCorpusConfig); + + if (pretranslate) + { + TranslationBuildConfig.Pretranslate!.Add( + new PretranslateCorpusConfig { ParallelCorpusId = parallelCorpus.Id, TextIds = filesToAdd.ToList() } + ); + } + + return parallelCorpus.Id; + } + public async Task> UploadFilesAsync( IEnumerable filesToAdd, FileFormat fileFormat, diff --git a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs index 59d24d0c..a71e8908 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/EngineServiceTests.cs @@ -136,7 +136,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified() Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }, @@ -155,7 +157,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified() Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -206,7 +210,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } }, @@ -226,7 +232,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } } @@ -277,7 +285,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } }, @@ -297,7 +307,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } } @@ -347,7 +359,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }, @@ -366,7 +380,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "text1" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -445,7 +461,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } }, @@ -475,7 +493,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } } @@ -525,7 +545,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } }, @@ -544,7 +566,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false } } } @@ -618,7 +642,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "MAT" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -632,7 +658,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "MRK" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }, @@ -653,7 +681,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "MAT" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -667,7 +697,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Text, TextId = "MRK" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -741,7 +773,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -755,7 +789,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file3.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }, @@ -776,7 +812,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -790,7 +828,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file4.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -866,7 +906,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -880,7 +922,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file3.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }, @@ -911,7 +955,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new() { @@ -925,7 +971,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file4.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -1001,7 +1049,9 @@ await env.Service.StartBuildAsync( "MRK", new ScriptureChapters { Chapters = { 1, 2 } } } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new V1.MonolingualCorpus() { @@ -1026,7 +1076,9 @@ await env.Service.StartBuildAsync( "MRK", new ScriptureChapters { Chapters = { 1 } } } - } + }, + PretranslateAll = true, + TrainOnAll = false } }, TargetCorpora = @@ -1054,7 +1106,9 @@ await env.Service.StartBuildAsync( "MRK", new ScriptureChapters { Chapters = { 2 } } } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new V1.MonolingualCorpus() { @@ -1079,7 +1133,9 @@ await env.Service.StartBuildAsync( "MRK", new ScriptureChapters { Chapters = { 1, 2 } } } - } + }, + PretranslateAll = true, + TrainOnAll = false } } } @@ -1127,7 +1183,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true }, new V1.MonolingualCorpus() { @@ -1141,7 +1199,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file3.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } }, TargetCorpora = @@ -1158,7 +1218,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true }, new V1.MonolingualCorpus() { @@ -1172,7 +1234,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file4.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -1213,7 +1277,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() Format = FileFormat.Paratext, TextId = "file1.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true }, new V1.MonolingualCorpus() { @@ -1227,7 +1293,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() Format = FileFormat.Paratext, TextId = "file3.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } }, TargetCorpora = @@ -1244,7 +1312,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true }, new V1.MonolingualCorpus() { @@ -1258,7 +1328,9 @@ public async Task StartBuildAsync_TrainOnNotSpecified_ParallelCorpus() Format = FileFormat.Paratext, TextId = "file4.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } } @@ -1327,7 +1399,9 @@ await env.Service.StartBuildAsync( "MRK", new ScriptureChapters { } } - } + }, + PretranslateAll = true, + TrainOnAll = false }, new V1.MonolingualCorpus() { @@ -1341,7 +1415,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file3.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } }, TargetCorpora = @@ -1358,7 +1434,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file2.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true }, new V1.MonolingualCorpus() { @@ -1372,7 +1450,9 @@ await env.Service.StartBuildAsync( Format = FileFormat.Paratext, TextId = "file4.zip" } - } + }, + PretranslateAll = true, + TrainOnAll = true } } }