From d5cc6edc717d108a135e10f779e7bfc0fd4bbf89 Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Tue, 9 Jul 2024 15:29:24 -0500 Subject: [PATCH] Revert "Move Machine engine to Serval repo (#421)" (#425) This reverts commit e62da1fb16a44298ece6e1771769730d49160ecd. --- .github/workflows/ci.yml | 4 +- Serval.sln | 36 - .../Serval.Machine.EngineServer/Program.cs | 39 - .../Properties/launchSettings.json | 11 - .../Serval.Machine.EngineServer.csproj | 34 - .../appsettings.Development.json | 21 - .../appsettings.Production.json | 8 - .../appsettings.Staging.json | 8 - .../appsettings.json | 43 - .../src/Serval.Machine.JobServer/Program.cs | 30 - .../Properties/launchSettings.json | 12 - .../Serval.Machine.JobServer.csproj | 37 - .../appsettings.Development.json | 21 - .../appsettings.Production.json | 8 - .../appsettings.Staging.json | 8 - .../Serval.Machine.JobServer/appsettings.json | 43 - .../Configuration/BuildJobOptions.cs | 8 - .../Configuration/ClearMLBuildQueue.cs | 9 - .../Configuration/ClearMLOptions.cs | 13 - .../IEndpointRouteBuilderExtensions.cs | 11 - .../Configuration/IMachineBuilder.cs | 7 - .../IMachineBuilderExtensions.cs | 437 - .../IServiceCollectionExtensions.cs | 54 - .../Configuration/MachineBuilder.cs | 7 - .../Configuration/MessageOutboxOptions.cs | 9 - .../Configuration/ServiceOptions.cs | 8 - .../Configuration/SharedFileOptions.cs | 11 - .../Configuration/SmtTransferEngineOptions.cs | 10 - .../Configuration/ThotSmtModelOptions.cs | 14 - .../src/Serval.Machine.Shared/Models/Build.cs | 32 - .../Models/ClearMLMetricsEvent.cs | 12 - .../Models/ClearMLProject.cs | 6 - .../Models/ClearMLTask.cs | 35 - .../Serval.Machine.Shared/Models/Corpus.cs | 14 - .../Models/CorpusFile.cs | 14 - .../src/Serval.Machine.Shared/Models/Lock.cs | 8 - .../Models/ModelDownloadUrl.cs | 8 - .../Serval.Machine.Shared/Models/Outbox.cs | 10 - .../Models/OutboxMessage.cs | 15 - .../Models/Pretranslation.cs | 9 - .../Serval.Machine.Shared/Models/RWLock.cs | 25 - .../Models/TrainSegmentPair.cs | 11 - .../Models/TranslationEngine.cs | 14 - .../Properties/AssemblyInfo.cs | 2 - .../Serval.Machine.Shared.csproj | 60 - .../Services/BuildJobService.cs | 211 - .../Services/BuildProgress.cs | 25 - .../Services/CancellationInterceptor.cs | 30 - .../Services/ClearMLAuthenticationService.cs | 77 - .../Services/ClearMLBuildJobRunner.cs | 88 - .../Services/ClearMLHealthCheck.cs | 100 - .../Services/ClearMLMonitorService.cs | 407 - .../Services/ClearMLService.cs | 221 - .../Services/CorpusService.cs | 51 - .../Services/DistributedReaderWriterLock.cs | 177 - .../DistributedReaderWriterLockFactory.cs | 77 - .../Services/FileSystem.cs | 25 - .../Services/HangfireBuildJob.cs | 180 - .../Services/HangfireBuildJobRunner.cs | 85 - .../Services/HangfireHealthCheck.cs | 25 - .../Services/IBuildJobRunner.cs | 29 - .../Services/IBuildJobService.cs | 41 - .../Services/IClearMLAuthenticationService.cs | 6 - .../Services/IClearMLBuildJobFactory.cs | 16 - .../Services/IClearMLQueueService.cs | 6 - .../Services/IClearMLService.cs | 30 - .../Services/ICorpusService.cs | 7 - .../Services/IDistributedReaderWriterLock.cs | 7 - .../IDistributedReaderWriterLockFactory.cs | 8 - .../Services/IFileStorage.cs | 20 - .../Services/IFileSystem.cs | 9 - .../Services/IHangfireBuildJobFactory.cs | 8 - .../Services/ILanguageTagService.cs | 6 - .../Services/IMessageOutboxService.cs | 13 - .../Services/IOutboxMessageHandler.cs | 13 - .../Services/IPlatformService.cs | 30 - .../Services/ISharedFileService.cs | 24 - .../Services/ISmtModelFactory.cs | 22 - .../Services/ITransferEngineFactory.cs | 14 - .../Services/ITranslationEngineService.cs | 49 - .../Services/ITruecaserFactory.cs | 13 - .../Services/InMemoryStorage.cs | 143 - .../Services/LanguageTagService.cs | 170 - .../Services/LocalStorage.cs | 78 - .../Services/MessageOutboxDeliveryService.cs | 174 - .../Services/MessageOutboxService.cs | 73 - .../Services/ModelCleanupService.cs | 59 - .../Services/NmtClearMLBuildJobFactory.cs | 58 - .../Services/NmtEngineService.cs | 204 - .../Services/NmtHangfireBuildJobFactory.cs | 26 - .../Services/NmtPreprocessBuildJob.cs | 31 - .../Services/PostprocessBuildJob.cs | 88 - .../Services/PreprocessBuildJob.cs | 434 - .../Services/S3FileStorage.cs | 125 - .../Services/S3HealthCheck.cs | 65 - .../Services/S3WriteStream.cs | 232 - .../Services/ServalPlatformOutboxConstants.cs | 14 - .../ServalPlatformOutboxMessageHandler.cs | 92 - .../Services/ServalPlatformService.cs | 140 - .../ServalTranslationEngineServiceV1.cs | 341 - .../Services/SharedFileService.cs | 104 - .../Services/SmtTransferBuildJob.cs | 166 - .../SmtTransferClearMLBuildJobFactory.cs | 49 - .../SmtTransferEngineCommitService.cs | 38 - .../Services/SmtTransferEngineService.cs | 272 - .../Services/SmtTransferEngineState.cs | 128 - .../Services/SmtTransferEngineStateService.cs | 72 - .../SmtTransferHangfireBuildJobFactory.cs | 33 - .../SmtTransferPostprocessBuildJob.cs | 74 - .../Services/SmtTransferTrainBuildJob.cs | 237 - .../Services/ThotSmtModelFactory.cs | 110 - .../Services/TransferEngineFactory.cs | 61 - .../Services/TranslationEngineType.cs | 7 - .../Services/UnigramTruecaserFactory.cs | 37 - .../Services/UnimplementedInterceptor.cs | 22 - .../src/Serval.Machine.Shared/Usings.cs | 59 - .../Utils/AsyncDisposableBase.cs | 19 - .../Serval.Machine.Shared/Utils/AsyncTimer.cs | 69 - .../Utils/CustomEnumConverterFactory.cs | 151 - .../Utils/DictionaryStringConverter.cs | 65 - .../Utils/EngineNotBuiltException.cs | 4 - .../Utils/RecurrentTask.cs | 40 - .../Utils/SharedFileUtils.cs | 20 - .../Utils/StartupTask.cs | 19 - .../Serval.Machine.Shared/data/.gitattributes | 5 - .../data/flores200languages.csv | 205 - .../data/thot-new-model/lm/trg.lm | 6 - .../data/thot-new-model/lm/trg.lm.weights | 1 - .../data/thot-new-model/lm/trg.lm.wp | 1 - .../data/thot-new-model/smt.cfg | 29 - .../data/thot-new-model/tm/src_trg.lambda | 1 - .../thot-new-model/tm/src_trg.seglentable | 10201 ---------------- .../thot-new-model/tm/src_trg.srcsegmlentable | 1 - .../thot-new-model/tm/src_trg.trgsegmlentable | 1 - .../data/thot-new-model/tm/src_trg.ttable | 0 .../thot-new-model/tm/src_trg_invswm.anji | 0 .../tm/src_trg_invswm.anjm1ip_anji | 0 .../tm/src_trg_invswm.asifactor | 1 - .../tm/src_trg_invswm.hmm_alignd | Bin 22 -> 0 bytes .../tm/src_trg_invswm.hmm_lexnd | Bin 32 -> 0 bytes .../thot-new-model/tm/src_trg_invswm.hmm_p0 | 1 - .../tm/src_trg_invswm.lsifactor | 1 - .../thot-new-model/tm/src_trg_invswm.msinfo | 2 - .../thot-new-model/tm/src_trg_invswm.slmodel | 3 - .../data/thot-new-model/tm/src_trg_invswm.src | 1 - .../thot-new-model/tm/src_trg_invswm.srctrgc | 1 - .../thot-new-model/tm/src_trg_invswm.svcb | 4 - .../data/thot-new-model/tm/src_trg_invswm.trg | 1 - .../thot-new-model/tm/src_trg_invswm.tvcb | 4 - .../data/thot-new-model/tm/src_trg_swm.anji | 0 .../tm/src_trg_swm.anjm1ip_anji | 0 .../thot-new-model/tm/src_trg_swm.asifactor | 1 - .../thot-new-model/tm/src_trg_swm.hmm_alignd | Bin 22 -> 0 bytes .../thot-new-model/tm/src_trg_swm.hmm_lexnd | Bin 32 -> 0 bytes .../data/thot-new-model/tm/src_trg_swm.hmm_p0 | 1 - .../thot-new-model/tm/src_trg_swm.lsifactor | 1 - .../data/thot-new-model/tm/src_trg_swm.msinfo | 2 - .../thot-new-model/tm/src_trg_swm.slmodel | 3 - .../data/thot-new-model/tm/src_trg_swm.src | 1 - .../thot-new-model/tm/src_trg_swm.srctrgc | 1 - .../data/thot-new-model/tm/src_trg_swm.svcb | 4 - .../data/thot-new-model/tm/src_trg_swm.trg | 1 - .../data/thot-new-model/tm/src_trg_swm.tvcb | 4 - .../Serval.Machine.Shared.Tests.csproj | 60 - .../Services/ClearMLServiceTests.cs | 50 - ...DistributedReaderWriterLockFactoryTests.cs | 81 - .../DistributedReaderWriterLockTests.cs | 405 - .../Services/InMemoryStorageTests.cs | 91 - .../Services/LanguageTagServiceTests.cs | 69 - .../Services/LocalStorageTests.cs | 96 - .../MessageOutboxDeliveryServiceTests.cs | 214 - .../Services/MessageOutboxServiceTests.cs | 93 - .../Services/ModelCleanupServiceTests.cs | 104 - .../NmtClearMLBuildJobFactoryTests.cs | 123 - .../Services/NmtEngineServiceTests.cs | 324 - .../Services/PreprocessBuildJobTests.cs | 581 - ...ServalPlatformOutboxMessageHandlerTests.cs | 113 - .../Services/SmtTransferEngineServiceTests.cs | 740 -- .../Services/data/pt-source1/04LEVTe1.SFM | 8 - .../Services/data/pt-source1/131CHTe1.SFM | 12 - .../Services/data/pt-source1/41MATTe1.SFM | 16 - .../Services/data/pt-source1/42MRKTe1.SFM | 4 - .../Services/data/pt-source1/Settings.xml | 34 - .../data/pt-source1/TermRenderings.xml | 9 - .../Services/data/pt-source1/custom.vrs | 31 - .../Services/data/pt-source2/04LEVTe3.SFM | 7 - .../Services/data/pt-source2/131CHTe3.SFM | 16 - .../Services/data/pt-source2/41MATTe3.SFM | 19 - .../Services/data/pt-source2/42MRKTe3.SFM | 7 - .../Services/data/pt-source2/Settings.xml | 34 - .../data/pt-source2/TermRenderings.xml | 9 - .../Services/data/pt-source2/custom.vrs | 31 - .../Services/data/pt-target1/41MATTe2.SFM | 17 - .../Services/data/pt-target1/42MRKTe2.SFM | 4 - .../Services/data/pt-target1/Settings.xml | 33 - .../data/pt-target1/TermRenderings.xml | 9 - .../Services/data/pt-target1/custom.vrs | 31 - .../Services/data/source1.txt | 7 - .../Services/data/source2.txt | 7 - .../Services/data/target1.txt | 7 - .../Serval.Machine.Shared.Tests/Usings.cs | 28 - 201 files changed, 2 insertions(+), 21800 deletions(-) delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/Program.cs delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json delete mode 100644 src/Machine/src/Serval.Machine.EngineServer/appsettings.json delete mode 100644 src/Machine/src/Serval.Machine.JobServer/Program.cs delete mode 100644 src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json delete mode 100644 src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj delete mode 100644 src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json delete mode 100644 src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json delete mode 100644 src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json delete mode 100644 src/Machine/src/Serval.Machine.JobServer/appsettings.json delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/Build.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/Lock.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Usings.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/.gitattributes delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.ttable delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anji delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anji delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg delete mode 100644 src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt delete mode 100644 src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e335e757..124ecbf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Setup .NET - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v3 with: dotnet-version: 8.0.x - name: Start MongoDB diff --git a/Serval.sln b/Serval.sln index 8188624d..6cb275c0 100644 --- a/Serval.sln +++ b/Serval.sln @@ -64,20 +64,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{92805246-528 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{BA044B98-3136-4FDE-B90F-B0975758C07F}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Machine", "Machine", "{F6142E52-4B58-4D12-980F-B07D8AA932C2}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared", "src\Machine\src\Serval.Machine.Shared\Serval.Machine.Shared.csproj", "{090ECB69-464F-42C8-B92C-0808BE2802FA}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.EngineServer", "src\Machine\src\Serval.Machine.EngineServer\Serval.Machine.EngineServer.csproj", "{C02494FB-663E-4430-9F2D-41F1A740B271}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer", "src\Machine\src\Serval.Machine.JobServer\Serval.Machine.JobServer.csproj", "{BC766753-E560-4ADF-9923-C7A96076EA47}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -148,22 +134,6 @@ Global {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.Build.0 = Release|Any CPU - {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.Build.0 = Release|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.Build.0 = Release|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.Build.0 = Release|Any CPU - {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -190,12 +160,6 @@ Global {3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B} {92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF} {BA044B98-3136-4FDE-B90F-B0975758C07F} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF} - {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} - {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} - {090ECB69-464F-42C8-B92C-0808BE2802FA} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} - {C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} - {BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} - {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/Machine/src/Serval.Machine.EngineServer/Program.cs b/src/Machine/src/Serval.Machine.EngineServer/Program.cs deleted file mode 100644 index 029e03df..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Program.cs +++ /dev/null @@ -1,39 +0,0 @@ -using Hangfire; -using OpenTelemetry.Trace; - -var builder = WebApplication.CreateBuilder(args); - -// Add services to the container. -builder - .Services.AddMachine(builder.Configuration) - .AddBuildJobService() - .AddMongoDataAccess() - .AddMongoHangfireJobClient() - .AddServalTranslationEngineService() - .AddModelCleanupService() - .AddMessageOutboxDeliveryService() - .AddClearMLService(); - -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -var app = builder.Build(); - -app.UseHttpsRedirection(); - -app.MapServalTranslationEngineService(); -app.MapHangfireDashboard(); - -app.Run(); diff --git a/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json deleted file mode 100644 index 34eb2e94..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.EngineServer": { - "commandName": "Project", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9000" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj b/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj deleted file mode 100644 index 89f4d8e6..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj +++ /dev/null @@ -1,34 +0,0 @@ - - - - net8.0 - enable - enable - 34e222a9-ef76-48f9-869e-338547f9bd25 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json deleted file mode 100644 index 1f2a4ef6..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://aqua-ml-data/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json deleted file mode 100644 index 1b2d3baf..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json deleted file mode 100644 index 1b2d3baf..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json deleted file mode 100644 index 271163ff..00000000 --- a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_engine" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "BuildJob": { - "ClearML": [ - { - "TranslationEngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "TranslationEngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ] - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": true - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Program.cs b/src/Machine/src/Serval.Machine.JobServer/Program.cs deleted file mode 100644 index d78bfed8..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/Program.cs +++ /dev/null @@ -1,30 +0,0 @@ -using OpenTelemetry.Trace; - -var builder = WebApplication.CreateBuilder(args); - -builder - .Services.AddMachine(builder.Configuration) - .AddBuildJobService() - .AddMongoDataAccess() - .AddMongoHangfireJobClient() - .AddHangfireJobServer() - .AddServalPlatformService() - .AddClearMLService(); -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -var app = builder.Build(); - -app.Run(); diff --git a/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json deleted file mode 100644 index f636d0c3..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.JobServer": { - "commandName": "Project", - "launchBrowser": false, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9100" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj b/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj deleted file mode 100644 index 8a466b1d..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - net8.0 - enable - enable - aa9e7440-5a04-4de6-ba51-bab9ef4a62e1 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json deleted file mode 100644 index 1f2a4ef6..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://aqua-ml-data/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json deleted file mode 100644 index 1b2d3baf..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json deleted file mode 100644 index 1b2d3baf..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.json deleted file mode 100644 index 738a4f28..00000000 --- a/src/Machine/src/Serval.Machine.JobServer/appsettings.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_job" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "BuildJob": { - "ClearML": [ - { - "TranslationEngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "TranslationEngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ] - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": false - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs deleted file mode 100644 index 547a9dbd..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class BuildJobOptions -{ - public const string Key = "BuildJob"; - - public IList ClearML { get; set; } = new List(); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs deleted file mode 100644 index 53e25245..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class ClearMLBuildQueue -{ - public TranslationEngineType TranslationEngineType { get; set; } - public string ModelType { get; set; } = ""; - public string Queue { get; set; } = "default"; - public string DockerImage { get; set; } = ""; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs deleted file mode 100644 index e72b7dec..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class ClearMLOptions -{ - public const string Key = "ClearML"; - - public string AccessKey { get; set; } = ""; - public string SecretKey { get; set; } = ""; - public bool BuildPollingEnabled { get; set; } = false; - public TimeSpan BuildPollingTimeout { get; set; } = TimeSpan.FromSeconds(10); - public string RootProject { get; set; } = "Machine"; - public string Project { get; set; } = "dev"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs deleted file mode 100644 index 694dd67e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Microsoft.AspNetCore.Builder; - -public static class IEndpointRouteBuilderExtensions -{ - public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - - return builder; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs deleted file mode 100644 index f8dfbcd5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public interface IMachineBuilder -{ - IServiceCollection Services { get; } - IConfiguration? Configuration { get; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs deleted file mode 100644 index 567a073e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ /dev/null @@ -1,437 +0,0 @@ -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddMessageOutboxOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddMessageOutboxOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddBuildJobOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddBuildJobOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - if (builder.Configuration is null) - return builder.AddThotSmtModel(o => { }); - else - return builder.AddThotSmtModel(builder.Configuration.GetSection(ThotSmtModelOptions.Key)); - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration?.GetConnectionString("ClearML"); - if (connectionString is null) - throw new InvalidOperationException("ClearML connection string is required"); - - builder - .Services.AddHttpClient("ClearML") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) - // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds - .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - builder.Services.AddSingleton(); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder - .Services.AddHttpClient("ClearML-NoRetry") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); - builder.Services.AddSingleton(); - - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - return builder; - } - - private static MongoStorageOptions GetMongoStorageOptions() - { - var mongoStorageOptions = new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - }; - return mongoStorageOptions; - } - - public static IMachineBuilder AddMongoHangfireJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - connectionString ??= builder.Configuration?.GetConnectionString("Hangfire"); - if (connectionString is null) - throw new InvalidOperationException("Hangfire connection string is required"); - - builder.Services.AddHangfire(c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage(connectionString, GetMongoStorageOptions()) - .UseFilter(new AutomaticRetryAttribute { Attempts = 0 }) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddHangfireJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration?.GetConnectionString("Mongo"); - if (connectionString is null) - throw new InvalidOperationException("Mongo connection string is required"); - builder.Services.AddMongoDataAccess( - connectionString!, - "Serval.Machine.Shared.Models", - o => - { - o.AddRepository( - "translation_engines", - mapSetup: m => m.SetIgnoreExtraElements(true), - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(e => e.EngineId) - .Ascending("currentBuild._id") - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) - ) - ); - } - ); - o.AddRepository("locks"); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - o.AddRepository( - "outbox_messages", - mapSetup: m => m.MapProperty(m => m.OutboxRef).SetSerializer(new StringSerializer()) - ); - o.AddRepository( - "outboxes", - mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - connectionString ??= builder.Configuration?.GetConnectionString("Serval"); - if (connectionString is null) - throw new InvalidOperationException("Serval connection string is required"); - - builder.Services.AddScoped(); - - builder.Services.AddSingleton(); - - builder.Services.AddScoped(); - - builder - .Services.AddGrpcClient(o => - { - o.Address = new Uri(connectionString); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => - { - options.Interceptors.Add(); - options.Interceptors.Add(); - }); - builder.AddServalPlatformService(connectionString); - - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.Services.AddScoped(); - break; - } - } - - return builder; - } - - public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, string? smtTransferEngineDir = null) - { - builder.Services.AddScoped(); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(x => x.GetRequiredService()); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - if (smtTransferEngineDir is null) - { - var smtTransferEngineOptions = new SmtTransferEngineOptions(); - builder.Configuration?.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); - smtTransferEngineDir = smtTransferEngineOptions.EnginesDir; - } - string? driveLetter = Path.GetPathRoot(smtTransferEngineDir)?[..1]; - if (driveLetter is null) - throw new InvalidOperationException("SMT Engine directory is required"); - // add health check for disk storage capacity - builder - .Services.AddHealthChecks() - .AddDiskStorageHealthCheck( - x => x.AddDrive(driveLetter, 1_000), // 1GB - "SMT Engine Storage Capacity", - HealthStatus.Degraded - ); - - return builder; - } - - public static IMachineBuilder AddModelCleanupService(this IMachineBuilder builder) - { - builder.Services.AddHostedService(); - return builder; - } - - public static IMachineBuilder AddMessageOutboxDeliveryService(this IMachineBuilder builder) - { - builder.Services.AddHostedService(); - return builder; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs deleted file mode 100644 index 7463e6ac..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServiceCollectionExtensions -{ - public static IMachineBuilder AddMachine(this IServiceCollection services, IConfiguration? configuration = null) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddHealthChecks().AddCheck("S3 Bucket"); - - services.AddSingleton(); - services.AddTransient(); - - services.AddScoped(); - services.AddSingleton(); - services.AddStartupTask( - (sp, cancellationToken) => - sp.GetRequiredService().InitAsync(cancellationToken) - ); - - var builder = new MachineBuilder(services, configuration); - if (configuration is null) - { - builder.AddServiceOptions(o => { }); - builder.AddSharedFileOptions(o => { }); - builder.AddSmtTransferEngineOptions(o => { }); - builder.AddClearMLOptions(o => { }); - builder.AddBuildJobOptions(o => { }); - builder.AddMessageOutboxOptions(o => { }); - } - else - { - builder.AddServiceOptions(configuration.GetSection(ServiceOptions.Key)); - builder.AddSharedFileOptions(configuration.GetSection(SharedFileOptions.Key)); - builder.AddSmtTransferEngineOptions(configuration.GetSection(SmtTransferEngineOptions.Key)); - builder.AddClearMLOptions(configuration.GetSection(ClearMLOptions.Key)); - builder.AddBuildJobOptions(configuration.GetSection(BuildJobOptions.Key)); - builder.AddMessageOutboxOptions(configuration.GetSection(MessageOutboxOptions.Key)); - } - return builder; - } - - public static IServiceCollection AddStartupTask( - this IServiceCollection services, - Func startupTask - ) - { - services.AddHostedService(sp => new StartupTask(sp, startupTask)); - return services; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs deleted file mode 100644 index 58ddf5c1..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -internal class MachineBuilder(IServiceCollection services, IConfiguration? configuration) : IMachineBuilder -{ - public IServiceCollection Services { get; } = services; - public IConfiguration? Configuration { get; } = configuration; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs deleted file mode 100644 index e2e88feb..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class MessageOutboxOptions -{ - public const string Key = "MessageOutbox"; - - public string OutboxDir { get; set; } = "outbox"; - public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs deleted file mode 100644 index 8011e7b7..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class ServiceOptions -{ - public const string Key = "Service"; - - public string ServiceId { get; set; } = "machine_api"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs deleted file mode 100644 index 4ae27e1e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class SharedFileOptions -{ - public const string Key = "SharedFile"; - - public string Uri { get; set; } = "file:///var/lib/machine/"; - public string S3AccessKeyId { get; set; } = ""; - public string S3SecretAccessKey { get; set; } = ""; - public string S3Region { get; set; } = "us-east-1"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs deleted file mode 100644 index 15002604..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class SmtTransferEngineOptions -{ - public const string Key = "SmtTransferEngine"; - - public string EnginesDir { get; set; } = "translation_engines"; - public TimeSpan EngineCommitFrequency { get; set; } = TimeSpan.FromMinutes(5); - public TimeSpan InactiveEngineTimeout { get; set; } = TimeSpan.FromMinutes(10); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs deleted file mode 100644 index 780eb7d2..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Configuration; - -public class ThotSmtModelOptions -{ - public const string Key = "ThotSmtModel"; - - public ThotSmtModelOptions() - { - string installDir = Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location)!; - NewModelFile = Path.Combine(installDir, "thot-new-model.zip"); - } - - public string NewModelFile { get; set; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs deleted file mode 100644 index aca20540..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs +++ /dev/null @@ -1,32 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public enum BuildJobState -{ - None, - Pending, - Active, - Canceling -} - -public enum BuildJobRunnerType -{ - Hangfire, - ClearML -} - -public enum BuildStage -{ - Preprocess, - Train, - Postprocess -} - -public record Build -{ - public required string BuildId { get; init; } - public required BuildJobState JobState { get; init; } - public required string JobId { get; init; } - public required BuildJobRunnerType BuildJobRunner { get; init; } - public required BuildStage Stage { get; init; } - public string? Options { get; set; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs deleted file mode 100644 index 5ae9fbfd..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record ClearMLMetricsEvent -{ - public string? Metric { get; init; } - public string? Variant { get; init; } - public required double Value { get; init; } - public double? MinValue { get; init; } - public int? MinValueIteration { get; init; } - public double? MaxValue { get; init; } - public int? MaxValueIteration { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs deleted file mode 100644 index c0cd0d7e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record ClearMLProject -{ - public required string Id { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs deleted file mode 100644 index 5b13fdaa..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public enum ClearMLTaskStatus -{ - Created, - Queued, - InProgress, - Stopped, - Published, - Publishing, - Closed, - Failed, - Completed, - Unknown -} - -public record ClearMLTask -{ - public required string Id { get; init; } - public required string Name { get; init; } - public required ClearMLProject Project { get; init; } - public required ClearMLTaskStatus Status { get; init; } - public string? StatusReason { get; init; } - public string? StatusMessage { get; init; } - public required DateTime Created { get; init; } - public int? LastIteration { get; init; } - public int ActiveDuration { get; init; } - public required IReadOnlyDictionary< - string, - IReadOnlyDictionary - > LastMetrics { get; init; } - - [JsonConverter(typeof(DictionaryStringStringConverter))] - public required IReadOnlyDictionary Runtime { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs b/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs deleted file mode 100644 index 9145e90d..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record Corpus -{ - public required string Id { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public IReadOnlyDictionary>? TrainOnChapters { get; init; } - public IReadOnlyDictionary>? PretranslateChapters { get; init; } - public required HashSet? TrainOnTextIds { get; init; } - public required HashSet? PretranslateTextIds { get; init; } - public required IReadOnlyList SourceFiles { get; init; } - public required IReadOnlyList TargetFiles { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs b/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs deleted file mode 100644 index a84bf7f6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public enum FileFormat -{ - Text = 0, - Paratext = 1 -} - -public record CorpusFile -{ - public required string Location { get; init; } - public required FileFormat Format { get; init; } - public required string TextId { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs b/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs deleted file mode 100644 index 39ceae87..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record Lock -{ - public required string Id { get; init; } - public DateTime? ExpiresAt { get; init; } - public required string HostId { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs b/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs deleted file mode 100644 index 798fe175..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record ModelDownloadUrl -{ - public required string Url { get; init; } - public required int ModelRevision { get; init; } - public required DateTime ExpiresAt { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs b/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs deleted file mode 100644 index ad9c0001..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record Outbox : IEntity -{ - public string Id { get; set; } = ""; - - public int Revision { get; set; } - - public int CurrentIndex { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs b/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs deleted file mode 100644 index 0e95e9a6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record OutboxMessage : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required int Index { get; init; } - public required string OutboxRef { get; init; } - public required string Method { get; init; } - public required string GroupId { get; init; } - public string? Content { get; init; } - public required bool HasContentStream { get; init; } - public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow; - public int Attempts { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs b/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs deleted file mode 100644 index 6e9807b5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record Pretranslation -{ - public required string CorpusId { get; init; } - public required string TextId { get; init; } - public required IReadOnlyList Refs { get; init; } - public required string Translation { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs b/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs deleted file mode 100644 index 2271aa9b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record RWLock : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public Lock? WriterLock { get; init; } - public required IReadOnlyList ReaderLocks { get; init; } - public required IReadOnlyList WriterQueue { get; init; } - - public bool IsAvailableForReading() - { - var now = DateTime.UtcNow; - return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) - && WriterQueue.Count == 0; - } - - public bool IsAvailableForWriting(string? lockId = null) - { - var now = DateTime.UtcNow; - return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) - && !ReaderLocks.Any(l => l.ExpiresAt is null || l.ExpiresAt > now) - && (lockId is null || WriterQueue.Count > 0 && WriterQueue[0].Id == lockId); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs b/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs deleted file mode 100644 index 30927345..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record TrainSegmentPair : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required string TranslationEngineRef { get; init; } - public required string Source { get; init; } - public required string Target { get; init; } - public required bool SentenceStart { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs b/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs deleted file mode 100644 index 80b1f648..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Models; - -public record TranslationEngine : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required string EngineId { get; init; } - public required TranslationEngineType Type { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public required bool IsModelPersisted { get; init; } - public int BuildRevision { get; init; } - public Build? CurrentBuild { get; init; } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs b/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs deleted file mode 100644 index 54a4902d..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,2 +0,0 @@ -[assembly: InternalsVisibleTo("Serval.Machine.Shared.Tests")] -[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj deleted file mode 100644 index 6b716479..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj +++ /dev/null @@ -1,60 +0,0 @@ - - - - net8.0 - An ASP.NET Core web API middleware for the Machine library. - enable - enable - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - PreserveNewest - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs deleted file mode 100644 index 244aa04a..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs +++ /dev/null @@ -1,211 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class BuildJobService(IEnumerable runners, IRepository engines) - : IBuildJobService -{ - private readonly Dictionary _runners = runners.ToDictionary(r => r.Type); - private readonly IRepository _engines = engines; - - public Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default) - { - return _engines.ExistsAsync(e => e.EngineId == engineId && e.CurrentBuild != null, cancellationToken); - } - - public Task> GetBuildingEnginesAsync( - BuildJobRunnerType runner, - CancellationToken cancellationToken = default - ) - { - return _engines.GetAllAsync( - e => e.CurrentBuild != null && e.CurrentBuild.BuildJobRunner == runner, - cancellationToken - ); - } - - public async Task GetBuildAsync( - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - cancellationToken - ); - return engine?.CurrentBuild; - } - - public async Task CreateEngineAsync( - string engineId, - string? name = null, - CancellationToken cancellationToken = default - ) - { - foreach (BuildJobRunnerType runnerType in _runners.Keys) - { - IBuildJobRunner runner = _runners[runnerType]; - await runner.CreateEngineAsync(engineId, name, cancellationToken); - } - } - - public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - foreach (BuildJobRunnerType runnerType in _runners.Keys) - { - IBuildJobRunner runner = _runners[runnerType]; - await runner.DeleteEngineAsync(engineId, cancellationToken); - } - } - - public async Task StartBuildJobAsync( - BuildJobRunnerType runnerType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => - e.EngineId == engineId - && (e.CurrentBuild == null || e.CurrentBuild.JobState != BuildJobState.Canceling), - cancellationToken - ); - if (engine is null) - return false; - - IBuildJobRunner runner = _runners[runnerType]; - string jobId = await runner.CreateJobAsync( - engine.Type, - engineId, - buildId, - stage, - data, - buildOptions, - cancellationToken - ); - try - { - await _engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set( - e => e.CurrentBuild, - new Build - { - BuildId = buildId, - JobId = jobId, - BuildJobRunner = runner.Type, - Stage = stage, - JobState = BuildJobState.Pending, - Options = buildOptions - } - ), - cancellationToken: cancellationToken - ); - await runner.EnqueueJobAsync(jobId, engine.Type, cancellationToken); - return true; - } - catch - { - await runner.DeleteJobAsync(jobId, CancellationToken.None); - throw; - } - } - - public async Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - cancellationToken - ); - if (engine is null || engine.CurrentBuild is null) - return (null, BuildJobState.None); - - IBuildJobRunner runner = _runners[engine.CurrentBuild.BuildJobRunner]; - - if (engine.CurrentBuild.JobState is BuildJobState.Pending) - { - // cancel a job that hasn't started yet - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - u => u.Unset(b => b.CurrentBuild), - returnOriginal: true, - cancellationToken: cancellationToken - ); - if (engine is not null && engine.CurrentBuild is not null) - { - // job will be deleted from the queue - await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); - return (engine.CurrentBuild.BuildId, BuildJobState.None); - } - } - else if (engine.CurrentBuild.JobState is BuildJobState.Active) - { - // cancel a job that is already running - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Canceling), - cancellationToken: cancellationToken - ); - if (engine is not null && engine.CurrentBuild is not null) - { - await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); - return (engine.CurrentBuild.BuildId, BuildJobState.Canceling); - } - } - - return (null, BuildJobState.None); - } - - public async Task BuildJobStartedAsync( - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.UpdateAsync( - e => - e.EngineId == engineId - && e.CurrentBuild != null - && e.CurrentBuild.BuildId == buildId - && e.CurrentBuild.JobState == BuildJobState.Pending, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Active), - cancellationToken: cancellationToken - ); - return engine is not null; - } - - public Task BuildJobFinishedAsync( - string engineId, - string buildId, - bool buildComplete, - CancellationToken cancellationToken = default - ) - { - return _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - u => - { - u.Unset(e => e.CurrentBuild); - if (buildComplete) - u.Inc(e => e.BuildRevision); - }, - cancellationToken: cancellationToken - ); - } - - public Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default) - { - return _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Pending), - cancellationToken: cancellationToken - ); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs deleted file mode 100644 index 88422c6c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class BuildProgress(IPlatformService platformService, string buildId) : IProgress -{ - private readonly IPlatformService _platformService = platformService; - private readonly string _buildId = buildId; - private ProgressStatus _prevStatus; - - private DateTime _lastReportTime = DateTime.Now; - - private const float ThrottleTimeSeconds = 1; - - public void Report(ProgressStatus value) - { - if (_prevStatus.Equals(value)) - return; - - if (DateTime.Now < _lastReportTime.AddSeconds(ThrottleTimeSeconds)) - return; - - _lastReportTime = DateTime.Now; - _platformService.UpdateBuildStatusAsync(_buildId, value); - _prevStatus = value; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs deleted file mode 100644 index 73b06c2b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class CancellationInterceptor(ILogger logger) : Interceptor -{ - private readonly ILogger _logger = logger; - - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (Exception ex) - { - if (ex is OperationCanceledException) - { - _logger.LogInformation("An operation was canceled."); - throw new RpcException(new Status(StatusCode.Cancelled, "An operation was canceled.")); - } - else - { - throw; - } - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs deleted file mode 100644 index 9603aeb6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ClearMLAuthenticationService( - IServiceProvider services, - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - ILogger logger -) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); - private readonly IOptionsMonitor _options = options; - private readonly ILogger _logger = logger; - private readonly AsyncLock _lock = new(); - - // technically, the token should be good for 30 days, but let's refresh each hour - // to know well ahead of time if something is wrong. - private static readonly TimeSpan RefreshPeriod = TimeSpan.FromSeconds(3600); - private string _authToken = ""; - - public async Task GetAuthTokenAsync(CancellationToken cancellationToken = default) - { - using (await _lock.LockAsync(cancellationToken)) - { - if (_authToken is "") - { - //Should only happen once, so no different in cost than previous solution - _logger.LogInformation("Token was empty; refreshing"); - await AuthorizeAsync(cancellationToken); - } - } - return _authToken; - } - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - using (await _lock.LockAsync(cancellationToken)) - await AuthorizeAsync(cancellationToken); - } - catch (Exception e) - { - if (_authToken is "") - { - _logger.LogError(e, "Error occurred while acquiring ClearML authentication token for the first time."); - // The ClearML token never was set. We can't continue without it. - throw; - } - else - { - _logger.LogError(e, "Error occurred while refreshing ClearML authentication token."); - } - } - } - - private async Task AuthorizeAsync(CancellationToken cancellationToken) - { - var request = new HttpRequestMessage(HttpMethod.Post, "auth.login") - { - Content = new StringContent("{}", Encoding.UTF8, "application/json") - }; - var authenticationString = $"{_options.CurrentValue.AccessKey}:{_options.CurrentValue.SecretKey}"; - var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString)); - request.Headers.Add("Authorization", $"Basic {base64EncodedAuthenticationString}"); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - string? refreshedToken = (string?)((JsonObject?)JsonNode.Parse(result))?["data"]?["token"]; - if (refreshedToken is null || refreshedToken is "") - { - throw new InvalidOperationException( - $"ClearML authentication failed - {response.StatusCode}: {response.ReasonPhrase}" - ); - } - _authToken = refreshedToken; - _logger.LogInformation("ClearML Authentication Token Refresh Successful."); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs deleted file mode 100644 index 910dd957..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs +++ /dev/null @@ -1,88 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ClearMLBuildJobRunner( - IClearMLService clearMLService, - IEnumerable buildJobFactories, - IOptionsMonitor options -) : IBuildJobRunner -{ - private readonly IClearMLService _clearMLService = clearMLService; - private readonly Dictionary _buildJobFactories = - buildJobFactories.ToDictionary(f => f.EngineType); - - private readonly Dictionary _options = - options.CurrentValue.ClearML.ToDictionary(o => o.TranslationEngineType); - - public BuildJobRunnerType Type => BuildJobRunnerType.ClearML; - - public async Task CreateEngineAsync( - string engineId, - string? name = null, - CancellationToken cancellationToken = default - ) - { - await _clearMLService.CreateProjectAsync(engineId, name, cancellationToken); - } - - public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, cancellationToken); - } - - public async Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - projectId ??= await _clearMLService.CreateProjectAsync(engineId, cancellationToken: cancellationToken); - - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (task is not null) - return task.Id; - - IClearMLBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; - string script = await buildJobFactory.CreateJobScriptAsync( - engineId, - buildId, - _options[engineType].ModelType, - stage, - data, - buildOptions, - cancellationToken - ); - return await _clearMLService.CreateTaskAsync( - buildId, - projectId, - script, - _options[engineType].DockerImage, - cancellationToken - ); - } - - public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return _clearMLService.DeleteTaskAsync(jobId, cancellationToken); - } - - public Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ) - { - return _clearMLService.EnqueueTaskAsync(jobId, _options[engineType].Queue, cancellationToken); - } - - public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return _clearMLService.StopTaskAsync(jobId, cancellationToken); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs deleted file mode 100644 index 929b14ed..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs +++ /dev/null @@ -1,100 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ClearMLHealthCheck( - IClearMLAuthenticationService clearMLAuthenticationService, - IHttpClientFactory httpClientFactory, - IOptionsMonitor buildJobOptions -) : IHealthCheck -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML-NoRetry"); - private readonly IClearMLAuthenticationService _clearMLAuthenticationService = clearMLAuthenticationService; - private readonly ISet _queuesMonitored = buildJobOptions - .CurrentValue.ClearML.Select(x => x.Queue) - .ToHashSet(); - - private int _numConsecutiveFailures = 0; - private readonly AsyncLock _lock = new AsyncLock(); - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - try - { - if (!await PingAsync(cancellationToken)) - return HealthCheckResult.Unhealthy("ClearML is unresponsive"); - IReadOnlySet queuesWithoutWorkers = await QueuesWithoutWorkers(cancellationToken); - if (queuesWithoutWorkers.Count > 0) - { - return HealthCheckResult.Unhealthy( - $"No ClearML agents are available for configured queues: {string.Join(", ", queuesWithoutWorkers)}" - ); - } - - using (await _lock.LockAsync(cancellationToken)) - _numConsecutiveFailures = 0; - return HealthCheckResult.Healthy("ClearML is available"); - } - catch (Exception e) - { - using (await _lock.LockAsync(cancellationToken)) - { - _numConsecutiveFailures++; - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy(exception: e) - : HealthCheckResult.Degraded(exception: e); - } - } - } - - private async Task CallAsync( - string service, - string action, - JsonNode body, - CancellationToken cancellationToken = default - ) - { - var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") - { - Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") - }; - request.Headers.Add( - "Authorization", - $"Bearer {await _clearMLAuthenticationService.GetAuthTokenAsync(cancellationToken)}" - ); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - return (JsonObject?)JsonNode.Parse(result); - } - - public async Task PingAsync(CancellationToken cancellationToken = default) - { - JsonObject? result = await CallAsync("debug", "ping", new JsonObject(), cancellationToken); - return result is not null; - } - - public async Task> QueuesWithoutWorkers(CancellationToken cancellationToken = default) - { - var queuesWithoutWorkers = _queuesMonitored.ToHashSet(); - JsonObject? result = await CallAsync("workers", "get_all", new JsonObject(), cancellationToken); - JsonNode? workers_node = result?["data"]?["workers"]; - if (workers_node is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - var workers = (JsonArray)workers_node; - foreach (var worker in workers) - { - JsonNode? queues_node = worker?["queues"]; - if (queues_node is null) - continue; - var queues = (JsonArray)queues_node; - foreach (var currentQueue in queues) - { - string? currentQueueName = (string?)currentQueue?["name"]; - if (currentQueueName is not null) - queuesWithoutWorkers.Remove(currentQueueName); - } - } - return queuesWithoutWorkers; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs deleted file mode 100644 index f577fdce..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ /dev/null @@ -1,407 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ClearMLMonitorService( - IServiceProvider services, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor clearMLOptions, - IOptionsMonitor buildJobOptions, - ILogger logger -) - : RecurrentTask( - "ClearML monitor service", - services, - clearMLOptions.CurrentValue.BuildPollingTimeout, - logger, - clearMLOptions.CurrentValue.BuildPollingEnabled - ), - IClearMLQueueService -{ - private static readonly string SummaryMetric = CreateMD5("Summary"); - private static readonly string TrainCorpusSizeVariant = CreateMD5("train_corpus_size"); - private static readonly string ConfidenceVariant = CreateMD5("confidence"); - - private readonly IClearMLService _clearMLService = clearMLService; - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILogger _logger = logger; - private readonly Dictionary _curBuildStatus = new(); - - private readonly IReadOnlyDictionary _queuePerEngineType = - buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => x.Queue); - - private readonly IDictionary _queueSizePerEngineType = new ConcurrentDictionary< - TranslationEngineType, - int - >(buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => 0)); - - public int GetQueueSize(TranslationEngineType engineType) - { - return _queueSizePerEngineType[engineType]; - } - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - var buildJobService = scope.ServiceProvider.GetRequiredService(); - IReadOnlyList trainingEngines = await buildJobService.GetBuildingEnginesAsync( - BuildJobRunnerType.ClearML, - cancellationToken - ); - if (trainingEngines.Count == 0) - return; - - Dictionary tasks = new(); - Dictionary queuePositions = new(); - - foreach (TranslationEngineType engineType in _queuePerEngineType.Keys) - { - var tasksPerEngineType = ( - await _clearMLService.GetTasksByIdAsync( - trainingEngines.Select(e => e.CurrentBuild!.JobId), - cancellationToken - ) - ) - .UnionBy( - await _clearMLService.GetTasksForQueueAsync(_queuePerEngineType[engineType], cancellationToken), - t => t.Id - ) - .ToDictionary(t => t.Id); - // add new keys to dictionary - foreach (KeyValuePair kvp in tasksPerEngineType) - tasks.TryAdd(kvp.Key, kvp.Value); - - var queuePositionsPerEngineType = tasksPerEngineType - .Values.Where(t => t.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created) - .OrderBy(t => t.Created) - .Select((t, i) => (Position: i, Task: t)) - .ToDictionary(e => e.Task.Name, e => e.Position); - // add new keys to dictionary - foreach (KeyValuePair kvp in queuePositionsPerEngineType) - queuePositions.TryAdd(kvp.Key, kvp.Value); - - _queueSizePerEngineType[engineType] = queuePositionsPerEngineType.Count; - } - - var dataAccessContext = scope.ServiceProvider.GetRequiredService(); - var platformService = scope.ServiceProvider.GetRequiredService(); - var lockFactory = scope.ServiceProvider.GetRequiredService(); - foreach (TranslationEngine engine in trainingEngines) - { - if (engine.CurrentBuild is null || !tasks.TryGetValue(engine.CurrentBuild.JobId, out ClearMLTask? task)) - continue; - - if ( - engine.CurrentBuild.JobState is BuildJobState.Pending - && task.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created - ) - { - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(step: 0, percentCompleted: 0.0), - //CurrentBuild.BuildId should always equal the corresponding task.Name - queuePositions[engine.CurrentBuild.BuildId] + 1, - cancellationToken - ); - } - - if (engine.CurrentBuild.Stage == BuildStage.Train) - { - if ( - engine.CurrentBuild.JobState is BuildJobState.Pending - && task.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - bool canceled = !await TrainJobStartedAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - if (canceled) - continue; - } - - switch (task.Status) - { - case ClearMLTaskStatus.InProgress: - { - double? percentCompleted = null; - if (task.Runtime.TryGetValue("progress", out string? progressStr)) - percentCompleted = int.Parse(progressStr, CultureInfo.InvariantCulture) / 100.0; - task.Runtime.TryGetValue("message", out string? message); - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted, message), - queueDepth: 0, - cancellationToken - ); - break; - } - - case ClearMLTaskStatus.Completed: - { - task.Runtime.TryGetValue("message", out string? message); - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted: 1.0, message), - queueDepth: 0, - cancellationToken - ); - bool canceling = !await TrainJobCompletedAsync( - lockFactory, - buildJobService, - engine.EngineId, - engine.CurrentBuild.BuildId, - (int)GetMetric(task, SummaryMetric, TrainCorpusSizeVariant), - GetMetric(task, SummaryMetric, ConfidenceVariant), - engine.CurrentBuild.Options, - cancellationToken - ); - if (canceling) - { - await TrainJobCanceledAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - } - break; - } - - case ClearMLTaskStatus.Stopped: - { - await TrainJobCanceledAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - break; - } - - case ClearMLTaskStatus.Failed: - { - await TrainJobFaultedAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - $"{task.StatusReason} : {task.StatusMessage}", - cancellationToken - ); - break; - } - } - } - } - } - catch (Exception e) - { - _logger.LogError(e, "Error occurred while monitoring ClearML tasks."); - } - } - - private async Task TrainJobStartedAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - bool success; - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - success = await dataAccessContext.WithTransactionAsync( - async (ct) => - { - if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, ct)) - return false; - await platformService.BuildStartedAsync(buildId, CancellationToken.None); - return true; - }, - cancellationToken: cancellationToken - ); - } - await UpdateTrainJobStatus(platformService, buildId, new ProgressStatus(0), 0, cancellationToken); - _logger.LogInformation("Build started ({BuildId})", buildId); - return success; - } - - private async Task TrainJobCompletedAsync( - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - string engineId, - string buildId, - int corpusSize, - double confidence, - string? buildOptions, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - return await buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Postprocess, - (corpusSize, confidence), - buildOptions, - cancellationToken - ); - } - } - finally - { - _curBuildStatus.Remove(buildId); - } - } - - private async Task TrainJobFaultedAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - string message, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await dataAccessContext.WithTransactionAsync( - async (ct) => - { - await platformService.BuildFaultedAsync(buildId, message, ct); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: cancellationToken - ); - } - _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message); - } - finally - { - _curBuildStatus.Remove(buildId); - } - } - - private async Task TrainJobCanceledAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await dataAccessContext.WithTransactionAsync( - async (ct) => - { - await platformService.BuildCanceledAsync(buildId, ct); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: cancellationToken - ); - } - _logger.LogInformation("Build canceled ({BuildId})", buildId); - } - finally - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e) - { - _logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - _curBuildStatus.Remove(buildId); - } - } - - private async Task UpdateTrainJobStatus( - IPlatformService platformService, - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ) - { - if ( - _curBuildStatus.TryGetValue(buildId, out ProgressStatus curProgressStatus) - && curProgressStatus.Equals(progressStatus) - ) - { - return; - } - await platformService.UpdateBuildStatusAsync(buildId, progressStatus, queueDepth, cancellationToken); - _curBuildStatus[buildId] = progressStatus; - } - - private static double GetMetric(ClearMLTask task, string metric, string variant) - { - if (!task.LastMetrics.TryGetValue(metric, out IReadOnlyDictionary? metricVariants)) - return 0; - - if (!metricVariants.TryGetValue(variant, out ClearMLMetricsEvent? metricEvent)) - return 0; - - return metricEvent.Value; - } - - private static string CreateMD5(string input) - { - byte[] inputBytes = Encoding.UTF8.GetBytes(input); - byte[] hashBytes = MD5.HashData(inputBytes); - - return Convert.ToHexString(hashBytes).ToLower(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs deleted file mode 100644 index d3d6540c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs +++ /dev/null @@ -1,221 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ClearMLService( - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - IClearMLAuthenticationService clearMLAuthService, - IHostEnvironment env -) : IClearMLService -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); - private readonly IOptionsMonitor _options = options; - private readonly IHostEnvironment _env = env; - private static readonly JsonNamingPolicy JsonNamingPolicy = new SnakeCaseJsonNamingPolicy(); - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() - { - PropertyNamingPolicy = JsonNamingPolicy, - Converters = { new CustomEnumConverterFactory(JsonNamingPolicy) } - }; - - private readonly IClearMLAuthenticationService _clearMLAuthService = clearMLAuthService; - - public async Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default) - { - var body = new JsonObject - { - ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}", - ["only_fields"] = new JsonArray("id") - }; - JsonObject? result = await CallAsync("projects", "get_all", body, cancellationToken); - var projects = (JsonArray?)result?["data"]?["projects"]; - if (projects is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - if (projects.Count == 0) - return null; - return (string?)projects[0]?["id"]; - } - - public async Task CreateProjectAsync( - string name, - string? description = null, - CancellationToken cancellationToken = default - ) - { - var body = new JsonObject - { - ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}" - }; - if (description != null) - body["description"] = description; - JsonObject? result = await CallAsync("projects", "create", body, cancellationToken); - var projectId = (string?)result?["data"]?["id"]; - if (projectId is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return projectId; - } - - public async Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject - { - ["project"] = id, - ["delete_contents"] = true, - ["force"] = true // needed if there are tasks already in that project. - }; - JsonObject? result = await CallAsync("projects", "delete", body, cancellationToken); - var deleted = (int?)result?["data"]?["deleted"]; - if (deleted is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return deleted == 1; - } - - public async Task CreateTaskAsync( - string buildId, - string projectId, - string script, - string dockerImage, - CancellationToken cancellationToken = default - ) - { - var snakeCaseEnvironment = JsonNamingPolicy.ConvertName(_env.EnvironmentName); - var body = new JsonObject - { - ["name"] = buildId, - ["project"] = projectId, - ["script"] = new JsonObject { ["diff"] = script }, - ["container"] = new JsonObject - { - ["image"] = dockerImage, - ["arguments"] = "--env ENV_FOR_DYNACONF=" + snakeCaseEnvironment, - }, - ["type"] = "training" - }; - JsonObject? result = await CallAsync("tasks", "create", body, cancellationToken); - var taskId = (string?)result?["data"]?["id"]; - if (taskId is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return taskId; - } - - public async Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id }; - JsonObject? result = await CallAsync("tasks", "delete", body, cancellationToken); - var deleted = (bool?)result?["data"]?["deleted"]; - if (deleted is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return deleted.Value; - } - - public async Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id, ["queue_name"] = queue }; - JsonObject? result = await CallAsync("tasks", "enqueue", body, cancellationToken); - var queued = (int?)result?["data"]?["queued"]; - if (queued is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return queued == 1; - } - - public async Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id }; - JsonObject? result = await CallAsync("tasks", "dequeue", body, cancellationToken); - var dequeued = (int?)result?["data"]?["dequeued"]; - if (dequeued is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return dequeued == 1; - } - - public async Task StopTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id, ["force"] = true }; - JsonObject? result = await CallAsync("tasks", "stop", body, cancellationToken); - var updated = (int?)result?["data"]?["updated"]; - if (updated is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return updated == 1; - } - - public async Task> GetTasksForQueueAsync( - string queue, - CancellationToken cancellationToken = default - ) - { - var body = new JsonObject { ["name"] = queue }; - JsonObject? result = await CallAsync("queues", "get_all_ex", body, cancellationToken); - var tasks = (JsonArray?)result?["data"]?["queues"]?[0]?["entries"]; - IEnumerable taskIds = tasks?.Select(t => (string)t?["id"]!) ?? new List(); - return await GetTasksByIdAsync(taskIds, cancellationToken); - } - - public async Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default) - { - IReadOnlyList tasks = await GetTasksAsync(new JsonObject { ["name"] = name }, cancellationToken); - if (tasks.Count == 0) - return null; - return tasks[0]; - } - - public Task> GetTasksByIdAsync( - IEnumerable ids, - CancellationToken cancellationToken = default - ) - { - return GetTasksAsync(new JsonObject { ["id"] = JsonValue.Create(ids.ToArray()) }, cancellationToken); - } - - private async Task> GetTasksAsync( - JsonObject body, - CancellationToken cancellationToken = default - ) - { - body["only_fields"] = new JsonArray( - "id", - "name", - "status", - "project", - "last_iteration", - "status_reason", - "status_message", - "created", - "active_duration", - "last_metrics", - "runtime" - ); - JsonObject? result = await CallAsync("tasks", "get_all_ex", body, cancellationToken); - var tasks = (JsonArray?)result?["data"]?["tasks"]; - return tasks?.Select(t => t.Deserialize(JsonSerializerOptions)!).ToArray() - ?? Array.Empty(); - } - - private async Task CallAsync( - string service, - string action, - JsonNode body, - CancellationToken cancellationToken = default - ) - { - var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") - { - Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") - }; - request.Headers.Add( - "Authorization", - $"Bearer {await _clearMLAuthService.GetAuthTokenAsync(cancellationToken)}" - ); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - return (JsonObject?)JsonNode.Parse(result); - } - - private class SnakeCaseJsonNamingPolicy : JsonNamingPolicy - { - public override string ConvertName(string name) - { - return string.Concat(name.Select((x, i) => i > 0 && char.IsUpper(x) ? "_" + x.ToString() : x.ToString())) - .ToLowerInvariant(); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs deleted file mode 100644 index 17d562ad..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class CorpusService : ICorpusService -{ - public IEnumerable CreateTextCorpora(IReadOnlyList files) - { - List corpora = []; - - List> textFileCorpora = []; - foreach (CorpusFile file in files) - { - switch (file.Format) - { - case FileFormat.Text: - // if there are multiple texts with the same id, then add it to a new corpus or the first - // corpus that doesn't contain a text with that id - Dictionary? corpus = textFileCorpora.FirstOrDefault(c => - !c.ContainsKey(file.TextId) - ); - if (corpus is null) - { - corpus = []; - textFileCorpora.Add(corpus); - } - corpus[file.TextId] = new TextFileText(file.TextId, file.Location); - break; - - case FileFormat.Paratext: - corpora.Add(new ParatextBackupTextCorpus(file.Location, includeAllText: true)); - break; - } - } - foreach (Dictionary corpus in textFileCorpora) - corpora.Add(new DictionaryTextCorpus(corpus.Values)); - - return corpora; - } - - public IEnumerable CreateTermCorpora(IReadOnlyList files) - { - foreach (CorpusFile file in files) - { - switch (file.Format) - { - case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); - break; - } - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs deleted file mode 100644 index 7ea8679f..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs +++ /dev/null @@ -1,177 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class DistributedReaderWriterLock(string hostId, IRepository locks, IIdGenerator idGenerator, string id) - : IDistributedReaderWriterLock -{ - private readonly string _hostId = hostId; - private readonly IRepository _locks = locks; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly string _id = id; - - public async Task ReaderLockAsync( - TimeSpan? lifetime = default, - CancellationToken cancellationToken = default - ) - { - string lockId = _idGenerator.GenerateId(); - if (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)) - { - using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); - do - { - RWLock? rwLock = sub.Change.Entity; - if (rwLock is not null && !rwLock.IsAvailableForReading()) - { - TimeSpan? timeout = default; - if (rwLock.WriterLock?.ExpiresAt is not null) - { - timeout = rwLock.WriterLock.ExpiresAt - DateTime.UtcNow; - if (timeout < TimeSpan.Zero) - timeout = TimeSpan.Zero; - } - if (timeout != TimeSpan.Zero) - await sub.WaitForChangeAsync(timeout, cancellationToken); - } - } while (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)); - } - return new ReaderLockReleaser(this, lockId); - } - - public async Task WriterLockAsync( - TimeSpan? lifetime = default, - CancellationToken cancellationToken = default - ) - { - string lockId = _idGenerator.GenerateId(); - if (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)) - { - await _locks.UpdateAsync( - _id, - u => u.Add(rwl => rwl.WriterQueue, new Lock { Id = lockId, HostId = _hostId }), - cancellationToken: cancellationToken - ); - try - { - using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); - do - { - RWLock? rwLock = sub.Change.Entity; - if (rwLock is not null && !rwLock.IsAvailableForWriting(lockId)) - { - var dateTimes = rwLock - .ReaderLocks.Where(l => l.ExpiresAt.HasValue) - .Select(l => l.ExpiresAt.GetValueOrDefault()) - .ToList(); - if (rwLock.WriterLock?.ExpiresAt is not null) - dateTimes.Add(rwLock.WriterLock.ExpiresAt.Value); - TimeSpan? timeout = default; - if (dateTimes.Count > 0) - { - timeout = dateTimes.Max() - DateTime.UtcNow; - if (timeout < TimeSpan.Zero) - timeout = TimeSpan.Zero; - } - if (timeout != TimeSpan.Zero) - await sub.WaitForChangeAsync(timeout, cancellationToken); - } - } while (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)); - } - catch - { - await _locks.UpdateAsync( - _id, - u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId), - cancellationToken: cancellationToken - ); - throw; - } - } - return new WriterLockReleaser(this, lockId); - } - - private async Task TryAcquireWriterLock( - string lockId, - TimeSpan? lifetime, - CancellationToken cancellationToken - ) - { - var now = DateTime.UtcNow; - Expression> filter = rwl => - rwl.Id == _id - && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) - && !rwl.ReaderLocks.Any(l => l.ExpiresAt == null || l.ExpiresAt > now) - && (!rwl.WriterQueue.Any() || rwl.WriterQueue[0].Id == lockId); - void Update(IUpdateBuilder u) - { - u.Set( - rwl => rwl.WriterLock, - new Lock - { - Id = lockId, - ExpiresAt = lifetime is null ? null : now + lifetime, - HostId = _hostId - } - ); - u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId); - } - RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); - return rwLock is not null; - } - - private async Task TryAcquireReaderLock( - string lockId, - TimeSpan? lifetime, - CancellationToken cancellationToken - ) - { - var now = DateTime.UtcNow; - Expression> filter = rwl => - rwl.Id == _id - && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) - && !rwl.WriterQueue.Any(); - void Update(IUpdateBuilder u) - { - u.Add( - rwl => rwl.ReaderLocks, - new Lock - { - Id = lockId, - ExpiresAt = lifetime is null ? null : now + lifetime, - HostId = _hostId - } - ); - } - - RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); - return rwLock is not null; - } - - private class WriterLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase - { - private readonly DistributedReaderWriterLock _distributedLock = distributedLock; - private readonly string _lockId = lockId; - - protected override async ValueTask DisposeAsyncCore() - { - Expression> filter = rwl => - rwl.Id == _distributedLock._id && rwl.WriterLock != null && rwl.WriterLock.Id == _lockId; - await _distributedLock._locks.UpdateAsync(filter, u => u.Unset(rwl => rwl.WriterLock)); - } - } - - private class ReaderLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase - { - private readonly DistributedReaderWriterLock _distributedLock = distributedLock; - private readonly string _lockId = lockId; - - protected override async ValueTask DisposeAsyncCore() - { - Expression> filter = rwl => - rwl.Id == _distributedLock._id && rwl.ReaderLocks.Any(l => l.Id == _lockId); - await _distributedLock._locks.UpdateAsync( - filter, - u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.Id == _lockId) - ); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs deleted file mode 100644 index 81810fb1..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class DistributedReaderWriterLockFactory( - IOptions serviceOptions, - IRepository locks, - IIdGenerator idGenerator -) : IDistributedReaderWriterLockFactory -{ - private readonly ServiceOptions _serviceOptions = serviceOptions.Value; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly IRepository _locks = locks; - - public async Task InitAsync(CancellationToken cancellationToken = default) - { - await RemoveAllWaitersAsync(cancellationToken); - await ReleaseAllWriterLocksAsync(cancellationToken); - await ReleaseAllReaderLocksAsync(cancellationToken); - } - - public async Task CreateAsync( - string id, - CancellationToken cancellationToken = default - ) - { - try - { - await _locks.InsertAsync( - new RWLock - { - Id = id, - ReaderLocks = [], - WriterQueue = [] - }, - cancellationToken - ); - } - catch (DuplicateKeyException) - { - // the lock is already made - no new one needs to be made - // This is done instead of checking if it exists first to prevent race conditions. - } - return new DistributedReaderWriterLock(_serviceOptions.ServiceId, _locks, _idGenerator, id); - } - - public async Task DeleteAsync(string id, CancellationToken cancellationToken = default) - { - RWLock? rwLock = await _locks.DeleteAsync(rwl => rwl.Id == id, cancellationToken); - return rwLock is not null; - } - - private async Task ReleaseAllWriterLocksAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.WriterLock != null && rwl.WriterLock.HostId == _serviceOptions.ServiceId, - u => u.Unset(rwl => rwl.WriterLock), - cancellationToken - ); - } - - private async Task ReleaseAllReaderLocksAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.ReaderLocks.Any(l => l.HostId == _serviceOptions.ServiceId), - u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken - ); - } - - private async Task RemoveAllWaitersAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.WriterQueue.Any(l => l.HostId == _serviceOptions.ServiceId), - u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken - ); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs b/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs deleted file mode 100644 index 78a3ceb2..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class FileSystem : IFileSystem -{ - public void CreateDirectory(string path) - { - Directory.CreateDirectory(path); - } - - public void DeleteFile(string path) - { - if (File.Exists(path)) - File.Delete(path); - } - - public Stream OpenWrite(string path) - { - return File.OpenWrite(path); - } - - public Stream OpenRead(string path) - { - return File.OpenRead(path); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs deleted file mode 100644 index 26fe58ed..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs +++ /dev/null @@ -1,180 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public abstract class HangfireBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger -) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - public virtual Task RunAsync( - string engineId, - string buildId, - string? buildOptions, - CancellationToken cancellationToken - ) - { - return RunAsync(engineId, buildId, null, buildOptions, cancellationToken); - } -} - -public abstract class HangfireBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger> logger -) -{ - protected IPlatformService PlatformService { get; } = platformService; - protected IRepository Engines { get; } = engines; - protected IDistributedReaderWriterLockFactory LockFactory { get; } = lockFactory; - protected IDataAccessContext DataAccessContext { get; } = dataAccessContext; - protected IBuildJobService BuildJobService { get; } = buildJobService; - protected ILogger> Logger { get; } = logger; - - public virtual async Task RunAsync( - string engineId, - string buildId, - T data, - string? buildOptions, - CancellationToken cancellationToken - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - JobCompletionStatus completionStatus = JobCompletionStatus.Completed; - try - { - await InitializeAsync(engineId, buildId, data, @lock, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await BuildJobService.BuildJobStartedAsync(engineId, buildId, cancellationToken)) - { - completionStatus = JobCompletionStatus.Canceled; - return; - } - } - - await DoWorkAsync(engineId, buildId, data, buildOptions, @lock, cancellationToken); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await Engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - CancellationToken.None - ); - if (engine?.CurrentBuild?.JobState is BuildJobState.Canceling) - { - completionStatus = JobCompletionStatus.Canceled; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildCanceledAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - Logger.LogInformation("Build canceled ({0})", buildId); - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - completionStatus = JobCompletionStatus.Restarting; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildRestartingAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobRestartingAsync(engineId, buildId, CancellationToken.None); - }, - cancellationToken: CancellationToken.None - ); - } - throw; - } - else - { - completionStatus = JobCompletionStatus.Canceled; - } - } - catch (Exception e) - { - completionStatus = JobCompletionStatus.Faulted; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - Logger.LogError(0, e, "Build faulted ({0})", buildId); - throw; - } - finally - { - await CleanupAsync(engineId, buildId, data, @lock, completionStatus); - } - } - - protected virtual Task InitializeAsync( - string engineId, - string buildId, - T data, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - return Task.CompletedTask; - } - - protected abstract Task DoWorkAsync( - string engineId, - string buildId, - T data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ); - - protected virtual Task CleanupAsync( - string engineId, - string buildId, - T data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - return Task.CompletedTask; - } - - protected enum JobCompletionStatus - { - Completed, - Faulted, - Canceled, - Restarting - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs deleted file mode 100644 index d5be7f30..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs +++ /dev/null @@ -1,85 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class HangfireBuildJobRunner( - IBackgroundJobClient jobClient, - IEnumerable buildJobFactories -) : IBuildJobRunner -{ - public static Job CreateJob( - string engineId, - string buildId, - string queue, - object? data, - string? buildOptions - ) - where TJob : HangfireBuildJob - { - ArgumentNullException.ThrowIfNull(data); - // Token "None" is used here because hangfire injects the proper cancellation token - return Job.FromExpression( - j => j.RunAsync(engineId, buildId, (TData)data, buildOptions, CancellationToken.None), - queue - ); - } - - public static Job CreateJob(string engineId, string buildId, string queue, string? buildOptions) - where TJob : HangfireBuildJob - { - // Token "None" is used here because hangfire injects the proper cancellation token - return Job.FromExpression( - j => j.RunAsync(engineId, buildId, buildOptions, CancellationToken.None), - queue - ); - } - - private readonly IBackgroundJobClient _jobClient = jobClient; - private readonly Dictionary _buildJobFactories = - buildJobFactories.ToDictionary(f => f.EngineType); - - public BuildJobRunnerType Type => BuildJobRunnerType.Hangfire; - - public Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } - - public Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } - - public Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - IHangfireBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; - Job job = buildJobFactory.CreateJob(engineId, buildId, stage, data, buildOptions); - return Task.FromResult(_jobClient.Create(job, new ScheduledState(TimeSpan.FromDays(10000)))); - } - - public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return Task.FromResult(_jobClient.Delete(jobId)); - } - - public Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ) - { - return Task.FromResult(_jobClient.Requeue(jobId)); - } - - public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) - { - // Trigger the cancellation token for the job - return Task.FromResult(_jobClient.Delete(jobId)); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs deleted file mode 100644 index c3c14751..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class HangfireHealthCheck(JobStorage jobStorage, IOptions options) : IHealthCheck -{ - private readonly JobStorage _jobStorage = jobStorage; - private readonly IOptions _options = options; - - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - if ( - _jobStorage - .GetMonitoringApi() - .Servers() - .Any(s => DateTime.UtcNow - s.Heartbeat < _options.Value.ServerTimeout) - ) - { - return Task.FromResult(HealthCheckResult.Healthy()); - } - - return Task.FromResult(HealthCheckResult.Unhealthy("There are no Hangfire servers running.")); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs deleted file mode 100644 index 6f6d3696..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IBuildJobRunner -{ - BuildJobRunnerType Type { get; } - - Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); - Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); - - Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ); - - Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default); - - Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ); - - Task StopJobAsync(string jobId, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs deleted file mode 100644 index c9ddf983..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs +++ /dev/null @@ -1,41 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IBuildJobService -{ - Task> GetBuildingEnginesAsync( - BuildJobRunnerType runner, - CancellationToken cancellationToken = default - ); - - Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default); - - Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); - - Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); - - Task StartBuildJobAsync( - BuildJobRunnerType jobType, - string engineId, - string buildId, - BuildStage stage, - object? data = default, - string? buildOptions = default, - CancellationToken cancellationToken = default - ); - - Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( - string engineId, - CancellationToken cancellationToken = default - ); - - Task BuildJobStartedAsync(string engineId, string buildId, CancellationToken cancellationToken = default); - - Task BuildJobFinishedAsync( - string engineId, - string buildId, - bool buildComplete, - CancellationToken cancellationToken = default - ); - - Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs deleted file mode 100644 index 4cacec2c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IClearMLAuthenticationService : IHostedService -{ - public Task GetAuthTokenAsync(CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs deleted file mode 100644 index bb5afc57..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IClearMLBuildJobFactory -{ - TranslationEngineType EngineType { get; } - - Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs deleted file mode 100644 index 1e2425a4..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IClearMLQueueService -{ - public int GetQueueSize(TranslationEngineType engineType); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs deleted file mode 100644 index 75f8be96..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IClearMLService -{ - Task CreateProjectAsync( - string name, - string? description = null, - CancellationToken cancellationToken = default - ); - Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default); - Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default); - - Task CreateTaskAsync( - string buildId, - string projectId, - string script, - string dockerImage, - CancellationToken cancellationToken = default - ); - Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default); - Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default); - Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default); - Task StopTaskAsync(string id, CancellationToken cancellationToken = default); - Task> GetTasksForQueueAsync(string queue, CancellationToken cancellationToken = default); - Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default); - Task> GetTasksByIdAsync( - IEnumerable ids, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs deleted file mode 100644 index bbcc9de3..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ICorpusService -{ - IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs deleted file mode 100644 index 026aff28..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IDistributedReaderWriterLock -{ - Task ReaderLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); - Task WriterLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs deleted file mode 100644 index 93e26c62..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IDistributedReaderWriterLockFactory -{ - Task InitAsync(CancellationToken cancellationToken = default); - Task CreateAsync(string id, CancellationToken cancellationToken = default); - Task DeleteAsync(string id, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs deleted file mode 100644 index 7df25380..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IFileStorage : IDisposable -{ - Task ExistsAsync(string path, CancellationToken cancellationToken = default); - - Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ); - - Task OpenReadAsync(string path, CancellationToken cancellationToken = default); - - Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); - - Task GetDownloadUrlAsync(string path, DateTime expiresAt, CancellationToken cancellationToken = default); - - Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs b/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs deleted file mode 100644 index fa5c8f6c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IFileSystem -{ - void DeleteFile(string path); - void CreateDirectory(string path); - Stream OpenWrite(string path); - Stream OpenRead(string path); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs deleted file mode 100644 index faabcfec..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IHangfireBuildJobFactory -{ - TranslationEngineType EngineType { get; } - - Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs deleted file mode 100644 index 761a3898..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ILanguageTagService -{ - bool ConvertToFlores200Code(string languageTag, out string flores200Code); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs deleted file mode 100644 index d9791ec8..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IMessageOutboxService -{ - public Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - string? content = null, - Stream? contentStream = null, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs deleted file mode 100644 index 014ab591..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IOutboxMessageHandler -{ - public string OutboxId { get; } - - public Task HandleMessageAsync( - string method, - string? content, - Stream? contentStream, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs deleted file mode 100644 index 79b30f6b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface IPlatformService -{ - Task IncrementTrainSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); - - Task UpdateBuildStatusAsync( - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ); - Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default); - Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default); - Task BuildCompletedAsync( - string buildId, - int trainSize, - double confidence, - CancellationToken cancellationToken = default - ); - Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default); - Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); - Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); - - Task InsertPretranslationsAsync( - string engineId, - Stream pretranslationsStream, - CancellationToken cancellationToken = default - ); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs deleted file mode 100644 index e8811f09..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ISharedFileService -{ - Uri GetBaseUri(); - - Uri GetResolvedUri(string path); - - Task GetDownloadUrlAsync(string path, DateTime expiresAt); - - Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ); - - Task OpenReadAsync(string path, CancellationToken cancellationToken = default); - - Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); - - Task ExistsAsync(string path, CancellationToken cancellationToken = default); - - Task DeleteAsync(string path, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs deleted file mode 100644 index 6612e11e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ISmtModelFactory -{ - Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ); - Task CreateTrainerAsync( - string engineDir, - IRangeTokenizer tokenizer, - IParallelTextCorpus corpus, - CancellationToken cancellationToken = default - ); - Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); - Task UpdateEngineFromAsync(string engineDir, Stream source, CancellationToken cancellationToken = default); - Task SaveEngineToAsync(string engineDir, Stream destination, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs deleted file mode 100644 index c76b8e91..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ITransferEngineFactory -{ - Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ); - Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs deleted file mode 100644 index 71ed5d94..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ITranslationEngineService -{ - TranslationEngineType Type { get; } - - Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ); - Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - - Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ); - - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - - Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ); - - Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ); - - Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - - Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); - - int GetQueueSize(); - - bool IsLanguageNativeToModel(string language, out string internalCode); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs deleted file mode 100644 index e83337d3..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public interface ITruecaserFactory -{ - Task CreateAsync(string engineDir, CancellationToken cancellationToken = default); - Task CreateTrainerAsync( - string engineDir, - ITokenizer tokenizer, - ITextCorpus corpus, - CancellationToken cancellationToken = default - ); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs deleted file mode 100644 index 998144fe..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs +++ /dev/null @@ -1,143 +0,0 @@ -using SIL.ObjectModel; -using static Serval.Machine.Shared.Utils.SharedFileUtils; - -namespace Serval.Machine.Shared.Services; - -public class InMemoryStorage : DisposableBase, IFileStorage -{ - public class Entry : Stream - { - public MemoryStream MemoryStream { get; } - public string Path { get; } - - private readonly InMemoryStorage _parent; - - public override bool CanRead => MemoryStream.CanRead; - - public override bool CanSeek => MemoryStream.CanSeek; - - public override bool CanWrite => MemoryStream.CanWrite; - - public override long Length => MemoryStream.Length; - - public override long Position - { - get => MemoryStream.Position; - set => MemoryStream.Position = value; - } - - public Entry(string path, InMemoryStorage parent) - { - Path = path; - MemoryStream = new(); - _parent = parent; - } - - public Entry(Entry other) - { - Path = other.Path; - MemoryStream = other.MemoryStream; - _parent = other._parent; - } - - protected override void Dispose(bool disposing) - { - _parent._memoryStreams[Path] = new Entry(this); - base.Dispose(disposing); - } - - public override void Flush() - { - MemoryStream.Flush(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - return MemoryStream.Read(buffer, offset, count); - } - - public override long Seek(long offset, SeekOrigin origin) - { - return MemoryStream.Seek(offset, origin); - } - - public override void SetLength(long value) - { - MemoryStream.SetLength(value); - } - - public override void Write(byte[] buffer, int offset, int count) - { - MemoryStream.Write(buffer, offset, count); - } - } - - private readonly ConcurrentDictionary _memoryStreams = new(); - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return Task.FromResult(_memoryStreams.TryGetValue(Normalize(path), out _)); - } - - public Task> ListFilesAsync( - string? path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - path = string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true); - if (recurse) - { - return Task.FromResult>( - _memoryStreams.Keys.Where(p => p.StartsWith(path)).ToList() - ); - } - - return Task.FromResult>( - _memoryStreams.Keys.Where(p => p.StartsWith(path) && !p[path.Length..].Contains('/')).ToList() - ); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - if (!_memoryStreams.TryGetValue(Normalize(path), out Entry? ret)) - throw new FileNotFoundException($"Unable to find file {path}"); - ret.Position = 0; - return Task.FromResult(ret); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return Task.FromResult(new Entry(Normalize(path), this)); - } - - public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) - { - if (_memoryStreams.ContainsKey(Normalize(path))) - { - _memoryStreams.Remove(Normalize(path), out _); - } - else - { - IEnumerable filesToRemove = await ListFilesAsync(path, recurse, cancellationToken); - foreach (string filePath in filesToRemove) - _memoryStreams.Remove(Normalize(filePath), out _); - } - } - - protected override void DisposeManagedResources() - { - foreach (Entry stream in _memoryStreams.Values) - stream.Dispose(); - _memoryStreams.Clear(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs deleted file mode 100644 index 30e065f5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs +++ /dev/null @@ -1,170 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class LanguageTagService : ILanguageTagService -{ - private static readonly Dictionary StandardLanguages = - new() - { - { "ar", "arb" }, - { "ms", "zsm" }, - { "lv", "lvs" }, - { "ne", "npi" }, - { "sw", "swh" }, - { "cmn", "zh" } - }; - - private static readonly Dictionary StandardScripts = new() { { "Kore", "Hang" } }; - - private readonly Dictionary _defaultScripts; - - private readonly Dictionary _flores200Languages; - - private static readonly Regex LangTagPattern = - new("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture); - - public LanguageTagService() - { - // initialize SLDR language tags to retrieve latest langtags.json file - _defaultScripts = InitializeDefaultScripts(); - _flores200Languages = InitializeFlores200Languages(); - } - - protected virtual void InitializeSldrLanguageTags() - { - Sldr.InitializeLanguageTags(); - } - - private Dictionary InitializeDefaultScripts() - { - InitializeSldrLanguageTags(); - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - JsonNode? json; - - if (!File.Exists(cachedAllTagsPath)) - { - using HttpClient client = new(); - using HttpResponseMessage response = client.Send( - new HttpRequestMessage( - HttpMethod.Get, - "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json" - ) - ); - response.EnsureSuccessStatusCode(); - using Stream responseStream = response.Content.ReadAsStream(); - using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create); - responseStream.CopyTo(fileStream); - } - using FileStream stream = new(cachedAllTagsPath, FileMode.Open); - json = JsonNode.Parse(stream); - - Dictionary tempDefaultScripts = new(); - foreach (JsonNode? entry in json!.AsArray()) - { - if (entry is null) - continue; - - var script = (string?)entry["script"]; - if (script is null) - continue; - - JsonNode? tags = entry["tags"]; - if (tags is not null) - { - foreach (var t in tags.AsArray().Select(v => (string?)v)) - { - if ( - t is not null - && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) - && s is null - ) - { - tempDefaultScripts[t] = script; - } - } - } - - var tag = (string?)entry["tag"]; - if (tag is not null) - tempDefaultScripts[tag] = script; - } - return tempDefaultScripts; - } - - private static Dictionary InitializeFlores200Languages() - { - var tempFlores200Languages = new Dictionary(); - using var floresStream = Assembly - .GetExecutingAssembly() - .GetManifestResourceStream("Serval.Machine.Shared.data.flores200languages.csv"); - Debug.Assert(floresStream is not null); - var reader = new StreamReader(floresStream); - var firstLine = reader.ReadLine(); - Debug.Assert(firstLine == "language, code"); - while (!reader.EndOfStream) - { - string? line = reader.ReadLine(); - if (line is null) - continue; - string[] values = line.Split(','); - tempFlores200Languages[values[1].Trim()] = values[0].Trim(); - } - return tempFlores200Languages; - } - - /** - * Converts a language tag to a Flores 200 code - * @param {string} languageTag - The language tag to convert - * @param out {string} flores200Code - The converted Flores 200 code - * @returns {bool} is the language is the Flores 200 list - */ - public bool ConvertToFlores200Code(string languageTag, out string flores200Code) - { - flores200Code = ResolveLanguageTag(languageTag); - return _flores200Languages.ContainsKey(flores200Code); - } - - private string ResolveLanguageTag(string languageTag) - { - // Try to find a pattern of {language code}_{script} - Match langTagMatch = LangTagPattern.Match(languageTag); - if (!langTagMatch.Success) - return languageTag; - string parsedLanguage = langTagMatch.Groups["language"].Value; - string languageSubtag = parsedLanguage; - string iso639_3Code = parsedLanguage; - - // Best attempt to convert language to a registered ISO 639-3 code - // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping - - // If they gave us the ISO code, revert it to the 2 character code - if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag)) - languageSubtag = tempSubtag.Code; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName)) - languageSubtag = tempName; - - if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj)) - iso639_3Code = languageSubtagObj.Iso3Code; - - // Use default script unless there is one parsed out of the language tag - Group scriptGroup = langTagMatch.Groups["script"]; - string? script = null; - - if (scriptGroup.Success) - script = scriptGroup.Value; - else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2)) - script = tempScript2; - else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript)) - script = tempScript; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (script is not null && StandardScripts.TryGetValue(script, out string? tempScript3)) - script = tempScript3; - - if (script is not null) - return $"{iso639_3Code}_{script}"; - else - return languageTag; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs deleted file mode 100644 index b666ea77..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs +++ /dev/null @@ -1,78 +0,0 @@ -using SIL.ObjectModel; -using static Serval.Machine.Shared.Utils.SharedFileUtils; - -namespace Serval.Machine.Shared.Services; - -public class LocalStorage : DisposableBase, IFileStorage -{ - private readonly Uri _basePath; - - public LocalStorage(string basePath) - { - _basePath = new Uri(basePath); - if (!_basePath.AbsoluteUri.EndsWith("/")) - _basePath = new Uri(_basePath.AbsoluteUri + "/"); - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - return Task.FromResult(File.Exists(pathUri.LocalPath)); - } - - public Task> ListFilesAsync( - string path = "", - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - Uri pathUri = new(_basePath, Normalize(path)); - string[] files = Directory.GetFiles( - pathUri.LocalPath, - "*", - new EnumerationOptions { RecurseSubdirectories = recurse } - ); - return Task.FromResult>( - files.Select(f => _basePath.MakeRelativeUri(new Uri(f)).ToString()).ToArray() - ); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - return Task.FromResult(File.OpenRead(pathUri.LocalPath)); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - Directory.CreateDirectory(Path.GetDirectoryName(pathUri.LocalPath)!); - return Task.FromResult(File.OpenWrite(pathUri.LocalPath)); - } - - public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - - if (File.Exists(pathUri.LocalPath)) - { - File.Delete(pathUri.LocalPath); - } - else if (Directory.Exists(pathUri.LocalPath)) - { - foreach (string filePath in await ListFilesAsync(path, recurse, cancellationToken)) - { - await DeleteAsync(filePath, false, cancellationToken); - } - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs deleted file mode 100644 index 09f49fb6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class MessageOutboxDeliveryService( - IServiceProvider services, - IEnumerable outboxMessageHandlers, - IFileSystem fileSystem, - IOptionsMonitor options, - ILogger logger -) : BackgroundService -{ - private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); - - private readonly IServiceProvider _services = services; - private readonly Dictionary _outboxMessageHandlers = - outboxMessageHandlers.ToDictionary(o => o.OutboxId); - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - private readonly ILogger _logger = logger; - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - Initialize(); - using IServiceScope scope = _services.CreateScope(); - var messages = scope.ServiceProvider.GetRequiredService>(); - using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); - while (true) - { - await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); - if (stoppingToken.IsCancellationRequested) - break; - await ProcessMessagesAsync(messages, stoppingToken); - } - } - - private void Initialize() - { - _fileSystem.CreateDirectory(_options.CurrentValue.OutboxDir); - } - - internal async Task ProcessMessagesAsync( - IRepository messages, - CancellationToken cancellationToken = default - ) - { - bool anyMessages = await messages.ExistsAsync(m => true, cancellationToken); - if (!anyMessages) - return; - - IReadOnlyList curMessages = await messages.GetAllAsync(cancellationToken); - - IEnumerable> messageGroups = curMessages - .OrderBy(m => m.Index) - .GroupBy(m => (m.OutboxRef, m.GroupId)); - - foreach (IGrouping<(string OutboxId, string GroupId), OutboxMessage> messageGroup in messageGroups) - { - bool abortMessageGroup = false; - IOutboxMessageHandler outboxMessageHandler = _outboxMessageHandlers[messageGroup.Key.OutboxId]; - foreach (OutboxMessage message in messageGroup) - { - try - { - await ProcessGroupMessagesAsync(messages, message, outboxMessageHandler, cancellationToken); - } - catch (RpcException e) - { - switch (e.StatusCode) - { - case StatusCode.Unavailable: - case StatusCode.Unauthenticated: - case StatusCode.PermissionDenied: - case StatusCode.Cancelled: - _logger.LogWarning(e, "Platform Message sending failure: {statusCode}", e.StatusCode); - return; - case StatusCode.Aborted: - case StatusCode.DeadlineExceeded: - case StatusCode.Internal: - case StatusCode.ResourceExhausted: - case StatusCode.Unknown: - abortMessageGroup = !await CheckIfFinalMessageAttempt(messages, message, e); - break; - case StatusCode.InvalidArgument: - default: - // log error - await PermanentlyFailedMessage(messages, message, e); - break; - } - } - catch (Exception e) - { - await PermanentlyFailedMessage(messages, message, e); - break; - } - if (abortMessageGroup) - break; - } - } - } - - private async Task ProcessGroupMessagesAsync( - IRepository messages, - OutboxMessage message, - IOutboxMessageHandler outboxMessageHandler, - CancellationToken cancellationToken = default - ) - { - Stream? contentStream = null; - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, message.Id); - if (message.HasContentStream) - contentStream = _fileSystem.OpenRead(filePath); - try - { - await outboxMessageHandler.HandleMessageAsync( - message.Method, - message.Content, - contentStream, - cancellationToken - ); - await messages.DeleteAsync(message.Id, CancellationToken.None); - } - finally - { - contentStream?.Dispose(); - } - _fileSystem.DeleteFile(filePath); - } - - private async Task CheckIfFinalMessageAttempt( - IRepository messages, - OutboxMessage message, - Exception e - ) - { - if (message.Created < DateTimeOffset.UtcNow.Subtract(_options.CurrentValue.MessageExpirationTimeout)) - { - await PermanentlyFailedMessage(messages, message, e); - return true; - } - else - { - await LogFailedAttempt(messages, message, e); - return false; - } - } - - private async Task PermanentlyFailedMessage(IRepository messages, OutboxMessage message, Exception e) - { - // log error - _logger.LogError( - e, - "Permanently failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Id, - message.Method, - message.Content, - e.Message - ); - await messages.DeleteAsync(message.Id); - } - - private async Task LogFailedAttempt(IRepository messages, OutboxMessage message, Exception e) - { - // log error - await messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); - _logger.LogError( - e, - "Attempt {Attempts}. Failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Attempts + 1, - message.Id, - message.Method, - message.Content, - e.Message - ); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs deleted file mode 100644 index 2c5d410d..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class MessageOutboxService( - IRepository outboxes, - IRepository messages, - IIdGenerator idGenerator, - IFileSystem fileSystem, - IOptionsMonitor options -) : IMessageOutboxService -{ - private readonly IRepository _outboxes = outboxes; - private readonly IRepository _messages = messages; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - internal int MaxDocumentSize { get; set; } = 1_000_000; - - public async Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - string? content = null, - Stream? contentStream = null, - CancellationToken cancellationToken = default - ) - { - if (content == null && contentStream == null) - throw new ArgumentException("Either content or contentStream must be specified."); - if (content is not null && content.Length > MaxDocumentSize) - { - throw new ArgumentException( - $"The content is too large for request {method} with group ID {groupId}. " - + $"It is {content.Length} bytes, but the maximum is {MaxDocumentSize} bytes." - ); - } - Outbox outbox = ( - await _outboxes.UpdateAsync( - outboxId, - u => u.Inc(o => o.CurrentIndex, 1), - upsert: true, - cancellationToken: cancellationToken - ) - )!; - OutboxMessage outboxMessage = - new() - { - Id = _idGenerator.GenerateId(), - Index = outbox.CurrentIndex, - OutboxRef = outboxId, - Method = method, - GroupId = groupId, - Content = content, - HasContentStream = contentStream is not null - }; - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, outboxMessage.Id); - try - { - if (contentStream is not null) - { - await using Stream fileStream = _fileSystem.OpenWrite(filePath); - await contentStream.CopyToAsync(fileStream, cancellationToken); - } - await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); - return outboxMessage.Id; - } - catch - { - if (contentStream is not null) - _fileSystem.DeleteFile(filePath); - throw; - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs deleted file mode 100644 index 92b38d6a..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs +++ /dev/null @@ -1,59 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ModelCleanupService( - IServiceProvider services, - ISharedFileService sharedFileService, - ILogger logger -) : RecurrentTask("Model Cleanup Service", services, RefreshPeriod, logger) -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILogger _logger = logger; - private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1); - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - var engines = scope.ServiceProvider.GetRequiredService>(); - await CheckModelsAsync(engines, cancellationToken); - } - - internal async Task CheckModelsAsync(IRepository engines, CancellationToken cancellationToken) - { - _logger.LogInformation("Running model cleanup job"); - IReadOnlyCollection paths = await _sharedFileService.ListFilesAsync( - NmtEngineService.ModelDirectory, - cancellationToken: cancellationToken - ); - // Get all NMT engine ids from the database - IReadOnlyList? allEngines = await engines.GetAllAsync(cancellationToken: cancellationToken); - IEnumerable validNmtFilenames = allEngines - .Where(e => e.Type == TranslationEngineType.Nmt) - .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision)); - // If there is a currently running build that creates and pushes a new file, but the database has not - // updated yet, don't delete the new file. - IEnumerable validNmtFilenamesForNextBuild = allEngines - .Where(e => e.Type == TranslationEngineType.Nmt) - .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision + 1)); - - var filenameFilter = validNmtFilenames.Concat(validNmtFilenamesForNextBuild).ToHashSet(); - - foreach (string path in paths) - { - if (!filenameFilter.Contains(path)) - { - await DeleteFileAsync( - path, - $"file in S3 bucket not found in database. It may be an old rev, etc.", - cancellationToken - ); - } - } - } - - private async Task DeleteFileAsync(string path, string message, CancellationToken cancellationToken = default) - { - // This may delete a file while it is being downloaded, but the chance is rare - // enough and the solution easy enough (just download again) to just live with it. - _logger.LogInformation("Deleting old model file {filename}: {message}", path, message); - await _sharedFileService.DeleteAsync(path, cancellationToken); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs deleted file mode 100644 index 4f465936..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class NmtClearMLBuildJobFactory( - ISharedFileService sharedFileService, - ILanguageTagService languageTagService, - IRepository engines -) : IClearMLBuildJobFactory -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILanguageTagService _languageTagService = languageTagService; - private readonly IRepository _engines = engines; - - public TranslationEngineType EngineType => TranslationEngineType.Nmt; - - public async Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - if (stage == BuildStage.Train) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException("The engine does not exist."); - - Uri sharedFileUri = _sharedFileService.GetBaseUri(); - string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); - string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); - _languageTagService.ConvertToFlores200Code(engine.SourceLanguage, out string srcLang); - _languageTagService.ConvertToFlores200Code(engine.TargetLanguage, out string trgLang); - return "from machine.jobs.build_nmt_engine import run\n" - + "args = {\n" - + $" 'model_type': '{modelType}',\n" - + $" 'engine_id': '{engineId}',\n" - + $" 'build_id': '{buildId}',\n" - + $" 'src_lang': '{srcLang}',\n" - + $" 'trg_lang': '{trgLang}',\n" - + $" 'shared_file_uri': '{baseUri}',\n" - + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") - // buildRevision + 1 because the build revision is incremented after the build job - // is finished successfully but the file should be saved with the new revision number - + (engine.IsModelPersisted ? $" 'save_model': '{engineId}_{engine.BuildRevision + 1}',\n" : $"") - + $" 'clearml': True\n" - + "}\n" - + "run(args)\n"; - } - else - { - throw new ArgumentException("Unknown build stage.", nameof(stage)); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs deleted file mode 100644 index 5a2fb912..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs +++ /dev/null @@ -1,204 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class NmtEngineService( - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IBuildJobService buildJobService, - ILanguageTagService languageTagService, - IClearMLQueueService clearMLQueueService, - ISharedFileService sharedFileService -) : ITranslationEngineService -{ - private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; - private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IRepository _engines = engines; - private readonly IBuildJobService _buildJobService = buildJobService; - private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - private readonly ILanguageTagService _languageTagService = languageTagService; - private readonly ISharedFileService _sharedFileService = sharedFileService; - public const string ModelDirectory = "models/"; - - public static string GetModelPath(string engineId, int buildRevision) - { - return $"{ModelDirectory}{engineId}_{buildRevision}.tar.gz"; - } - - public TranslationEngineType Type => TranslationEngineType.Nmt; - - private const int MinutesToExpire = 60; - - public async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ) - { - var translationEngine = await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - var translationEngine = new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage, - Type = TranslationEngineType.Nmt, - IsModelPersisted = isModelPersisted ?? false // models are not persisted if not specified - }; - await _engines.InsertAsync(translationEngine, ct); - await _buildJobService.CreateEngineAsync(engineId, engineName, ct); - return translationEngine; - }, - cancellationToken: cancellationToken - ); - return translationEngine; - } - - public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildJobAsync(engineId, cancellationToken); - - await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); - } - await _lockFactory.DeleteAsync(engineId, CancellationToken.None); - } - - public async Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - // If there is a pending/running build, then no need to start a new one. - if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is already building or in the process of canceling."); - - await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - cancellationToken - ); - } - } - - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await CancelBuildJobAsync(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is not currently building."); - } - } - - public async Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.IsModelPersisted != true) - { - throw new NotSupportedException( - "The model cannot be downloaded. " - + "To enable downloading the model, recreate the engine with IsModelPersisted property to true." - ); - } - - if (engine.BuildRevision == 0) - throw new InvalidOperationException("The engine has not been built yet."); - string filepath = GetModelPath(engineId, engine.BuildRevision); - bool fileExists = await _sharedFileService.ExistsAsync(filepath, cancellationToken); - if (!fileExists) - throw new FileNotFoundException($"The model for build revision , {engine.BuildRevision}, does not exist."); - var expiresAt = DateTime.UtcNow.AddMinutes(MinutesToExpire); - var modelInfo = new ModelDownloadUrl - { - Url = await _sharedFileService.GetDownloadUrlAsync(filepath, expiresAt), - ModelRevision = engine.BuildRevision, - ExpiresAt = expiresAt - }; - return modelInfo; - } - - public Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public int GetQueueSize() - { - return _clearMLQueueService.GetQueueSize(Type); - } - - public bool IsLanguageNativeToModel(string language, out string internalCode) - { - return _languageTagService.ConvertToFlores200Code(language, out internalCode); - } - - private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) - { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken - ); - return buildId is not null; - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException($"The engine {engineId} does not exist."); - return engine; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs deleted file mode 100644 index a8b3d52f..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs +++ /dev/null @@ -1,26 +0,0 @@ -using static Serval.Machine.Shared.Services.HangfireBuildJobRunner; - -namespace Serval.Machine.Shared.Services; - -public class NmtHangfireBuildJobFactory : IHangfireBuildJobFactory -{ - public TranslationEngineType EngineType => TranslationEngineType.Nmt; - - public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) - { - return stage switch - { - BuildStage.Preprocess - => CreateJob>( - engineId, - buildId, - "nmt", - data, - buildOptions - ), - BuildStage.Postprocess - => CreateJob(engineId, buildId, "nmt", data, buildOptions), - _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), - }; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs deleted file mode 100644 index b4c61648..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class NmtPreprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - ILogger logger, - IBuildJobService buildJobService, - ISharedFileService sharedFileService, - ICorpusService corpusService, - ILanguageTagService languageTagService -) - : PreprocessBuildJob( - platformService, - engines, - lockFactory, - dataAccessContext, - logger, - buildJobService, - sharedFileService, - corpusService - ) -{ - private readonly ILanguageTagService _languageTagService = languageTagService; - - protected override bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) - { - return _languageTagService.ConvertToFlores200Code(languageCode, out resolvedCode); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs deleted file mode 100644 index 25e34892..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs +++ /dev/null @@ -1,88 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class PostprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService -) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - protected ISharedFileService SharedFileService { get; } = sharedFileService; - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - (int, double) data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - (int corpusSize, double confidence) = data; - - await using ( - Stream pretranslationsStream = await SharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ) - ) - { - await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken); - } - - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - int additionalCorpusSize = await SaveModelAsync(engineId, buildId); - await PlatformService.BuildCompletedAsync( - buildId, - corpusSize + additionalCorpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: true, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - - Logger.LogInformation("Build completed ({0}).", buildId); - } - - protected virtual Task SaveModelAsync(string engineId, string buildId) - { - return Task.FromResult(0); - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - (int, double) data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Restarting) - return; - - try - { - if (completionStatus is not JobCompletionStatus.Faulted) - await SharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {0}.", buildId); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs deleted file mode 100644 index 214a1818..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ /dev/null @@ -1,434 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class PreprocessBuildJob : HangfireBuildJob> -{ - private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; - - internal BuildJobRunnerType TrainJobRunnerType { get; init; } = BuildJobRunnerType.ClearML; - - private readonly ISharedFileService _sharedFileService; - private readonly ICorpusService _corpusService; - private int _seed = 1234; - private Random _random; - - public PreprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - ILogger logger, - IBuildJobService buildJobService, - ISharedFileService sharedFileService, - ICorpusService corpusService - ) - : base(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) - { - _sharedFileService = sharedFileService; - _corpusService = corpusService; - _random = new Random(_seed); - } - - internal int Seed - { - get => _seed; - set - { - if (_seed != value) - { - _seed = value; - _random = new Random(_seed); - } - } - } - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - IReadOnlyList data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - (int trainCount, int pretranslateCount) = await WriteDataFilesAsync( - buildId, - data, - buildOptions, - cancellationToken - ); - - // Log summary of build data - JsonObject buildPreprocessSummary = - new() - { - { "Event", "BuildPreprocess" }, - { "EngineId", engineId }, - { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount } - }; - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - - bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); - buildPreprocessSummary.Add("SourceLanguageResolved", srcLang); - bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); - buildPreprocessSummary.Add("TargetLanguageResolved", trgLang); - Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); - - if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) - { - throw new InvalidOperationException( - $"Neither language code in build {buildId} are known to the base model, and the data specified for training was empty. Build canceled." - ); - } - - cancellationToken.ThrowIfCancellationRequested(); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - bool canceling = !await BuildJobService.StartBuildJobAsync( - TrainJobRunnerType, - engineId, - buildId, - BuildStage.Train, - buildOptions: buildOptions, - cancellationToken: cancellationToken - ); - if (canceling) - throw new OperationCanceledException(); - } - } - - private async Task<(int TrainCount, int PretranslateCount)> WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - string? buildOptions, - CancellationToken cancellationToken - ) - { - JsonObject? buildOptionsObject = null; - if (buildOptions is not null) - buildOptionsObject = JsonSerializer.Deserialize(buildOptions); - await using StreamWriter sourceTrainWriter = - new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken)); - await using StreamWriter targetTrainWriter = - new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken)); - - await using Stream pretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, PretranslateWriterOptions); - - int trainCount = 0; - int pretranslateCount = 0; - pretranslateWriter.WriteStartArray(); - foreach (Corpus corpus in corpora) - { - ITextCorpus[] sourceTextCorpora = _corpusService.CreateTextCorpora(corpus.SourceFiles).ToArray(); - ITextCorpus targetTextCorpus = - _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault() ?? new DictionaryTextCorpus(); - - if (sourceTextCorpora.Length == 0) - continue; - - int skipCount = 0; - foreach (Row?[] rows in AlignTrainCorpus(sourceTextCorpora, targetTextCorpus)) - { - if (skipCount > 0) - { - skipCount--; - continue; - } - - Row[] trainRows = rows.Where(r => r is not null && IsInTrain(r, corpus)).Cast().ToArray(); - if (trainRows.Length > 0) - { - Row row = trainRows[0]; - if (rows.Length > 1) - { - Row[] nonEmptyRows = trainRows.Where(r => r.SourceSegment.Length > 0).ToArray(); - if (nonEmptyRows.Length > 0) - row = nonEmptyRows[_random.Next(nonEmptyRows.Length)]; - } - - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - skipCount = row.RowCount - 1; - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - } - } - - if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) - { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTermCorpus is not null && targetTermCorpus is not null) - { - IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); - foreach (ParallelTextRow row in parallelKeyTermsCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - trainCount++; - } - } - } - - foreach (Row row in AlignPretranslateCorpus(sourceTextCorpora[0], targetTextCorpus)) - { - if ( - IsInPretranslate(row, corpus) - && row.SourceSegment.Length > 0 - && (row.TargetSegment.Length == 0 || !IsInTrain(row, corpus)) - ) - { - pretranslateWriter.WriteStartObject(); - pretranslateWriter.WriteString("corpusId", corpus.Id); - pretranslateWriter.WriteString("textId", row.TextId); - pretranslateWriter.WriteStartArray("refs"); - foreach (object rowRef in row.Refs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteString("translation", row.SourceSegment); - pretranslateWriter.WriteEndObject(); - pretranslateCount++; - } - } - } - - pretranslateWriter.WriteEndArray(); - - return (trainCount, pretranslateCount); - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - IReadOnlyList data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Canceled) - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - } - } - - private static bool IsInTrain(Row row, Corpus corpus) - { - return IsIncluded(row, corpus.TrainOnTextIds, corpus.TrainOnChapters); - } - - private static bool IsInPretranslate(Row row, Corpus corpus) - { - return IsIncluded(row, corpus.PretranslateTextIds, corpus.PretranslateChapters); - } - - private static bool IsIncluded( - Row? row, - IReadOnlySet? textIds, - IReadOnlyDictionary>? chapters - ) - { - if (row is null) - return false; - if (chapters is not null) - return row.Refs.Any(r => IsInChapters(chapters, r)); - if (textIds is not null) - return textIds.Contains(row.TextId); - return true; - } - - private static bool IsInChapters(IReadOnlyDictionary> bookChapters, object rowRef) - { - if (rowRef is not ScriptureRef sr) - return false; - return bookChapters.TryGetValue(sr.Book, out HashSet? chapters) - && (chapters.Contains(sr.ChapterNum) || chapters.Count == 0); - } - - private static IEnumerable AlignTrainCorpus(IReadOnlyList srcCorpora, ITextCorpus trgCorpus) - { - if (trgCorpus.IsScripture()) - { - return srcCorpora - .Select(sc => AlignScripture(sc, trgCorpus)) - .ZipMany(rows => rows.ToArray()) - // filter out every list that only contains completely empty rows - .Where(rows => rows.Any(r => r is null || r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); - } - - IEnumerable sourceOnlyRows = srcCorpora - .Select(sc => sc.AlignRows(trgCorpus, allSourceRows: true)) - .ZipMany(rows => - rows.Where(r => r.TargetSegment.Count == 0) - .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) - .ToArray() - ); - - IEnumerable targetRows = srcCorpora - .Select(sc => sc.AlignRows(trgCorpus, allTargetRows: true)) - .ZipMany(rows => - rows.Where(r => r.TargetSegment.Count > 0) - .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) - .ToArray() - ); - - return sourceOnlyRows - .Concat(targetRows) - // filter out every list that only contains completely empty rows - .Where(rows => rows.Any(r => r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); - } - - private static IEnumerable AlignScripture(ITextCorpus srcCorpus, ITextCorpus trgCorpus) - { - int rowCount = 0; - StringBuilder srcSegBuffer = new(); - StringBuilder trgSegBuffer = new(); - HashSet vrefs = []; - foreach ( - (VerseRef vref, string srcSegment, string trgSegment) in srcCorpus - .ExtractScripture() - .Select(r => (r.CorpusVerseRef, r.Text)) - .Zip( - trgCorpus.ExtractScripture().Select(r => r.Text), - (s, t) => (VerseRef: s.CorpusVerseRef, SourceSegment: s.Text, TargetSegment: t) - ) - ) - { - if (srcSegment == "" && trgSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - rowCount++; - } - else if (srcSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - if (trgSegment.Length > 0) - { - if (trgSegBuffer.Length > 0) - trgSegBuffer.Append(' '); - trgSegBuffer.Append(trgSegment); - } - rowCount++; - } - else if (trgSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - if (srcSegment.Length > 0) - { - if (srcSegBuffer.Length > 0) - srcSegBuffer.Append(' '); - srcSegBuffer.Append(srcSegment); - } - rowCount++; - } - else - { - if (rowCount > 0) - { - yield return new( - vrefs.First().Book, - vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), - srcSegBuffer.ToString(), - trgSegBuffer.ToString(), - rowCount - ); - for (int i = 0; i < rowCount - 1; i++) - yield return null; - srcSegBuffer.Clear(); - trgSegBuffer.Clear(); - vrefs.Clear(); - rowCount = 0; - } - vrefs.UnionWith(vref.AllVerses()); - srcSegBuffer.Append(srcSegment); - trgSegBuffer.Append(trgSegment); - rowCount++; - } - } - - if (rowCount > 0) - { - yield return new( - vrefs.First().Book, - vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), - srcSegBuffer.ToString(), - trgSegBuffer.ToString(), - rowCount - ); - for (int i = 0; i < rowCount - 1; i++) - yield return null; - } - } - - private static IEnumerable AlignPretranslateCorpus(ITextCorpus srcCorpus, ITextCorpus trgCorpus) - { - int rowCount = 0; - StringBuilder srcSegBuffer = new(); - StringBuilder trgSegBuffer = new(); - List refs = []; - string textId = ""; - foreach (ParallelTextRow row in srcCorpus.AlignRows(trgCorpus, allSourceRows: true)) - { - if (!row.IsTargetRangeStart && row.IsTargetInRange) - { - refs.AddRange(row.Refs); - if (row.SourceText.Length > 0) - { - if (srcSegBuffer.Length > 0) - srcSegBuffer.Append(' '); - srcSegBuffer.Append(row.SourceText); - } - rowCount++; - } - else - { - if (rowCount > 0) - { - yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); - textId = ""; - srcSegBuffer.Clear(); - trgSegBuffer.Clear(); - refs.Clear(); - rowCount = 0; - } - - textId = row.TextId; - refs.AddRange(row.Refs); - srcSegBuffer.Append(row.SourceText); - trgSegBuffer.Append(row.TargetText); - rowCount++; - } - } - - if (rowCount > 0) - yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); - } - - private record Row( - string TextId, - IReadOnlyList Refs, - string SourceSegment, - string TargetSegment, - int RowCount - ); - - protected virtual bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) - { - resolvedCode = languageCode; - return true; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs deleted file mode 100644 index 3a3cb11e..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs +++ /dev/null @@ -1,125 +0,0 @@ -using SIL.ObjectModel; -using static Serval.Machine.Shared.Utils.SharedFileUtils; - -namespace Serval.Machine.Shared.Services; - -public class S3FileStorage : DisposableBase, IFileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config { RegionEndpoint = RegionEndpoint.GetBySystemName(region) } - ); - - _bucketName = bucketName; - // Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level - // implementation - _basePath = Normalize(basePath, includeTrailingSlash: true); - _loggerFactory = loggerFactory; - } - - public async Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public async Task> ListFilesAsync( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + (string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true)), - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key[_basePath.Length..]).ToList(); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - return Task.FromResult( - _client.GetPreSignedURL( - new GetPreSignedUrlRequest - { - BucketName = _bucketName, - Key = _basePath + Normalize(path), - Expires = expiresAt, - ResponseHeaderOverrides = new ResponseHeaderOverrides - { - ContentDisposition = new ContentDisposition() { FileName = Path.GetFileName(path) }.ToString() - } - } - ) - ); - } - - public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - GetObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {path} does not exist"); - return response.ResponseStream; - } - - public async Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - string fullPath = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = fullPath }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync( - request, - cancellationToken - ); - return new BufferedStream( - new S3WriteStream(_client, fullPath, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.MaxPartSize - ); - } - - public async Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.NoContent)) - { - throw new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs deleted file mode 100644 index 69eedc14..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs +++ /dev/null @@ -1,65 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class S3HealthCheck(IOptions options) : IHealthCheck -{ - private readonly IOptions _options = options; - private int _numConsecutiveFailures = 0; - private readonly AsyncLock _lock = new AsyncLock(); - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - try - { - var request = new ListObjectsV2Request - { - BucketName = new Uri(_options.Value.Uri).Host, - Prefix = new Uri(_options.Value.Uri).AbsolutePath + "/models/", - MaxKeys = 1, - Delimiter = "" - }; - - await new AmazonS3Client( - _options.Value.S3AccessKeyId, - _options.Value.S3SecretAccessKey, - new AmazonS3Config - { - MaxErrorRetry = 0, //Do not let health check hang - RegionEndpoint = RegionEndpoint.GetBySystemName(_options.Value.S3Region) - } - ).ListObjectsV2Async(request, cancellationToken); - using (await _lock.LockAsync(cancellationToken)) - _numConsecutiveFailures = 0; - return HealthCheckResult.Healthy("The S3 bucket is available"); - } - catch (Exception e) - { - using (await _lock.LockAsync(cancellationToken)) - { - _numConsecutiveFailures++; - if ( - e is HttpRequestException httpRequestException - && httpRequestException.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.Unauthorized - ) - { - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy( - "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." - ) - : HealthCheckResult.Degraded( - "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." - ); - } - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy( - "S3 bucket is not available. The following exception occurred: " + e.Message - ) - : HealthCheckResult.Degraded( - "S3 bucket is not available. The following exception occurred: " + e.Message - ); - } - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs deleted file mode 100644 index 4b623d6d..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs +++ /dev/null @@ -1,232 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory -) : Stream -{ - private readonly AmazonS3Client _client = client; - private readonly string _key = key; - private readonly string _uploadId = uploadId; - private readonly string _bucketName = bucketName; - private readonly List _uploadResponses = new List(); - private readonly ILogger _logger = loggerFactory.CreateLogger(); - - public const int MaxPartSize = 5 * 1024 * 1024; - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async ValueTask WriteAsync( - ReadOnlyMemory buffer, - CancellationToken cancellationToken = default - ) - { - try - { - using Stream stream = buffer.AsStream(); - - int bytesWritten = 0; - - while (stream.Length > bytesWritten) - { - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = stream, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug( - "Transferred {e.TransferredBytes}/{e.TotalBytes}", - e.TransferredBytes, - e.TotalBytes - ); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - - _uploadResponses.Add(response); - - bytesWritten += MaxPartSize; - } - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - await WriteAsync(buffer.AsMemory(offset, count), cancellationToken); - } - - protected override void Dispose(bool disposing) - { - try - { - if (disposing) - { - if (_uploadResponses.Count == 0) - { - AbortAsync().WaitAndUnwrapException(); - PutObjectRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - ContentBody = "" - }; - PutObjectResponse response = _client.PutObjectAsync(request).WaitAndUnwrapException(); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - else - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - } - } - finally - { - base.Dispose(disposing); - } - } - - public override async ValueTask DisposeAsync() - { - try - { - if (_uploadResponses.Count == 0) - { - await AbortAsync(); - PutObjectRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - ContentBody = "" - }; - PutObjectResponse response = await _client.PutObjectAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - - return; - } - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - catch (Exception e) - { - await AbortAsync(e); - } - } - finally - { - await base.DisposeAsync(); - GC.SuppressFinalize(this); - } - } - - private async Task AbortAsync(Exception? e = null) - { - if (e is not null) - _logger.LogError(e, "Aborted upload {UploadId} to {BucketName}/{Key}", _uploadId, _bucketName, _key); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs deleted file mode 100644 index 493cb9ed..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public static class ServalPlatformOutboxConstants -{ - public const string OutboxId = "ServalPlatform"; - - public const string BuildStarted = "BuildStarted"; - public const string BuildCompleted = "BuildCompleted"; - public const string BuildCanceled = "BuildCanceled"; - public const string BuildFaulted = "BuildFaulted"; - public const string BuildRestarting = "BuildRestarting"; - public const string InsertPretranslations = "InsertPretranslations"; - public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize"; -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs deleted file mode 100644 index 9b1cf788..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs +++ /dev/null @@ -1,92 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalPlatformOutboxMessageHandler(TranslationPlatformApi.TranslationPlatformApiClient client) - : IOutboxMessageHandler -{ - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - - public string OutboxId => ServalPlatformOutboxConstants.OutboxId; - - public async Task HandleMessageAsync( - string method, - string? content, - Stream? contentStream, - CancellationToken cancellationToken = default - ) - { - switch (method) - { - case ServalPlatformOutboxConstants.BuildStarted: - await _client.BuildStartedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildCompleted: - await _client.BuildCompletedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildCanceled: - await _client.BuildCanceledAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildFaulted: - await _client.BuildFaultedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildRestarting: - await _client.BuildRestartingAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.InsertPretranslations: - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - contentStream!, - JsonSerializerOptions, - cancellationToken - ) - .OfType(); - - using (var call = _client.InsertPretranslations(cancellationToken: cancellationToken)) - { - await foreach (Pretranslation pretranslation in pretranslations) - { - await call.RequestStream.WriteAsync( - new InsertPretranslationRequest - { - EngineId = content!, - CorpusId = pretranslation.CorpusId, - TextId = pretranslation.TextId, - Refs = { pretranslation.Refs }, - Translation = pretranslation.Translation - }, - cancellationToken - ); - } - await call.RequestStream.CompleteAsync(); - await call; - } - break; - case ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize: - await _client.IncrementTranslationEngineCorpusSizeAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - default: - throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'."); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs deleted file mode 100644 index 429fbb72..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs +++ /dev/null @@ -1,140 +0,0 @@ -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalPlatformService( - TranslationPlatformApi.TranslationPlatformApiClient client, - IMessageOutboxService outboxService -) : IPlatformService -{ - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private readonly IMessageOutboxService _outboxService = outboxService; - - public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildStarted, - buildId, - JsonSerializer.Serialize(new BuildStartedRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildCompletedAsync( - string buildId, - int trainSize, - double confidence, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildCompleted, - buildId, - JsonSerializer.Serialize( - new BuildCompletedRequest - { - BuildId = buildId, - CorpusSize = trainSize, - Confidence = confidence - } - ), - cancellationToken: cancellationToken - ); - } - - public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildCanceled, - buildId, - JsonSerializer.Serialize(new BuildCanceledRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildFaulted, - buildId, - JsonSerializer.Serialize(new BuildFaultedRequest { BuildId = buildId, Message = message }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildRestarting, - buildId, - JsonSerializer.Serialize(new BuildRestartingRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task UpdateBuildStatusAsync( - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; - if (progressStatus.PercentCompleted.HasValue) - request.PercentCompleted = progressStatus.PercentCompleted.Value; - if (progressStatus.Message is not null) - request.Message = progressStatus.Message; - if (queueDepth is not null) - request.QueueDepth = queueDepth.Value; - - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); - } - - public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) - { - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync( - new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, - cancellationToken: cancellationToken - ); - } - - public async Task InsertPretranslationsAsync( - string engineId, - Stream pretranslationsStream, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.InsertPretranslations, - engineId, - engineId, - pretranslationsStream, - cancellationToken: cancellationToken - ); - } - - public async Task IncrementTrainSizeAsync( - string engineId, - int count = 1, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize, - engineId, - JsonSerializer.Serialize( - new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count } - ), - cancellationToken: cancellationToken - ); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs deleted file mode 100644 index 863fd158..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs +++ /dev/null @@ -1,341 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Services; - -public class ServalTranslationEngineServiceV1( - IEnumerable engineServices, - HealthCheckService healthCheckService -) : TranslationEngineApi.TranslationEngineApiBase -{ - private static readonly Empty Empty = new(); - - private readonly Dictionary _engineServices = - engineServices.ToDictionary(es => es.Type); - - private readonly HealthCheckService _healthCheckService = healthCheckService; - - public override async Task Create(CreateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - TranslationEngine translationEngine = await engineService.CreateAsync( - request.EngineId, - request.HasEngineName ? request.EngineName : null, - request.SourceLanguage, - request.TargetLanguage, - request.HasIsModelPersisted ? request.IsModelPersisted : null, - context.CancellationToken - ); - return new CreateResponse { IsModelPersisted = translationEngine.IsModelPersisted }; - } - - public override async Task Delete(DeleteRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.DeleteAsync(request.EngineId, context.CancellationToken); - return Empty; - } - - public override async Task Translate(TranslateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - IEnumerable results; - try - { - results = await engineService.TranslateAsync( - request.EngineId, - request.N, - request.Segment, - context.CancellationToken - ); - } - catch (EngineNotBuiltException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - - return new TranslateResponse { Results = { results.Select(Map) } }; - } - - public override async Task GetWordGraph( - GetWordGraphRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - SIL.Machine.Translation.WordGraph wordGraph; - try - { - wordGraph = await engineService.GetWordGraphAsync( - request.EngineId, - request.Segment, - context.CancellationToken - ); - } - catch (EngineNotBuiltException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return new GetWordGraphResponse { WordGraph = Map(wordGraph) }; - } - - public override async Task TrainSegmentPair(TrainSegmentPairRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.TrainSegmentPairAsync( - request.EngineId, - request.SourceSegment, - request.TargetSegment, - request.SentenceStart, - context.CancellationToken - ); - return Empty; - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - Models.Corpus[] corpora = request.Corpora.Select(Map).ToArray(); - try - { - await engineService.StartBuildAsync( - request.EngineId, - request.BuildId, - request.HasOptions ? request.Options : null, - corpora, - context.CancellationToken - ); - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return Empty; - } - - public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - try - { - await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return Empty; - } - - public override async Task GetModelDownloadUrl( - GetModelDownloadUrlRequest request, - ServerCallContext context - ) - { - try - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - ModelDownloadUrl modelDownloadUrl = await engineService.GetModelDownloadUrlAsync( - request.EngineId, - context.CancellationToken - ); - return new GetModelDownloadUrlResponse - { - Url = modelDownloadUrl.Url, - ModelRevision = modelDownloadUrl.ModelRevision, - ExpiresAt = modelDownloadUrl.ExpiresAt.ToTimestamp() - }; - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message)); - } - catch (FileNotFoundException e) - { - throw new RpcException(new Status(StatusCode.NotFound, e.Message)); - } - } - - public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); - } - - public override Task GetLanguageInfo( - GetLanguageInfoRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - bool isNative = engineService.IsLanguageNativeToModel(request.Language, out string internalCode); - return Task.FromResult(new GetLanguageInfoResponse { InternalCode = internalCode, IsNative = isNative, }); - } - - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } - - private ITranslationEngineService GetEngineService(string engineTypeStr) - { - if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) - return service; - throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); - } - - private static TranslationEngineType GetEngineType(string engineTypeStr) - { - engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; - if (System.Enum.TryParse(engineTypeStr, out TranslationEngineType engineType)) - return engineType; - throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); - } - - private static Translation.V1.TranslationResult Map(SIL.Machine.Translation.TranslationResult source) - { - return new Translation.V1.TranslationResult - { - Translation = source.Translation, - SourceTokens = { source.SourceTokens }, - TargetTokens = { source.TargetTokens }, - Confidences = { source.Confidences }, - Sources = { source.Sources.Select(Map) }, - Alignment = { Map(source.Alignment) }, - Phrases = { source.Phrases.Select(Map) } - }; - } - - private static Translation.V1.WordGraph Map(SIL.Machine.Translation.WordGraph source) - { - return new Translation.V1.WordGraph - { - SourceTokens = { source.SourceTokens }, - InitialStateScore = source.InitialStateScore, - FinalStates = { source.FinalStates }, - Arcs = { source.Arcs.Select(Map) } - }; - } - - private static Translation.V1.WordGraphArc Map(SIL.Machine.Translation.WordGraphArc source) - { - return new Translation.V1.WordGraphArc - { - PrevState = source.PrevState, - NextState = source.NextState, - Score = source.Score, - TargetTokens = { source.TargetTokens }, - Alignment = { Map(source.Alignment) }, - Confidences = { source.Confidences }, - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - Sources = { source.Sources.Select(Map) } - }; - } - - private static Translation.V1.TranslationSources Map(SIL.Machine.Translation.TranslationSources source) - { - return new Translation.V1.TranslationSources - { - Values = - { - System - .Enum.GetValues() - .Where(s => s != SIL.Machine.Translation.TranslationSources.None && source.HasFlag(s)) - .Select(s => - s switch - { - SIL.Machine.Translation.TranslationSources.Smt => TranslationSource.Primary, - SIL.Machine.Translation.TranslationSources.Nmt => TranslationSource.Primary, - SIL.Machine.Translation.TranslationSources.Transfer => TranslationSource.Secondary, - SIL.Machine.Translation.TranslationSources.Prefix => TranslationSource.Human, - _ => TranslationSource.Primary - } - ) - } - }; - } - - private static IEnumerable Map(WordAlignmentMatrix source) - { - for (int i = 0; i < source.RowCount; i++) - { - for (int j = 0; j < source.ColumnCount; j++) - { - if (source[i, j]) - yield return new Translation.V1.AlignedWordPair { SourceIndex = i, TargetIndex = j }; - } - } - } - - private static Translation.V1.Phrase Map(SIL.Machine.Translation.Phrase source) - { - return new Translation.V1.Phrase - { - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - TargetSegmentCut = source.TargetSegmentCut - }; - } - - private static Models.Corpus Map(Translation.V1.Corpus source) - { - var pretranslateChapters = source.PretranslateChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - FilterChoice pretranslateFilter = GetFilterChoice(source.PretranslateAll, pretranslateChapters); - - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - FilterChoice trainingFilter = GetFilterChoice(source.TrainOnAll, trainOnChapters); - - return new Models.Corpus - { - Id = source.Id, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - PretranslateChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? source.TrainOnTextIds.ToHashSet() : null, - PretranslateTextIds = - pretranslateFilter == FilterChoice.TextIds ? source.PretranslateTextIds.ToHashSet() : null, - SourceFiles = source.SourceFiles.Select(Map).ToList(), - TargetFiles = source.TargetFiles.Select(Map).ToList() - }; - } - - private static Models.CorpusFile Map(Translation.V1.CorpusFile source) - { - return new Models.CorpusFile - { - Location = source.Location, - Format = (Models.FileFormat)source.Format, - TextId = source.TextId - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None - } - - private static FilterChoice GetFilterChoice(bool all, IReadOnlyDictionary> chapters) - { - if (all) - return FilterChoice.None; - - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (chapters.Count == 0) - return FilterChoice.TextIds; - else - return FilterChoice.Chapters; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs deleted file mode 100644 index 8d6ef0b9..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs +++ /dev/null @@ -1,104 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly IFileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public async Task GetDownloadUrlAsync(string path, DateTime expiresAt) - { - return await _fileStorage.GetDownloadUrlAsync(path, expiresAt); - } - - public Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.ListFilesAsync(path, recurse, cancellationToken); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenReadAsync(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWriteAsync(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.ListFilesAsync( - path, - recurse: true, - cancellationToken - ); - foreach (string file in files) - await _fileStorage.DeleteAsync(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.DeleteAsync(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.ExistsAsync(path, cancellationToken); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs deleted file mode 100644 index c83f0703..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs +++ /dev/null @@ -1,166 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - IRepository trainSegmentPairs, - ITruecaserFactory truecaserFactory, - ISmtModelFactory smtModelFactory, - ICorpusService corpusService -) - : HangfireBuildJob>( - platformService, - engines, - lockFactory, - dataAccessContext, - buildJobService, - logger - ) -{ - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ICorpusService _corpusService = corpusService; - - protected override Task InitializeAsync( - string engineId, - string buildId, - IReadOnlyList data, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - return _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, cancellationToken); - } - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - IReadOnlyList data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - await PlatformService.BuildStartedAsync(buildId, cancellationToken); - Logger.LogInformation("Build started ({0})", buildId); - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - cancellationToken.ThrowIfCancellationRequested(); - - JsonObject? buildOptionsObject = null; - if (buildOptions is not null) - buildOptionsObject = JsonSerializer.Deserialize(buildOptions); - - var targetCorpora = new List(); - var parallelCorpora = new List(); - foreach (Corpus corpus in data) - { - ITextCorpus? sourceTextCorpus = _corpusService.CreateTextCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTextCorpus = _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTextCorpus is null || targetTextCorpus is null) - continue; - - targetCorpora.Add(targetTextCorpus); - parallelCorpora.Add(sourceTextCorpus.AlignRows(targetTextCorpus)); - - if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) - { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTermCorpus is not null && targetTermCorpus is not null) - { - IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); - parallelCorpora.Add(parallelKeyTermsCorpus); - } - } - } - - IParallelTextCorpus parallelCorpus = parallelCorpora.Flatten(); - ITextCorpus targetCorpus = targetCorpora.Flatten(); - - var tokenizer = new LatinWordTokenizer(); - var detokenizer = new LatinWordDetokenizer(); - - using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync( - engineId, - tokenizer, - parallelCorpus, - cancellationToken - ); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( - engineId, - tokenizer, - targetCorpus, - cancellationToken - ); - - cancellationToken.ThrowIfCancellationRequested(); - - var progress = new BuildProgress(PlatformService, buildId); - await smtModelTrainer.TrainAsync(progress, cancellationToken); - await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); - - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException(); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - cancellationToken.ThrowIfCancellationRequested(); - await smtModelTrainer.SaveAsync(CancellationToken.None); - await truecaseTrainer.SaveAsync(CancellationToken.None); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineId, CancellationToken.None); - IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync( - p => p.TranslationEngineRef == engine.Id, - CancellationToken.None - ); - using ( - IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineId, - tokenizer, - detokenizer, - truecaser, - CancellationToken.None - ) - ) - { - foreach (TrainSegmentPair segmentPair in segmentPairs) - { - await smtModel.TrainSegmentAsync( - segmentPair.Source, - segmentPair.Target, - cancellationToken: CancellationToken.None - ); - } - } - - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildCompletedAsync( - buildId, - smtModelTrainer.Stats.TrainCorpusSize + segmentPairs.Count, - smtModelTrainer.Stats.Metrics["bleu"] * 100.0, - CancellationToken.None - ); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: true, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - - stopwatch.Stop(); - Logger.LogInformation("Build completed in {0}s ({1})", stopwatch.Elapsed.TotalSeconds, buildId); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs deleted file mode 100644 index 6e0b6b9c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferClearMLBuildJobFactory( - ISharedFileService sharedFileService, - IRepository engines -) : IClearMLBuildJobFactory -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly IRepository _engines = engines; - - public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; - - public async Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - if (stage == BuildStage.Train) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException("The engine does not exist."); - - Uri sharedFileUri = _sharedFileService.GetBaseUri(); - string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); - string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); - return "from machine.jobs.build_smt_engine import run\n" - + "args = {\n" - + $" 'model_type': '{modelType}',\n" - + $" 'engine_id': '{engineId}',\n" - + $" 'build_id': '{buildId}',\n" - + $" 'shared_file_uri': '{baseUri}',\n" - + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") - + $" 'clearml': True\n" - + "}\n" - + "run(args)\n"; - } - else - { - throw new ArgumentException("Unknown build stage.", nameof(stage)); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs deleted file mode 100644 index 8e802ce6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferEngineCommitService( - IServiceProvider services, - IOptionsMonitor engineOptions, - SmtTransferEngineStateService stateService, - ILogger logger -) - : RecurrentTask( - "SMT transfer engine commit service", - services, - engineOptions.CurrentValue.EngineCommitFrequency, - logger - ) -{ - private readonly IOptionsMonitor _engineOptions = engineOptions; - private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly ILogger _logger = logger; - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - var engines = scope.ServiceProvider.GetRequiredService>(); - var lockFactory = scope.ServiceProvider.GetRequiredService(); - await _stateService.CommitAsync( - lockFactory, - engines, - _engineOptions.CurrentValue.InactiveEngineTimeout, - cancellationToken - ); - } - catch (Exception e) - { - _logger.LogError(e, "Error occurred while committing SMT transfer engines."); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs deleted file mode 100644 index bdda5353..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs +++ /dev/null @@ -1,272 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferEngineService( - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService, - IBuildJobService buildJobService, - IClearMLQueueService clearMLQueueService -) : ITranslationEngineService -{ - private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; - private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IRepository _engines = engines; - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly IBuildJobService _buildJobService = buildJobService; - private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - - public TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ) - { - if (isModelPersisted == false) - { - throw new NotSupportedException( - "SMT transfer engines do not support non-persisted models." - + "Please remove the isModelPersisted parameter or set it to true." - ); - } - - TranslationEngine translationEngine = await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - var translationEngine = new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage, - Type = TranslationEngineType.SmtTransfer, - IsModelPersisted = isModelPersisted ?? true // models are persisted if not specified - }; - await _engines.InsertAsync(translationEngine, ct); - await _buildJobService.CreateEngineAsync(engineId, engineName, ct); - return translationEngine; - }, - cancellationToken: cancellationToken - ); - - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - SmtTransferEngineState state = _stateService.Get(engineId); - await state.InitNewAsync(CancellationToken.None); - } - return translationEngine; - } - - public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildJobAsync(engineId, cancellationToken); - - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - await _engines.DeleteAsync(e => e.EngineId == engineId, ct); - await _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, ct); - }, - cancellationToken: cancellationToken - ); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); - - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - await _lockFactory.DeleteAsync(engineId, CancellationToken.None); - } - - public async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - SmtTransferEngineState state = _stateService.Get(engineId); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - SmtTransferEngineState state = _stateService.Get(engineId); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - - async Task TrainSubroutineAsync(SmtTransferEngineState state, CancellationToken ct) - { - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, ct); - await _platformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - } - - SmtTransferEngineState state = _stateService.Get(engineId); - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - if (engine.CurrentBuild?.JobState is BuildJobState.Active) - { - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engineId, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - CancellationToken.None - ); - await TrainSubroutineAsync(state, CancellationToken.None); - } - else - { - await TrainSubroutineAsync(state, ct); - } - }, - cancellationToken: cancellationToken - ); - - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public async Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - // If there is a pending/running build, then no need to start a new one. - if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is already building or in the process of canceling."); - - await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - cancellationToken - ); - SmtTransferEngineState state = _stateService.Get(engineId); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await CancelBuildJobAsync(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is not currently building."); - SmtTransferEngineState state = _stateService.Get(engineId); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public int GetQueueSize() - { - return _clearMLQueueService.GetQueueSize(Type); - } - - public bool IsLanguageNativeToModel(string language, out string internalCode) - { - throw new NotSupportedException("SMT transfer engines do not support language info."); - } - - private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) - { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken - ); - return buildId is not null; - } - - public Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException($"The engine {engineId} does not exist."); - return engine; - } - - private async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0) - throw new EngineNotBuiltException("The engine must be built first."); - return engine; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs deleted file mode 100644 index a5f4300a..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs +++ /dev/null @@ -1,128 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferEngineState( - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options, - string engineId -) : AsyncDisposableBase -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IOptionsMonitor _options = options; - private readonly AsyncLock _lock = new(); - - private IInteractiveTranslationModel? _smtModel; - private HybridTranslationEngine? _hybridEngine; - - public string EngineId { get; } = engineId; - - public bool IsUpdated { get; set; } - public int CurrentBuildRevision { get; set; } = -1; - public DateTime LastUsedTime { get; set; } = DateTime.UtcNow; - public bool IsLoaded => _hybridEngine != null; - - private string EngineDir => Path.Combine(_options.CurrentValue.EnginesDir, EngineId); - - public async Task InitNewAsync(CancellationToken cancellationToken = default) - { - await _smtModelFactory.InitNewAsync(EngineDir, cancellationToken); - await _transferEngineFactory.InitNewAsync(EngineDir, cancellationToken); - } - - public async Task GetHybridEngineAsync(int buildRevision) - { - using (await _lock.LockAsync()) - { - if (_hybridEngine is not null && CurrentBuildRevision != -1 && buildRevision != CurrentBuildRevision) - { - IsUpdated = false; - await UnloadAsync(); - } - - if (_hybridEngine is null) - { - LatinWordTokenizer tokenizer = new(); - LatinWordDetokenizer detokenizer = new(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(EngineDir); - _smtModel = await _smtModelFactory.CreateAsync(EngineDir, tokenizer, detokenizer, truecaser); - ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( - EngineDir, - tokenizer, - detokenizer, - truecaser - ); - _hybridEngine = new HybridTranslationEngine(_smtModel, transferEngine) - { - TargetDetokenizer = detokenizer - }; - } - CurrentBuildRevision = buildRevision; - return _hybridEngine; - } - } - - public async Task DeleteDataAsync() - { - await UnloadAsync(); - await _smtModelFactory.CleanupAsync(EngineDir); - await _transferEngineFactory.CleanupAsync(EngineDir); - await _truecaserFactory.CleanupAsync(EngineDir); - } - - public async Task CommitAsync( - int buildRevision, - TimeSpan inactiveTimeout, - CancellationToken cancellationToken = default - ) - { - if (_hybridEngine is null) - return; - - if (CurrentBuildRevision == -1) - CurrentBuildRevision = buildRevision; - if (buildRevision != CurrentBuildRevision) - { - await UnloadAsync(cancellationToken); - CurrentBuildRevision = buildRevision; - } - else if (DateTime.Now - LastUsedTime > inactiveTimeout) - { - await UnloadAsync(cancellationToken); - } - else - { - await SaveModelAsync(cancellationToken); - } - } - - private async Task SaveModelAsync(CancellationToken cancellationToken = default) - { - if (_smtModel is not null && IsUpdated) - { - await _smtModel.SaveAsync(cancellationToken); - IsUpdated = false; - } - } - - private async Task UnloadAsync(CancellationToken cancellationToken = default) - { - if (_hybridEngine is null) - return; - - await SaveModelAsync(cancellationToken); - - _hybridEngine.Dispose(); - - _smtModel = null; - _hybridEngine = null; - CurrentBuildRevision = -1; - } - - protected override async ValueTask DisposeAsyncCore() - { - await UnloadAsync(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs deleted file mode 100644 index 03ef2ad8..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs +++ /dev/null @@ -1,72 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferEngineStateService( - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options -) : AsyncDisposableBase -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IOptionsMonitor _options = options; - - private readonly ConcurrentDictionary _engineStates = - new ConcurrentDictionary(); - - public SmtTransferEngineState Get(string engineId) - { - return _engineStates.GetOrAdd(engineId, CreateState); - } - - public bool TryRemove(string engineId, [MaybeNullWhen(false)] out SmtTransferEngineState state) - { - return _engineStates.TryRemove(engineId, out state); - } - - public async Task CommitAsync( - IDistributedReaderWriterLockFactory lockFactory, - IRepository engines, - TimeSpan inactiveTimeout, - CancellationToken cancellationToken = default - ) - { - foreach (SmtTransferEngineState state in _engineStates.Values) - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(state.EngineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine? engine = await engines.GetAsync( - e => e.EngineId == state.EngineId, - cancellationToken - ); - if ( - engine is not null - && (engine.CurrentBuild is null || engine.CurrentBuild.JobState is BuildJobState.Pending) - ) - { - await state.CommitAsync(engine.BuildRevision, inactiveTimeout, cancellationToken); - } - } - } - } - - private SmtTransferEngineState CreateState(string engineId) - { - return new SmtTransferEngineState( - _smtModelFactory, - _transferEngineFactory, - _truecaserFactory, - _options, - engineId - ); - } - - protected override async ValueTask DisposeAsyncCore() - { - foreach (SmtTransferEngineState state in _engineStates.Values) - await state.DisposeAsync(); - _engineStates.Clear(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs deleted file mode 100644 index 9f532b2b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using static Serval.Machine.Shared.Services.HangfireBuildJobRunner; - -namespace Serval.Machine.Shared.Services; - -public class SmtTransferHangfireBuildJobFactory : IHangfireBuildJobFactory -{ - public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; - - public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) - { - return stage switch - { - BuildStage.Preprocess - => CreateJob>( - engineId, - buildId, - "smt_transfer", - data, - buildOptions - ), - BuildStage.Postprocess - => CreateJob( - engineId, - buildId, - "smt_transfer", - data, - buildOptions - ), - BuildStage.Train => CreateJob(engineId, buildId, "smt_transfer", buildOptions), - _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), - }; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs deleted file mode 100644 index d0d25fe5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs +++ /dev/null @@ -1,74 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferPostprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService, - IRepository trainSegmentPairs, - ISmtModelFactory smtModelFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options -) - : PostprocessBuildJob( - platformService, - engines, - lockFactory, - dataAccessContext, - buildJobService, - logger, - sharedFileService - ) -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly IOptionsMonitor _options = options; - - protected override async Task SaveModelAsync(string engineId, string buildId) - { - await using ( - Stream engineStream = await SharedFileService.OpenReadAsync( - $"builds/{buildId}/model.tar.gz", - CancellationToken.None - ) - ) - { - await _smtModelFactory.UpdateEngineFromAsync( - Path.Combine(_options.CurrentValue.EnginesDir, engineId), - engineStream, - CancellationToken.None - ); - } - return await TrainOnNewSegmentPairsAsync(engineId); - } - - private async Task TrainOnNewSegmentPairsAsync(string engineId) - { - IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync(p => - p.TranslationEngineRef == engineId - ); - if (segmentPairs.Count == 0) - return segmentPairs.Count; - - string engineDir = Path.Combine(_options.CurrentValue.EnginesDir, engineId); - var tokenizer = new LatinWordTokenizer(); - var detokenizer = new LatinWordDetokenizer(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir); - using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser - ); - foreach (TrainSegmentPair segmentPair in segmentPairs) - { - await smtModel.TrainSegmentAsync(segmentPair.Source, segmentPair.Target); - } - await smtModel.SaveAsync(); - return segmentPairs.Count; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs deleted file mode 100644 index bb4870c1..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs +++ /dev/null @@ -1,237 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class SmtTransferTrainBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService, - ITruecaserFactory truecaserFactory, - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory -) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - private const int BatchSize = 128; - - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - object? data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - using TempDirectory tempDir = new(buildId); - string corpusDir = Path.Combine(tempDir.Path, "corpus"); - await DownloadDataAsync(buildId, corpusDir, cancellationToken); - - // assemble corpus - ITextCorpus sourceCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.src.txt")); - ITextCorpus targetCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.trg.txt")); - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - // train SMT model - string engineDir = Path.Combine(tempDir.Path, "engine"); - (int trainCorpusSize, double confidence) = await TrainAsync( - buildId, - engineDir, - targetCorpus, - parallelCorpus, - cancellationToken - ); - - cancellationToken.ThrowIfCancellationRequested(); - - await GeneratePretranslationsAsync(buildId, engineDir, cancellationToken); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - bool canceling = !await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Postprocess, - data: (trainCorpusSize, confidence), - buildOptions: buildOptions, - cancellationToken: cancellationToken - ); - if (canceling) - throw new OperationCanceledException(); - } - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - object? data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Canceled) - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - } - } - - private async Task DownloadDataAsync(string buildId, string corpusDir, CancellationToken cancellationToken) - { - Directory.CreateDirectory(corpusDir); - await using Stream srcText = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/train.src.txt", - cancellationToken - ); - await using FileStream srcFileStream = File.Create(Path.Combine(corpusDir, "train.src.txt")); - await srcText.CopyToAsync(srcFileStream, cancellationToken); - - await using Stream tgtText = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/train.trg.txt", - cancellationToken - ); - await using FileStream tgtFileStream = File.Create(Path.Combine(corpusDir, "train.trg.txt")); - await tgtText.CopyToAsync(tgtFileStream, cancellationToken); - } - - private async Task<(int TrainCorpusSize, double Confidence)> TrainAsync( - string buildId, - string engineDir, - ITextCorpus targetCorpus, - IParallelTextCorpus parallelCorpus, - CancellationToken cancellationToken - ) - { - await _smtModelFactory.InitNewAsync(engineDir, cancellationToken); - LatinWordTokenizer tokenizer = new(); - int trainCorpusSize; - double confidence; - using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync( - engineDir, - tokenizer, - parallelCorpus, - cancellationToken - ); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( - engineDir, - tokenizer, - targetCorpus, - cancellationToken - ); - cancellationToken.ThrowIfCancellationRequested(); - - var progress = new BuildProgress(PlatformService, buildId); - await smtModelTrainer.TrainAsync(progress, cancellationToken); - await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); - - trainCorpusSize = smtModelTrainer.Stats.TrainCorpusSize; - confidence = smtModelTrainer.Stats.Metrics["bleu"] * 100.0; - - cancellationToken.ThrowIfCancellationRequested(); - - await smtModelTrainer.SaveAsync(cancellationToken); - await truecaseTrainer.SaveAsync(cancellationToken); - - await using Stream engineStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/model.tar.gz", - cancellationToken - ); - await _smtModelFactory.SaveEngineToAsync(engineDir, engineStream, cancellationToken); - return (trainCorpusSize, confidence); - } - - private async Task GeneratePretranslationsAsync( - string buildId, - string engineDir, - CancellationToken cancellationToken - ) - { - await using Stream sourceStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable(sourceStream, JsonSerializerOptions, cancellationToken) - .OfType(); - - await using Stream targetStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - await using Utf8JsonWriter targetWriter = new(targetStream, PretranslateWriterOptions); - - LatinWordTokenizer tokenizer = new(); - LatinWordDetokenizer detokenizer = new(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir, CancellationToken.None); - using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser, - cancellationToken - ); - using ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser, - cancellationToken - ); - HybridTranslationEngine hybridEngine = new(smtModel, transferEngine) { TargetDetokenizer = detokenizer }; - - await foreach (IReadOnlyList batch in BatchAsync(pretranslations)) - { - string[] segments = batch.Select(p => p.Translation).ToArray(); - IReadOnlyList results = await hybridEngine.TranslateBatchAsync( - segments, - cancellationToken - ); - foreach ((Pretranslation pretranslation, TranslationResult result) in batch.Zip(results)) - { - JsonSerializer.Serialize( - targetWriter, - pretranslation with - { - Translation = result.Translation - }, - JsonSerializerOptions - ); - } - } - } - - public static async IAsyncEnumerable> BatchAsync( - IAsyncEnumerable pretranslations - ) - { - List batch = []; - await foreach (Pretranslation item in pretranslations) - { - batch.Add(item); - if (batch.Count == BatchSize) - { - yield return batch; - batch = []; - } - } - if (batch.Count > 0) - yield return batch; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs deleted file mode 100644 index 031891c4..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class ThotSmtModelFactory(IOptionsMonitor options) : ISmtModelFactory -{ - private readonly IOptionsMonitor _options = options; - - public Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ) - { - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - IInteractiveTranslationModel model = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, smtConfigFileName) - { - SourceTokenizer = tokenizer, - TargetTokenizer = tokenizer, - TargetDetokenizer = detokenizer, - LowercaseSource = true, - LowercaseTarget = true, - Truecaser = truecaser - }; - return Task.FromResult(model); - } - - public Task CreateTrainerAsync( - string engineDir, - IRangeTokenizer tokenizer, - IParallelTextCorpus corpus, - CancellationToken cancellationToken = default - ) - { - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - ITrainer trainer = new ThotSmtModelTrainer(ThotWordAlignmentModelType.Hmm, corpus, smtConfigFileName) - { - SourceTokenizer = tokenizer, - TargetTokenizer = tokenizer, - LowercaseSource = true, - LowercaseTarget = true - }; - return Task.FromResult(trainer); - } - - public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - Directory.CreateDirectory(engineDir); - ZipFile.ExtractToDirectory(_options.CurrentValue.NewModelFile, engineDir); - return Task.CompletedTask; - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - return Task.CompletedTask; - DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "lm")); - DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "tm")); - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - if (File.Exists(smtConfigFileName)) - File.Delete(smtConfigFileName); - if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) - Directory.Delete(engineDir); - return Task.CompletedTask; - } - - public async Task UpdateEngineFromAsync( - string engineDir, - Stream source, - CancellationToken cancellationToken = default - ) - { - if (!Directory.Exists(engineDir)) - Directory.CreateDirectory(engineDir); - - await using MemoryStream memoryStream = new(); - await using (GZipStream gzipStream = new(source, CompressionMode.Decompress)) - { - await gzipStream.CopyToAsync(memoryStream, cancellationToken); - } - memoryStream.Seek(0, SeekOrigin.Begin); - await TarFile.ExtractToDirectoryAsync( - memoryStream, - engineDir, - overwriteFiles: true, - cancellationToken: cancellationToken - ); - } - - public async Task SaveEngineToAsync( - string engineDir, - Stream destination, - CancellationToken cancellationToken = default - ) - { - // create zip archive in memory stream - // This cannot be created directly to the shared stream because it all needs to be written at once - await using MemoryStream memoryStream = new(); - await TarFile.CreateFromDirectoryAsync( - engineDir, - memoryStream, - includeBaseDirectory: false, - cancellationToken: cancellationToken - ); - memoryStream.Seek(0, SeekOrigin.Begin); - await using GZipStream gzipStream = new(destination, CompressionMode.Compress); - await memoryStream.CopyToAsync(gzipStream, cancellationToken); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs deleted file mode 100644 index a140792b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs +++ /dev/null @@ -1,61 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class TransferEngineFactory : ITransferEngineFactory -{ - public Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ) - { - string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); - string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); - ITranslationEngine? transferEngine = null; - if (File.Exists(hcSrcConfigFileName) && File.Exists(hcTrgConfigFileName)) - { - var hcTraceManager = new TraceManager(); - - Language srcLang = XmlLanguageLoader.Load(hcSrcConfigFileName); - var srcMorpher = new Morpher(hcTraceManager, srcLang); - - Language trgLang = XmlLanguageLoader.Load(hcTrgConfigFileName); - var trgMorpher = new Morpher(hcTraceManager, trgLang); - - transferEngine = new TransferEngine( - srcMorpher, - new SimpleTransferer(new GlossMorphemeMapper(trgMorpher)), - trgMorpher - ) - { - SourceTokenizer = tokenizer, - TargetDetokenizer = detokenizer, - LowercaseSource = true, - Truecaser = truecaser - }; - } - return Task.FromResult(transferEngine); - } - - public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) - { - // TODO: generate source and target config files - return Task.CompletedTask; - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - return Task.CompletedTask; - string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); - if (File.Exists(hcSrcConfigFileName)) - File.Delete(hcSrcConfigFileName); - string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); - if (File.Exists(hcTrgConfigFileName)) - File.Delete(hcTrgConfigFileName); - if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) - Directory.Delete(engineDir); - return Task.CompletedTask; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs deleted file mode 100644 index 61df1966..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public enum TranslationEngineType -{ - SmtTransfer, - Nmt -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs deleted file mode 100644 index cbf9c8b5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class UnigramTruecaserFactory : ITruecaserFactory -{ - public async Task CreateAsync(string engineDir, CancellationToken cancellationToken = default) - { - var truecaser = new UnigramTruecaser(); - string path = GetModelPath(engineDir); - await truecaser.LoadAsync(path); - return truecaser; - } - - public Task CreateTrainerAsync( - string engineDir, - ITokenizer tokenizer, - ITextCorpus corpus, - CancellationToken cancellationToken = default - ) - { - string path = GetModelPath(engineDir); - ITrainer trainer = new UnigramTruecaserTrainer(path, corpus) { Tokenizer = tokenizer }; - return Task.FromResult(trainer); - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - string path = GetModelPath(engineDir); - if (File.Exists(path)) - File.Delete(path); - return Task.CompletedTask; - } - - private static string GetModelPath(string engineDir) - { - return Path.Combine(engineDir, "unigram-casing-model.txt"); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs deleted file mode 100644 index c75812d6..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -public class UnimplementedInterceptor : Interceptor -{ - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (NotSupportedException) - { - throw new RpcException( - new Status(StatusCode.Unimplemented, "The call is not supported by the specified engine.") - ); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs deleted file mode 100644 index 244e090b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Usings.cs +++ /dev/null @@ -1,59 +0,0 @@ -global using System.Collections.Concurrent; -global using System.Data; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.Formats.Tar; -global using System.Globalization; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Net.Mime; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Security.Cryptography; -global using System.Text; -global using System.Text.Encodings.Web; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using System.Text.Json.Serialization; -global using System.Text.RegularExpressions; -global using Amazon; -global using Amazon.Runtime; -global using Amazon.S3; -global using Amazon.S3.Model; -global using CommunityToolkit.HighPerformance; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Common; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Hangfire.States; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Bson.Serialization.Serializers; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using Serval.Machine.Shared.Configuration; -global using Serval.Machine.Shared.Models; -global using Serval.Machine.Shared.Services; -global using Serval.Machine.Shared.Utils; -global using SIL.DataAccess; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.Scripture; -global using SIL.WritingSystems; diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs deleted file mode 100644 index 6c4a5c0f..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs +++ /dev/null @@ -1,19 +0,0 @@ -using SIL.ObjectModel; - -namespace Serval.Machine.Shared.Utils; - -public class AsyncDisposableBase : DisposableBase, IAsyncDisposable -{ - public async ValueTask DisposeAsync() - { - await DisposeAsyncCore(); - - Dispose(false); - GC.SuppressFinalize(this); - } - - protected virtual ValueTask DisposeAsyncCore() - { - return default; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs deleted file mode 100644 index 711826d5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs +++ /dev/null @@ -1,69 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -public class AsyncTimer : AsyncDisposableBase -{ - private readonly Timer _timer; - private readonly Func _callback; - private readonly AsyncLock _lock; - private bool _running; - - public AsyncTimer(Func callback) - { - _callback = callback; - _lock = new AsyncLock(); - _timer = new Timer(FireTimerAsync, null, Timeout.Infinite, Timeout.Infinite); - } - - public void Start(TimeSpan period) - { - _running = true; - _timer.Change(period, period); - } - - private async void FireTimerAsync(object? state) - { - using (await _lock.LockAsync()) - { - if (_running) - await _callback(); - } - } - - public async Task StopAsync() - { - using (await _lock.LockAsync()) - { - // FireTimer is *not* running _callback (since we got the lock) - StopTimer(); - } - // Now FireTimer will *never* run _callback - } - - public void Stop() - { - using (_lock.Lock()) - { - // FireTimer is *not* running _callback (since we got the lock) - StopTimer(); - } - // Now FireTimer will *never* run _callback - } - - private void StopTimer() - { - _timer.Change(Timeout.Infinite, Timeout.Infinite); - _running = false; - } - - protected override async ValueTask DisposeAsyncCore() - { - await StopAsync(); - _timer.Dispose(); - } - - protected override void DisposeManagedResources() - { - Stop(); - _timer.Dispose(); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs b/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs deleted file mode 100644 index c5e07809..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -public sealed class CustomEnumConverterFactory(JsonNamingPolicy namingPolicy) : JsonConverterFactory -{ - private readonly JsonNamingPolicy _namingPolicy = namingPolicy; - - public override bool CanConvert(Type typeToConvert) => typeToConvert.IsEnum; - - public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) - { - object[]? knownValues = null; - - if (typeToConvert == typeof(BindingFlags)) - knownValues = new object[] { BindingFlags.CreateInstance | BindingFlags.DeclaredOnly }; - - return (JsonConverter) - Activator.CreateInstance( - typeof(CustomEnumConverter<>).MakeGenericType(typeToConvert), - BindingFlags.Instance | BindingFlags.Public, - binder: null, - args: new object?[] { _namingPolicy, options, knownValues }, - culture: null - )!; - } -} - -public sealed class CustomEnumConverter : JsonConverter - where T : Enum -{ - private readonly JsonNamingPolicy _namingPolicy; - - private readonly Dictionary _readCache = new(); - private readonly Dictionary _writeCache = new(); - - // This converter will only support up to 64 enum values (including flags) on serialization and deserialization - private const int NameCacheLimit = 64; - - private const string ValueSeparator = ", "; - - public CustomEnumConverter(JsonNamingPolicy namingPolicy, JsonSerializerOptions options, object[]? knownValues) - { - _namingPolicy = namingPolicy; - - bool continueProcessing = true; - for (int i = 0; i < knownValues?.Length; i++) - { - if (!TryProcessValue((T)knownValues[i])) - { - continueProcessing = false; - break; - } - } - - if (continueProcessing) - { - Array values = Enum.GetValues(typeof(T)); - - for (int i = 0; i < values.Length; i++) - { - var value = (T)values.GetValue(i)!; - - if (!TryProcessValue(value)) - break; - } - } - - bool TryProcessValue(T value) - { - if (_readCache.Count == NameCacheLimit) - { - Debug.Assert(_writeCache.Count == NameCacheLimit); - return false; - } - - FormatAndAddToCaches(value, options.Encoder); - return true; - } - } - - public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - string? json; - - if ( - reader.TokenType != JsonTokenType.String - || (json = reader.GetString()) == null - || !_readCache.TryGetValue(json, out T? value) - ) - { - throw new JsonException(); - } - - return value; - } - - public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) - { - if (!_writeCache.TryGetValue(value, out JsonEncodedText formatted)) - { - if (_writeCache.Count == NameCacheLimit) - { - Debug.Assert(_readCache.Count == NameCacheLimit); - throw new InvalidOperationException("The JSON value contains too many enumerations."); - } - - formatted = FormatAndAddToCaches(value, options.Encoder); - } - - writer.WriteStringValue(formatted); - } - - private JsonEncodedText FormatAndAddToCaches(T value, JavaScriptEncoder? encoder) - { - (string valueFormattedToStr, JsonEncodedText valueEncoded) = CustomEnumConverter.FormatEnumValue( - value.ToString(), - _namingPolicy, - encoder - ); - _readCache[valueFormattedToStr] = value; - _writeCache[value] = valueEncoded; - return valueEncoded; - } - - private static ValueTuple FormatEnumValue( - string value, - JsonNamingPolicy namingPolicy, - JavaScriptEncoder? encoder - ) - { - string converted; - - if (!value.Contains(ValueSeparator)) - { - converted = namingPolicy.ConvertName(value); - } - else - { - // todo: optimize implementation here by leveraging https://github.com/dotnet/runtime/issues/934. - string[] enumValues = value.Split(ValueSeparator); - - for (int i = 0; i < enumValues.Length; i++) - { - enumValues[i] = namingPolicy.ConvertName(enumValues[i]); - } - - converted = string.Join(ValueSeparator, enumValues); - } - - return (converted, JsonEncodedText.Encode(converted, encoder)); - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs b/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs deleted file mode 100644 index 792fa9b2..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs +++ /dev/null @@ -1,65 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -internal sealed class DictionaryStringStringConverter : JsonConverter> -{ - public override IReadOnlyDictionary Read( - ref Utf8JsonReader reader, - Type typeToConvert, - JsonSerializerOptions options - ) - { - if (reader.TokenType != JsonTokenType.StartObject) - throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported"); - - var dictionary = new Dictionary(); - while (reader.Read()) - { - if (reader.TokenType == JsonTokenType.EndObject) - return dictionary; - - if (reader.TokenType != JsonTokenType.PropertyName) - throw new JsonException("JsonTokenType was not PropertyName"); - - var propertyName = reader.GetString(); - - if (string.IsNullOrWhiteSpace(propertyName)) - throw new JsonException("Failed to get property name"); - - reader.Read(); - - dictionary.Add(propertyName!, ExtractValue(ref reader)); - } - - return dictionary; - } - - public override void Write( - Utf8JsonWriter writer, - IReadOnlyDictionary value, - JsonSerializerOptions options - ) - { - JsonSerializer.Serialize(writer, value, options); - } - - private static string ExtractValue(ref Utf8JsonReader reader) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - return reader.GetString() ?? "Error Reading String."; - case JsonTokenType.False: - return "false"; - case JsonTokenType.True: - return "true"; - case JsonTokenType.Null: - return "null"; - case JsonTokenType.Number: - if (reader.TryGetDouble(out var result)) - return result.ToString(CultureInfo.InvariantCulture); - return "Error Reading Number."; - default: - throw new JsonException($"'{reader.TokenType}' is not supported"); - } - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs b/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs deleted file mode 100644 index 6378535f..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -/// This exception is thrown when an unbuilt engine is requested to perform an action that requires it being built -public class EngineNotBuiltException(string message) : Exception(message) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs b/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs deleted file mode 100644 index 2e2f91ec..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs +++ /dev/null @@ -1,40 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -public abstract class RecurrentTask( - string serviceName, - IServiceProvider services, - TimeSpan period, - ILogger logger, - bool enable = true -) : BackgroundService -{ - private readonly bool _enable = enable; - private readonly string _serviceName = serviceName; - private readonly IServiceProvider _services = services; - private readonly TimeSpan _period = period; - private readonly ILogger _logger = logger; - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - if (!_enable) - return; - - using PeriodicTimer timer = new(_period); - - _logger.LogInformation("{ServiceName} started.", _serviceName); - - try - { - while (await timer.WaitForNextTickAsync(stoppingToken)) - { - using IServiceScope scope = _services.CreateScope(); - await DoWorkAsync(scope, stoppingToken); - } - } - catch (OperationCanceledException) { } - - _logger.LogInformation("{ServiceName} stopped.", _serviceName); - } - - protected abstract Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken); -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs b/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs deleted file mode 100644 index 36409c2b..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -public static class SharedFileUtils -{ - public static string Normalize(string path, bool includeLeadingSlash = false, bool includeTrailingSlash = false) - { - string normalizedPath = path; - if (normalizedPath == "/") - return normalizedPath; - if (!includeLeadingSlash && normalizedPath.StartsWith("/")) - normalizedPath = normalizedPath.Remove(0, 1); - else if (includeLeadingSlash && !normalizedPath.StartsWith("/")) - normalizedPath = "/" + normalizedPath; - if (!includeTrailingSlash && normalizedPath.EndsWith("/")) - normalizedPath = normalizedPath.Remove(normalizedPath.Length - 1, 1); - else if (includeTrailingSlash && !normalizedPath.EndsWith("/")) - normalizedPath += "/"; - return normalizedPath; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs b/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs deleted file mode 100644 index 37cecdd0..00000000 --- a/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Serval.Machine.Shared.Utils; - -public class StartupTask(IServiceProvider services, Func task) - : IHostedService -{ - private readonly IServiceProvider _services = services; - private readonly Func _task = task; - - public async Task StartAsync(CancellationToken cancellationToken) - { - using IServiceScope scope = _services.CreateScope(); - await _task(scope.ServiceProvider, cancellationToken); - } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } -} diff --git a/src/Machine/src/Serval.Machine.Shared/data/.gitattributes b/src/Machine/src/Serval.Machine.Shared/data/.gitattributes deleted file mode 100644 index be61261c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/.gitattributes +++ /dev/null @@ -1,5 +0,0 @@ -thot-new-model/lm/* eol=lf -thot-new-model/tm/* eol=lf -thot-new-model/tm/*.hmm_alignd binary -thot-new-model/tm/*.hmm_lexnd binary -thot-new-model/smt.cfg eol=lf \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv b/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv deleted file mode 100644 index a9734554..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv +++ /dev/null @@ -1,205 +0,0 @@ -language, code -Acehnese (Arabic script), ace_Arab -Acehnese (Latin script), ace_Latn -Mesopotamian Arabic, acm_Arab -Ta’izzi-Adeni Arabic, acq_Arab -Tunisian Arabic, aeb_Arab -Afrikaans, afr_Latn -South Levantine Arabic, ajp_Arab -Akan, aka_Latn -Amharic, amh_Ethi -North Levantine Arabic, apc_Arab -Modern Standard Arabic, arb_Arab -Modern Standard Arabic (Romanized), arb_Latn -Najdi Arabic, ars_Arab -Moroccan Arabic, ary_Arab -Egyptian Arabic, arz_Arab -Assamese, asm_Beng -Asturian, ast_Latn -Awadhi, awa_Deva -Central Aymara, ayr_Latn -South Azerbaijani, azb_Arab -North Azerbaijani, azj_Latn -Bashkir, bak_Cyrl -Bambara, bam_Latn -Balinese, ban_Latn -Belarusian, bel_Cyrl -Bemba, bem_Latn -Bengali, ben_Beng -Bhojpuri, bho_Deva -Banjar (Arabic script), bjn_Arab -Banjar (Latin script), bjn_Latn -Standard Tibetan, bod_Tibt -Bosnian, bos_Latn -Buginese, bug_Latn -Bulgarian, bul_Cyrl -Catalan, cat_Latn -Cebuano, ceb_Latn -Czech, ces_Latn -Chokwe, cjk_Latn -Central Kurdish, ckb_Arab -Crimean Tatar, crh_Latn -Welsh, cym_Latn -Danish, dan_Latn -German, deu_Latn -Southwestern Dinka, dik_Latn -Dyula, dyu_Latn -Dzongkha, dzo_Tibt -Greek, ell_Grek -English, eng_Latn -Esperanto, epo_Latn -Estonian, est_Latn -Basque, eus_Latn -Ewe, ewe_Latn -Faroese, fao_Latn -Fijian, fij_Latn -Finnish, fin_Latn -Fon, fon_Latn -French, fra_Latn -Friulian, fur_Latn -Nigerian Fulfulde, fuv_Latn -Scottish Gaelic, gla_Latn -Irish, gle_Latn -Galician, glg_Latn -Guarani, grn_Latn -Gujarati, guj_Gujr -Haitian Creole, hat_Latn -Hausa, hau_Latn -Hebrew, heb_Hebr -Hindi, hin_Deva -Chhattisgarhi, hne_Deva -Croatian, hrv_Latn -Hungarian, hun_Latn -Armenian, hye_Armn -Igbo, ibo_Latn -Ilocano, ilo_Latn -Indonesian, ind_Latn -Icelandic, isl_Latn -Italian, ita_Latn -Javanese, jav_Latn -Japanese, jpn_Jpan -Kabyle, kab_Latn -Jingpho, kac_Latn -Kamba, kam_Latn -Kannada, kan_Knda -Kashmiri (Arabic script), kas_Arab -Kashmiri (Devanagari script), kas_Deva -Georgian, kat_Geor -Central Kanuri (Arabic script), knc_Arab -Central Kanuri (Latin script), knc_Latn -Kazakh, kaz_Cyrl -Kabiyè, kbp_Latn -Kabuverdianu, kea_Latn -Khmer, khm_Khmr -Kikuyu, kik_Latn -Kinyarwanda, kin_Latn -Kyrgyz, kir_Cyrl -Kimbundu, kmb_Latn -Northern Kurdish, kmr_Latn -Kikongo, kon_Latn -Korean, kor_Hang -Lao, lao_Laoo -Ligurian, lij_Latn -Limburgish, lim_Latn -Lingala, lin_Latn -Lithuanian, lit_Latn -Lombard, lmo_Latn -Latgalian, ltg_Latn -Luxembourgish, ltz_Latn -Luba-Kasai, lua_Latn -Ganda, lug_Latn -Luo, luo_Latn -Mizo, lus_Latn -Standard Latvian, lvs_Latn -Magahi, mag_Deva -Maithili, mai_Deva -Malayalam, mal_Mlym -Marathi, mar_Deva -Minangkabau (Arabic script), min_Arab -Minangkabau (Latin script), min_Latn -Macedonian, mkd_Cyrl -Plateau Malagasy, plt_Latn -Maltese, mlt_Latn -Meitei (Bengali script), mni_Beng -Halh Mongolian, khk_Cyrl -Mossi, mos_Latn -Maori, mri_Latn -Burmese, mya_Mymr -Dutch, nld_Latn -Norwegian Nynorsk, nno_Latn -Norwegian Bokmål, nob_Latn -Nepali, npi_Deva -Northern Sotho, nso_Latn -Nuer, nus_Latn -Nyanja, nya_Latn -Occitan, oci_Latn -West Central Oromo, gaz_Latn -Odia, ory_Orya -Pangasinan, pag_Latn -Eastern Panjabi, pan_Guru -Papiamento, pap_Latn -Western Persian, pes_Arab -Polish, pol_Latn -Portuguese, por_Latn -Dari, prs_Arab -Southern Pashto, pbt_Arab -Ayacucho Quechua, quy_Latn -Romanian, ron_Latn -Rundi, run_Latn -Russian, rus_Cyrl -Sango, sag_Latn -Sanskrit, san_Deva -Santali, sat_Olck -Sicilian, scn_Latn -Shan, shn_Mymr -Sinhala, sin_Sinh -Slovak, slk_Latn -Slovenian, slv_Latn -Samoan, smo_Latn -Shona, sna_Latn -Sindhi, snd_Arab -Somali, som_Latn -Southern Sotho, sot_Latn -Spanish, spa_Latn -Tosk Albanian, als_Latn -Sardinian, srd_Latn -Serbian, srp_Cyrl -Swati, ssw_Latn -Sundanese, sun_Latn -Swedish, swe_Latn -Swahili, swh_Latn -Silesian, szl_Latn -Tamil, tam_Taml -Tatar, tat_Cyrl -Telugu, tel_Telu -Tajik, tgk_Cyrl -Tagalog, tgl_Latn -Thai, tha_Thai -Tigrinya, tir_Ethi -Tamasheq (Latin script), taq_Latn -Tamasheq (Tifinagh script), taq_Tfng -Tok Pisin, tpi_Latn -Tswana, tsn_Latn -Tsonga, tso_Latn -Turkmen, tuk_Latn -Tumbuka, tum_Latn -Turkish, tur_Latn -Twi, twi_Latn -Central Atlas Tamazight, tzm_Tfng -Uyghur, uig_Arab -Ukrainian, ukr_Cyrl -Umbundu, umb_Latn -Urdu, urd_Arab -Northern Uzbek, uzn_Latn -Venetian, vec_Latn -Vietnamese, vie_Latn -Waray, war_Latn -Wolof, wol_Latn -Xhosa, xho_Latn -Eastern Yiddish, ydd_Hebr -Yoruba, yor_Latn -Yue Chinese, yue_Hant -Chinese (Simplified), zho_Hans -Chinese (Traditional), zho_Hant -Standard Malay, zsm_Latn -Zulu, zul_Latn \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm deleted file mode 100644 index 2bdf4838..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm +++ /dev/null @@ -1,6 +0,0 @@ - 1.00000000 1.00000000 - 3.00000000 1.00000000 - 1.00000000 1.00000000 - 1.00000000 1.00000000 - 3.00000000 1.00000000 - 3.00000000 1.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights deleted file mode 100644 index 67a757d9..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights +++ /dev/null @@ -1 +0,0 @@ -3 3 10.000000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp deleted file mode 100644 index a3fd47b9..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg deleted file mode 100644 index b438a898..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg +++ /dev/null @@ -1,29 +0,0 @@ -# Translation model prefix --tm tm/src_trg - -# Language model --lm lm/trg.lm - -# W parameter (maximum number of translation options to be considered per each source phrase) --W 10 - -# S parameter (maximum number of hypotheses that can be stored in each stack) --S 10 - -# A parameter (Maximum length in words of the source phrases to be translated) --A 7 - -# Degree of non-monotonicity --nomon 0 - -# Heuristic function used --h 6 - -# Best-first search flag --be - -# Translation model weights --tmw 0 0.5 1 1 1 1 0 1 - -# Set online learning parameters (ol_alg, lr_policy, l_stepsize, em_iters, e_par, r_par) --olp 0 0 1 5 1 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda deleted file mode 100644 index 6e6566ce..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda +++ /dev/null @@ -1 +0,0 @@ -0.01 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable deleted file mode 100644 index 7277bf4c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable +++ /dev/null @@ -1,10201 +0,0 @@ -0 0 1 -0 1 1 -0 2 1 -0 3 1 -0 4 1 -0 5 1 -0 6 1 -0 7 1 -0 8 1 -0 9 1 -0 10 1 -0 11 1 -0 12 1 -0 13 1 -0 14 1 -0 15 1 -0 16 1 -0 17 1 -0 18 1 -0 19 1 -0 20 1 -0 21 1 -0 22 1 -0 23 1 -0 24 1 -0 25 1 -0 26 1 -0 27 1 -0 28 1 -0 29 1 -0 30 1 -0 31 1 -0 32 1 -0 33 1 -0 34 1 -0 35 1 -0 36 1 -0 37 1 -0 38 1 -0 39 1 -0 40 1 -0 41 1 -0 42 1 -0 43 1 -0 44 1 -0 45 1 -0 46 1 -0 47 1 -0 48 1 -0 49 1 -0 50 1 -0 51 1 -0 52 1 -0 53 1 -0 54 1 -0 55 1 -0 56 1 -0 57 1 -0 58 1 -0 59 1 -0 60 1 -0 61 1 -0 62 1 -0 63 1 -0 64 1 -0 65 1 -0 66 1 -0 67 1 -0 68 1 -0 69 1 -0 70 1 -0 71 1 -0 72 1 -0 73 1 -0 74 1 -0 75 1 -0 76 1 -0 77 1 -0 78 1 -0 79 1 -0 80 1 -0 81 1 -0 82 1 -0 83 1 -0 84 1 -0 85 1 -0 86 1 -0 87 1 -0 88 1 -0 89 1 -0 90 1 -0 91 1 -0 92 1 -0 93 1 -0 94 1 -0 95 1 -0 96 1 -0 97 1 -0 98 1 -0 99 1 -0 100 1 -1 0 1 -1 1 1 -1 2 1 -1 3 1 -1 4 1 -1 5 1 -1 6 1 -1 7 1 -1 8 1 -1 9 1 -1 10 1 -1 11 1 -1 12 1 -1 13 1 -1 14 1 -1 15 1 -1 16 1 -1 17 1 -1 18 1 -1 19 1 -1 20 1 -1 21 1 -1 22 1 -1 23 1 -1 24 1 -1 25 1 -1 26 1 -1 27 1 -1 28 1 -1 29 1 -1 30 1 -1 31 1 -1 32 1 -1 33 1 -1 34 1 -1 35 1 -1 36 1 -1 37 1 -1 38 1 -1 39 1 -1 40 1 -1 41 1 -1 42 1 -1 43 1 -1 44 1 -1 45 1 -1 46 1 -1 47 1 -1 48 1 -1 49 1 -1 50 1 -1 51 1 -1 52 1 -1 53 1 -1 54 1 -1 55 1 -1 56 1 -1 57 1 -1 58 1 -1 59 1 -1 60 1 -1 61 1 -1 62 1 -1 63 1 -1 64 1 -1 65 1 -1 66 1 -1 67 1 -1 68 1 -1 69 1 -1 70 1 -1 71 1 -1 72 1 -1 73 1 -1 74 1 -1 75 1 -1 76 1 -1 77 1 -1 78 1 -1 79 1 -1 80 1 -1 81 1 -1 82 1 -1 83 1 -1 84 1 -1 85 1 -1 86 1 -1 87 1 -1 88 1 -1 89 1 -1 90 1 -1 91 1 -1 92 1 -1 93 1 -1 94 1 -1 95 1 -1 96 1 -1 97 1 -1 98 1 -1 99 1 -1 100 1 -2 0 1 -2 1 1 -2 2 1 -2 3 1 -2 4 1 -2 5 1 -2 6 1 -2 7 1 -2 8 1 -2 9 1 -2 10 1 -2 11 1 -2 12 1 -2 13 1 -2 14 1 -2 15 1 -2 16 1 -2 17 1 -2 18 1 -2 19 1 -2 20 1 -2 21 1 -2 22 1 -2 23 1 -2 24 1 -2 25 1 -2 26 1 -2 27 1 -2 28 1 -2 29 1 -2 30 1 -2 31 1 -2 32 1 -2 33 1 -2 34 1 -2 35 1 -2 36 1 -2 37 1 -2 38 1 -2 39 1 -2 40 1 -2 41 1 -2 42 1 -2 43 1 -2 44 1 -2 45 1 -2 46 1 -2 47 1 -2 48 1 -2 49 1 -2 50 1 -2 51 1 -2 52 1 -2 53 1 -2 54 1 -2 55 1 -2 56 1 -2 57 1 -2 58 1 -2 59 1 -2 60 1 -2 61 1 -2 62 1 -2 63 1 -2 64 1 -2 65 1 -2 66 1 -2 67 1 -2 68 1 -2 69 1 -2 70 1 -2 71 1 -2 72 1 -2 73 1 -2 74 1 -2 75 1 -2 76 1 -2 77 1 -2 78 1 -2 79 1 -2 80 1 -2 81 1 -2 82 1 -2 83 1 -2 84 1 -2 85 1 -2 86 1 -2 87 1 -2 88 1 -2 89 1 -2 90 1 -2 91 1 -2 92 1 -2 93 1 -2 94 1 -2 95 1 -2 96 1 -2 97 1 -2 98 1 -2 99 1 -2 100 1 -3 0 1 -3 1 1 -3 2 1 -3 3 1 -3 4 1 -3 5 1 -3 6 1 -3 7 1 -3 8 1 -3 9 1 -3 10 1 -3 11 1 -3 12 1 -3 13 1 -3 14 1 -3 15 1 -3 16 1 -3 17 1 -3 18 1 -3 19 1 -3 20 1 -3 21 1 -3 22 1 -3 23 1 -3 24 1 -3 25 1 -3 26 1 -3 27 1 -3 28 1 -3 29 1 -3 30 1 -3 31 1 -3 32 1 -3 33 1 -3 34 1 -3 35 1 -3 36 1 -3 37 1 -3 38 1 -3 39 1 -3 40 1 -3 41 1 -3 42 1 -3 43 1 -3 44 1 -3 45 1 -3 46 1 -3 47 1 -3 48 1 -3 49 1 -3 50 1 -3 51 1 -3 52 1 -3 53 1 -3 54 1 -3 55 1 -3 56 1 -3 57 1 -3 58 1 -3 59 1 -3 60 1 -3 61 1 -3 62 1 -3 63 1 -3 64 1 -3 65 1 -3 66 1 -3 67 1 -3 68 1 -3 69 1 -3 70 1 -3 71 1 -3 72 1 -3 73 1 -3 74 1 -3 75 1 -3 76 1 -3 77 1 -3 78 1 -3 79 1 -3 80 1 -3 81 1 -3 82 1 -3 83 1 -3 84 1 -3 85 1 -3 86 1 -3 87 1 -3 88 1 -3 89 1 -3 90 1 -3 91 1 -3 92 1 -3 93 1 -3 94 1 -3 95 1 -3 96 1 -3 97 1 -3 98 1 -3 99 1 -3 100 1 -4 0 1 -4 1 1 -4 2 1 -4 3 1 -4 4 1 -4 5 1 -4 6 1 -4 7 1 -4 8 1 -4 9 1 -4 10 1 -4 11 1 -4 12 1 -4 13 1 -4 14 1 -4 15 1 -4 16 1 -4 17 1 -4 18 1 -4 19 1 -4 20 1 -4 21 1 -4 22 1 -4 23 1 -4 24 1 -4 25 1 -4 26 1 -4 27 1 -4 28 1 -4 29 1 -4 30 1 -4 31 1 -4 32 1 -4 33 1 -4 34 1 -4 35 1 -4 36 1 -4 37 1 -4 38 1 -4 39 1 -4 40 1 -4 41 1 -4 42 1 -4 43 1 -4 44 1 -4 45 1 -4 46 1 -4 47 1 -4 48 1 -4 49 1 -4 50 1 -4 51 1 -4 52 1 -4 53 1 -4 54 1 -4 55 1 -4 56 1 -4 57 1 -4 58 1 -4 59 1 -4 60 1 -4 61 1 -4 62 1 -4 63 1 -4 64 1 -4 65 1 -4 66 1 -4 67 1 -4 68 1 -4 69 1 -4 70 1 -4 71 1 -4 72 1 -4 73 1 -4 74 1 -4 75 1 -4 76 1 -4 77 1 -4 78 1 -4 79 1 -4 80 1 -4 81 1 -4 82 1 -4 83 1 -4 84 1 -4 85 1 -4 86 1 -4 87 1 -4 88 1 -4 89 1 -4 90 1 -4 91 1 -4 92 1 -4 93 1 -4 94 1 -4 95 1 -4 96 1 -4 97 1 -4 98 1 -4 99 1 -4 100 1 -5 0 1 -5 1 1 -5 2 1 -5 3 1 -5 4 1 -5 5 1 -5 6 1 -5 7 1 -5 8 1 -5 9 1 -5 10 1 -5 11 1 -5 12 1 -5 13 1 -5 14 1 -5 15 1 -5 16 1 -5 17 1 -5 18 1 -5 19 1 -5 20 1 -5 21 1 -5 22 1 -5 23 1 -5 24 1 -5 25 1 -5 26 1 -5 27 1 -5 28 1 -5 29 1 -5 30 1 -5 31 1 -5 32 1 -5 33 1 -5 34 1 -5 35 1 -5 36 1 -5 37 1 -5 38 1 -5 39 1 -5 40 1 -5 41 1 -5 42 1 -5 43 1 -5 44 1 -5 45 1 -5 46 1 -5 47 1 -5 48 1 -5 49 1 -5 50 1 -5 51 1 -5 52 1 -5 53 1 -5 54 1 -5 55 1 -5 56 1 -5 57 1 -5 58 1 -5 59 1 -5 60 1 -5 61 1 -5 62 1 -5 63 1 -5 64 1 -5 65 1 -5 66 1 -5 67 1 -5 68 1 -5 69 1 -5 70 1 -5 71 1 -5 72 1 -5 73 1 -5 74 1 -5 75 1 -5 76 1 -5 77 1 -5 78 1 -5 79 1 -5 80 1 -5 81 1 -5 82 1 -5 83 1 -5 84 1 -5 85 1 -5 86 1 -5 87 1 -5 88 1 -5 89 1 -5 90 1 -5 91 1 -5 92 1 -5 93 1 -5 94 1 -5 95 1 -5 96 1 -5 97 1 -5 98 1 -5 99 1 -5 100 1 -6 0 1 -6 1 1 -6 2 1 -6 3 1 -6 4 1 -6 5 1 -6 6 1 -6 7 1 -6 8 1 -6 9 1 -6 10 1 -6 11 1 -6 12 1 -6 13 1 -6 14 1 -6 15 1 -6 16 1 -6 17 1 -6 18 1 -6 19 1 -6 20 1 -6 21 1 -6 22 1 -6 23 1 -6 24 1 -6 25 1 -6 26 1 -6 27 1 -6 28 1 -6 29 1 -6 30 1 -6 31 1 -6 32 1 -6 33 1 -6 34 1 -6 35 1 -6 36 1 -6 37 1 -6 38 1 -6 39 1 -6 40 1 -6 41 1 -6 42 1 -6 43 1 -6 44 1 -6 45 1 -6 46 1 -6 47 1 -6 48 1 -6 49 1 -6 50 1 -6 51 1 -6 52 1 -6 53 1 -6 54 1 -6 55 1 -6 56 1 -6 57 1 -6 58 1 -6 59 1 -6 60 1 -6 61 1 -6 62 1 -6 63 1 -6 64 1 -6 65 1 -6 66 1 -6 67 1 -6 68 1 -6 69 1 -6 70 1 -6 71 1 -6 72 1 -6 73 1 -6 74 1 -6 75 1 -6 76 1 -6 77 1 -6 78 1 -6 79 1 -6 80 1 -6 81 1 -6 82 1 -6 83 1 -6 84 1 -6 85 1 -6 86 1 -6 87 1 -6 88 1 -6 89 1 -6 90 1 -6 91 1 -6 92 1 -6 93 1 -6 94 1 -6 95 1 -6 96 1 -6 97 1 -6 98 1 -6 99 1 -6 100 1 -7 0 1 -7 1 1 -7 2 1 -7 3 1 -7 4 1 -7 5 1 -7 6 1 -7 7 1 -7 8 1 -7 9 1 -7 10 1 -7 11 1 -7 12 1 -7 13 1 -7 14 1 -7 15 1 -7 16 1 -7 17 1 -7 18 1 -7 19 1 -7 20 1 -7 21 1 -7 22 1 -7 23 1 -7 24 1 -7 25 1 -7 26 1 -7 27 1 -7 28 1 -7 29 1 -7 30 1 -7 31 1 -7 32 1 -7 33 1 -7 34 1 -7 35 1 -7 36 1 -7 37 1 -7 38 1 -7 39 1 -7 40 1 -7 41 1 -7 42 1 -7 43 1 -7 44 1 -7 45 1 -7 46 1 -7 47 1 -7 48 1 -7 49 1 -7 50 1 -7 51 1 -7 52 1 -7 53 1 -7 54 1 -7 55 1 -7 56 1 -7 57 1 -7 58 1 -7 59 1 -7 60 1 -7 61 1 -7 62 1 -7 63 1 -7 64 1 -7 65 1 -7 66 1 -7 67 1 -7 68 1 -7 69 1 -7 70 1 -7 71 1 -7 72 1 -7 73 1 -7 74 1 -7 75 1 -7 76 1 -7 77 1 -7 78 1 -7 79 1 -7 80 1 -7 81 1 -7 82 1 -7 83 1 -7 84 1 -7 85 1 -7 86 1 -7 87 1 -7 88 1 -7 89 1 -7 90 1 -7 91 1 -7 92 1 -7 93 1 -7 94 1 -7 95 1 -7 96 1 -7 97 1 -7 98 1 -7 99 1 -7 100 1 -8 0 1 -8 1 1 -8 2 1 -8 3 1 -8 4 1 -8 5 1 -8 6 1 -8 7 1 -8 8 1 -8 9 1 -8 10 1 -8 11 1 -8 12 1 -8 13 1 -8 14 1 -8 15 1 -8 16 1 -8 17 1 -8 18 1 -8 19 1 -8 20 1 -8 21 1 -8 22 1 -8 23 1 -8 24 1 -8 25 1 -8 26 1 -8 27 1 -8 28 1 -8 29 1 -8 30 1 -8 31 1 -8 32 1 -8 33 1 -8 34 1 -8 35 1 -8 36 1 -8 37 1 -8 38 1 -8 39 1 -8 40 1 -8 41 1 -8 42 1 -8 43 1 -8 44 1 -8 45 1 -8 46 1 -8 47 1 -8 48 1 -8 49 1 -8 50 1 -8 51 1 -8 52 1 -8 53 1 -8 54 1 -8 55 1 -8 56 1 -8 57 1 -8 58 1 -8 59 1 -8 60 1 -8 61 1 -8 62 1 -8 63 1 -8 64 1 -8 65 1 -8 66 1 -8 67 1 -8 68 1 -8 69 1 -8 70 1 -8 71 1 -8 72 1 -8 73 1 -8 74 1 -8 75 1 -8 76 1 -8 77 1 -8 78 1 -8 79 1 -8 80 1 -8 81 1 -8 82 1 -8 83 1 -8 84 1 -8 85 1 -8 86 1 -8 87 1 -8 88 1 -8 89 1 -8 90 1 -8 91 1 -8 92 1 -8 93 1 -8 94 1 -8 95 1 -8 96 1 -8 97 1 -8 98 1 -8 99 1 -8 100 1 -9 0 1 -9 1 1 -9 2 1 -9 3 1 -9 4 1 -9 5 1 -9 6 1 -9 7 1 -9 8 1 -9 9 1 -9 10 1 -9 11 1 -9 12 1 -9 13 1 -9 14 1 -9 15 1 -9 16 1 -9 17 1 -9 18 1 -9 19 1 -9 20 1 -9 21 1 -9 22 1 -9 23 1 -9 24 1 -9 25 1 -9 26 1 -9 27 1 -9 28 1 -9 29 1 -9 30 1 -9 31 1 -9 32 1 -9 33 1 -9 34 1 -9 35 1 -9 36 1 -9 37 1 -9 38 1 -9 39 1 -9 40 1 -9 41 1 -9 42 1 -9 43 1 -9 44 1 -9 45 1 -9 46 1 -9 47 1 -9 48 1 -9 49 1 -9 50 1 -9 51 1 -9 52 1 -9 53 1 -9 54 1 -9 55 1 -9 56 1 -9 57 1 -9 58 1 -9 59 1 -9 60 1 -9 61 1 -9 62 1 -9 63 1 -9 64 1 -9 65 1 -9 66 1 -9 67 1 -9 68 1 -9 69 1 -9 70 1 -9 71 1 -9 72 1 -9 73 1 -9 74 1 -9 75 1 -9 76 1 -9 77 1 -9 78 1 -9 79 1 -9 80 1 -9 81 1 -9 82 1 -9 83 1 -9 84 1 -9 85 1 -9 86 1 -9 87 1 -9 88 1 -9 89 1 -9 90 1 -9 91 1 -9 92 1 -9 93 1 -9 94 1 -9 95 1 -9 96 1 -9 97 1 -9 98 1 -9 99 1 -9 100 1 -10 0 1 -10 1 1 -10 2 1 -10 3 1 -10 4 1 -10 5 1 -10 6 1 -10 7 1 -10 8 1 -10 9 1 -10 10 1 -10 11 1 -10 12 1 -10 13 1 -10 14 1 -10 15 1 -10 16 1 -10 17 1 -10 18 1 -10 19 1 -10 20 1 -10 21 1 -10 22 1 -10 23 1 -10 24 1 -10 25 1 -10 26 1 -10 27 1 -10 28 1 -10 29 1 -10 30 1 -10 31 1 -10 32 1 -10 33 1 -10 34 1 -10 35 1 -10 36 1 -10 37 1 -10 38 1 -10 39 1 -10 40 1 -10 41 1 -10 42 1 -10 43 1 -10 44 1 -10 45 1 -10 46 1 -10 47 1 -10 48 1 -10 49 1 -10 50 1 -10 51 1 -10 52 1 -10 53 1 -10 54 1 -10 55 1 -10 56 1 -10 57 1 -10 58 1 -10 59 1 -10 60 1 -10 61 1 -10 62 1 -10 63 1 -10 64 1 -10 65 1 -10 66 1 -10 67 1 -10 68 1 -10 69 1 -10 70 1 -10 71 1 -10 72 1 -10 73 1 -10 74 1 -10 75 1 -10 76 1 -10 77 1 -10 78 1 -10 79 1 -10 80 1 -10 81 1 -10 82 1 -10 83 1 -10 84 1 -10 85 1 -10 86 1 -10 87 1 -10 88 1 -10 89 1 -10 90 1 -10 91 1 -10 92 1 -10 93 1 -10 94 1 -10 95 1 -10 96 1 -10 97 1 -10 98 1 -10 99 1 -10 100 1 -11 0 1 -11 1 1 -11 2 1 -11 3 1 -11 4 1 -11 5 1 -11 6 1 -11 7 1 -11 8 1 -11 9 1 -11 10 1 -11 11 1 -11 12 1 -11 13 1 -11 14 1 -11 15 1 -11 16 1 -11 17 1 -11 18 1 -11 19 1 -11 20 1 -11 21 1 -11 22 1 -11 23 1 -11 24 1 -11 25 1 -11 26 1 -11 27 1 -11 28 1 -11 29 1 -11 30 1 -11 31 1 -11 32 1 -11 33 1 -11 34 1 -11 35 1 -11 36 1 -11 37 1 -11 38 1 -11 39 1 -11 40 1 -11 41 1 -11 42 1 -11 43 1 -11 44 1 -11 45 1 -11 46 1 -11 47 1 -11 48 1 -11 49 1 -11 50 1 -11 51 1 -11 52 1 -11 53 1 -11 54 1 -11 55 1 -11 56 1 -11 57 1 -11 58 1 -11 59 1 -11 60 1 -11 61 1 -11 62 1 -11 63 1 -11 64 1 -11 65 1 -11 66 1 -11 67 1 -11 68 1 -11 69 1 -11 70 1 -11 71 1 -11 72 1 -11 73 1 -11 74 1 -11 75 1 -11 76 1 -11 77 1 -11 78 1 -11 79 1 -11 80 1 -11 81 1 -11 82 1 -11 83 1 -11 84 1 -11 85 1 -11 86 1 -11 87 1 -11 88 1 -11 89 1 -11 90 1 -11 91 1 -11 92 1 -11 93 1 -11 94 1 -11 95 1 -11 96 1 -11 97 1 -11 98 1 -11 99 1 -11 100 1 -12 0 1 -12 1 1 -12 2 1 -12 3 1 -12 4 1 -12 5 1 -12 6 1 -12 7 1 -12 8 1 -12 9 1 -12 10 1 -12 11 1 -12 12 1 -12 13 1 -12 14 1 -12 15 1 -12 16 1 -12 17 1 -12 18 1 -12 19 1 -12 20 1 -12 21 1 -12 22 1 -12 23 1 -12 24 1 -12 25 1 -12 26 1 -12 27 1 -12 28 1 -12 29 1 -12 30 1 -12 31 1 -12 32 1 -12 33 1 -12 34 1 -12 35 1 -12 36 1 -12 37 1 -12 38 1 -12 39 1 -12 40 1 -12 41 1 -12 42 1 -12 43 1 -12 44 1 -12 45 1 -12 46 1 -12 47 1 -12 48 1 -12 49 1 -12 50 1 -12 51 1 -12 52 1 -12 53 1 -12 54 1 -12 55 1 -12 56 1 -12 57 1 -12 58 1 -12 59 1 -12 60 1 -12 61 1 -12 62 1 -12 63 1 -12 64 1 -12 65 1 -12 66 1 -12 67 1 -12 68 1 -12 69 1 -12 70 1 -12 71 1 -12 72 1 -12 73 1 -12 74 1 -12 75 1 -12 76 1 -12 77 1 -12 78 1 -12 79 1 -12 80 1 -12 81 1 -12 82 1 -12 83 1 -12 84 1 -12 85 1 -12 86 1 -12 87 1 -12 88 1 -12 89 1 -12 90 1 -12 91 1 -12 92 1 -12 93 1 -12 94 1 -12 95 1 -12 96 1 -12 97 1 -12 98 1 -12 99 1 -12 100 1 -13 0 1 -13 1 1 -13 2 1 -13 3 1 -13 4 1 -13 5 1 -13 6 1 -13 7 1 -13 8 1 -13 9 1 -13 10 1 -13 11 1 -13 12 1 -13 13 1 -13 14 1 -13 15 1 -13 16 1 -13 17 1 -13 18 1 -13 19 1 -13 20 1 -13 21 1 -13 22 1 -13 23 1 -13 24 1 -13 25 1 -13 26 1 -13 27 1 -13 28 1 -13 29 1 -13 30 1 -13 31 1 -13 32 1 -13 33 1 -13 34 1 -13 35 1 -13 36 1 -13 37 1 -13 38 1 -13 39 1 -13 40 1 -13 41 1 -13 42 1 -13 43 1 -13 44 1 -13 45 1 -13 46 1 -13 47 1 -13 48 1 -13 49 1 -13 50 1 -13 51 1 -13 52 1 -13 53 1 -13 54 1 -13 55 1 -13 56 1 -13 57 1 -13 58 1 -13 59 1 -13 60 1 -13 61 1 -13 62 1 -13 63 1 -13 64 1 -13 65 1 -13 66 1 -13 67 1 -13 68 1 -13 69 1 -13 70 1 -13 71 1 -13 72 1 -13 73 1 -13 74 1 -13 75 1 -13 76 1 -13 77 1 -13 78 1 -13 79 1 -13 80 1 -13 81 1 -13 82 1 -13 83 1 -13 84 1 -13 85 1 -13 86 1 -13 87 1 -13 88 1 -13 89 1 -13 90 1 -13 91 1 -13 92 1 -13 93 1 -13 94 1 -13 95 1 -13 96 1 -13 97 1 -13 98 1 -13 99 1 -13 100 1 -14 0 1 -14 1 1 -14 2 1 -14 3 1 -14 4 1 -14 5 1 -14 6 1 -14 7 1 -14 8 1 -14 9 1 -14 10 1 -14 11 1 -14 12 1 -14 13 1 -14 14 1 -14 15 1 -14 16 1 -14 17 1 -14 18 1 -14 19 1 -14 20 1 -14 21 1 -14 22 1 -14 23 1 -14 24 1 -14 25 1 -14 26 1 -14 27 1 -14 28 1 -14 29 1 -14 30 1 -14 31 1 -14 32 1 -14 33 1 -14 34 1 -14 35 1 -14 36 1 -14 37 1 -14 38 1 -14 39 1 -14 40 1 -14 41 1 -14 42 1 -14 43 1 -14 44 1 -14 45 1 -14 46 1 -14 47 1 -14 48 1 -14 49 1 -14 50 1 -14 51 1 -14 52 1 -14 53 1 -14 54 1 -14 55 1 -14 56 1 -14 57 1 -14 58 1 -14 59 1 -14 60 1 -14 61 1 -14 62 1 -14 63 1 -14 64 1 -14 65 1 -14 66 1 -14 67 1 -14 68 1 -14 69 1 -14 70 1 -14 71 1 -14 72 1 -14 73 1 -14 74 1 -14 75 1 -14 76 1 -14 77 1 -14 78 1 -14 79 1 -14 80 1 -14 81 1 -14 82 1 -14 83 1 -14 84 1 -14 85 1 -14 86 1 -14 87 1 -14 88 1 -14 89 1 -14 90 1 -14 91 1 -14 92 1 -14 93 1 -14 94 1 -14 95 1 -14 96 1 -14 97 1 -14 98 1 -14 99 1 -14 100 1 -15 0 1 -15 1 1 -15 2 1 -15 3 1 -15 4 1 -15 5 1 -15 6 1 -15 7 1 -15 8 1 -15 9 1 -15 10 1 -15 11 1 -15 12 1 -15 13 1 -15 14 1 -15 15 1 -15 16 1 -15 17 1 -15 18 1 -15 19 1 -15 20 1 -15 21 1 -15 22 1 -15 23 1 -15 24 1 -15 25 1 -15 26 1 -15 27 1 -15 28 1 -15 29 1 -15 30 1 -15 31 1 -15 32 1 -15 33 1 -15 34 1 -15 35 1 -15 36 1 -15 37 1 -15 38 1 -15 39 1 -15 40 1 -15 41 1 -15 42 1 -15 43 1 -15 44 1 -15 45 1 -15 46 1 -15 47 1 -15 48 1 -15 49 1 -15 50 1 -15 51 1 -15 52 1 -15 53 1 -15 54 1 -15 55 1 -15 56 1 -15 57 1 -15 58 1 -15 59 1 -15 60 1 -15 61 1 -15 62 1 -15 63 1 -15 64 1 -15 65 1 -15 66 1 -15 67 1 -15 68 1 -15 69 1 -15 70 1 -15 71 1 -15 72 1 -15 73 1 -15 74 1 -15 75 1 -15 76 1 -15 77 1 -15 78 1 -15 79 1 -15 80 1 -15 81 1 -15 82 1 -15 83 1 -15 84 1 -15 85 1 -15 86 1 -15 87 1 -15 88 1 -15 89 1 -15 90 1 -15 91 1 -15 92 1 -15 93 1 -15 94 1 -15 95 1 -15 96 1 -15 97 1 -15 98 1 -15 99 1 -15 100 1 -16 0 1 -16 1 1 -16 2 1 -16 3 1 -16 4 1 -16 5 1 -16 6 1 -16 7 1 -16 8 1 -16 9 1 -16 10 1 -16 11 1 -16 12 1 -16 13 1 -16 14 1 -16 15 1 -16 16 1 -16 17 1 -16 18 1 -16 19 1 -16 20 1 -16 21 1 -16 22 1 -16 23 1 -16 24 1 -16 25 1 -16 26 1 -16 27 1 -16 28 1 -16 29 1 -16 30 1 -16 31 1 -16 32 1 -16 33 1 -16 34 1 -16 35 1 -16 36 1 -16 37 1 -16 38 1 -16 39 1 -16 40 1 -16 41 1 -16 42 1 -16 43 1 -16 44 1 -16 45 1 -16 46 1 -16 47 1 -16 48 1 -16 49 1 -16 50 1 -16 51 1 -16 52 1 -16 53 1 -16 54 1 -16 55 1 -16 56 1 -16 57 1 -16 58 1 -16 59 1 -16 60 1 -16 61 1 -16 62 1 -16 63 1 -16 64 1 -16 65 1 -16 66 1 -16 67 1 -16 68 1 -16 69 1 -16 70 1 -16 71 1 -16 72 1 -16 73 1 -16 74 1 -16 75 1 -16 76 1 -16 77 1 -16 78 1 -16 79 1 -16 80 1 -16 81 1 -16 82 1 -16 83 1 -16 84 1 -16 85 1 -16 86 1 -16 87 1 -16 88 1 -16 89 1 -16 90 1 -16 91 1 -16 92 1 -16 93 1 -16 94 1 -16 95 1 -16 96 1 -16 97 1 -16 98 1 -16 99 1 -16 100 1 -17 0 1 -17 1 1 -17 2 1 -17 3 1 -17 4 1 -17 5 1 -17 6 1 -17 7 1 -17 8 1 -17 9 1 -17 10 1 -17 11 1 -17 12 1 -17 13 1 -17 14 1 -17 15 1 -17 16 1 -17 17 1 -17 18 1 -17 19 1 -17 20 1 -17 21 1 -17 22 1 -17 23 1 -17 24 1 -17 25 1 -17 26 1 -17 27 1 -17 28 1 -17 29 1 -17 30 1 -17 31 1 -17 32 1 -17 33 1 -17 34 1 -17 35 1 -17 36 1 -17 37 1 -17 38 1 -17 39 1 -17 40 1 -17 41 1 -17 42 1 -17 43 1 -17 44 1 -17 45 1 -17 46 1 -17 47 1 -17 48 1 -17 49 1 -17 50 1 -17 51 1 -17 52 1 -17 53 1 -17 54 1 -17 55 1 -17 56 1 -17 57 1 -17 58 1 -17 59 1 -17 60 1 -17 61 1 -17 62 1 -17 63 1 -17 64 1 -17 65 1 -17 66 1 -17 67 1 -17 68 1 -17 69 1 -17 70 1 -17 71 1 -17 72 1 -17 73 1 -17 74 1 -17 75 1 -17 76 1 -17 77 1 -17 78 1 -17 79 1 -17 80 1 -17 81 1 -17 82 1 -17 83 1 -17 84 1 -17 85 1 -17 86 1 -17 87 1 -17 88 1 -17 89 1 -17 90 1 -17 91 1 -17 92 1 -17 93 1 -17 94 1 -17 95 1 -17 96 1 -17 97 1 -17 98 1 -17 99 1 -17 100 1 -18 0 1 -18 1 1 -18 2 1 -18 3 1 -18 4 1 -18 5 1 -18 6 1 -18 7 1 -18 8 1 -18 9 1 -18 10 1 -18 11 1 -18 12 1 -18 13 1 -18 14 1 -18 15 1 -18 16 1 -18 17 1 -18 18 1 -18 19 1 -18 20 1 -18 21 1 -18 22 1 -18 23 1 -18 24 1 -18 25 1 -18 26 1 -18 27 1 -18 28 1 -18 29 1 -18 30 1 -18 31 1 -18 32 1 -18 33 1 -18 34 1 -18 35 1 -18 36 1 -18 37 1 -18 38 1 -18 39 1 -18 40 1 -18 41 1 -18 42 1 -18 43 1 -18 44 1 -18 45 1 -18 46 1 -18 47 1 -18 48 1 -18 49 1 -18 50 1 -18 51 1 -18 52 1 -18 53 1 -18 54 1 -18 55 1 -18 56 1 -18 57 1 -18 58 1 -18 59 1 -18 60 1 -18 61 1 -18 62 1 -18 63 1 -18 64 1 -18 65 1 -18 66 1 -18 67 1 -18 68 1 -18 69 1 -18 70 1 -18 71 1 -18 72 1 -18 73 1 -18 74 1 -18 75 1 -18 76 1 -18 77 1 -18 78 1 -18 79 1 -18 80 1 -18 81 1 -18 82 1 -18 83 1 -18 84 1 -18 85 1 -18 86 1 -18 87 1 -18 88 1 -18 89 1 -18 90 1 -18 91 1 -18 92 1 -18 93 1 -18 94 1 -18 95 1 -18 96 1 -18 97 1 -18 98 1 -18 99 1 -18 100 1 -19 0 1 -19 1 1 -19 2 1 -19 3 1 -19 4 1 -19 5 1 -19 6 1 -19 7 1 -19 8 1 -19 9 1 -19 10 1 -19 11 1 -19 12 1 -19 13 1 -19 14 1 -19 15 1 -19 16 1 -19 17 1 -19 18 1 -19 19 1 -19 20 1 -19 21 1 -19 22 1 -19 23 1 -19 24 1 -19 25 1 -19 26 1 -19 27 1 -19 28 1 -19 29 1 -19 30 1 -19 31 1 -19 32 1 -19 33 1 -19 34 1 -19 35 1 -19 36 1 -19 37 1 -19 38 1 -19 39 1 -19 40 1 -19 41 1 -19 42 1 -19 43 1 -19 44 1 -19 45 1 -19 46 1 -19 47 1 -19 48 1 -19 49 1 -19 50 1 -19 51 1 -19 52 1 -19 53 1 -19 54 1 -19 55 1 -19 56 1 -19 57 1 -19 58 1 -19 59 1 -19 60 1 -19 61 1 -19 62 1 -19 63 1 -19 64 1 -19 65 1 -19 66 1 -19 67 1 -19 68 1 -19 69 1 -19 70 1 -19 71 1 -19 72 1 -19 73 1 -19 74 1 -19 75 1 -19 76 1 -19 77 1 -19 78 1 -19 79 1 -19 80 1 -19 81 1 -19 82 1 -19 83 1 -19 84 1 -19 85 1 -19 86 1 -19 87 1 -19 88 1 -19 89 1 -19 90 1 -19 91 1 -19 92 1 -19 93 1 -19 94 1 -19 95 1 -19 96 1 -19 97 1 -19 98 1 -19 99 1 -19 100 1 -20 0 1 -20 1 1 -20 2 1 -20 3 1 -20 4 1 -20 5 1 -20 6 1 -20 7 1 -20 8 1 -20 9 1 -20 10 1 -20 11 1 -20 12 1 -20 13 1 -20 14 1 -20 15 1 -20 16 1 -20 17 1 -20 18 1 -20 19 1 -20 20 1 -20 21 1 -20 22 1 -20 23 1 -20 24 1 -20 25 1 -20 26 1 -20 27 1 -20 28 1 -20 29 1 -20 30 1 -20 31 1 -20 32 1 -20 33 1 -20 34 1 -20 35 1 -20 36 1 -20 37 1 -20 38 1 -20 39 1 -20 40 1 -20 41 1 -20 42 1 -20 43 1 -20 44 1 -20 45 1 -20 46 1 -20 47 1 -20 48 1 -20 49 1 -20 50 1 -20 51 1 -20 52 1 -20 53 1 -20 54 1 -20 55 1 -20 56 1 -20 57 1 -20 58 1 -20 59 1 -20 60 1 -20 61 1 -20 62 1 -20 63 1 -20 64 1 -20 65 1 -20 66 1 -20 67 1 -20 68 1 -20 69 1 -20 70 1 -20 71 1 -20 72 1 -20 73 1 -20 74 1 -20 75 1 -20 76 1 -20 77 1 -20 78 1 -20 79 1 -20 80 1 -20 81 1 -20 82 1 -20 83 1 -20 84 1 -20 85 1 -20 86 1 -20 87 1 -20 88 1 -20 89 1 -20 90 1 -20 91 1 -20 92 1 -20 93 1 -20 94 1 -20 95 1 -20 96 1 -20 97 1 -20 98 1 -20 99 1 -20 100 1 -21 0 1 -21 1 1 -21 2 1 -21 3 1 -21 4 1 -21 5 1 -21 6 1 -21 7 1 -21 8 1 -21 9 1 -21 10 1 -21 11 1 -21 12 1 -21 13 1 -21 14 1 -21 15 1 -21 16 1 -21 17 1 -21 18 1 -21 19 1 -21 20 1 -21 21 1 -21 22 1 -21 23 1 -21 24 1 -21 25 1 -21 26 1 -21 27 1 -21 28 1 -21 29 1 -21 30 1 -21 31 1 -21 32 1 -21 33 1 -21 34 1 -21 35 1 -21 36 1 -21 37 1 -21 38 1 -21 39 1 -21 40 1 -21 41 1 -21 42 1 -21 43 1 -21 44 1 -21 45 1 -21 46 1 -21 47 1 -21 48 1 -21 49 1 -21 50 1 -21 51 1 -21 52 1 -21 53 1 -21 54 1 -21 55 1 -21 56 1 -21 57 1 -21 58 1 -21 59 1 -21 60 1 -21 61 1 -21 62 1 -21 63 1 -21 64 1 -21 65 1 -21 66 1 -21 67 1 -21 68 1 -21 69 1 -21 70 1 -21 71 1 -21 72 1 -21 73 1 -21 74 1 -21 75 1 -21 76 1 -21 77 1 -21 78 1 -21 79 1 -21 80 1 -21 81 1 -21 82 1 -21 83 1 -21 84 1 -21 85 1 -21 86 1 -21 87 1 -21 88 1 -21 89 1 -21 90 1 -21 91 1 -21 92 1 -21 93 1 -21 94 1 -21 95 1 -21 96 1 -21 97 1 -21 98 1 -21 99 1 -21 100 1 -22 0 1 -22 1 1 -22 2 1 -22 3 1 -22 4 1 -22 5 1 -22 6 1 -22 7 1 -22 8 1 -22 9 1 -22 10 1 -22 11 1 -22 12 1 -22 13 1 -22 14 1 -22 15 1 -22 16 1 -22 17 1 -22 18 1 -22 19 1 -22 20 1 -22 21 1 -22 22 1 -22 23 1 -22 24 1 -22 25 1 -22 26 1 -22 27 1 -22 28 1 -22 29 1 -22 30 1 -22 31 1 -22 32 1 -22 33 1 -22 34 1 -22 35 1 -22 36 1 -22 37 1 -22 38 1 -22 39 1 -22 40 1 -22 41 1 -22 42 1 -22 43 1 -22 44 1 -22 45 1 -22 46 1 -22 47 1 -22 48 1 -22 49 1 -22 50 1 -22 51 1 -22 52 1 -22 53 1 -22 54 1 -22 55 1 -22 56 1 -22 57 1 -22 58 1 -22 59 1 -22 60 1 -22 61 1 -22 62 1 -22 63 1 -22 64 1 -22 65 1 -22 66 1 -22 67 1 -22 68 1 -22 69 1 -22 70 1 -22 71 1 -22 72 1 -22 73 1 -22 74 1 -22 75 1 -22 76 1 -22 77 1 -22 78 1 -22 79 1 -22 80 1 -22 81 1 -22 82 1 -22 83 1 -22 84 1 -22 85 1 -22 86 1 -22 87 1 -22 88 1 -22 89 1 -22 90 1 -22 91 1 -22 92 1 -22 93 1 -22 94 1 -22 95 1 -22 96 1 -22 97 1 -22 98 1 -22 99 1 -22 100 1 -23 0 1 -23 1 1 -23 2 1 -23 3 1 -23 4 1 -23 5 1 -23 6 1 -23 7 1 -23 8 1 -23 9 1 -23 10 1 -23 11 1 -23 12 1 -23 13 1 -23 14 1 -23 15 1 -23 16 1 -23 17 1 -23 18 1 -23 19 1 -23 20 1 -23 21 1 -23 22 1 -23 23 1 -23 24 1 -23 25 1 -23 26 1 -23 27 1 -23 28 1 -23 29 1 -23 30 1 -23 31 1 -23 32 1 -23 33 1 -23 34 1 -23 35 1 -23 36 1 -23 37 1 -23 38 1 -23 39 1 -23 40 1 -23 41 1 -23 42 1 -23 43 1 -23 44 1 -23 45 1 -23 46 1 -23 47 1 -23 48 1 -23 49 1 -23 50 1 -23 51 1 -23 52 1 -23 53 1 -23 54 1 -23 55 1 -23 56 1 -23 57 1 -23 58 1 -23 59 1 -23 60 1 -23 61 1 -23 62 1 -23 63 1 -23 64 1 -23 65 1 -23 66 1 -23 67 1 -23 68 1 -23 69 1 -23 70 1 -23 71 1 -23 72 1 -23 73 1 -23 74 1 -23 75 1 -23 76 1 -23 77 1 -23 78 1 -23 79 1 -23 80 1 -23 81 1 -23 82 1 -23 83 1 -23 84 1 -23 85 1 -23 86 1 -23 87 1 -23 88 1 -23 89 1 -23 90 1 -23 91 1 -23 92 1 -23 93 1 -23 94 1 -23 95 1 -23 96 1 -23 97 1 -23 98 1 -23 99 1 -23 100 1 -24 0 1 -24 1 1 -24 2 1 -24 3 1 -24 4 1 -24 5 1 -24 6 1 -24 7 1 -24 8 1 -24 9 1 -24 10 1 -24 11 1 -24 12 1 -24 13 1 -24 14 1 -24 15 1 -24 16 1 -24 17 1 -24 18 1 -24 19 1 -24 20 1 -24 21 1 -24 22 1 -24 23 1 -24 24 1 -24 25 1 -24 26 1 -24 27 1 -24 28 1 -24 29 1 -24 30 1 -24 31 1 -24 32 1 -24 33 1 -24 34 1 -24 35 1 -24 36 1 -24 37 1 -24 38 1 -24 39 1 -24 40 1 -24 41 1 -24 42 1 -24 43 1 -24 44 1 -24 45 1 -24 46 1 -24 47 1 -24 48 1 -24 49 1 -24 50 1 -24 51 1 -24 52 1 -24 53 1 -24 54 1 -24 55 1 -24 56 1 -24 57 1 -24 58 1 -24 59 1 -24 60 1 -24 61 1 -24 62 1 -24 63 1 -24 64 1 -24 65 1 -24 66 1 -24 67 1 -24 68 1 -24 69 1 -24 70 1 -24 71 1 -24 72 1 -24 73 1 -24 74 1 -24 75 1 -24 76 1 -24 77 1 -24 78 1 -24 79 1 -24 80 1 -24 81 1 -24 82 1 -24 83 1 -24 84 1 -24 85 1 -24 86 1 -24 87 1 -24 88 1 -24 89 1 -24 90 1 -24 91 1 -24 92 1 -24 93 1 -24 94 1 -24 95 1 -24 96 1 -24 97 1 -24 98 1 -24 99 1 -24 100 1 -25 0 1 -25 1 1 -25 2 1 -25 3 1 -25 4 1 -25 5 1 -25 6 1 -25 7 1 -25 8 1 -25 9 1 -25 10 1 -25 11 1 -25 12 1 -25 13 1 -25 14 1 -25 15 1 -25 16 1 -25 17 1 -25 18 1 -25 19 1 -25 20 1 -25 21 1 -25 22 1 -25 23 1 -25 24 1 -25 25 1 -25 26 1 -25 27 1 -25 28 1 -25 29 1 -25 30 1 -25 31 1 -25 32 1 -25 33 1 -25 34 1 -25 35 1 -25 36 1 -25 37 1 -25 38 1 -25 39 1 -25 40 1 -25 41 1 -25 42 1 -25 43 1 -25 44 1 -25 45 1 -25 46 1 -25 47 1 -25 48 1 -25 49 1 -25 50 1 -25 51 1 -25 52 1 -25 53 1 -25 54 1 -25 55 1 -25 56 1 -25 57 1 -25 58 1 -25 59 1 -25 60 1 -25 61 1 -25 62 1 -25 63 1 -25 64 1 -25 65 1 -25 66 1 -25 67 1 -25 68 1 -25 69 1 -25 70 1 -25 71 1 -25 72 1 -25 73 1 -25 74 1 -25 75 1 -25 76 1 -25 77 1 -25 78 1 -25 79 1 -25 80 1 -25 81 1 -25 82 1 -25 83 1 -25 84 1 -25 85 1 -25 86 1 -25 87 1 -25 88 1 -25 89 1 -25 90 1 -25 91 1 -25 92 1 -25 93 1 -25 94 1 -25 95 1 -25 96 1 -25 97 1 -25 98 1 -25 99 1 -25 100 1 -26 0 1 -26 1 1 -26 2 1 -26 3 1 -26 4 1 -26 5 1 -26 6 1 -26 7 1 -26 8 1 -26 9 1 -26 10 1 -26 11 1 -26 12 1 -26 13 1 -26 14 1 -26 15 1 -26 16 1 -26 17 1 -26 18 1 -26 19 1 -26 20 1 -26 21 1 -26 22 1 -26 23 1 -26 24 1 -26 25 1 -26 26 1 -26 27 1 -26 28 1 -26 29 1 -26 30 1 -26 31 1 -26 32 1 -26 33 1 -26 34 1 -26 35 1 -26 36 1 -26 37 1 -26 38 1 -26 39 1 -26 40 1 -26 41 1 -26 42 1 -26 43 1 -26 44 1 -26 45 1 -26 46 1 -26 47 1 -26 48 1 -26 49 1 -26 50 1 -26 51 1 -26 52 1 -26 53 1 -26 54 1 -26 55 1 -26 56 1 -26 57 1 -26 58 1 -26 59 1 -26 60 1 -26 61 1 -26 62 1 -26 63 1 -26 64 1 -26 65 1 -26 66 1 -26 67 1 -26 68 1 -26 69 1 -26 70 1 -26 71 1 -26 72 1 -26 73 1 -26 74 1 -26 75 1 -26 76 1 -26 77 1 -26 78 1 -26 79 1 -26 80 1 -26 81 1 -26 82 1 -26 83 1 -26 84 1 -26 85 1 -26 86 1 -26 87 1 -26 88 1 -26 89 1 -26 90 1 -26 91 1 -26 92 1 -26 93 1 -26 94 1 -26 95 1 -26 96 1 -26 97 1 -26 98 1 -26 99 1 -26 100 1 -27 0 1 -27 1 1 -27 2 1 -27 3 1 -27 4 1 -27 5 1 -27 6 1 -27 7 1 -27 8 1 -27 9 1 -27 10 1 -27 11 1 -27 12 1 -27 13 1 -27 14 1 -27 15 1 -27 16 1 -27 17 1 -27 18 1 -27 19 1 -27 20 1 -27 21 1 -27 22 1 -27 23 1 -27 24 1 -27 25 1 -27 26 1 -27 27 1 -27 28 1 -27 29 1 -27 30 1 -27 31 1 -27 32 1 -27 33 1 -27 34 1 -27 35 1 -27 36 1 -27 37 1 -27 38 1 -27 39 1 -27 40 1 -27 41 1 -27 42 1 -27 43 1 -27 44 1 -27 45 1 -27 46 1 -27 47 1 -27 48 1 -27 49 1 -27 50 1 -27 51 1 -27 52 1 -27 53 1 -27 54 1 -27 55 1 -27 56 1 -27 57 1 -27 58 1 -27 59 1 -27 60 1 -27 61 1 -27 62 1 -27 63 1 -27 64 1 -27 65 1 -27 66 1 -27 67 1 -27 68 1 -27 69 1 -27 70 1 -27 71 1 -27 72 1 -27 73 1 -27 74 1 -27 75 1 -27 76 1 -27 77 1 -27 78 1 -27 79 1 -27 80 1 -27 81 1 -27 82 1 -27 83 1 -27 84 1 -27 85 1 -27 86 1 -27 87 1 -27 88 1 -27 89 1 -27 90 1 -27 91 1 -27 92 1 -27 93 1 -27 94 1 -27 95 1 -27 96 1 -27 97 1 -27 98 1 -27 99 1 -27 100 1 -28 0 1 -28 1 1 -28 2 1 -28 3 1 -28 4 1 -28 5 1 -28 6 1 -28 7 1 -28 8 1 -28 9 1 -28 10 1 -28 11 1 -28 12 1 -28 13 1 -28 14 1 -28 15 1 -28 16 1 -28 17 1 -28 18 1 -28 19 1 -28 20 1 -28 21 1 -28 22 1 -28 23 1 -28 24 1 -28 25 1 -28 26 1 -28 27 1 -28 28 1 -28 29 1 -28 30 1 -28 31 1 -28 32 1 -28 33 1 -28 34 1 -28 35 1 -28 36 1 -28 37 1 -28 38 1 -28 39 1 -28 40 1 -28 41 1 -28 42 1 -28 43 1 -28 44 1 -28 45 1 -28 46 1 -28 47 1 -28 48 1 -28 49 1 -28 50 1 -28 51 1 -28 52 1 -28 53 1 -28 54 1 -28 55 1 -28 56 1 -28 57 1 -28 58 1 -28 59 1 -28 60 1 -28 61 1 -28 62 1 -28 63 1 -28 64 1 -28 65 1 -28 66 1 -28 67 1 -28 68 1 -28 69 1 -28 70 1 -28 71 1 -28 72 1 -28 73 1 -28 74 1 -28 75 1 -28 76 1 -28 77 1 -28 78 1 -28 79 1 -28 80 1 -28 81 1 -28 82 1 -28 83 1 -28 84 1 -28 85 1 -28 86 1 -28 87 1 -28 88 1 -28 89 1 -28 90 1 -28 91 1 -28 92 1 -28 93 1 -28 94 1 -28 95 1 -28 96 1 -28 97 1 -28 98 1 -28 99 1 -28 100 1 -29 0 1 -29 1 1 -29 2 1 -29 3 1 -29 4 1 -29 5 1 -29 6 1 -29 7 1 -29 8 1 -29 9 1 -29 10 1 -29 11 1 -29 12 1 -29 13 1 -29 14 1 -29 15 1 -29 16 1 -29 17 1 -29 18 1 -29 19 1 -29 20 1 -29 21 1 -29 22 1 -29 23 1 -29 24 1 -29 25 1 -29 26 1 -29 27 1 -29 28 1 -29 29 1 -29 30 1 -29 31 1 -29 32 1 -29 33 1 -29 34 1 -29 35 1 -29 36 1 -29 37 1 -29 38 1 -29 39 1 -29 40 1 -29 41 1 -29 42 1 -29 43 1 -29 44 1 -29 45 1 -29 46 1 -29 47 1 -29 48 1 -29 49 1 -29 50 1 -29 51 1 -29 52 1 -29 53 1 -29 54 1 -29 55 1 -29 56 1 -29 57 1 -29 58 1 -29 59 1 -29 60 1 -29 61 1 -29 62 1 -29 63 1 -29 64 1 -29 65 1 -29 66 1 -29 67 1 -29 68 1 -29 69 1 -29 70 1 -29 71 1 -29 72 1 -29 73 1 -29 74 1 -29 75 1 -29 76 1 -29 77 1 -29 78 1 -29 79 1 -29 80 1 -29 81 1 -29 82 1 -29 83 1 -29 84 1 -29 85 1 -29 86 1 -29 87 1 -29 88 1 -29 89 1 -29 90 1 -29 91 1 -29 92 1 -29 93 1 -29 94 1 -29 95 1 -29 96 1 -29 97 1 -29 98 1 -29 99 1 -29 100 1 -30 0 1 -30 1 1 -30 2 1 -30 3 1 -30 4 1 -30 5 1 -30 6 1 -30 7 1 -30 8 1 -30 9 1 -30 10 1 -30 11 1 -30 12 1 -30 13 1 -30 14 1 -30 15 1 -30 16 1 -30 17 1 -30 18 1 -30 19 1 -30 20 1 -30 21 1 -30 22 1 -30 23 1 -30 24 1 -30 25 1 -30 26 1 -30 27 1 -30 28 1 -30 29 1 -30 30 1 -30 31 1 -30 32 1 -30 33 1 -30 34 1 -30 35 1 -30 36 1 -30 37 1 -30 38 1 -30 39 1 -30 40 1 -30 41 1 -30 42 1 -30 43 1 -30 44 1 -30 45 1 -30 46 1 -30 47 1 -30 48 1 -30 49 1 -30 50 1 -30 51 1 -30 52 1 -30 53 1 -30 54 1 -30 55 1 -30 56 1 -30 57 1 -30 58 1 -30 59 1 -30 60 1 -30 61 1 -30 62 1 -30 63 1 -30 64 1 -30 65 1 -30 66 1 -30 67 1 -30 68 1 -30 69 1 -30 70 1 -30 71 1 -30 72 1 -30 73 1 -30 74 1 -30 75 1 -30 76 1 -30 77 1 -30 78 1 -30 79 1 -30 80 1 -30 81 1 -30 82 1 -30 83 1 -30 84 1 -30 85 1 -30 86 1 -30 87 1 -30 88 1 -30 89 1 -30 90 1 -30 91 1 -30 92 1 -30 93 1 -30 94 1 -30 95 1 -30 96 1 -30 97 1 -30 98 1 -30 99 1 -30 100 1 -31 0 1 -31 1 1 -31 2 1 -31 3 1 -31 4 1 -31 5 1 -31 6 1 -31 7 1 -31 8 1 -31 9 1 -31 10 1 -31 11 1 -31 12 1 -31 13 1 -31 14 1 -31 15 1 -31 16 1 -31 17 1 -31 18 1 -31 19 1 -31 20 1 -31 21 1 -31 22 1 -31 23 1 -31 24 1 -31 25 1 -31 26 1 -31 27 1 -31 28 1 -31 29 1 -31 30 1 -31 31 1 -31 32 1 -31 33 1 -31 34 1 -31 35 1 -31 36 1 -31 37 1 -31 38 1 -31 39 1 -31 40 1 -31 41 1 -31 42 1 -31 43 1 -31 44 1 -31 45 1 -31 46 1 -31 47 1 -31 48 1 -31 49 1 -31 50 1 -31 51 1 -31 52 1 -31 53 1 -31 54 1 -31 55 1 -31 56 1 -31 57 1 -31 58 1 -31 59 1 -31 60 1 -31 61 1 -31 62 1 -31 63 1 -31 64 1 -31 65 1 -31 66 1 -31 67 1 -31 68 1 -31 69 1 -31 70 1 -31 71 1 -31 72 1 -31 73 1 -31 74 1 -31 75 1 -31 76 1 -31 77 1 -31 78 1 -31 79 1 -31 80 1 -31 81 1 -31 82 1 -31 83 1 -31 84 1 -31 85 1 -31 86 1 -31 87 1 -31 88 1 -31 89 1 -31 90 1 -31 91 1 -31 92 1 -31 93 1 -31 94 1 -31 95 1 -31 96 1 -31 97 1 -31 98 1 -31 99 1 -31 100 1 -32 0 1 -32 1 1 -32 2 1 -32 3 1 -32 4 1 -32 5 1 -32 6 1 -32 7 1 -32 8 1 -32 9 1 -32 10 1 -32 11 1 -32 12 1 -32 13 1 -32 14 1 -32 15 1 -32 16 1 -32 17 1 -32 18 1 -32 19 1 -32 20 1 -32 21 1 -32 22 1 -32 23 1 -32 24 1 -32 25 1 -32 26 1 -32 27 1 -32 28 1 -32 29 1 -32 30 1 -32 31 1 -32 32 1 -32 33 1 -32 34 1 -32 35 1 -32 36 1 -32 37 1 -32 38 1 -32 39 1 -32 40 1 -32 41 1 -32 42 1 -32 43 1 -32 44 1 -32 45 1 -32 46 1 -32 47 1 -32 48 1 -32 49 1 -32 50 1 -32 51 1 -32 52 1 -32 53 1 -32 54 1 -32 55 1 -32 56 1 -32 57 1 -32 58 1 -32 59 1 -32 60 1 -32 61 1 -32 62 1 -32 63 1 -32 64 1 -32 65 1 -32 66 1 -32 67 1 -32 68 1 -32 69 1 -32 70 1 -32 71 1 -32 72 1 -32 73 1 -32 74 1 -32 75 1 -32 76 1 -32 77 1 -32 78 1 -32 79 1 -32 80 1 -32 81 1 -32 82 1 -32 83 1 -32 84 1 -32 85 1 -32 86 1 -32 87 1 -32 88 1 -32 89 1 -32 90 1 -32 91 1 -32 92 1 -32 93 1 -32 94 1 -32 95 1 -32 96 1 -32 97 1 -32 98 1 -32 99 1 -32 100 1 -33 0 1 -33 1 1 -33 2 1 -33 3 1 -33 4 1 -33 5 1 -33 6 1 -33 7 1 -33 8 1 -33 9 1 -33 10 1 -33 11 1 -33 12 1 -33 13 1 -33 14 1 -33 15 1 -33 16 1 -33 17 1 -33 18 1 -33 19 1 -33 20 1 -33 21 1 -33 22 1 -33 23 1 -33 24 1 -33 25 1 -33 26 1 -33 27 1 -33 28 1 -33 29 1 -33 30 1 -33 31 1 -33 32 1 -33 33 1 -33 34 1 -33 35 1 -33 36 1 -33 37 1 -33 38 1 -33 39 1 -33 40 1 -33 41 1 -33 42 1 -33 43 1 -33 44 1 -33 45 1 -33 46 1 -33 47 1 -33 48 1 -33 49 1 -33 50 1 -33 51 1 -33 52 1 -33 53 1 -33 54 1 -33 55 1 -33 56 1 -33 57 1 -33 58 1 -33 59 1 -33 60 1 -33 61 1 -33 62 1 -33 63 1 -33 64 1 -33 65 1 -33 66 1 -33 67 1 -33 68 1 -33 69 1 -33 70 1 -33 71 1 -33 72 1 -33 73 1 -33 74 1 -33 75 1 -33 76 1 -33 77 1 -33 78 1 -33 79 1 -33 80 1 -33 81 1 -33 82 1 -33 83 1 -33 84 1 -33 85 1 -33 86 1 -33 87 1 -33 88 1 -33 89 1 -33 90 1 -33 91 1 -33 92 1 -33 93 1 -33 94 1 -33 95 1 -33 96 1 -33 97 1 -33 98 1 -33 99 1 -33 100 1 -34 0 1 -34 1 1 -34 2 1 -34 3 1 -34 4 1 -34 5 1 -34 6 1 -34 7 1 -34 8 1 -34 9 1 -34 10 1 -34 11 1 -34 12 1 -34 13 1 -34 14 1 -34 15 1 -34 16 1 -34 17 1 -34 18 1 -34 19 1 -34 20 1 -34 21 1 -34 22 1 -34 23 1 -34 24 1 -34 25 1 -34 26 1 -34 27 1 -34 28 1 -34 29 1 -34 30 1 -34 31 1 -34 32 1 -34 33 1 -34 34 1 -34 35 1 -34 36 1 -34 37 1 -34 38 1 -34 39 1 -34 40 1 -34 41 1 -34 42 1 -34 43 1 -34 44 1 -34 45 1 -34 46 1 -34 47 1 -34 48 1 -34 49 1 -34 50 1 -34 51 1 -34 52 1 -34 53 1 -34 54 1 -34 55 1 -34 56 1 -34 57 1 -34 58 1 -34 59 1 -34 60 1 -34 61 1 -34 62 1 -34 63 1 -34 64 1 -34 65 1 -34 66 1 -34 67 1 -34 68 1 -34 69 1 -34 70 1 -34 71 1 -34 72 1 -34 73 1 -34 74 1 -34 75 1 -34 76 1 -34 77 1 -34 78 1 -34 79 1 -34 80 1 -34 81 1 -34 82 1 -34 83 1 -34 84 1 -34 85 1 -34 86 1 -34 87 1 -34 88 1 -34 89 1 -34 90 1 -34 91 1 -34 92 1 -34 93 1 -34 94 1 -34 95 1 -34 96 1 -34 97 1 -34 98 1 -34 99 1 -34 100 1 -35 0 1 -35 1 1 -35 2 1 -35 3 1 -35 4 1 -35 5 1 -35 6 1 -35 7 1 -35 8 1 -35 9 1 -35 10 1 -35 11 1 -35 12 1 -35 13 1 -35 14 1 -35 15 1 -35 16 1 -35 17 1 -35 18 1 -35 19 1 -35 20 1 -35 21 1 -35 22 1 -35 23 1 -35 24 1 -35 25 1 -35 26 1 -35 27 1 -35 28 1 -35 29 1 -35 30 1 -35 31 1 -35 32 1 -35 33 1 -35 34 1 -35 35 1 -35 36 1 -35 37 1 -35 38 1 -35 39 1 -35 40 1 -35 41 1 -35 42 1 -35 43 1 -35 44 1 -35 45 1 -35 46 1 -35 47 1 -35 48 1 -35 49 1 -35 50 1 -35 51 1 -35 52 1 -35 53 1 -35 54 1 -35 55 1 -35 56 1 -35 57 1 -35 58 1 -35 59 1 -35 60 1 -35 61 1 -35 62 1 -35 63 1 -35 64 1 -35 65 1 -35 66 1 -35 67 1 -35 68 1 -35 69 1 -35 70 1 -35 71 1 -35 72 1 -35 73 1 -35 74 1 -35 75 1 -35 76 1 -35 77 1 -35 78 1 -35 79 1 -35 80 1 -35 81 1 -35 82 1 -35 83 1 -35 84 1 -35 85 1 -35 86 1 -35 87 1 -35 88 1 -35 89 1 -35 90 1 -35 91 1 -35 92 1 -35 93 1 -35 94 1 -35 95 1 -35 96 1 -35 97 1 -35 98 1 -35 99 1 -35 100 1 -36 0 1 -36 1 1 -36 2 1 -36 3 1 -36 4 1 -36 5 1 -36 6 1 -36 7 1 -36 8 1 -36 9 1 -36 10 1 -36 11 1 -36 12 1 -36 13 1 -36 14 1 -36 15 1 -36 16 1 -36 17 1 -36 18 1 -36 19 1 -36 20 1 -36 21 1 -36 22 1 -36 23 1 -36 24 1 -36 25 1 -36 26 1 -36 27 1 -36 28 1 -36 29 1 -36 30 1 -36 31 1 -36 32 1 -36 33 1 -36 34 1 -36 35 1 -36 36 1 -36 37 1 -36 38 1 -36 39 1 -36 40 1 -36 41 1 -36 42 1 -36 43 1 -36 44 1 -36 45 1 -36 46 1 -36 47 1 -36 48 1 -36 49 1 -36 50 1 -36 51 1 -36 52 1 -36 53 1 -36 54 1 -36 55 1 -36 56 1 -36 57 1 -36 58 1 -36 59 1 -36 60 1 -36 61 1 -36 62 1 -36 63 1 -36 64 1 -36 65 1 -36 66 1 -36 67 1 -36 68 1 -36 69 1 -36 70 1 -36 71 1 -36 72 1 -36 73 1 -36 74 1 -36 75 1 -36 76 1 -36 77 1 -36 78 1 -36 79 1 -36 80 1 -36 81 1 -36 82 1 -36 83 1 -36 84 1 -36 85 1 -36 86 1 -36 87 1 -36 88 1 -36 89 1 -36 90 1 -36 91 1 -36 92 1 -36 93 1 -36 94 1 -36 95 1 -36 96 1 -36 97 1 -36 98 1 -36 99 1 -36 100 1 -37 0 1 -37 1 1 -37 2 1 -37 3 1 -37 4 1 -37 5 1 -37 6 1 -37 7 1 -37 8 1 -37 9 1 -37 10 1 -37 11 1 -37 12 1 -37 13 1 -37 14 1 -37 15 1 -37 16 1 -37 17 1 -37 18 1 -37 19 1 -37 20 1 -37 21 1 -37 22 1 -37 23 1 -37 24 1 -37 25 1 -37 26 1 -37 27 1 -37 28 1 -37 29 1 -37 30 1 -37 31 1 -37 32 1 -37 33 1 -37 34 1 -37 35 1 -37 36 1 -37 37 1 -37 38 1 -37 39 1 -37 40 1 -37 41 1 -37 42 1 -37 43 1 -37 44 1 -37 45 1 -37 46 1 -37 47 1 -37 48 1 -37 49 1 -37 50 1 -37 51 1 -37 52 1 -37 53 1 -37 54 1 -37 55 1 -37 56 1 -37 57 1 -37 58 1 -37 59 1 -37 60 1 -37 61 1 -37 62 1 -37 63 1 -37 64 1 -37 65 1 -37 66 1 -37 67 1 -37 68 1 -37 69 1 -37 70 1 -37 71 1 -37 72 1 -37 73 1 -37 74 1 -37 75 1 -37 76 1 -37 77 1 -37 78 1 -37 79 1 -37 80 1 -37 81 1 -37 82 1 -37 83 1 -37 84 1 -37 85 1 -37 86 1 -37 87 1 -37 88 1 -37 89 1 -37 90 1 -37 91 1 -37 92 1 -37 93 1 -37 94 1 -37 95 1 -37 96 1 -37 97 1 -37 98 1 -37 99 1 -37 100 1 -38 0 1 -38 1 1 -38 2 1 -38 3 1 -38 4 1 -38 5 1 -38 6 1 -38 7 1 -38 8 1 -38 9 1 -38 10 1 -38 11 1 -38 12 1 -38 13 1 -38 14 1 -38 15 1 -38 16 1 -38 17 1 -38 18 1 -38 19 1 -38 20 1 -38 21 1 -38 22 1 -38 23 1 -38 24 1 -38 25 1 -38 26 1 -38 27 1 -38 28 1 -38 29 1 -38 30 1 -38 31 1 -38 32 1 -38 33 1 -38 34 1 -38 35 1 -38 36 1 -38 37 1 -38 38 1 -38 39 1 -38 40 1 -38 41 1 -38 42 1 -38 43 1 -38 44 1 -38 45 1 -38 46 1 -38 47 1 -38 48 1 -38 49 1 -38 50 1 -38 51 1 -38 52 1 -38 53 1 -38 54 1 -38 55 1 -38 56 1 -38 57 1 -38 58 1 -38 59 1 -38 60 1 -38 61 1 -38 62 1 -38 63 1 -38 64 1 -38 65 1 -38 66 1 -38 67 1 -38 68 1 -38 69 1 -38 70 1 -38 71 1 -38 72 1 -38 73 1 -38 74 1 -38 75 1 -38 76 1 -38 77 1 -38 78 1 -38 79 1 -38 80 1 -38 81 1 -38 82 1 -38 83 1 -38 84 1 -38 85 1 -38 86 1 -38 87 1 -38 88 1 -38 89 1 -38 90 1 -38 91 1 -38 92 1 -38 93 1 -38 94 1 -38 95 1 -38 96 1 -38 97 1 -38 98 1 -38 99 1 -38 100 1 -39 0 1 -39 1 1 -39 2 1 -39 3 1 -39 4 1 -39 5 1 -39 6 1 -39 7 1 -39 8 1 -39 9 1 -39 10 1 -39 11 1 -39 12 1 -39 13 1 -39 14 1 -39 15 1 -39 16 1 -39 17 1 -39 18 1 -39 19 1 -39 20 1 -39 21 1 -39 22 1 -39 23 1 -39 24 1 -39 25 1 -39 26 1 -39 27 1 -39 28 1 -39 29 1 -39 30 1 -39 31 1 -39 32 1 -39 33 1 -39 34 1 -39 35 1 -39 36 1 -39 37 1 -39 38 1 -39 39 1 -39 40 1 -39 41 1 -39 42 1 -39 43 1 -39 44 1 -39 45 1 -39 46 1 -39 47 1 -39 48 1 -39 49 1 -39 50 1 -39 51 1 -39 52 1 -39 53 1 -39 54 1 -39 55 1 -39 56 1 -39 57 1 -39 58 1 -39 59 1 -39 60 1 -39 61 1 -39 62 1 -39 63 1 -39 64 1 -39 65 1 -39 66 1 -39 67 1 -39 68 1 -39 69 1 -39 70 1 -39 71 1 -39 72 1 -39 73 1 -39 74 1 -39 75 1 -39 76 1 -39 77 1 -39 78 1 -39 79 1 -39 80 1 -39 81 1 -39 82 1 -39 83 1 -39 84 1 -39 85 1 -39 86 1 -39 87 1 -39 88 1 -39 89 1 -39 90 1 -39 91 1 -39 92 1 -39 93 1 -39 94 1 -39 95 1 -39 96 1 -39 97 1 -39 98 1 -39 99 1 -39 100 1 -40 0 1 -40 1 1 -40 2 1 -40 3 1 -40 4 1 -40 5 1 -40 6 1 -40 7 1 -40 8 1 -40 9 1 -40 10 1 -40 11 1 -40 12 1 -40 13 1 -40 14 1 -40 15 1 -40 16 1 -40 17 1 -40 18 1 -40 19 1 -40 20 1 -40 21 1 -40 22 1 -40 23 1 -40 24 1 -40 25 1 -40 26 1 -40 27 1 -40 28 1 -40 29 1 -40 30 1 -40 31 1 -40 32 1 -40 33 1 -40 34 1 -40 35 1 -40 36 1 -40 37 1 -40 38 1 -40 39 1 -40 40 1 -40 41 1 -40 42 1 -40 43 1 -40 44 1 -40 45 1 -40 46 1 -40 47 1 -40 48 1 -40 49 1 -40 50 1 -40 51 1 -40 52 1 -40 53 1 -40 54 1 -40 55 1 -40 56 1 -40 57 1 -40 58 1 -40 59 1 -40 60 1 -40 61 1 -40 62 1 -40 63 1 -40 64 1 -40 65 1 -40 66 1 -40 67 1 -40 68 1 -40 69 1 -40 70 1 -40 71 1 -40 72 1 -40 73 1 -40 74 1 -40 75 1 -40 76 1 -40 77 1 -40 78 1 -40 79 1 -40 80 1 -40 81 1 -40 82 1 -40 83 1 -40 84 1 -40 85 1 -40 86 1 -40 87 1 -40 88 1 -40 89 1 -40 90 1 -40 91 1 -40 92 1 -40 93 1 -40 94 1 -40 95 1 -40 96 1 -40 97 1 -40 98 1 -40 99 1 -40 100 1 -41 0 1 -41 1 1 -41 2 1 -41 3 1 -41 4 1 -41 5 1 -41 6 1 -41 7 1 -41 8 1 -41 9 1 -41 10 1 -41 11 1 -41 12 1 -41 13 1 -41 14 1 -41 15 1 -41 16 1 -41 17 1 -41 18 1 -41 19 1 -41 20 1 -41 21 1 -41 22 1 -41 23 1 -41 24 1 -41 25 1 -41 26 1 -41 27 1 -41 28 1 -41 29 1 -41 30 1 -41 31 1 -41 32 1 -41 33 1 -41 34 1 -41 35 1 -41 36 1 -41 37 1 -41 38 1 -41 39 1 -41 40 1 -41 41 1 -41 42 1 -41 43 1 -41 44 1 -41 45 1 -41 46 1 -41 47 1 -41 48 1 -41 49 1 -41 50 1 -41 51 1 -41 52 1 -41 53 1 -41 54 1 -41 55 1 -41 56 1 -41 57 1 -41 58 1 -41 59 1 -41 60 1 -41 61 1 -41 62 1 -41 63 1 -41 64 1 -41 65 1 -41 66 1 -41 67 1 -41 68 1 -41 69 1 -41 70 1 -41 71 1 -41 72 1 -41 73 1 -41 74 1 -41 75 1 -41 76 1 -41 77 1 -41 78 1 -41 79 1 -41 80 1 -41 81 1 -41 82 1 -41 83 1 -41 84 1 -41 85 1 -41 86 1 -41 87 1 -41 88 1 -41 89 1 -41 90 1 -41 91 1 -41 92 1 -41 93 1 -41 94 1 -41 95 1 -41 96 1 -41 97 1 -41 98 1 -41 99 1 -41 100 1 -42 0 1 -42 1 1 -42 2 1 -42 3 1 -42 4 1 -42 5 1 -42 6 1 -42 7 1 -42 8 1 -42 9 1 -42 10 1 -42 11 1 -42 12 1 -42 13 1 -42 14 1 -42 15 1 -42 16 1 -42 17 1 -42 18 1 -42 19 1 -42 20 1 -42 21 1 -42 22 1 -42 23 1 -42 24 1 -42 25 1 -42 26 1 -42 27 1 -42 28 1 -42 29 1 -42 30 1 -42 31 1 -42 32 1 -42 33 1 -42 34 1 -42 35 1 -42 36 1 -42 37 1 -42 38 1 -42 39 1 -42 40 1 -42 41 1 -42 42 1 -42 43 1 -42 44 1 -42 45 1 -42 46 1 -42 47 1 -42 48 1 -42 49 1 -42 50 1 -42 51 1 -42 52 1 -42 53 1 -42 54 1 -42 55 1 -42 56 1 -42 57 1 -42 58 1 -42 59 1 -42 60 1 -42 61 1 -42 62 1 -42 63 1 -42 64 1 -42 65 1 -42 66 1 -42 67 1 -42 68 1 -42 69 1 -42 70 1 -42 71 1 -42 72 1 -42 73 1 -42 74 1 -42 75 1 -42 76 1 -42 77 1 -42 78 1 -42 79 1 -42 80 1 -42 81 1 -42 82 1 -42 83 1 -42 84 1 -42 85 1 -42 86 1 -42 87 1 -42 88 1 -42 89 1 -42 90 1 -42 91 1 -42 92 1 -42 93 1 -42 94 1 -42 95 1 -42 96 1 -42 97 1 -42 98 1 -42 99 1 -42 100 1 -43 0 1 -43 1 1 -43 2 1 -43 3 1 -43 4 1 -43 5 1 -43 6 1 -43 7 1 -43 8 1 -43 9 1 -43 10 1 -43 11 1 -43 12 1 -43 13 1 -43 14 1 -43 15 1 -43 16 1 -43 17 1 -43 18 1 -43 19 1 -43 20 1 -43 21 1 -43 22 1 -43 23 1 -43 24 1 -43 25 1 -43 26 1 -43 27 1 -43 28 1 -43 29 1 -43 30 1 -43 31 1 -43 32 1 -43 33 1 -43 34 1 -43 35 1 -43 36 1 -43 37 1 -43 38 1 -43 39 1 -43 40 1 -43 41 1 -43 42 1 -43 43 1 -43 44 1 -43 45 1 -43 46 1 -43 47 1 -43 48 1 -43 49 1 -43 50 1 -43 51 1 -43 52 1 -43 53 1 -43 54 1 -43 55 1 -43 56 1 -43 57 1 -43 58 1 -43 59 1 -43 60 1 -43 61 1 -43 62 1 -43 63 1 -43 64 1 -43 65 1 -43 66 1 -43 67 1 -43 68 1 -43 69 1 -43 70 1 -43 71 1 -43 72 1 -43 73 1 -43 74 1 -43 75 1 -43 76 1 -43 77 1 -43 78 1 -43 79 1 -43 80 1 -43 81 1 -43 82 1 -43 83 1 -43 84 1 -43 85 1 -43 86 1 -43 87 1 -43 88 1 -43 89 1 -43 90 1 -43 91 1 -43 92 1 -43 93 1 -43 94 1 -43 95 1 -43 96 1 -43 97 1 -43 98 1 -43 99 1 -43 100 1 -44 0 1 -44 1 1 -44 2 1 -44 3 1 -44 4 1 -44 5 1 -44 6 1 -44 7 1 -44 8 1 -44 9 1 -44 10 1 -44 11 1 -44 12 1 -44 13 1 -44 14 1 -44 15 1 -44 16 1 -44 17 1 -44 18 1 -44 19 1 -44 20 1 -44 21 1 -44 22 1 -44 23 1 -44 24 1 -44 25 1 -44 26 1 -44 27 1 -44 28 1 -44 29 1 -44 30 1 -44 31 1 -44 32 1 -44 33 1 -44 34 1 -44 35 1 -44 36 1 -44 37 1 -44 38 1 -44 39 1 -44 40 1 -44 41 1 -44 42 1 -44 43 1 -44 44 1 -44 45 1 -44 46 1 -44 47 1 -44 48 1 -44 49 1 -44 50 1 -44 51 1 -44 52 1 -44 53 1 -44 54 1 -44 55 1 -44 56 1 -44 57 1 -44 58 1 -44 59 1 -44 60 1 -44 61 1 -44 62 1 -44 63 1 -44 64 1 -44 65 1 -44 66 1 -44 67 1 -44 68 1 -44 69 1 -44 70 1 -44 71 1 -44 72 1 -44 73 1 -44 74 1 -44 75 1 -44 76 1 -44 77 1 -44 78 1 -44 79 1 -44 80 1 -44 81 1 -44 82 1 -44 83 1 -44 84 1 -44 85 1 -44 86 1 -44 87 1 -44 88 1 -44 89 1 -44 90 1 -44 91 1 -44 92 1 -44 93 1 -44 94 1 -44 95 1 -44 96 1 -44 97 1 -44 98 1 -44 99 1 -44 100 1 -45 0 1 -45 1 1 -45 2 1 -45 3 1 -45 4 1 -45 5 1 -45 6 1 -45 7 1 -45 8 1 -45 9 1 -45 10 1 -45 11 1 -45 12 1 -45 13 1 -45 14 1 -45 15 1 -45 16 1 -45 17 1 -45 18 1 -45 19 1 -45 20 1 -45 21 1 -45 22 1 -45 23 1 -45 24 1 -45 25 1 -45 26 1 -45 27 1 -45 28 1 -45 29 1 -45 30 1 -45 31 1 -45 32 1 -45 33 1 -45 34 1 -45 35 1 -45 36 1 -45 37 1 -45 38 1 -45 39 1 -45 40 1 -45 41 1 -45 42 1 -45 43 1 -45 44 1 -45 45 1 -45 46 1 -45 47 1 -45 48 1 -45 49 1 -45 50 1 -45 51 1 -45 52 1 -45 53 1 -45 54 1 -45 55 1 -45 56 1 -45 57 1 -45 58 1 -45 59 1 -45 60 1 -45 61 1 -45 62 1 -45 63 1 -45 64 1 -45 65 1 -45 66 1 -45 67 1 -45 68 1 -45 69 1 -45 70 1 -45 71 1 -45 72 1 -45 73 1 -45 74 1 -45 75 1 -45 76 1 -45 77 1 -45 78 1 -45 79 1 -45 80 1 -45 81 1 -45 82 1 -45 83 1 -45 84 1 -45 85 1 -45 86 1 -45 87 1 -45 88 1 -45 89 1 -45 90 1 -45 91 1 -45 92 1 -45 93 1 -45 94 1 -45 95 1 -45 96 1 -45 97 1 -45 98 1 -45 99 1 -45 100 1 -46 0 1 -46 1 1 -46 2 1 -46 3 1 -46 4 1 -46 5 1 -46 6 1 -46 7 1 -46 8 1 -46 9 1 -46 10 1 -46 11 1 -46 12 1 -46 13 1 -46 14 1 -46 15 1 -46 16 1 -46 17 1 -46 18 1 -46 19 1 -46 20 1 -46 21 1 -46 22 1 -46 23 1 -46 24 1 -46 25 1 -46 26 1 -46 27 1 -46 28 1 -46 29 1 -46 30 1 -46 31 1 -46 32 1 -46 33 1 -46 34 1 -46 35 1 -46 36 1 -46 37 1 -46 38 1 -46 39 1 -46 40 1 -46 41 1 -46 42 1 -46 43 1 -46 44 1 -46 45 1 -46 46 1 -46 47 1 -46 48 1 -46 49 1 -46 50 1 -46 51 1 -46 52 1 -46 53 1 -46 54 1 -46 55 1 -46 56 1 -46 57 1 -46 58 1 -46 59 1 -46 60 1 -46 61 1 -46 62 1 -46 63 1 -46 64 1 -46 65 1 -46 66 1 -46 67 1 -46 68 1 -46 69 1 -46 70 1 -46 71 1 -46 72 1 -46 73 1 -46 74 1 -46 75 1 -46 76 1 -46 77 1 -46 78 1 -46 79 1 -46 80 1 -46 81 1 -46 82 1 -46 83 1 -46 84 1 -46 85 1 -46 86 1 -46 87 1 -46 88 1 -46 89 1 -46 90 1 -46 91 1 -46 92 1 -46 93 1 -46 94 1 -46 95 1 -46 96 1 -46 97 1 -46 98 1 -46 99 1 -46 100 1 -47 0 1 -47 1 1 -47 2 1 -47 3 1 -47 4 1 -47 5 1 -47 6 1 -47 7 1 -47 8 1 -47 9 1 -47 10 1 -47 11 1 -47 12 1 -47 13 1 -47 14 1 -47 15 1 -47 16 1 -47 17 1 -47 18 1 -47 19 1 -47 20 1 -47 21 1 -47 22 1 -47 23 1 -47 24 1 -47 25 1 -47 26 1 -47 27 1 -47 28 1 -47 29 1 -47 30 1 -47 31 1 -47 32 1 -47 33 1 -47 34 1 -47 35 1 -47 36 1 -47 37 1 -47 38 1 -47 39 1 -47 40 1 -47 41 1 -47 42 1 -47 43 1 -47 44 1 -47 45 1 -47 46 1 -47 47 1 -47 48 1 -47 49 1 -47 50 1 -47 51 1 -47 52 1 -47 53 1 -47 54 1 -47 55 1 -47 56 1 -47 57 1 -47 58 1 -47 59 1 -47 60 1 -47 61 1 -47 62 1 -47 63 1 -47 64 1 -47 65 1 -47 66 1 -47 67 1 -47 68 1 -47 69 1 -47 70 1 -47 71 1 -47 72 1 -47 73 1 -47 74 1 -47 75 1 -47 76 1 -47 77 1 -47 78 1 -47 79 1 -47 80 1 -47 81 1 -47 82 1 -47 83 1 -47 84 1 -47 85 1 -47 86 1 -47 87 1 -47 88 1 -47 89 1 -47 90 1 -47 91 1 -47 92 1 -47 93 1 -47 94 1 -47 95 1 -47 96 1 -47 97 1 -47 98 1 -47 99 1 -47 100 1 -48 0 1 -48 1 1 -48 2 1 -48 3 1 -48 4 1 -48 5 1 -48 6 1 -48 7 1 -48 8 1 -48 9 1 -48 10 1 -48 11 1 -48 12 1 -48 13 1 -48 14 1 -48 15 1 -48 16 1 -48 17 1 -48 18 1 -48 19 1 -48 20 1 -48 21 1 -48 22 1 -48 23 1 -48 24 1 -48 25 1 -48 26 1 -48 27 1 -48 28 1 -48 29 1 -48 30 1 -48 31 1 -48 32 1 -48 33 1 -48 34 1 -48 35 1 -48 36 1 -48 37 1 -48 38 1 -48 39 1 -48 40 1 -48 41 1 -48 42 1 -48 43 1 -48 44 1 -48 45 1 -48 46 1 -48 47 1 -48 48 1 -48 49 1 -48 50 1 -48 51 1 -48 52 1 -48 53 1 -48 54 1 -48 55 1 -48 56 1 -48 57 1 -48 58 1 -48 59 1 -48 60 1 -48 61 1 -48 62 1 -48 63 1 -48 64 1 -48 65 1 -48 66 1 -48 67 1 -48 68 1 -48 69 1 -48 70 1 -48 71 1 -48 72 1 -48 73 1 -48 74 1 -48 75 1 -48 76 1 -48 77 1 -48 78 1 -48 79 1 -48 80 1 -48 81 1 -48 82 1 -48 83 1 -48 84 1 -48 85 1 -48 86 1 -48 87 1 -48 88 1 -48 89 1 -48 90 1 -48 91 1 -48 92 1 -48 93 1 -48 94 1 -48 95 1 -48 96 1 -48 97 1 -48 98 1 -48 99 1 -48 100 1 -49 0 1 -49 1 1 -49 2 1 -49 3 1 -49 4 1 -49 5 1 -49 6 1 -49 7 1 -49 8 1 -49 9 1 -49 10 1 -49 11 1 -49 12 1 -49 13 1 -49 14 1 -49 15 1 -49 16 1 -49 17 1 -49 18 1 -49 19 1 -49 20 1 -49 21 1 -49 22 1 -49 23 1 -49 24 1 -49 25 1 -49 26 1 -49 27 1 -49 28 1 -49 29 1 -49 30 1 -49 31 1 -49 32 1 -49 33 1 -49 34 1 -49 35 1 -49 36 1 -49 37 1 -49 38 1 -49 39 1 -49 40 1 -49 41 1 -49 42 1 -49 43 1 -49 44 1 -49 45 1 -49 46 1 -49 47 1 -49 48 1 -49 49 1 -49 50 1 -49 51 1 -49 52 1 -49 53 1 -49 54 1 -49 55 1 -49 56 1 -49 57 1 -49 58 1 -49 59 1 -49 60 1 -49 61 1 -49 62 1 -49 63 1 -49 64 1 -49 65 1 -49 66 1 -49 67 1 -49 68 1 -49 69 1 -49 70 1 -49 71 1 -49 72 1 -49 73 1 -49 74 1 -49 75 1 -49 76 1 -49 77 1 -49 78 1 -49 79 1 -49 80 1 -49 81 1 -49 82 1 -49 83 1 -49 84 1 -49 85 1 -49 86 1 -49 87 1 -49 88 1 -49 89 1 -49 90 1 -49 91 1 -49 92 1 -49 93 1 -49 94 1 -49 95 1 -49 96 1 -49 97 1 -49 98 1 -49 99 1 -49 100 1 -50 0 1 -50 1 1 -50 2 1 -50 3 1 -50 4 1 -50 5 1 -50 6 1 -50 7 1 -50 8 1 -50 9 1 -50 10 1 -50 11 1 -50 12 1 -50 13 1 -50 14 1 -50 15 1 -50 16 1 -50 17 1 -50 18 1 -50 19 1 -50 20 1 -50 21 1 -50 22 1 -50 23 1 -50 24 1 -50 25 1 -50 26 1 -50 27 1 -50 28 1 -50 29 1 -50 30 1 -50 31 1 -50 32 1 -50 33 1 -50 34 1 -50 35 1 -50 36 1 -50 37 1 -50 38 1 -50 39 1 -50 40 1 -50 41 1 -50 42 1 -50 43 1 -50 44 1 -50 45 1 -50 46 1 -50 47 1 -50 48 1 -50 49 1 -50 50 1 -50 51 1 -50 52 1 -50 53 1 -50 54 1 -50 55 1 -50 56 1 -50 57 1 -50 58 1 -50 59 1 -50 60 1 -50 61 1 -50 62 1 -50 63 1 -50 64 1 -50 65 1 -50 66 1 -50 67 1 -50 68 1 -50 69 1 -50 70 1 -50 71 1 -50 72 1 -50 73 1 -50 74 1 -50 75 1 -50 76 1 -50 77 1 -50 78 1 -50 79 1 -50 80 1 -50 81 1 -50 82 1 -50 83 1 -50 84 1 -50 85 1 -50 86 1 -50 87 1 -50 88 1 -50 89 1 -50 90 1 -50 91 1 -50 92 1 -50 93 1 -50 94 1 -50 95 1 -50 96 1 -50 97 1 -50 98 1 -50 99 1 -50 100 1 -51 0 1 -51 1 1 -51 2 1 -51 3 1 -51 4 1 -51 5 1 -51 6 1 -51 7 1 -51 8 1 -51 9 1 -51 10 1 -51 11 1 -51 12 1 -51 13 1 -51 14 1 -51 15 1 -51 16 1 -51 17 1 -51 18 1 -51 19 1 -51 20 1 -51 21 1 -51 22 1 -51 23 1 -51 24 1 -51 25 1 -51 26 1 -51 27 1 -51 28 1 -51 29 1 -51 30 1 -51 31 1 -51 32 1 -51 33 1 -51 34 1 -51 35 1 -51 36 1 -51 37 1 -51 38 1 -51 39 1 -51 40 1 -51 41 1 -51 42 1 -51 43 1 -51 44 1 -51 45 1 -51 46 1 -51 47 1 -51 48 1 -51 49 1 -51 50 1 -51 51 1 -51 52 1 -51 53 1 -51 54 1 -51 55 1 -51 56 1 -51 57 1 -51 58 1 -51 59 1 -51 60 1 -51 61 1 -51 62 1 -51 63 1 -51 64 1 -51 65 1 -51 66 1 -51 67 1 -51 68 1 -51 69 1 -51 70 1 -51 71 1 -51 72 1 -51 73 1 -51 74 1 -51 75 1 -51 76 1 -51 77 1 -51 78 1 -51 79 1 -51 80 1 -51 81 1 -51 82 1 -51 83 1 -51 84 1 -51 85 1 -51 86 1 -51 87 1 -51 88 1 -51 89 1 -51 90 1 -51 91 1 -51 92 1 -51 93 1 -51 94 1 -51 95 1 -51 96 1 -51 97 1 -51 98 1 -51 99 1 -51 100 1 -52 0 1 -52 1 1 -52 2 1 -52 3 1 -52 4 1 -52 5 1 -52 6 1 -52 7 1 -52 8 1 -52 9 1 -52 10 1 -52 11 1 -52 12 1 -52 13 1 -52 14 1 -52 15 1 -52 16 1 -52 17 1 -52 18 1 -52 19 1 -52 20 1 -52 21 1 -52 22 1 -52 23 1 -52 24 1 -52 25 1 -52 26 1 -52 27 1 -52 28 1 -52 29 1 -52 30 1 -52 31 1 -52 32 1 -52 33 1 -52 34 1 -52 35 1 -52 36 1 -52 37 1 -52 38 1 -52 39 1 -52 40 1 -52 41 1 -52 42 1 -52 43 1 -52 44 1 -52 45 1 -52 46 1 -52 47 1 -52 48 1 -52 49 1 -52 50 1 -52 51 1 -52 52 1 -52 53 1 -52 54 1 -52 55 1 -52 56 1 -52 57 1 -52 58 1 -52 59 1 -52 60 1 -52 61 1 -52 62 1 -52 63 1 -52 64 1 -52 65 1 -52 66 1 -52 67 1 -52 68 1 -52 69 1 -52 70 1 -52 71 1 -52 72 1 -52 73 1 -52 74 1 -52 75 1 -52 76 1 -52 77 1 -52 78 1 -52 79 1 -52 80 1 -52 81 1 -52 82 1 -52 83 1 -52 84 1 -52 85 1 -52 86 1 -52 87 1 -52 88 1 -52 89 1 -52 90 1 -52 91 1 -52 92 1 -52 93 1 -52 94 1 -52 95 1 -52 96 1 -52 97 1 -52 98 1 -52 99 1 -52 100 1 -53 0 1 -53 1 1 -53 2 1 -53 3 1 -53 4 1 -53 5 1 -53 6 1 -53 7 1 -53 8 1 -53 9 1 -53 10 1 -53 11 1 -53 12 1 -53 13 1 -53 14 1 -53 15 1 -53 16 1 -53 17 1 -53 18 1 -53 19 1 -53 20 1 -53 21 1 -53 22 1 -53 23 1 -53 24 1 -53 25 1 -53 26 1 -53 27 1 -53 28 1 -53 29 1 -53 30 1 -53 31 1 -53 32 1 -53 33 1 -53 34 1 -53 35 1 -53 36 1 -53 37 1 -53 38 1 -53 39 1 -53 40 1 -53 41 1 -53 42 1 -53 43 1 -53 44 1 -53 45 1 -53 46 1 -53 47 1 -53 48 1 -53 49 1 -53 50 1 -53 51 1 -53 52 1 -53 53 1 -53 54 1 -53 55 1 -53 56 1 -53 57 1 -53 58 1 -53 59 1 -53 60 1 -53 61 1 -53 62 1 -53 63 1 -53 64 1 -53 65 1 -53 66 1 -53 67 1 -53 68 1 -53 69 1 -53 70 1 -53 71 1 -53 72 1 -53 73 1 -53 74 1 -53 75 1 -53 76 1 -53 77 1 -53 78 1 -53 79 1 -53 80 1 -53 81 1 -53 82 1 -53 83 1 -53 84 1 -53 85 1 -53 86 1 -53 87 1 -53 88 1 -53 89 1 -53 90 1 -53 91 1 -53 92 1 -53 93 1 -53 94 1 -53 95 1 -53 96 1 -53 97 1 -53 98 1 -53 99 1 -53 100 1 -54 0 1 -54 1 1 -54 2 1 -54 3 1 -54 4 1 -54 5 1 -54 6 1 -54 7 1 -54 8 1 -54 9 1 -54 10 1 -54 11 1 -54 12 1 -54 13 1 -54 14 1 -54 15 1 -54 16 1 -54 17 1 -54 18 1 -54 19 1 -54 20 1 -54 21 1 -54 22 1 -54 23 1 -54 24 1 -54 25 1 -54 26 1 -54 27 1 -54 28 1 -54 29 1 -54 30 1 -54 31 1 -54 32 1 -54 33 1 -54 34 1 -54 35 1 -54 36 1 -54 37 1 -54 38 1 -54 39 1 -54 40 1 -54 41 1 -54 42 1 -54 43 1 -54 44 1 -54 45 1 -54 46 1 -54 47 1 -54 48 1 -54 49 1 -54 50 1 -54 51 1 -54 52 1 -54 53 1 -54 54 1 -54 55 1 -54 56 1 -54 57 1 -54 58 1 -54 59 1 -54 60 1 -54 61 1 -54 62 1 -54 63 1 -54 64 1 -54 65 1 -54 66 1 -54 67 1 -54 68 1 -54 69 1 -54 70 1 -54 71 1 -54 72 1 -54 73 1 -54 74 1 -54 75 1 -54 76 1 -54 77 1 -54 78 1 -54 79 1 -54 80 1 -54 81 1 -54 82 1 -54 83 1 -54 84 1 -54 85 1 -54 86 1 -54 87 1 -54 88 1 -54 89 1 -54 90 1 -54 91 1 -54 92 1 -54 93 1 -54 94 1 -54 95 1 -54 96 1 -54 97 1 -54 98 1 -54 99 1 -54 100 1 -55 0 1 -55 1 1 -55 2 1 -55 3 1 -55 4 1 -55 5 1 -55 6 1 -55 7 1 -55 8 1 -55 9 1 -55 10 1 -55 11 1 -55 12 1 -55 13 1 -55 14 1 -55 15 1 -55 16 1 -55 17 1 -55 18 1 -55 19 1 -55 20 1 -55 21 1 -55 22 1 -55 23 1 -55 24 1 -55 25 1 -55 26 1 -55 27 1 -55 28 1 -55 29 1 -55 30 1 -55 31 1 -55 32 1 -55 33 1 -55 34 1 -55 35 1 -55 36 1 -55 37 1 -55 38 1 -55 39 1 -55 40 1 -55 41 1 -55 42 1 -55 43 1 -55 44 1 -55 45 1 -55 46 1 -55 47 1 -55 48 1 -55 49 1 -55 50 1 -55 51 1 -55 52 1 -55 53 1 -55 54 1 -55 55 1 -55 56 1 -55 57 1 -55 58 1 -55 59 1 -55 60 1 -55 61 1 -55 62 1 -55 63 1 -55 64 1 -55 65 1 -55 66 1 -55 67 1 -55 68 1 -55 69 1 -55 70 1 -55 71 1 -55 72 1 -55 73 1 -55 74 1 -55 75 1 -55 76 1 -55 77 1 -55 78 1 -55 79 1 -55 80 1 -55 81 1 -55 82 1 -55 83 1 -55 84 1 -55 85 1 -55 86 1 -55 87 1 -55 88 1 -55 89 1 -55 90 1 -55 91 1 -55 92 1 -55 93 1 -55 94 1 -55 95 1 -55 96 1 -55 97 1 -55 98 1 -55 99 1 -55 100 1 -56 0 1 -56 1 1 -56 2 1 -56 3 1 -56 4 1 -56 5 1 -56 6 1 -56 7 1 -56 8 1 -56 9 1 -56 10 1 -56 11 1 -56 12 1 -56 13 1 -56 14 1 -56 15 1 -56 16 1 -56 17 1 -56 18 1 -56 19 1 -56 20 1 -56 21 1 -56 22 1 -56 23 1 -56 24 1 -56 25 1 -56 26 1 -56 27 1 -56 28 1 -56 29 1 -56 30 1 -56 31 1 -56 32 1 -56 33 1 -56 34 1 -56 35 1 -56 36 1 -56 37 1 -56 38 1 -56 39 1 -56 40 1 -56 41 1 -56 42 1 -56 43 1 -56 44 1 -56 45 1 -56 46 1 -56 47 1 -56 48 1 -56 49 1 -56 50 1 -56 51 1 -56 52 1 -56 53 1 -56 54 1 -56 55 1 -56 56 1 -56 57 1 -56 58 1 -56 59 1 -56 60 1 -56 61 1 -56 62 1 -56 63 1 -56 64 1 -56 65 1 -56 66 1 -56 67 1 -56 68 1 -56 69 1 -56 70 1 -56 71 1 -56 72 1 -56 73 1 -56 74 1 -56 75 1 -56 76 1 -56 77 1 -56 78 1 -56 79 1 -56 80 1 -56 81 1 -56 82 1 -56 83 1 -56 84 1 -56 85 1 -56 86 1 -56 87 1 -56 88 1 -56 89 1 -56 90 1 -56 91 1 -56 92 1 -56 93 1 -56 94 1 -56 95 1 -56 96 1 -56 97 1 -56 98 1 -56 99 1 -56 100 1 -57 0 1 -57 1 1 -57 2 1 -57 3 1 -57 4 1 -57 5 1 -57 6 1 -57 7 1 -57 8 1 -57 9 1 -57 10 1 -57 11 1 -57 12 1 -57 13 1 -57 14 1 -57 15 1 -57 16 1 -57 17 1 -57 18 1 -57 19 1 -57 20 1 -57 21 1 -57 22 1 -57 23 1 -57 24 1 -57 25 1 -57 26 1 -57 27 1 -57 28 1 -57 29 1 -57 30 1 -57 31 1 -57 32 1 -57 33 1 -57 34 1 -57 35 1 -57 36 1 -57 37 1 -57 38 1 -57 39 1 -57 40 1 -57 41 1 -57 42 1 -57 43 1 -57 44 1 -57 45 1 -57 46 1 -57 47 1 -57 48 1 -57 49 1 -57 50 1 -57 51 1 -57 52 1 -57 53 1 -57 54 1 -57 55 1 -57 56 1 -57 57 1 -57 58 1 -57 59 1 -57 60 1 -57 61 1 -57 62 1 -57 63 1 -57 64 1 -57 65 1 -57 66 1 -57 67 1 -57 68 1 -57 69 1 -57 70 1 -57 71 1 -57 72 1 -57 73 1 -57 74 1 -57 75 1 -57 76 1 -57 77 1 -57 78 1 -57 79 1 -57 80 1 -57 81 1 -57 82 1 -57 83 1 -57 84 1 -57 85 1 -57 86 1 -57 87 1 -57 88 1 -57 89 1 -57 90 1 -57 91 1 -57 92 1 -57 93 1 -57 94 1 -57 95 1 -57 96 1 -57 97 1 -57 98 1 -57 99 1 -57 100 1 -58 0 1 -58 1 1 -58 2 1 -58 3 1 -58 4 1 -58 5 1 -58 6 1 -58 7 1 -58 8 1 -58 9 1 -58 10 1 -58 11 1 -58 12 1 -58 13 1 -58 14 1 -58 15 1 -58 16 1 -58 17 1 -58 18 1 -58 19 1 -58 20 1 -58 21 1 -58 22 1 -58 23 1 -58 24 1 -58 25 1 -58 26 1 -58 27 1 -58 28 1 -58 29 1 -58 30 1 -58 31 1 -58 32 1 -58 33 1 -58 34 1 -58 35 1 -58 36 1 -58 37 1 -58 38 1 -58 39 1 -58 40 1 -58 41 1 -58 42 1 -58 43 1 -58 44 1 -58 45 1 -58 46 1 -58 47 1 -58 48 1 -58 49 1 -58 50 1 -58 51 1 -58 52 1 -58 53 1 -58 54 1 -58 55 1 -58 56 1 -58 57 1 -58 58 1 -58 59 1 -58 60 1 -58 61 1 -58 62 1 -58 63 1 -58 64 1 -58 65 1 -58 66 1 -58 67 1 -58 68 1 -58 69 1 -58 70 1 -58 71 1 -58 72 1 -58 73 1 -58 74 1 -58 75 1 -58 76 1 -58 77 1 -58 78 1 -58 79 1 -58 80 1 -58 81 1 -58 82 1 -58 83 1 -58 84 1 -58 85 1 -58 86 1 -58 87 1 -58 88 1 -58 89 1 -58 90 1 -58 91 1 -58 92 1 -58 93 1 -58 94 1 -58 95 1 -58 96 1 -58 97 1 -58 98 1 -58 99 1 -58 100 1 -59 0 1 -59 1 1 -59 2 1 -59 3 1 -59 4 1 -59 5 1 -59 6 1 -59 7 1 -59 8 1 -59 9 1 -59 10 1 -59 11 1 -59 12 1 -59 13 1 -59 14 1 -59 15 1 -59 16 1 -59 17 1 -59 18 1 -59 19 1 -59 20 1 -59 21 1 -59 22 1 -59 23 1 -59 24 1 -59 25 1 -59 26 1 -59 27 1 -59 28 1 -59 29 1 -59 30 1 -59 31 1 -59 32 1 -59 33 1 -59 34 1 -59 35 1 -59 36 1 -59 37 1 -59 38 1 -59 39 1 -59 40 1 -59 41 1 -59 42 1 -59 43 1 -59 44 1 -59 45 1 -59 46 1 -59 47 1 -59 48 1 -59 49 1 -59 50 1 -59 51 1 -59 52 1 -59 53 1 -59 54 1 -59 55 1 -59 56 1 -59 57 1 -59 58 1 -59 59 1 -59 60 1 -59 61 1 -59 62 1 -59 63 1 -59 64 1 -59 65 1 -59 66 1 -59 67 1 -59 68 1 -59 69 1 -59 70 1 -59 71 1 -59 72 1 -59 73 1 -59 74 1 -59 75 1 -59 76 1 -59 77 1 -59 78 1 -59 79 1 -59 80 1 -59 81 1 -59 82 1 -59 83 1 -59 84 1 -59 85 1 -59 86 1 -59 87 1 -59 88 1 -59 89 1 -59 90 1 -59 91 1 -59 92 1 -59 93 1 -59 94 1 -59 95 1 -59 96 1 -59 97 1 -59 98 1 -59 99 1 -59 100 1 -60 0 1 -60 1 1 -60 2 1 -60 3 1 -60 4 1 -60 5 1 -60 6 1 -60 7 1 -60 8 1 -60 9 1 -60 10 1 -60 11 1 -60 12 1 -60 13 1 -60 14 1 -60 15 1 -60 16 1 -60 17 1 -60 18 1 -60 19 1 -60 20 1 -60 21 1 -60 22 1 -60 23 1 -60 24 1 -60 25 1 -60 26 1 -60 27 1 -60 28 1 -60 29 1 -60 30 1 -60 31 1 -60 32 1 -60 33 1 -60 34 1 -60 35 1 -60 36 1 -60 37 1 -60 38 1 -60 39 1 -60 40 1 -60 41 1 -60 42 1 -60 43 1 -60 44 1 -60 45 1 -60 46 1 -60 47 1 -60 48 1 -60 49 1 -60 50 1 -60 51 1 -60 52 1 -60 53 1 -60 54 1 -60 55 1 -60 56 1 -60 57 1 -60 58 1 -60 59 1 -60 60 1 -60 61 1 -60 62 1 -60 63 1 -60 64 1 -60 65 1 -60 66 1 -60 67 1 -60 68 1 -60 69 1 -60 70 1 -60 71 1 -60 72 1 -60 73 1 -60 74 1 -60 75 1 -60 76 1 -60 77 1 -60 78 1 -60 79 1 -60 80 1 -60 81 1 -60 82 1 -60 83 1 -60 84 1 -60 85 1 -60 86 1 -60 87 1 -60 88 1 -60 89 1 -60 90 1 -60 91 1 -60 92 1 -60 93 1 -60 94 1 -60 95 1 -60 96 1 -60 97 1 -60 98 1 -60 99 1 -60 100 1 -61 0 1 -61 1 1 -61 2 1 -61 3 1 -61 4 1 -61 5 1 -61 6 1 -61 7 1 -61 8 1 -61 9 1 -61 10 1 -61 11 1 -61 12 1 -61 13 1 -61 14 1 -61 15 1 -61 16 1 -61 17 1 -61 18 1 -61 19 1 -61 20 1 -61 21 1 -61 22 1 -61 23 1 -61 24 1 -61 25 1 -61 26 1 -61 27 1 -61 28 1 -61 29 1 -61 30 1 -61 31 1 -61 32 1 -61 33 1 -61 34 1 -61 35 1 -61 36 1 -61 37 1 -61 38 1 -61 39 1 -61 40 1 -61 41 1 -61 42 1 -61 43 1 -61 44 1 -61 45 1 -61 46 1 -61 47 1 -61 48 1 -61 49 1 -61 50 1 -61 51 1 -61 52 1 -61 53 1 -61 54 1 -61 55 1 -61 56 1 -61 57 1 -61 58 1 -61 59 1 -61 60 1 -61 61 1 -61 62 1 -61 63 1 -61 64 1 -61 65 1 -61 66 1 -61 67 1 -61 68 1 -61 69 1 -61 70 1 -61 71 1 -61 72 1 -61 73 1 -61 74 1 -61 75 1 -61 76 1 -61 77 1 -61 78 1 -61 79 1 -61 80 1 -61 81 1 -61 82 1 -61 83 1 -61 84 1 -61 85 1 -61 86 1 -61 87 1 -61 88 1 -61 89 1 -61 90 1 -61 91 1 -61 92 1 -61 93 1 -61 94 1 -61 95 1 -61 96 1 -61 97 1 -61 98 1 -61 99 1 -61 100 1 -62 0 1 -62 1 1 -62 2 1 -62 3 1 -62 4 1 -62 5 1 -62 6 1 -62 7 1 -62 8 1 -62 9 1 -62 10 1 -62 11 1 -62 12 1 -62 13 1 -62 14 1 -62 15 1 -62 16 1 -62 17 1 -62 18 1 -62 19 1 -62 20 1 -62 21 1 -62 22 1 -62 23 1 -62 24 1 -62 25 1 -62 26 1 -62 27 1 -62 28 1 -62 29 1 -62 30 1 -62 31 1 -62 32 1 -62 33 1 -62 34 1 -62 35 1 -62 36 1 -62 37 1 -62 38 1 -62 39 1 -62 40 1 -62 41 1 -62 42 1 -62 43 1 -62 44 1 -62 45 1 -62 46 1 -62 47 1 -62 48 1 -62 49 1 -62 50 1 -62 51 1 -62 52 1 -62 53 1 -62 54 1 -62 55 1 -62 56 1 -62 57 1 -62 58 1 -62 59 1 -62 60 1 -62 61 1 -62 62 1 -62 63 1 -62 64 1 -62 65 1 -62 66 1 -62 67 1 -62 68 1 -62 69 1 -62 70 1 -62 71 1 -62 72 1 -62 73 1 -62 74 1 -62 75 1 -62 76 1 -62 77 1 -62 78 1 -62 79 1 -62 80 1 -62 81 1 -62 82 1 -62 83 1 -62 84 1 -62 85 1 -62 86 1 -62 87 1 -62 88 1 -62 89 1 -62 90 1 -62 91 1 -62 92 1 -62 93 1 -62 94 1 -62 95 1 -62 96 1 -62 97 1 -62 98 1 -62 99 1 -62 100 1 -63 0 1 -63 1 1 -63 2 1 -63 3 1 -63 4 1 -63 5 1 -63 6 1 -63 7 1 -63 8 1 -63 9 1 -63 10 1 -63 11 1 -63 12 1 -63 13 1 -63 14 1 -63 15 1 -63 16 1 -63 17 1 -63 18 1 -63 19 1 -63 20 1 -63 21 1 -63 22 1 -63 23 1 -63 24 1 -63 25 1 -63 26 1 -63 27 1 -63 28 1 -63 29 1 -63 30 1 -63 31 1 -63 32 1 -63 33 1 -63 34 1 -63 35 1 -63 36 1 -63 37 1 -63 38 1 -63 39 1 -63 40 1 -63 41 1 -63 42 1 -63 43 1 -63 44 1 -63 45 1 -63 46 1 -63 47 1 -63 48 1 -63 49 1 -63 50 1 -63 51 1 -63 52 1 -63 53 1 -63 54 1 -63 55 1 -63 56 1 -63 57 1 -63 58 1 -63 59 1 -63 60 1 -63 61 1 -63 62 1 -63 63 1 -63 64 1 -63 65 1 -63 66 1 -63 67 1 -63 68 1 -63 69 1 -63 70 1 -63 71 1 -63 72 1 -63 73 1 -63 74 1 -63 75 1 -63 76 1 -63 77 1 -63 78 1 -63 79 1 -63 80 1 -63 81 1 -63 82 1 -63 83 1 -63 84 1 -63 85 1 -63 86 1 -63 87 1 -63 88 1 -63 89 1 -63 90 1 -63 91 1 -63 92 1 -63 93 1 -63 94 1 -63 95 1 -63 96 1 -63 97 1 -63 98 1 -63 99 1 -63 100 1 -64 0 1 -64 1 1 -64 2 1 -64 3 1 -64 4 1 -64 5 1 -64 6 1 -64 7 1 -64 8 1 -64 9 1 -64 10 1 -64 11 1 -64 12 1 -64 13 1 -64 14 1 -64 15 1 -64 16 1 -64 17 1 -64 18 1 -64 19 1 -64 20 1 -64 21 1 -64 22 1 -64 23 1 -64 24 1 -64 25 1 -64 26 1 -64 27 1 -64 28 1 -64 29 1 -64 30 1 -64 31 1 -64 32 1 -64 33 1 -64 34 1 -64 35 1 -64 36 1 -64 37 1 -64 38 1 -64 39 1 -64 40 1 -64 41 1 -64 42 1 -64 43 1 -64 44 1 -64 45 1 -64 46 1 -64 47 1 -64 48 1 -64 49 1 -64 50 1 -64 51 1 -64 52 1 -64 53 1 -64 54 1 -64 55 1 -64 56 1 -64 57 1 -64 58 1 -64 59 1 -64 60 1 -64 61 1 -64 62 1 -64 63 1 -64 64 1 -64 65 1 -64 66 1 -64 67 1 -64 68 1 -64 69 1 -64 70 1 -64 71 1 -64 72 1 -64 73 1 -64 74 1 -64 75 1 -64 76 1 -64 77 1 -64 78 1 -64 79 1 -64 80 1 -64 81 1 -64 82 1 -64 83 1 -64 84 1 -64 85 1 -64 86 1 -64 87 1 -64 88 1 -64 89 1 -64 90 1 -64 91 1 -64 92 1 -64 93 1 -64 94 1 -64 95 1 -64 96 1 -64 97 1 -64 98 1 -64 99 1 -64 100 1 -65 0 1 -65 1 1 -65 2 1 -65 3 1 -65 4 1 -65 5 1 -65 6 1 -65 7 1 -65 8 1 -65 9 1 -65 10 1 -65 11 1 -65 12 1 -65 13 1 -65 14 1 -65 15 1 -65 16 1 -65 17 1 -65 18 1 -65 19 1 -65 20 1 -65 21 1 -65 22 1 -65 23 1 -65 24 1 -65 25 1 -65 26 1 -65 27 1 -65 28 1 -65 29 1 -65 30 1 -65 31 1 -65 32 1 -65 33 1 -65 34 1 -65 35 1 -65 36 1 -65 37 1 -65 38 1 -65 39 1 -65 40 1 -65 41 1 -65 42 1 -65 43 1 -65 44 1 -65 45 1 -65 46 1 -65 47 1 -65 48 1 -65 49 1 -65 50 1 -65 51 1 -65 52 1 -65 53 1 -65 54 1 -65 55 1 -65 56 1 -65 57 1 -65 58 1 -65 59 1 -65 60 1 -65 61 1 -65 62 1 -65 63 1 -65 64 1 -65 65 1 -65 66 1 -65 67 1 -65 68 1 -65 69 1 -65 70 1 -65 71 1 -65 72 1 -65 73 1 -65 74 1 -65 75 1 -65 76 1 -65 77 1 -65 78 1 -65 79 1 -65 80 1 -65 81 1 -65 82 1 -65 83 1 -65 84 1 -65 85 1 -65 86 1 -65 87 1 -65 88 1 -65 89 1 -65 90 1 -65 91 1 -65 92 1 -65 93 1 -65 94 1 -65 95 1 -65 96 1 -65 97 1 -65 98 1 -65 99 1 -65 100 1 -66 0 1 -66 1 1 -66 2 1 -66 3 1 -66 4 1 -66 5 1 -66 6 1 -66 7 1 -66 8 1 -66 9 1 -66 10 1 -66 11 1 -66 12 1 -66 13 1 -66 14 1 -66 15 1 -66 16 1 -66 17 1 -66 18 1 -66 19 1 -66 20 1 -66 21 1 -66 22 1 -66 23 1 -66 24 1 -66 25 1 -66 26 1 -66 27 1 -66 28 1 -66 29 1 -66 30 1 -66 31 1 -66 32 1 -66 33 1 -66 34 1 -66 35 1 -66 36 1 -66 37 1 -66 38 1 -66 39 1 -66 40 1 -66 41 1 -66 42 1 -66 43 1 -66 44 1 -66 45 1 -66 46 1 -66 47 1 -66 48 1 -66 49 1 -66 50 1 -66 51 1 -66 52 1 -66 53 1 -66 54 1 -66 55 1 -66 56 1 -66 57 1 -66 58 1 -66 59 1 -66 60 1 -66 61 1 -66 62 1 -66 63 1 -66 64 1 -66 65 1 -66 66 1 -66 67 1 -66 68 1 -66 69 1 -66 70 1 -66 71 1 -66 72 1 -66 73 1 -66 74 1 -66 75 1 -66 76 1 -66 77 1 -66 78 1 -66 79 1 -66 80 1 -66 81 1 -66 82 1 -66 83 1 -66 84 1 -66 85 1 -66 86 1 -66 87 1 -66 88 1 -66 89 1 -66 90 1 -66 91 1 -66 92 1 -66 93 1 -66 94 1 -66 95 1 -66 96 1 -66 97 1 -66 98 1 -66 99 1 -66 100 1 -67 0 1 -67 1 1 -67 2 1 -67 3 1 -67 4 1 -67 5 1 -67 6 1 -67 7 1 -67 8 1 -67 9 1 -67 10 1 -67 11 1 -67 12 1 -67 13 1 -67 14 1 -67 15 1 -67 16 1 -67 17 1 -67 18 1 -67 19 1 -67 20 1 -67 21 1 -67 22 1 -67 23 1 -67 24 1 -67 25 1 -67 26 1 -67 27 1 -67 28 1 -67 29 1 -67 30 1 -67 31 1 -67 32 1 -67 33 1 -67 34 1 -67 35 1 -67 36 1 -67 37 1 -67 38 1 -67 39 1 -67 40 1 -67 41 1 -67 42 1 -67 43 1 -67 44 1 -67 45 1 -67 46 1 -67 47 1 -67 48 1 -67 49 1 -67 50 1 -67 51 1 -67 52 1 -67 53 1 -67 54 1 -67 55 1 -67 56 1 -67 57 1 -67 58 1 -67 59 1 -67 60 1 -67 61 1 -67 62 1 -67 63 1 -67 64 1 -67 65 1 -67 66 1 -67 67 1 -67 68 1 -67 69 1 -67 70 1 -67 71 1 -67 72 1 -67 73 1 -67 74 1 -67 75 1 -67 76 1 -67 77 1 -67 78 1 -67 79 1 -67 80 1 -67 81 1 -67 82 1 -67 83 1 -67 84 1 -67 85 1 -67 86 1 -67 87 1 -67 88 1 -67 89 1 -67 90 1 -67 91 1 -67 92 1 -67 93 1 -67 94 1 -67 95 1 -67 96 1 -67 97 1 -67 98 1 -67 99 1 -67 100 1 -68 0 1 -68 1 1 -68 2 1 -68 3 1 -68 4 1 -68 5 1 -68 6 1 -68 7 1 -68 8 1 -68 9 1 -68 10 1 -68 11 1 -68 12 1 -68 13 1 -68 14 1 -68 15 1 -68 16 1 -68 17 1 -68 18 1 -68 19 1 -68 20 1 -68 21 1 -68 22 1 -68 23 1 -68 24 1 -68 25 1 -68 26 1 -68 27 1 -68 28 1 -68 29 1 -68 30 1 -68 31 1 -68 32 1 -68 33 1 -68 34 1 -68 35 1 -68 36 1 -68 37 1 -68 38 1 -68 39 1 -68 40 1 -68 41 1 -68 42 1 -68 43 1 -68 44 1 -68 45 1 -68 46 1 -68 47 1 -68 48 1 -68 49 1 -68 50 1 -68 51 1 -68 52 1 -68 53 1 -68 54 1 -68 55 1 -68 56 1 -68 57 1 -68 58 1 -68 59 1 -68 60 1 -68 61 1 -68 62 1 -68 63 1 -68 64 1 -68 65 1 -68 66 1 -68 67 1 -68 68 1 -68 69 1 -68 70 1 -68 71 1 -68 72 1 -68 73 1 -68 74 1 -68 75 1 -68 76 1 -68 77 1 -68 78 1 -68 79 1 -68 80 1 -68 81 1 -68 82 1 -68 83 1 -68 84 1 -68 85 1 -68 86 1 -68 87 1 -68 88 1 -68 89 1 -68 90 1 -68 91 1 -68 92 1 -68 93 1 -68 94 1 -68 95 1 -68 96 1 -68 97 1 -68 98 1 -68 99 1 -68 100 1 -69 0 1 -69 1 1 -69 2 1 -69 3 1 -69 4 1 -69 5 1 -69 6 1 -69 7 1 -69 8 1 -69 9 1 -69 10 1 -69 11 1 -69 12 1 -69 13 1 -69 14 1 -69 15 1 -69 16 1 -69 17 1 -69 18 1 -69 19 1 -69 20 1 -69 21 1 -69 22 1 -69 23 1 -69 24 1 -69 25 1 -69 26 1 -69 27 1 -69 28 1 -69 29 1 -69 30 1 -69 31 1 -69 32 1 -69 33 1 -69 34 1 -69 35 1 -69 36 1 -69 37 1 -69 38 1 -69 39 1 -69 40 1 -69 41 1 -69 42 1 -69 43 1 -69 44 1 -69 45 1 -69 46 1 -69 47 1 -69 48 1 -69 49 1 -69 50 1 -69 51 1 -69 52 1 -69 53 1 -69 54 1 -69 55 1 -69 56 1 -69 57 1 -69 58 1 -69 59 1 -69 60 1 -69 61 1 -69 62 1 -69 63 1 -69 64 1 -69 65 1 -69 66 1 -69 67 1 -69 68 1 -69 69 1 -69 70 1 -69 71 1 -69 72 1 -69 73 1 -69 74 1 -69 75 1 -69 76 1 -69 77 1 -69 78 1 -69 79 1 -69 80 1 -69 81 1 -69 82 1 -69 83 1 -69 84 1 -69 85 1 -69 86 1 -69 87 1 -69 88 1 -69 89 1 -69 90 1 -69 91 1 -69 92 1 -69 93 1 -69 94 1 -69 95 1 -69 96 1 -69 97 1 -69 98 1 -69 99 1 -69 100 1 -70 0 1 -70 1 1 -70 2 1 -70 3 1 -70 4 1 -70 5 1 -70 6 1 -70 7 1 -70 8 1 -70 9 1 -70 10 1 -70 11 1 -70 12 1 -70 13 1 -70 14 1 -70 15 1 -70 16 1 -70 17 1 -70 18 1 -70 19 1 -70 20 1 -70 21 1 -70 22 1 -70 23 1 -70 24 1 -70 25 1 -70 26 1 -70 27 1 -70 28 1 -70 29 1 -70 30 1 -70 31 1 -70 32 1 -70 33 1 -70 34 1 -70 35 1 -70 36 1 -70 37 1 -70 38 1 -70 39 1 -70 40 1 -70 41 1 -70 42 1 -70 43 1 -70 44 1 -70 45 1 -70 46 1 -70 47 1 -70 48 1 -70 49 1 -70 50 1 -70 51 1 -70 52 1 -70 53 1 -70 54 1 -70 55 1 -70 56 1 -70 57 1 -70 58 1 -70 59 1 -70 60 1 -70 61 1 -70 62 1 -70 63 1 -70 64 1 -70 65 1 -70 66 1 -70 67 1 -70 68 1 -70 69 1 -70 70 1 -70 71 1 -70 72 1 -70 73 1 -70 74 1 -70 75 1 -70 76 1 -70 77 1 -70 78 1 -70 79 1 -70 80 1 -70 81 1 -70 82 1 -70 83 1 -70 84 1 -70 85 1 -70 86 1 -70 87 1 -70 88 1 -70 89 1 -70 90 1 -70 91 1 -70 92 1 -70 93 1 -70 94 1 -70 95 1 -70 96 1 -70 97 1 -70 98 1 -70 99 1 -70 100 1 -71 0 1 -71 1 1 -71 2 1 -71 3 1 -71 4 1 -71 5 1 -71 6 1 -71 7 1 -71 8 1 -71 9 1 -71 10 1 -71 11 1 -71 12 1 -71 13 1 -71 14 1 -71 15 1 -71 16 1 -71 17 1 -71 18 1 -71 19 1 -71 20 1 -71 21 1 -71 22 1 -71 23 1 -71 24 1 -71 25 1 -71 26 1 -71 27 1 -71 28 1 -71 29 1 -71 30 1 -71 31 1 -71 32 1 -71 33 1 -71 34 1 -71 35 1 -71 36 1 -71 37 1 -71 38 1 -71 39 1 -71 40 1 -71 41 1 -71 42 1 -71 43 1 -71 44 1 -71 45 1 -71 46 1 -71 47 1 -71 48 1 -71 49 1 -71 50 1 -71 51 1 -71 52 1 -71 53 1 -71 54 1 -71 55 1 -71 56 1 -71 57 1 -71 58 1 -71 59 1 -71 60 1 -71 61 1 -71 62 1 -71 63 1 -71 64 1 -71 65 1 -71 66 1 -71 67 1 -71 68 1 -71 69 1 -71 70 1 -71 71 1 -71 72 1 -71 73 1 -71 74 1 -71 75 1 -71 76 1 -71 77 1 -71 78 1 -71 79 1 -71 80 1 -71 81 1 -71 82 1 -71 83 1 -71 84 1 -71 85 1 -71 86 1 -71 87 1 -71 88 1 -71 89 1 -71 90 1 -71 91 1 -71 92 1 -71 93 1 -71 94 1 -71 95 1 -71 96 1 -71 97 1 -71 98 1 -71 99 1 -71 100 1 -72 0 1 -72 1 1 -72 2 1 -72 3 1 -72 4 1 -72 5 1 -72 6 1 -72 7 1 -72 8 1 -72 9 1 -72 10 1 -72 11 1 -72 12 1 -72 13 1 -72 14 1 -72 15 1 -72 16 1 -72 17 1 -72 18 1 -72 19 1 -72 20 1 -72 21 1 -72 22 1 -72 23 1 -72 24 1 -72 25 1 -72 26 1 -72 27 1 -72 28 1 -72 29 1 -72 30 1 -72 31 1 -72 32 1 -72 33 1 -72 34 1 -72 35 1 -72 36 1 -72 37 1 -72 38 1 -72 39 1 -72 40 1 -72 41 1 -72 42 1 -72 43 1 -72 44 1 -72 45 1 -72 46 1 -72 47 1 -72 48 1 -72 49 1 -72 50 1 -72 51 1 -72 52 1 -72 53 1 -72 54 1 -72 55 1 -72 56 1 -72 57 1 -72 58 1 -72 59 1 -72 60 1 -72 61 1 -72 62 1 -72 63 1 -72 64 1 -72 65 1 -72 66 1 -72 67 1 -72 68 1 -72 69 1 -72 70 1 -72 71 1 -72 72 1 -72 73 1 -72 74 1 -72 75 1 -72 76 1 -72 77 1 -72 78 1 -72 79 1 -72 80 1 -72 81 1 -72 82 1 -72 83 1 -72 84 1 -72 85 1 -72 86 1 -72 87 1 -72 88 1 -72 89 1 -72 90 1 -72 91 1 -72 92 1 -72 93 1 -72 94 1 -72 95 1 -72 96 1 -72 97 1 -72 98 1 -72 99 1 -72 100 1 -73 0 1 -73 1 1 -73 2 1 -73 3 1 -73 4 1 -73 5 1 -73 6 1 -73 7 1 -73 8 1 -73 9 1 -73 10 1 -73 11 1 -73 12 1 -73 13 1 -73 14 1 -73 15 1 -73 16 1 -73 17 1 -73 18 1 -73 19 1 -73 20 1 -73 21 1 -73 22 1 -73 23 1 -73 24 1 -73 25 1 -73 26 1 -73 27 1 -73 28 1 -73 29 1 -73 30 1 -73 31 1 -73 32 1 -73 33 1 -73 34 1 -73 35 1 -73 36 1 -73 37 1 -73 38 1 -73 39 1 -73 40 1 -73 41 1 -73 42 1 -73 43 1 -73 44 1 -73 45 1 -73 46 1 -73 47 1 -73 48 1 -73 49 1 -73 50 1 -73 51 1 -73 52 1 -73 53 1 -73 54 1 -73 55 1 -73 56 1 -73 57 1 -73 58 1 -73 59 1 -73 60 1 -73 61 1 -73 62 1 -73 63 1 -73 64 1 -73 65 1 -73 66 1 -73 67 1 -73 68 1 -73 69 1 -73 70 1 -73 71 1 -73 72 1 -73 73 1 -73 74 1 -73 75 1 -73 76 1 -73 77 1 -73 78 1 -73 79 1 -73 80 1 -73 81 1 -73 82 1 -73 83 1 -73 84 1 -73 85 1 -73 86 1 -73 87 1 -73 88 1 -73 89 1 -73 90 1 -73 91 1 -73 92 1 -73 93 1 -73 94 1 -73 95 1 -73 96 1 -73 97 1 -73 98 1 -73 99 1 -73 100 1 -74 0 1 -74 1 1 -74 2 1 -74 3 1 -74 4 1 -74 5 1 -74 6 1 -74 7 1 -74 8 1 -74 9 1 -74 10 1 -74 11 1 -74 12 1 -74 13 1 -74 14 1 -74 15 1 -74 16 1 -74 17 1 -74 18 1 -74 19 1 -74 20 1 -74 21 1 -74 22 1 -74 23 1 -74 24 1 -74 25 1 -74 26 1 -74 27 1 -74 28 1 -74 29 1 -74 30 1 -74 31 1 -74 32 1 -74 33 1 -74 34 1 -74 35 1 -74 36 1 -74 37 1 -74 38 1 -74 39 1 -74 40 1 -74 41 1 -74 42 1 -74 43 1 -74 44 1 -74 45 1 -74 46 1 -74 47 1 -74 48 1 -74 49 1 -74 50 1 -74 51 1 -74 52 1 -74 53 1 -74 54 1 -74 55 1 -74 56 1 -74 57 1 -74 58 1 -74 59 1 -74 60 1 -74 61 1 -74 62 1 -74 63 1 -74 64 1 -74 65 1 -74 66 1 -74 67 1 -74 68 1 -74 69 1 -74 70 1 -74 71 1 -74 72 1 -74 73 1 -74 74 1 -74 75 1 -74 76 1 -74 77 1 -74 78 1 -74 79 1 -74 80 1 -74 81 1 -74 82 1 -74 83 1 -74 84 1 -74 85 1 -74 86 1 -74 87 1 -74 88 1 -74 89 1 -74 90 1 -74 91 1 -74 92 1 -74 93 1 -74 94 1 -74 95 1 -74 96 1 -74 97 1 -74 98 1 -74 99 1 -74 100 1 -75 0 1 -75 1 1 -75 2 1 -75 3 1 -75 4 1 -75 5 1 -75 6 1 -75 7 1 -75 8 1 -75 9 1 -75 10 1 -75 11 1 -75 12 1 -75 13 1 -75 14 1 -75 15 1 -75 16 1 -75 17 1 -75 18 1 -75 19 1 -75 20 1 -75 21 1 -75 22 1 -75 23 1 -75 24 1 -75 25 1 -75 26 1 -75 27 1 -75 28 1 -75 29 1 -75 30 1 -75 31 1 -75 32 1 -75 33 1 -75 34 1 -75 35 1 -75 36 1 -75 37 1 -75 38 1 -75 39 1 -75 40 1 -75 41 1 -75 42 1 -75 43 1 -75 44 1 -75 45 1 -75 46 1 -75 47 1 -75 48 1 -75 49 1 -75 50 1 -75 51 1 -75 52 1 -75 53 1 -75 54 1 -75 55 1 -75 56 1 -75 57 1 -75 58 1 -75 59 1 -75 60 1 -75 61 1 -75 62 1 -75 63 1 -75 64 1 -75 65 1 -75 66 1 -75 67 1 -75 68 1 -75 69 1 -75 70 1 -75 71 1 -75 72 1 -75 73 1 -75 74 1 -75 75 1 -75 76 1 -75 77 1 -75 78 1 -75 79 1 -75 80 1 -75 81 1 -75 82 1 -75 83 1 -75 84 1 -75 85 1 -75 86 1 -75 87 1 -75 88 1 -75 89 1 -75 90 1 -75 91 1 -75 92 1 -75 93 1 -75 94 1 -75 95 1 -75 96 1 -75 97 1 -75 98 1 -75 99 1 -75 100 1 -76 0 1 -76 1 1 -76 2 1 -76 3 1 -76 4 1 -76 5 1 -76 6 1 -76 7 1 -76 8 1 -76 9 1 -76 10 1 -76 11 1 -76 12 1 -76 13 1 -76 14 1 -76 15 1 -76 16 1 -76 17 1 -76 18 1 -76 19 1 -76 20 1 -76 21 1 -76 22 1 -76 23 1 -76 24 1 -76 25 1 -76 26 1 -76 27 1 -76 28 1 -76 29 1 -76 30 1 -76 31 1 -76 32 1 -76 33 1 -76 34 1 -76 35 1 -76 36 1 -76 37 1 -76 38 1 -76 39 1 -76 40 1 -76 41 1 -76 42 1 -76 43 1 -76 44 1 -76 45 1 -76 46 1 -76 47 1 -76 48 1 -76 49 1 -76 50 1 -76 51 1 -76 52 1 -76 53 1 -76 54 1 -76 55 1 -76 56 1 -76 57 1 -76 58 1 -76 59 1 -76 60 1 -76 61 1 -76 62 1 -76 63 1 -76 64 1 -76 65 1 -76 66 1 -76 67 1 -76 68 1 -76 69 1 -76 70 1 -76 71 1 -76 72 1 -76 73 1 -76 74 1 -76 75 1 -76 76 1 -76 77 1 -76 78 1 -76 79 1 -76 80 1 -76 81 1 -76 82 1 -76 83 1 -76 84 1 -76 85 1 -76 86 1 -76 87 1 -76 88 1 -76 89 1 -76 90 1 -76 91 1 -76 92 1 -76 93 1 -76 94 1 -76 95 1 -76 96 1 -76 97 1 -76 98 1 -76 99 1 -76 100 1 -77 0 1 -77 1 1 -77 2 1 -77 3 1 -77 4 1 -77 5 1 -77 6 1 -77 7 1 -77 8 1 -77 9 1 -77 10 1 -77 11 1 -77 12 1 -77 13 1 -77 14 1 -77 15 1 -77 16 1 -77 17 1 -77 18 1 -77 19 1 -77 20 1 -77 21 1 -77 22 1 -77 23 1 -77 24 1 -77 25 1 -77 26 1 -77 27 1 -77 28 1 -77 29 1 -77 30 1 -77 31 1 -77 32 1 -77 33 1 -77 34 1 -77 35 1 -77 36 1 -77 37 1 -77 38 1 -77 39 1 -77 40 1 -77 41 1 -77 42 1 -77 43 1 -77 44 1 -77 45 1 -77 46 1 -77 47 1 -77 48 1 -77 49 1 -77 50 1 -77 51 1 -77 52 1 -77 53 1 -77 54 1 -77 55 1 -77 56 1 -77 57 1 -77 58 1 -77 59 1 -77 60 1 -77 61 1 -77 62 1 -77 63 1 -77 64 1 -77 65 1 -77 66 1 -77 67 1 -77 68 1 -77 69 1 -77 70 1 -77 71 1 -77 72 1 -77 73 1 -77 74 1 -77 75 1 -77 76 1 -77 77 1 -77 78 1 -77 79 1 -77 80 1 -77 81 1 -77 82 1 -77 83 1 -77 84 1 -77 85 1 -77 86 1 -77 87 1 -77 88 1 -77 89 1 -77 90 1 -77 91 1 -77 92 1 -77 93 1 -77 94 1 -77 95 1 -77 96 1 -77 97 1 -77 98 1 -77 99 1 -77 100 1 -78 0 1 -78 1 1 -78 2 1 -78 3 1 -78 4 1 -78 5 1 -78 6 1 -78 7 1 -78 8 1 -78 9 1 -78 10 1 -78 11 1 -78 12 1 -78 13 1 -78 14 1 -78 15 1 -78 16 1 -78 17 1 -78 18 1 -78 19 1 -78 20 1 -78 21 1 -78 22 1 -78 23 1 -78 24 1 -78 25 1 -78 26 1 -78 27 1 -78 28 1 -78 29 1 -78 30 1 -78 31 1 -78 32 1 -78 33 1 -78 34 1 -78 35 1 -78 36 1 -78 37 1 -78 38 1 -78 39 1 -78 40 1 -78 41 1 -78 42 1 -78 43 1 -78 44 1 -78 45 1 -78 46 1 -78 47 1 -78 48 1 -78 49 1 -78 50 1 -78 51 1 -78 52 1 -78 53 1 -78 54 1 -78 55 1 -78 56 1 -78 57 1 -78 58 1 -78 59 1 -78 60 1 -78 61 1 -78 62 1 -78 63 1 -78 64 1 -78 65 1 -78 66 1 -78 67 1 -78 68 1 -78 69 1 -78 70 1 -78 71 1 -78 72 1 -78 73 1 -78 74 1 -78 75 1 -78 76 1 -78 77 1 -78 78 1 -78 79 1 -78 80 1 -78 81 1 -78 82 1 -78 83 1 -78 84 1 -78 85 1 -78 86 1 -78 87 1 -78 88 1 -78 89 1 -78 90 1 -78 91 1 -78 92 1 -78 93 1 -78 94 1 -78 95 1 -78 96 1 -78 97 1 -78 98 1 -78 99 1 -78 100 1 -79 0 1 -79 1 1 -79 2 1 -79 3 1 -79 4 1 -79 5 1 -79 6 1 -79 7 1 -79 8 1 -79 9 1 -79 10 1 -79 11 1 -79 12 1 -79 13 1 -79 14 1 -79 15 1 -79 16 1 -79 17 1 -79 18 1 -79 19 1 -79 20 1 -79 21 1 -79 22 1 -79 23 1 -79 24 1 -79 25 1 -79 26 1 -79 27 1 -79 28 1 -79 29 1 -79 30 1 -79 31 1 -79 32 1 -79 33 1 -79 34 1 -79 35 1 -79 36 1 -79 37 1 -79 38 1 -79 39 1 -79 40 1 -79 41 1 -79 42 1 -79 43 1 -79 44 1 -79 45 1 -79 46 1 -79 47 1 -79 48 1 -79 49 1 -79 50 1 -79 51 1 -79 52 1 -79 53 1 -79 54 1 -79 55 1 -79 56 1 -79 57 1 -79 58 1 -79 59 1 -79 60 1 -79 61 1 -79 62 1 -79 63 1 -79 64 1 -79 65 1 -79 66 1 -79 67 1 -79 68 1 -79 69 1 -79 70 1 -79 71 1 -79 72 1 -79 73 1 -79 74 1 -79 75 1 -79 76 1 -79 77 1 -79 78 1 -79 79 1 -79 80 1 -79 81 1 -79 82 1 -79 83 1 -79 84 1 -79 85 1 -79 86 1 -79 87 1 -79 88 1 -79 89 1 -79 90 1 -79 91 1 -79 92 1 -79 93 1 -79 94 1 -79 95 1 -79 96 1 -79 97 1 -79 98 1 -79 99 1 -79 100 1 -80 0 1 -80 1 1 -80 2 1 -80 3 1 -80 4 1 -80 5 1 -80 6 1 -80 7 1 -80 8 1 -80 9 1 -80 10 1 -80 11 1 -80 12 1 -80 13 1 -80 14 1 -80 15 1 -80 16 1 -80 17 1 -80 18 1 -80 19 1 -80 20 1 -80 21 1 -80 22 1 -80 23 1 -80 24 1 -80 25 1 -80 26 1 -80 27 1 -80 28 1 -80 29 1 -80 30 1 -80 31 1 -80 32 1 -80 33 1 -80 34 1 -80 35 1 -80 36 1 -80 37 1 -80 38 1 -80 39 1 -80 40 1 -80 41 1 -80 42 1 -80 43 1 -80 44 1 -80 45 1 -80 46 1 -80 47 1 -80 48 1 -80 49 1 -80 50 1 -80 51 1 -80 52 1 -80 53 1 -80 54 1 -80 55 1 -80 56 1 -80 57 1 -80 58 1 -80 59 1 -80 60 1 -80 61 1 -80 62 1 -80 63 1 -80 64 1 -80 65 1 -80 66 1 -80 67 1 -80 68 1 -80 69 1 -80 70 1 -80 71 1 -80 72 1 -80 73 1 -80 74 1 -80 75 1 -80 76 1 -80 77 1 -80 78 1 -80 79 1 -80 80 1 -80 81 1 -80 82 1 -80 83 1 -80 84 1 -80 85 1 -80 86 1 -80 87 1 -80 88 1 -80 89 1 -80 90 1 -80 91 1 -80 92 1 -80 93 1 -80 94 1 -80 95 1 -80 96 1 -80 97 1 -80 98 1 -80 99 1 -80 100 1 -81 0 1 -81 1 1 -81 2 1 -81 3 1 -81 4 1 -81 5 1 -81 6 1 -81 7 1 -81 8 1 -81 9 1 -81 10 1 -81 11 1 -81 12 1 -81 13 1 -81 14 1 -81 15 1 -81 16 1 -81 17 1 -81 18 1 -81 19 1 -81 20 1 -81 21 1 -81 22 1 -81 23 1 -81 24 1 -81 25 1 -81 26 1 -81 27 1 -81 28 1 -81 29 1 -81 30 1 -81 31 1 -81 32 1 -81 33 1 -81 34 1 -81 35 1 -81 36 1 -81 37 1 -81 38 1 -81 39 1 -81 40 1 -81 41 1 -81 42 1 -81 43 1 -81 44 1 -81 45 1 -81 46 1 -81 47 1 -81 48 1 -81 49 1 -81 50 1 -81 51 1 -81 52 1 -81 53 1 -81 54 1 -81 55 1 -81 56 1 -81 57 1 -81 58 1 -81 59 1 -81 60 1 -81 61 1 -81 62 1 -81 63 1 -81 64 1 -81 65 1 -81 66 1 -81 67 1 -81 68 1 -81 69 1 -81 70 1 -81 71 1 -81 72 1 -81 73 1 -81 74 1 -81 75 1 -81 76 1 -81 77 1 -81 78 1 -81 79 1 -81 80 1 -81 81 1 -81 82 1 -81 83 1 -81 84 1 -81 85 1 -81 86 1 -81 87 1 -81 88 1 -81 89 1 -81 90 1 -81 91 1 -81 92 1 -81 93 1 -81 94 1 -81 95 1 -81 96 1 -81 97 1 -81 98 1 -81 99 1 -81 100 1 -82 0 1 -82 1 1 -82 2 1 -82 3 1 -82 4 1 -82 5 1 -82 6 1 -82 7 1 -82 8 1 -82 9 1 -82 10 1 -82 11 1 -82 12 1 -82 13 1 -82 14 1 -82 15 1 -82 16 1 -82 17 1 -82 18 1 -82 19 1 -82 20 1 -82 21 1 -82 22 1 -82 23 1 -82 24 1 -82 25 1 -82 26 1 -82 27 1 -82 28 1 -82 29 1 -82 30 1 -82 31 1 -82 32 1 -82 33 1 -82 34 1 -82 35 1 -82 36 1 -82 37 1 -82 38 1 -82 39 1 -82 40 1 -82 41 1 -82 42 1 -82 43 1 -82 44 1 -82 45 1 -82 46 1 -82 47 1 -82 48 1 -82 49 1 -82 50 1 -82 51 1 -82 52 1 -82 53 1 -82 54 1 -82 55 1 -82 56 1 -82 57 1 -82 58 1 -82 59 1 -82 60 1 -82 61 1 -82 62 1 -82 63 1 -82 64 1 -82 65 1 -82 66 1 -82 67 1 -82 68 1 -82 69 1 -82 70 1 -82 71 1 -82 72 1 -82 73 1 -82 74 1 -82 75 1 -82 76 1 -82 77 1 -82 78 1 -82 79 1 -82 80 1 -82 81 1 -82 82 1 -82 83 1 -82 84 1 -82 85 1 -82 86 1 -82 87 1 -82 88 1 -82 89 1 -82 90 1 -82 91 1 -82 92 1 -82 93 1 -82 94 1 -82 95 1 -82 96 1 -82 97 1 -82 98 1 -82 99 1 -82 100 1 -83 0 1 -83 1 1 -83 2 1 -83 3 1 -83 4 1 -83 5 1 -83 6 1 -83 7 1 -83 8 1 -83 9 1 -83 10 1 -83 11 1 -83 12 1 -83 13 1 -83 14 1 -83 15 1 -83 16 1 -83 17 1 -83 18 1 -83 19 1 -83 20 1 -83 21 1 -83 22 1 -83 23 1 -83 24 1 -83 25 1 -83 26 1 -83 27 1 -83 28 1 -83 29 1 -83 30 1 -83 31 1 -83 32 1 -83 33 1 -83 34 1 -83 35 1 -83 36 1 -83 37 1 -83 38 1 -83 39 1 -83 40 1 -83 41 1 -83 42 1 -83 43 1 -83 44 1 -83 45 1 -83 46 1 -83 47 1 -83 48 1 -83 49 1 -83 50 1 -83 51 1 -83 52 1 -83 53 1 -83 54 1 -83 55 1 -83 56 1 -83 57 1 -83 58 1 -83 59 1 -83 60 1 -83 61 1 -83 62 1 -83 63 1 -83 64 1 -83 65 1 -83 66 1 -83 67 1 -83 68 1 -83 69 1 -83 70 1 -83 71 1 -83 72 1 -83 73 1 -83 74 1 -83 75 1 -83 76 1 -83 77 1 -83 78 1 -83 79 1 -83 80 1 -83 81 1 -83 82 1 -83 83 1 -83 84 1 -83 85 1 -83 86 1 -83 87 1 -83 88 1 -83 89 1 -83 90 1 -83 91 1 -83 92 1 -83 93 1 -83 94 1 -83 95 1 -83 96 1 -83 97 1 -83 98 1 -83 99 1 -83 100 1 -84 0 1 -84 1 1 -84 2 1 -84 3 1 -84 4 1 -84 5 1 -84 6 1 -84 7 1 -84 8 1 -84 9 1 -84 10 1 -84 11 1 -84 12 1 -84 13 1 -84 14 1 -84 15 1 -84 16 1 -84 17 1 -84 18 1 -84 19 1 -84 20 1 -84 21 1 -84 22 1 -84 23 1 -84 24 1 -84 25 1 -84 26 1 -84 27 1 -84 28 1 -84 29 1 -84 30 1 -84 31 1 -84 32 1 -84 33 1 -84 34 1 -84 35 1 -84 36 1 -84 37 1 -84 38 1 -84 39 1 -84 40 1 -84 41 1 -84 42 1 -84 43 1 -84 44 1 -84 45 1 -84 46 1 -84 47 1 -84 48 1 -84 49 1 -84 50 1 -84 51 1 -84 52 1 -84 53 1 -84 54 1 -84 55 1 -84 56 1 -84 57 1 -84 58 1 -84 59 1 -84 60 1 -84 61 1 -84 62 1 -84 63 1 -84 64 1 -84 65 1 -84 66 1 -84 67 1 -84 68 1 -84 69 1 -84 70 1 -84 71 1 -84 72 1 -84 73 1 -84 74 1 -84 75 1 -84 76 1 -84 77 1 -84 78 1 -84 79 1 -84 80 1 -84 81 1 -84 82 1 -84 83 1 -84 84 1 -84 85 1 -84 86 1 -84 87 1 -84 88 1 -84 89 1 -84 90 1 -84 91 1 -84 92 1 -84 93 1 -84 94 1 -84 95 1 -84 96 1 -84 97 1 -84 98 1 -84 99 1 -84 100 1 -85 0 1 -85 1 1 -85 2 1 -85 3 1 -85 4 1 -85 5 1 -85 6 1 -85 7 1 -85 8 1 -85 9 1 -85 10 1 -85 11 1 -85 12 1 -85 13 1 -85 14 1 -85 15 1 -85 16 1 -85 17 1 -85 18 1 -85 19 1 -85 20 1 -85 21 1 -85 22 1 -85 23 1 -85 24 1 -85 25 1 -85 26 1 -85 27 1 -85 28 1 -85 29 1 -85 30 1 -85 31 1 -85 32 1 -85 33 1 -85 34 1 -85 35 1 -85 36 1 -85 37 1 -85 38 1 -85 39 1 -85 40 1 -85 41 1 -85 42 1 -85 43 1 -85 44 1 -85 45 1 -85 46 1 -85 47 1 -85 48 1 -85 49 1 -85 50 1 -85 51 1 -85 52 1 -85 53 1 -85 54 1 -85 55 1 -85 56 1 -85 57 1 -85 58 1 -85 59 1 -85 60 1 -85 61 1 -85 62 1 -85 63 1 -85 64 1 -85 65 1 -85 66 1 -85 67 1 -85 68 1 -85 69 1 -85 70 1 -85 71 1 -85 72 1 -85 73 1 -85 74 1 -85 75 1 -85 76 1 -85 77 1 -85 78 1 -85 79 1 -85 80 1 -85 81 1 -85 82 1 -85 83 1 -85 84 1 -85 85 1 -85 86 1 -85 87 1 -85 88 1 -85 89 1 -85 90 1 -85 91 1 -85 92 1 -85 93 1 -85 94 1 -85 95 1 -85 96 1 -85 97 1 -85 98 1 -85 99 1 -85 100 1 -86 0 1 -86 1 1 -86 2 1 -86 3 1 -86 4 1 -86 5 1 -86 6 1 -86 7 1 -86 8 1 -86 9 1 -86 10 1 -86 11 1 -86 12 1 -86 13 1 -86 14 1 -86 15 1 -86 16 1 -86 17 1 -86 18 1 -86 19 1 -86 20 1 -86 21 1 -86 22 1 -86 23 1 -86 24 1 -86 25 1 -86 26 1 -86 27 1 -86 28 1 -86 29 1 -86 30 1 -86 31 1 -86 32 1 -86 33 1 -86 34 1 -86 35 1 -86 36 1 -86 37 1 -86 38 1 -86 39 1 -86 40 1 -86 41 1 -86 42 1 -86 43 1 -86 44 1 -86 45 1 -86 46 1 -86 47 1 -86 48 1 -86 49 1 -86 50 1 -86 51 1 -86 52 1 -86 53 1 -86 54 1 -86 55 1 -86 56 1 -86 57 1 -86 58 1 -86 59 1 -86 60 1 -86 61 1 -86 62 1 -86 63 1 -86 64 1 -86 65 1 -86 66 1 -86 67 1 -86 68 1 -86 69 1 -86 70 1 -86 71 1 -86 72 1 -86 73 1 -86 74 1 -86 75 1 -86 76 1 -86 77 1 -86 78 1 -86 79 1 -86 80 1 -86 81 1 -86 82 1 -86 83 1 -86 84 1 -86 85 1 -86 86 1 -86 87 1 -86 88 1 -86 89 1 -86 90 1 -86 91 1 -86 92 1 -86 93 1 -86 94 1 -86 95 1 -86 96 1 -86 97 1 -86 98 1 -86 99 1 -86 100 1 -87 0 1 -87 1 1 -87 2 1 -87 3 1 -87 4 1 -87 5 1 -87 6 1 -87 7 1 -87 8 1 -87 9 1 -87 10 1 -87 11 1 -87 12 1 -87 13 1 -87 14 1 -87 15 1 -87 16 1 -87 17 1 -87 18 1 -87 19 1 -87 20 1 -87 21 1 -87 22 1 -87 23 1 -87 24 1 -87 25 1 -87 26 1 -87 27 1 -87 28 1 -87 29 1 -87 30 1 -87 31 1 -87 32 1 -87 33 1 -87 34 1 -87 35 1 -87 36 1 -87 37 1 -87 38 1 -87 39 1 -87 40 1 -87 41 1 -87 42 1 -87 43 1 -87 44 1 -87 45 1 -87 46 1 -87 47 1 -87 48 1 -87 49 1 -87 50 1 -87 51 1 -87 52 1 -87 53 1 -87 54 1 -87 55 1 -87 56 1 -87 57 1 -87 58 1 -87 59 1 -87 60 1 -87 61 1 -87 62 1 -87 63 1 -87 64 1 -87 65 1 -87 66 1 -87 67 1 -87 68 1 -87 69 1 -87 70 1 -87 71 1 -87 72 1 -87 73 1 -87 74 1 -87 75 1 -87 76 1 -87 77 1 -87 78 1 -87 79 1 -87 80 1 -87 81 1 -87 82 1 -87 83 1 -87 84 1 -87 85 1 -87 86 1 -87 87 1 -87 88 1 -87 89 1 -87 90 1 -87 91 1 -87 92 1 -87 93 1 -87 94 1 -87 95 1 -87 96 1 -87 97 1 -87 98 1 -87 99 1 -87 100 1 -88 0 1 -88 1 1 -88 2 1 -88 3 1 -88 4 1 -88 5 1 -88 6 1 -88 7 1 -88 8 1 -88 9 1 -88 10 1 -88 11 1 -88 12 1 -88 13 1 -88 14 1 -88 15 1 -88 16 1 -88 17 1 -88 18 1 -88 19 1 -88 20 1 -88 21 1 -88 22 1 -88 23 1 -88 24 1 -88 25 1 -88 26 1 -88 27 1 -88 28 1 -88 29 1 -88 30 1 -88 31 1 -88 32 1 -88 33 1 -88 34 1 -88 35 1 -88 36 1 -88 37 1 -88 38 1 -88 39 1 -88 40 1 -88 41 1 -88 42 1 -88 43 1 -88 44 1 -88 45 1 -88 46 1 -88 47 1 -88 48 1 -88 49 1 -88 50 1 -88 51 1 -88 52 1 -88 53 1 -88 54 1 -88 55 1 -88 56 1 -88 57 1 -88 58 1 -88 59 1 -88 60 1 -88 61 1 -88 62 1 -88 63 1 -88 64 1 -88 65 1 -88 66 1 -88 67 1 -88 68 1 -88 69 1 -88 70 1 -88 71 1 -88 72 1 -88 73 1 -88 74 1 -88 75 1 -88 76 1 -88 77 1 -88 78 1 -88 79 1 -88 80 1 -88 81 1 -88 82 1 -88 83 1 -88 84 1 -88 85 1 -88 86 1 -88 87 1 -88 88 1 -88 89 1 -88 90 1 -88 91 1 -88 92 1 -88 93 1 -88 94 1 -88 95 1 -88 96 1 -88 97 1 -88 98 1 -88 99 1 -88 100 1 -89 0 1 -89 1 1 -89 2 1 -89 3 1 -89 4 1 -89 5 1 -89 6 1 -89 7 1 -89 8 1 -89 9 1 -89 10 1 -89 11 1 -89 12 1 -89 13 1 -89 14 1 -89 15 1 -89 16 1 -89 17 1 -89 18 1 -89 19 1 -89 20 1 -89 21 1 -89 22 1 -89 23 1 -89 24 1 -89 25 1 -89 26 1 -89 27 1 -89 28 1 -89 29 1 -89 30 1 -89 31 1 -89 32 1 -89 33 1 -89 34 1 -89 35 1 -89 36 1 -89 37 1 -89 38 1 -89 39 1 -89 40 1 -89 41 1 -89 42 1 -89 43 1 -89 44 1 -89 45 1 -89 46 1 -89 47 1 -89 48 1 -89 49 1 -89 50 1 -89 51 1 -89 52 1 -89 53 1 -89 54 1 -89 55 1 -89 56 1 -89 57 1 -89 58 1 -89 59 1 -89 60 1 -89 61 1 -89 62 1 -89 63 1 -89 64 1 -89 65 1 -89 66 1 -89 67 1 -89 68 1 -89 69 1 -89 70 1 -89 71 1 -89 72 1 -89 73 1 -89 74 1 -89 75 1 -89 76 1 -89 77 1 -89 78 1 -89 79 1 -89 80 1 -89 81 1 -89 82 1 -89 83 1 -89 84 1 -89 85 1 -89 86 1 -89 87 1 -89 88 1 -89 89 1 -89 90 1 -89 91 1 -89 92 1 -89 93 1 -89 94 1 -89 95 1 -89 96 1 -89 97 1 -89 98 1 -89 99 1 -89 100 1 -90 0 1 -90 1 1 -90 2 1 -90 3 1 -90 4 1 -90 5 1 -90 6 1 -90 7 1 -90 8 1 -90 9 1 -90 10 1 -90 11 1 -90 12 1 -90 13 1 -90 14 1 -90 15 1 -90 16 1 -90 17 1 -90 18 1 -90 19 1 -90 20 1 -90 21 1 -90 22 1 -90 23 1 -90 24 1 -90 25 1 -90 26 1 -90 27 1 -90 28 1 -90 29 1 -90 30 1 -90 31 1 -90 32 1 -90 33 1 -90 34 1 -90 35 1 -90 36 1 -90 37 1 -90 38 1 -90 39 1 -90 40 1 -90 41 1 -90 42 1 -90 43 1 -90 44 1 -90 45 1 -90 46 1 -90 47 1 -90 48 1 -90 49 1 -90 50 1 -90 51 1 -90 52 1 -90 53 1 -90 54 1 -90 55 1 -90 56 1 -90 57 1 -90 58 1 -90 59 1 -90 60 1 -90 61 1 -90 62 1 -90 63 1 -90 64 1 -90 65 1 -90 66 1 -90 67 1 -90 68 1 -90 69 1 -90 70 1 -90 71 1 -90 72 1 -90 73 1 -90 74 1 -90 75 1 -90 76 1 -90 77 1 -90 78 1 -90 79 1 -90 80 1 -90 81 1 -90 82 1 -90 83 1 -90 84 1 -90 85 1 -90 86 1 -90 87 1 -90 88 1 -90 89 1 -90 90 1 -90 91 1 -90 92 1 -90 93 1 -90 94 1 -90 95 1 -90 96 1 -90 97 1 -90 98 1 -90 99 1 -90 100 1 -91 0 1 -91 1 1 -91 2 1 -91 3 1 -91 4 1 -91 5 1 -91 6 1 -91 7 1 -91 8 1 -91 9 1 -91 10 1 -91 11 1 -91 12 1 -91 13 1 -91 14 1 -91 15 1 -91 16 1 -91 17 1 -91 18 1 -91 19 1 -91 20 1 -91 21 1 -91 22 1 -91 23 1 -91 24 1 -91 25 1 -91 26 1 -91 27 1 -91 28 1 -91 29 1 -91 30 1 -91 31 1 -91 32 1 -91 33 1 -91 34 1 -91 35 1 -91 36 1 -91 37 1 -91 38 1 -91 39 1 -91 40 1 -91 41 1 -91 42 1 -91 43 1 -91 44 1 -91 45 1 -91 46 1 -91 47 1 -91 48 1 -91 49 1 -91 50 1 -91 51 1 -91 52 1 -91 53 1 -91 54 1 -91 55 1 -91 56 1 -91 57 1 -91 58 1 -91 59 1 -91 60 1 -91 61 1 -91 62 1 -91 63 1 -91 64 1 -91 65 1 -91 66 1 -91 67 1 -91 68 1 -91 69 1 -91 70 1 -91 71 1 -91 72 1 -91 73 1 -91 74 1 -91 75 1 -91 76 1 -91 77 1 -91 78 1 -91 79 1 -91 80 1 -91 81 1 -91 82 1 -91 83 1 -91 84 1 -91 85 1 -91 86 1 -91 87 1 -91 88 1 -91 89 1 -91 90 1 -91 91 1 -91 92 1 -91 93 1 -91 94 1 -91 95 1 -91 96 1 -91 97 1 -91 98 1 -91 99 1 -91 100 1 -92 0 1 -92 1 1 -92 2 1 -92 3 1 -92 4 1 -92 5 1 -92 6 1 -92 7 1 -92 8 1 -92 9 1 -92 10 1 -92 11 1 -92 12 1 -92 13 1 -92 14 1 -92 15 1 -92 16 1 -92 17 1 -92 18 1 -92 19 1 -92 20 1 -92 21 1 -92 22 1 -92 23 1 -92 24 1 -92 25 1 -92 26 1 -92 27 1 -92 28 1 -92 29 1 -92 30 1 -92 31 1 -92 32 1 -92 33 1 -92 34 1 -92 35 1 -92 36 1 -92 37 1 -92 38 1 -92 39 1 -92 40 1 -92 41 1 -92 42 1 -92 43 1 -92 44 1 -92 45 1 -92 46 1 -92 47 1 -92 48 1 -92 49 1 -92 50 1 -92 51 1 -92 52 1 -92 53 1 -92 54 1 -92 55 1 -92 56 1 -92 57 1 -92 58 1 -92 59 1 -92 60 1 -92 61 1 -92 62 1 -92 63 1 -92 64 1 -92 65 1 -92 66 1 -92 67 1 -92 68 1 -92 69 1 -92 70 1 -92 71 1 -92 72 1 -92 73 1 -92 74 1 -92 75 1 -92 76 1 -92 77 1 -92 78 1 -92 79 1 -92 80 1 -92 81 1 -92 82 1 -92 83 1 -92 84 1 -92 85 1 -92 86 1 -92 87 1 -92 88 1 -92 89 1 -92 90 1 -92 91 1 -92 92 1 -92 93 1 -92 94 1 -92 95 1 -92 96 1 -92 97 1 -92 98 1 -92 99 1 -92 100 1 -93 0 1 -93 1 1 -93 2 1 -93 3 1 -93 4 1 -93 5 1 -93 6 1 -93 7 1 -93 8 1 -93 9 1 -93 10 1 -93 11 1 -93 12 1 -93 13 1 -93 14 1 -93 15 1 -93 16 1 -93 17 1 -93 18 1 -93 19 1 -93 20 1 -93 21 1 -93 22 1 -93 23 1 -93 24 1 -93 25 1 -93 26 1 -93 27 1 -93 28 1 -93 29 1 -93 30 1 -93 31 1 -93 32 1 -93 33 1 -93 34 1 -93 35 1 -93 36 1 -93 37 1 -93 38 1 -93 39 1 -93 40 1 -93 41 1 -93 42 1 -93 43 1 -93 44 1 -93 45 1 -93 46 1 -93 47 1 -93 48 1 -93 49 1 -93 50 1 -93 51 1 -93 52 1 -93 53 1 -93 54 1 -93 55 1 -93 56 1 -93 57 1 -93 58 1 -93 59 1 -93 60 1 -93 61 1 -93 62 1 -93 63 1 -93 64 1 -93 65 1 -93 66 1 -93 67 1 -93 68 1 -93 69 1 -93 70 1 -93 71 1 -93 72 1 -93 73 1 -93 74 1 -93 75 1 -93 76 1 -93 77 1 -93 78 1 -93 79 1 -93 80 1 -93 81 1 -93 82 1 -93 83 1 -93 84 1 -93 85 1 -93 86 1 -93 87 1 -93 88 1 -93 89 1 -93 90 1 -93 91 1 -93 92 1 -93 93 1 -93 94 1 -93 95 1 -93 96 1 -93 97 1 -93 98 1 -93 99 1 -93 100 1 -94 0 1 -94 1 1 -94 2 1 -94 3 1 -94 4 1 -94 5 1 -94 6 1 -94 7 1 -94 8 1 -94 9 1 -94 10 1 -94 11 1 -94 12 1 -94 13 1 -94 14 1 -94 15 1 -94 16 1 -94 17 1 -94 18 1 -94 19 1 -94 20 1 -94 21 1 -94 22 1 -94 23 1 -94 24 1 -94 25 1 -94 26 1 -94 27 1 -94 28 1 -94 29 1 -94 30 1 -94 31 1 -94 32 1 -94 33 1 -94 34 1 -94 35 1 -94 36 1 -94 37 1 -94 38 1 -94 39 1 -94 40 1 -94 41 1 -94 42 1 -94 43 1 -94 44 1 -94 45 1 -94 46 1 -94 47 1 -94 48 1 -94 49 1 -94 50 1 -94 51 1 -94 52 1 -94 53 1 -94 54 1 -94 55 1 -94 56 1 -94 57 1 -94 58 1 -94 59 1 -94 60 1 -94 61 1 -94 62 1 -94 63 1 -94 64 1 -94 65 1 -94 66 1 -94 67 1 -94 68 1 -94 69 1 -94 70 1 -94 71 1 -94 72 1 -94 73 1 -94 74 1 -94 75 1 -94 76 1 -94 77 1 -94 78 1 -94 79 1 -94 80 1 -94 81 1 -94 82 1 -94 83 1 -94 84 1 -94 85 1 -94 86 1 -94 87 1 -94 88 1 -94 89 1 -94 90 1 -94 91 1 -94 92 1 -94 93 1 -94 94 1 -94 95 1 -94 96 1 -94 97 1 -94 98 1 -94 99 1 -94 100 1 -95 0 1 -95 1 1 -95 2 1 -95 3 1 -95 4 1 -95 5 1 -95 6 1 -95 7 1 -95 8 1 -95 9 1 -95 10 1 -95 11 1 -95 12 1 -95 13 1 -95 14 1 -95 15 1 -95 16 1 -95 17 1 -95 18 1 -95 19 1 -95 20 1 -95 21 1 -95 22 1 -95 23 1 -95 24 1 -95 25 1 -95 26 1 -95 27 1 -95 28 1 -95 29 1 -95 30 1 -95 31 1 -95 32 1 -95 33 1 -95 34 1 -95 35 1 -95 36 1 -95 37 1 -95 38 1 -95 39 1 -95 40 1 -95 41 1 -95 42 1 -95 43 1 -95 44 1 -95 45 1 -95 46 1 -95 47 1 -95 48 1 -95 49 1 -95 50 1 -95 51 1 -95 52 1 -95 53 1 -95 54 1 -95 55 1 -95 56 1 -95 57 1 -95 58 1 -95 59 1 -95 60 1 -95 61 1 -95 62 1 -95 63 1 -95 64 1 -95 65 1 -95 66 1 -95 67 1 -95 68 1 -95 69 1 -95 70 1 -95 71 1 -95 72 1 -95 73 1 -95 74 1 -95 75 1 -95 76 1 -95 77 1 -95 78 1 -95 79 1 -95 80 1 -95 81 1 -95 82 1 -95 83 1 -95 84 1 -95 85 1 -95 86 1 -95 87 1 -95 88 1 -95 89 1 -95 90 1 -95 91 1 -95 92 1 -95 93 1 -95 94 1 -95 95 1 -95 96 1 -95 97 1 -95 98 1 -95 99 1 -95 100 1 -96 0 1 -96 1 1 -96 2 1 -96 3 1 -96 4 1 -96 5 1 -96 6 1 -96 7 1 -96 8 1 -96 9 1 -96 10 1 -96 11 1 -96 12 1 -96 13 1 -96 14 1 -96 15 1 -96 16 1 -96 17 1 -96 18 1 -96 19 1 -96 20 1 -96 21 1 -96 22 1 -96 23 1 -96 24 1 -96 25 1 -96 26 1 -96 27 1 -96 28 1 -96 29 1 -96 30 1 -96 31 1 -96 32 1 -96 33 1 -96 34 1 -96 35 1 -96 36 1 -96 37 1 -96 38 1 -96 39 1 -96 40 1 -96 41 1 -96 42 1 -96 43 1 -96 44 1 -96 45 1 -96 46 1 -96 47 1 -96 48 1 -96 49 1 -96 50 1 -96 51 1 -96 52 1 -96 53 1 -96 54 1 -96 55 1 -96 56 1 -96 57 1 -96 58 1 -96 59 1 -96 60 1 -96 61 1 -96 62 1 -96 63 1 -96 64 1 -96 65 1 -96 66 1 -96 67 1 -96 68 1 -96 69 1 -96 70 1 -96 71 1 -96 72 1 -96 73 1 -96 74 1 -96 75 1 -96 76 1 -96 77 1 -96 78 1 -96 79 1 -96 80 1 -96 81 1 -96 82 1 -96 83 1 -96 84 1 -96 85 1 -96 86 1 -96 87 1 -96 88 1 -96 89 1 -96 90 1 -96 91 1 -96 92 1 -96 93 1 -96 94 1 -96 95 1 -96 96 1 -96 97 1 -96 98 1 -96 99 1 -96 100 1 -97 0 1 -97 1 1 -97 2 1 -97 3 1 -97 4 1 -97 5 1 -97 6 1 -97 7 1 -97 8 1 -97 9 1 -97 10 1 -97 11 1 -97 12 1 -97 13 1 -97 14 1 -97 15 1 -97 16 1 -97 17 1 -97 18 1 -97 19 1 -97 20 1 -97 21 1 -97 22 1 -97 23 1 -97 24 1 -97 25 1 -97 26 1 -97 27 1 -97 28 1 -97 29 1 -97 30 1 -97 31 1 -97 32 1 -97 33 1 -97 34 1 -97 35 1 -97 36 1 -97 37 1 -97 38 1 -97 39 1 -97 40 1 -97 41 1 -97 42 1 -97 43 1 -97 44 1 -97 45 1 -97 46 1 -97 47 1 -97 48 1 -97 49 1 -97 50 1 -97 51 1 -97 52 1 -97 53 1 -97 54 1 -97 55 1 -97 56 1 -97 57 1 -97 58 1 -97 59 1 -97 60 1 -97 61 1 -97 62 1 -97 63 1 -97 64 1 -97 65 1 -97 66 1 -97 67 1 -97 68 1 -97 69 1 -97 70 1 -97 71 1 -97 72 1 -97 73 1 -97 74 1 -97 75 1 -97 76 1 -97 77 1 -97 78 1 -97 79 1 -97 80 1 -97 81 1 -97 82 1 -97 83 1 -97 84 1 -97 85 1 -97 86 1 -97 87 1 -97 88 1 -97 89 1 -97 90 1 -97 91 1 -97 92 1 -97 93 1 -97 94 1 -97 95 1 -97 96 1 -97 97 1 -97 98 1 -97 99 1 -97 100 1 -98 0 1 -98 1 1 -98 2 1 -98 3 1 -98 4 1 -98 5 1 -98 6 1 -98 7 1 -98 8 1 -98 9 1 -98 10 1 -98 11 1 -98 12 1 -98 13 1 -98 14 1 -98 15 1 -98 16 1 -98 17 1 -98 18 1 -98 19 1 -98 20 1 -98 21 1 -98 22 1 -98 23 1 -98 24 1 -98 25 1 -98 26 1 -98 27 1 -98 28 1 -98 29 1 -98 30 1 -98 31 1 -98 32 1 -98 33 1 -98 34 1 -98 35 1 -98 36 1 -98 37 1 -98 38 1 -98 39 1 -98 40 1 -98 41 1 -98 42 1 -98 43 1 -98 44 1 -98 45 1 -98 46 1 -98 47 1 -98 48 1 -98 49 1 -98 50 1 -98 51 1 -98 52 1 -98 53 1 -98 54 1 -98 55 1 -98 56 1 -98 57 1 -98 58 1 -98 59 1 -98 60 1 -98 61 1 -98 62 1 -98 63 1 -98 64 1 -98 65 1 -98 66 1 -98 67 1 -98 68 1 -98 69 1 -98 70 1 -98 71 1 -98 72 1 -98 73 1 -98 74 1 -98 75 1 -98 76 1 -98 77 1 -98 78 1 -98 79 1 -98 80 1 -98 81 1 -98 82 1 -98 83 1 -98 84 1 -98 85 1 -98 86 1 -98 87 1 -98 88 1 -98 89 1 -98 90 1 -98 91 1 -98 92 1 -98 93 1 -98 94 1 -98 95 1 -98 96 1 -98 97 1 -98 98 1 -98 99 1 -98 100 1 -99 0 1 -99 1 1 -99 2 1 -99 3 1 -99 4 1 -99 5 1 -99 6 1 -99 7 1 -99 8 1 -99 9 1 -99 10 1 -99 11 1 -99 12 1 -99 13 1 -99 14 1 -99 15 1 -99 16 1 -99 17 1 -99 18 1 -99 19 1 -99 20 1 -99 21 1 -99 22 1 -99 23 1 -99 24 1 -99 25 1 -99 26 1 -99 27 1 -99 28 1 -99 29 1 -99 30 1 -99 31 1 -99 32 1 -99 33 1 -99 34 1 -99 35 1 -99 36 1 -99 37 1 -99 38 1 -99 39 1 -99 40 1 -99 41 1 -99 42 1 -99 43 1 -99 44 1 -99 45 1 -99 46 1 -99 47 1 -99 48 1 -99 49 1 -99 50 1 -99 51 1 -99 52 1 -99 53 1 -99 54 1 -99 55 1 -99 56 1 -99 57 1 -99 58 1 -99 59 1 -99 60 1 -99 61 1 -99 62 1 -99 63 1 -99 64 1 -99 65 1 -99 66 1 -99 67 1 -99 68 1 -99 69 1 -99 70 1 -99 71 1 -99 72 1 -99 73 1 -99 74 1 -99 75 1 -99 76 1 -99 77 1 -99 78 1 -99 79 1 -99 80 1 -99 81 1 -99 82 1 -99 83 1 -99 84 1 -99 85 1 -99 86 1 -99 87 1 -99 88 1 -99 89 1 -99 90 1 -99 91 1 -99 92 1 -99 93 1 -99 94 1 -99 95 1 -99 96 1 -99 97 1 -99 98 1 -99 99 1 -99 100 1 -100 0 1 -100 1 1 -100 2 1 -100 3 1 -100 4 1 -100 5 1 -100 6 1 -100 7 1 -100 8 1 -100 9 1 -100 10 1 -100 11 1 -100 12 1 -100 13 1 -100 14 1 -100 15 1 -100 16 1 -100 17 1 -100 18 1 -100 19 1 -100 20 1 -100 21 1 -100 22 1 -100 23 1 -100 24 1 -100 25 1 -100 26 1 -100 27 1 -100 28 1 -100 29 1 -100 30 1 -100 31 1 -100 32 1 -100 33 1 -100 34 1 -100 35 1 -100 36 1 -100 37 1 -100 38 1 -100 39 1 -100 40 1 -100 41 1 -100 42 1 -100 43 1 -100 44 1 -100 45 1 -100 46 1 -100 47 1 -100 48 1 -100 49 1 -100 50 1 -100 51 1 -100 52 1 -100 53 1 -100 54 1 -100 55 1 -100 56 1 -100 57 1 -100 58 1 -100 59 1 -100 60 1 -100 61 1 -100 62 1 -100 63 1 -100 64 1 -100 65 1 -100 66 1 -100 67 1 -100 68 1 -100 69 1 -100 70 1 -100 71 1 -100 72 1 -100 73 1 -100 74 1 -100 75 1 -100 76 1 -100 77 1 -100 78 1 -100 79 1 -100 80 1 -100 81 1 -100 82 1 -100 83 1 -100 84 1 -100 85 1 -100 86 1 -100 87 1 -100 88 1 -100 89 1 -100 90 1 -100 91 1 -100 92 1 -100 93 1 -100 94 1 -100 95 1 -100 96 1 -100 97 1 -100 98 1 -100 99 1 -100 100 1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable deleted file mode 100644 index 44ee42e2..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable +++ /dev/null @@ -1 +0,0 @@ -Uniform diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable deleted file mode 100644 index 80dd323c..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable +++ /dev/null @@ -1 +0,0 @@ -Geometric diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.ttable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.ttable deleted file mode 100644 index e69de29b..00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anji deleted file mode 100644 index e69de29b..00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji deleted file mode 100644 index e69de29b..00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor deleted file mode 100644 index be586341..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor +++ /dev/null @@ -1 +0,0 @@ -0.3 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd deleted file mode 100644 index d17ce6ae62383450a8678872b637c8acf7c6307a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 VcmZQzU|?W`VqRXZ>&kmTBmfN}0@wfm diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd deleted file mode 100644 index 67392c2b8f0fccb590a0bc1afd6e029d3d68be3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ccmZQzU|?Vd;t%Hz?gwEI8-!Qn)a?Ud0GiSZCIA2c diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 deleted file mode 100644 index 49d59571..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor deleted file mode 100644 index 49d59571..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo deleted file mode 100644 index aa47d0d4..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo +++ /dev/null @@ -1,2 +0,0 @@ -0 -0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel deleted file mode 100644 index 1e5ffcdb..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel +++ /dev/null @@ -1,3 +0,0 @@ -Weighted incr. gaussian sentence length model... -numsents: 1 ; slensum: 1 ; tlensum: 1 -1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src deleted file mode 100644 index 8b137891..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc deleted file mode 100644 index d00491fd..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb deleted file mode 100644 index e04006a5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg deleted file mode 100644 index 8b137891..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb deleted file mode 100644 index e04006a5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anji deleted file mode 100644 index e69de29b..00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji deleted file mode 100644 index e69de29b..00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor deleted file mode 100644 index be586341..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor +++ /dev/null @@ -1 +0,0 @@ -0.3 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd deleted file mode 100644 index d17ce6ae62383450a8678872b637c8acf7c6307a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 VcmZQzU|?W`VqRXZ>&kmTBmfN}0@wfm diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd deleted file mode 100644 index 67392c2b8f0fccb590a0bc1afd6e029d3d68be3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ccmZQzU|?Vd;t%Hz?gwEI8-!Qn)a?Ud0GiSZCIA2c diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 deleted file mode 100644 index 49d59571..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor deleted file mode 100644 index 49d59571..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo deleted file mode 100644 index aa47d0d4..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo +++ /dev/null @@ -1,2 +0,0 @@ -0 -0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel deleted file mode 100644 index 1e5ffcdb..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel +++ /dev/null @@ -1,3 +0,0 @@ -Weighted incr. gaussian sentence length model... -numsents: 1 ; slensum: 1 ; tlensum: 1 -1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src deleted file mode 100644 index 8b137891..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc deleted file mode 100644 index d00491fd..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb deleted file mode 100644 index e04006a5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg deleted file mode 100644 index 8b137891..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb deleted file mode 100644 index e04006a5..00000000 --- a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj b/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj deleted file mode 100644 index b8c398f9..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj +++ /dev/null @@ -1,60 +0,0 @@ - - - - net8.0 - Serval.Machine.Shared - enable - enable - false - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs deleted file mode 100644 index 3cd02524..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class ClearMLServiceTests -{ - private const string ApiServer = "https://clearml.com"; - private const string AccessKey = "accessKey"; - private const string SecretKey = "secretKey"; - - [Test] - public async Task CreateTaskAsync() - { - var mockHttp = new MockHttpMessageHandler(); - mockHttp - .Expect(HttpMethod.Post, $"{ApiServer}/tasks.create") - .WithHeaders("Authorization", $"Bearer accessToken") - .WithPartialContent("\\u0027src_lang\\u0027: \\u0027spa_Latn\\u0027") - .WithPartialContent("\\u0027trg_lang\\u0027: \\u0027eng_Latn\\u0027") - .Respond("application/json", "{ \"data\": { \"id\": \"projectId\" } }"); - HttpClient httpClient = mockHttp.ToHttpClient(); - httpClient.BaseAddress = new Uri(ApiServer); - - var options = Substitute.For>(); - options.CurrentValue.Returns(new ClearMLOptions { AccessKey = AccessKey, SecretKey = SecretKey }); - var authService = Substitute.For(); - authService.GetAuthTokenAsync().Returns(Task.FromResult("accessToken")); - var env = new HostingEnvironment { EnvironmentName = Environments.Development }; - var httpClientFactory = Substitute.For(); - httpClientFactory.CreateClient("ClearML").Returns(httpClient); - var service = new ClearMLService(httpClientFactory, options, authService, env); - - string script = - "from machine.jobs.build_nmt_engine import run\n" - + "args = {\n" - + " 'model_type': 'huggingface',\n" - + " 'engine_id': 'engine1',\n" - + " 'build_id': 'build1',\n" - + " 'src_lang': 'spa_Latn',\n" - + " 'trg_lang': 'eng_Latn',\n" - + " 'max_steps': 20000,\n" - + " 'shared_file_uri': 's3://aqua-ml-data',\n" - + " 'clearml': True\n" - + "}\n" - + "run(args)\n"; - - string projectId = await service.CreateTaskAsync("build1", "project1", "dockerImage", script); - Assert.That(projectId, Is.EqualTo("projectId")); - mockHttp.VerifyNoOutstandingExpectation(); - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs deleted file mode 100644 index d9389a69..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs +++ /dev/null @@ -1,81 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class DistributedReaderWriterLockFactoryTests -{ - [Test] - public async Task InitAsync_ReleaseWriterLocks() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - WriterLock = new() { Id = "lock1", HostId = "this_service" }, - ReaderLocks = [], - WriterQueue = [] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.WriterLock, Is.Null); - } - - [Test] - public async Task InitAsync_ReleaseReaderLocks() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - ReaderLocks = [new() { Id = "lock1", HostId = "this_service" }], - WriterQueue = [] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.ReaderLocks, Is.Empty); - } - - [Test] - public async Task InitAsync_RemoveWaiters() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - WriterLock = new() { Id = "lock1", HostId = "other_service" }, - ReaderLocks = [], - WriterQueue = [new() { Id = "lock2", HostId = "this_service" }] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.WriterQueue, Is.Empty); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Locks = new MemoryRepository(); - ServiceOptions serviceOptions = new() { ServiceId = "this_service" }; - Factory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(serviceOptions), - Locks, - new ObjectIdGenerator() - ); - } - - public MemoryRepository Locks { get; } - public DistributedReaderWriterLockFactory Factory { get; } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs deleted file mode 100644 index dae41b35..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs +++ /dev/null @@ -1,405 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class DistributedReaderWriterLockTests -{ - [Test] - public async Task ReaderLockAsync_NoLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_ReaderLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndNotReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.WriterLockAsync(); - var task = rwLock.ReaderLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - task = rwLock.ReaderLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndExpired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) - { - var task = rwLock.ReaderLockAsync(); - await Task.Delay(500); - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_Cancelled() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - var cts = new CancellationTokenSource(); - task = rwLock.ReaderLockAsync(cancellationToken: cts.Token); - cts.Cancel(); - Assert.CatchAsync(async () => await task); - } - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_NoLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_ReaderLockAcquiredAndNotReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.ReaderLockAsync(); - var task = rwLock.WriterLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task WriterLockAsync_ReaderLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.ReaderLockAsync()) - { - task = rwLock.WriterLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndNeverReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.WriterLockAsync(); - var task = rwLock.WriterLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - task = rwLock.WriterLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_WriterLockTakesPriorityOverReaderLock() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task writeTask, - readTask; - await using (await rwLock.WriterLockAsync()) - { - readTask = rwLock.ReaderLockAsync(); - Assert.That(readTask.IsCompleted, Is.False); - writeTask = rwLock.WriterLockAsync(); - Assert.That(writeTask.IsCompleted, Is.False); - } - - await writeTask; - await AssertNeverCompletesAsync(readTask); - } - - [Test] - public async Task WriterLockAsync_FirstWriterLockHasPriority() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task1, - task2; - await using (await rwLock.WriterLockAsync()) - { - task1 = rwLock.WriterLockAsync(); - Assert.That(task1.IsCompleted, Is.False); - task2 = rwLock.WriterLockAsync(); - Assert.That(task2.IsCompleted, Is.False); - } - - await task1; - await AssertNeverCompletesAsync(task2); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndExpired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) - { - var task = rwLock.WriterLockAsync(); - await Task.Delay(500); - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_Cancelled() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - var cts = new CancellationTokenSource(); - task = rwLock.WriterLockAsync(cancellationToken: cts.Token); - cts.Cancel(); - Assert.CatchAsync(async () => await task); - } - - RWLock entity; - await using (await rwLock.WriterLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - private static async Task AssertNeverCompletesAsync(Task task, int timeout = 100) - { - if (task.IsCompleted) - Assert.Fail("Task completed unexpectedly."); - Task completedTask = await Task.WhenAny(task, Task.Delay(timeout)).ConfigureAwait(false); - if (completedTask == task) - Assert.Fail("Task completed unexpectedly."); - var _ = task.ContinueWith(_ => Assert.Fail("Task completed unexpectedly."), TaskScheduler.Default); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Locks = new MemoryRepository(); - var idGenerator = new ObjectIdGenerator(); - var options = Substitute.For>(); - options.Value.Returns(new ServiceOptions { ServiceId = "host" }); - Factory = new DistributedReaderWriterLockFactory(options, Locks, idGenerator); - } - - public DistributedReaderWriterLockFactory Factory { get; } - public MemoryRepository Locks { get; } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs deleted file mode 100644 index 687bb477..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class InMemoryStorageTests -{ - [Test] - public async Task ExistsAsync() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - bool exists = await fs.ExistsAsync("file1"); - Assert.That(exists, Is.True); - } - - [Test] - public async Task OpenReadAsync() - { - using InMemoryStorage fs = new(); - string input; - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - input = "Hello"; - sw.WriteLine(input); - } - string? output; - using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) - { - output = sr.ReadLine(); - } - Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); - } - - [Test] - public async Task ListFilesAsync_Recurse() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); - } - - [Test] - public async Task ListFilesAsync_DoNotRecurse() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); - } - - [Test] - public async Task DeleteAsync() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - await fs.DeleteAsync("test", recurse: true); - var files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.Empty); - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs deleted file mode 100644 index 008b13e0..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class LanguageTagServiceTests -{ - [Test] - [TestCase("es", "spa_Latn", Description = "Iso639_1Code")] - [TestCase("hne", "hne_Deva", Description = "Iso639_3Code")] - [TestCase("ks-Arab", "kas_Arab", Description = "ScriptCode")] - [TestCase("srp_Cyrl", "srp_Cyrl", Description = "InvalidLangTag")] - [TestCase("zh", "zho_Hans", Description = "ChineseNoScript")] - [TestCase("zh-Hant", "zho_Hant", Description = "ChineseScript")] - [TestCase("zh-TW", "zho_Hant", Description = "ChineseRegion")] - [TestCase("cmn", "zho_Hans", Description = "MandarinChineseNoScript")] - [TestCase("cmn-Hant", "zho_Hant", Description = "MandarinChineseScript")] - [TestCase("ms", "zsm_Latn", Description = "Macrolanguage")] - [TestCase("arb", "arb_Arab", Description = "Arabic")] - [TestCase("eng", "eng_Latn", Description = "InsteadOfISO639_1")] - [TestCase("eng-Latn", "eng_Latn", Description = "DashToUnderscore")] - [TestCase("kor", "kor_Hang", Description = "KoreanScript")] - [TestCase("kor_Kore", "kor_Hang", Description = "KoreanScriptCorrection")] - public void ConvertToFlores200CodeTest(string language, string internalCodeTruth) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - new LanguageTagService().ConvertToFlores200Code(language, out string internalCode); - Assert.That(internalCode, Is.EqualTo(internalCodeTruth)); - } - - [Test] - [TestCase("en", "eng_Latn", true)] - [TestCase("ms", "zsm_Latn", true)] - [TestCase("cmn", "zho_Hans", true)] - [TestCase("xyz", "xyz", false)] - public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCode, bool nativeLanguageSupport) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - bool isNative = new LanguageTagService().ConvertToFlores200Code(languageCode, out string internalCode); - Assert.Multiple(() => - { - Assert.That(internalCode, Is.EqualTo(resolvedLanguageCode)); - Assert.That(isNative, Is.EqualTo(nativeLanguageSupport)); - }); - } - - public class TestLanguageTagService : LanguageTagService - { - // Don't call Sldr initialize to call - protected override void InitializeSldrLanguageTags() - { - // remove langtags.json to force download - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - if (File.Exists(cachedAllTagsPath)) - File.Delete(cachedAllTagsPath); - Directory.CreateDirectory(Sldr.SldrCachePath); - } - } - - [Test] - public void BackupLangtagsJsonTest() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - var service = new TestLanguageTagService(); - service.ConvertToFlores200Code("en", out string internalCode); - Assert.That(internalCode, Is.EqualTo("eng_Latn")); - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs deleted file mode 100644 index c1b7ff04..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class LocalStorageTests -{ - [Test] - public async Task ExistsAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - bool exists = await fs.ExistsAsync("file1"); - Assert.That(exists, Is.True); - } - - [Test] - public async Task OpenReadAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - string input; - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - input = "Hello"; - sw.WriteLine(input); - } - string? output; - using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) - { - output = sr.ReadLine(); - } - Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); - } - - [Test] - public async Task ListFilesAsync_Recurse() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); - } - - [Test] - public async Task ListFilesAsync_DoNotRecurse() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); - } - - [Test] - public async Task DeleteFileAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - await fs.DeleteAsync("test", recurse: true); - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.Empty); - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs deleted file mode 100644 index 2a5e517b..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs +++ /dev/null @@ -1,214 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class MessageOutboxDeliveryServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method1 = "Method1"; - private const string Method2 = "Method2"; - - [Test] - public async Task ProcessMessagesAsync() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - await env.ProcessMessagesAsync(); - Received.InOrder(() => - { - env.Handler.HandleMessageAsync(Method2, "B", null, Arg.Any()); - env.Handler.HandleMessageAsync(Method1, "A", null, Arg.Any()); - env.Handler.HandleMessageAsync(Method2, "C", null, Arg.Any()); - }); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - } - - [Test] - public async Task ProcessMessagesAsync_Timeout() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - - // Timeout is long enough where the message attempt will be incremented, but not deleted. - env.EnableHandlerFailure(StatusCode.Internal); - await env.ProcessMessagesAsync(); - // Each group should try to send one message - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); - - // with now shorter timeout, the messages will be deleted. - // 4 start build attempts, and only one build completed attempt - env.Options.CurrentValue.Returns( - new MessageOutboxOptions { MessageExpirationTimeout = TimeSpan.FromMilliseconds(1) } - ); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env.Handler.Received(1) - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); - _ = env.Handler.Received(4) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_UnavailableFailure() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - - env.EnableHandlerFailure(StatusCode.Unavailable); - await env.ProcessMessagesAsync(); - // Only the first group should be attempted - but not recorded as attempted - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(0)); - _ = env.Handler.Received(1) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - - env.Handler.ClearReceivedCalls(); - env.EnableHandlerFailure(StatusCode.Internal); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); - _ = env.Handler.Received(2) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - - env.Handler.ClearReceivedCalls(); - env.DisableHandlerFailure(); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env.Handler.Received(1) - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); - _ = env.Handler.Received(2) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_File() - { - var env = new TestEnvironment(); - env.AddContentStreamMessages(); - - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env.Handler.Received(1) - .HandleMessageAsync(Method1, "A", Arg.Is(s => s != null), Arg.Any()); - env.FileSystem.Received().DeleteFile(Path.Combine("outbox", "A")); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - - Handler = Substitute.For(); - Handler.OutboxId.Returns(OutboxId); - FileSystem = Substitute.For(); - Options = Substitute.For>(); - Options.CurrentValue.Returns(new MessageOutboxOptions()); - - Service = new MessageOutboxDeliveryService( - Substitute.For(), - [Handler], - FileSystem, - Options, - Substitute.For>() - ); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public MessageOutboxDeliveryService Service { get; } - public IOutboxMessageHandler Handler { get; } - public IOptionsMonitor Options { get; } - public IFileSystem FileSystem { get; } - - public Task ProcessMessagesAsync() - { - return Service.ProcessMessagesAsync(Messages); - } - - public void AddStandardMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "A", - HasContentStream = false - } - ); - Messages.Add( - new OutboxMessage - { - Id = "B", - Index = 1, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "A", - Content = "B", - HasContentStream = false - } - ); - Messages.Add( - new OutboxMessage - { - Id = "C", - Index = 3, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "B", - Content = "C", - HasContentStream = false - } - ); - } - - public void AddContentStreamMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "A", - HasContentStream = true - } - ); - FileSystem - .OpenRead(Path.Combine("outbox", "A")) - .Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes("Content"))); - } - - public void EnableHandlerFailure(StatusCode code) - { - Handler - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - Handler - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - } - - public void DisableHandlerFailure() - { - Handler - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - Handler - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs deleted file mode 100644 index 876568d9..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class MessageOutboxServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method = "TestMethod"; - - [Test] - public async Task EnqueueMessageAsync_NoContentStream() - { - TestEnvironment env = new(); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(1)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_ExistingOutbox() - { - TestEnvironment env = new(); - env.Outboxes.Add(new Outbox { Id = OutboxId, CurrentIndex = 1 }); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(2)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(2)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_HasContentStream() - { - TestEnvironment env = new(); - await using MemoryStream fileStream = new(); - env.FileSystem.OpenWrite(Path.Combine("outbox", "1")).Returns(fileStream); - - await using MemoryStream stream = new(Encoding.UTF8.GetBytes("content")); - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content", stream); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.True); - Assert.That(fileStream.ToArray(), Is.EqualTo(stream.ToArray())); - } - - [Test] - public void EnqueueMessageAsync_ContentTooLarge() - { - TestEnvironment env = new(); - env.Service.MaxDocumentSize = 5; - - Assert.ThrowsAsync(() => env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content")); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - var idGenerator = Substitute.For(); - idGenerator.GenerateId().Returns("1"); - FileSystem = Substitute.For(); - var options = Substitute.For>(); - options.CurrentValue.Returns(new MessageOutboxOptions()); - Service = new MessageOutboxService(Outboxes, Messages, idGenerator, FileSystem, options); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public IFileSystem FileSystem { get; } - public MessageOutboxService Service { get; } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs deleted file mode 100644 index 49923372..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs +++ /dev/null @@ -1,104 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class ModelCleanupServiceTests -{ - private static readonly List ValidFiles = - [ - "models/engineId1_1.tar.gz", - "models/engineId2_2.tar.gz", - "models/engineId2_3.tar.gz" // only one build ahead - keep - ]; - private static readonly List InvalidFiles = - [ - "models/engineId2_1.tar.gz", // 1 build behind - "models/engineId2_4.tar.gz", // 2 builds ahead - "models/wrongId_1.tar.gz", - "models/engineId1_badBuildNumber.tar.gz", - "models/noBuildNumber.tar.gz", - "models/engineId1_1.differentExtension" - ]; - - [Test] - public async Task CheckModelsAsync_ValidFiles() - { - TestEnvironment env = new(); - await env.CreateFilesAsync(); - - Assert.That( - await env.SharedFileService.ListFilesAsync("models"), - Is.EquivalentTo(ValidFiles.Concat(InvalidFiles)) - ); - await env.CheckModelsAsync(); - // only valid files exist after running service - Assert.That(await env.SharedFileService.ListFilesAsync("models"), Is.EquivalentTo(ValidFiles)); - } - - private class TestEnvironment - { - private readonly MemoryRepository _engines; - - public TestEnvironment() - { - _engines = new MemoryRepository(); - _engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engineId1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = true - } - ); - _engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engineId2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 2, - IsModelPersisted = true - } - ); - - SharedFileService = new SharedFileService(Substitute.For()); - - Service = new ModelCleanupService( - Substitute.For(), - SharedFileService, - Substitute.For>() - ); - } - - public ModelCleanupService Service { get; } - public ISharedFileService SharedFileService { get; } - - public async Task CreateFilesAsync() - { - foreach (string path in ValidFiles) - { - await WriteFileStubAsync(path, "content"); - } - foreach (string path in InvalidFiles) - { - await WriteFileStubAsync(path, "content"); - } - } - - public Task CheckModelsAsync() - { - return Service.CheckModelsAsync(_engines, CancellationToken.None); - } - - private async Task WriteFileStubAsync(string path, string content) - { - using StreamWriter streamWriter = new(await SharedFileService.OpenWriteAsync(path, CancellationToken.None)); - await streamWriter.WriteAsync(content); - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs deleted file mode 100644 index 439b8d7c..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs +++ /dev/null @@ -1,123 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class NmtClearMLBuildJobFactoryTests -{ - [Test] - public async Task CreateJobScriptAsync_BuildOptions() - { - var env = new TestEnvironment(); - string script = await env.BuildJobFactory.CreateJobScriptAsync( - "engine1", - "build1", - "test_model", - BuildStage.Train, - buildOptions: "{ \"max_steps\": \"10\" }" - ); - Assert.That( - script, - Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { - 'model_type': 'test_model', - 'engine_id': 'engine1', - 'build_id': 'build1', - 'src_lang': 'spa_Latn', - 'trg_lang': 'eng_Latn', - 'shared_file_uri': 's3://bucket', - 'shared_file_folder': 'folder1/folder2', - 'build_options': '''{ ""max_steps"": ""10"" }''', - 'clearml': True -} -run(args) -".ReplaceLineEndings("\n") - ) - ); - } - - [Test] - public async Task CreateJobScriptAsync_NoBuildOptions() - { - var env = new TestEnvironment(); - string script = await env.BuildJobFactory.CreateJobScriptAsync( - "engine1", - "build1", - "test_model", - BuildStage.Train - ); - Assert.That( - script, - Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { - 'model_type': 'test_model', - 'engine_id': 'engine1', - 'build_id': 'build1', - 'src_lang': 'spa_Latn', - 'trg_lang': 'eng_Latn', - 'shared_file_uri': 's3://bucket', - 'shared_file_folder': 'folder1/folder2', - 'clearml': True -} -run(args) -".ReplaceLineEndings("\n") - ) - ); - } - - private class TestEnvironment - { - public ISharedFileService SharedFileService { get; } - public MemoryRepository Engines { get; } - public IOptionsMonitor Options { get; } - public ILanguageTagService LanguageTagService { get; } - public NmtClearMLBuildJobFactory BuildJobFactory { get; } - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - BuildJobRunner = BuildJobRunnerType.ClearML, - Stage = BuildStage.Train, - JobState = BuildJobState.Pending - } - } - ); - Options = Substitute.For>(); - Options.CurrentValue.Returns(new ClearMLOptions { }); - SharedFileService = Substitute.For(); - SharedFileService.GetBaseUri().Returns(new Uri("s3://bucket/folder1/folder2")); - LanguageTagService = Substitute.For(); - LanguageTagService.ConvertToFlores200Code("es", out string spa); - var anyStringArg = Arg.Any(); - LanguageTagService - .ConvertToFlores200Code("es", out anyStringArg) - .Returns(x => - { - x[1] = "spa_Latn"; - return true; - }); - LanguageTagService - .ConvertToFlores200Code("en", out anyStringArg) - .Returns(x => - { - x[1] = "eng_Latn"; - return true; - }); - BuildJobFactory = new NmtClearMLBuildJobFactory(SharedFileService, LanguageTagService, Engines); - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs deleted file mode 100644 index 5463e613..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs +++ /dev/null @@ -1,324 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class NmtEngineServiceTests -{ - [Test] - public async Task StartBuildAsync() - { - using var env = new TestEnvironment(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.Multiple(() => - { - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - Assert.That(engine.IsModelPersisted, Is.False); - }); - } - - [Test] - public async Task CancelBuildAsync_Building() - { - using var env = new TestEnvironment(); - env.UseInfiniteTrainJob(); - - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - } - - [Test] - public void CancelBuildAsync_NotBuilding() - { - using var env = new TestEnvironment(); - Assert.ThrowsAsync(() => env.Service.CancelBuildAsync("engine1")); - } - - [Test] - public async Task DeleteAsync_WhileBuilding() - { - using var env = new TestEnvironment(); - env.UseInfiniteTrainJob(); - - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.DeleteAsync("engine1"); - // ensure that the train job has completed - await env.WaitForBuildToFinishAsync(); - Assert.That(env.Engines.Contains("engine1"), Is.False); - } - - private class TestEnvironment : DisposableBase - { - private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private Func _trainJobFunc; - private Task? _trainJobTask; - - public TestEnvironment() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(offlineMode: true); - - _trainJobFunc = RunNormalTrainJob; - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false - } - ); - _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - ClearMLService - .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) - .Do(_ => _trainJobTask = Task.Run(_trainJobFunc)); - ClearMLService - .When(x => x.StopTaskAsync("job1", Arg.Any())) - .Do(_ => _cancellationTokenSource.Cancel()); - SharedFileService = new SharedFileService(Substitute.For()); - var buildJobOptions = Substitute.For>(); - buildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]), - new ClearMLBuildJobRunner( - ClearMLService, - [ - new NmtClearMLBuildJobFactory( - SharedFileService, - Substitute.For(), - Engines - ) - ], - buildJobOptions - ) - ], - Engines - ); - var clearMLOptions = Substitute.For>(); - clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); - ClearMLQueueService = new ClearMLMonitorService( - Substitute.For(), - ClearMLService, - SharedFileService, - clearMLOptions, - buildJobOptions, - Substitute.For>() - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public NmtEngineService Service { get; private set; } - public IClearMLQueueService ClearMLQueueService { get; } - public MemoryRepository Engines { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - public ISharedFileService SharedFileService { get; } - public IBuildJobService BuildJobService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(50), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private NmtEngineService CreateService() - { - return new NmtEngineService( - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - BuildJobService, - new LanguageTagService(), - ClearMLQueueService, - SharedFileService - ); - } - - public async Task WaitForBuildToFinishAsync() - { - await WaitForBuildState(e => e.CurrentBuild is null); - if (_trainJobTask is not null) - await _trainJobTask; - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => - e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage == BuildStage.Train - ); - } - - public void UseInfiniteTrainJob() - { - _trainJobFunc = RunInfiniteTrainJob; - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync(e => - e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? engine = subscription.Change.Entity; - if (engine is null || predicate(engine)) - break; - await subscription.WaitForChangeAsync(); - } - } - - private async Task RunNormalTrainJob() - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1"); - - await using Stream stream = await SharedFileService.OpenWriteAsync("builds/build1/pretranslate.trg.json"); - - await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - "engine1", - "build1", - BuildStage.Postprocess, - (0, 0.0) - ); - } - - private async Task RunInfiniteTrainJob() - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1"); - - while (!_cancellationTokenSource.IsCancellationRequested) - await Task.Delay(50); - - await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - _cancellationTokenSource.Dispose(); - } - - private class EnvActivator(TestEnvironment env) : JobActivator - { - private readonly TestEnvironment _env = env; - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(NmtPreprocessBuildJob)) - { - return new NmtPreprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - _env.BuildJobService, - _env.SharedFileService, - Substitute.For(), - new LanguageTagService() - ); - } - if (jobType == typeof(PostprocessBuildJob)) - { - return new PostprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs deleted file mode 100644 index 08b6d414..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ /dev/null @@ -1,581 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class PreprocessBuildJobTests -{ - [Test] - public async Task RunAsync_FilterOutEverything() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnTextIds() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = ["textId1"] }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainAndPretranslateAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null, TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_PretranslateAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); - } - - [Test] - public async Task RunAsync_PretranslateTextIds() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = ["textId1"], TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_EnableKeyTerms() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(1)); - }); - } - - [Test] - public async Task RunAsync_DisableKeyTerms() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_PretranslateChapters() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with - { - PretranslateChapters = new Dictionary> - { - { - "1CH", - new HashSet { 12 } - } - } - }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); - } - - [Test] - public async Task RunAsync_TrainOnChapters() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with - { - TrainOnChapters = new Dictionary> - { - { - "MAT", - new HashSet { 1 } - } - } - }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(5)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_MixedSource_Paratext() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceParatextCorpus with - { - TrainOnTextIds = null, - PretranslateTextIds = null - }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(12)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(12)); - } - - [Test] - public async Task RunAsync_MixedSource_Text() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceTextFileCorpus with - { - TrainOnTextIds = null, - PretranslateTextIds = null, - TrainOnChapters = null, - PretranslateChapters = null - }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(3)); - Assert.That(src2Count, Is.EqualTo(2)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public void RunAsync_UnknownLanguageTagsNoData() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; - - Assert.ThrowsAsync(async () => - { - await env.RunBuildJobAsync(corpus1, engineId: "engine2"); - }); - } - - [Test] - public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; - - await env.RunBuildJobAsync(corpus1, engineId: "engine2", engineType: TranslationEngineType.SmtTransfer); - } - - private class TestEnvironment : DisposableBase - { - private static readonly string TestDataPath = Path.Combine( - AppContext.BaseDirectory, - "..", - "..", - "..", - "Services", - "data" - ); - - private readonly TempDirectory _tempDir; - - public ISharedFileService SharedFileService { get; } - public ICorpusService CorpusService { get; } - public IPlatformService PlatformService { get; } - public MemoryRepository Engines { get; } - public IDistributedReaderWriterLockFactory LockFactory { get; } - public IBuildJobService BuildJobService { get; } - public IClearMLService ClearMLService { get; } - public IOptionsMonitor BuildJobOptions { get; } - - public Corpus DefaultTextFileCorpus { get; } - public Corpus DefaultMixedSourceTextFileCorpus { get; } - public Corpus DefaultParatextCorpus { get; } - public Corpus DefaultMixedSourceParatextCorpus { get; } - - public TestEnvironment() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(offlineMode: true); - - _tempDir = new TempDirectory("PreprocessBuildJobTests"); - - ZipParatextProject("pt-source1"); - ZipParatextProject("pt-source2"); - ZipParatextProject("pt-target1"); - - DefaultTextFileCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [TextFile("source1")], - TargetFiles = [TextFile("target1")] - }; - - DefaultMixedSourceTextFileCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [TextFile("source1"), TextFile("source2")], - TargetFiles = [TextFile("target1")] - }; - - DefaultParatextCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [ParatextFile("pt-source1")], - TargetFiles = [ParatextFile("pt-target1")] - }; - - DefaultMixedSourceParatextCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [ParatextFile("pt-source1"), ParatextFile("pt-source2")], - TargetFiles = [ParatextFile("pt-target1")] - }; - - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - Engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engine2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "xxx", - TargetLanguage = "zzz", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - Engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engine2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "xxx", - TargetLanguage = "zzz", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - CorpusService = new CorpusService(); - PlatformService = Substitute.For(); - LockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - BuildJobOptions = Substitute.For>(); - BuildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .GetProjectIdAsync("engine2", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .GetProjectIdAsync("engine2", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - SharedFileService = new SharedFileService(Substitute.For()); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner( - Substitute.For(), - [new NmtHangfireBuildJobFactory()] - ), - new ClearMLBuildJobRunner( - ClearMLService, - [ - new NmtClearMLBuildJobFactory( - SharedFileService, - Substitute.For(), - Engines - ) - ], - BuildJobOptions - ) - ], - Engines - ); - } - - public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) - { - switch (engineType) - { - case TranslationEngineType.Nmt: - { - return new NmtPreprocessBuildJob( - PlatformService, - Engines, - LockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - BuildJobService, - SharedFileService, - CorpusService, - new LanguageTagService() - ) - { - Seed = 1234 - }; - } - case TranslationEngineType.SmtTransfer: - { - return new PreprocessBuildJob( - PlatformService, - Engines, - LockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - BuildJobService, - SharedFileService, - CorpusService - ) - { - Seed = 1234 - }; - } - default: - throw new InvalidOperationException("Unknown engine type."); - } - ; - } - - public Task RunBuildJobAsync( - Corpus corpus, - bool useKeyTerms = true, - string engineId = "engine1", - TranslationEngineType engineType = TranslationEngineType.Nmt - ) - { - return GetBuildJob(engineType) - .RunAsync(engineId, "build1", [corpus], useKeyTerms ? null : "{\"use_key_terms\":false}", default); - } - - public async Task<(int Source1Count, int Source2Count, int TargetCount, int TermCount)> GetTrainCountAsync() - { - using StreamReader srcReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.src.txt")); - using StreamReader trgReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.trg.txt")); - int src1Count = 0; - int src2Count = 0; - int trgCount = 0; - int termCount = 0; - string? srcLine; - string? trgLine; - while ( - (srcLine = await srcReader.ReadLineAsync()) is not null - && (trgLine = await trgReader.ReadLineAsync()) is not null - ) - { - srcLine = srcLine.Trim(); - trgLine = trgLine.Trim(); - if (srcLine.StartsWith("Source one")) - src1Count++; - else if (srcLine.StartsWith("Source two")) - src2Count++; - else if (srcLine.Length == 0) - trgCount++; - else - termCount++; - } - return (src1Count, src2Count, trgCount, termCount); - } - - public async Task GetPretranslateCountAsync() - { - using StreamReader reader = - new(await SharedFileService.OpenReadAsync("builds/build1/pretranslate.src.json")); - JsonArray? pretranslationJsonObject = JsonSerializer.Deserialize(await reader.ReadToEndAsync()); - return pretranslationJsonObject?.Count ?? 0; - } - - private void ZipParatextProject(string name) - { - ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), Path.Combine(_tempDir.Path, $"{name}.zip")); - } - - private CorpusFile ParatextFile(string name) - { - return new() - { - TextId = name, - Format = FileFormat.Paratext, - Location = Path.Combine(_tempDir.Path, $"{name}.zip") - }; - } - - private static CorpusFile TextFile(string name) - { - return new() - { - TextId = "textId1", - Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, $"{name}.txt") - }; - } - - protected override void DisposeManagedResources() - { - _tempDir.Dispose(); - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs deleted file mode 100644 index f3667838..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs +++ /dev/null @@ -1,113 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class ServalPlatformOutboxMessageHandlerTests -{ - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - - [Test] - public async Task HandleMessageAsync_BuildStarted() - { - TestEnvironment env = new(); - - await env.Handler.HandleMessageAsync( - ServalPlatformOutboxConstants.BuildStarted, - JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), - null - ); - - _ = env.Client.Received(1).BuildStartedAsync(Arg.Is(x => x.BuildId == "C")); - } - - [Test] - public async Task HandleMessageAsync_InsertPretranslations() - { - TestEnvironment env = new(); - await using (MemoryStream stream = new()) - { - await JsonSerializer.SerializeAsync( - stream, - new[] - { - new Pretranslation - { - CorpusId = "corpus1", - TextId = "MAT", - Refs = ["MAT 1:1"], - Translation = "translation" - } - }, - JsonSerializerOptions - ); - stream.Seek(0, SeekOrigin.Begin); - await env.Handler.HandleMessageAsync( - ServalPlatformOutboxConstants.InsertPretranslations, - "engine1", - stream - ); - } - - _ = env.Client.Received(1).InsertPretranslations(); - _ = env.PretranslationWriter.Received(1) - .WriteAsync( - new InsertPretranslationRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - Refs = { "MAT 1:1" }, - Translation = "translation" - }, - Arg.Any() - ); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Client = Substitute.For(); - Client.BuildStartedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildCanceledAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildFaultedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildCompletedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client - .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) - .Returns(CreateEmptyUnaryCall()); - PretranslationWriter = Substitute.For>(); - Client - .InsertPretranslations(cancellationToken: Arg.Any()) - .Returns( - TestCalls.AsyncClientStreamingCall( - PretranslationWriter, - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ) - ); - - Handler = new ServalPlatformOutboxMessageHandler(Client); - } - - public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } - public ServalPlatformOutboxMessageHandler Handler { get; } - public IClientStreamWriter PretranslationWriter { get; } - - private static AsyncUnaryCall CreateEmptyUnaryCall() - { - return new AsyncUnaryCall( - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs deleted file mode 100644 index 5517aabf..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs +++ /dev/null @@ -1,740 +0,0 @@ -namespace Serval.Machine.Shared.Services; - -[TestFixture] -public class SmtTransferEngineServiceTests -{ - const string EngineId1 = "engine1"; - const string EngineId2 = "engine2"; - const string BuildId1 = "build1"; - const string CorpusId1 = "corpus1"; - - [Test] - public async Task CreateAsync() - { - using var env = new TestEnvironment(); - await env.Service.CreateAsync(EngineId2, "Engine 2", "es", "en"); - TranslationEngine? engine = await env.Engines.GetAsync(e => e.EngineId == EngineId2); - Assert.Multiple(() => - { - Assert.That(engine, Is.Not.Null); - Assert.That(engine?.EngineId, Is.EqualTo(EngineId2)); - Assert.That(engine?.BuildRevision, Is.EqualTo(0)); - Assert.That(engine?.IsModelPersisted, Is.True); - }); - string engineDir = Path.Combine("translation_engines", EngineId2); - _ = env.SmtModelFactory.Received().InitNewAsync(engineDir); - _ = env.TransferEngineFactory.Received().InitNewAsync(engineDir); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task StartBuildAsync(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - // ensure that the SMT model was loaded before training - await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); - await env.Service.StartBuildAsync( - EngineId1, - BuildId1, - null, - [ - new Corpus() - { - Id = CorpusId1, - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = [], - TargetFiles = [], - TrainOnTextIds = null, - PretranslateTextIds = null - } - ] - ); - await env.WaitForBuildToFinishAsync(); - _ = env.SmtBatchTrainer.Received() - .TrainAsync(Arg.Any>(), Arg.Any()); - _ = env.TruecaserTrainer.Received() - .TrainAsync(Arg.Any>(), Arg.Any()); - _ = env.SmtBatchTrainer.Received().SaveAsync(Arg.Any()); - _ = env.TruecaserTrainer.Received().SaveAsync(Arg.Any()); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - // check if SMT model was reloaded upon first use after training - env.SmtModel.ClearReceivedCalls(); - await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); - env.SmtModel.Received().Dispose(); - _ = env.SmtModel.DidNotReceive().SaveAsync(); - _ = env.Truecaser.DidNotReceive().SaveAsync(); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task CancelBuildAsync_Building(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.CancelBuildAsync(EngineId1); - await env.WaitForBuildToFinishAsync(); - _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); - _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - } - - [Test] - public void CancelBuildAsync_NotBuilding() - { - using var env = new TestEnvironment(); - Assert.ThrowsAsync(() => env.Service.CancelBuildAsync(EngineId1)); - } - - [Test] - public async Task StartBuildAsync_RestartUnfinishedBuild() - { - using var env = new TestEnvironment(BuildJobRunnerType.Hangfire); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - env.StopServer(); - await env.WaitForBuildToRestartAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Pending)); - _ = env.PlatformService.Received().BuildRestartingAsync(BuildId1); - env.SmtBatchTrainer.ClearSubstitute(ClearOptions.CallActions); - env.StartServer(); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.DeleteAsync(EngineId1); - await env.WaitForBuildToFinishAsync(); - await env.WaitForAllHangfireJobsToFinishAsync(); - _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); - _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); - Assert.That(env.Engines.Contains(EngineId1), Is.False); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task TrainSegmentPairAsync(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - env.StopTraining(); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - _ = env.SmtModel.Received(2).TrainSegmentAsync("esto es una prueba.", "this is a test.", true); - } - - [Test] - public async Task CommitAsync_LoadedInactive() - { - using var env = new TestEnvironment(); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - await Task.Delay(10); - await env.CommitAsync(TimeSpan.Zero); - _ = env.SmtModel.Received().SaveAsync(); - Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.False); - } - - [Test] - public async Task CommitAsync_LoadedActive() - { - using var env = new TestEnvironment(); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - await env.CommitAsync(TimeSpan.FromHours(1)); - _ = env.SmtModel.Received().SaveAsync(); - Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.True); - } - - [Test] - public async Task TranslateAsync() - { - using var env = new TestEnvironment(); - TranslationResult result = (await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."))[0]; - Assert.That(result.Translation, Is.EqualTo("this is a TEST.")); - } - - [Test] - public async Task GetWordGraphAsync() - { - using var env = new TestEnvironment(); - WordGraph result = await env.Service.GetWordGraphAsync(EngineId1, "esto es una prueba."); - Assert.That( - result.Arcs.Select(a => string.Join(' ', a.TargetTokens)), - Is.EqualTo(new[] { "this is", "a test", "." }) - ); - } - - private class TestEnvironment : DisposableBase - { - private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly ITruecaserFactory _truecaserFactory; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly BuildJobRunnerType _trainJobRunnerType; - private Task? _trainJobTask; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private bool _training = true; - - public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerType.ClearML) - { - _trainJobRunnerType = trainJobRunnerType; - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = EngineId1, - EngineId = EngineId1, - Type = TranslationEngineType.SmtTransfer, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false - } - ); - TrainSegmentPairs = new MemoryRepository(); - _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - SmtModel = Substitute.For(); - SmtBatchTrainer = Substitute.For(); - SmtBatchTrainer.Stats.Returns( - new TrainStats { TrainCorpusSize = 0, Metrics = { { "bleu", 0.0 }, { "perplexity", 0.0 } } } - ); - Truecaser = Substitute.For(); - TruecaserTrainer = Substitute.For(); - - SmtModelFactory = CreateSmtModelFactory(); - TransferEngineFactory = CreateTransferEngineFactory(); - _truecaserFactory = CreateTruecaserFactory(); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - SharedFileService = new SharedFileService(Substitute.For()); - var clearMLOptions = Substitute.For>(); - clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); - var buildJobOptions = Substitute.For>(); - buildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - ClearMLService - .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) - .Do(_ => _trainJobTask = Task.Run(RunTrainJob)); - ClearMLService - .When(x => x.StopTaskAsync("job1", Arg.Any())) - .Do(_ => _cancellationTokenSource.Cancel()); - ClearMLMonitorService = new ClearMLMonitorService( - Substitute.For(), - ClearMLService, - SharedFileService, - clearMLOptions, - buildJobOptions, - Substitute.For>() - ); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner(_jobClient, [new SmtTransferHangfireBuildJobFactory()]), - new ClearMLBuildJobRunner( - ClearMLService, - [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], - buildJobOptions - ) - ], - Engines - ); - _jobServer = CreateJobServer(); - StateService = CreateStateService(); - Service = CreateService(); - } - - public SmtTransferEngineService Service { get; private set; } - public SmtTransferEngineStateService StateService { get; private set; } - public MemoryRepository Engines { get; } - public MemoryRepository TrainSegmentPairs { get; } - public ISmtModelFactory SmtModelFactory { get; } - public ITransferEngineFactory TransferEngineFactory { get; } - public ITrainer SmtBatchTrainer { get; } - public IInteractiveTranslationModel SmtModel { get; } - public ITruecaser Truecaser { get; } - public ITrainer TruecaserTrainer { get; } - public IPlatformService PlatformService { get; } - - public IClearMLService ClearMLService { get; } - public IClearMLQueueService ClearMLMonitorService { get; } - - public ISharedFileService SharedFileService { get; } - - public IBuildJobService BuildJobService { get; } - - public async Task CommitAsync(TimeSpan inactiveTimeout) - { - await StateService.CommitAsync(_lockFactory, Engines, inactiveTimeout); - } - - public void StopServer() - { - _jobServer.Dispose(); - StateService.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - StateService = CreateStateService(); - Service = CreateService(); - } - - public void UseInfiniteTrainJob() - { - SmtBatchTrainer.TrainAsync( - Arg.Any>(), - Arg.Do(cancellationToken => - { - while (_training) - { - cancellationToken.ThrowIfCancellationRequested(); - Thread.Sleep(100); - } - }) - ); - } - - public void StopTraining() - { - _training = false; - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "smt_transfer" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(50), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private SmtTransferEngineStateService CreateStateService() - { - var options = Substitute.For>(); - options.CurrentValue.Returns(new SmtTransferEngineOptions()); - return new SmtTransferEngineStateService( - SmtModelFactory, - TransferEngineFactory, - _truecaserFactory, - options - ); - } - - private SmtTransferEngineService CreateService() - { - return new SmtTransferEngineService( - _lockFactory, - PlatformService, - new MemoryDataAccessContext(), - Engines, - TrainSegmentPairs, - StateService, - BuildJobService, - ClearMLMonitorService - ); - } - - private ISmtModelFactory CreateSmtModelFactory() - { - ISmtModelFactory factory = Substitute.For(); - - var translationResult = new TranslationResult( - "this is a TEST.", - "esto es una prueba .".Split(), - "this is a TEST .".Split(), - [1.0, 1.0, 1.0, 1.0, 1.0], - [ - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt - ], - new WordAlignmentMatrix(5, 5) - { - [0, 0] = true, - [1, 1] = true, - [2, 2] = true, - [3, 3] = true, - [4, 4] = true - }, - [new Phrase(Range.Create(0, 5), 5)] - ); - SmtModel - .TranslateAsync(1, Arg.Any()) - .Returns(Task.FromResult>([translationResult])); - SmtModel - .GetWordGraphAsync(Arg.Any()) - .Returns( - Task.FromResult( - new WordGraph( - "esto es una prueba .".Split(), - new[] - { - new WordGraphArc( - 0, - 1, - 1.0, - "this is".Split(), - new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, - Range.Create(0, 2), - GetSources(2, false), - [1.0, 1.0] - ), - new WordGraphArc( - 1, - 2, - 1.0, - "a test".Split(), - new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, - Range.Create(2, 4), - GetSources(2, false), - [1.0, 1.0] - ), - new WordGraphArc( - 2, - 3, - 1.0, - ".".Split(), - new WordAlignmentMatrix(1, 1) { [0, 0] = true }, - Range.Create(4, 5), - GetSources(1, false), - [1.0] - ) - }, - [3] - ) - ) - ); - - factory - .CreateAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(SmtModel)); - factory - .CreateTrainerAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(SmtBatchTrainer)); - return factory; - } - - private static ITransferEngineFactory CreateTransferEngineFactory() - { - ITransferEngineFactory factory = Substitute.For(); - ITranslationEngine engine = Substitute.For(); - engine - .TranslateAsync(Arg.Any()) - .Returns( - Task.FromResult( - new TranslationResult( - "this is a TEST.", - "esto es una prueba .".Split(), - "this is a TEST .".Split(), - [1.0, 1.0, 1.0, 1.0, 1.0], - [ - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer - ], - new WordAlignmentMatrix(5, 5) - { - [0, 0] = true, - [1, 1] = true, - [2, 2] = true, - [3, 3] = true, - [4, 4] = true - }, - [new Phrase(Range.Create(0, 5), 5)] - ) - ) - ); - factory - .CreateAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(engine)); - return factory; - } - - private ITruecaserFactory CreateTruecaserFactory() - { - ITruecaserFactory factory = Substitute.For(); - factory.CreateAsync(Arg.Any()).Returns(Task.FromResult(Truecaser)); - factory - .CreateTrainerAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(TruecaserTrainer)); - return factory; - } - - private static TranslationSources[] GetSources(int count, bool isUnknown) - { - var sources = new TranslationSources[count]; - for (int i = 0; i < count; i++) - sources[i] = isUnknown ? TranslationSources.None : TranslationSources.Smt; - return sources; - } - - public async Task WaitForAllHangfireJobsToFinishAsync() - { - IMonitoringApi monitoringApi = _memoryStorage.GetMonitoringApi(); - while (monitoringApi.EnqueuedCount("smt_transfer") > 0 || monitoringApi.ProcessingCount() > 0) - await Task.Delay(50); - } - - public async Task WaitForBuildToFinishAsync() - { - await WaitForBuildState(e => e.CurrentBuild is null); - if (_trainJobTask is not null) - await _trainJobTask; - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Active); - } - - public Task WaitForTrainingToStartAsync() - { - return WaitForBuildState(e => - e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage is BuildStage.Train - ); - } - - public Task WaitForBuildToRestartAsync() - { - return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Pending); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync(e => - e.EngineId == EngineId1 - ); - while (true) - { - TranslationEngine? engine = subscription.Change.Entity; - if (engine is null || predicate(engine)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - StateService.Dispose(); - _jobServer.Dispose(); - } - - private async Task RunTrainJob() - { - try - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1", _cancellationTokenSource.Token); - - string engineDir = Path.Combine("translation_engines", EngineId1); - await SmtModelFactory.InitNewAsync(engineDir, _cancellationTokenSource.Token); - ITextCorpus sourceCorpus = new DictionaryTextCorpus(); - ITextCorpus targetCorpus = new DictionaryTextCorpus(); - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - LatinWordTokenizer tokenizer = new(); - using ITrainer smtModelTrainer = await SmtModelFactory.CreateTrainerAsync( - engineDir, - tokenizer, - parallelCorpus, - _cancellationTokenSource.Token - ); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( - engineDir, - tokenizer, - targetCorpus, - _cancellationTokenSource.Token - ); - await smtModelTrainer.TrainAsync(null, _cancellationTokenSource.Token); - await truecaseTrainer.TrainAsync(cancellationToken: _cancellationTokenSource.Token); - - await smtModelTrainer.SaveAsync(_cancellationTokenSource.Token); - await truecaseTrainer.SaveAsync(_cancellationTokenSource.Token); - - await using Stream engineStream = await SharedFileService.OpenWriteAsync( - $"builds/{BuildId1}/model.tar.gz", - _cancellationTokenSource.Token - ); - - await using Stream targetStream = await SharedFileService.OpenWriteAsync( - $"builds/{BuildId1}/pretranslate.trg.json", - _cancellationTokenSource.Token - ); - - await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - EngineId1, - BuildId1, - BuildStage.Postprocess, - data: (0, 0.0) - ); - } - catch (OperationCanceledException) - { - await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); - } - } - - private class EnvActivator(TestEnvironment env) : JobActivator - { - private readonly TestEnvironment _env = env; - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(PreprocessBuildJob)) - { - return new PreprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - _env.BuildJobService, - _env.SharedFileService, - Substitute.For() - ) - { - TrainJobRunnerType = _env._trainJobRunnerType - }; - } - if (jobType == typeof(SmtTransferPostprocessBuildJob)) - { - var options = Substitute.For>(); - options.CurrentValue.Returns(new SmtTransferEngineOptions()); - return new SmtTransferPostprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService, - _env.TrainSegmentPairs, - _env.SmtModelFactory, - _env._truecaserFactory, - options - ); - } - if (jobType == typeof(SmtTransferTrainBuildJob)) - { - return new SmtTransferTrainBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService, - _env._truecaserFactory, - _env.SmtModelFactory, - _env.TransferEngineFactory - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM deleted file mode 100644 index b8665290..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM +++ /dev/null @@ -1,8 +0,0 @@ -\id LEV - Test -\h Leviticus -\mt Leviticus -\c 14 -\p -\v 55 Source one, chapter fourteen, verse fifty-five. -\v 55b Segment b. -\v 56 Source one, chapter fourteen, verse fifty-six. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM deleted file mode 100644 index 4eb8b5fd..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM +++ /dev/null @@ -1,12 +0,0 @@ -\id 1CH - Test -\h 1 Chronicles -\mt 1 Chronicles -\c 12 -\p -\v 1 Source one, chapter twelve, verse one. -\v 2 Source one, chapter twelve, verse two. -\v 3-7 Source one, chapter twelve, verses three through seven. -\v 8 Source one, chapter twelve, verse eight. -\c 13 -\p -\v 1 Source one, chapter thirteen, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM deleted file mode 100644 index ccf166e2..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM +++ /dev/null @@ -1,16 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Source one, chapter one, verse one. -\v 2-3 Source one, chapter one, verse two and three. -\v 4 Source one, chapter one, verse four. -\v 5 Source one, chapter one, verse five. -\v 6 Source one, chapter one, verse six. -\v 7-9 Source one, chapter one, verse seven, eight, and nine. -\v 10 Source one, chapter one, verse ten. -\c 2 -\p -\v 1 Source one, chapter two, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM deleted file mode 100644 index ff8aaf6e..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml deleted file mode 100644 index c80caedf..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml +++ /dev/null @@ -1,34 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test1 - 65001 - T - - NFC - Te1 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Tes.SFM - Major::BiblicalTerms.xml - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml deleted file mode 100644 index 03e45020..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs deleted file mode 100644 index 9c1cd387..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM deleted file mode 100644 index a0edc38b..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM +++ /dev/null @@ -1,7 +0,0 @@ -\id LEV - Test -\h Leviticus -\mt Leviticus -\c 14 -\p -\v 55 Source two, chapter fourteen, verse fifty-five. -\v 56 Source two, chapter fourteen, verse fifty-six. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM deleted file mode 100644 index 05cdff2c..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM +++ /dev/null @@ -1,16 +0,0 @@ -\id 1CH - Test -\h 1 Chronicles -\mt 1 Chronicles -\c 12 -\p -\v 1 Source two, chapter twelve, verse one. -\v 2 Source two, chapter twelve, verse two. -\v 3 Source two, chapter twelve, verse three. -\v 4 Source two, chapter twelve, verse four. -\v 5 Source two, chapter twelve, verse five. -\v 6 Source two, chapter twelve, verse six. -\v 7 Source two, chapter twelve, verse seven. -\v 8 Source two, chapter twelve, verse eight. -\c 13 -\p -\v 1 Source two, chapter thirteen, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM deleted file mode 100644 index 7208a72d..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM +++ /dev/null @@ -1,19 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Source two, chapter one, verse one. -\v 2 Source two, chapter one, verse two. -\v 3 Source two, chapter one, verse three. -\v 4 Source two, chapter one, verse four. -\v 5 Source two, chapter one, verse five. -\v 6 Source two, chapter one, verse six. -\v 7 Source two, chapter one, verse seven. -\v 8 Source two, chapter one, verse eight. -\v 9 Source two, chapter one, verse nine. -\v 10 Source two, chapter one, verse ten. -\c 2 -\p -\v 1 Source two, chapter two, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM deleted file mode 100644 index 22380983..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM +++ /dev/null @@ -1,7 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark -\c 1 -\p -\v 1 Source two, chapter one, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml deleted file mode 100644 index affce0ec..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml +++ /dev/null @@ -1,34 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test3 - 65001 - T - - NFC - Te3 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Tes.SFM - Major::BiblicalTerms.xml - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml deleted file mode 100644 index 03e45020..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs deleted file mode 100644 index 9c1cd387..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM deleted file mode 100644 index 69f46250..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM +++ /dev/null @@ -1,17 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Target one, chapter one, verse one. -\v 2 Target one, chapter one, verse two. -\v 3 Target one, chapter one, verse three. -\v 4 -\v 5-6 Target one, chapter one, verse five and six. -\v 7-8 Target one, chapter one, verse seven and eight. -\v 9-10 Target one, chapter one, verse nine and ten. -\c 2 -\p -\v 1 Target one, chapter two, verse one. -\v 2 Target one, chapter two, verse two. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM deleted file mode 100644 index 46000963..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml deleted file mode 100644 index 37d2772a..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml +++ /dev/null @@ -1,33 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test2 - 65001 - T - - NFC - Te2 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Ten.SFM - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml deleted file mode 100644 index 03e45020..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs deleted file mode 100644 index 9c1cd387..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt deleted file mode 100644 index 2aeb971c..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source one, Line 1 -Source one, Line 2 - -Source one, Line 4 - -Source one, Line 6 - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt deleted file mode 100644 index 7f4a0669..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source two, Line 1 -Source two, Line 2 - -Source two, Line 4 -Source two, Line 5 -Source two, Line 6 - diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt deleted file mode 100644 index 816e9435..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Target one, Line 1 - - -Target one, Line 4 - - -Target one, Line 7 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs deleted file mode 100644 index 4115517a..00000000 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs +++ /dev/null @@ -1,28 +0,0 @@ -global using System.IO.Compression; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Grpc.Core; -global using Grpc.Core.Testing; -global using Hangfire; -global using Hangfire.Storage; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Hosting.Internal; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using NSubstitute; -global using NSubstitute.ClearExtensions; -global using NSubstitute.ExceptionExtensions; -global using NSubstitute.ReceivedExtensions; -global using NUnit.Framework; -global using RichardSzalay.MockHttp; -global using Serval.Machine.Shared.Configuration; -global using Serval.Machine.Shared.Models; -global using SIL.DataAccess; -global using SIL.Machine.Annotations; -global using SIL.Machine.Corpora; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems;