diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 124ecbf7..e335e757 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup .NET - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: 8.0.x - name: Start MongoDB diff --git a/Serval.sln b/Serval.sln index 6cb275c0..8188624d 100644 --- a/Serval.sln +++ b/Serval.sln @@ -64,6 +64,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{92805246-528 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{BA044B98-3136-4FDE-B90F-B0975758C07F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Machine", "Machine", "{F6142E52-4B58-4D12-980F-B07D8AA932C2}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared", "src\Machine\src\Serval.Machine.Shared\Serval.Machine.Shared.csproj", "{090ECB69-464F-42C8-B92C-0808BE2802FA}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.EngineServer", "src\Machine\src\Serval.Machine.EngineServer\Serval.Machine.EngineServer.csproj", "{C02494FB-663E-4430-9F2D-41F1A740B271}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer", "src\Machine\src\Serval.Machine.JobServer\Serval.Machine.JobServer.csproj", "{BC766753-E560-4ADF-9923-C7A96076EA47}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -134,6 +148,22 @@ Global {0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.Build.0 = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.Build.0 = Release|Any CPU + {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.Build.0 = Release|Any CPU + {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.Build.0 = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -160,6 +190,12 @@ Global {3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B} {92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF} {BA044B98-3136-4FDE-B90F-B0975758C07F} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF} + {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} + {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} = {F6142E52-4B58-4D12-980F-B07D8AA932C2} + {090ECB69-464F-42C8-B92C-0808BE2802FA} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} + {C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} + {BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} + {B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/src/Machine/src/Serval.Machine.EngineServer/Program.cs b/src/Machine/src/Serval.Machine.EngineServer/Program.cs new file mode 100644 index 00000000..029e03df --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/Program.cs @@ -0,0 +1,39 @@ +using Hangfire; +using OpenTelemetry.Trace; + +var builder = WebApplication.CreateBuilder(args); + +// Add services to the container. +builder + .Services.AddMachine(builder.Configuration) + .AddBuildJobService() + .AddMongoDataAccess() + .AddMongoHangfireJobClient() + .AddServalTranslationEngineService() + .AddModelCleanupService() + .AddMessageOutboxDeliveryService() + .AddClearMLService(); + +if (builder.Environment.IsDevelopment()) +{ + builder + .Services.AddOpenTelemetry() + .WithTracing(builder => + { + builder + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddGrpcClientInstrumentation() + .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") + .AddConsoleExporter(); + }); +} + +var app = builder.Build(); + +app.UseHttpsRedirection(); + +app.MapServalTranslationEngineService(); +app.MapHangfireDashboard(); + +app.Run(); diff --git a/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json new file mode 100644 index 00000000..34eb2e94 --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "SIL.Machine.Serval.EngineServer": { + "commandName": "Project", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:9000" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj b/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj new file mode 100644 index 00000000..89f4d8e6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj @@ -0,0 +1,34 @@ + + + + net8.0 + enable + enable + 34e222a9-ef76-48f9-869e-338547f9bd25 + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + + + + icu.net.dll.config + + + + diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json new file mode 100644 index 00000000..1f2a4ef6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json @@ -0,0 +1,21 @@ +{ + "ConnectionStrings": { + "Hangfire": "mongodb://localhost:27017/machine_jobs", + "Mongo": "mongodb://localhost:27017/machine", + "Serval": "https://localhost:8444" + }, + "ClearML": { + "MaxSteps": 1000, + "Project": "dev" + }, + "SharedFile": { + "Uri": "s3://aqua-ml-data/dev/" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "System.Net.Http.HttpClient.Default": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.EngineServer/appsettings.json b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json new file mode 100644 index 00000000..271163ff --- /dev/null +++ b/src/Machine/src/Serval.Machine.EngineServer/appsettings.json @@ -0,0 +1,43 @@ +{ + "ConnectionStrings": { + "ClearML": "https://api.sil.hosted.allegro.ai" + }, + "AllowedHosts": "*", + "Service": { + "ServiceId": "machine_engine" + }, + "TranslationEngines": [ + "SmtTransfer", + "Nmt" + ], + "BuildJob": { + "ClearML": [ + { + "TranslationEngineType": "Nmt", + "ModelType": "huggingface", + "Queue": "jobs_backlog", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "TranslationEngineType": "SmtTransfer", + "ModelType": "thot", + "Queue": "cpu_only", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + } + ] + }, + "SmtTransferEngine": { + "EnginesDir": "/var/lib/machine/engines" + }, + "ClearML": { + "BuildPollingEnabled": true + }, + "MessageOutbox": { + "OutboxDir": "/var/lib/machine/outbox" + }, + "Logging": { + "LogLevel": { + "System.Net.Http.HttpClient.Default": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Program.cs b/src/Machine/src/Serval.Machine.JobServer/Program.cs new file mode 100644 index 00000000..d78bfed8 --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/Program.cs @@ -0,0 +1,30 @@ +using OpenTelemetry.Trace; + +var builder = WebApplication.CreateBuilder(args); + +builder + .Services.AddMachine(builder.Configuration) + .AddBuildJobService() + .AddMongoDataAccess() + .AddMongoHangfireJobClient() + .AddHangfireJobServer() + .AddServalPlatformService() + .AddClearMLService(); +if (builder.Environment.IsDevelopment()) +{ + builder + .Services.AddOpenTelemetry() + .WithTracing(builder => + { + builder + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddGrpcClientInstrumentation() + .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") + .AddConsoleExporter(); + }); +} + +var app = builder.Build(); + +app.Run(); diff --git a/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json b/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json new file mode 100644 index 00000000..f636d0c3 --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "SIL.Machine.Serval.JobServer": { + "commandName": "Project", + "launchBrowser": false, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:9100" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj b/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj new file mode 100644 index 00000000..8a466b1d --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj @@ -0,0 +1,37 @@ + + + + net8.0 + enable + enable + aa9e7440-5a04-4de6-ba51-bab9ef4a62e1 + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + + + + + + + + + + + + + + icu.net.dll.config + + + + diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json new file mode 100644 index 00000000..1f2a4ef6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json @@ -0,0 +1,21 @@ +{ + "ConnectionStrings": { + "Hangfire": "mongodb://localhost:27017/machine_jobs", + "Mongo": "mongodb://localhost:27017/machine", + "Serval": "https://localhost:8444" + }, + "ClearML": { + "MaxSteps": 1000, + "Project": "dev" + }, + "SharedFile": { + "Uri": "s3://aqua-ml-data/dev/" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "System.Net.Http.HttpClient.Default": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json new file mode 100644 index 00000000..1b2d3baf --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.JobServer/appsettings.json b/src/Machine/src/Serval.Machine.JobServer/appsettings.json new file mode 100644 index 00000000..738a4f28 --- /dev/null +++ b/src/Machine/src/Serval.Machine.JobServer/appsettings.json @@ -0,0 +1,43 @@ +{ + "ConnectionStrings": { + "ClearML": "https://api.sil.hosted.allegro.ai" + }, + "AllowedHosts": "*", + "Service": { + "ServiceId": "machine_job" + }, + "TranslationEngines": [ + "SmtTransfer", + "Nmt" + ], + "BuildJob": { + "ClearML": [ + { + "TranslationEngineType": "Nmt", + "ModelType": "huggingface", + "Queue": "jobs_backlog", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + }, + { + "TranslationEngineType": "SmtTransfer", + "ModelType": "thot", + "Queue": "jobs_backlog", + "DockerImage": "ghcr.io/sillsdev/machine.py:latest" + } + ] + }, + "SmtTransferEngine": { + "EnginesDir": "/var/lib/machine/engines" + }, + "ClearML": { + "BuildPollingEnabled": false + }, + "MessageOutbox": { + "OutboxDir": "/var/lib/machine/outbox" + }, + "Logging": { + "LogLevel": { + "System.Net.Http.HttpClient.Default": "Warning" + } + } +} \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs new file mode 100644 index 00000000..547a9dbd --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Configuration; + +public class BuildJobOptions +{ + public const string Key = "BuildJob"; + + public IList ClearML { get; set; } = new List(); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs new file mode 100644 index 00000000..53e25245 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLBuildQueue.cs @@ -0,0 +1,9 @@ +namespace Serval.Machine.Shared.Configuration; + +public class ClearMLBuildQueue +{ + public TranslationEngineType TranslationEngineType { get; set; } + public string ModelType { get; set; } = ""; + public string Queue { get; set; } = "default"; + public string DockerImage { get; set; } = ""; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs new file mode 100644 index 00000000..e72b7dec --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/ClearMLOptions.cs @@ -0,0 +1,13 @@ +namespace Serval.Machine.Shared.Configuration; + +public class ClearMLOptions +{ + public const string Key = "ClearML"; + + public string AccessKey { get; set; } = ""; + public string SecretKey { get; set; } = ""; + public bool BuildPollingEnabled { get; set; } = false; + public TimeSpan BuildPollingTimeout { get; set; } = TimeSpan.FromSeconds(10); + public string RootProject { get; set; } = "Machine"; + public string Project { get; set; } = "dev"; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs new file mode 100644 index 00000000..694dd67e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IEndpointRouteBuilderExtensions.cs @@ -0,0 +1,11 @@ +namespace Microsoft.AspNetCore.Builder; + +public static class IEndpointRouteBuilderExtensions +{ + public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndpointRouteBuilder builder) + { + builder.MapGrpcService(); + + return builder; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs new file mode 100644 index 00000000..f8dfbcd5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilder.cs @@ -0,0 +1,7 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public interface IMachineBuilder +{ + IServiceCollection Services { get; } + IConfiguration? Configuration { get; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs new file mode 100644 index 00000000..567a073e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -0,0 +1,437 @@ +using Serval.Translation.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMachineBuilderExtensions +{ + public static IMachineBuilder AddServiceOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddClearMLOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddClearMLOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddMessageOutboxOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddMessageOutboxOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddBuildJobOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddBuildJobOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) + { + if (builder.Configuration is null) + return builder.AddThotSmtModel(o => { }); + else + return builder.AddThotSmtModel(builder.Configuration.GetSection(ThotSmtModelOptions.Key)); + } + + public static IMachineBuilder AddThotSmtModel( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration?.GetConnectionString("ClearML"); + if (connectionString is null) + throw new InvalidOperationException("ClearML connection string is required"); + + builder + .Services.AddHttpClient("ClearML") + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) + // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds + .AddTransientHttpErrorPolicy(b => + b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + ); + + builder.Services.AddSingleton(); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder + .Services.AddHttpClient("ClearML-NoRetry") + .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); + builder.Services.AddSingleton(); + + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + + return builder; + } + + private static MongoStorageOptions GetMongoStorageOptions() + { + var mongoStorageOptions = new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy() + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + }; + return mongoStorageOptions; + } + + public static IMachineBuilder AddMongoHangfireJobClient( + this IMachineBuilder builder, + string? connectionString = null + ) + { + connectionString ??= builder.Configuration?.GetConnectionString("Hangfire"); + if (connectionString is null) + throw new InvalidOperationException("Hangfire connection string is required"); + + builder.Services.AddHangfire(c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage(connectionString, GetMongoStorageOptions()) + .UseFilter(new AutomaticRetryAttribute { Attempts = 0 }) + ); + builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + return builder; + } + + public static IMachineBuilder AddHangfireJobServer( + this IMachineBuilder builder, + IEnumerable? engineTypes = null + ) + { + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; + var queues = new List(); + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + queues.Add("smt_transfer"); + break; + case TranslationEngineType.Nmt: + queues.Add("nmt"); + break; + } + } + + builder.Services.AddHangfireServer(o => + { + o.Queues = queues.ToArray(); + }); + return builder; + } + + public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) + { + builder.Services.AddMemoryDataAccess(o => + { + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + }); + + return builder; + } + + public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration?.GetConnectionString("Mongo"); + if (connectionString is null) + throw new InvalidOperationException("Mongo connection string is required"); + builder.Services.AddMongoDataAccess( + connectionString!, + "Serval.Machine.Shared.Models", + o => + { + o.AddRepository( + "translation_engines", + mapSetup: m => m.SetIgnoreExtraElements(true), + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders + .IndexKeys.Ascending(e => e.EngineId) + .Ascending("currentBuild._id") + ) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) + ) + ); + } + ); + o.AddRepository("locks"); + o.AddRepository( + "train_segment_pairs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ) + ); + o.AddRepository( + "outbox_messages", + mapSetup: m => m.MapProperty(m => m.OutboxRef).SetSerializer(new StringSerializer()) + ); + o.AddRepository( + "outboxes", + mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) + ); + } + ); + builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); + + return builder; + } + + public static IMachineBuilder AddServalPlatformService( + this IMachineBuilder builder, + string? connectionString = null + ) + { + connectionString ??= builder.Configuration?.GetConnectionString("Serval"); + if (connectionString is null) + throw new InvalidOperationException("Serval connection string is required"); + + builder.Services.AddScoped(); + + builder.Services.AddSingleton(); + + builder.Services.AddScoped(); + + builder + .Services.AddGrpcClient(o => + { + o.Address = new Uri(connectionString); + }) + .ConfigureChannel(o => + { + o.MaxRetryAttempts = null; + o.ServiceConfig = new ServiceConfig + { + MethodConfigs = + { + new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy + { + MaxAttempts = 10, + InitialBackoff = TimeSpan.FromSeconds(1), + MaxBackoff = TimeSpan.FromSeconds(5), + BackoffMultiplier = 1.5, + RetryableStatusCodes = { StatusCode.Unavailable } + } + }, + new MethodConfig + { + Names = + { + new MethodName + { + Service = "serval.translation.v1.TranslationPlatformApi", + Method = "UpdateBuildStatus" + } + } + }, + } + }; + }); + + return builder; + } + + public static IMachineBuilder AddServalTranslationEngineService( + this IMachineBuilder builder, + string? connectionString = null, + IEnumerable? engineTypes = null + ) + { + builder.Services.AddGrpc(options => + { + options.Interceptors.Add(); + options.Interceptors.Add(); + }); + builder.AddServalPlatformService(connectionString); + + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + builder.Services.AddScoped(); + break; + case TranslationEngineType.Nmt: + builder.Services.AddScoped(); + break; + } + } + + return builder; + } + + public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, string? smtTransferEngineDir = null) + { + builder.Services.AddScoped(); + + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(x => x.GetRequiredService()); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder.Services.AddScoped(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); + + if (smtTransferEngineDir is null) + { + var smtTransferEngineOptions = new SmtTransferEngineOptions(); + builder.Configuration?.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); + smtTransferEngineDir = smtTransferEngineOptions.EnginesDir; + } + string? driveLetter = Path.GetPathRoot(smtTransferEngineDir)?[..1]; + if (driveLetter is null) + throw new InvalidOperationException("SMT Engine directory is required"); + // add health check for disk storage capacity + builder + .Services.AddHealthChecks() + .AddDiskStorageHealthCheck( + x => x.AddDrive(driveLetter, 1_000), // 1GB + "SMT Engine Storage Capacity", + HealthStatus.Degraded + ); + + return builder; + } + + public static IMachineBuilder AddModelCleanupService(this IMachineBuilder builder) + { + builder.Services.AddHostedService(); + return builder; + } + + public static IMachineBuilder AddMessageOutboxDeliveryService(this IMachineBuilder builder) + { + builder.Services.AddHostedService(); + return builder; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs new file mode 100644 index 00000000..7463e6ac --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IServiceCollectionExtensions.cs @@ -0,0 +1,54 @@ +namespace Microsoft.Extensions.DependencyInjection; + +public static class IServiceCollectionExtensions +{ + public static IMachineBuilder AddMachine(this IServiceCollection services, IConfiguration? configuration = null) + { + if (!Sldr.IsInitialized) + Sldr.Initialize(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddHealthChecks().AddCheck("S3 Bucket"); + + services.AddSingleton(); + services.AddTransient(); + + services.AddScoped(); + services.AddSingleton(); + services.AddStartupTask( + (sp, cancellationToken) => + sp.GetRequiredService().InitAsync(cancellationToken) + ); + + var builder = new MachineBuilder(services, configuration); + if (configuration is null) + { + builder.AddServiceOptions(o => { }); + builder.AddSharedFileOptions(o => { }); + builder.AddSmtTransferEngineOptions(o => { }); + builder.AddClearMLOptions(o => { }); + builder.AddBuildJobOptions(o => { }); + builder.AddMessageOutboxOptions(o => { }); + } + else + { + builder.AddServiceOptions(configuration.GetSection(ServiceOptions.Key)); + builder.AddSharedFileOptions(configuration.GetSection(SharedFileOptions.Key)); + builder.AddSmtTransferEngineOptions(configuration.GetSection(SmtTransferEngineOptions.Key)); + builder.AddClearMLOptions(configuration.GetSection(ClearMLOptions.Key)); + builder.AddBuildJobOptions(configuration.GetSection(BuildJobOptions.Key)); + builder.AddMessageOutboxOptions(configuration.GetSection(MessageOutboxOptions.Key)); + } + return builder; + } + + public static IServiceCollection AddStartupTask( + this IServiceCollection services, + Func startupTask + ) + { + services.AddHostedService(sp => new StartupTask(sp, startupTask)); + return services; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs new file mode 100644 index 00000000..58ddf5c1 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/MachineBuilder.cs @@ -0,0 +1,7 @@ +namespace Microsoft.Extensions.DependencyInjection; + +internal class MachineBuilder(IServiceCollection services, IConfiguration? configuration) : IMachineBuilder +{ + public IServiceCollection Services { get; } = services; + public IConfiguration? Configuration { get; } = configuration; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs new file mode 100644 index 00000000..e2e88feb --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/MessageOutboxOptions.cs @@ -0,0 +1,9 @@ +namespace Serval.Machine.Shared.Configuration; + +public class MessageOutboxOptions +{ + public const string Key = "MessageOutbox"; + + public string OutboxDir { get; set; } = "outbox"; + public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs new file mode 100644 index 00000000..8011e7b7 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/ServiceOptions.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Configuration; + +public class ServiceOptions +{ + public const string Key = "Service"; + + public string ServiceId { get; set; } = "machine_api"; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs new file mode 100644 index 00000000..4ae27e1e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/SharedFileOptions.cs @@ -0,0 +1,11 @@ +namespace Serval.Machine.Shared.Configuration; + +public class SharedFileOptions +{ + public const string Key = "SharedFile"; + + public string Uri { get; set; } = "file:///var/lib/machine/"; + public string S3AccessKeyId { get; set; } = ""; + public string S3SecretAccessKey { get; set; } = ""; + public string S3Region { get; set; } = "us-east-1"; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs new file mode 100644 index 00000000..15002604 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/SmtTransferEngineOptions.cs @@ -0,0 +1,10 @@ +namespace Serval.Machine.Shared.Configuration; + +public class SmtTransferEngineOptions +{ + public const string Key = "SmtTransferEngine"; + + public string EnginesDir { get; set; } = "translation_engines"; + public TimeSpan EngineCommitFrequency { get; set; } = TimeSpan.FromMinutes(5); + public TimeSpan InactiveEngineTimeout { get; set; } = TimeSpan.FromMinutes(10); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs new file mode 100644 index 00000000..780eb7d2 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/ThotSmtModelOptions.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Configuration; + +public class ThotSmtModelOptions +{ + public const string Key = "ThotSmtModel"; + + public ThotSmtModelOptions() + { + string installDir = Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location)!; + NewModelFile = Path.Combine(installDir, "thot-new-model.zip"); + } + + public string NewModelFile { get; set; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs new file mode 100644 index 00000000..aca20540 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs @@ -0,0 +1,32 @@ +namespace Serval.Machine.Shared.Models; + +public enum BuildJobState +{ + None, + Pending, + Active, + Canceling +} + +public enum BuildJobRunnerType +{ + Hangfire, + ClearML +} + +public enum BuildStage +{ + Preprocess, + Train, + Postprocess +} + +public record Build +{ + public required string BuildId { get; init; } + public required BuildJobState JobState { get; init; } + public required string JobId { get; init; } + public required BuildJobRunnerType BuildJobRunner { get; init; } + public required BuildStage Stage { get; init; } + public string? Options { get; set; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs new file mode 100644 index 00000000..5ae9fbfd --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLMetricsEvent.cs @@ -0,0 +1,12 @@ +namespace Serval.Machine.Shared.Models; + +public record ClearMLMetricsEvent +{ + public string? Metric { get; init; } + public string? Variant { get; init; } + public required double Value { get; init; } + public double? MinValue { get; init; } + public int? MinValueIteration { get; init; } + public double? MaxValue { get; init; } + public int? MaxValueIteration { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs new file mode 100644 index 00000000..c0cd0d7e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLProject.cs @@ -0,0 +1,6 @@ +namespace Serval.Machine.Shared.Models; + +public record ClearMLProject +{ + public required string Id { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs new file mode 100644 index 00000000..5b13fdaa --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/ClearMLTask.cs @@ -0,0 +1,35 @@ +namespace Serval.Machine.Shared.Models; + +public enum ClearMLTaskStatus +{ + Created, + Queued, + InProgress, + Stopped, + Published, + Publishing, + Closed, + Failed, + Completed, + Unknown +} + +public record ClearMLTask +{ + public required string Id { get; init; } + public required string Name { get; init; } + public required ClearMLProject Project { get; init; } + public required ClearMLTaskStatus Status { get; init; } + public string? StatusReason { get; init; } + public string? StatusMessage { get; init; } + public required DateTime Created { get; init; } + public int? LastIteration { get; init; } + public int ActiveDuration { get; init; } + public required IReadOnlyDictionary< + string, + IReadOnlyDictionary + > LastMetrics { get; init; } + + [JsonConverter(typeof(DictionaryStringStringConverter))] + public required IReadOnlyDictionary Runtime { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs b/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs new file mode 100644 index 00000000..9145e90d --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/Corpus.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Models; + +public record Corpus +{ + public required string Id { get; init; } + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public IReadOnlyDictionary>? TrainOnChapters { get; init; } + public IReadOnlyDictionary>? PretranslateChapters { get; init; } + public required HashSet? TrainOnTextIds { get; init; } + public required HashSet? PretranslateTextIds { get; init; } + public required IReadOnlyList SourceFiles { get; init; } + public required IReadOnlyList TargetFiles { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs b/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs new file mode 100644 index 00000000..a84bf7f6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/CorpusFile.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Models; + +public enum FileFormat +{ + Text = 0, + Paratext = 1 +} + +public record CorpusFile +{ + public required string Location { get; init; } + public required FileFormat Format { get; init; } + public required string TextId { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs b/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs new file mode 100644 index 00000000..39ceae87 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/Lock.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Models; + +public record Lock +{ + public required string Id { get; init; } + public DateTime? ExpiresAt { get; init; } + public required string HostId { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs b/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs new file mode 100644 index 00000000..798fe175 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/ModelDownloadUrl.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Models; + +public record ModelDownloadUrl +{ + public required string Url { get; init; } + public required int ModelRevision { get; init; } + public required DateTime ExpiresAt { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs b/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs new file mode 100644 index 00000000..ad9c0001 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/Outbox.cs @@ -0,0 +1,10 @@ +namespace Serval.Machine.Shared.Models; + +public record Outbox : IEntity +{ + public string Id { get; set; } = ""; + + public int Revision { get; set; } + + public int CurrentIndex { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs b/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs new file mode 100644 index 00000000..0e95e9a6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/OutboxMessage.cs @@ -0,0 +1,15 @@ +namespace Serval.Machine.Shared.Models; + +public record OutboxMessage : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required int Index { get; init; } + public required string OutboxRef { get; init; } + public required string Method { get; init; } + public required string GroupId { get; init; } + public string? Content { get; init; } + public required bool HasContentStream { get; init; } + public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow; + public int Attempts { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs b/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs new file mode 100644 index 00000000..6e9807b5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/Pretranslation.cs @@ -0,0 +1,9 @@ +namespace Serval.Machine.Shared.Models; + +public record Pretranslation +{ + public required string CorpusId { get; init; } + public required string TextId { get; init; } + public required IReadOnlyList Refs { get; init; } + public required string Translation { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs b/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs new file mode 100644 index 00000000..2271aa9b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/RWLock.cs @@ -0,0 +1,25 @@ +namespace Serval.Machine.Shared.Models; + +public record RWLock : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public Lock? WriterLock { get; init; } + public required IReadOnlyList ReaderLocks { get; init; } + public required IReadOnlyList WriterQueue { get; init; } + + public bool IsAvailableForReading() + { + var now = DateTime.UtcNow; + return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) + && WriterQueue.Count == 0; + } + + public bool IsAvailableForWriting(string? lockId = null) + { + var now = DateTime.UtcNow; + return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) + && !ReaderLocks.Any(l => l.ExpiresAt is null || l.ExpiresAt > now) + && (lockId is null || WriterQueue.Count > 0 && WriterQueue[0].Id == lockId); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs b/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs new file mode 100644 index 00000000..30927345 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/TrainSegmentPair.cs @@ -0,0 +1,11 @@ +namespace Serval.Machine.Shared.Models; + +public record TrainSegmentPair : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string TranslationEngineRef { get; init; } + public required string Source { get; init; } + public required string Target { get; init; } + public required bool SentenceStart { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs b/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs new file mode 100644 index 00000000..80b1f648 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/TranslationEngine.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Models; + +public record TranslationEngine : IEntity +{ + public string Id { get; set; } = ""; + public int Revision { get; set; } = 1; + public required string EngineId { get; init; } + public required TranslationEngineType Type { get; init; } + public required string SourceLanguage { get; init; } + public required string TargetLanguage { get; init; } + public required bool IsModelPersisted { get; init; } + public int BuildRevision { get; init; } + public Build? CurrentBuild { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs b/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..54a4902d --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Properties/AssemblyInfo.cs @@ -0,0 +1,2 @@ +[assembly: InternalsVisibleTo("Serval.Machine.Shared.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj new file mode 100644 index 00000000..6b716479 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj @@ -0,0 +1,60 @@ + + + + net8.0 + An ASP.NET Core web API middleware for the Machine library. + enable + enable + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs new file mode 100644 index 00000000..244aa04a --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs @@ -0,0 +1,211 @@ +namespace Serval.Machine.Shared.Services; + +public class BuildJobService(IEnumerable runners, IRepository engines) + : IBuildJobService +{ + private readonly Dictionary _runners = runners.ToDictionary(r => r.Type); + private readonly IRepository _engines = engines; + + public Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default) + { + return _engines.ExistsAsync(e => e.EngineId == engineId && e.CurrentBuild != null, cancellationToken); + } + + public Task> GetBuildingEnginesAsync( + BuildJobRunnerType runner, + CancellationToken cancellationToken = default + ) + { + return _engines.GetAllAsync( + e => e.CurrentBuild != null && e.CurrentBuild.BuildJobRunner == runner, + cancellationToken + ); + } + + public async Task GetBuildAsync( + string engineId, + string buildId, + CancellationToken cancellationToken = default + ) + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, + cancellationToken + ); + return engine?.CurrentBuild; + } + + public async Task CreateEngineAsync( + string engineId, + string? name = null, + CancellationToken cancellationToken = default + ) + { + foreach (BuildJobRunnerType runnerType in _runners.Keys) + { + IBuildJobRunner runner = _runners[runnerType]; + await runner.CreateEngineAsync(engineId, name, cancellationToken); + } + } + + public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) + { + foreach (BuildJobRunnerType runnerType in _runners.Keys) + { + IBuildJobRunner runner = _runners[runnerType]; + await runner.DeleteEngineAsync(engineId, cancellationToken); + } + } + + public async Task StartBuildJobAsync( + BuildJobRunnerType runnerType, + string engineId, + string buildId, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ) + { + TranslationEngine? engine = await _engines.GetAsync( + e => + e.EngineId == engineId + && (e.CurrentBuild == null || e.CurrentBuild.JobState != BuildJobState.Canceling), + cancellationToken + ); + if (engine is null) + return false; + + IBuildJobRunner runner = _runners[runnerType]; + string jobId = await runner.CreateJobAsync( + engine.Type, + engineId, + buildId, + stage, + data, + buildOptions, + cancellationToken + ); + try + { + await _engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set( + e => e.CurrentBuild, + new Build + { + BuildId = buildId, + JobId = jobId, + BuildJobRunner = runner.Type, + Stage = stage, + JobState = BuildJobState.Pending, + Options = buildOptions + } + ), + cancellationToken: cancellationToken + ); + await runner.EnqueueJobAsync(jobId, engine.Type, cancellationToken); + return true; + } + catch + { + await runner.DeleteJobAsync(jobId, CancellationToken.None); + throw; + } + } + + public async Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( + string engineId, + CancellationToken cancellationToken = default + ) + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.CurrentBuild != null, + cancellationToken + ); + if (engine is null || engine.CurrentBuild is null) + return (null, BuildJobState.None); + + IBuildJobRunner runner = _runners[engine.CurrentBuild.BuildJobRunner]; + + if (engine.CurrentBuild.JobState is BuildJobState.Pending) + { + // cancel a job that hasn't started yet + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.CurrentBuild != null, + u => u.Unset(b => b.CurrentBuild), + returnOriginal: true, + cancellationToken: cancellationToken + ); + if (engine is not null && engine.CurrentBuild is not null) + { + // job will be deleted from the queue + await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); + return (engine.CurrentBuild.BuildId, BuildJobState.None); + } + } + else if (engine.CurrentBuild.JobState is BuildJobState.Active) + { + // cancel a job that is already running + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.CurrentBuild != null, + u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Canceling), + cancellationToken: cancellationToken + ); + if (engine is not null && engine.CurrentBuild is not null) + { + await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); + return (engine.CurrentBuild.BuildId, BuildJobState.Canceling); + } + } + + return (null, BuildJobState.None); + } + + public async Task BuildJobStartedAsync( + string engineId, + string buildId, + CancellationToken cancellationToken = default + ) + { + TranslationEngine? engine = await _engines.UpdateAsync( + e => + e.EngineId == engineId + && e.CurrentBuild != null + && e.CurrentBuild.BuildId == buildId + && e.CurrentBuild.JobState == BuildJobState.Pending, + u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Active), + cancellationToken: cancellationToken + ); + return engine is not null; + } + + public Task BuildJobFinishedAsync( + string engineId, + string buildId, + bool buildComplete, + CancellationToken cancellationToken = default + ) + { + return _engines.UpdateAsync( + e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, + u => + { + u.Unset(e => e.CurrentBuild); + if (buildComplete) + u.Inc(e => e.BuildRevision); + }, + cancellationToken: cancellationToken + ); + } + + public Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default) + { + return _engines.UpdateAsync( + e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, + u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Pending), + cancellationToken: cancellationToken + ); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs new file mode 100644 index 00000000..88422c6c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/BuildProgress.cs @@ -0,0 +1,25 @@ +namespace Serval.Machine.Shared.Services; + +public class BuildProgress(IPlatformService platformService, string buildId) : IProgress +{ + private readonly IPlatformService _platformService = platformService; + private readonly string _buildId = buildId; + private ProgressStatus _prevStatus; + + private DateTime _lastReportTime = DateTime.Now; + + private const float ThrottleTimeSeconds = 1; + + public void Report(ProgressStatus value) + { + if (_prevStatus.Equals(value)) + return; + + if (DateTime.Now < _lastReportTime.AddSeconds(ThrottleTimeSeconds)) + return; + + _lastReportTime = DateTime.Now; + _platformService.UpdateBuildStatusAsync(_buildId, value); + _prevStatus = value; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs new file mode 100644 index 00000000..73b06c2b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/CancellationInterceptor.cs @@ -0,0 +1,30 @@ +namespace Serval.Machine.Shared.Services; + +public class CancellationInterceptor(ILogger logger) : Interceptor +{ + private readonly ILogger _logger = logger; + + public override async Task UnaryServerHandler( + TRequest request, + ServerCallContext context, + UnaryServerMethod continuation + ) + { + try + { + return await continuation(request, context); + } + catch (Exception ex) + { + if (ex is OperationCanceledException) + { + _logger.LogInformation("An operation was canceled."); + throw new RpcException(new Status(StatusCode.Cancelled, "An operation was canceled.")); + } + else + { + throw; + } + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs new file mode 100644 index 00000000..9603aeb6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLAuthenticationService.cs @@ -0,0 +1,77 @@ +namespace Serval.Machine.Shared.Services; + +public class ClearMLAuthenticationService( + IServiceProvider services, + IHttpClientFactory httpClientFactory, + IOptionsMonitor options, + ILogger logger +) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService +{ + private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); + private readonly IOptionsMonitor _options = options; + private readonly ILogger _logger = logger; + private readonly AsyncLock _lock = new(); + + // technically, the token should be good for 30 days, but let's refresh each hour + // to know well ahead of time if something is wrong. + private static readonly TimeSpan RefreshPeriod = TimeSpan.FromSeconds(3600); + private string _authToken = ""; + + public async Task GetAuthTokenAsync(CancellationToken cancellationToken = default) + { + using (await _lock.LockAsync(cancellationToken)) + { + if (_authToken is "") + { + //Should only happen once, so no different in cost than previous solution + _logger.LogInformation("Token was empty; refreshing"); + await AuthorizeAsync(cancellationToken); + } + } + return _authToken; + } + + protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) + { + try + { + using (await _lock.LockAsync(cancellationToken)) + await AuthorizeAsync(cancellationToken); + } + catch (Exception e) + { + if (_authToken is "") + { + _logger.LogError(e, "Error occurred while acquiring ClearML authentication token for the first time."); + // The ClearML token never was set. We can't continue without it. + throw; + } + else + { + _logger.LogError(e, "Error occurred while refreshing ClearML authentication token."); + } + } + } + + private async Task AuthorizeAsync(CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Post, "auth.login") + { + Content = new StringContent("{}", Encoding.UTF8, "application/json") + }; + var authenticationString = $"{_options.CurrentValue.AccessKey}:{_options.CurrentValue.SecretKey}"; + var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString)); + request.Headers.Add("Authorization", $"Basic {base64EncodedAuthenticationString}"); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + string result = await response.Content.ReadAsStringAsync(cancellationToken); + string? refreshedToken = (string?)((JsonObject?)JsonNode.Parse(result))?["data"]?["token"]; + if (refreshedToken is null || refreshedToken is "") + { + throw new InvalidOperationException( + $"ClearML authentication failed - {response.StatusCode}: {response.ReasonPhrase}" + ); + } + _authToken = refreshedToken; + _logger.LogInformation("ClearML Authentication Token Refresh Successful."); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs new file mode 100644 index 00000000..910dd957 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs @@ -0,0 +1,88 @@ +namespace Serval.Machine.Shared.Services; + +public class ClearMLBuildJobRunner( + IClearMLService clearMLService, + IEnumerable buildJobFactories, + IOptionsMonitor options +) : IBuildJobRunner +{ + private readonly IClearMLService _clearMLService = clearMLService; + private readonly Dictionary _buildJobFactories = + buildJobFactories.ToDictionary(f => f.EngineType); + + private readonly Dictionary _options = + options.CurrentValue.ClearML.ToDictionary(o => o.TranslationEngineType); + + public BuildJobRunnerType Type => BuildJobRunnerType.ClearML; + + public async Task CreateEngineAsync( + string engineId, + string? name = null, + CancellationToken cancellationToken = default + ) + { + await _clearMLService.CreateProjectAsync(engineId, name, cancellationToken); + } + + public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) + { + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, cancellationToken); + } + + public async Task CreateJobAsync( + TranslationEngineType engineType, + string engineId, + string buildId, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ) + { + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + projectId ??= await _clearMLService.CreateProjectAsync(engineId, cancellationToken: cancellationToken); + + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (task is not null) + return task.Id; + + IClearMLBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; + string script = await buildJobFactory.CreateJobScriptAsync( + engineId, + buildId, + _options[engineType].ModelType, + stage, + data, + buildOptions, + cancellationToken + ); + return await _clearMLService.CreateTaskAsync( + buildId, + projectId, + script, + _options[engineType].DockerImage, + cancellationToken + ); + } + + public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) + { + return _clearMLService.DeleteTaskAsync(jobId, cancellationToken); + } + + public Task EnqueueJobAsync( + string jobId, + TranslationEngineType engineType, + CancellationToken cancellationToken = default + ) + { + return _clearMLService.EnqueueTaskAsync(jobId, _options[engineType].Queue, cancellationToken); + } + + public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) + { + return _clearMLService.StopTaskAsync(jobId, cancellationToken); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs new file mode 100644 index 00000000..929b14ed --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLHealthCheck.cs @@ -0,0 +1,100 @@ +namespace Serval.Machine.Shared.Services; + +public class ClearMLHealthCheck( + IClearMLAuthenticationService clearMLAuthenticationService, + IHttpClientFactory httpClientFactory, + IOptionsMonitor buildJobOptions +) : IHealthCheck +{ + private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML-NoRetry"); + private readonly IClearMLAuthenticationService _clearMLAuthenticationService = clearMLAuthenticationService; + private readonly ISet _queuesMonitored = buildJobOptions + .CurrentValue.ClearML.Select(x => x.Queue) + .ToHashSet(); + + private int _numConsecutiveFailures = 0; + private readonly AsyncLock _lock = new AsyncLock(); + + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default + ) + { + try + { + if (!await PingAsync(cancellationToken)) + return HealthCheckResult.Unhealthy("ClearML is unresponsive"); + IReadOnlySet queuesWithoutWorkers = await QueuesWithoutWorkers(cancellationToken); + if (queuesWithoutWorkers.Count > 0) + { + return HealthCheckResult.Unhealthy( + $"No ClearML agents are available for configured queues: {string.Join(", ", queuesWithoutWorkers)}" + ); + } + + using (await _lock.LockAsync(cancellationToken)) + _numConsecutiveFailures = 0; + return HealthCheckResult.Healthy("ClearML is available"); + } + catch (Exception e) + { + using (await _lock.LockAsync(cancellationToken)) + { + _numConsecutiveFailures++; + return _numConsecutiveFailures > 3 + ? HealthCheckResult.Unhealthy(exception: e) + : HealthCheckResult.Degraded(exception: e); + } + } + } + + private async Task CallAsync( + string service, + string action, + JsonNode body, + CancellationToken cancellationToken = default + ) + { + var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") + { + Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") + }; + request.Headers.Add( + "Authorization", + $"Bearer {await _clearMLAuthenticationService.GetAuthTokenAsync(cancellationToken)}" + ); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + string result = await response.Content.ReadAsStringAsync(cancellationToken); + return (JsonObject?)JsonNode.Parse(result); + } + + public async Task PingAsync(CancellationToken cancellationToken = default) + { + JsonObject? result = await CallAsync("debug", "ping", new JsonObject(), cancellationToken); + return result is not null; + } + + public async Task> QueuesWithoutWorkers(CancellationToken cancellationToken = default) + { + var queuesWithoutWorkers = _queuesMonitored.ToHashSet(); + JsonObject? result = await CallAsync("workers", "get_all", new JsonObject(), cancellationToken); + JsonNode? workers_node = result?["data"]?["workers"]; + if (workers_node is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + var workers = (JsonArray)workers_node; + foreach (var worker in workers) + { + JsonNode? queues_node = worker?["queues"]; + if (queues_node is null) + continue; + var queues = (JsonArray)queues_node; + foreach (var currentQueue in queues) + { + string? currentQueueName = (string?)currentQueue?["name"]; + if (currentQueueName is not null) + queuesWithoutWorkers.Remove(currentQueueName); + } + } + return queuesWithoutWorkers; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs new file mode 100644 index 00000000..f577fdce --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -0,0 +1,407 @@ +namespace Serval.Machine.Shared.Services; + +public class ClearMLMonitorService( + IServiceProvider services, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor clearMLOptions, + IOptionsMonitor buildJobOptions, + ILogger logger +) + : RecurrentTask( + "ClearML monitor service", + services, + clearMLOptions.CurrentValue.BuildPollingTimeout, + logger, + clearMLOptions.CurrentValue.BuildPollingEnabled + ), + IClearMLQueueService +{ + private static readonly string SummaryMetric = CreateMD5("Summary"); + private static readonly string TrainCorpusSizeVariant = CreateMD5("train_corpus_size"); + private static readonly string ConfidenceVariant = CreateMD5("confidence"); + + private readonly IClearMLService _clearMLService = clearMLService; + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly ILogger _logger = logger; + private readonly Dictionary _curBuildStatus = new(); + + private readonly IReadOnlyDictionary _queuePerEngineType = + buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => x.Queue); + + private readonly IDictionary _queueSizePerEngineType = new ConcurrentDictionary< + TranslationEngineType, + int + >(buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => 0)); + + public int GetQueueSize(TranslationEngineType engineType) + { + return _queueSizePerEngineType[engineType]; + } + + protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) + { + try + { + var buildJobService = scope.ServiceProvider.GetRequiredService(); + IReadOnlyList trainingEngines = await buildJobService.GetBuildingEnginesAsync( + BuildJobRunnerType.ClearML, + cancellationToken + ); + if (trainingEngines.Count == 0) + return; + + Dictionary tasks = new(); + Dictionary queuePositions = new(); + + foreach (TranslationEngineType engineType in _queuePerEngineType.Keys) + { + var tasksPerEngineType = ( + await _clearMLService.GetTasksByIdAsync( + trainingEngines.Select(e => e.CurrentBuild!.JobId), + cancellationToken + ) + ) + .UnionBy( + await _clearMLService.GetTasksForQueueAsync(_queuePerEngineType[engineType], cancellationToken), + t => t.Id + ) + .ToDictionary(t => t.Id); + // add new keys to dictionary + foreach (KeyValuePair kvp in tasksPerEngineType) + tasks.TryAdd(kvp.Key, kvp.Value); + + var queuePositionsPerEngineType = tasksPerEngineType + .Values.Where(t => t.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created) + .OrderBy(t => t.Created) + .Select((t, i) => (Position: i, Task: t)) + .ToDictionary(e => e.Task.Name, e => e.Position); + // add new keys to dictionary + foreach (KeyValuePair kvp in queuePositionsPerEngineType) + queuePositions.TryAdd(kvp.Key, kvp.Value); + + _queueSizePerEngineType[engineType] = queuePositionsPerEngineType.Count; + } + + var dataAccessContext = scope.ServiceProvider.GetRequiredService(); + var platformService = scope.ServiceProvider.GetRequiredService(); + var lockFactory = scope.ServiceProvider.GetRequiredService(); + foreach (TranslationEngine engine in trainingEngines) + { + if (engine.CurrentBuild is null || !tasks.TryGetValue(engine.CurrentBuild.JobId, out ClearMLTask? task)) + continue; + + if ( + engine.CurrentBuild.JobState is BuildJobState.Pending + && task.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created + ) + { + await UpdateTrainJobStatus( + platformService, + engine.CurrentBuild.BuildId, + new ProgressStatus(step: 0, percentCompleted: 0.0), + //CurrentBuild.BuildId should always equal the corresponding task.Name + queuePositions[engine.CurrentBuild.BuildId] + 1, + cancellationToken + ); + } + + if (engine.CurrentBuild.Stage == BuildStage.Train) + { + if ( + engine.CurrentBuild.JobState is BuildJobState.Pending + && task.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + bool canceled = !await TrainJobStartedAsync( + dataAccessContext, + lockFactory, + buildJobService, + platformService, + engine.EngineId, + engine.CurrentBuild.BuildId, + cancellationToken + ); + if (canceled) + continue; + } + + switch (task.Status) + { + case ClearMLTaskStatus.InProgress: + { + double? percentCompleted = null; + if (task.Runtime.TryGetValue("progress", out string? progressStr)) + percentCompleted = int.Parse(progressStr, CultureInfo.InvariantCulture) / 100.0; + task.Runtime.TryGetValue("message", out string? message); + await UpdateTrainJobStatus( + platformService, + engine.CurrentBuild.BuildId, + new ProgressStatus(task.LastIteration ?? 0, percentCompleted, message), + queueDepth: 0, + cancellationToken + ); + break; + } + + case ClearMLTaskStatus.Completed: + { + task.Runtime.TryGetValue("message", out string? message); + await UpdateTrainJobStatus( + platformService, + engine.CurrentBuild.BuildId, + new ProgressStatus(task.LastIteration ?? 0, percentCompleted: 1.0, message), + queueDepth: 0, + cancellationToken + ); + bool canceling = !await TrainJobCompletedAsync( + lockFactory, + buildJobService, + engine.EngineId, + engine.CurrentBuild.BuildId, + (int)GetMetric(task, SummaryMetric, TrainCorpusSizeVariant), + GetMetric(task, SummaryMetric, ConfidenceVariant), + engine.CurrentBuild.Options, + cancellationToken + ); + if (canceling) + { + await TrainJobCanceledAsync( + dataAccessContext, + lockFactory, + buildJobService, + platformService, + engine.EngineId, + engine.CurrentBuild.BuildId, + cancellationToken + ); + } + break; + } + + case ClearMLTaskStatus.Stopped: + { + await TrainJobCanceledAsync( + dataAccessContext, + lockFactory, + buildJobService, + platformService, + engine.EngineId, + engine.CurrentBuild.BuildId, + cancellationToken + ); + break; + } + + case ClearMLTaskStatus.Failed: + { + await TrainJobFaultedAsync( + dataAccessContext, + lockFactory, + buildJobService, + platformService, + engine.EngineId, + engine.CurrentBuild.BuildId, + $"{task.StatusReason} : {task.StatusMessage}", + cancellationToken + ); + break; + } + } + } + } + } + catch (Exception e) + { + _logger.LogError(e, "Error occurred while monitoring ClearML tasks."); + } + } + + private async Task TrainJobStartedAsync( + IDataAccessContext dataAccessContext, + IDistributedReaderWriterLockFactory lockFactory, + IBuildJobService buildJobService, + IPlatformService platformService, + string engineId, + string buildId, + CancellationToken cancellationToken = default + ) + { + bool success; + IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + success = await dataAccessContext.WithTransactionAsync( + async (ct) => + { + if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, ct)) + return false; + await platformService.BuildStartedAsync(buildId, CancellationToken.None); + return true; + }, + cancellationToken: cancellationToken + ); + } + await UpdateTrainJobStatus(platformService, buildId, new ProgressStatus(0), 0, cancellationToken); + _logger.LogInformation("Build started ({BuildId})", buildId); + return success; + } + + private async Task TrainJobCompletedAsync( + IDistributedReaderWriterLockFactory lockFactory, + IBuildJobService buildJobService, + string engineId, + string buildId, + int corpusSize, + double confidence, + string? buildOptions, + CancellationToken cancellationToken + ) + { + try + { + IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + return await buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + engineId, + buildId, + BuildStage.Postprocess, + (corpusSize, confidence), + buildOptions, + cancellationToken + ); + } + } + finally + { + _curBuildStatus.Remove(buildId); + } + } + + private async Task TrainJobFaultedAsync( + IDataAccessContext dataAccessContext, + IDistributedReaderWriterLockFactory lockFactory, + IBuildJobService buildJobService, + IPlatformService platformService, + string engineId, + string buildId, + string message, + CancellationToken cancellationToken + ) + { + try + { + IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await dataAccessContext.WithTransactionAsync( + async (ct) => + { + await platformService.BuildFaultedAsync(buildId, message, ct); + await buildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: cancellationToken + ); + } + _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message); + } + finally + { + _curBuildStatus.Remove(buildId); + } + } + + private async Task TrainJobCanceledAsync( + IDataAccessContext dataAccessContext, + IDistributedReaderWriterLockFactory lockFactory, + IBuildJobService buildJobService, + IPlatformService platformService, + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + try + { + IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await dataAccessContext.WithTransactionAsync( + async (ct) => + { + await platformService.BuildCanceledAsync(buildId, ct); + await buildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: cancellationToken + ); + } + _logger.LogInformation("Build canceled ({BuildId})", buildId); + } + finally + { + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e) + { + _logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); + } + _curBuildStatus.Remove(buildId); + } + } + + private async Task UpdateTrainJobStatus( + IPlatformService platformService, + string buildId, + ProgressStatus progressStatus, + int? queueDepth = null, + CancellationToken cancellationToken = default + ) + { + if ( + _curBuildStatus.TryGetValue(buildId, out ProgressStatus curProgressStatus) + && curProgressStatus.Equals(progressStatus) + ) + { + return; + } + await platformService.UpdateBuildStatusAsync(buildId, progressStatus, queueDepth, cancellationToken); + _curBuildStatus[buildId] = progressStatus; + } + + private static double GetMetric(ClearMLTask task, string metric, string variant) + { + if (!task.LastMetrics.TryGetValue(metric, out IReadOnlyDictionary? metricVariants)) + return 0; + + if (!metricVariants.TryGetValue(variant, out ClearMLMetricsEvent? metricEvent)) + return 0; + + return metricEvent.Value; + } + + private static string CreateMD5(string input) + { + byte[] inputBytes = Encoding.UTF8.GetBytes(input); + byte[] hashBytes = MD5.HashData(inputBytes); + + return Convert.ToHexString(hashBytes).ToLower(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs new file mode 100644 index 00000000..d3d6540c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLService.cs @@ -0,0 +1,221 @@ +namespace Serval.Machine.Shared.Services; + +public class ClearMLService( + IHttpClientFactory httpClientFactory, + IOptionsMonitor options, + IClearMLAuthenticationService clearMLAuthService, + IHostEnvironment env +) : IClearMLService +{ + private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); + private readonly IOptionsMonitor _options = options; + private readonly IHostEnvironment _env = env; + private static readonly JsonNamingPolicy JsonNamingPolicy = new SnakeCaseJsonNamingPolicy(); + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() + { + PropertyNamingPolicy = JsonNamingPolicy, + Converters = { new CustomEnumConverterFactory(JsonNamingPolicy) } + }; + + private readonly IClearMLAuthenticationService _clearMLAuthService = clearMLAuthService; + + public async Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default) + { + var body = new JsonObject + { + ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}", + ["only_fields"] = new JsonArray("id") + }; + JsonObject? result = await CallAsync("projects", "get_all", body, cancellationToken); + var projects = (JsonArray?)result?["data"]?["projects"]; + if (projects is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + if (projects.Count == 0) + return null; + return (string?)projects[0]?["id"]; + } + + public async Task CreateProjectAsync( + string name, + string? description = null, + CancellationToken cancellationToken = default + ) + { + var body = new JsonObject + { + ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}" + }; + if (description != null) + body["description"] = description; + JsonObject? result = await CallAsync("projects", "create", body, cancellationToken); + var projectId = (string?)result?["data"]?["id"]; + if (projectId is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return projectId; + } + + public async Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default) + { + var body = new JsonObject + { + ["project"] = id, + ["delete_contents"] = true, + ["force"] = true // needed if there are tasks already in that project. + }; + JsonObject? result = await CallAsync("projects", "delete", body, cancellationToken); + var deleted = (int?)result?["data"]?["deleted"]; + if (deleted is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return deleted == 1; + } + + public async Task CreateTaskAsync( + string buildId, + string projectId, + string script, + string dockerImage, + CancellationToken cancellationToken = default + ) + { + var snakeCaseEnvironment = JsonNamingPolicy.ConvertName(_env.EnvironmentName); + var body = new JsonObject + { + ["name"] = buildId, + ["project"] = projectId, + ["script"] = new JsonObject { ["diff"] = script }, + ["container"] = new JsonObject + { + ["image"] = dockerImage, + ["arguments"] = "--env ENV_FOR_DYNACONF=" + snakeCaseEnvironment, + }, + ["type"] = "training" + }; + JsonObject? result = await CallAsync("tasks", "create", body, cancellationToken); + var taskId = (string?)result?["data"]?["id"]; + if (taskId is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return taskId; + } + + public async Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default) + { + var body = new JsonObject { ["task"] = id }; + JsonObject? result = await CallAsync("tasks", "delete", body, cancellationToken); + var deleted = (bool?)result?["data"]?["deleted"]; + if (deleted is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return deleted.Value; + } + + public async Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default) + { + var body = new JsonObject { ["task"] = id, ["queue_name"] = queue }; + JsonObject? result = await CallAsync("tasks", "enqueue", body, cancellationToken); + var queued = (int?)result?["data"]?["queued"]; + if (queued is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return queued == 1; + } + + public async Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default) + { + var body = new JsonObject { ["task"] = id }; + JsonObject? result = await CallAsync("tasks", "dequeue", body, cancellationToken); + var dequeued = (int?)result?["data"]?["dequeued"]; + if (dequeued is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return dequeued == 1; + } + + public async Task StopTaskAsync(string id, CancellationToken cancellationToken = default) + { + var body = new JsonObject { ["task"] = id, ["force"] = true }; + JsonObject? result = await CallAsync("tasks", "stop", body, cancellationToken); + var updated = (int?)result?["data"]?["updated"]; + if (updated is null) + throw new InvalidOperationException("Malformed response from ClearML server."); + return updated == 1; + } + + public async Task> GetTasksForQueueAsync( + string queue, + CancellationToken cancellationToken = default + ) + { + var body = new JsonObject { ["name"] = queue }; + JsonObject? result = await CallAsync("queues", "get_all_ex", body, cancellationToken); + var tasks = (JsonArray?)result?["data"]?["queues"]?[0]?["entries"]; + IEnumerable taskIds = tasks?.Select(t => (string)t?["id"]!) ?? new List(); + return await GetTasksByIdAsync(taskIds, cancellationToken); + } + + public async Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default) + { + IReadOnlyList tasks = await GetTasksAsync(new JsonObject { ["name"] = name }, cancellationToken); + if (tasks.Count == 0) + return null; + return tasks[0]; + } + + public Task> GetTasksByIdAsync( + IEnumerable ids, + CancellationToken cancellationToken = default + ) + { + return GetTasksAsync(new JsonObject { ["id"] = JsonValue.Create(ids.ToArray()) }, cancellationToken); + } + + private async Task> GetTasksAsync( + JsonObject body, + CancellationToken cancellationToken = default + ) + { + body["only_fields"] = new JsonArray( + "id", + "name", + "status", + "project", + "last_iteration", + "status_reason", + "status_message", + "created", + "active_duration", + "last_metrics", + "runtime" + ); + JsonObject? result = await CallAsync("tasks", "get_all_ex", body, cancellationToken); + var tasks = (JsonArray?)result?["data"]?["tasks"]; + return tasks?.Select(t => t.Deserialize(JsonSerializerOptions)!).ToArray() + ?? Array.Empty(); + } + + private async Task CallAsync( + string service, + string action, + JsonNode body, + CancellationToken cancellationToken = default + ) + { + var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") + { + Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") + }; + request.Headers.Add( + "Authorization", + $"Bearer {await _clearMLAuthService.GetAuthTokenAsync(cancellationToken)}" + ); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + string result = await response.Content.ReadAsStringAsync(cancellationToken); + return (JsonObject?)JsonNode.Parse(result); + } + + private class SnakeCaseJsonNamingPolicy : JsonNamingPolicy + { + public override string ConvertName(string name) + { + return string.Concat(name.Select((x, i) => i > 0 && char.IsUpper(x) ? "_" + x.ToString() : x.ToString())) + .ToLowerInvariant(); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs new file mode 100644 index 00000000..17d562ad --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/CorpusService.cs @@ -0,0 +1,51 @@ +namespace Serval.Machine.Shared.Services; + +public class CorpusService : ICorpusService +{ + public IEnumerable CreateTextCorpora(IReadOnlyList files) + { + List corpora = []; + + List> textFileCorpora = []; + foreach (CorpusFile file in files) + { + switch (file.Format) + { + case FileFormat.Text: + // if there are multiple texts with the same id, then add it to a new corpus or the first + // corpus that doesn't contain a text with that id + Dictionary? corpus = textFileCorpora.FirstOrDefault(c => + !c.ContainsKey(file.TextId) + ); + if (corpus is null) + { + corpus = []; + textFileCorpora.Add(corpus); + } + corpus[file.TextId] = new TextFileText(file.TextId, file.Location); + break; + + case FileFormat.Paratext: + corpora.Add(new ParatextBackupTextCorpus(file.Location, includeAllText: true)); + break; + } + } + foreach (Dictionary corpus in textFileCorpora) + corpora.Add(new DictionaryTextCorpus(corpus.Values)); + + return corpora; + } + + public IEnumerable CreateTermCorpora(IReadOnlyList files) + { + foreach (CorpusFile file in files) + { + switch (file.Format) + { + case FileFormat.Paratext: + yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); + break; + } + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs new file mode 100644 index 00000000..7ea8679f --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLock.cs @@ -0,0 +1,177 @@ +namespace Serval.Machine.Shared.Services; + +public class DistributedReaderWriterLock(string hostId, IRepository locks, IIdGenerator idGenerator, string id) + : IDistributedReaderWriterLock +{ + private readonly string _hostId = hostId; + private readonly IRepository _locks = locks; + private readonly IIdGenerator _idGenerator = idGenerator; + private readonly string _id = id; + + public async Task ReaderLockAsync( + TimeSpan? lifetime = default, + CancellationToken cancellationToken = default + ) + { + string lockId = _idGenerator.GenerateId(); + if (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)) + { + using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); + do + { + RWLock? rwLock = sub.Change.Entity; + if (rwLock is not null && !rwLock.IsAvailableForReading()) + { + TimeSpan? timeout = default; + if (rwLock.WriterLock?.ExpiresAt is not null) + { + timeout = rwLock.WriterLock.ExpiresAt - DateTime.UtcNow; + if (timeout < TimeSpan.Zero) + timeout = TimeSpan.Zero; + } + if (timeout != TimeSpan.Zero) + await sub.WaitForChangeAsync(timeout, cancellationToken); + } + } while (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)); + } + return new ReaderLockReleaser(this, lockId); + } + + public async Task WriterLockAsync( + TimeSpan? lifetime = default, + CancellationToken cancellationToken = default + ) + { + string lockId = _idGenerator.GenerateId(); + if (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)) + { + await _locks.UpdateAsync( + _id, + u => u.Add(rwl => rwl.WriterQueue, new Lock { Id = lockId, HostId = _hostId }), + cancellationToken: cancellationToken + ); + try + { + using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); + do + { + RWLock? rwLock = sub.Change.Entity; + if (rwLock is not null && !rwLock.IsAvailableForWriting(lockId)) + { + var dateTimes = rwLock + .ReaderLocks.Where(l => l.ExpiresAt.HasValue) + .Select(l => l.ExpiresAt.GetValueOrDefault()) + .ToList(); + if (rwLock.WriterLock?.ExpiresAt is not null) + dateTimes.Add(rwLock.WriterLock.ExpiresAt.Value); + TimeSpan? timeout = default; + if (dateTimes.Count > 0) + { + timeout = dateTimes.Max() - DateTime.UtcNow; + if (timeout < TimeSpan.Zero) + timeout = TimeSpan.Zero; + } + if (timeout != TimeSpan.Zero) + await sub.WaitForChangeAsync(timeout, cancellationToken); + } + } while (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)); + } + catch + { + await _locks.UpdateAsync( + _id, + u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId), + cancellationToken: cancellationToken + ); + throw; + } + } + return new WriterLockReleaser(this, lockId); + } + + private async Task TryAcquireWriterLock( + string lockId, + TimeSpan? lifetime, + CancellationToken cancellationToken + ) + { + var now = DateTime.UtcNow; + Expression> filter = rwl => + rwl.Id == _id + && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) + && !rwl.ReaderLocks.Any(l => l.ExpiresAt == null || l.ExpiresAt > now) + && (!rwl.WriterQueue.Any() || rwl.WriterQueue[0].Id == lockId); + void Update(IUpdateBuilder u) + { + u.Set( + rwl => rwl.WriterLock, + new Lock + { + Id = lockId, + ExpiresAt = lifetime is null ? null : now + lifetime, + HostId = _hostId + } + ); + u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId); + } + RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); + return rwLock is not null; + } + + private async Task TryAcquireReaderLock( + string lockId, + TimeSpan? lifetime, + CancellationToken cancellationToken + ) + { + var now = DateTime.UtcNow; + Expression> filter = rwl => + rwl.Id == _id + && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) + && !rwl.WriterQueue.Any(); + void Update(IUpdateBuilder u) + { + u.Add( + rwl => rwl.ReaderLocks, + new Lock + { + Id = lockId, + ExpiresAt = lifetime is null ? null : now + lifetime, + HostId = _hostId + } + ); + } + + RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); + return rwLock is not null; + } + + private class WriterLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase + { + private readonly DistributedReaderWriterLock _distributedLock = distributedLock; + private readonly string _lockId = lockId; + + protected override async ValueTask DisposeAsyncCore() + { + Expression> filter = rwl => + rwl.Id == _distributedLock._id && rwl.WriterLock != null && rwl.WriterLock.Id == _lockId; + await _distributedLock._locks.UpdateAsync(filter, u => u.Unset(rwl => rwl.WriterLock)); + } + } + + private class ReaderLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase + { + private readonly DistributedReaderWriterLock _distributedLock = distributedLock; + private readonly string _lockId = lockId; + + protected override async ValueTask DisposeAsyncCore() + { + Expression> filter = rwl => + rwl.Id == _distributedLock._id && rwl.ReaderLocks.Any(l => l.Id == _lockId); + await _distributedLock._locks.UpdateAsync( + filter, + u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.Id == _lockId) + ); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs new file mode 100644 index 00000000..81810fb1 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/DistributedReaderWriterLockFactory.cs @@ -0,0 +1,77 @@ +namespace Serval.Machine.Shared.Services; + +public class DistributedReaderWriterLockFactory( + IOptions serviceOptions, + IRepository locks, + IIdGenerator idGenerator +) : IDistributedReaderWriterLockFactory +{ + private readonly ServiceOptions _serviceOptions = serviceOptions.Value; + private readonly IIdGenerator _idGenerator = idGenerator; + private readonly IRepository _locks = locks; + + public async Task InitAsync(CancellationToken cancellationToken = default) + { + await RemoveAllWaitersAsync(cancellationToken); + await ReleaseAllWriterLocksAsync(cancellationToken); + await ReleaseAllReaderLocksAsync(cancellationToken); + } + + public async Task CreateAsync( + string id, + CancellationToken cancellationToken = default + ) + { + try + { + await _locks.InsertAsync( + new RWLock + { + Id = id, + ReaderLocks = [], + WriterQueue = [] + }, + cancellationToken + ); + } + catch (DuplicateKeyException) + { + // the lock is already made - no new one needs to be made + // This is done instead of checking if it exists first to prevent race conditions. + } + return new DistributedReaderWriterLock(_serviceOptions.ServiceId, _locks, _idGenerator, id); + } + + public async Task DeleteAsync(string id, CancellationToken cancellationToken = default) + { + RWLock? rwLock = await _locks.DeleteAsync(rwl => rwl.Id == id, cancellationToken); + return rwLock is not null; + } + + private async Task ReleaseAllWriterLocksAsync(CancellationToken cancellationToken) + { + await _locks.UpdateAllAsync( + rwl => rwl.WriterLock != null && rwl.WriterLock.HostId == _serviceOptions.ServiceId, + u => u.Unset(rwl => rwl.WriterLock), + cancellationToken + ); + } + + private async Task ReleaseAllReaderLocksAsync(CancellationToken cancellationToken) + { + await _locks.UpdateAllAsync( + rwl => rwl.ReaderLocks.Any(l => l.HostId == _serviceOptions.ServiceId), + u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.HostId == _serviceOptions.ServiceId), + cancellationToken + ); + } + + private async Task RemoveAllWaitersAsync(CancellationToken cancellationToken) + { + await _locks.UpdateAllAsync( + rwl => rwl.WriterQueue.Any(l => l.HostId == _serviceOptions.ServiceId), + u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.HostId == _serviceOptions.ServiceId), + cancellationToken + ); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs b/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs new file mode 100644 index 00000000..78a3ceb2 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/FileSystem.cs @@ -0,0 +1,25 @@ +namespace Serval.Machine.Shared.Services; + +public class FileSystem : IFileSystem +{ + public void CreateDirectory(string path) + { + Directory.CreateDirectory(path); + } + + public void DeleteFile(string path) + { + if (File.Exists(path)) + File.Delete(path); + } + + public Stream OpenWrite(string path) + { + return File.OpenWrite(path); + } + + public Stream OpenRead(string path) + { + return File.OpenRead(path); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs new file mode 100644 index 00000000..26fe58ed --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJob.cs @@ -0,0 +1,180 @@ +namespace Serval.Machine.Shared.Services; + +public abstract class HangfireBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger +) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) +{ + public virtual Task RunAsync( + string engineId, + string buildId, + string? buildOptions, + CancellationToken cancellationToken + ) + { + return RunAsync(engineId, buildId, null, buildOptions, cancellationToken); + } +} + +public abstract class HangfireBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger> logger +) +{ + protected IPlatformService PlatformService { get; } = platformService; + protected IRepository Engines { get; } = engines; + protected IDistributedReaderWriterLockFactory LockFactory { get; } = lockFactory; + protected IDataAccessContext DataAccessContext { get; } = dataAccessContext; + protected IBuildJobService BuildJobService { get; } = buildJobService; + protected ILogger> Logger { get; } = logger; + + public virtual async Task RunAsync( + string engineId, + string buildId, + T data, + string? buildOptions, + CancellationToken cancellationToken + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + JobCompletionStatus completionStatus = JobCompletionStatus.Completed; + try + { + await InitializeAsync(engineId, buildId, data, @lock, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + if (!await BuildJobService.BuildJobStartedAsync(engineId, buildId, cancellationToken)) + { + completionStatus = JobCompletionStatus.Canceled; + return; + } + } + + await DoWorkAsync(engineId, buildId, data, buildOptions, @lock, cancellationToken); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await Engines.GetAsync( + e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, + CancellationToken.None + ); + if (engine?.CurrentBuild?.JobState is BuildJobState.Canceling) + { + completionStatus = JobCompletionStatus.Canceled; + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildCanceledAsync(buildId, CancellationToken.None); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None + ); + } + Logger.LogInformation("Build canceled ({0})", buildId); + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + completionStatus = JobCompletionStatus.Restarting; + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildRestartingAsync(buildId, CancellationToken.None); + await BuildJobService.BuildJobRestartingAsync(engineId, buildId, CancellationToken.None); + }, + cancellationToken: CancellationToken.None + ); + } + throw; + } + else + { + completionStatus = JobCompletionStatus.Canceled; + } + } + catch (Exception e) + { + completionStatus = JobCompletionStatus.Faulted; + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: false, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None + ); + } + Logger.LogError(0, e, "Build faulted ({0})", buildId); + throw; + } + finally + { + await CleanupAsync(engineId, buildId, data, @lock, completionStatus); + } + } + + protected virtual Task InitializeAsync( + string engineId, + string buildId, + T data, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + return Task.CompletedTask; + } + + protected abstract Task DoWorkAsync( + string engineId, + string buildId, + T data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ); + + protected virtual Task CleanupAsync( + string engineId, + string buildId, + T data, + IDistributedReaderWriterLock @lock, + JobCompletionStatus completionStatus + ) + { + return Task.CompletedTask; + } + + protected enum JobCompletionStatus + { + Completed, + Faulted, + Canceled, + Restarting + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs new file mode 100644 index 00000000..d5be7f30 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/HangfireBuildJobRunner.cs @@ -0,0 +1,85 @@ +namespace Serval.Machine.Shared.Services; + +public class HangfireBuildJobRunner( + IBackgroundJobClient jobClient, + IEnumerable buildJobFactories +) : IBuildJobRunner +{ + public static Job CreateJob( + string engineId, + string buildId, + string queue, + object? data, + string? buildOptions + ) + where TJob : HangfireBuildJob + { + ArgumentNullException.ThrowIfNull(data); + // Token "None" is used here because hangfire injects the proper cancellation token + return Job.FromExpression( + j => j.RunAsync(engineId, buildId, (TData)data, buildOptions, CancellationToken.None), + queue + ); + } + + public static Job CreateJob(string engineId, string buildId, string queue, string? buildOptions) + where TJob : HangfireBuildJob + { + // Token "None" is used here because hangfire injects the proper cancellation token + return Job.FromExpression( + j => j.RunAsync(engineId, buildId, buildOptions, CancellationToken.None), + queue + ); + } + + private readonly IBackgroundJobClient _jobClient = jobClient; + private readonly Dictionary _buildJobFactories = + buildJobFactories.ToDictionary(f => f.EngineType); + + public BuildJobRunnerType Type => BuildJobRunnerType.Hangfire; + + public Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task CreateJobAsync( + TranslationEngineType engineType, + string engineId, + string buildId, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ) + { + IHangfireBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; + Job job = buildJobFactory.CreateJob(engineId, buildId, stage, data, buildOptions); + return Task.FromResult(_jobClient.Create(job, new ScheduledState(TimeSpan.FromDays(10000)))); + } + + public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) + { + return Task.FromResult(_jobClient.Delete(jobId)); + } + + public Task EnqueueJobAsync( + string jobId, + TranslationEngineType engineType, + CancellationToken cancellationToken = default + ) + { + return Task.FromResult(_jobClient.Requeue(jobId)); + } + + public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) + { + // Trigger the cancellation token for the job + return Task.FromResult(_jobClient.Delete(jobId)); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs new file mode 100644 index 00000000..c3c14751 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/HangfireHealthCheck.cs @@ -0,0 +1,25 @@ +namespace Serval.Machine.Shared.Services; + +public class HangfireHealthCheck(JobStorage jobStorage, IOptions options) : IHealthCheck +{ + private readonly JobStorage _jobStorage = jobStorage; + private readonly IOptions _options = options; + + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default + ) + { + if ( + _jobStorage + .GetMonitoringApi() + .Servers() + .Any(s => DateTime.UtcNow - s.Heartbeat < _options.Value.ServerTimeout) + ) + { + return Task.FromResult(HealthCheckResult.Healthy()); + } + + return Task.FromResult(HealthCheckResult.Unhealthy("There are no Hangfire servers running.")); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs new file mode 100644 index 00000000..6f6d3696 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs @@ -0,0 +1,29 @@ +namespace Serval.Machine.Shared.Services; + +public interface IBuildJobRunner +{ + BuildJobRunnerType Type { get; } + + Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); + Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); + + Task CreateJobAsync( + TranslationEngineType engineType, + string engineId, + string buildId, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ); + + Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default); + + Task EnqueueJobAsync( + string jobId, + TranslationEngineType engineType, + CancellationToken cancellationToken = default + ); + + Task StopJobAsync(string jobId, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs new file mode 100644 index 00000000..c9ddf983 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs @@ -0,0 +1,41 @@ +namespace Serval.Machine.Shared.Services; + +public interface IBuildJobService +{ + Task> GetBuildingEnginesAsync( + BuildJobRunnerType runner, + CancellationToken cancellationToken = default + ); + + Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default); + + Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); + + Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); + + Task StartBuildJobAsync( + BuildJobRunnerType jobType, + string engineId, + string buildId, + BuildStage stage, + object? data = default, + string? buildOptions = default, + CancellationToken cancellationToken = default + ); + + Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( + string engineId, + CancellationToken cancellationToken = default + ); + + Task BuildJobStartedAsync(string engineId, string buildId, CancellationToken cancellationToken = default); + + Task BuildJobFinishedAsync( + string engineId, + string buildId, + bool buildComplete, + CancellationToken cancellationToken = default + ); + + Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs new file mode 100644 index 00000000..4cacec2c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLAuthenticationService.cs @@ -0,0 +1,6 @@ +namespace Serval.Machine.Shared.Services; + +public interface IClearMLAuthenticationService : IHostedService +{ + public Task GetAuthTokenAsync(CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs new file mode 100644 index 00000000..bb5afc57 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs @@ -0,0 +1,16 @@ +namespace Serval.Machine.Shared.Services; + +public interface IClearMLBuildJobFactory +{ + TranslationEngineType EngineType { get; } + + Task CreateJobScriptAsync( + string engineId, + string buildId, + string modelType, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs new file mode 100644 index 00000000..1e2425a4 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLQueueService.cs @@ -0,0 +1,6 @@ +namespace Serval.Machine.Shared.Services; + +public interface IClearMLQueueService +{ + public int GetQueueSize(TranslationEngineType engineType); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs new file mode 100644 index 00000000..75f8be96 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLService.cs @@ -0,0 +1,30 @@ +namespace Serval.Machine.Shared.Services; + +public interface IClearMLService +{ + Task CreateProjectAsync( + string name, + string? description = null, + CancellationToken cancellationToken = default + ); + Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default); + Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default); + + Task CreateTaskAsync( + string buildId, + string projectId, + string script, + string dockerImage, + CancellationToken cancellationToken = default + ); + Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default); + Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default); + Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default); + Task StopTaskAsync(string id, CancellationToken cancellationToken = default); + Task> GetTasksForQueueAsync(string queue, CancellationToken cancellationToken = default); + Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default); + Task> GetTasksByIdAsync( + IEnumerable ids, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs new file mode 100644 index 00000000..bbcc9de3 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ICorpusService.cs @@ -0,0 +1,7 @@ +namespace Serval.Machine.Shared.Services; + +public interface ICorpusService +{ + IEnumerable CreateTextCorpora(IReadOnlyList files); + IEnumerable CreateTermCorpora(IReadOnlyList files); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs new file mode 100644 index 00000000..026aff28 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLock.cs @@ -0,0 +1,7 @@ +namespace Serval.Machine.Shared.Services; + +public interface IDistributedReaderWriterLock +{ + Task ReaderLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); + Task WriterLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs new file mode 100644 index 00000000..93e26c62 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IDistributedReaderWriterLockFactory.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Services; + +public interface IDistributedReaderWriterLockFactory +{ + Task InitAsync(CancellationToken cancellationToken = default); + Task CreateAsync(string id, CancellationToken cancellationToken = default); + Task DeleteAsync(string id, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs new file mode 100644 index 00000000..7df25380 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IFileStorage.cs @@ -0,0 +1,20 @@ +namespace Serval.Machine.Shared.Services; + +public interface IFileStorage : IDisposable +{ + Task ExistsAsync(string path, CancellationToken cancellationToken = default); + + Task> ListFilesAsync( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ); + + Task OpenReadAsync(string path, CancellationToken cancellationToken = default); + + Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); + + Task GetDownloadUrlAsync(string path, DateTime expiresAt, CancellationToken cancellationToken = default); + + Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs b/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs new file mode 100644 index 00000000..fa5c8f6c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IFileSystem.cs @@ -0,0 +1,9 @@ +namespace Serval.Machine.Shared.Services; + +public interface IFileSystem +{ + void DeleteFile(string path); + void CreateDirectory(string path); + Stream OpenWrite(string path); + Stream OpenRead(string path); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs new file mode 100644 index 00000000..faabcfec --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IHangfireBuildJobFactory.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Services; + +public interface IHangfireBuildJobFactory +{ + TranslationEngineType EngineType { get; } + + Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs new file mode 100644 index 00000000..761a3898 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ILanguageTagService.cs @@ -0,0 +1,6 @@ +namespace Serval.Machine.Shared.Services; + +public interface ILanguageTagService +{ + bool ConvertToFlores200Code(string languageTag, out string flores200Code); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs new file mode 100644 index 00000000..d9791ec8 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IMessageOutboxService.cs @@ -0,0 +1,13 @@ +namespace Serval.Machine.Shared.Services; + +public interface IMessageOutboxService +{ + public Task EnqueueMessageAsync( + string outboxId, + string method, + string groupId, + string? content = null, + Stream? contentStream = null, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs new file mode 100644 index 00000000..014ab591 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IOutboxMessageHandler.cs @@ -0,0 +1,13 @@ +namespace Serval.Machine.Shared.Services; + +public interface IOutboxMessageHandler +{ + public string OutboxId { get; } + + public Task HandleMessageAsync( + string method, + string? content, + Stream? contentStream, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs new file mode 100644 index 00000000..79b30f6b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs @@ -0,0 +1,30 @@ +namespace Serval.Machine.Shared.Services; + +public interface IPlatformService +{ + Task IncrementTrainSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); + + Task UpdateBuildStatusAsync( + string buildId, + ProgressStatus progressStatus, + int? queueDepth = null, + CancellationToken cancellationToken = default + ); + Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default); + Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildCompletedAsync( + string buildId, + int trainSize, + double confidence, + CancellationToken cancellationToken = default + ); + Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default); + Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); + Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); + + Task InsertPretranslationsAsync( + string engineId, + Stream pretranslationsStream, + CancellationToken cancellationToken = default + ); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs new file mode 100644 index 00000000..e8811f09 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ISharedFileService.cs @@ -0,0 +1,24 @@ +namespace Serval.Machine.Shared.Services; + +public interface ISharedFileService +{ + Uri GetBaseUri(); + + Uri GetResolvedUri(string path); + + Task GetDownloadUrlAsync(string path, DateTime expiresAt); + + Task> ListFilesAsync( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ); + + Task OpenReadAsync(string path, CancellationToken cancellationToken = default); + + Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); + + Task ExistsAsync(string path, CancellationToken cancellationToken = default); + + Task DeleteAsync(string path, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs new file mode 100644 index 00000000..6612e11e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ISmtModelFactory.cs @@ -0,0 +1,22 @@ +namespace Serval.Machine.Shared.Services; + +public interface ISmtModelFactory +{ + Task CreateAsync( + string engineDir, + IRangeTokenizer tokenizer, + IDetokenizer detokenizer, + ITruecaser truecaser, + CancellationToken cancellationToken = default + ); + Task CreateTrainerAsync( + string engineDir, + IRangeTokenizer tokenizer, + IParallelTextCorpus corpus, + CancellationToken cancellationToken = default + ); + Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); + Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); + Task UpdateEngineFromAsync(string engineDir, Stream source, CancellationToken cancellationToken = default); + Task SaveEngineToAsync(string engineDir, Stream destination, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs new file mode 100644 index 00000000..c76b8e91 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ITransferEngineFactory.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Services; + +public interface ITransferEngineFactory +{ + Task CreateAsync( + string engineDir, + IRangeTokenizer tokenizer, + IDetokenizer detokenizer, + ITruecaser truecaser, + CancellationToken cancellationToken = default + ); + Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); + Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs new file mode 100644 index 00000000..71ed5d94 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ITranslationEngineService.cs @@ -0,0 +1,49 @@ +namespace Serval.Machine.Shared.Services; + +public interface ITranslationEngineService +{ + TranslationEngineType Type { get; } + + Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + bool? isModelPersisted = null, + CancellationToken cancellationToken = default + ); + Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); + + Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ); + + Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); + + Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ); + + Task StartBuildAsync( + string engineId, + string buildId, + string? buildOptions, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ); + + Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); + + Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); + + int GetQueueSize(); + + bool IsLanguageNativeToModel(string language, out string internalCode); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs new file mode 100644 index 00000000..e83337d3 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ITruecaserFactory.cs @@ -0,0 +1,13 @@ +namespace Serval.Machine.Shared.Services; + +public interface ITruecaserFactory +{ + Task CreateAsync(string engineDir, CancellationToken cancellationToken = default); + Task CreateTrainerAsync( + string engineDir, + ITokenizer tokenizer, + ITextCorpus corpus, + CancellationToken cancellationToken = default + ); + Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs new file mode 100644 index 00000000..998144fe --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/InMemoryStorage.cs @@ -0,0 +1,143 @@ +using SIL.ObjectModel; +using static Serval.Machine.Shared.Utils.SharedFileUtils; + +namespace Serval.Machine.Shared.Services; + +public class InMemoryStorage : DisposableBase, IFileStorage +{ + public class Entry : Stream + { + public MemoryStream MemoryStream { get; } + public string Path { get; } + + private readonly InMemoryStorage _parent; + + public override bool CanRead => MemoryStream.CanRead; + + public override bool CanSeek => MemoryStream.CanSeek; + + public override bool CanWrite => MemoryStream.CanWrite; + + public override long Length => MemoryStream.Length; + + public override long Position + { + get => MemoryStream.Position; + set => MemoryStream.Position = value; + } + + public Entry(string path, InMemoryStorage parent) + { + Path = path; + MemoryStream = new(); + _parent = parent; + } + + public Entry(Entry other) + { + Path = other.Path; + MemoryStream = other.MemoryStream; + _parent = other._parent; + } + + protected override void Dispose(bool disposing) + { + _parent._memoryStreams[Path] = new Entry(this); + base.Dispose(disposing); + } + + public override void Flush() + { + MemoryStream.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return MemoryStream.Read(buffer, offset, count); + } + + public override long Seek(long offset, SeekOrigin origin) + { + return MemoryStream.Seek(offset, origin); + } + + public override void SetLength(long value) + { + MemoryStream.SetLength(value); + } + + public override void Write(byte[] buffer, int offset, int count) + { + MemoryStream.Write(buffer, offset, count); + } + } + + private readonly ConcurrentDictionary _memoryStreams = new(); + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return Task.FromResult(_memoryStreams.TryGetValue(Normalize(path), out _)); + } + + public Task> ListFilesAsync( + string? path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + path = string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true); + if (recurse) + { + return Task.FromResult>( + _memoryStreams.Keys.Where(p => p.StartsWith(path)).ToList() + ); + } + + return Task.FromResult>( + _memoryStreams.Keys.Where(p => p.StartsWith(path) && !p[path.Length..].Contains('/')).ToList() + ); + } + + public Task GetDownloadUrlAsync( + string path, + DateTime expiresAt, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + if (!_memoryStreams.TryGetValue(Normalize(path), out Entry? ret)) + throw new FileNotFoundException($"Unable to find file {path}"); + ret.Position = 0; + return Task.FromResult(ret); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return Task.FromResult(new Entry(Normalize(path), this)); + } + + public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) + { + if (_memoryStreams.ContainsKey(Normalize(path))) + { + _memoryStreams.Remove(Normalize(path), out _); + } + else + { + IEnumerable filesToRemove = await ListFilesAsync(path, recurse, cancellationToken); + foreach (string filePath in filesToRemove) + _memoryStreams.Remove(Normalize(filePath), out _); + } + } + + protected override void DisposeManagedResources() + { + foreach (Entry stream in _memoryStreams.Values) + stream.Dispose(); + _memoryStreams.Clear(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs new file mode 100644 index 00000000..30e065f5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/LanguageTagService.cs @@ -0,0 +1,170 @@ +namespace Serval.Machine.Shared.Services; + +public class LanguageTagService : ILanguageTagService +{ + private static readonly Dictionary StandardLanguages = + new() + { + { "ar", "arb" }, + { "ms", "zsm" }, + { "lv", "lvs" }, + { "ne", "npi" }, + { "sw", "swh" }, + { "cmn", "zh" } + }; + + private static readonly Dictionary StandardScripts = new() { { "Kore", "Hang" } }; + + private readonly Dictionary _defaultScripts; + + private readonly Dictionary _flores200Languages; + + private static readonly Regex LangTagPattern = + new("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture); + + public LanguageTagService() + { + // initialize SLDR language tags to retrieve latest langtags.json file + _defaultScripts = InitializeDefaultScripts(); + _flores200Languages = InitializeFlores200Languages(); + } + + protected virtual void InitializeSldrLanguageTags() + { + Sldr.InitializeLanguageTags(); + } + + private Dictionary InitializeDefaultScripts() + { + InitializeSldrLanguageTags(); + var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); + JsonNode? json; + + if (!File.Exists(cachedAllTagsPath)) + { + using HttpClient client = new(); + using HttpResponseMessage response = client.Send( + new HttpRequestMessage( + HttpMethod.Get, + "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json" + ) + ); + response.EnsureSuccessStatusCode(); + using Stream responseStream = response.Content.ReadAsStream(); + using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create); + responseStream.CopyTo(fileStream); + } + using FileStream stream = new(cachedAllTagsPath, FileMode.Open); + json = JsonNode.Parse(stream); + + Dictionary tempDefaultScripts = new(); + foreach (JsonNode? entry in json!.AsArray()) + { + if (entry is null) + continue; + + var script = (string?)entry["script"]; + if (script is null) + continue; + + JsonNode? tags = entry["tags"]; + if (tags is not null) + { + foreach (var t in tags.AsArray().Select(v => (string?)v)) + { + if ( + t is not null + && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) + && s is null + ) + { + tempDefaultScripts[t] = script; + } + } + } + + var tag = (string?)entry["tag"]; + if (tag is not null) + tempDefaultScripts[tag] = script; + } + return tempDefaultScripts; + } + + private static Dictionary InitializeFlores200Languages() + { + var tempFlores200Languages = new Dictionary(); + using var floresStream = Assembly + .GetExecutingAssembly() + .GetManifestResourceStream("Serval.Machine.Shared.data.flores200languages.csv"); + Debug.Assert(floresStream is not null); + var reader = new StreamReader(floresStream); + var firstLine = reader.ReadLine(); + Debug.Assert(firstLine == "language, code"); + while (!reader.EndOfStream) + { + string? line = reader.ReadLine(); + if (line is null) + continue; + string[] values = line.Split(','); + tempFlores200Languages[values[1].Trim()] = values[0].Trim(); + } + return tempFlores200Languages; + } + + /** + * Converts a language tag to a Flores 200 code + * @param {string} languageTag - The language tag to convert + * @param out {string} flores200Code - The converted Flores 200 code + * @returns {bool} is the language is the Flores 200 list + */ + public bool ConvertToFlores200Code(string languageTag, out string flores200Code) + { + flores200Code = ResolveLanguageTag(languageTag); + return _flores200Languages.ContainsKey(flores200Code); + } + + private string ResolveLanguageTag(string languageTag) + { + // Try to find a pattern of {language code}_{script} + Match langTagMatch = LangTagPattern.Match(languageTag); + if (!langTagMatch.Success) + return languageTag; + string parsedLanguage = langTagMatch.Groups["language"].Value; + string languageSubtag = parsedLanguage; + string iso639_3Code = parsedLanguage; + + // Best attempt to convert language to a registered ISO 639-3 code + // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping + + // If they gave us the ISO code, revert it to the 2 character code + if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag)) + languageSubtag = tempSubtag.Code; + + // There are a few extra conversions not in SIL Writing Systems that we need to handle + if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName)) + languageSubtag = tempName; + + if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj)) + iso639_3Code = languageSubtagObj.Iso3Code; + + // Use default script unless there is one parsed out of the language tag + Group scriptGroup = langTagMatch.Groups["script"]; + string? script = null; + + if (scriptGroup.Success) + script = scriptGroup.Value; + else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2)) + script = tempScript2; + else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript)) + script = tempScript; + + // There are a few extra conversions not in SIL Writing Systems that we need to handle + if (script is not null && StandardScripts.TryGetValue(script, out string? tempScript3)) + script = tempScript3; + + if (script is not null) + return $"{iso639_3Code}_{script}"; + else + return languageTag; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs new file mode 100644 index 00000000..b666ea77 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/LocalStorage.cs @@ -0,0 +1,78 @@ +using SIL.ObjectModel; +using static Serval.Machine.Shared.Utils.SharedFileUtils; + +namespace Serval.Machine.Shared.Services; + +public class LocalStorage : DisposableBase, IFileStorage +{ + private readonly Uri _basePath; + + public LocalStorage(string basePath) + { + _basePath = new Uri(basePath); + if (!_basePath.AbsoluteUri.EndsWith("/")) + _basePath = new Uri(_basePath.AbsoluteUri + "/"); + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + Uri pathUri = new(_basePath, Normalize(path)); + return Task.FromResult(File.Exists(pathUri.LocalPath)); + } + + public Task> ListFilesAsync( + string path = "", + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + Uri pathUri = new(_basePath, Normalize(path)); + string[] files = Directory.GetFiles( + pathUri.LocalPath, + "*", + new EnumerationOptions { RecurseSubdirectories = recurse } + ); + return Task.FromResult>( + files.Select(f => _basePath.MakeRelativeUri(new Uri(f)).ToString()).ToArray() + ); + } + + public Task GetDownloadUrlAsync( + string path, + DateTime expiresAt, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + Uri pathUri = new(_basePath, Normalize(path)); + return Task.FromResult(File.OpenRead(pathUri.LocalPath)); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + Uri pathUri = new(_basePath, Normalize(path)); + Directory.CreateDirectory(Path.GetDirectoryName(pathUri.LocalPath)!); + return Task.FromResult(File.OpenWrite(pathUri.LocalPath)); + } + + public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) + { + Uri pathUri = new(_basePath, Normalize(path)); + + if (File.Exists(pathUri.LocalPath)) + { + File.Delete(pathUri.LocalPath); + } + else if (Directory.Exists(pathUri.LocalPath)) + { + foreach (string filePath in await ListFilesAsync(path, recurse, cancellationToken)) + { + await DeleteAsync(filePath, false, cancellationToken); + } + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs new file mode 100644 index 00000000..09f49fb6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxDeliveryService.cs @@ -0,0 +1,174 @@ +namespace Serval.Machine.Shared.Services; + +public class MessageOutboxDeliveryService( + IServiceProvider services, + IEnumerable outboxMessageHandlers, + IFileSystem fileSystem, + IOptionsMonitor options, + ILogger logger +) : BackgroundService +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); + + private readonly IServiceProvider _services = services; + private readonly Dictionary _outboxMessageHandlers = + outboxMessageHandlers.ToDictionary(o => o.OutboxId); + private readonly IFileSystem _fileSystem = fileSystem; + private readonly IOptionsMonitor _options = options; + private readonly ILogger _logger = logger; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + Initialize(); + using IServiceScope scope = _services.CreateScope(); + var messages = scope.ServiceProvider.GetRequiredService>(); + using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); + while (true) + { + await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); + if (stoppingToken.IsCancellationRequested) + break; + await ProcessMessagesAsync(messages, stoppingToken); + } + } + + private void Initialize() + { + _fileSystem.CreateDirectory(_options.CurrentValue.OutboxDir); + } + + internal async Task ProcessMessagesAsync( + IRepository messages, + CancellationToken cancellationToken = default + ) + { + bool anyMessages = await messages.ExistsAsync(m => true, cancellationToken); + if (!anyMessages) + return; + + IReadOnlyList curMessages = await messages.GetAllAsync(cancellationToken); + + IEnumerable> messageGroups = curMessages + .OrderBy(m => m.Index) + .GroupBy(m => (m.OutboxRef, m.GroupId)); + + foreach (IGrouping<(string OutboxId, string GroupId), OutboxMessage> messageGroup in messageGroups) + { + bool abortMessageGroup = false; + IOutboxMessageHandler outboxMessageHandler = _outboxMessageHandlers[messageGroup.Key.OutboxId]; + foreach (OutboxMessage message in messageGroup) + { + try + { + await ProcessGroupMessagesAsync(messages, message, outboxMessageHandler, cancellationToken); + } + catch (RpcException e) + { + switch (e.StatusCode) + { + case StatusCode.Unavailable: + case StatusCode.Unauthenticated: + case StatusCode.PermissionDenied: + case StatusCode.Cancelled: + _logger.LogWarning(e, "Platform Message sending failure: {statusCode}", e.StatusCode); + return; + case StatusCode.Aborted: + case StatusCode.DeadlineExceeded: + case StatusCode.Internal: + case StatusCode.ResourceExhausted: + case StatusCode.Unknown: + abortMessageGroup = !await CheckIfFinalMessageAttempt(messages, message, e); + break; + case StatusCode.InvalidArgument: + default: + // log error + await PermanentlyFailedMessage(messages, message, e); + break; + } + } + catch (Exception e) + { + await PermanentlyFailedMessage(messages, message, e); + break; + } + if (abortMessageGroup) + break; + } + } + } + + private async Task ProcessGroupMessagesAsync( + IRepository messages, + OutboxMessage message, + IOutboxMessageHandler outboxMessageHandler, + CancellationToken cancellationToken = default + ) + { + Stream? contentStream = null; + string filePath = Path.Combine(_options.CurrentValue.OutboxDir, message.Id); + if (message.HasContentStream) + contentStream = _fileSystem.OpenRead(filePath); + try + { + await outboxMessageHandler.HandleMessageAsync( + message.Method, + message.Content, + contentStream, + cancellationToken + ); + await messages.DeleteAsync(message.Id, CancellationToken.None); + } + finally + { + contentStream?.Dispose(); + } + _fileSystem.DeleteFile(filePath); + } + + private async Task CheckIfFinalMessageAttempt( + IRepository messages, + OutboxMessage message, + Exception e + ) + { + if (message.Created < DateTimeOffset.UtcNow.Subtract(_options.CurrentValue.MessageExpirationTimeout)) + { + await PermanentlyFailedMessage(messages, message, e); + return true; + } + else + { + await LogFailedAttempt(messages, message, e); + return false; + } + } + + private async Task PermanentlyFailedMessage(IRepository messages, OutboxMessage message, Exception e) + { + // log error + _logger.LogError( + e, + "Permanently failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", + message.Id, + message.Method, + message.Content, + e.Message + ); + await messages.DeleteAsync(message.Id); + } + + private async Task LogFailedAttempt(IRepository messages, OutboxMessage message, Exception e) + { + // log error + await messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); + _logger.LogError( + e, + "Attempt {Attempts}. Failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", + message.Attempts + 1, + message.Id, + message.Method, + message.Content, + e.Message + ); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs new file mode 100644 index 00000000..2c5d410d --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/MessageOutboxService.cs @@ -0,0 +1,73 @@ +namespace Serval.Machine.Shared.Services; + +public class MessageOutboxService( + IRepository outboxes, + IRepository messages, + IIdGenerator idGenerator, + IFileSystem fileSystem, + IOptionsMonitor options +) : IMessageOutboxService +{ + private readonly IRepository _outboxes = outboxes; + private readonly IRepository _messages = messages; + private readonly IIdGenerator _idGenerator = idGenerator; + private readonly IFileSystem _fileSystem = fileSystem; + private readonly IOptionsMonitor _options = options; + internal int MaxDocumentSize { get; set; } = 1_000_000; + + public async Task EnqueueMessageAsync( + string outboxId, + string method, + string groupId, + string? content = null, + Stream? contentStream = null, + CancellationToken cancellationToken = default + ) + { + if (content == null && contentStream == null) + throw new ArgumentException("Either content or contentStream must be specified."); + if (content is not null && content.Length > MaxDocumentSize) + { + throw new ArgumentException( + $"The content is too large for request {method} with group ID {groupId}. " + + $"It is {content.Length} bytes, but the maximum is {MaxDocumentSize} bytes." + ); + } + Outbox outbox = ( + await _outboxes.UpdateAsync( + outboxId, + u => u.Inc(o => o.CurrentIndex, 1), + upsert: true, + cancellationToken: cancellationToken + ) + )!; + OutboxMessage outboxMessage = + new() + { + Id = _idGenerator.GenerateId(), + Index = outbox.CurrentIndex, + OutboxRef = outboxId, + Method = method, + GroupId = groupId, + Content = content, + HasContentStream = contentStream is not null + }; + string filePath = Path.Combine(_options.CurrentValue.OutboxDir, outboxMessage.Id); + try + { + if (contentStream is not null) + { + await using Stream fileStream = _fileSystem.OpenWrite(filePath); + await contentStream.CopyToAsync(fileStream, cancellationToken); + } + await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); + return outboxMessage.Id; + } + catch + { + if (contentStream is not null) + _fileSystem.DeleteFile(filePath); + throw; + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs new file mode 100644 index 00000000..92b38d6a --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ModelCleanupService.cs @@ -0,0 +1,59 @@ +namespace Serval.Machine.Shared.Services; + +public class ModelCleanupService( + IServiceProvider services, + ISharedFileService sharedFileService, + ILogger logger +) : RecurrentTask("Model Cleanup Service", services, RefreshPeriod, logger) +{ + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly ILogger _logger = logger; + private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1); + + protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) + { + var engines = scope.ServiceProvider.GetRequiredService>(); + await CheckModelsAsync(engines, cancellationToken); + } + + internal async Task CheckModelsAsync(IRepository engines, CancellationToken cancellationToken) + { + _logger.LogInformation("Running model cleanup job"); + IReadOnlyCollection paths = await _sharedFileService.ListFilesAsync( + NmtEngineService.ModelDirectory, + cancellationToken: cancellationToken + ); + // Get all NMT engine ids from the database + IReadOnlyList? allEngines = await engines.GetAllAsync(cancellationToken: cancellationToken); + IEnumerable validNmtFilenames = allEngines + .Where(e => e.Type == TranslationEngineType.Nmt) + .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision)); + // If there is a currently running build that creates and pushes a new file, but the database has not + // updated yet, don't delete the new file. + IEnumerable validNmtFilenamesForNextBuild = allEngines + .Where(e => e.Type == TranslationEngineType.Nmt) + .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision + 1)); + + var filenameFilter = validNmtFilenames.Concat(validNmtFilenamesForNextBuild).ToHashSet(); + + foreach (string path in paths) + { + if (!filenameFilter.Contains(path)) + { + await DeleteFileAsync( + path, + $"file in S3 bucket not found in database. It may be an old rev, etc.", + cancellationToken + ); + } + } + } + + private async Task DeleteFileAsync(string path, string message, CancellationToken cancellationToken = default) + { + // This may delete a file while it is being downloaded, but the chance is rare + // enough and the solution easy enough (just download again) to just live with it. + _logger.LogInformation("Deleting old model file {filename}: {message}", path, message); + await _sharedFileService.DeleteAsync(path, cancellationToken); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs new file mode 100644 index 00000000..4f465936 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtClearMLBuildJobFactory.cs @@ -0,0 +1,58 @@ +namespace Serval.Machine.Shared.Services; + +public class NmtClearMLBuildJobFactory( + ISharedFileService sharedFileService, + ILanguageTagService languageTagService, + IRepository engines +) : IClearMLBuildJobFactory +{ + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly ILanguageTagService _languageTagService = languageTagService; + private readonly IRepository _engines = engines; + + public TranslationEngineType EngineType => TranslationEngineType.Nmt; + + public async Task CreateJobScriptAsync( + string engineId, + string buildId, + string modelType, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ) + { + if (stage == BuildStage.Train) + { + TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException("The engine does not exist."); + + Uri sharedFileUri = _sharedFileService.GetBaseUri(); + string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); + string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); + _languageTagService.ConvertToFlores200Code(engine.SourceLanguage, out string srcLang); + _languageTagService.ConvertToFlores200Code(engine.TargetLanguage, out string trgLang); + return "from machine.jobs.build_nmt_engine import run\n" + + "args = {\n" + + $" 'model_type': '{modelType}',\n" + + $" 'engine_id': '{engineId}',\n" + + $" 'build_id': '{buildId}',\n" + + $" 'src_lang': '{srcLang}',\n" + + $" 'trg_lang': '{trgLang}',\n" + + $" 'shared_file_uri': '{baseUri}',\n" + + $" 'shared_file_folder': '{folder}',\n" + + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + // buildRevision + 1 because the build revision is incremented after the build job + // is finished successfully but the file should be saved with the new revision number + + (engine.IsModelPersisted ? $" 'save_model': '{engineId}_{engine.BuildRevision + 1}',\n" : $"") + + $" 'clearml': True\n" + + "}\n" + + "run(args)\n"; + } + else + { + throw new ArgumentException("Unknown build stage.", nameof(stage)); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs new file mode 100644 index 00000000..5a2fb912 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtEngineService.cs @@ -0,0 +1,204 @@ +namespace Serval.Machine.Shared.Services; + +public class NmtEngineService( + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IBuildJobService buildJobService, + ILanguageTagService languageTagService, + IClearMLQueueService clearMLQueueService, + ISharedFileService sharedFileService +) : ITranslationEngineService +{ + private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; + private readonly IPlatformService _platformService = platformService; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IRepository _engines = engines; + private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; + private readonly ILanguageTagService _languageTagService = languageTagService; + private readonly ISharedFileService _sharedFileService = sharedFileService; + public const string ModelDirectory = "models/"; + + public static string GetModelPath(string engineId, int buildRevision) + { + return $"{ModelDirectory}{engineId}_{buildRevision}.tar.gz"; + } + + public TranslationEngineType Type => TranslationEngineType.Nmt; + + private const int MinutesToExpire = 60; + + public async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + bool? isModelPersisted = null, + CancellationToken cancellationToken = default + ) + { + var translationEngine = await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + var translationEngine = new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage, + Type = TranslationEngineType.Nmt, + IsModelPersisted = isModelPersisted ?? false // models are not persisted if not specified + }; + await _engines.InsertAsync(translationEngine, ct); + await _buildJobService.CreateEngineAsync(engineId, engineName, ct); + return translationEngine; + }, + cancellationToken: cancellationToken + ); + return translationEngine; + } + + public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildJobAsync(engineId, cancellationToken); + + await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); + } + await _lockFactory.DeleteAsync(engineId, CancellationToken.None); + } + + public async Task StartBuildAsync( + string engineId, + string buildId, + string? buildOptions, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + // If there is a pending/running build, then no need to start a new one. + if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) + throw new InvalidOperationException("The engine is already building or in the process of canceling."); + + await _buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + engineId, + buildId, + BuildStage.Preprocess, + corpora, + buildOptions, + cancellationToken + ); + } + } + + public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + if (!await CancelBuildJobAsync(engineId, cancellationToken)) + throw new InvalidOperationException("The engine is not currently building."); + } + } + + public async Task GetModelDownloadUrlAsync( + string engineId, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.IsModelPersisted != true) + { + throw new NotSupportedException( + "The model cannot be downloaded. " + + "To enable downloading the model, recreate the engine with IsModelPersisted property to true." + ); + } + + if (engine.BuildRevision == 0) + throw new InvalidOperationException("The engine has not been built yet."); + string filepath = GetModelPath(engineId, engine.BuildRevision); + bool fileExists = await _sharedFileService.ExistsAsync(filepath, cancellationToken); + if (!fileExists) + throw new FileNotFoundException($"The model for build revision , {engine.BuildRevision}, does not exist."); + var expiresAt = DateTime.UtcNow.AddMinutes(MinutesToExpire); + var modelInfo = new ModelDownloadUrl + { + Url = await _sharedFileService.GetDownloadUrlAsync(filepath, expiresAt), + ModelRevision = engine.BuildRevision, + ExpiresAt = expiresAt + }; + return modelInfo; + } + + public Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public int GetQueueSize() + { + return _clearMLQueueService.GetQueueSize(Type); + } + + public bool IsLanguageNativeToModel(string language, out string internalCode) + { + return _languageTagService.ConvertToFlores200Code(language, out internalCode); + } + + private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) + { + string? buildId = null; + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + }, + cancellationToken: cancellationToken + ); + return buildId is not null; + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException($"The engine {engineId} does not exist."); + return engine; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs new file mode 100644 index 00000000..a8b3d52f --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtHangfireBuildJobFactory.cs @@ -0,0 +1,26 @@ +using static Serval.Machine.Shared.Services.HangfireBuildJobRunner; + +namespace Serval.Machine.Shared.Services; + +public class NmtHangfireBuildJobFactory : IHangfireBuildJobFactory +{ + public TranslationEngineType EngineType => TranslationEngineType.Nmt; + + public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) + { + return stage switch + { + BuildStage.Preprocess + => CreateJob>( + engineId, + buildId, + "nmt", + data, + buildOptions + ), + BuildStage.Postprocess + => CreateJob(engineId, buildId, "nmt", data, buildOptions), + _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), + }; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs new file mode 100644 index 00000000..b4c61648 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -0,0 +1,31 @@ +namespace Serval.Machine.Shared.Services; + +public class NmtPreprocessBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + ILogger logger, + IBuildJobService buildJobService, + ISharedFileService sharedFileService, + ICorpusService corpusService, + ILanguageTagService languageTagService +) + : PreprocessBuildJob( + platformService, + engines, + lockFactory, + dataAccessContext, + logger, + buildJobService, + sharedFileService, + corpusService + ) +{ + private readonly ILanguageTagService _languageTagService = languageTagService; + + protected override bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) + { + return _languageTagService.ConvertToFlores200Code(languageCode, out resolvedCode); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs new file mode 100644 index 00000000..25e34892 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/PostprocessBuildJob.cs @@ -0,0 +1,88 @@ +namespace Serval.Machine.Shared.Services; + +public class PostprocessBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger, + ISharedFileService sharedFileService +) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) +{ + protected ISharedFileService SharedFileService { get; } = sharedFileService; + + protected override async Task DoWorkAsync( + string engineId, + string buildId, + (int, double) data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + (int corpusSize, double confidence) = data; + + await using ( + Stream pretranslationsStream = await SharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ) + ) + { + await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken); + } + + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + int additionalCorpusSize = await SaveModelAsync(engineId, buildId); + await PlatformService.BuildCompletedAsync( + buildId, + corpusSize + additionalCorpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: true, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None + ); + } + + Logger.LogInformation("Build completed ({0}).", buildId); + } + + protected virtual Task SaveModelAsync(string engineId, string buildId) + { + return Task.FromResult(0); + } + + protected override async Task CleanupAsync( + string engineId, + string buildId, + (int, double) data, + IDistributedReaderWriterLock @lock, + JobCompletionStatus completionStatus + ) + { + if (completionStatus is JobCompletionStatus.Restarting) + return; + + try + { + if (completionStatus is not JobCompletionStatus.Faulted) + await SharedFileService.DeleteAsync($"builds/{buildId}/"); + } + catch (Exception e) + { + Logger.LogWarning(e, "Unable to to delete job data for build {0}.", buildId); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs new file mode 100644 index 00000000..214a1818 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -0,0 +1,434 @@ +namespace Serval.Machine.Shared.Services; + +public class PreprocessBuildJob : HangfireBuildJob> +{ + private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; + + internal BuildJobRunnerType TrainJobRunnerType { get; init; } = BuildJobRunnerType.ClearML; + + private readonly ISharedFileService _sharedFileService; + private readonly ICorpusService _corpusService; + private int _seed = 1234; + private Random _random; + + public PreprocessBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + ILogger logger, + IBuildJobService buildJobService, + ISharedFileService sharedFileService, + ICorpusService corpusService + ) + : base(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) + { + _sharedFileService = sharedFileService; + _corpusService = corpusService; + _random = new Random(_seed); + } + + internal int Seed + { + get => _seed; + set + { + if (_seed != value) + { + _seed = value; + _random = new Random(_seed); + } + } + } + + protected override async Task DoWorkAsync( + string engineId, + string buildId, + IReadOnlyList data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + (int trainCount, int pretranslateCount) = await WriteDataFilesAsync( + buildId, + data, + buildOptions, + cancellationToken + ); + + // Log summary of build data + JsonObject buildPreprocessSummary = + new() + { + { "Event", "BuildPreprocess" }, + { "EngineId", engineId }, + { "BuildId", buildId }, + { "NumTrainRows", trainCount }, + { "NumPretranslateRows", pretranslateCount } + }; + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); + + bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); + buildPreprocessSummary.Add("SourceLanguageResolved", srcLang); + bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); + buildPreprocessSummary.Add("TargetLanguageResolved", trgLang); + Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); + + if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) + { + throw new InvalidOperationException( + $"Neither language code in build {buildId} are known to the base model, and the data specified for training was empty. Build canceled." + ); + } + + cancellationToken.ThrowIfCancellationRequested(); + + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + bool canceling = !await BuildJobService.StartBuildJobAsync( + TrainJobRunnerType, + engineId, + buildId, + BuildStage.Train, + buildOptions: buildOptions, + cancellationToken: cancellationToken + ); + if (canceling) + throw new OperationCanceledException(); + } + } + + private async Task<(int TrainCount, int PretranslateCount)> WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + string? buildOptions, + CancellationToken cancellationToken + ) + { + JsonObject? buildOptionsObject = null; + if (buildOptions is not null) + buildOptionsObject = JsonSerializer.Deserialize(buildOptions); + await using StreamWriter sourceTrainWriter = + new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken)); + await using StreamWriter targetTrainWriter = + new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken)); + + await using Stream pretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, PretranslateWriterOptions); + + int trainCount = 0; + int pretranslateCount = 0; + pretranslateWriter.WriteStartArray(); + foreach (Corpus corpus in corpora) + { + ITextCorpus[] sourceTextCorpora = _corpusService.CreateTextCorpora(corpus.SourceFiles).ToArray(); + ITextCorpus targetTextCorpus = + _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault() ?? new DictionaryTextCorpus(); + + if (sourceTextCorpora.Length == 0) + continue; + + int skipCount = 0; + foreach (Row?[] rows in AlignTrainCorpus(sourceTextCorpora, targetTextCorpus)) + { + if (skipCount > 0) + { + skipCount--; + continue; + } + + Row[] trainRows = rows.Where(r => r is not null && IsInTrain(r, corpus)).Cast().ToArray(); + if (trainRows.Length > 0) + { + Row row = trainRows[0]; + if (rows.Length > 1) + { + Row[] nonEmptyRows = trainRows.Where(r => r.SourceSegment.Length > 0).ToArray(); + if (nonEmptyRows.Length > 0) + row = nonEmptyRows[_random.Next(nonEmptyRows.Length)]; + } + + await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); + skipCount = row.RowCount - 1; + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + trainCount++; + } + } + + if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) + { + ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); + if (sourceTermCorpus is not null && targetTermCorpus is not null) + { + IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); + foreach (ParallelTextRow row in parallelKeyTermsCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + trainCount++; + } + } + } + + foreach (Row row in AlignPretranslateCorpus(sourceTextCorpora[0], targetTextCorpus)) + { + if ( + IsInPretranslate(row, corpus) + && row.SourceSegment.Length > 0 + && (row.TargetSegment.Length == 0 || !IsInTrain(row, corpus)) + ) + { + pretranslateWriter.WriteStartObject(); + pretranslateWriter.WriteString("corpusId", corpus.Id); + pretranslateWriter.WriteString("textId", row.TextId); + pretranslateWriter.WriteStartArray("refs"); + foreach (object rowRef in row.Refs) + pretranslateWriter.WriteStringValue(rowRef.ToString()); + pretranslateWriter.WriteEndArray(); + pretranslateWriter.WriteString("translation", row.SourceSegment); + pretranslateWriter.WriteEndObject(); + pretranslateCount++; + } + } + } + + pretranslateWriter.WriteEndArray(); + + return (trainCount, pretranslateCount); + } + + protected override async Task CleanupAsync( + string engineId, + string buildId, + IReadOnlyList data, + IDistributedReaderWriterLock @lock, + JobCompletionStatus completionStatus + ) + { + if (completionStatus is JobCompletionStatus.Canceled) + { + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/"); + } + catch (Exception e) + { + Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); + } + } + } + + private static bool IsInTrain(Row row, Corpus corpus) + { + return IsIncluded(row, corpus.TrainOnTextIds, corpus.TrainOnChapters); + } + + private static bool IsInPretranslate(Row row, Corpus corpus) + { + return IsIncluded(row, corpus.PretranslateTextIds, corpus.PretranslateChapters); + } + + private static bool IsIncluded( + Row? row, + IReadOnlySet? textIds, + IReadOnlyDictionary>? chapters + ) + { + if (row is null) + return false; + if (chapters is not null) + return row.Refs.Any(r => IsInChapters(chapters, r)); + if (textIds is not null) + return textIds.Contains(row.TextId); + return true; + } + + private static bool IsInChapters(IReadOnlyDictionary> bookChapters, object rowRef) + { + if (rowRef is not ScriptureRef sr) + return false; + return bookChapters.TryGetValue(sr.Book, out HashSet? chapters) + && (chapters.Contains(sr.ChapterNum) || chapters.Count == 0); + } + + private static IEnumerable AlignTrainCorpus(IReadOnlyList srcCorpora, ITextCorpus trgCorpus) + { + if (trgCorpus.IsScripture()) + { + return srcCorpora + .Select(sc => AlignScripture(sc, trgCorpus)) + .ZipMany(rows => rows.ToArray()) + // filter out every list that only contains completely empty rows + .Where(rows => rows.Any(r => r is null || r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); + } + + IEnumerable sourceOnlyRows = srcCorpora + .Select(sc => sc.AlignRows(trgCorpus, allSourceRows: true)) + .ZipMany(rows => + rows.Where(r => r.TargetSegment.Count == 0) + .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) + .ToArray() + ); + + IEnumerable targetRows = srcCorpora + .Select(sc => sc.AlignRows(trgCorpus, allTargetRows: true)) + .ZipMany(rows => + rows.Where(r => r.TargetSegment.Count > 0) + .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) + .ToArray() + ); + + return sourceOnlyRows + .Concat(targetRows) + // filter out every list that only contains completely empty rows + .Where(rows => rows.Any(r => r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); + } + + private static IEnumerable AlignScripture(ITextCorpus srcCorpus, ITextCorpus trgCorpus) + { + int rowCount = 0; + StringBuilder srcSegBuffer = new(); + StringBuilder trgSegBuffer = new(); + HashSet vrefs = []; + foreach ( + (VerseRef vref, string srcSegment, string trgSegment) in srcCorpus + .ExtractScripture() + .Select(r => (r.CorpusVerseRef, r.Text)) + .Zip( + trgCorpus.ExtractScripture().Select(r => r.Text), + (s, t) => (VerseRef: s.CorpusVerseRef, SourceSegment: s.Text, TargetSegment: t) + ) + ) + { + if (srcSegment == "" && trgSegment == "") + { + vrefs.UnionWith(vref.AllVerses()); + rowCount++; + } + else if (srcSegment == "") + { + vrefs.UnionWith(vref.AllVerses()); + if (trgSegment.Length > 0) + { + if (trgSegBuffer.Length > 0) + trgSegBuffer.Append(' '); + trgSegBuffer.Append(trgSegment); + } + rowCount++; + } + else if (trgSegment == "") + { + vrefs.UnionWith(vref.AllVerses()); + if (srcSegment.Length > 0) + { + if (srcSegBuffer.Length > 0) + srcSegBuffer.Append(' '); + srcSegBuffer.Append(srcSegment); + } + rowCount++; + } + else + { + if (rowCount > 0) + { + yield return new( + vrefs.First().Book, + vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + rowCount + ); + for (int i = 0; i < rowCount - 1; i++) + yield return null; + srcSegBuffer.Clear(); + trgSegBuffer.Clear(); + vrefs.Clear(); + rowCount = 0; + } + vrefs.UnionWith(vref.AllVerses()); + srcSegBuffer.Append(srcSegment); + trgSegBuffer.Append(trgSegment); + rowCount++; + } + } + + if (rowCount > 0) + { + yield return new( + vrefs.First().Book, + vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), + srcSegBuffer.ToString(), + trgSegBuffer.ToString(), + rowCount + ); + for (int i = 0; i < rowCount - 1; i++) + yield return null; + } + } + + private static IEnumerable AlignPretranslateCorpus(ITextCorpus srcCorpus, ITextCorpus trgCorpus) + { + int rowCount = 0; + StringBuilder srcSegBuffer = new(); + StringBuilder trgSegBuffer = new(); + List refs = []; + string textId = ""; + foreach (ParallelTextRow row in srcCorpus.AlignRows(trgCorpus, allSourceRows: true)) + { + if (!row.IsTargetRangeStart && row.IsTargetInRange) + { + refs.AddRange(row.Refs); + if (row.SourceText.Length > 0) + { + if (srcSegBuffer.Length > 0) + srcSegBuffer.Append(' '); + srcSegBuffer.Append(row.SourceText); + } + rowCount++; + } + else + { + if (rowCount > 0) + { + yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); + textId = ""; + srcSegBuffer.Clear(); + trgSegBuffer.Clear(); + refs.Clear(); + rowCount = 0; + } + + textId = row.TextId; + refs.AddRange(row.Refs); + srcSegBuffer.Append(row.SourceText); + trgSegBuffer.Append(row.TargetText); + rowCount++; + } + } + + if (rowCount > 0) + yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); + } + + private record Row( + string TextId, + IReadOnlyList Refs, + string SourceSegment, + string TargetSegment, + int RowCount + ); + + protected virtual bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) + { + resolvedCode = languageCode; + return true; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs new file mode 100644 index 00000000..3a3cb11e --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/S3FileStorage.cs @@ -0,0 +1,125 @@ +using SIL.ObjectModel; +using static Serval.Machine.Shared.Utils.SharedFileUtils; + +namespace Serval.Machine.Shared.Services; + +public class S3FileStorage : DisposableBase, IFileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config { RegionEndpoint = RegionEndpoint.GetBySystemName(region) } + ); + + _bucketName = bucketName; + // Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level + // implementation + _basePath = Normalize(basePath, includeTrailingSlash: true); + _loggerFactory = loggerFactory; + } + + public async Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public async Task> ListFilesAsync( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + (string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true)), + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key[_basePath.Length..]).ToList(); + } + + public Task GetDownloadUrlAsync( + string path, + DateTime expiresAt, + CancellationToken cancellationToken = default + ) + { + return Task.FromResult( + _client.GetPreSignedURL( + new GetPreSignedUrlRequest + { + BucketName = _bucketName, + Key = _basePath + Normalize(path), + Expires = expiresAt, + ResponseHeaderOverrides = new ResponseHeaderOverrides + { + ContentDisposition = new ContentDisposition() { FileName = Path.GetFileName(path) }.ToString() + } + } + ) + ); + } + + public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + GetObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {path} does not exist"); + return response.ResponseStream; + } + + public async Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + string fullPath = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = fullPath }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync( + request, + cancellationToken + ); + return new BufferedStream( + new S3WriteStream(_client, fullPath, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.MaxPartSize + ); + } + + public async Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.NoContent)) + { + throw new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs new file mode 100644 index 00000000..69eedc14 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/S3HealthCheck.cs @@ -0,0 +1,65 @@ +namespace Serval.Machine.Shared.Services; + +public class S3HealthCheck(IOptions options) : IHealthCheck +{ + private readonly IOptions _options = options; + private int _numConsecutiveFailures = 0; + private readonly AsyncLock _lock = new AsyncLock(); + + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default + ) + { + try + { + var request = new ListObjectsV2Request + { + BucketName = new Uri(_options.Value.Uri).Host, + Prefix = new Uri(_options.Value.Uri).AbsolutePath + "/models/", + MaxKeys = 1, + Delimiter = "" + }; + + await new AmazonS3Client( + _options.Value.S3AccessKeyId, + _options.Value.S3SecretAccessKey, + new AmazonS3Config + { + MaxErrorRetry = 0, //Do not let health check hang + RegionEndpoint = RegionEndpoint.GetBySystemName(_options.Value.S3Region) + } + ).ListObjectsV2Async(request, cancellationToken); + using (await _lock.LockAsync(cancellationToken)) + _numConsecutiveFailures = 0; + return HealthCheckResult.Healthy("The S3 bucket is available"); + } + catch (Exception e) + { + using (await _lock.LockAsync(cancellationToken)) + { + _numConsecutiveFailures++; + if ( + e is HttpRequestException httpRequestException + && httpRequestException.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.Unauthorized + ) + { + return _numConsecutiveFailures > 3 + ? HealthCheckResult.Unhealthy( + "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." + ) + : HealthCheckResult.Degraded( + "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." + ); + } + return _numConsecutiveFailures > 3 + ? HealthCheckResult.Unhealthy( + "S3 bucket is not available. The following exception occurred: " + e.Message + ) + : HealthCheckResult.Degraded( + "S3 bucket is not available. The following exception occurred: " + e.Message + ); + } + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs b/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs new file mode 100644 index 00000000..4b623d6d --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/S3WriteStream.cs @@ -0,0 +1,232 @@ +namespace Serval.Machine.Shared.Services; + +public class S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory +) : Stream +{ + private readonly AmazonS3Client _client = client; + private readonly string _key = key; + private readonly string _uploadId = uploadId; + private readonly string _bucketName = bucketName; + private readonly List _uploadResponses = new List(); + private readonly ILogger _logger = loggerFactory.CreateLogger(); + + public const int MaxPartSize = 5 * 1024 * 1024; + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default + ) + { + try + { + using Stream stream = buffer.AsStream(); + + int bytesWritten = 0; + + while (stream.Length > bytesWritten) + { + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = stream, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug( + "Transferred {e.TransferredBytes}/{e.TotalBytes}", + e.TransferredBytes, + e.TotalBytes + ); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + + _uploadResponses.Add(response); + + bytesWritten += MaxPartSize; + } + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await WriteAsync(buffer.AsMemory(offset, count), cancellationToken); + } + + protected override void Dispose(bool disposing) + { + try + { + if (disposing) + { + if (_uploadResponses.Count == 0) + { + AbortAsync().WaitAndUnwrapException(); + PutObjectRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + ContentBody = "" + }; + PutObjectResponse response = _client.PutObjectAsync(request).WaitAndUnwrapException(); + if (response.HttpStatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException( + $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + } + else + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + if (response.HttpStatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + } + } + finally + { + base.Dispose(disposing); + } + } + + public override async ValueTask DisposeAsync() + { + try + { + if (_uploadResponses.Count == 0) + { + await AbortAsync(); + PutObjectRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + ContentBody = "" + }; + PutObjectResponse response = await _client.PutObjectAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException( + $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + + return; + } + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + } + catch (Exception e) + { + await AbortAsync(e); + } + } + finally + { + await base.DisposeAsync(); + GC.SuppressFinalize(this); + } + } + + private async Task AbortAsync(Exception? e = null) + { + if (e is not null) + _logger.LogError(e, "Aborted upload {UploadId} to {BucketName}/{Key}", _uploadId, _bucketName, _key); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs new file mode 100644 index 00000000..493cb9ed --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxConstants.cs @@ -0,0 +1,14 @@ +namespace Serval.Machine.Shared.Services; + +public static class ServalPlatformOutboxConstants +{ + public const string OutboxId = "ServalPlatform"; + + public const string BuildStarted = "BuildStarted"; + public const string BuildCompleted = "BuildCompleted"; + public const string BuildCanceled = "BuildCanceled"; + public const string BuildFaulted = "BuildFaulted"; + public const string BuildRestarting = "BuildRestarting"; + public const string InsertPretranslations = "InsertPretranslations"; + public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize"; +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs new file mode 100644 index 00000000..9b1cf788 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformOutboxMessageHandler.cs @@ -0,0 +1,92 @@ +using Serval.Translation.V1; + +namespace Serval.Machine.Shared.Services; + +public class ServalPlatformOutboxMessageHandler(TranslationPlatformApi.TranslationPlatformApiClient client) + : IOutboxMessageHandler +{ + private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + public string OutboxId => ServalPlatformOutboxConstants.OutboxId; + + public async Task HandleMessageAsync( + string method, + string? content, + Stream? contentStream, + CancellationToken cancellationToken = default + ) + { + switch (method) + { + case ServalPlatformOutboxConstants.BuildStarted: + await _client.BuildStartedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildCompleted: + await _client.BuildCompletedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildCanceled: + await _client.BuildCanceledAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildFaulted: + await _client.BuildFaultedAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.BuildRestarting: + await _client.BuildRestartingAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + case ServalPlatformOutboxConstants.InsertPretranslations: + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + contentStream!, + JsonSerializerOptions, + cancellationToken + ) + .OfType(); + + using (var call = _client.InsertPretranslations(cancellationToken: cancellationToken)) + { + await foreach (Pretranslation pretranslation in pretranslations) + { + await call.RequestStream.WriteAsync( + new InsertPretranslationRequest + { + EngineId = content!, + CorpusId = pretranslation.CorpusId, + TextId = pretranslation.TextId, + Refs = { pretranslation.Refs }, + Translation = pretranslation.Translation + }, + cancellationToken + ); + } + await call.RequestStream.CompleteAsync(); + await call; + } + break; + case ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize: + await _client.IncrementTranslationEngineCorpusSizeAsync( + JsonSerializer.Deserialize(content!), + cancellationToken: cancellationToken + ); + break; + default: + throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'."); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs new file mode 100644 index 00000000..429fbb72 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalPlatformService.cs @@ -0,0 +1,140 @@ +using Serval.Translation.V1; + +namespace Serval.Machine.Shared.Services; + +public class ServalPlatformService( + TranslationPlatformApi.TranslationPlatformApiClient client, + IMessageOutboxService outboxService +) : IPlatformService +{ + private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; + private readonly IMessageOutboxService _outboxService = outboxService; + + public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildStarted, + buildId, + JsonSerializer.Serialize(new BuildStartedRequest { BuildId = buildId }), + cancellationToken: cancellationToken + ); + } + + public async Task BuildCompletedAsync( + string buildId, + int trainSize, + double confidence, + CancellationToken cancellationToken = default + ) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildCompleted, + buildId, + JsonSerializer.Serialize( + new BuildCompletedRequest + { + BuildId = buildId, + CorpusSize = trainSize, + Confidence = confidence + } + ), + cancellationToken: cancellationToken + ); + } + + public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildCanceled, + buildId, + JsonSerializer.Serialize(new BuildCanceledRequest { BuildId = buildId }), + cancellationToken: cancellationToken + ); + } + + public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildFaulted, + buildId, + JsonSerializer.Serialize(new BuildFaultedRequest { BuildId = buildId, Message = message }), + cancellationToken: cancellationToken + ); + } + + public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.BuildRestarting, + buildId, + JsonSerializer.Serialize(new BuildRestartingRequest { BuildId = buildId }), + cancellationToken: cancellationToken + ); + } + + public async Task UpdateBuildStatusAsync( + string buildId, + ProgressStatus progressStatus, + int? queueDepth = null, + CancellationToken cancellationToken = default + ) + { + var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; + if (progressStatus.PercentCompleted.HasValue) + request.PercentCompleted = progressStatus.PercentCompleted.Value; + if (progressStatus.Message is not null) + request.Message = progressStatus.Message; + if (queueDepth is not null) + request.QueueDepth = queueDepth.Value; + + // just try to send it - if it fails, it fails. + await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); + } + + public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) + { + // just try to send it - if it fails, it fails. + await _client.UpdateBuildStatusAsync( + new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, + cancellationToken: cancellationToken + ); + } + + public async Task InsertPretranslationsAsync( + string engineId, + Stream pretranslationsStream, + CancellationToken cancellationToken = default + ) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.InsertPretranslations, + engineId, + engineId, + pretranslationsStream, + cancellationToken: cancellationToken + ); + } + + public async Task IncrementTrainSizeAsync( + string engineId, + int count = 1, + CancellationToken cancellationToken = default + ) + { + await _outboxService.EnqueueMessageAsync( + ServalPlatformOutboxConstants.OutboxId, + ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize, + engineId, + JsonSerializer.Serialize( + new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count } + ), + cancellationToken: cancellationToken + ); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs new file mode 100644 index 00000000..863fd158 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationEngineServiceV1.cs @@ -0,0 +1,341 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Translation.V1; + +namespace Serval.Machine.Shared.Services; + +public class ServalTranslationEngineServiceV1( + IEnumerable engineServices, + HealthCheckService healthCheckService +) : TranslationEngineApi.TranslationEngineApiBase +{ + private static readonly Empty Empty = new(); + + private readonly Dictionary _engineServices = + engineServices.ToDictionary(es => es.Type); + + private readonly HealthCheckService _healthCheckService = healthCheckService; + + public override async Task Create(CreateRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + TranslationEngine translationEngine = await engineService.CreateAsync( + request.EngineId, + request.HasEngineName ? request.EngineName : null, + request.SourceLanguage, + request.TargetLanguage, + request.HasIsModelPersisted ? request.IsModelPersisted : null, + context.CancellationToken + ); + return new CreateResponse { IsModelPersisted = translationEngine.IsModelPersisted }; + } + + public override async Task Delete(DeleteRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + await engineService.DeleteAsync(request.EngineId, context.CancellationToken); + return Empty; + } + + public override async Task Translate(TranslateRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + IEnumerable results; + try + { + results = await engineService.TranslateAsync( + request.EngineId, + request.N, + request.Segment, + context.CancellationToken + ); + } + catch (EngineNotBuiltException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + + return new TranslateResponse { Results = { results.Select(Map) } }; + } + + public override async Task GetWordGraph( + GetWordGraphRequest request, + ServerCallContext context + ) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + SIL.Machine.Translation.WordGraph wordGraph; + try + { + wordGraph = await engineService.GetWordGraphAsync( + request.EngineId, + request.Segment, + context.CancellationToken + ); + } + catch (EngineNotBuiltException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + return new GetWordGraphResponse { WordGraph = Map(wordGraph) }; + } + + public override async Task TrainSegmentPair(TrainSegmentPairRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + await engineService.TrainSegmentPairAsync( + request.EngineId, + request.SourceSegment, + request.TargetSegment, + request.SentenceStart, + context.CancellationToken + ); + return Empty; + } + + public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + Models.Corpus[] corpora = request.Corpora.Select(Map).ToArray(); + try + { + await engineService.StartBuildAsync( + request.EngineId, + request.BuildId, + request.HasOptions ? request.Options : null, + corpora, + context.CancellationToken + ); + } + catch (InvalidOperationException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + return Empty; + } + + public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + try + { + await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); + } + catch (InvalidOperationException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); + } + return Empty; + } + + public override async Task GetModelDownloadUrl( + GetModelDownloadUrlRequest request, + ServerCallContext context + ) + { + try + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + ModelDownloadUrl modelDownloadUrl = await engineService.GetModelDownloadUrlAsync( + request.EngineId, + context.CancellationToken + ); + return new GetModelDownloadUrlResponse + { + Url = modelDownloadUrl.Url, + ModelRevision = modelDownloadUrl.ModelRevision, + ExpiresAt = modelDownloadUrl.ExpiresAt.ToTimestamp() + }; + } + catch (InvalidOperationException e) + { + throw new RpcException(new Status(StatusCode.Aborted, e.Message)); + } + catch (FileNotFoundException e) + { + throw new RpcException(new Status(StatusCode.NotFound, e.Message)); + } + } + + public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); + } + + public override Task GetLanguageInfo( + GetLanguageInfoRequest request, + ServerCallContext context + ) + { + ITranslationEngineService engineService = GetEngineService(request.EngineType); + bool isNative = engineService.IsLanguageNativeToModel(request.Language, out string internalCode); + return Task.FromResult(new GetLanguageInfoResponse { InternalCode = internalCode, IsNative = isNative, }); + } + + public override async Task HealthCheck(Empty request, ServerCallContext context) + { + HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); + HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); + return healthCheckResponse; + } + + private ITranslationEngineService GetEngineService(string engineTypeStr) + { + if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) + return service; + throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); + } + + private static TranslationEngineType GetEngineType(string engineTypeStr) + { + engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; + if (System.Enum.TryParse(engineTypeStr, out TranslationEngineType engineType)) + return engineType; + throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); + } + + private static Translation.V1.TranslationResult Map(SIL.Machine.Translation.TranslationResult source) + { + return new Translation.V1.TranslationResult + { + Translation = source.Translation, + SourceTokens = { source.SourceTokens }, + TargetTokens = { source.TargetTokens }, + Confidences = { source.Confidences }, + Sources = { source.Sources.Select(Map) }, + Alignment = { Map(source.Alignment) }, + Phrases = { source.Phrases.Select(Map) } + }; + } + + private static Translation.V1.WordGraph Map(SIL.Machine.Translation.WordGraph source) + { + return new Translation.V1.WordGraph + { + SourceTokens = { source.SourceTokens }, + InitialStateScore = source.InitialStateScore, + FinalStates = { source.FinalStates }, + Arcs = { source.Arcs.Select(Map) } + }; + } + + private static Translation.V1.WordGraphArc Map(SIL.Machine.Translation.WordGraphArc source) + { + return new Translation.V1.WordGraphArc + { + PrevState = source.PrevState, + NextState = source.NextState, + Score = source.Score, + TargetTokens = { source.TargetTokens }, + Alignment = { Map(source.Alignment) }, + Confidences = { source.Confidences }, + SourceSegmentStart = source.SourceSegmentRange.Start, + SourceSegmentEnd = source.SourceSegmentRange.End, + Sources = { source.Sources.Select(Map) } + }; + } + + private static Translation.V1.TranslationSources Map(SIL.Machine.Translation.TranslationSources source) + { + return new Translation.V1.TranslationSources + { + Values = + { + System + .Enum.GetValues() + .Where(s => s != SIL.Machine.Translation.TranslationSources.None && source.HasFlag(s)) + .Select(s => + s switch + { + SIL.Machine.Translation.TranslationSources.Smt => TranslationSource.Primary, + SIL.Machine.Translation.TranslationSources.Nmt => TranslationSource.Primary, + SIL.Machine.Translation.TranslationSources.Transfer => TranslationSource.Secondary, + SIL.Machine.Translation.TranslationSources.Prefix => TranslationSource.Human, + _ => TranslationSource.Primary + } + ) + } + }; + } + + private static IEnumerable Map(WordAlignmentMatrix source) + { + for (int i = 0; i < source.RowCount; i++) + { + for (int j = 0; j < source.ColumnCount; j++) + { + if (source[i, j]) + yield return new Translation.V1.AlignedWordPair { SourceIndex = i, TargetIndex = j }; + } + } + } + + private static Translation.V1.Phrase Map(SIL.Machine.Translation.Phrase source) + { + return new Translation.V1.Phrase + { + SourceSegmentStart = source.SourceSegmentRange.Start, + SourceSegmentEnd = source.SourceSegmentRange.End, + TargetSegmentCut = source.TargetSegmentCut + }; + } + + private static Models.Corpus Map(Translation.V1.Corpus source) + { + var pretranslateChapters = source.PretranslateChapters.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.Chapters.ToHashSet() + ); + FilterChoice pretranslateFilter = GetFilterChoice(source.PretranslateAll, pretranslateChapters); + + var trainOnChapters = source.TrainOnChapters.ToDictionary( + kvp => kvp.Key, + kvp => kvp.Value.Chapters.ToHashSet() + ); + FilterChoice trainingFilter = GetFilterChoice(source.TrainOnAll, trainOnChapters); + + return new Models.Corpus + { + Id = source.Id, + SourceLanguage = source.SourceLanguage, + TargetLanguage = source.TargetLanguage, + TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, + PretranslateChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, + TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? source.TrainOnTextIds.ToHashSet() : null, + PretranslateTextIds = + pretranslateFilter == FilterChoice.TextIds ? source.PretranslateTextIds.ToHashSet() : null, + SourceFiles = source.SourceFiles.Select(Map).ToList(), + TargetFiles = source.TargetFiles.Select(Map).ToList() + }; + } + + private static Models.CorpusFile Map(Translation.V1.CorpusFile source) + { + return new Models.CorpusFile + { + Location = source.Location, + Format = (Models.FileFormat)source.Format, + TextId = source.TextId + }; + } + + private enum FilterChoice + { + Chapters, + TextIds, + None + } + + private static FilterChoice GetFilterChoice(bool all, IReadOnlyDictionary> chapters) + { + if (all) + return FilterChoice.None; + + // Only either textIds or Scripture Range will be used at a time + // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) + if (chapters.Count == 0) + return FilterChoice.TextIds; + else + return FilterChoice.Chapters; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs new file mode 100644 index 00000000..8d6ef0b9 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SharedFileService.cs @@ -0,0 +1,104 @@ +namespace Serval.Machine.Shared.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly IFileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public async Task GetDownloadUrlAsync(string path, DateTime expiresAt) + { + return await _fileStorage.GetDownloadUrlAsync(path, expiresAt); + } + + public Task> ListFilesAsync( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.ListFilesAsync(path, recurse, cancellationToken); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenReadAsync(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWriteAsync(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.ListFilesAsync( + path, + recurse: true, + cancellationToken + ); + foreach (string file in files) + await _fileStorage.DeleteAsync(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.DeleteAsync(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.ExistsAsync(path, cancellationToken); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs new file mode 100644 index 00000000..c83f0703 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferBuildJob.cs @@ -0,0 +1,166 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger, + IRepository trainSegmentPairs, + ITruecaserFactory truecaserFactory, + ISmtModelFactory smtModelFactory, + ICorpusService corpusService +) + : HangfireBuildJob>( + platformService, + engines, + lockFactory, + dataAccessContext, + buildJobService, + logger + ) +{ + private readonly IRepository _trainSegmentPairs = trainSegmentPairs; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ICorpusService _corpusService = corpusService; + + protected override Task InitializeAsync( + string engineId, + string buildId, + IReadOnlyList data, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + return _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, cancellationToken); + } + + protected override async Task DoWorkAsync( + string engineId, + string buildId, + IReadOnlyList data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + await PlatformService.BuildStartedAsync(buildId, cancellationToken); + Logger.LogInformation("Build started ({0})", buildId); + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + cancellationToken.ThrowIfCancellationRequested(); + + JsonObject? buildOptionsObject = null; + if (buildOptions is not null) + buildOptionsObject = JsonSerializer.Deserialize(buildOptions); + + var targetCorpora = new List(); + var parallelCorpora = new List(); + foreach (Corpus corpus in data) + { + ITextCorpus? sourceTextCorpus = _corpusService.CreateTextCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTextCorpus = _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault(); + if (sourceTextCorpus is null || targetTextCorpus is null) + continue; + + targetCorpora.Add(targetTextCorpus); + parallelCorpora.Add(sourceTextCorpus.AlignRows(targetTextCorpus)); + + if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) + { + ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); + ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); + if (sourceTermCorpus is not null && targetTermCorpus is not null) + { + IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); + parallelCorpora.Add(parallelKeyTermsCorpus); + } + } + } + + IParallelTextCorpus parallelCorpus = parallelCorpora.Flatten(); + ITextCorpus targetCorpus = targetCorpora.Flatten(); + + var tokenizer = new LatinWordTokenizer(); + var detokenizer = new LatinWordDetokenizer(); + + using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync( + engineId, + tokenizer, + parallelCorpus, + cancellationToken + ); + using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( + engineId, + tokenizer, + targetCorpus, + cancellationToken + ); + + cancellationToken.ThrowIfCancellationRequested(); + + var progress = new BuildProgress(PlatformService, buildId); + await smtModelTrainer.TrainAsync(progress, cancellationToken); + await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); + + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new OperationCanceledException(); + + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + await smtModelTrainer.SaveAsync(CancellationToken.None); + await truecaseTrainer.SaveAsync(CancellationToken.None); + ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineId, CancellationToken.None); + IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync( + p => p.TranslationEngineRef == engine.Id, + CancellationToken.None + ); + using ( + IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( + engineId, + tokenizer, + detokenizer, + truecaser, + CancellationToken.None + ) + ) + { + foreach (TrainSegmentPair segmentPair in segmentPairs) + { + await smtModel.TrainSegmentAsync( + segmentPair.Source, + segmentPair.Target, + cancellationToken: CancellationToken.None + ); + } + } + + await DataAccessContext.WithTransactionAsync( + async (ct) => + { + await PlatformService.BuildCompletedAsync( + buildId, + smtModelTrainer.Stats.TrainCorpusSize + segmentPairs.Count, + smtModelTrainer.Stats.Metrics["bleu"] * 100.0, + CancellationToken.None + ); + await BuildJobService.BuildJobFinishedAsync( + engineId, + buildId, + buildComplete: true, + CancellationToken.None + ); + }, + cancellationToken: CancellationToken.None + ); + } + + stopwatch.Stop(); + Logger.LogInformation("Build completed in {0}s ({1})", stopwatch.Elapsed.TotalSeconds, buildId); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs new file mode 100644 index 00000000..6e0b6b9c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferClearMLBuildJobFactory.cs @@ -0,0 +1,49 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferClearMLBuildJobFactory( + ISharedFileService sharedFileService, + IRepository engines +) : IClearMLBuildJobFactory +{ + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly IRepository _engines = engines; + + public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; + + public async Task CreateJobScriptAsync( + string engineId, + string buildId, + string modelType, + BuildStage stage, + object? data = null, + string? buildOptions = null, + CancellationToken cancellationToken = default + ) + { + if (stage == BuildStage.Train) + { + TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException("The engine does not exist."); + + Uri sharedFileUri = _sharedFileService.GetBaseUri(); + string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); + string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); + return "from machine.jobs.build_smt_engine import run\n" + + "args = {\n" + + $" 'model_type': '{modelType}',\n" + + $" 'engine_id': '{engineId}',\n" + + $" 'build_id': '{buildId}',\n" + + $" 'shared_file_uri': '{baseUri}',\n" + + $" 'shared_file_folder': '{folder}',\n" + + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") + + $" 'clearml': True\n" + + "}\n" + + "run(args)\n"; + } + else + { + throw new ArgumentException("Unknown build stage.", nameof(stage)); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs new file mode 100644 index 00000000..8e802ce6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineCommitService.cs @@ -0,0 +1,38 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferEngineCommitService( + IServiceProvider services, + IOptionsMonitor engineOptions, + SmtTransferEngineStateService stateService, + ILogger logger +) + : RecurrentTask( + "SMT transfer engine commit service", + services, + engineOptions.CurrentValue.EngineCommitFrequency, + logger + ) +{ + private readonly IOptionsMonitor _engineOptions = engineOptions; + private readonly SmtTransferEngineStateService _stateService = stateService; + private readonly ILogger _logger = logger; + + protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) + { + try + { + var engines = scope.ServiceProvider.GetRequiredService>(); + var lockFactory = scope.ServiceProvider.GetRequiredService(); + await _stateService.CommitAsync( + lockFactory, + engines, + _engineOptions.CurrentValue.InactiveEngineTimeout, + cancellationToken + ); + } + catch (Exception e) + { + _logger.LogError(e, "Error occurred while committing SMT transfer engines."); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs new file mode 100644 index 00000000..bdda5353 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineService.cs @@ -0,0 +1,272 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferEngineService( + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService, + IBuildJobService buildJobService, + IClearMLQueueService clearMLQueueService +) : ITranslationEngineService +{ + private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; + private readonly IPlatformService _platformService = platformService; + private readonly IDataAccessContext _dataAccessContext = dataAccessContext; + private readonly IRepository _engines = engines; + private readonly IRepository _trainSegmentPairs = trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService = stateService; + private readonly IBuildJobService _buildJobService = buildJobService; + private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; + + public TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + bool? isModelPersisted = null, + CancellationToken cancellationToken = default + ) + { + if (isModelPersisted == false) + { + throw new NotSupportedException( + "SMT transfer engines do not support non-persisted models." + + "Please remove the isModelPersisted parameter or set it to true." + ); + } + + TranslationEngine translationEngine = await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + var translationEngine = new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage, + Type = TranslationEngineType.SmtTransfer, + IsModelPersisted = isModelPersisted ?? true // models are persisted if not specified + }; + await _engines.InsertAsync(translationEngine, ct); + await _buildJobService.CreateEngineAsync(engineId, engineName, ct); + return translationEngine; + }, + cancellationToken: cancellationToken + ); + + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + SmtTransferEngineState state = _stateService.Get(engineId); + await state.InitNewAsync(CancellationToken.None); + } + return translationEngine; + } + + public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildJobAsync(engineId, cancellationToken); + + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + await _engines.DeleteAsync(e => e.EngineId == engineId, ct); + await _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, ct); + }, + cancellationToken: cancellationToken + ); + await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); + + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + await _lockFactory.DeleteAsync(engineId, CancellationToken.None); + } + + public async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + SmtTransferEngineState state = _stateService.Get(engineId); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + SmtTransferEngineState state = _stateService.Get(engineId); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + + async Task TrainSubroutineAsync(SmtTransferEngineState state, CancellationToken ct) + { + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, ct); + await _platformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + } + + SmtTransferEngineState state = _stateService.Get(engineId); + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + if (engine.CurrentBuild?.JobState is BuildJobState.Active) + { + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engineId, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + CancellationToken.None + ); + await TrainSubroutineAsync(state, CancellationToken.None); + } + else + { + await TrainSubroutineAsync(state, ct); + } + }, + cancellationToken: cancellationToken + ); + + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public async Task StartBuildAsync( + string engineId, + string buildId, + string? buildOptions, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + // If there is a pending/running build, then no need to start a new one. + if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) + throw new InvalidOperationException("The engine is already building or in the process of canceling."); + + await _buildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + engineId, + buildId, + BuildStage.Preprocess, + corpora, + buildOptions, + cancellationToken + ); + SmtTransferEngineState state = _stateService.Get(engineId); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + if (!await CancelBuildJobAsync(engineId, cancellationToken)) + throw new InvalidOperationException("The engine is not currently building."); + SmtTransferEngineState state = _stateService.Get(engineId); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public int GetQueueSize() + { + return _clearMLQueueService.GetQueueSize(Type); + } + + public bool IsLanguageNativeToModel(string language, out string internalCode) + { + throw new NotSupportedException("SMT transfer engines do not support language info."); + } + + private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) + { + string? buildId = null; + await _dataAccessContext.WithTransactionAsync( + async (ct) => + { + (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); + if (buildId is not null && jobState is BuildJobState.None) + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + }, + cancellationToken: cancellationToken + ); + return buildId is not null; + } + + public Task GetModelDownloadUrlAsync( + string engineId, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException($"The engine {engineId} does not exist."); + return engine; + } + + private async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0) + throw new EngineNotBuiltException("The engine must be built first."); + return engine; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs new file mode 100644 index 00000000..a5f4300a --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineState.cs @@ -0,0 +1,128 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferEngineState( + ISmtModelFactory smtModelFactory, + ITransferEngineFactory transferEngineFactory, + ITruecaserFactory truecaserFactory, + IOptionsMonitor options, + string engineId +) : AsyncDisposableBase +{ + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly IOptionsMonitor _options = options; + private readonly AsyncLock _lock = new(); + + private IInteractiveTranslationModel? _smtModel; + private HybridTranslationEngine? _hybridEngine; + + public string EngineId { get; } = engineId; + + public bool IsUpdated { get; set; } + public int CurrentBuildRevision { get; set; } = -1; + public DateTime LastUsedTime { get; set; } = DateTime.UtcNow; + public bool IsLoaded => _hybridEngine != null; + + private string EngineDir => Path.Combine(_options.CurrentValue.EnginesDir, EngineId); + + public async Task InitNewAsync(CancellationToken cancellationToken = default) + { + await _smtModelFactory.InitNewAsync(EngineDir, cancellationToken); + await _transferEngineFactory.InitNewAsync(EngineDir, cancellationToken); + } + + public async Task GetHybridEngineAsync(int buildRevision) + { + using (await _lock.LockAsync()) + { + if (_hybridEngine is not null && CurrentBuildRevision != -1 && buildRevision != CurrentBuildRevision) + { + IsUpdated = false; + await UnloadAsync(); + } + + if (_hybridEngine is null) + { + LatinWordTokenizer tokenizer = new(); + LatinWordDetokenizer detokenizer = new(); + ITruecaser truecaser = await _truecaserFactory.CreateAsync(EngineDir); + _smtModel = await _smtModelFactory.CreateAsync(EngineDir, tokenizer, detokenizer, truecaser); + ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( + EngineDir, + tokenizer, + detokenizer, + truecaser + ); + _hybridEngine = new HybridTranslationEngine(_smtModel, transferEngine) + { + TargetDetokenizer = detokenizer + }; + } + CurrentBuildRevision = buildRevision; + return _hybridEngine; + } + } + + public async Task DeleteDataAsync() + { + await UnloadAsync(); + await _smtModelFactory.CleanupAsync(EngineDir); + await _transferEngineFactory.CleanupAsync(EngineDir); + await _truecaserFactory.CleanupAsync(EngineDir); + } + + public async Task CommitAsync( + int buildRevision, + TimeSpan inactiveTimeout, + CancellationToken cancellationToken = default + ) + { + if (_hybridEngine is null) + return; + + if (CurrentBuildRevision == -1) + CurrentBuildRevision = buildRevision; + if (buildRevision != CurrentBuildRevision) + { + await UnloadAsync(cancellationToken); + CurrentBuildRevision = buildRevision; + } + else if (DateTime.Now - LastUsedTime > inactiveTimeout) + { + await UnloadAsync(cancellationToken); + } + else + { + await SaveModelAsync(cancellationToken); + } + } + + private async Task SaveModelAsync(CancellationToken cancellationToken = default) + { + if (_smtModel is not null && IsUpdated) + { + await _smtModel.SaveAsync(cancellationToken); + IsUpdated = false; + } + } + + private async Task UnloadAsync(CancellationToken cancellationToken = default) + { + if (_hybridEngine is null) + return; + + await SaveModelAsync(cancellationToken); + + _hybridEngine.Dispose(); + + _smtModel = null; + _hybridEngine = null; + CurrentBuildRevision = -1; + } + + protected override async ValueTask DisposeAsyncCore() + { + await UnloadAsync(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs new file mode 100644 index 00000000..03ef2ad8 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferEngineStateService.cs @@ -0,0 +1,72 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferEngineStateService( + ISmtModelFactory smtModelFactory, + ITransferEngineFactory transferEngineFactory, + ITruecaserFactory truecaserFactory, + IOptionsMonitor options +) : AsyncDisposableBase +{ + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly IOptionsMonitor _options = options; + + private readonly ConcurrentDictionary _engineStates = + new ConcurrentDictionary(); + + public SmtTransferEngineState Get(string engineId) + { + return _engineStates.GetOrAdd(engineId, CreateState); + } + + public bool TryRemove(string engineId, [MaybeNullWhen(false)] out SmtTransferEngineState state) + { + return _engineStates.TryRemove(engineId, out state); + } + + public async Task CommitAsync( + IDistributedReaderWriterLockFactory lockFactory, + IRepository engines, + TimeSpan inactiveTimeout, + CancellationToken cancellationToken = default + ) + { + foreach (SmtTransferEngineState state in _engineStates.Values) + { + IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(state.EngineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine? engine = await engines.GetAsync( + e => e.EngineId == state.EngineId, + cancellationToken + ); + if ( + engine is not null + && (engine.CurrentBuild is null || engine.CurrentBuild.JobState is BuildJobState.Pending) + ) + { + await state.CommitAsync(engine.BuildRevision, inactiveTimeout, cancellationToken); + } + } + } + } + + private SmtTransferEngineState CreateState(string engineId) + { + return new SmtTransferEngineState( + _smtModelFactory, + _transferEngineFactory, + _truecaserFactory, + _options, + engineId + ); + } + + protected override async ValueTask DisposeAsyncCore() + { + foreach (SmtTransferEngineState state in _engineStates.Values) + await state.DisposeAsync(); + _engineStates.Clear(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs new file mode 100644 index 00000000..9f532b2b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferHangfireBuildJobFactory.cs @@ -0,0 +1,33 @@ +using static Serval.Machine.Shared.Services.HangfireBuildJobRunner; + +namespace Serval.Machine.Shared.Services; + +public class SmtTransferHangfireBuildJobFactory : IHangfireBuildJobFactory +{ + public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; + + public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) + { + return stage switch + { + BuildStage.Preprocess + => CreateJob>( + engineId, + buildId, + "smt_transfer", + data, + buildOptions + ), + BuildStage.Postprocess + => CreateJob( + engineId, + buildId, + "smt_transfer", + data, + buildOptions + ), + BuildStage.Train => CreateJob(engineId, buildId, "smt_transfer", buildOptions), + _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), + }; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs new file mode 100644 index 00000000..d0d25fe5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferPostprocessBuildJob.cs @@ -0,0 +1,74 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferPostprocessBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger, + ISharedFileService sharedFileService, + IRepository trainSegmentPairs, + ISmtModelFactory smtModelFactory, + ITruecaserFactory truecaserFactory, + IOptionsMonitor options +) + : PostprocessBuildJob( + platformService, + engines, + lockFactory, + dataAccessContext, + buildJobService, + logger, + sharedFileService + ) +{ + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly IRepository _trainSegmentPairs = trainSegmentPairs; + private readonly IOptionsMonitor _options = options; + + protected override async Task SaveModelAsync(string engineId, string buildId) + { + await using ( + Stream engineStream = await SharedFileService.OpenReadAsync( + $"builds/{buildId}/model.tar.gz", + CancellationToken.None + ) + ) + { + await _smtModelFactory.UpdateEngineFromAsync( + Path.Combine(_options.CurrentValue.EnginesDir, engineId), + engineStream, + CancellationToken.None + ); + } + return await TrainOnNewSegmentPairsAsync(engineId); + } + + private async Task TrainOnNewSegmentPairsAsync(string engineId) + { + IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync(p => + p.TranslationEngineRef == engineId + ); + if (segmentPairs.Count == 0) + return segmentPairs.Count; + + string engineDir = Path.Combine(_options.CurrentValue.EnginesDir, engineId); + var tokenizer = new LatinWordTokenizer(); + var detokenizer = new LatinWordDetokenizer(); + ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir); + using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( + engineDir, + tokenizer, + detokenizer, + truecaser + ); + foreach (TrainSegmentPair segmentPair in segmentPairs) + { + await smtModel.TrainSegmentAsync(segmentPair.Source, segmentPair.Target); + } + await smtModel.SaveAsync(); + return segmentPairs.Count; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs new file mode 100644 index 00000000..bb4870c1 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/SmtTransferTrainBuildJob.cs @@ -0,0 +1,237 @@ +namespace Serval.Machine.Shared.Services; + +public class SmtTransferTrainBuildJob( + IPlatformService platformService, + IRepository engines, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IBuildJobService buildJobService, + ILogger logger, + ISharedFileService sharedFileService, + ITruecaserFactory truecaserFactory, + ISmtModelFactory smtModelFactory, + ITransferEngineFactory transferEngineFactory +) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) +{ + private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + private const int BatchSize = 128; + + private readonly ISharedFileService _sharedFileService = sharedFileService; + private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; + private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; + private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; + + protected override async Task DoWorkAsync( + string engineId, + string buildId, + object? data, + string? buildOptions, + IDistributedReaderWriterLock @lock, + CancellationToken cancellationToken + ) + { + using TempDirectory tempDir = new(buildId); + string corpusDir = Path.Combine(tempDir.Path, "corpus"); + await DownloadDataAsync(buildId, corpusDir, cancellationToken); + + // assemble corpus + ITextCorpus sourceCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.src.txt")); + ITextCorpus targetCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.trg.txt")); + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + // train SMT model + string engineDir = Path.Combine(tempDir.Path, "engine"); + (int trainCorpusSize, double confidence) = await TrainAsync( + buildId, + engineDir, + targetCorpus, + parallelCorpus, + cancellationToken + ); + + cancellationToken.ThrowIfCancellationRequested(); + + await GeneratePretranslationsAsync(buildId, engineDir, cancellationToken); + + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + bool canceling = !await BuildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + engineId, + buildId, + BuildStage.Postprocess, + data: (trainCorpusSize, confidence), + buildOptions: buildOptions, + cancellationToken: cancellationToken + ); + if (canceling) + throw new OperationCanceledException(); + } + } + + protected override async Task CleanupAsync( + string engineId, + string buildId, + object? data, + IDistributedReaderWriterLock @lock, + JobCompletionStatus completionStatus + ) + { + if (completionStatus is JobCompletionStatus.Canceled) + { + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/"); + } + catch (Exception e) + { + Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); + } + } + } + + private async Task DownloadDataAsync(string buildId, string corpusDir, CancellationToken cancellationToken) + { + Directory.CreateDirectory(corpusDir); + await using Stream srcText = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/train.src.txt", + cancellationToken + ); + await using FileStream srcFileStream = File.Create(Path.Combine(corpusDir, "train.src.txt")); + await srcText.CopyToAsync(srcFileStream, cancellationToken); + + await using Stream tgtText = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/train.trg.txt", + cancellationToken + ); + await using FileStream tgtFileStream = File.Create(Path.Combine(corpusDir, "train.trg.txt")); + await tgtText.CopyToAsync(tgtFileStream, cancellationToken); + } + + private async Task<(int TrainCorpusSize, double Confidence)> TrainAsync( + string buildId, + string engineDir, + ITextCorpus targetCorpus, + IParallelTextCorpus parallelCorpus, + CancellationToken cancellationToken + ) + { + await _smtModelFactory.InitNewAsync(engineDir, cancellationToken); + LatinWordTokenizer tokenizer = new(); + int trainCorpusSize; + double confidence; + using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync( + engineDir, + tokenizer, + parallelCorpus, + cancellationToken + ); + using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( + engineDir, + tokenizer, + targetCorpus, + cancellationToken + ); + cancellationToken.ThrowIfCancellationRequested(); + + var progress = new BuildProgress(PlatformService, buildId); + await smtModelTrainer.TrainAsync(progress, cancellationToken); + await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); + + trainCorpusSize = smtModelTrainer.Stats.TrainCorpusSize; + confidence = smtModelTrainer.Stats.Metrics["bleu"] * 100.0; + + cancellationToken.ThrowIfCancellationRequested(); + + await smtModelTrainer.SaveAsync(cancellationToken); + await truecaseTrainer.SaveAsync(cancellationToken); + + await using Stream engineStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/model.tar.gz", + cancellationToken + ); + await _smtModelFactory.SaveEngineToAsync(engineDir, engineStream, cancellationToken); + return (trainCorpusSize, confidence); + } + + private async Task GeneratePretranslationsAsync( + string buildId, + string engineDir, + CancellationToken cancellationToken + ) + { + await using Stream sourceStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable(sourceStream, JsonSerializerOptions, cancellationToken) + .OfType(); + + await using Stream targetStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + await using Utf8JsonWriter targetWriter = new(targetStream, PretranslateWriterOptions); + + LatinWordTokenizer tokenizer = new(); + LatinWordDetokenizer detokenizer = new(); + ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir, CancellationToken.None); + using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( + engineDir, + tokenizer, + detokenizer, + truecaser, + cancellationToken + ); + using ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( + engineDir, + tokenizer, + detokenizer, + truecaser, + cancellationToken + ); + HybridTranslationEngine hybridEngine = new(smtModel, transferEngine) { TargetDetokenizer = detokenizer }; + + await foreach (IReadOnlyList batch in BatchAsync(pretranslations)) + { + string[] segments = batch.Select(p => p.Translation).ToArray(); + IReadOnlyList results = await hybridEngine.TranslateBatchAsync( + segments, + cancellationToken + ); + foreach ((Pretranslation pretranslation, TranslationResult result) in batch.Zip(results)) + { + JsonSerializer.Serialize( + targetWriter, + pretranslation with + { + Translation = result.Translation + }, + JsonSerializerOptions + ); + } + } + } + + public static async IAsyncEnumerable> BatchAsync( + IAsyncEnumerable pretranslations + ) + { + List batch = []; + await foreach (Pretranslation item in pretranslations) + { + batch.Add(item); + if (batch.Count == BatchSize) + { + yield return batch; + batch = []; + } + } + if (batch.Count > 0) + yield return batch; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs new file mode 100644 index 00000000..031891c4 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/ThotSmtModelFactory.cs @@ -0,0 +1,110 @@ +namespace Serval.Machine.Shared.Services; + +public class ThotSmtModelFactory(IOptionsMonitor options) : ISmtModelFactory +{ + private readonly IOptionsMonitor _options = options; + + public Task CreateAsync( + string engineDir, + IRangeTokenizer tokenizer, + IDetokenizer detokenizer, + ITruecaser truecaser, + CancellationToken cancellationToken = default + ) + { + string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); + IInteractiveTranslationModel model = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, smtConfigFileName) + { + SourceTokenizer = tokenizer, + TargetTokenizer = tokenizer, + TargetDetokenizer = detokenizer, + LowercaseSource = true, + LowercaseTarget = true, + Truecaser = truecaser + }; + return Task.FromResult(model); + } + + public Task CreateTrainerAsync( + string engineDir, + IRangeTokenizer tokenizer, + IParallelTextCorpus corpus, + CancellationToken cancellationToken = default + ) + { + string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); + ITrainer trainer = new ThotSmtModelTrainer(ThotWordAlignmentModelType.Hmm, corpus, smtConfigFileName) + { + SourceTokenizer = tokenizer, + TargetTokenizer = tokenizer, + LowercaseSource = true, + LowercaseTarget = true + }; + return Task.FromResult(trainer); + } + + public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) + { + if (!Directory.Exists(engineDir)) + Directory.CreateDirectory(engineDir); + ZipFile.ExtractToDirectory(_options.CurrentValue.NewModelFile, engineDir); + return Task.CompletedTask; + } + + public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) + { + if (!Directory.Exists(engineDir)) + return Task.CompletedTask; + DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "lm")); + DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "tm")); + string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); + if (File.Exists(smtConfigFileName)) + File.Delete(smtConfigFileName); + if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) + Directory.Delete(engineDir); + return Task.CompletedTask; + } + + public async Task UpdateEngineFromAsync( + string engineDir, + Stream source, + CancellationToken cancellationToken = default + ) + { + if (!Directory.Exists(engineDir)) + Directory.CreateDirectory(engineDir); + + await using MemoryStream memoryStream = new(); + await using (GZipStream gzipStream = new(source, CompressionMode.Decompress)) + { + await gzipStream.CopyToAsync(memoryStream, cancellationToken); + } + memoryStream.Seek(0, SeekOrigin.Begin); + await TarFile.ExtractToDirectoryAsync( + memoryStream, + engineDir, + overwriteFiles: true, + cancellationToken: cancellationToken + ); + } + + public async Task SaveEngineToAsync( + string engineDir, + Stream destination, + CancellationToken cancellationToken = default + ) + { + // create zip archive in memory stream + // This cannot be created directly to the shared stream because it all needs to be written at once + await using MemoryStream memoryStream = new(); + await TarFile.CreateFromDirectoryAsync( + engineDir, + memoryStream, + includeBaseDirectory: false, + cancellationToken: cancellationToken + ); + memoryStream.Seek(0, SeekOrigin.Begin); + await using GZipStream gzipStream = new(destination, CompressionMode.Compress); + await memoryStream.CopyToAsync(gzipStream, cancellationToken); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs new file mode 100644 index 00000000..a140792b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/TransferEngineFactory.cs @@ -0,0 +1,61 @@ +namespace Serval.Machine.Shared.Services; + +public class TransferEngineFactory : ITransferEngineFactory +{ + public Task CreateAsync( + string engineDir, + IRangeTokenizer tokenizer, + IDetokenizer detokenizer, + ITruecaser truecaser, + CancellationToken cancellationToken = default + ) + { + string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); + string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); + ITranslationEngine? transferEngine = null; + if (File.Exists(hcSrcConfigFileName) && File.Exists(hcTrgConfigFileName)) + { + var hcTraceManager = new TraceManager(); + + Language srcLang = XmlLanguageLoader.Load(hcSrcConfigFileName); + var srcMorpher = new Morpher(hcTraceManager, srcLang); + + Language trgLang = XmlLanguageLoader.Load(hcTrgConfigFileName); + var trgMorpher = new Morpher(hcTraceManager, trgLang); + + transferEngine = new TransferEngine( + srcMorpher, + new SimpleTransferer(new GlossMorphemeMapper(trgMorpher)), + trgMorpher + ) + { + SourceTokenizer = tokenizer, + TargetDetokenizer = detokenizer, + LowercaseSource = true, + Truecaser = truecaser + }; + } + return Task.FromResult(transferEngine); + } + + public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) + { + // TODO: generate source and target config files + return Task.CompletedTask; + } + + public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) + { + if (!Directory.Exists(engineDir)) + return Task.CompletedTask; + string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); + if (File.Exists(hcSrcConfigFileName)) + File.Delete(hcSrcConfigFileName); + string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); + if (File.Exists(hcTrgConfigFileName)) + File.Delete(hcTrgConfigFileName); + if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) + Directory.Delete(engineDir); + return Task.CompletedTask; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs b/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs new file mode 100644 index 00000000..61df1966 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/TranslationEngineType.cs @@ -0,0 +1,7 @@ +namespace Serval.Machine.Shared.Services; + +public enum TranslationEngineType +{ + SmtTransfer, + Nmt +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs new file mode 100644 index 00000000..cbf9c8b5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/UnigramTruecaserFactory.cs @@ -0,0 +1,37 @@ +namespace Serval.Machine.Shared.Services; + +public class UnigramTruecaserFactory : ITruecaserFactory +{ + public async Task CreateAsync(string engineDir, CancellationToken cancellationToken = default) + { + var truecaser = new UnigramTruecaser(); + string path = GetModelPath(engineDir); + await truecaser.LoadAsync(path); + return truecaser; + } + + public Task CreateTrainerAsync( + string engineDir, + ITokenizer tokenizer, + ITextCorpus corpus, + CancellationToken cancellationToken = default + ) + { + string path = GetModelPath(engineDir); + ITrainer trainer = new UnigramTruecaserTrainer(path, corpus) { Tokenizer = tokenizer }; + return Task.FromResult(trainer); + } + + public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) + { + string path = GetModelPath(engineDir); + if (File.Exists(path)) + File.Delete(path); + return Task.CompletedTask; + } + + private static string GetModelPath(string engineDir) + { + return Path.Combine(engineDir, "unigram-casing-model.txt"); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs b/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs new file mode 100644 index 00000000..c75812d6 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Services/UnimplementedInterceptor.cs @@ -0,0 +1,22 @@ +namespace Serval.Machine.Shared.Services; + +public class UnimplementedInterceptor : Interceptor +{ + public override async Task UnaryServerHandler( + TRequest request, + ServerCallContext context, + UnaryServerMethod continuation + ) + { + try + { + return await continuation(request, context); + } + catch (NotSupportedException) + { + throw new RpcException( + new Status(StatusCode.Unimplemented, "The call is not supported by the specified engine.") + ); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs new file mode 100644 index 00000000..244e090b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Usings.cs @@ -0,0 +1,59 @@ +global using System.Collections.Concurrent; +global using System.Data; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.Formats.Tar; +global using System.Globalization; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Net.Mime; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Security.Cryptography; +global using System.Text; +global using System.Text.Encodings.Web; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using System.Text.Json.Serialization; +global using System.Text.RegularExpressions; +global using Amazon; +global using Amazon.Runtime; +global using Amazon.S3; +global using Amazon.S3.Model; +global using CommunityToolkit.HighPerformance; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Common; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Hangfire.States; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Bson.Serialization.Serializers; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Nito.AsyncEx.Synchronous; +global using Polly; +global using Serval.Machine.Shared.Configuration; +global using Serval.Machine.Shared.Models; +global using Serval.Machine.Shared.Services; +global using Serval.Machine.Shared.Utils; +global using SIL.DataAccess; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.Scripture; +global using SIL.WritingSystems; diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs new file mode 100644 index 00000000..6c4a5c0f --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncDisposableBase.cs @@ -0,0 +1,19 @@ +using SIL.ObjectModel; + +namespace Serval.Machine.Shared.Utils; + +public class AsyncDisposableBase : DisposableBase, IAsyncDisposable +{ + public async ValueTask DisposeAsync() + { + await DisposeAsyncCore(); + + Dispose(false); + GC.SuppressFinalize(this); + } + + protected virtual ValueTask DisposeAsyncCore() + { + return default; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs new file mode 100644 index 00000000..711826d5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/AsyncTimer.cs @@ -0,0 +1,69 @@ +namespace Serval.Machine.Shared.Utils; + +public class AsyncTimer : AsyncDisposableBase +{ + private readonly Timer _timer; + private readonly Func _callback; + private readonly AsyncLock _lock; + private bool _running; + + public AsyncTimer(Func callback) + { + _callback = callback; + _lock = new AsyncLock(); + _timer = new Timer(FireTimerAsync, null, Timeout.Infinite, Timeout.Infinite); + } + + public void Start(TimeSpan period) + { + _running = true; + _timer.Change(period, period); + } + + private async void FireTimerAsync(object? state) + { + using (await _lock.LockAsync()) + { + if (_running) + await _callback(); + } + } + + public async Task StopAsync() + { + using (await _lock.LockAsync()) + { + // FireTimer is *not* running _callback (since we got the lock) + StopTimer(); + } + // Now FireTimer will *never* run _callback + } + + public void Stop() + { + using (_lock.Lock()) + { + // FireTimer is *not* running _callback (since we got the lock) + StopTimer(); + } + // Now FireTimer will *never* run _callback + } + + private void StopTimer() + { + _timer.Change(Timeout.Infinite, Timeout.Infinite); + _running = false; + } + + protected override async ValueTask DisposeAsyncCore() + { + await StopAsync(); + _timer.Dispose(); + } + + protected override void DisposeManagedResources() + { + Stop(); + _timer.Dispose(); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs b/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs new file mode 100644 index 00000000..c5e07809 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/CustomEnumConverterFactory.cs @@ -0,0 +1,151 @@ +namespace Serval.Machine.Shared.Utils; + +public sealed class CustomEnumConverterFactory(JsonNamingPolicy namingPolicy) : JsonConverterFactory +{ + private readonly JsonNamingPolicy _namingPolicy = namingPolicy; + + public override bool CanConvert(Type typeToConvert) => typeToConvert.IsEnum; + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + object[]? knownValues = null; + + if (typeToConvert == typeof(BindingFlags)) + knownValues = new object[] { BindingFlags.CreateInstance | BindingFlags.DeclaredOnly }; + + return (JsonConverter) + Activator.CreateInstance( + typeof(CustomEnumConverter<>).MakeGenericType(typeToConvert), + BindingFlags.Instance | BindingFlags.Public, + binder: null, + args: new object?[] { _namingPolicy, options, knownValues }, + culture: null + )!; + } +} + +public sealed class CustomEnumConverter : JsonConverter + where T : Enum +{ + private readonly JsonNamingPolicy _namingPolicy; + + private readonly Dictionary _readCache = new(); + private readonly Dictionary _writeCache = new(); + + // This converter will only support up to 64 enum values (including flags) on serialization and deserialization + private const int NameCacheLimit = 64; + + private const string ValueSeparator = ", "; + + public CustomEnumConverter(JsonNamingPolicy namingPolicy, JsonSerializerOptions options, object[]? knownValues) + { + _namingPolicy = namingPolicy; + + bool continueProcessing = true; + for (int i = 0; i < knownValues?.Length; i++) + { + if (!TryProcessValue((T)knownValues[i])) + { + continueProcessing = false; + break; + } + } + + if (continueProcessing) + { + Array values = Enum.GetValues(typeof(T)); + + for (int i = 0; i < values.Length; i++) + { + var value = (T)values.GetValue(i)!; + + if (!TryProcessValue(value)) + break; + } + } + + bool TryProcessValue(T value) + { + if (_readCache.Count == NameCacheLimit) + { + Debug.Assert(_writeCache.Count == NameCacheLimit); + return false; + } + + FormatAndAddToCaches(value, options.Encoder); + return true; + } + } + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string? json; + + if ( + reader.TokenType != JsonTokenType.String + || (json = reader.GetString()) == null + || !_readCache.TryGetValue(json, out T? value) + ) + { + throw new JsonException(); + } + + return value; + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (!_writeCache.TryGetValue(value, out JsonEncodedText formatted)) + { + if (_writeCache.Count == NameCacheLimit) + { + Debug.Assert(_readCache.Count == NameCacheLimit); + throw new InvalidOperationException("The JSON value contains too many enumerations."); + } + + formatted = FormatAndAddToCaches(value, options.Encoder); + } + + writer.WriteStringValue(formatted); + } + + private JsonEncodedText FormatAndAddToCaches(T value, JavaScriptEncoder? encoder) + { + (string valueFormattedToStr, JsonEncodedText valueEncoded) = CustomEnumConverter.FormatEnumValue( + value.ToString(), + _namingPolicy, + encoder + ); + _readCache[valueFormattedToStr] = value; + _writeCache[value] = valueEncoded; + return valueEncoded; + } + + private static ValueTuple FormatEnumValue( + string value, + JsonNamingPolicy namingPolicy, + JavaScriptEncoder? encoder + ) + { + string converted; + + if (!value.Contains(ValueSeparator)) + { + converted = namingPolicy.ConvertName(value); + } + else + { + // todo: optimize implementation here by leveraging https://github.com/dotnet/runtime/issues/934. + string[] enumValues = value.Split(ValueSeparator); + + for (int i = 0; i < enumValues.Length; i++) + { + enumValues[i] = namingPolicy.ConvertName(enumValues[i]); + } + + converted = string.Join(ValueSeparator, enumValues); + } + + return (converted, JsonEncodedText.Encode(converted, encoder)); + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs b/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs new file mode 100644 index 00000000..792fa9b2 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/DictionaryStringConverter.cs @@ -0,0 +1,65 @@ +namespace Serval.Machine.Shared.Utils; + +internal sealed class DictionaryStringStringConverter : JsonConverter> +{ + public override IReadOnlyDictionary Read( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options + ) + { + if (reader.TokenType != JsonTokenType.StartObject) + throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported"); + + var dictionary = new Dictionary(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + return dictionary; + + if (reader.TokenType != JsonTokenType.PropertyName) + throw new JsonException("JsonTokenType was not PropertyName"); + + var propertyName = reader.GetString(); + + if (string.IsNullOrWhiteSpace(propertyName)) + throw new JsonException("Failed to get property name"); + + reader.Read(); + + dictionary.Add(propertyName!, ExtractValue(ref reader)); + } + + return dictionary; + } + + public override void Write( + Utf8JsonWriter writer, + IReadOnlyDictionary value, + JsonSerializerOptions options + ) + { + JsonSerializer.Serialize(writer, value, options); + } + + private static string ExtractValue(ref Utf8JsonReader reader) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + return reader.GetString() ?? "Error Reading String."; + case JsonTokenType.False: + return "false"; + case JsonTokenType.True: + return "true"; + case JsonTokenType.Null: + return "null"; + case JsonTokenType.Number: + if (reader.TryGetDouble(out var result)) + return result.ToString(CultureInfo.InvariantCulture); + return "Error Reading Number."; + default: + throw new JsonException($"'{reader.TokenType}' is not supported"); + } + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs b/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs new file mode 100644 index 00000000..6378535f --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/EngineNotBuiltException.cs @@ -0,0 +1,4 @@ +namespace Serval.Machine.Shared.Utils; + +/// This exception is thrown when an unbuilt engine is requested to perform an action that requires it being built +public class EngineNotBuiltException(string message) : Exception(message) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs b/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs new file mode 100644 index 00000000..2e2f91ec --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/RecurrentTask.cs @@ -0,0 +1,40 @@ +namespace Serval.Machine.Shared.Utils; + +public abstract class RecurrentTask( + string serviceName, + IServiceProvider services, + TimeSpan period, + ILogger logger, + bool enable = true +) : BackgroundService +{ + private readonly bool _enable = enable; + private readonly string _serviceName = serviceName; + private readonly IServiceProvider _services = services; + private readonly TimeSpan _period = period; + private readonly ILogger _logger = logger; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!_enable) + return; + + using PeriodicTimer timer = new(_period); + + _logger.LogInformation("{ServiceName} started.", _serviceName); + + try + { + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + using IServiceScope scope = _services.CreateScope(); + await DoWorkAsync(scope, stoppingToken); + } + } + catch (OperationCanceledException) { } + + _logger.LogInformation("{ServiceName} stopped.", _serviceName); + } + + protected abstract Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken); +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs b/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs new file mode 100644 index 00000000..36409c2b --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/SharedFileUtils.cs @@ -0,0 +1,20 @@ +namespace Serval.Machine.Shared.Utils; + +public static class SharedFileUtils +{ + public static string Normalize(string path, bool includeLeadingSlash = false, bool includeTrailingSlash = false) + { + string normalizedPath = path; + if (normalizedPath == "/") + return normalizedPath; + if (!includeLeadingSlash && normalizedPath.StartsWith("/")) + normalizedPath = normalizedPath.Remove(0, 1); + else if (includeLeadingSlash && !normalizedPath.StartsWith("/")) + normalizedPath = "/" + normalizedPath; + if (!includeTrailingSlash && normalizedPath.EndsWith("/")) + normalizedPath = normalizedPath.Remove(normalizedPath.Length - 1, 1); + else if (includeTrailingSlash && !normalizedPath.EndsWith("/")) + normalizedPath += "/"; + return normalizedPath; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs b/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs new file mode 100644 index 00000000..37cecdd0 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Utils/StartupTask.cs @@ -0,0 +1,19 @@ +namespace Serval.Machine.Shared.Utils; + +public class StartupTask(IServiceProvider services, Func task) + : IHostedService +{ + private readonly IServiceProvider _services = services; + private readonly Func _task = task; + + public async Task StartAsync(CancellationToken cancellationToken) + { + using IServiceScope scope = _services.CreateScope(); + await _task(scope.ServiceProvider, cancellationToken); + } + + public Task StopAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } +} diff --git a/src/Machine/src/Serval.Machine.Shared/data/.gitattributes b/src/Machine/src/Serval.Machine.Shared/data/.gitattributes new file mode 100644 index 00000000..be61261c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/.gitattributes @@ -0,0 +1,5 @@ +thot-new-model/lm/* eol=lf +thot-new-model/tm/* eol=lf +thot-new-model/tm/*.hmm_alignd binary +thot-new-model/tm/*.hmm_lexnd binary +thot-new-model/smt.cfg eol=lf \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv b/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv new file mode 100644 index 00000000..a9734554 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/flores200languages.csv @@ -0,0 +1,205 @@ +language, code +Acehnese (Arabic script), ace_Arab +Acehnese (Latin script), ace_Latn +Mesopotamian Arabic, acm_Arab +Ta’izzi-Adeni Arabic, acq_Arab +Tunisian Arabic, aeb_Arab +Afrikaans, afr_Latn +South Levantine Arabic, ajp_Arab +Akan, aka_Latn +Amharic, amh_Ethi +North Levantine Arabic, apc_Arab +Modern Standard Arabic, arb_Arab +Modern Standard Arabic (Romanized), arb_Latn +Najdi Arabic, ars_Arab +Moroccan Arabic, ary_Arab +Egyptian Arabic, arz_Arab +Assamese, asm_Beng +Asturian, ast_Latn +Awadhi, awa_Deva +Central Aymara, ayr_Latn +South Azerbaijani, azb_Arab +North Azerbaijani, azj_Latn +Bashkir, bak_Cyrl +Bambara, bam_Latn +Balinese, ban_Latn +Belarusian, bel_Cyrl +Bemba, bem_Latn +Bengali, ben_Beng +Bhojpuri, bho_Deva +Banjar (Arabic script), bjn_Arab +Banjar (Latin script), bjn_Latn +Standard Tibetan, bod_Tibt +Bosnian, bos_Latn +Buginese, bug_Latn +Bulgarian, bul_Cyrl +Catalan, cat_Latn +Cebuano, ceb_Latn +Czech, ces_Latn +Chokwe, cjk_Latn +Central Kurdish, ckb_Arab +Crimean Tatar, crh_Latn +Welsh, cym_Latn +Danish, dan_Latn +German, deu_Latn +Southwestern Dinka, dik_Latn +Dyula, dyu_Latn +Dzongkha, dzo_Tibt +Greek, ell_Grek +English, eng_Latn +Esperanto, epo_Latn +Estonian, est_Latn +Basque, eus_Latn +Ewe, ewe_Latn +Faroese, fao_Latn +Fijian, fij_Latn +Finnish, fin_Latn +Fon, fon_Latn +French, fra_Latn +Friulian, fur_Latn +Nigerian Fulfulde, fuv_Latn +Scottish Gaelic, gla_Latn +Irish, gle_Latn +Galician, glg_Latn +Guarani, grn_Latn +Gujarati, guj_Gujr +Haitian Creole, hat_Latn +Hausa, hau_Latn +Hebrew, heb_Hebr +Hindi, hin_Deva +Chhattisgarhi, hne_Deva +Croatian, hrv_Latn +Hungarian, hun_Latn +Armenian, hye_Armn +Igbo, ibo_Latn +Ilocano, ilo_Latn +Indonesian, ind_Latn +Icelandic, isl_Latn +Italian, ita_Latn +Javanese, jav_Latn +Japanese, jpn_Jpan +Kabyle, kab_Latn +Jingpho, kac_Latn +Kamba, kam_Latn +Kannada, kan_Knda +Kashmiri (Arabic script), kas_Arab +Kashmiri (Devanagari script), kas_Deva +Georgian, kat_Geor +Central Kanuri (Arabic script), knc_Arab +Central Kanuri (Latin script), knc_Latn +Kazakh, kaz_Cyrl +Kabiyè, kbp_Latn +Kabuverdianu, kea_Latn +Khmer, khm_Khmr +Kikuyu, kik_Latn +Kinyarwanda, kin_Latn +Kyrgyz, kir_Cyrl +Kimbundu, kmb_Latn +Northern Kurdish, kmr_Latn +Kikongo, kon_Latn +Korean, kor_Hang +Lao, lao_Laoo +Ligurian, lij_Latn +Limburgish, lim_Latn +Lingala, lin_Latn +Lithuanian, lit_Latn +Lombard, lmo_Latn +Latgalian, ltg_Latn +Luxembourgish, ltz_Latn +Luba-Kasai, lua_Latn +Ganda, lug_Latn +Luo, luo_Latn +Mizo, lus_Latn +Standard Latvian, lvs_Latn +Magahi, mag_Deva +Maithili, mai_Deva +Malayalam, mal_Mlym +Marathi, mar_Deva +Minangkabau (Arabic script), min_Arab +Minangkabau (Latin script), min_Latn +Macedonian, mkd_Cyrl +Plateau Malagasy, plt_Latn +Maltese, mlt_Latn +Meitei (Bengali script), mni_Beng +Halh Mongolian, khk_Cyrl +Mossi, mos_Latn +Maori, mri_Latn +Burmese, mya_Mymr +Dutch, nld_Latn +Norwegian Nynorsk, nno_Latn +Norwegian Bokmål, nob_Latn +Nepali, npi_Deva +Northern Sotho, nso_Latn +Nuer, nus_Latn +Nyanja, nya_Latn +Occitan, oci_Latn +West Central Oromo, gaz_Latn +Odia, ory_Orya +Pangasinan, pag_Latn +Eastern Panjabi, pan_Guru +Papiamento, pap_Latn +Western Persian, pes_Arab +Polish, pol_Latn +Portuguese, por_Latn +Dari, prs_Arab +Southern Pashto, pbt_Arab +Ayacucho Quechua, quy_Latn +Romanian, ron_Latn +Rundi, run_Latn +Russian, rus_Cyrl +Sango, sag_Latn +Sanskrit, san_Deva +Santali, sat_Olck +Sicilian, scn_Latn +Shan, shn_Mymr +Sinhala, sin_Sinh +Slovak, slk_Latn +Slovenian, slv_Latn +Samoan, smo_Latn +Shona, sna_Latn +Sindhi, snd_Arab +Somali, som_Latn +Southern Sotho, sot_Latn +Spanish, spa_Latn +Tosk Albanian, als_Latn +Sardinian, srd_Latn +Serbian, srp_Cyrl +Swati, ssw_Latn +Sundanese, sun_Latn +Swedish, swe_Latn +Swahili, swh_Latn +Silesian, szl_Latn +Tamil, tam_Taml +Tatar, tat_Cyrl +Telugu, tel_Telu +Tajik, tgk_Cyrl +Tagalog, tgl_Latn +Thai, tha_Thai +Tigrinya, tir_Ethi +Tamasheq (Latin script), taq_Latn +Tamasheq (Tifinagh script), taq_Tfng +Tok Pisin, tpi_Latn +Tswana, tsn_Latn +Tsonga, tso_Latn +Turkmen, tuk_Latn +Tumbuka, tum_Latn +Turkish, tur_Latn +Twi, twi_Latn +Central Atlas Tamazight, tzm_Tfng +Uyghur, uig_Arab +Ukrainian, ukr_Cyrl +Umbundu, umb_Latn +Urdu, urd_Arab +Northern Uzbek, uzn_Latn +Venetian, vec_Latn +Vietnamese, vie_Latn +Waray, war_Latn +Wolof, wol_Latn +Xhosa, xho_Latn +Eastern Yiddish, ydd_Hebr +Yoruba, yor_Latn +Yue Chinese, yue_Hant +Chinese (Simplified), zho_Hans +Chinese (Traditional), zho_Hant +Standard Malay, zsm_Latn +Zulu, zul_Latn \ No newline at end of file diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm new file mode 100644 index 00000000..2bdf4838 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm @@ -0,0 +1,6 @@ + 1.00000000 1.00000000 + 3.00000000 1.00000000 + 1.00000000 1.00000000 + 1.00000000 1.00000000 + 3.00000000 1.00000000 + 3.00000000 1.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights new file mode 100644 index 00000000..67a757d9 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.weights @@ -0,0 +1 @@ +3 3 10.000000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp new file mode 100644 index 00000000..a3fd47b9 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/lm/trg.lm.wp @@ -0,0 +1 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg new file mode 100644 index 00000000..b438a898 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/smt.cfg @@ -0,0 +1,29 @@ +# Translation model prefix +-tm tm/src_trg + +# Language model +-lm lm/trg.lm + +# W parameter (maximum number of translation options to be considered per each source phrase) +-W 10 + +# S parameter (maximum number of hypotheses that can be stored in each stack) +-S 10 + +# A parameter (Maximum length in words of the source phrases to be translated) +-A 7 + +# Degree of non-monotonicity +-nomon 0 + +# Heuristic function used +-h 6 + +# Best-first search flag +-be + +# Translation model weights +-tmw 0 0.5 1 1 1 1 0 1 + +# Set online learning parameters (ol_alg, lr_policy, l_stepsize, em_iters, e_par, r_par) +-olp 0 0 1 5 1 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda new file mode 100644 index 00000000..6e6566ce --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.lambda @@ -0,0 +1 @@ +0.01 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable new file mode 100644 index 00000000..7277bf4c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.seglentable @@ -0,0 +1,10201 @@ +0 0 1 +0 1 1 +0 2 1 +0 3 1 +0 4 1 +0 5 1 +0 6 1 +0 7 1 +0 8 1 +0 9 1 +0 10 1 +0 11 1 +0 12 1 +0 13 1 +0 14 1 +0 15 1 +0 16 1 +0 17 1 +0 18 1 +0 19 1 +0 20 1 +0 21 1 +0 22 1 +0 23 1 +0 24 1 +0 25 1 +0 26 1 +0 27 1 +0 28 1 +0 29 1 +0 30 1 +0 31 1 +0 32 1 +0 33 1 +0 34 1 +0 35 1 +0 36 1 +0 37 1 +0 38 1 +0 39 1 +0 40 1 +0 41 1 +0 42 1 +0 43 1 +0 44 1 +0 45 1 +0 46 1 +0 47 1 +0 48 1 +0 49 1 +0 50 1 +0 51 1 +0 52 1 +0 53 1 +0 54 1 +0 55 1 +0 56 1 +0 57 1 +0 58 1 +0 59 1 +0 60 1 +0 61 1 +0 62 1 +0 63 1 +0 64 1 +0 65 1 +0 66 1 +0 67 1 +0 68 1 +0 69 1 +0 70 1 +0 71 1 +0 72 1 +0 73 1 +0 74 1 +0 75 1 +0 76 1 +0 77 1 +0 78 1 +0 79 1 +0 80 1 +0 81 1 +0 82 1 +0 83 1 +0 84 1 +0 85 1 +0 86 1 +0 87 1 +0 88 1 +0 89 1 +0 90 1 +0 91 1 +0 92 1 +0 93 1 +0 94 1 +0 95 1 +0 96 1 +0 97 1 +0 98 1 +0 99 1 +0 100 1 +1 0 1 +1 1 1 +1 2 1 +1 3 1 +1 4 1 +1 5 1 +1 6 1 +1 7 1 +1 8 1 +1 9 1 +1 10 1 +1 11 1 +1 12 1 +1 13 1 +1 14 1 +1 15 1 +1 16 1 +1 17 1 +1 18 1 +1 19 1 +1 20 1 +1 21 1 +1 22 1 +1 23 1 +1 24 1 +1 25 1 +1 26 1 +1 27 1 +1 28 1 +1 29 1 +1 30 1 +1 31 1 +1 32 1 +1 33 1 +1 34 1 +1 35 1 +1 36 1 +1 37 1 +1 38 1 +1 39 1 +1 40 1 +1 41 1 +1 42 1 +1 43 1 +1 44 1 +1 45 1 +1 46 1 +1 47 1 +1 48 1 +1 49 1 +1 50 1 +1 51 1 +1 52 1 +1 53 1 +1 54 1 +1 55 1 +1 56 1 +1 57 1 +1 58 1 +1 59 1 +1 60 1 +1 61 1 +1 62 1 +1 63 1 +1 64 1 +1 65 1 +1 66 1 +1 67 1 +1 68 1 +1 69 1 +1 70 1 +1 71 1 +1 72 1 +1 73 1 +1 74 1 +1 75 1 +1 76 1 +1 77 1 +1 78 1 +1 79 1 +1 80 1 +1 81 1 +1 82 1 +1 83 1 +1 84 1 +1 85 1 +1 86 1 +1 87 1 +1 88 1 +1 89 1 +1 90 1 +1 91 1 +1 92 1 +1 93 1 +1 94 1 +1 95 1 +1 96 1 +1 97 1 +1 98 1 +1 99 1 +1 100 1 +2 0 1 +2 1 1 +2 2 1 +2 3 1 +2 4 1 +2 5 1 +2 6 1 +2 7 1 +2 8 1 +2 9 1 +2 10 1 +2 11 1 +2 12 1 +2 13 1 +2 14 1 +2 15 1 +2 16 1 +2 17 1 +2 18 1 +2 19 1 +2 20 1 +2 21 1 +2 22 1 +2 23 1 +2 24 1 +2 25 1 +2 26 1 +2 27 1 +2 28 1 +2 29 1 +2 30 1 +2 31 1 +2 32 1 +2 33 1 +2 34 1 +2 35 1 +2 36 1 +2 37 1 +2 38 1 +2 39 1 +2 40 1 +2 41 1 +2 42 1 +2 43 1 +2 44 1 +2 45 1 +2 46 1 +2 47 1 +2 48 1 +2 49 1 +2 50 1 +2 51 1 +2 52 1 +2 53 1 +2 54 1 +2 55 1 +2 56 1 +2 57 1 +2 58 1 +2 59 1 +2 60 1 +2 61 1 +2 62 1 +2 63 1 +2 64 1 +2 65 1 +2 66 1 +2 67 1 +2 68 1 +2 69 1 +2 70 1 +2 71 1 +2 72 1 +2 73 1 +2 74 1 +2 75 1 +2 76 1 +2 77 1 +2 78 1 +2 79 1 +2 80 1 +2 81 1 +2 82 1 +2 83 1 +2 84 1 +2 85 1 +2 86 1 +2 87 1 +2 88 1 +2 89 1 +2 90 1 +2 91 1 +2 92 1 +2 93 1 +2 94 1 +2 95 1 +2 96 1 +2 97 1 +2 98 1 +2 99 1 +2 100 1 +3 0 1 +3 1 1 +3 2 1 +3 3 1 +3 4 1 +3 5 1 +3 6 1 +3 7 1 +3 8 1 +3 9 1 +3 10 1 +3 11 1 +3 12 1 +3 13 1 +3 14 1 +3 15 1 +3 16 1 +3 17 1 +3 18 1 +3 19 1 +3 20 1 +3 21 1 +3 22 1 +3 23 1 +3 24 1 +3 25 1 +3 26 1 +3 27 1 +3 28 1 +3 29 1 +3 30 1 +3 31 1 +3 32 1 +3 33 1 +3 34 1 +3 35 1 +3 36 1 +3 37 1 +3 38 1 +3 39 1 +3 40 1 +3 41 1 +3 42 1 +3 43 1 +3 44 1 +3 45 1 +3 46 1 +3 47 1 +3 48 1 +3 49 1 +3 50 1 +3 51 1 +3 52 1 +3 53 1 +3 54 1 +3 55 1 +3 56 1 +3 57 1 +3 58 1 +3 59 1 +3 60 1 +3 61 1 +3 62 1 +3 63 1 +3 64 1 +3 65 1 +3 66 1 +3 67 1 +3 68 1 +3 69 1 +3 70 1 +3 71 1 +3 72 1 +3 73 1 +3 74 1 +3 75 1 +3 76 1 +3 77 1 +3 78 1 +3 79 1 +3 80 1 +3 81 1 +3 82 1 +3 83 1 +3 84 1 +3 85 1 +3 86 1 +3 87 1 +3 88 1 +3 89 1 +3 90 1 +3 91 1 +3 92 1 +3 93 1 +3 94 1 +3 95 1 +3 96 1 +3 97 1 +3 98 1 +3 99 1 +3 100 1 +4 0 1 +4 1 1 +4 2 1 +4 3 1 +4 4 1 +4 5 1 +4 6 1 +4 7 1 +4 8 1 +4 9 1 +4 10 1 +4 11 1 +4 12 1 +4 13 1 +4 14 1 +4 15 1 +4 16 1 +4 17 1 +4 18 1 +4 19 1 +4 20 1 +4 21 1 +4 22 1 +4 23 1 +4 24 1 +4 25 1 +4 26 1 +4 27 1 +4 28 1 +4 29 1 +4 30 1 +4 31 1 +4 32 1 +4 33 1 +4 34 1 +4 35 1 +4 36 1 +4 37 1 +4 38 1 +4 39 1 +4 40 1 +4 41 1 +4 42 1 +4 43 1 +4 44 1 +4 45 1 +4 46 1 +4 47 1 +4 48 1 +4 49 1 +4 50 1 +4 51 1 +4 52 1 +4 53 1 +4 54 1 +4 55 1 +4 56 1 +4 57 1 +4 58 1 +4 59 1 +4 60 1 +4 61 1 +4 62 1 +4 63 1 +4 64 1 +4 65 1 +4 66 1 +4 67 1 +4 68 1 +4 69 1 +4 70 1 +4 71 1 +4 72 1 +4 73 1 +4 74 1 +4 75 1 +4 76 1 +4 77 1 +4 78 1 +4 79 1 +4 80 1 +4 81 1 +4 82 1 +4 83 1 +4 84 1 +4 85 1 +4 86 1 +4 87 1 +4 88 1 +4 89 1 +4 90 1 +4 91 1 +4 92 1 +4 93 1 +4 94 1 +4 95 1 +4 96 1 +4 97 1 +4 98 1 +4 99 1 +4 100 1 +5 0 1 +5 1 1 +5 2 1 +5 3 1 +5 4 1 +5 5 1 +5 6 1 +5 7 1 +5 8 1 +5 9 1 +5 10 1 +5 11 1 +5 12 1 +5 13 1 +5 14 1 +5 15 1 +5 16 1 +5 17 1 +5 18 1 +5 19 1 +5 20 1 +5 21 1 +5 22 1 +5 23 1 +5 24 1 +5 25 1 +5 26 1 +5 27 1 +5 28 1 +5 29 1 +5 30 1 +5 31 1 +5 32 1 +5 33 1 +5 34 1 +5 35 1 +5 36 1 +5 37 1 +5 38 1 +5 39 1 +5 40 1 +5 41 1 +5 42 1 +5 43 1 +5 44 1 +5 45 1 +5 46 1 +5 47 1 +5 48 1 +5 49 1 +5 50 1 +5 51 1 +5 52 1 +5 53 1 +5 54 1 +5 55 1 +5 56 1 +5 57 1 +5 58 1 +5 59 1 +5 60 1 +5 61 1 +5 62 1 +5 63 1 +5 64 1 +5 65 1 +5 66 1 +5 67 1 +5 68 1 +5 69 1 +5 70 1 +5 71 1 +5 72 1 +5 73 1 +5 74 1 +5 75 1 +5 76 1 +5 77 1 +5 78 1 +5 79 1 +5 80 1 +5 81 1 +5 82 1 +5 83 1 +5 84 1 +5 85 1 +5 86 1 +5 87 1 +5 88 1 +5 89 1 +5 90 1 +5 91 1 +5 92 1 +5 93 1 +5 94 1 +5 95 1 +5 96 1 +5 97 1 +5 98 1 +5 99 1 +5 100 1 +6 0 1 +6 1 1 +6 2 1 +6 3 1 +6 4 1 +6 5 1 +6 6 1 +6 7 1 +6 8 1 +6 9 1 +6 10 1 +6 11 1 +6 12 1 +6 13 1 +6 14 1 +6 15 1 +6 16 1 +6 17 1 +6 18 1 +6 19 1 +6 20 1 +6 21 1 +6 22 1 +6 23 1 +6 24 1 +6 25 1 +6 26 1 +6 27 1 +6 28 1 +6 29 1 +6 30 1 +6 31 1 +6 32 1 +6 33 1 +6 34 1 +6 35 1 +6 36 1 +6 37 1 +6 38 1 +6 39 1 +6 40 1 +6 41 1 +6 42 1 +6 43 1 +6 44 1 +6 45 1 +6 46 1 +6 47 1 +6 48 1 +6 49 1 +6 50 1 +6 51 1 +6 52 1 +6 53 1 +6 54 1 +6 55 1 +6 56 1 +6 57 1 +6 58 1 +6 59 1 +6 60 1 +6 61 1 +6 62 1 +6 63 1 +6 64 1 +6 65 1 +6 66 1 +6 67 1 +6 68 1 +6 69 1 +6 70 1 +6 71 1 +6 72 1 +6 73 1 +6 74 1 +6 75 1 +6 76 1 +6 77 1 +6 78 1 +6 79 1 +6 80 1 +6 81 1 +6 82 1 +6 83 1 +6 84 1 +6 85 1 +6 86 1 +6 87 1 +6 88 1 +6 89 1 +6 90 1 +6 91 1 +6 92 1 +6 93 1 +6 94 1 +6 95 1 +6 96 1 +6 97 1 +6 98 1 +6 99 1 +6 100 1 +7 0 1 +7 1 1 +7 2 1 +7 3 1 +7 4 1 +7 5 1 +7 6 1 +7 7 1 +7 8 1 +7 9 1 +7 10 1 +7 11 1 +7 12 1 +7 13 1 +7 14 1 +7 15 1 +7 16 1 +7 17 1 +7 18 1 +7 19 1 +7 20 1 +7 21 1 +7 22 1 +7 23 1 +7 24 1 +7 25 1 +7 26 1 +7 27 1 +7 28 1 +7 29 1 +7 30 1 +7 31 1 +7 32 1 +7 33 1 +7 34 1 +7 35 1 +7 36 1 +7 37 1 +7 38 1 +7 39 1 +7 40 1 +7 41 1 +7 42 1 +7 43 1 +7 44 1 +7 45 1 +7 46 1 +7 47 1 +7 48 1 +7 49 1 +7 50 1 +7 51 1 +7 52 1 +7 53 1 +7 54 1 +7 55 1 +7 56 1 +7 57 1 +7 58 1 +7 59 1 +7 60 1 +7 61 1 +7 62 1 +7 63 1 +7 64 1 +7 65 1 +7 66 1 +7 67 1 +7 68 1 +7 69 1 +7 70 1 +7 71 1 +7 72 1 +7 73 1 +7 74 1 +7 75 1 +7 76 1 +7 77 1 +7 78 1 +7 79 1 +7 80 1 +7 81 1 +7 82 1 +7 83 1 +7 84 1 +7 85 1 +7 86 1 +7 87 1 +7 88 1 +7 89 1 +7 90 1 +7 91 1 +7 92 1 +7 93 1 +7 94 1 +7 95 1 +7 96 1 +7 97 1 +7 98 1 +7 99 1 +7 100 1 +8 0 1 +8 1 1 +8 2 1 +8 3 1 +8 4 1 +8 5 1 +8 6 1 +8 7 1 +8 8 1 +8 9 1 +8 10 1 +8 11 1 +8 12 1 +8 13 1 +8 14 1 +8 15 1 +8 16 1 +8 17 1 +8 18 1 +8 19 1 +8 20 1 +8 21 1 +8 22 1 +8 23 1 +8 24 1 +8 25 1 +8 26 1 +8 27 1 +8 28 1 +8 29 1 +8 30 1 +8 31 1 +8 32 1 +8 33 1 +8 34 1 +8 35 1 +8 36 1 +8 37 1 +8 38 1 +8 39 1 +8 40 1 +8 41 1 +8 42 1 +8 43 1 +8 44 1 +8 45 1 +8 46 1 +8 47 1 +8 48 1 +8 49 1 +8 50 1 +8 51 1 +8 52 1 +8 53 1 +8 54 1 +8 55 1 +8 56 1 +8 57 1 +8 58 1 +8 59 1 +8 60 1 +8 61 1 +8 62 1 +8 63 1 +8 64 1 +8 65 1 +8 66 1 +8 67 1 +8 68 1 +8 69 1 +8 70 1 +8 71 1 +8 72 1 +8 73 1 +8 74 1 +8 75 1 +8 76 1 +8 77 1 +8 78 1 +8 79 1 +8 80 1 +8 81 1 +8 82 1 +8 83 1 +8 84 1 +8 85 1 +8 86 1 +8 87 1 +8 88 1 +8 89 1 +8 90 1 +8 91 1 +8 92 1 +8 93 1 +8 94 1 +8 95 1 +8 96 1 +8 97 1 +8 98 1 +8 99 1 +8 100 1 +9 0 1 +9 1 1 +9 2 1 +9 3 1 +9 4 1 +9 5 1 +9 6 1 +9 7 1 +9 8 1 +9 9 1 +9 10 1 +9 11 1 +9 12 1 +9 13 1 +9 14 1 +9 15 1 +9 16 1 +9 17 1 +9 18 1 +9 19 1 +9 20 1 +9 21 1 +9 22 1 +9 23 1 +9 24 1 +9 25 1 +9 26 1 +9 27 1 +9 28 1 +9 29 1 +9 30 1 +9 31 1 +9 32 1 +9 33 1 +9 34 1 +9 35 1 +9 36 1 +9 37 1 +9 38 1 +9 39 1 +9 40 1 +9 41 1 +9 42 1 +9 43 1 +9 44 1 +9 45 1 +9 46 1 +9 47 1 +9 48 1 +9 49 1 +9 50 1 +9 51 1 +9 52 1 +9 53 1 +9 54 1 +9 55 1 +9 56 1 +9 57 1 +9 58 1 +9 59 1 +9 60 1 +9 61 1 +9 62 1 +9 63 1 +9 64 1 +9 65 1 +9 66 1 +9 67 1 +9 68 1 +9 69 1 +9 70 1 +9 71 1 +9 72 1 +9 73 1 +9 74 1 +9 75 1 +9 76 1 +9 77 1 +9 78 1 +9 79 1 +9 80 1 +9 81 1 +9 82 1 +9 83 1 +9 84 1 +9 85 1 +9 86 1 +9 87 1 +9 88 1 +9 89 1 +9 90 1 +9 91 1 +9 92 1 +9 93 1 +9 94 1 +9 95 1 +9 96 1 +9 97 1 +9 98 1 +9 99 1 +9 100 1 +10 0 1 +10 1 1 +10 2 1 +10 3 1 +10 4 1 +10 5 1 +10 6 1 +10 7 1 +10 8 1 +10 9 1 +10 10 1 +10 11 1 +10 12 1 +10 13 1 +10 14 1 +10 15 1 +10 16 1 +10 17 1 +10 18 1 +10 19 1 +10 20 1 +10 21 1 +10 22 1 +10 23 1 +10 24 1 +10 25 1 +10 26 1 +10 27 1 +10 28 1 +10 29 1 +10 30 1 +10 31 1 +10 32 1 +10 33 1 +10 34 1 +10 35 1 +10 36 1 +10 37 1 +10 38 1 +10 39 1 +10 40 1 +10 41 1 +10 42 1 +10 43 1 +10 44 1 +10 45 1 +10 46 1 +10 47 1 +10 48 1 +10 49 1 +10 50 1 +10 51 1 +10 52 1 +10 53 1 +10 54 1 +10 55 1 +10 56 1 +10 57 1 +10 58 1 +10 59 1 +10 60 1 +10 61 1 +10 62 1 +10 63 1 +10 64 1 +10 65 1 +10 66 1 +10 67 1 +10 68 1 +10 69 1 +10 70 1 +10 71 1 +10 72 1 +10 73 1 +10 74 1 +10 75 1 +10 76 1 +10 77 1 +10 78 1 +10 79 1 +10 80 1 +10 81 1 +10 82 1 +10 83 1 +10 84 1 +10 85 1 +10 86 1 +10 87 1 +10 88 1 +10 89 1 +10 90 1 +10 91 1 +10 92 1 +10 93 1 +10 94 1 +10 95 1 +10 96 1 +10 97 1 +10 98 1 +10 99 1 +10 100 1 +11 0 1 +11 1 1 +11 2 1 +11 3 1 +11 4 1 +11 5 1 +11 6 1 +11 7 1 +11 8 1 +11 9 1 +11 10 1 +11 11 1 +11 12 1 +11 13 1 +11 14 1 +11 15 1 +11 16 1 +11 17 1 +11 18 1 +11 19 1 +11 20 1 +11 21 1 +11 22 1 +11 23 1 +11 24 1 +11 25 1 +11 26 1 +11 27 1 +11 28 1 +11 29 1 +11 30 1 +11 31 1 +11 32 1 +11 33 1 +11 34 1 +11 35 1 +11 36 1 +11 37 1 +11 38 1 +11 39 1 +11 40 1 +11 41 1 +11 42 1 +11 43 1 +11 44 1 +11 45 1 +11 46 1 +11 47 1 +11 48 1 +11 49 1 +11 50 1 +11 51 1 +11 52 1 +11 53 1 +11 54 1 +11 55 1 +11 56 1 +11 57 1 +11 58 1 +11 59 1 +11 60 1 +11 61 1 +11 62 1 +11 63 1 +11 64 1 +11 65 1 +11 66 1 +11 67 1 +11 68 1 +11 69 1 +11 70 1 +11 71 1 +11 72 1 +11 73 1 +11 74 1 +11 75 1 +11 76 1 +11 77 1 +11 78 1 +11 79 1 +11 80 1 +11 81 1 +11 82 1 +11 83 1 +11 84 1 +11 85 1 +11 86 1 +11 87 1 +11 88 1 +11 89 1 +11 90 1 +11 91 1 +11 92 1 +11 93 1 +11 94 1 +11 95 1 +11 96 1 +11 97 1 +11 98 1 +11 99 1 +11 100 1 +12 0 1 +12 1 1 +12 2 1 +12 3 1 +12 4 1 +12 5 1 +12 6 1 +12 7 1 +12 8 1 +12 9 1 +12 10 1 +12 11 1 +12 12 1 +12 13 1 +12 14 1 +12 15 1 +12 16 1 +12 17 1 +12 18 1 +12 19 1 +12 20 1 +12 21 1 +12 22 1 +12 23 1 +12 24 1 +12 25 1 +12 26 1 +12 27 1 +12 28 1 +12 29 1 +12 30 1 +12 31 1 +12 32 1 +12 33 1 +12 34 1 +12 35 1 +12 36 1 +12 37 1 +12 38 1 +12 39 1 +12 40 1 +12 41 1 +12 42 1 +12 43 1 +12 44 1 +12 45 1 +12 46 1 +12 47 1 +12 48 1 +12 49 1 +12 50 1 +12 51 1 +12 52 1 +12 53 1 +12 54 1 +12 55 1 +12 56 1 +12 57 1 +12 58 1 +12 59 1 +12 60 1 +12 61 1 +12 62 1 +12 63 1 +12 64 1 +12 65 1 +12 66 1 +12 67 1 +12 68 1 +12 69 1 +12 70 1 +12 71 1 +12 72 1 +12 73 1 +12 74 1 +12 75 1 +12 76 1 +12 77 1 +12 78 1 +12 79 1 +12 80 1 +12 81 1 +12 82 1 +12 83 1 +12 84 1 +12 85 1 +12 86 1 +12 87 1 +12 88 1 +12 89 1 +12 90 1 +12 91 1 +12 92 1 +12 93 1 +12 94 1 +12 95 1 +12 96 1 +12 97 1 +12 98 1 +12 99 1 +12 100 1 +13 0 1 +13 1 1 +13 2 1 +13 3 1 +13 4 1 +13 5 1 +13 6 1 +13 7 1 +13 8 1 +13 9 1 +13 10 1 +13 11 1 +13 12 1 +13 13 1 +13 14 1 +13 15 1 +13 16 1 +13 17 1 +13 18 1 +13 19 1 +13 20 1 +13 21 1 +13 22 1 +13 23 1 +13 24 1 +13 25 1 +13 26 1 +13 27 1 +13 28 1 +13 29 1 +13 30 1 +13 31 1 +13 32 1 +13 33 1 +13 34 1 +13 35 1 +13 36 1 +13 37 1 +13 38 1 +13 39 1 +13 40 1 +13 41 1 +13 42 1 +13 43 1 +13 44 1 +13 45 1 +13 46 1 +13 47 1 +13 48 1 +13 49 1 +13 50 1 +13 51 1 +13 52 1 +13 53 1 +13 54 1 +13 55 1 +13 56 1 +13 57 1 +13 58 1 +13 59 1 +13 60 1 +13 61 1 +13 62 1 +13 63 1 +13 64 1 +13 65 1 +13 66 1 +13 67 1 +13 68 1 +13 69 1 +13 70 1 +13 71 1 +13 72 1 +13 73 1 +13 74 1 +13 75 1 +13 76 1 +13 77 1 +13 78 1 +13 79 1 +13 80 1 +13 81 1 +13 82 1 +13 83 1 +13 84 1 +13 85 1 +13 86 1 +13 87 1 +13 88 1 +13 89 1 +13 90 1 +13 91 1 +13 92 1 +13 93 1 +13 94 1 +13 95 1 +13 96 1 +13 97 1 +13 98 1 +13 99 1 +13 100 1 +14 0 1 +14 1 1 +14 2 1 +14 3 1 +14 4 1 +14 5 1 +14 6 1 +14 7 1 +14 8 1 +14 9 1 +14 10 1 +14 11 1 +14 12 1 +14 13 1 +14 14 1 +14 15 1 +14 16 1 +14 17 1 +14 18 1 +14 19 1 +14 20 1 +14 21 1 +14 22 1 +14 23 1 +14 24 1 +14 25 1 +14 26 1 +14 27 1 +14 28 1 +14 29 1 +14 30 1 +14 31 1 +14 32 1 +14 33 1 +14 34 1 +14 35 1 +14 36 1 +14 37 1 +14 38 1 +14 39 1 +14 40 1 +14 41 1 +14 42 1 +14 43 1 +14 44 1 +14 45 1 +14 46 1 +14 47 1 +14 48 1 +14 49 1 +14 50 1 +14 51 1 +14 52 1 +14 53 1 +14 54 1 +14 55 1 +14 56 1 +14 57 1 +14 58 1 +14 59 1 +14 60 1 +14 61 1 +14 62 1 +14 63 1 +14 64 1 +14 65 1 +14 66 1 +14 67 1 +14 68 1 +14 69 1 +14 70 1 +14 71 1 +14 72 1 +14 73 1 +14 74 1 +14 75 1 +14 76 1 +14 77 1 +14 78 1 +14 79 1 +14 80 1 +14 81 1 +14 82 1 +14 83 1 +14 84 1 +14 85 1 +14 86 1 +14 87 1 +14 88 1 +14 89 1 +14 90 1 +14 91 1 +14 92 1 +14 93 1 +14 94 1 +14 95 1 +14 96 1 +14 97 1 +14 98 1 +14 99 1 +14 100 1 +15 0 1 +15 1 1 +15 2 1 +15 3 1 +15 4 1 +15 5 1 +15 6 1 +15 7 1 +15 8 1 +15 9 1 +15 10 1 +15 11 1 +15 12 1 +15 13 1 +15 14 1 +15 15 1 +15 16 1 +15 17 1 +15 18 1 +15 19 1 +15 20 1 +15 21 1 +15 22 1 +15 23 1 +15 24 1 +15 25 1 +15 26 1 +15 27 1 +15 28 1 +15 29 1 +15 30 1 +15 31 1 +15 32 1 +15 33 1 +15 34 1 +15 35 1 +15 36 1 +15 37 1 +15 38 1 +15 39 1 +15 40 1 +15 41 1 +15 42 1 +15 43 1 +15 44 1 +15 45 1 +15 46 1 +15 47 1 +15 48 1 +15 49 1 +15 50 1 +15 51 1 +15 52 1 +15 53 1 +15 54 1 +15 55 1 +15 56 1 +15 57 1 +15 58 1 +15 59 1 +15 60 1 +15 61 1 +15 62 1 +15 63 1 +15 64 1 +15 65 1 +15 66 1 +15 67 1 +15 68 1 +15 69 1 +15 70 1 +15 71 1 +15 72 1 +15 73 1 +15 74 1 +15 75 1 +15 76 1 +15 77 1 +15 78 1 +15 79 1 +15 80 1 +15 81 1 +15 82 1 +15 83 1 +15 84 1 +15 85 1 +15 86 1 +15 87 1 +15 88 1 +15 89 1 +15 90 1 +15 91 1 +15 92 1 +15 93 1 +15 94 1 +15 95 1 +15 96 1 +15 97 1 +15 98 1 +15 99 1 +15 100 1 +16 0 1 +16 1 1 +16 2 1 +16 3 1 +16 4 1 +16 5 1 +16 6 1 +16 7 1 +16 8 1 +16 9 1 +16 10 1 +16 11 1 +16 12 1 +16 13 1 +16 14 1 +16 15 1 +16 16 1 +16 17 1 +16 18 1 +16 19 1 +16 20 1 +16 21 1 +16 22 1 +16 23 1 +16 24 1 +16 25 1 +16 26 1 +16 27 1 +16 28 1 +16 29 1 +16 30 1 +16 31 1 +16 32 1 +16 33 1 +16 34 1 +16 35 1 +16 36 1 +16 37 1 +16 38 1 +16 39 1 +16 40 1 +16 41 1 +16 42 1 +16 43 1 +16 44 1 +16 45 1 +16 46 1 +16 47 1 +16 48 1 +16 49 1 +16 50 1 +16 51 1 +16 52 1 +16 53 1 +16 54 1 +16 55 1 +16 56 1 +16 57 1 +16 58 1 +16 59 1 +16 60 1 +16 61 1 +16 62 1 +16 63 1 +16 64 1 +16 65 1 +16 66 1 +16 67 1 +16 68 1 +16 69 1 +16 70 1 +16 71 1 +16 72 1 +16 73 1 +16 74 1 +16 75 1 +16 76 1 +16 77 1 +16 78 1 +16 79 1 +16 80 1 +16 81 1 +16 82 1 +16 83 1 +16 84 1 +16 85 1 +16 86 1 +16 87 1 +16 88 1 +16 89 1 +16 90 1 +16 91 1 +16 92 1 +16 93 1 +16 94 1 +16 95 1 +16 96 1 +16 97 1 +16 98 1 +16 99 1 +16 100 1 +17 0 1 +17 1 1 +17 2 1 +17 3 1 +17 4 1 +17 5 1 +17 6 1 +17 7 1 +17 8 1 +17 9 1 +17 10 1 +17 11 1 +17 12 1 +17 13 1 +17 14 1 +17 15 1 +17 16 1 +17 17 1 +17 18 1 +17 19 1 +17 20 1 +17 21 1 +17 22 1 +17 23 1 +17 24 1 +17 25 1 +17 26 1 +17 27 1 +17 28 1 +17 29 1 +17 30 1 +17 31 1 +17 32 1 +17 33 1 +17 34 1 +17 35 1 +17 36 1 +17 37 1 +17 38 1 +17 39 1 +17 40 1 +17 41 1 +17 42 1 +17 43 1 +17 44 1 +17 45 1 +17 46 1 +17 47 1 +17 48 1 +17 49 1 +17 50 1 +17 51 1 +17 52 1 +17 53 1 +17 54 1 +17 55 1 +17 56 1 +17 57 1 +17 58 1 +17 59 1 +17 60 1 +17 61 1 +17 62 1 +17 63 1 +17 64 1 +17 65 1 +17 66 1 +17 67 1 +17 68 1 +17 69 1 +17 70 1 +17 71 1 +17 72 1 +17 73 1 +17 74 1 +17 75 1 +17 76 1 +17 77 1 +17 78 1 +17 79 1 +17 80 1 +17 81 1 +17 82 1 +17 83 1 +17 84 1 +17 85 1 +17 86 1 +17 87 1 +17 88 1 +17 89 1 +17 90 1 +17 91 1 +17 92 1 +17 93 1 +17 94 1 +17 95 1 +17 96 1 +17 97 1 +17 98 1 +17 99 1 +17 100 1 +18 0 1 +18 1 1 +18 2 1 +18 3 1 +18 4 1 +18 5 1 +18 6 1 +18 7 1 +18 8 1 +18 9 1 +18 10 1 +18 11 1 +18 12 1 +18 13 1 +18 14 1 +18 15 1 +18 16 1 +18 17 1 +18 18 1 +18 19 1 +18 20 1 +18 21 1 +18 22 1 +18 23 1 +18 24 1 +18 25 1 +18 26 1 +18 27 1 +18 28 1 +18 29 1 +18 30 1 +18 31 1 +18 32 1 +18 33 1 +18 34 1 +18 35 1 +18 36 1 +18 37 1 +18 38 1 +18 39 1 +18 40 1 +18 41 1 +18 42 1 +18 43 1 +18 44 1 +18 45 1 +18 46 1 +18 47 1 +18 48 1 +18 49 1 +18 50 1 +18 51 1 +18 52 1 +18 53 1 +18 54 1 +18 55 1 +18 56 1 +18 57 1 +18 58 1 +18 59 1 +18 60 1 +18 61 1 +18 62 1 +18 63 1 +18 64 1 +18 65 1 +18 66 1 +18 67 1 +18 68 1 +18 69 1 +18 70 1 +18 71 1 +18 72 1 +18 73 1 +18 74 1 +18 75 1 +18 76 1 +18 77 1 +18 78 1 +18 79 1 +18 80 1 +18 81 1 +18 82 1 +18 83 1 +18 84 1 +18 85 1 +18 86 1 +18 87 1 +18 88 1 +18 89 1 +18 90 1 +18 91 1 +18 92 1 +18 93 1 +18 94 1 +18 95 1 +18 96 1 +18 97 1 +18 98 1 +18 99 1 +18 100 1 +19 0 1 +19 1 1 +19 2 1 +19 3 1 +19 4 1 +19 5 1 +19 6 1 +19 7 1 +19 8 1 +19 9 1 +19 10 1 +19 11 1 +19 12 1 +19 13 1 +19 14 1 +19 15 1 +19 16 1 +19 17 1 +19 18 1 +19 19 1 +19 20 1 +19 21 1 +19 22 1 +19 23 1 +19 24 1 +19 25 1 +19 26 1 +19 27 1 +19 28 1 +19 29 1 +19 30 1 +19 31 1 +19 32 1 +19 33 1 +19 34 1 +19 35 1 +19 36 1 +19 37 1 +19 38 1 +19 39 1 +19 40 1 +19 41 1 +19 42 1 +19 43 1 +19 44 1 +19 45 1 +19 46 1 +19 47 1 +19 48 1 +19 49 1 +19 50 1 +19 51 1 +19 52 1 +19 53 1 +19 54 1 +19 55 1 +19 56 1 +19 57 1 +19 58 1 +19 59 1 +19 60 1 +19 61 1 +19 62 1 +19 63 1 +19 64 1 +19 65 1 +19 66 1 +19 67 1 +19 68 1 +19 69 1 +19 70 1 +19 71 1 +19 72 1 +19 73 1 +19 74 1 +19 75 1 +19 76 1 +19 77 1 +19 78 1 +19 79 1 +19 80 1 +19 81 1 +19 82 1 +19 83 1 +19 84 1 +19 85 1 +19 86 1 +19 87 1 +19 88 1 +19 89 1 +19 90 1 +19 91 1 +19 92 1 +19 93 1 +19 94 1 +19 95 1 +19 96 1 +19 97 1 +19 98 1 +19 99 1 +19 100 1 +20 0 1 +20 1 1 +20 2 1 +20 3 1 +20 4 1 +20 5 1 +20 6 1 +20 7 1 +20 8 1 +20 9 1 +20 10 1 +20 11 1 +20 12 1 +20 13 1 +20 14 1 +20 15 1 +20 16 1 +20 17 1 +20 18 1 +20 19 1 +20 20 1 +20 21 1 +20 22 1 +20 23 1 +20 24 1 +20 25 1 +20 26 1 +20 27 1 +20 28 1 +20 29 1 +20 30 1 +20 31 1 +20 32 1 +20 33 1 +20 34 1 +20 35 1 +20 36 1 +20 37 1 +20 38 1 +20 39 1 +20 40 1 +20 41 1 +20 42 1 +20 43 1 +20 44 1 +20 45 1 +20 46 1 +20 47 1 +20 48 1 +20 49 1 +20 50 1 +20 51 1 +20 52 1 +20 53 1 +20 54 1 +20 55 1 +20 56 1 +20 57 1 +20 58 1 +20 59 1 +20 60 1 +20 61 1 +20 62 1 +20 63 1 +20 64 1 +20 65 1 +20 66 1 +20 67 1 +20 68 1 +20 69 1 +20 70 1 +20 71 1 +20 72 1 +20 73 1 +20 74 1 +20 75 1 +20 76 1 +20 77 1 +20 78 1 +20 79 1 +20 80 1 +20 81 1 +20 82 1 +20 83 1 +20 84 1 +20 85 1 +20 86 1 +20 87 1 +20 88 1 +20 89 1 +20 90 1 +20 91 1 +20 92 1 +20 93 1 +20 94 1 +20 95 1 +20 96 1 +20 97 1 +20 98 1 +20 99 1 +20 100 1 +21 0 1 +21 1 1 +21 2 1 +21 3 1 +21 4 1 +21 5 1 +21 6 1 +21 7 1 +21 8 1 +21 9 1 +21 10 1 +21 11 1 +21 12 1 +21 13 1 +21 14 1 +21 15 1 +21 16 1 +21 17 1 +21 18 1 +21 19 1 +21 20 1 +21 21 1 +21 22 1 +21 23 1 +21 24 1 +21 25 1 +21 26 1 +21 27 1 +21 28 1 +21 29 1 +21 30 1 +21 31 1 +21 32 1 +21 33 1 +21 34 1 +21 35 1 +21 36 1 +21 37 1 +21 38 1 +21 39 1 +21 40 1 +21 41 1 +21 42 1 +21 43 1 +21 44 1 +21 45 1 +21 46 1 +21 47 1 +21 48 1 +21 49 1 +21 50 1 +21 51 1 +21 52 1 +21 53 1 +21 54 1 +21 55 1 +21 56 1 +21 57 1 +21 58 1 +21 59 1 +21 60 1 +21 61 1 +21 62 1 +21 63 1 +21 64 1 +21 65 1 +21 66 1 +21 67 1 +21 68 1 +21 69 1 +21 70 1 +21 71 1 +21 72 1 +21 73 1 +21 74 1 +21 75 1 +21 76 1 +21 77 1 +21 78 1 +21 79 1 +21 80 1 +21 81 1 +21 82 1 +21 83 1 +21 84 1 +21 85 1 +21 86 1 +21 87 1 +21 88 1 +21 89 1 +21 90 1 +21 91 1 +21 92 1 +21 93 1 +21 94 1 +21 95 1 +21 96 1 +21 97 1 +21 98 1 +21 99 1 +21 100 1 +22 0 1 +22 1 1 +22 2 1 +22 3 1 +22 4 1 +22 5 1 +22 6 1 +22 7 1 +22 8 1 +22 9 1 +22 10 1 +22 11 1 +22 12 1 +22 13 1 +22 14 1 +22 15 1 +22 16 1 +22 17 1 +22 18 1 +22 19 1 +22 20 1 +22 21 1 +22 22 1 +22 23 1 +22 24 1 +22 25 1 +22 26 1 +22 27 1 +22 28 1 +22 29 1 +22 30 1 +22 31 1 +22 32 1 +22 33 1 +22 34 1 +22 35 1 +22 36 1 +22 37 1 +22 38 1 +22 39 1 +22 40 1 +22 41 1 +22 42 1 +22 43 1 +22 44 1 +22 45 1 +22 46 1 +22 47 1 +22 48 1 +22 49 1 +22 50 1 +22 51 1 +22 52 1 +22 53 1 +22 54 1 +22 55 1 +22 56 1 +22 57 1 +22 58 1 +22 59 1 +22 60 1 +22 61 1 +22 62 1 +22 63 1 +22 64 1 +22 65 1 +22 66 1 +22 67 1 +22 68 1 +22 69 1 +22 70 1 +22 71 1 +22 72 1 +22 73 1 +22 74 1 +22 75 1 +22 76 1 +22 77 1 +22 78 1 +22 79 1 +22 80 1 +22 81 1 +22 82 1 +22 83 1 +22 84 1 +22 85 1 +22 86 1 +22 87 1 +22 88 1 +22 89 1 +22 90 1 +22 91 1 +22 92 1 +22 93 1 +22 94 1 +22 95 1 +22 96 1 +22 97 1 +22 98 1 +22 99 1 +22 100 1 +23 0 1 +23 1 1 +23 2 1 +23 3 1 +23 4 1 +23 5 1 +23 6 1 +23 7 1 +23 8 1 +23 9 1 +23 10 1 +23 11 1 +23 12 1 +23 13 1 +23 14 1 +23 15 1 +23 16 1 +23 17 1 +23 18 1 +23 19 1 +23 20 1 +23 21 1 +23 22 1 +23 23 1 +23 24 1 +23 25 1 +23 26 1 +23 27 1 +23 28 1 +23 29 1 +23 30 1 +23 31 1 +23 32 1 +23 33 1 +23 34 1 +23 35 1 +23 36 1 +23 37 1 +23 38 1 +23 39 1 +23 40 1 +23 41 1 +23 42 1 +23 43 1 +23 44 1 +23 45 1 +23 46 1 +23 47 1 +23 48 1 +23 49 1 +23 50 1 +23 51 1 +23 52 1 +23 53 1 +23 54 1 +23 55 1 +23 56 1 +23 57 1 +23 58 1 +23 59 1 +23 60 1 +23 61 1 +23 62 1 +23 63 1 +23 64 1 +23 65 1 +23 66 1 +23 67 1 +23 68 1 +23 69 1 +23 70 1 +23 71 1 +23 72 1 +23 73 1 +23 74 1 +23 75 1 +23 76 1 +23 77 1 +23 78 1 +23 79 1 +23 80 1 +23 81 1 +23 82 1 +23 83 1 +23 84 1 +23 85 1 +23 86 1 +23 87 1 +23 88 1 +23 89 1 +23 90 1 +23 91 1 +23 92 1 +23 93 1 +23 94 1 +23 95 1 +23 96 1 +23 97 1 +23 98 1 +23 99 1 +23 100 1 +24 0 1 +24 1 1 +24 2 1 +24 3 1 +24 4 1 +24 5 1 +24 6 1 +24 7 1 +24 8 1 +24 9 1 +24 10 1 +24 11 1 +24 12 1 +24 13 1 +24 14 1 +24 15 1 +24 16 1 +24 17 1 +24 18 1 +24 19 1 +24 20 1 +24 21 1 +24 22 1 +24 23 1 +24 24 1 +24 25 1 +24 26 1 +24 27 1 +24 28 1 +24 29 1 +24 30 1 +24 31 1 +24 32 1 +24 33 1 +24 34 1 +24 35 1 +24 36 1 +24 37 1 +24 38 1 +24 39 1 +24 40 1 +24 41 1 +24 42 1 +24 43 1 +24 44 1 +24 45 1 +24 46 1 +24 47 1 +24 48 1 +24 49 1 +24 50 1 +24 51 1 +24 52 1 +24 53 1 +24 54 1 +24 55 1 +24 56 1 +24 57 1 +24 58 1 +24 59 1 +24 60 1 +24 61 1 +24 62 1 +24 63 1 +24 64 1 +24 65 1 +24 66 1 +24 67 1 +24 68 1 +24 69 1 +24 70 1 +24 71 1 +24 72 1 +24 73 1 +24 74 1 +24 75 1 +24 76 1 +24 77 1 +24 78 1 +24 79 1 +24 80 1 +24 81 1 +24 82 1 +24 83 1 +24 84 1 +24 85 1 +24 86 1 +24 87 1 +24 88 1 +24 89 1 +24 90 1 +24 91 1 +24 92 1 +24 93 1 +24 94 1 +24 95 1 +24 96 1 +24 97 1 +24 98 1 +24 99 1 +24 100 1 +25 0 1 +25 1 1 +25 2 1 +25 3 1 +25 4 1 +25 5 1 +25 6 1 +25 7 1 +25 8 1 +25 9 1 +25 10 1 +25 11 1 +25 12 1 +25 13 1 +25 14 1 +25 15 1 +25 16 1 +25 17 1 +25 18 1 +25 19 1 +25 20 1 +25 21 1 +25 22 1 +25 23 1 +25 24 1 +25 25 1 +25 26 1 +25 27 1 +25 28 1 +25 29 1 +25 30 1 +25 31 1 +25 32 1 +25 33 1 +25 34 1 +25 35 1 +25 36 1 +25 37 1 +25 38 1 +25 39 1 +25 40 1 +25 41 1 +25 42 1 +25 43 1 +25 44 1 +25 45 1 +25 46 1 +25 47 1 +25 48 1 +25 49 1 +25 50 1 +25 51 1 +25 52 1 +25 53 1 +25 54 1 +25 55 1 +25 56 1 +25 57 1 +25 58 1 +25 59 1 +25 60 1 +25 61 1 +25 62 1 +25 63 1 +25 64 1 +25 65 1 +25 66 1 +25 67 1 +25 68 1 +25 69 1 +25 70 1 +25 71 1 +25 72 1 +25 73 1 +25 74 1 +25 75 1 +25 76 1 +25 77 1 +25 78 1 +25 79 1 +25 80 1 +25 81 1 +25 82 1 +25 83 1 +25 84 1 +25 85 1 +25 86 1 +25 87 1 +25 88 1 +25 89 1 +25 90 1 +25 91 1 +25 92 1 +25 93 1 +25 94 1 +25 95 1 +25 96 1 +25 97 1 +25 98 1 +25 99 1 +25 100 1 +26 0 1 +26 1 1 +26 2 1 +26 3 1 +26 4 1 +26 5 1 +26 6 1 +26 7 1 +26 8 1 +26 9 1 +26 10 1 +26 11 1 +26 12 1 +26 13 1 +26 14 1 +26 15 1 +26 16 1 +26 17 1 +26 18 1 +26 19 1 +26 20 1 +26 21 1 +26 22 1 +26 23 1 +26 24 1 +26 25 1 +26 26 1 +26 27 1 +26 28 1 +26 29 1 +26 30 1 +26 31 1 +26 32 1 +26 33 1 +26 34 1 +26 35 1 +26 36 1 +26 37 1 +26 38 1 +26 39 1 +26 40 1 +26 41 1 +26 42 1 +26 43 1 +26 44 1 +26 45 1 +26 46 1 +26 47 1 +26 48 1 +26 49 1 +26 50 1 +26 51 1 +26 52 1 +26 53 1 +26 54 1 +26 55 1 +26 56 1 +26 57 1 +26 58 1 +26 59 1 +26 60 1 +26 61 1 +26 62 1 +26 63 1 +26 64 1 +26 65 1 +26 66 1 +26 67 1 +26 68 1 +26 69 1 +26 70 1 +26 71 1 +26 72 1 +26 73 1 +26 74 1 +26 75 1 +26 76 1 +26 77 1 +26 78 1 +26 79 1 +26 80 1 +26 81 1 +26 82 1 +26 83 1 +26 84 1 +26 85 1 +26 86 1 +26 87 1 +26 88 1 +26 89 1 +26 90 1 +26 91 1 +26 92 1 +26 93 1 +26 94 1 +26 95 1 +26 96 1 +26 97 1 +26 98 1 +26 99 1 +26 100 1 +27 0 1 +27 1 1 +27 2 1 +27 3 1 +27 4 1 +27 5 1 +27 6 1 +27 7 1 +27 8 1 +27 9 1 +27 10 1 +27 11 1 +27 12 1 +27 13 1 +27 14 1 +27 15 1 +27 16 1 +27 17 1 +27 18 1 +27 19 1 +27 20 1 +27 21 1 +27 22 1 +27 23 1 +27 24 1 +27 25 1 +27 26 1 +27 27 1 +27 28 1 +27 29 1 +27 30 1 +27 31 1 +27 32 1 +27 33 1 +27 34 1 +27 35 1 +27 36 1 +27 37 1 +27 38 1 +27 39 1 +27 40 1 +27 41 1 +27 42 1 +27 43 1 +27 44 1 +27 45 1 +27 46 1 +27 47 1 +27 48 1 +27 49 1 +27 50 1 +27 51 1 +27 52 1 +27 53 1 +27 54 1 +27 55 1 +27 56 1 +27 57 1 +27 58 1 +27 59 1 +27 60 1 +27 61 1 +27 62 1 +27 63 1 +27 64 1 +27 65 1 +27 66 1 +27 67 1 +27 68 1 +27 69 1 +27 70 1 +27 71 1 +27 72 1 +27 73 1 +27 74 1 +27 75 1 +27 76 1 +27 77 1 +27 78 1 +27 79 1 +27 80 1 +27 81 1 +27 82 1 +27 83 1 +27 84 1 +27 85 1 +27 86 1 +27 87 1 +27 88 1 +27 89 1 +27 90 1 +27 91 1 +27 92 1 +27 93 1 +27 94 1 +27 95 1 +27 96 1 +27 97 1 +27 98 1 +27 99 1 +27 100 1 +28 0 1 +28 1 1 +28 2 1 +28 3 1 +28 4 1 +28 5 1 +28 6 1 +28 7 1 +28 8 1 +28 9 1 +28 10 1 +28 11 1 +28 12 1 +28 13 1 +28 14 1 +28 15 1 +28 16 1 +28 17 1 +28 18 1 +28 19 1 +28 20 1 +28 21 1 +28 22 1 +28 23 1 +28 24 1 +28 25 1 +28 26 1 +28 27 1 +28 28 1 +28 29 1 +28 30 1 +28 31 1 +28 32 1 +28 33 1 +28 34 1 +28 35 1 +28 36 1 +28 37 1 +28 38 1 +28 39 1 +28 40 1 +28 41 1 +28 42 1 +28 43 1 +28 44 1 +28 45 1 +28 46 1 +28 47 1 +28 48 1 +28 49 1 +28 50 1 +28 51 1 +28 52 1 +28 53 1 +28 54 1 +28 55 1 +28 56 1 +28 57 1 +28 58 1 +28 59 1 +28 60 1 +28 61 1 +28 62 1 +28 63 1 +28 64 1 +28 65 1 +28 66 1 +28 67 1 +28 68 1 +28 69 1 +28 70 1 +28 71 1 +28 72 1 +28 73 1 +28 74 1 +28 75 1 +28 76 1 +28 77 1 +28 78 1 +28 79 1 +28 80 1 +28 81 1 +28 82 1 +28 83 1 +28 84 1 +28 85 1 +28 86 1 +28 87 1 +28 88 1 +28 89 1 +28 90 1 +28 91 1 +28 92 1 +28 93 1 +28 94 1 +28 95 1 +28 96 1 +28 97 1 +28 98 1 +28 99 1 +28 100 1 +29 0 1 +29 1 1 +29 2 1 +29 3 1 +29 4 1 +29 5 1 +29 6 1 +29 7 1 +29 8 1 +29 9 1 +29 10 1 +29 11 1 +29 12 1 +29 13 1 +29 14 1 +29 15 1 +29 16 1 +29 17 1 +29 18 1 +29 19 1 +29 20 1 +29 21 1 +29 22 1 +29 23 1 +29 24 1 +29 25 1 +29 26 1 +29 27 1 +29 28 1 +29 29 1 +29 30 1 +29 31 1 +29 32 1 +29 33 1 +29 34 1 +29 35 1 +29 36 1 +29 37 1 +29 38 1 +29 39 1 +29 40 1 +29 41 1 +29 42 1 +29 43 1 +29 44 1 +29 45 1 +29 46 1 +29 47 1 +29 48 1 +29 49 1 +29 50 1 +29 51 1 +29 52 1 +29 53 1 +29 54 1 +29 55 1 +29 56 1 +29 57 1 +29 58 1 +29 59 1 +29 60 1 +29 61 1 +29 62 1 +29 63 1 +29 64 1 +29 65 1 +29 66 1 +29 67 1 +29 68 1 +29 69 1 +29 70 1 +29 71 1 +29 72 1 +29 73 1 +29 74 1 +29 75 1 +29 76 1 +29 77 1 +29 78 1 +29 79 1 +29 80 1 +29 81 1 +29 82 1 +29 83 1 +29 84 1 +29 85 1 +29 86 1 +29 87 1 +29 88 1 +29 89 1 +29 90 1 +29 91 1 +29 92 1 +29 93 1 +29 94 1 +29 95 1 +29 96 1 +29 97 1 +29 98 1 +29 99 1 +29 100 1 +30 0 1 +30 1 1 +30 2 1 +30 3 1 +30 4 1 +30 5 1 +30 6 1 +30 7 1 +30 8 1 +30 9 1 +30 10 1 +30 11 1 +30 12 1 +30 13 1 +30 14 1 +30 15 1 +30 16 1 +30 17 1 +30 18 1 +30 19 1 +30 20 1 +30 21 1 +30 22 1 +30 23 1 +30 24 1 +30 25 1 +30 26 1 +30 27 1 +30 28 1 +30 29 1 +30 30 1 +30 31 1 +30 32 1 +30 33 1 +30 34 1 +30 35 1 +30 36 1 +30 37 1 +30 38 1 +30 39 1 +30 40 1 +30 41 1 +30 42 1 +30 43 1 +30 44 1 +30 45 1 +30 46 1 +30 47 1 +30 48 1 +30 49 1 +30 50 1 +30 51 1 +30 52 1 +30 53 1 +30 54 1 +30 55 1 +30 56 1 +30 57 1 +30 58 1 +30 59 1 +30 60 1 +30 61 1 +30 62 1 +30 63 1 +30 64 1 +30 65 1 +30 66 1 +30 67 1 +30 68 1 +30 69 1 +30 70 1 +30 71 1 +30 72 1 +30 73 1 +30 74 1 +30 75 1 +30 76 1 +30 77 1 +30 78 1 +30 79 1 +30 80 1 +30 81 1 +30 82 1 +30 83 1 +30 84 1 +30 85 1 +30 86 1 +30 87 1 +30 88 1 +30 89 1 +30 90 1 +30 91 1 +30 92 1 +30 93 1 +30 94 1 +30 95 1 +30 96 1 +30 97 1 +30 98 1 +30 99 1 +30 100 1 +31 0 1 +31 1 1 +31 2 1 +31 3 1 +31 4 1 +31 5 1 +31 6 1 +31 7 1 +31 8 1 +31 9 1 +31 10 1 +31 11 1 +31 12 1 +31 13 1 +31 14 1 +31 15 1 +31 16 1 +31 17 1 +31 18 1 +31 19 1 +31 20 1 +31 21 1 +31 22 1 +31 23 1 +31 24 1 +31 25 1 +31 26 1 +31 27 1 +31 28 1 +31 29 1 +31 30 1 +31 31 1 +31 32 1 +31 33 1 +31 34 1 +31 35 1 +31 36 1 +31 37 1 +31 38 1 +31 39 1 +31 40 1 +31 41 1 +31 42 1 +31 43 1 +31 44 1 +31 45 1 +31 46 1 +31 47 1 +31 48 1 +31 49 1 +31 50 1 +31 51 1 +31 52 1 +31 53 1 +31 54 1 +31 55 1 +31 56 1 +31 57 1 +31 58 1 +31 59 1 +31 60 1 +31 61 1 +31 62 1 +31 63 1 +31 64 1 +31 65 1 +31 66 1 +31 67 1 +31 68 1 +31 69 1 +31 70 1 +31 71 1 +31 72 1 +31 73 1 +31 74 1 +31 75 1 +31 76 1 +31 77 1 +31 78 1 +31 79 1 +31 80 1 +31 81 1 +31 82 1 +31 83 1 +31 84 1 +31 85 1 +31 86 1 +31 87 1 +31 88 1 +31 89 1 +31 90 1 +31 91 1 +31 92 1 +31 93 1 +31 94 1 +31 95 1 +31 96 1 +31 97 1 +31 98 1 +31 99 1 +31 100 1 +32 0 1 +32 1 1 +32 2 1 +32 3 1 +32 4 1 +32 5 1 +32 6 1 +32 7 1 +32 8 1 +32 9 1 +32 10 1 +32 11 1 +32 12 1 +32 13 1 +32 14 1 +32 15 1 +32 16 1 +32 17 1 +32 18 1 +32 19 1 +32 20 1 +32 21 1 +32 22 1 +32 23 1 +32 24 1 +32 25 1 +32 26 1 +32 27 1 +32 28 1 +32 29 1 +32 30 1 +32 31 1 +32 32 1 +32 33 1 +32 34 1 +32 35 1 +32 36 1 +32 37 1 +32 38 1 +32 39 1 +32 40 1 +32 41 1 +32 42 1 +32 43 1 +32 44 1 +32 45 1 +32 46 1 +32 47 1 +32 48 1 +32 49 1 +32 50 1 +32 51 1 +32 52 1 +32 53 1 +32 54 1 +32 55 1 +32 56 1 +32 57 1 +32 58 1 +32 59 1 +32 60 1 +32 61 1 +32 62 1 +32 63 1 +32 64 1 +32 65 1 +32 66 1 +32 67 1 +32 68 1 +32 69 1 +32 70 1 +32 71 1 +32 72 1 +32 73 1 +32 74 1 +32 75 1 +32 76 1 +32 77 1 +32 78 1 +32 79 1 +32 80 1 +32 81 1 +32 82 1 +32 83 1 +32 84 1 +32 85 1 +32 86 1 +32 87 1 +32 88 1 +32 89 1 +32 90 1 +32 91 1 +32 92 1 +32 93 1 +32 94 1 +32 95 1 +32 96 1 +32 97 1 +32 98 1 +32 99 1 +32 100 1 +33 0 1 +33 1 1 +33 2 1 +33 3 1 +33 4 1 +33 5 1 +33 6 1 +33 7 1 +33 8 1 +33 9 1 +33 10 1 +33 11 1 +33 12 1 +33 13 1 +33 14 1 +33 15 1 +33 16 1 +33 17 1 +33 18 1 +33 19 1 +33 20 1 +33 21 1 +33 22 1 +33 23 1 +33 24 1 +33 25 1 +33 26 1 +33 27 1 +33 28 1 +33 29 1 +33 30 1 +33 31 1 +33 32 1 +33 33 1 +33 34 1 +33 35 1 +33 36 1 +33 37 1 +33 38 1 +33 39 1 +33 40 1 +33 41 1 +33 42 1 +33 43 1 +33 44 1 +33 45 1 +33 46 1 +33 47 1 +33 48 1 +33 49 1 +33 50 1 +33 51 1 +33 52 1 +33 53 1 +33 54 1 +33 55 1 +33 56 1 +33 57 1 +33 58 1 +33 59 1 +33 60 1 +33 61 1 +33 62 1 +33 63 1 +33 64 1 +33 65 1 +33 66 1 +33 67 1 +33 68 1 +33 69 1 +33 70 1 +33 71 1 +33 72 1 +33 73 1 +33 74 1 +33 75 1 +33 76 1 +33 77 1 +33 78 1 +33 79 1 +33 80 1 +33 81 1 +33 82 1 +33 83 1 +33 84 1 +33 85 1 +33 86 1 +33 87 1 +33 88 1 +33 89 1 +33 90 1 +33 91 1 +33 92 1 +33 93 1 +33 94 1 +33 95 1 +33 96 1 +33 97 1 +33 98 1 +33 99 1 +33 100 1 +34 0 1 +34 1 1 +34 2 1 +34 3 1 +34 4 1 +34 5 1 +34 6 1 +34 7 1 +34 8 1 +34 9 1 +34 10 1 +34 11 1 +34 12 1 +34 13 1 +34 14 1 +34 15 1 +34 16 1 +34 17 1 +34 18 1 +34 19 1 +34 20 1 +34 21 1 +34 22 1 +34 23 1 +34 24 1 +34 25 1 +34 26 1 +34 27 1 +34 28 1 +34 29 1 +34 30 1 +34 31 1 +34 32 1 +34 33 1 +34 34 1 +34 35 1 +34 36 1 +34 37 1 +34 38 1 +34 39 1 +34 40 1 +34 41 1 +34 42 1 +34 43 1 +34 44 1 +34 45 1 +34 46 1 +34 47 1 +34 48 1 +34 49 1 +34 50 1 +34 51 1 +34 52 1 +34 53 1 +34 54 1 +34 55 1 +34 56 1 +34 57 1 +34 58 1 +34 59 1 +34 60 1 +34 61 1 +34 62 1 +34 63 1 +34 64 1 +34 65 1 +34 66 1 +34 67 1 +34 68 1 +34 69 1 +34 70 1 +34 71 1 +34 72 1 +34 73 1 +34 74 1 +34 75 1 +34 76 1 +34 77 1 +34 78 1 +34 79 1 +34 80 1 +34 81 1 +34 82 1 +34 83 1 +34 84 1 +34 85 1 +34 86 1 +34 87 1 +34 88 1 +34 89 1 +34 90 1 +34 91 1 +34 92 1 +34 93 1 +34 94 1 +34 95 1 +34 96 1 +34 97 1 +34 98 1 +34 99 1 +34 100 1 +35 0 1 +35 1 1 +35 2 1 +35 3 1 +35 4 1 +35 5 1 +35 6 1 +35 7 1 +35 8 1 +35 9 1 +35 10 1 +35 11 1 +35 12 1 +35 13 1 +35 14 1 +35 15 1 +35 16 1 +35 17 1 +35 18 1 +35 19 1 +35 20 1 +35 21 1 +35 22 1 +35 23 1 +35 24 1 +35 25 1 +35 26 1 +35 27 1 +35 28 1 +35 29 1 +35 30 1 +35 31 1 +35 32 1 +35 33 1 +35 34 1 +35 35 1 +35 36 1 +35 37 1 +35 38 1 +35 39 1 +35 40 1 +35 41 1 +35 42 1 +35 43 1 +35 44 1 +35 45 1 +35 46 1 +35 47 1 +35 48 1 +35 49 1 +35 50 1 +35 51 1 +35 52 1 +35 53 1 +35 54 1 +35 55 1 +35 56 1 +35 57 1 +35 58 1 +35 59 1 +35 60 1 +35 61 1 +35 62 1 +35 63 1 +35 64 1 +35 65 1 +35 66 1 +35 67 1 +35 68 1 +35 69 1 +35 70 1 +35 71 1 +35 72 1 +35 73 1 +35 74 1 +35 75 1 +35 76 1 +35 77 1 +35 78 1 +35 79 1 +35 80 1 +35 81 1 +35 82 1 +35 83 1 +35 84 1 +35 85 1 +35 86 1 +35 87 1 +35 88 1 +35 89 1 +35 90 1 +35 91 1 +35 92 1 +35 93 1 +35 94 1 +35 95 1 +35 96 1 +35 97 1 +35 98 1 +35 99 1 +35 100 1 +36 0 1 +36 1 1 +36 2 1 +36 3 1 +36 4 1 +36 5 1 +36 6 1 +36 7 1 +36 8 1 +36 9 1 +36 10 1 +36 11 1 +36 12 1 +36 13 1 +36 14 1 +36 15 1 +36 16 1 +36 17 1 +36 18 1 +36 19 1 +36 20 1 +36 21 1 +36 22 1 +36 23 1 +36 24 1 +36 25 1 +36 26 1 +36 27 1 +36 28 1 +36 29 1 +36 30 1 +36 31 1 +36 32 1 +36 33 1 +36 34 1 +36 35 1 +36 36 1 +36 37 1 +36 38 1 +36 39 1 +36 40 1 +36 41 1 +36 42 1 +36 43 1 +36 44 1 +36 45 1 +36 46 1 +36 47 1 +36 48 1 +36 49 1 +36 50 1 +36 51 1 +36 52 1 +36 53 1 +36 54 1 +36 55 1 +36 56 1 +36 57 1 +36 58 1 +36 59 1 +36 60 1 +36 61 1 +36 62 1 +36 63 1 +36 64 1 +36 65 1 +36 66 1 +36 67 1 +36 68 1 +36 69 1 +36 70 1 +36 71 1 +36 72 1 +36 73 1 +36 74 1 +36 75 1 +36 76 1 +36 77 1 +36 78 1 +36 79 1 +36 80 1 +36 81 1 +36 82 1 +36 83 1 +36 84 1 +36 85 1 +36 86 1 +36 87 1 +36 88 1 +36 89 1 +36 90 1 +36 91 1 +36 92 1 +36 93 1 +36 94 1 +36 95 1 +36 96 1 +36 97 1 +36 98 1 +36 99 1 +36 100 1 +37 0 1 +37 1 1 +37 2 1 +37 3 1 +37 4 1 +37 5 1 +37 6 1 +37 7 1 +37 8 1 +37 9 1 +37 10 1 +37 11 1 +37 12 1 +37 13 1 +37 14 1 +37 15 1 +37 16 1 +37 17 1 +37 18 1 +37 19 1 +37 20 1 +37 21 1 +37 22 1 +37 23 1 +37 24 1 +37 25 1 +37 26 1 +37 27 1 +37 28 1 +37 29 1 +37 30 1 +37 31 1 +37 32 1 +37 33 1 +37 34 1 +37 35 1 +37 36 1 +37 37 1 +37 38 1 +37 39 1 +37 40 1 +37 41 1 +37 42 1 +37 43 1 +37 44 1 +37 45 1 +37 46 1 +37 47 1 +37 48 1 +37 49 1 +37 50 1 +37 51 1 +37 52 1 +37 53 1 +37 54 1 +37 55 1 +37 56 1 +37 57 1 +37 58 1 +37 59 1 +37 60 1 +37 61 1 +37 62 1 +37 63 1 +37 64 1 +37 65 1 +37 66 1 +37 67 1 +37 68 1 +37 69 1 +37 70 1 +37 71 1 +37 72 1 +37 73 1 +37 74 1 +37 75 1 +37 76 1 +37 77 1 +37 78 1 +37 79 1 +37 80 1 +37 81 1 +37 82 1 +37 83 1 +37 84 1 +37 85 1 +37 86 1 +37 87 1 +37 88 1 +37 89 1 +37 90 1 +37 91 1 +37 92 1 +37 93 1 +37 94 1 +37 95 1 +37 96 1 +37 97 1 +37 98 1 +37 99 1 +37 100 1 +38 0 1 +38 1 1 +38 2 1 +38 3 1 +38 4 1 +38 5 1 +38 6 1 +38 7 1 +38 8 1 +38 9 1 +38 10 1 +38 11 1 +38 12 1 +38 13 1 +38 14 1 +38 15 1 +38 16 1 +38 17 1 +38 18 1 +38 19 1 +38 20 1 +38 21 1 +38 22 1 +38 23 1 +38 24 1 +38 25 1 +38 26 1 +38 27 1 +38 28 1 +38 29 1 +38 30 1 +38 31 1 +38 32 1 +38 33 1 +38 34 1 +38 35 1 +38 36 1 +38 37 1 +38 38 1 +38 39 1 +38 40 1 +38 41 1 +38 42 1 +38 43 1 +38 44 1 +38 45 1 +38 46 1 +38 47 1 +38 48 1 +38 49 1 +38 50 1 +38 51 1 +38 52 1 +38 53 1 +38 54 1 +38 55 1 +38 56 1 +38 57 1 +38 58 1 +38 59 1 +38 60 1 +38 61 1 +38 62 1 +38 63 1 +38 64 1 +38 65 1 +38 66 1 +38 67 1 +38 68 1 +38 69 1 +38 70 1 +38 71 1 +38 72 1 +38 73 1 +38 74 1 +38 75 1 +38 76 1 +38 77 1 +38 78 1 +38 79 1 +38 80 1 +38 81 1 +38 82 1 +38 83 1 +38 84 1 +38 85 1 +38 86 1 +38 87 1 +38 88 1 +38 89 1 +38 90 1 +38 91 1 +38 92 1 +38 93 1 +38 94 1 +38 95 1 +38 96 1 +38 97 1 +38 98 1 +38 99 1 +38 100 1 +39 0 1 +39 1 1 +39 2 1 +39 3 1 +39 4 1 +39 5 1 +39 6 1 +39 7 1 +39 8 1 +39 9 1 +39 10 1 +39 11 1 +39 12 1 +39 13 1 +39 14 1 +39 15 1 +39 16 1 +39 17 1 +39 18 1 +39 19 1 +39 20 1 +39 21 1 +39 22 1 +39 23 1 +39 24 1 +39 25 1 +39 26 1 +39 27 1 +39 28 1 +39 29 1 +39 30 1 +39 31 1 +39 32 1 +39 33 1 +39 34 1 +39 35 1 +39 36 1 +39 37 1 +39 38 1 +39 39 1 +39 40 1 +39 41 1 +39 42 1 +39 43 1 +39 44 1 +39 45 1 +39 46 1 +39 47 1 +39 48 1 +39 49 1 +39 50 1 +39 51 1 +39 52 1 +39 53 1 +39 54 1 +39 55 1 +39 56 1 +39 57 1 +39 58 1 +39 59 1 +39 60 1 +39 61 1 +39 62 1 +39 63 1 +39 64 1 +39 65 1 +39 66 1 +39 67 1 +39 68 1 +39 69 1 +39 70 1 +39 71 1 +39 72 1 +39 73 1 +39 74 1 +39 75 1 +39 76 1 +39 77 1 +39 78 1 +39 79 1 +39 80 1 +39 81 1 +39 82 1 +39 83 1 +39 84 1 +39 85 1 +39 86 1 +39 87 1 +39 88 1 +39 89 1 +39 90 1 +39 91 1 +39 92 1 +39 93 1 +39 94 1 +39 95 1 +39 96 1 +39 97 1 +39 98 1 +39 99 1 +39 100 1 +40 0 1 +40 1 1 +40 2 1 +40 3 1 +40 4 1 +40 5 1 +40 6 1 +40 7 1 +40 8 1 +40 9 1 +40 10 1 +40 11 1 +40 12 1 +40 13 1 +40 14 1 +40 15 1 +40 16 1 +40 17 1 +40 18 1 +40 19 1 +40 20 1 +40 21 1 +40 22 1 +40 23 1 +40 24 1 +40 25 1 +40 26 1 +40 27 1 +40 28 1 +40 29 1 +40 30 1 +40 31 1 +40 32 1 +40 33 1 +40 34 1 +40 35 1 +40 36 1 +40 37 1 +40 38 1 +40 39 1 +40 40 1 +40 41 1 +40 42 1 +40 43 1 +40 44 1 +40 45 1 +40 46 1 +40 47 1 +40 48 1 +40 49 1 +40 50 1 +40 51 1 +40 52 1 +40 53 1 +40 54 1 +40 55 1 +40 56 1 +40 57 1 +40 58 1 +40 59 1 +40 60 1 +40 61 1 +40 62 1 +40 63 1 +40 64 1 +40 65 1 +40 66 1 +40 67 1 +40 68 1 +40 69 1 +40 70 1 +40 71 1 +40 72 1 +40 73 1 +40 74 1 +40 75 1 +40 76 1 +40 77 1 +40 78 1 +40 79 1 +40 80 1 +40 81 1 +40 82 1 +40 83 1 +40 84 1 +40 85 1 +40 86 1 +40 87 1 +40 88 1 +40 89 1 +40 90 1 +40 91 1 +40 92 1 +40 93 1 +40 94 1 +40 95 1 +40 96 1 +40 97 1 +40 98 1 +40 99 1 +40 100 1 +41 0 1 +41 1 1 +41 2 1 +41 3 1 +41 4 1 +41 5 1 +41 6 1 +41 7 1 +41 8 1 +41 9 1 +41 10 1 +41 11 1 +41 12 1 +41 13 1 +41 14 1 +41 15 1 +41 16 1 +41 17 1 +41 18 1 +41 19 1 +41 20 1 +41 21 1 +41 22 1 +41 23 1 +41 24 1 +41 25 1 +41 26 1 +41 27 1 +41 28 1 +41 29 1 +41 30 1 +41 31 1 +41 32 1 +41 33 1 +41 34 1 +41 35 1 +41 36 1 +41 37 1 +41 38 1 +41 39 1 +41 40 1 +41 41 1 +41 42 1 +41 43 1 +41 44 1 +41 45 1 +41 46 1 +41 47 1 +41 48 1 +41 49 1 +41 50 1 +41 51 1 +41 52 1 +41 53 1 +41 54 1 +41 55 1 +41 56 1 +41 57 1 +41 58 1 +41 59 1 +41 60 1 +41 61 1 +41 62 1 +41 63 1 +41 64 1 +41 65 1 +41 66 1 +41 67 1 +41 68 1 +41 69 1 +41 70 1 +41 71 1 +41 72 1 +41 73 1 +41 74 1 +41 75 1 +41 76 1 +41 77 1 +41 78 1 +41 79 1 +41 80 1 +41 81 1 +41 82 1 +41 83 1 +41 84 1 +41 85 1 +41 86 1 +41 87 1 +41 88 1 +41 89 1 +41 90 1 +41 91 1 +41 92 1 +41 93 1 +41 94 1 +41 95 1 +41 96 1 +41 97 1 +41 98 1 +41 99 1 +41 100 1 +42 0 1 +42 1 1 +42 2 1 +42 3 1 +42 4 1 +42 5 1 +42 6 1 +42 7 1 +42 8 1 +42 9 1 +42 10 1 +42 11 1 +42 12 1 +42 13 1 +42 14 1 +42 15 1 +42 16 1 +42 17 1 +42 18 1 +42 19 1 +42 20 1 +42 21 1 +42 22 1 +42 23 1 +42 24 1 +42 25 1 +42 26 1 +42 27 1 +42 28 1 +42 29 1 +42 30 1 +42 31 1 +42 32 1 +42 33 1 +42 34 1 +42 35 1 +42 36 1 +42 37 1 +42 38 1 +42 39 1 +42 40 1 +42 41 1 +42 42 1 +42 43 1 +42 44 1 +42 45 1 +42 46 1 +42 47 1 +42 48 1 +42 49 1 +42 50 1 +42 51 1 +42 52 1 +42 53 1 +42 54 1 +42 55 1 +42 56 1 +42 57 1 +42 58 1 +42 59 1 +42 60 1 +42 61 1 +42 62 1 +42 63 1 +42 64 1 +42 65 1 +42 66 1 +42 67 1 +42 68 1 +42 69 1 +42 70 1 +42 71 1 +42 72 1 +42 73 1 +42 74 1 +42 75 1 +42 76 1 +42 77 1 +42 78 1 +42 79 1 +42 80 1 +42 81 1 +42 82 1 +42 83 1 +42 84 1 +42 85 1 +42 86 1 +42 87 1 +42 88 1 +42 89 1 +42 90 1 +42 91 1 +42 92 1 +42 93 1 +42 94 1 +42 95 1 +42 96 1 +42 97 1 +42 98 1 +42 99 1 +42 100 1 +43 0 1 +43 1 1 +43 2 1 +43 3 1 +43 4 1 +43 5 1 +43 6 1 +43 7 1 +43 8 1 +43 9 1 +43 10 1 +43 11 1 +43 12 1 +43 13 1 +43 14 1 +43 15 1 +43 16 1 +43 17 1 +43 18 1 +43 19 1 +43 20 1 +43 21 1 +43 22 1 +43 23 1 +43 24 1 +43 25 1 +43 26 1 +43 27 1 +43 28 1 +43 29 1 +43 30 1 +43 31 1 +43 32 1 +43 33 1 +43 34 1 +43 35 1 +43 36 1 +43 37 1 +43 38 1 +43 39 1 +43 40 1 +43 41 1 +43 42 1 +43 43 1 +43 44 1 +43 45 1 +43 46 1 +43 47 1 +43 48 1 +43 49 1 +43 50 1 +43 51 1 +43 52 1 +43 53 1 +43 54 1 +43 55 1 +43 56 1 +43 57 1 +43 58 1 +43 59 1 +43 60 1 +43 61 1 +43 62 1 +43 63 1 +43 64 1 +43 65 1 +43 66 1 +43 67 1 +43 68 1 +43 69 1 +43 70 1 +43 71 1 +43 72 1 +43 73 1 +43 74 1 +43 75 1 +43 76 1 +43 77 1 +43 78 1 +43 79 1 +43 80 1 +43 81 1 +43 82 1 +43 83 1 +43 84 1 +43 85 1 +43 86 1 +43 87 1 +43 88 1 +43 89 1 +43 90 1 +43 91 1 +43 92 1 +43 93 1 +43 94 1 +43 95 1 +43 96 1 +43 97 1 +43 98 1 +43 99 1 +43 100 1 +44 0 1 +44 1 1 +44 2 1 +44 3 1 +44 4 1 +44 5 1 +44 6 1 +44 7 1 +44 8 1 +44 9 1 +44 10 1 +44 11 1 +44 12 1 +44 13 1 +44 14 1 +44 15 1 +44 16 1 +44 17 1 +44 18 1 +44 19 1 +44 20 1 +44 21 1 +44 22 1 +44 23 1 +44 24 1 +44 25 1 +44 26 1 +44 27 1 +44 28 1 +44 29 1 +44 30 1 +44 31 1 +44 32 1 +44 33 1 +44 34 1 +44 35 1 +44 36 1 +44 37 1 +44 38 1 +44 39 1 +44 40 1 +44 41 1 +44 42 1 +44 43 1 +44 44 1 +44 45 1 +44 46 1 +44 47 1 +44 48 1 +44 49 1 +44 50 1 +44 51 1 +44 52 1 +44 53 1 +44 54 1 +44 55 1 +44 56 1 +44 57 1 +44 58 1 +44 59 1 +44 60 1 +44 61 1 +44 62 1 +44 63 1 +44 64 1 +44 65 1 +44 66 1 +44 67 1 +44 68 1 +44 69 1 +44 70 1 +44 71 1 +44 72 1 +44 73 1 +44 74 1 +44 75 1 +44 76 1 +44 77 1 +44 78 1 +44 79 1 +44 80 1 +44 81 1 +44 82 1 +44 83 1 +44 84 1 +44 85 1 +44 86 1 +44 87 1 +44 88 1 +44 89 1 +44 90 1 +44 91 1 +44 92 1 +44 93 1 +44 94 1 +44 95 1 +44 96 1 +44 97 1 +44 98 1 +44 99 1 +44 100 1 +45 0 1 +45 1 1 +45 2 1 +45 3 1 +45 4 1 +45 5 1 +45 6 1 +45 7 1 +45 8 1 +45 9 1 +45 10 1 +45 11 1 +45 12 1 +45 13 1 +45 14 1 +45 15 1 +45 16 1 +45 17 1 +45 18 1 +45 19 1 +45 20 1 +45 21 1 +45 22 1 +45 23 1 +45 24 1 +45 25 1 +45 26 1 +45 27 1 +45 28 1 +45 29 1 +45 30 1 +45 31 1 +45 32 1 +45 33 1 +45 34 1 +45 35 1 +45 36 1 +45 37 1 +45 38 1 +45 39 1 +45 40 1 +45 41 1 +45 42 1 +45 43 1 +45 44 1 +45 45 1 +45 46 1 +45 47 1 +45 48 1 +45 49 1 +45 50 1 +45 51 1 +45 52 1 +45 53 1 +45 54 1 +45 55 1 +45 56 1 +45 57 1 +45 58 1 +45 59 1 +45 60 1 +45 61 1 +45 62 1 +45 63 1 +45 64 1 +45 65 1 +45 66 1 +45 67 1 +45 68 1 +45 69 1 +45 70 1 +45 71 1 +45 72 1 +45 73 1 +45 74 1 +45 75 1 +45 76 1 +45 77 1 +45 78 1 +45 79 1 +45 80 1 +45 81 1 +45 82 1 +45 83 1 +45 84 1 +45 85 1 +45 86 1 +45 87 1 +45 88 1 +45 89 1 +45 90 1 +45 91 1 +45 92 1 +45 93 1 +45 94 1 +45 95 1 +45 96 1 +45 97 1 +45 98 1 +45 99 1 +45 100 1 +46 0 1 +46 1 1 +46 2 1 +46 3 1 +46 4 1 +46 5 1 +46 6 1 +46 7 1 +46 8 1 +46 9 1 +46 10 1 +46 11 1 +46 12 1 +46 13 1 +46 14 1 +46 15 1 +46 16 1 +46 17 1 +46 18 1 +46 19 1 +46 20 1 +46 21 1 +46 22 1 +46 23 1 +46 24 1 +46 25 1 +46 26 1 +46 27 1 +46 28 1 +46 29 1 +46 30 1 +46 31 1 +46 32 1 +46 33 1 +46 34 1 +46 35 1 +46 36 1 +46 37 1 +46 38 1 +46 39 1 +46 40 1 +46 41 1 +46 42 1 +46 43 1 +46 44 1 +46 45 1 +46 46 1 +46 47 1 +46 48 1 +46 49 1 +46 50 1 +46 51 1 +46 52 1 +46 53 1 +46 54 1 +46 55 1 +46 56 1 +46 57 1 +46 58 1 +46 59 1 +46 60 1 +46 61 1 +46 62 1 +46 63 1 +46 64 1 +46 65 1 +46 66 1 +46 67 1 +46 68 1 +46 69 1 +46 70 1 +46 71 1 +46 72 1 +46 73 1 +46 74 1 +46 75 1 +46 76 1 +46 77 1 +46 78 1 +46 79 1 +46 80 1 +46 81 1 +46 82 1 +46 83 1 +46 84 1 +46 85 1 +46 86 1 +46 87 1 +46 88 1 +46 89 1 +46 90 1 +46 91 1 +46 92 1 +46 93 1 +46 94 1 +46 95 1 +46 96 1 +46 97 1 +46 98 1 +46 99 1 +46 100 1 +47 0 1 +47 1 1 +47 2 1 +47 3 1 +47 4 1 +47 5 1 +47 6 1 +47 7 1 +47 8 1 +47 9 1 +47 10 1 +47 11 1 +47 12 1 +47 13 1 +47 14 1 +47 15 1 +47 16 1 +47 17 1 +47 18 1 +47 19 1 +47 20 1 +47 21 1 +47 22 1 +47 23 1 +47 24 1 +47 25 1 +47 26 1 +47 27 1 +47 28 1 +47 29 1 +47 30 1 +47 31 1 +47 32 1 +47 33 1 +47 34 1 +47 35 1 +47 36 1 +47 37 1 +47 38 1 +47 39 1 +47 40 1 +47 41 1 +47 42 1 +47 43 1 +47 44 1 +47 45 1 +47 46 1 +47 47 1 +47 48 1 +47 49 1 +47 50 1 +47 51 1 +47 52 1 +47 53 1 +47 54 1 +47 55 1 +47 56 1 +47 57 1 +47 58 1 +47 59 1 +47 60 1 +47 61 1 +47 62 1 +47 63 1 +47 64 1 +47 65 1 +47 66 1 +47 67 1 +47 68 1 +47 69 1 +47 70 1 +47 71 1 +47 72 1 +47 73 1 +47 74 1 +47 75 1 +47 76 1 +47 77 1 +47 78 1 +47 79 1 +47 80 1 +47 81 1 +47 82 1 +47 83 1 +47 84 1 +47 85 1 +47 86 1 +47 87 1 +47 88 1 +47 89 1 +47 90 1 +47 91 1 +47 92 1 +47 93 1 +47 94 1 +47 95 1 +47 96 1 +47 97 1 +47 98 1 +47 99 1 +47 100 1 +48 0 1 +48 1 1 +48 2 1 +48 3 1 +48 4 1 +48 5 1 +48 6 1 +48 7 1 +48 8 1 +48 9 1 +48 10 1 +48 11 1 +48 12 1 +48 13 1 +48 14 1 +48 15 1 +48 16 1 +48 17 1 +48 18 1 +48 19 1 +48 20 1 +48 21 1 +48 22 1 +48 23 1 +48 24 1 +48 25 1 +48 26 1 +48 27 1 +48 28 1 +48 29 1 +48 30 1 +48 31 1 +48 32 1 +48 33 1 +48 34 1 +48 35 1 +48 36 1 +48 37 1 +48 38 1 +48 39 1 +48 40 1 +48 41 1 +48 42 1 +48 43 1 +48 44 1 +48 45 1 +48 46 1 +48 47 1 +48 48 1 +48 49 1 +48 50 1 +48 51 1 +48 52 1 +48 53 1 +48 54 1 +48 55 1 +48 56 1 +48 57 1 +48 58 1 +48 59 1 +48 60 1 +48 61 1 +48 62 1 +48 63 1 +48 64 1 +48 65 1 +48 66 1 +48 67 1 +48 68 1 +48 69 1 +48 70 1 +48 71 1 +48 72 1 +48 73 1 +48 74 1 +48 75 1 +48 76 1 +48 77 1 +48 78 1 +48 79 1 +48 80 1 +48 81 1 +48 82 1 +48 83 1 +48 84 1 +48 85 1 +48 86 1 +48 87 1 +48 88 1 +48 89 1 +48 90 1 +48 91 1 +48 92 1 +48 93 1 +48 94 1 +48 95 1 +48 96 1 +48 97 1 +48 98 1 +48 99 1 +48 100 1 +49 0 1 +49 1 1 +49 2 1 +49 3 1 +49 4 1 +49 5 1 +49 6 1 +49 7 1 +49 8 1 +49 9 1 +49 10 1 +49 11 1 +49 12 1 +49 13 1 +49 14 1 +49 15 1 +49 16 1 +49 17 1 +49 18 1 +49 19 1 +49 20 1 +49 21 1 +49 22 1 +49 23 1 +49 24 1 +49 25 1 +49 26 1 +49 27 1 +49 28 1 +49 29 1 +49 30 1 +49 31 1 +49 32 1 +49 33 1 +49 34 1 +49 35 1 +49 36 1 +49 37 1 +49 38 1 +49 39 1 +49 40 1 +49 41 1 +49 42 1 +49 43 1 +49 44 1 +49 45 1 +49 46 1 +49 47 1 +49 48 1 +49 49 1 +49 50 1 +49 51 1 +49 52 1 +49 53 1 +49 54 1 +49 55 1 +49 56 1 +49 57 1 +49 58 1 +49 59 1 +49 60 1 +49 61 1 +49 62 1 +49 63 1 +49 64 1 +49 65 1 +49 66 1 +49 67 1 +49 68 1 +49 69 1 +49 70 1 +49 71 1 +49 72 1 +49 73 1 +49 74 1 +49 75 1 +49 76 1 +49 77 1 +49 78 1 +49 79 1 +49 80 1 +49 81 1 +49 82 1 +49 83 1 +49 84 1 +49 85 1 +49 86 1 +49 87 1 +49 88 1 +49 89 1 +49 90 1 +49 91 1 +49 92 1 +49 93 1 +49 94 1 +49 95 1 +49 96 1 +49 97 1 +49 98 1 +49 99 1 +49 100 1 +50 0 1 +50 1 1 +50 2 1 +50 3 1 +50 4 1 +50 5 1 +50 6 1 +50 7 1 +50 8 1 +50 9 1 +50 10 1 +50 11 1 +50 12 1 +50 13 1 +50 14 1 +50 15 1 +50 16 1 +50 17 1 +50 18 1 +50 19 1 +50 20 1 +50 21 1 +50 22 1 +50 23 1 +50 24 1 +50 25 1 +50 26 1 +50 27 1 +50 28 1 +50 29 1 +50 30 1 +50 31 1 +50 32 1 +50 33 1 +50 34 1 +50 35 1 +50 36 1 +50 37 1 +50 38 1 +50 39 1 +50 40 1 +50 41 1 +50 42 1 +50 43 1 +50 44 1 +50 45 1 +50 46 1 +50 47 1 +50 48 1 +50 49 1 +50 50 1 +50 51 1 +50 52 1 +50 53 1 +50 54 1 +50 55 1 +50 56 1 +50 57 1 +50 58 1 +50 59 1 +50 60 1 +50 61 1 +50 62 1 +50 63 1 +50 64 1 +50 65 1 +50 66 1 +50 67 1 +50 68 1 +50 69 1 +50 70 1 +50 71 1 +50 72 1 +50 73 1 +50 74 1 +50 75 1 +50 76 1 +50 77 1 +50 78 1 +50 79 1 +50 80 1 +50 81 1 +50 82 1 +50 83 1 +50 84 1 +50 85 1 +50 86 1 +50 87 1 +50 88 1 +50 89 1 +50 90 1 +50 91 1 +50 92 1 +50 93 1 +50 94 1 +50 95 1 +50 96 1 +50 97 1 +50 98 1 +50 99 1 +50 100 1 +51 0 1 +51 1 1 +51 2 1 +51 3 1 +51 4 1 +51 5 1 +51 6 1 +51 7 1 +51 8 1 +51 9 1 +51 10 1 +51 11 1 +51 12 1 +51 13 1 +51 14 1 +51 15 1 +51 16 1 +51 17 1 +51 18 1 +51 19 1 +51 20 1 +51 21 1 +51 22 1 +51 23 1 +51 24 1 +51 25 1 +51 26 1 +51 27 1 +51 28 1 +51 29 1 +51 30 1 +51 31 1 +51 32 1 +51 33 1 +51 34 1 +51 35 1 +51 36 1 +51 37 1 +51 38 1 +51 39 1 +51 40 1 +51 41 1 +51 42 1 +51 43 1 +51 44 1 +51 45 1 +51 46 1 +51 47 1 +51 48 1 +51 49 1 +51 50 1 +51 51 1 +51 52 1 +51 53 1 +51 54 1 +51 55 1 +51 56 1 +51 57 1 +51 58 1 +51 59 1 +51 60 1 +51 61 1 +51 62 1 +51 63 1 +51 64 1 +51 65 1 +51 66 1 +51 67 1 +51 68 1 +51 69 1 +51 70 1 +51 71 1 +51 72 1 +51 73 1 +51 74 1 +51 75 1 +51 76 1 +51 77 1 +51 78 1 +51 79 1 +51 80 1 +51 81 1 +51 82 1 +51 83 1 +51 84 1 +51 85 1 +51 86 1 +51 87 1 +51 88 1 +51 89 1 +51 90 1 +51 91 1 +51 92 1 +51 93 1 +51 94 1 +51 95 1 +51 96 1 +51 97 1 +51 98 1 +51 99 1 +51 100 1 +52 0 1 +52 1 1 +52 2 1 +52 3 1 +52 4 1 +52 5 1 +52 6 1 +52 7 1 +52 8 1 +52 9 1 +52 10 1 +52 11 1 +52 12 1 +52 13 1 +52 14 1 +52 15 1 +52 16 1 +52 17 1 +52 18 1 +52 19 1 +52 20 1 +52 21 1 +52 22 1 +52 23 1 +52 24 1 +52 25 1 +52 26 1 +52 27 1 +52 28 1 +52 29 1 +52 30 1 +52 31 1 +52 32 1 +52 33 1 +52 34 1 +52 35 1 +52 36 1 +52 37 1 +52 38 1 +52 39 1 +52 40 1 +52 41 1 +52 42 1 +52 43 1 +52 44 1 +52 45 1 +52 46 1 +52 47 1 +52 48 1 +52 49 1 +52 50 1 +52 51 1 +52 52 1 +52 53 1 +52 54 1 +52 55 1 +52 56 1 +52 57 1 +52 58 1 +52 59 1 +52 60 1 +52 61 1 +52 62 1 +52 63 1 +52 64 1 +52 65 1 +52 66 1 +52 67 1 +52 68 1 +52 69 1 +52 70 1 +52 71 1 +52 72 1 +52 73 1 +52 74 1 +52 75 1 +52 76 1 +52 77 1 +52 78 1 +52 79 1 +52 80 1 +52 81 1 +52 82 1 +52 83 1 +52 84 1 +52 85 1 +52 86 1 +52 87 1 +52 88 1 +52 89 1 +52 90 1 +52 91 1 +52 92 1 +52 93 1 +52 94 1 +52 95 1 +52 96 1 +52 97 1 +52 98 1 +52 99 1 +52 100 1 +53 0 1 +53 1 1 +53 2 1 +53 3 1 +53 4 1 +53 5 1 +53 6 1 +53 7 1 +53 8 1 +53 9 1 +53 10 1 +53 11 1 +53 12 1 +53 13 1 +53 14 1 +53 15 1 +53 16 1 +53 17 1 +53 18 1 +53 19 1 +53 20 1 +53 21 1 +53 22 1 +53 23 1 +53 24 1 +53 25 1 +53 26 1 +53 27 1 +53 28 1 +53 29 1 +53 30 1 +53 31 1 +53 32 1 +53 33 1 +53 34 1 +53 35 1 +53 36 1 +53 37 1 +53 38 1 +53 39 1 +53 40 1 +53 41 1 +53 42 1 +53 43 1 +53 44 1 +53 45 1 +53 46 1 +53 47 1 +53 48 1 +53 49 1 +53 50 1 +53 51 1 +53 52 1 +53 53 1 +53 54 1 +53 55 1 +53 56 1 +53 57 1 +53 58 1 +53 59 1 +53 60 1 +53 61 1 +53 62 1 +53 63 1 +53 64 1 +53 65 1 +53 66 1 +53 67 1 +53 68 1 +53 69 1 +53 70 1 +53 71 1 +53 72 1 +53 73 1 +53 74 1 +53 75 1 +53 76 1 +53 77 1 +53 78 1 +53 79 1 +53 80 1 +53 81 1 +53 82 1 +53 83 1 +53 84 1 +53 85 1 +53 86 1 +53 87 1 +53 88 1 +53 89 1 +53 90 1 +53 91 1 +53 92 1 +53 93 1 +53 94 1 +53 95 1 +53 96 1 +53 97 1 +53 98 1 +53 99 1 +53 100 1 +54 0 1 +54 1 1 +54 2 1 +54 3 1 +54 4 1 +54 5 1 +54 6 1 +54 7 1 +54 8 1 +54 9 1 +54 10 1 +54 11 1 +54 12 1 +54 13 1 +54 14 1 +54 15 1 +54 16 1 +54 17 1 +54 18 1 +54 19 1 +54 20 1 +54 21 1 +54 22 1 +54 23 1 +54 24 1 +54 25 1 +54 26 1 +54 27 1 +54 28 1 +54 29 1 +54 30 1 +54 31 1 +54 32 1 +54 33 1 +54 34 1 +54 35 1 +54 36 1 +54 37 1 +54 38 1 +54 39 1 +54 40 1 +54 41 1 +54 42 1 +54 43 1 +54 44 1 +54 45 1 +54 46 1 +54 47 1 +54 48 1 +54 49 1 +54 50 1 +54 51 1 +54 52 1 +54 53 1 +54 54 1 +54 55 1 +54 56 1 +54 57 1 +54 58 1 +54 59 1 +54 60 1 +54 61 1 +54 62 1 +54 63 1 +54 64 1 +54 65 1 +54 66 1 +54 67 1 +54 68 1 +54 69 1 +54 70 1 +54 71 1 +54 72 1 +54 73 1 +54 74 1 +54 75 1 +54 76 1 +54 77 1 +54 78 1 +54 79 1 +54 80 1 +54 81 1 +54 82 1 +54 83 1 +54 84 1 +54 85 1 +54 86 1 +54 87 1 +54 88 1 +54 89 1 +54 90 1 +54 91 1 +54 92 1 +54 93 1 +54 94 1 +54 95 1 +54 96 1 +54 97 1 +54 98 1 +54 99 1 +54 100 1 +55 0 1 +55 1 1 +55 2 1 +55 3 1 +55 4 1 +55 5 1 +55 6 1 +55 7 1 +55 8 1 +55 9 1 +55 10 1 +55 11 1 +55 12 1 +55 13 1 +55 14 1 +55 15 1 +55 16 1 +55 17 1 +55 18 1 +55 19 1 +55 20 1 +55 21 1 +55 22 1 +55 23 1 +55 24 1 +55 25 1 +55 26 1 +55 27 1 +55 28 1 +55 29 1 +55 30 1 +55 31 1 +55 32 1 +55 33 1 +55 34 1 +55 35 1 +55 36 1 +55 37 1 +55 38 1 +55 39 1 +55 40 1 +55 41 1 +55 42 1 +55 43 1 +55 44 1 +55 45 1 +55 46 1 +55 47 1 +55 48 1 +55 49 1 +55 50 1 +55 51 1 +55 52 1 +55 53 1 +55 54 1 +55 55 1 +55 56 1 +55 57 1 +55 58 1 +55 59 1 +55 60 1 +55 61 1 +55 62 1 +55 63 1 +55 64 1 +55 65 1 +55 66 1 +55 67 1 +55 68 1 +55 69 1 +55 70 1 +55 71 1 +55 72 1 +55 73 1 +55 74 1 +55 75 1 +55 76 1 +55 77 1 +55 78 1 +55 79 1 +55 80 1 +55 81 1 +55 82 1 +55 83 1 +55 84 1 +55 85 1 +55 86 1 +55 87 1 +55 88 1 +55 89 1 +55 90 1 +55 91 1 +55 92 1 +55 93 1 +55 94 1 +55 95 1 +55 96 1 +55 97 1 +55 98 1 +55 99 1 +55 100 1 +56 0 1 +56 1 1 +56 2 1 +56 3 1 +56 4 1 +56 5 1 +56 6 1 +56 7 1 +56 8 1 +56 9 1 +56 10 1 +56 11 1 +56 12 1 +56 13 1 +56 14 1 +56 15 1 +56 16 1 +56 17 1 +56 18 1 +56 19 1 +56 20 1 +56 21 1 +56 22 1 +56 23 1 +56 24 1 +56 25 1 +56 26 1 +56 27 1 +56 28 1 +56 29 1 +56 30 1 +56 31 1 +56 32 1 +56 33 1 +56 34 1 +56 35 1 +56 36 1 +56 37 1 +56 38 1 +56 39 1 +56 40 1 +56 41 1 +56 42 1 +56 43 1 +56 44 1 +56 45 1 +56 46 1 +56 47 1 +56 48 1 +56 49 1 +56 50 1 +56 51 1 +56 52 1 +56 53 1 +56 54 1 +56 55 1 +56 56 1 +56 57 1 +56 58 1 +56 59 1 +56 60 1 +56 61 1 +56 62 1 +56 63 1 +56 64 1 +56 65 1 +56 66 1 +56 67 1 +56 68 1 +56 69 1 +56 70 1 +56 71 1 +56 72 1 +56 73 1 +56 74 1 +56 75 1 +56 76 1 +56 77 1 +56 78 1 +56 79 1 +56 80 1 +56 81 1 +56 82 1 +56 83 1 +56 84 1 +56 85 1 +56 86 1 +56 87 1 +56 88 1 +56 89 1 +56 90 1 +56 91 1 +56 92 1 +56 93 1 +56 94 1 +56 95 1 +56 96 1 +56 97 1 +56 98 1 +56 99 1 +56 100 1 +57 0 1 +57 1 1 +57 2 1 +57 3 1 +57 4 1 +57 5 1 +57 6 1 +57 7 1 +57 8 1 +57 9 1 +57 10 1 +57 11 1 +57 12 1 +57 13 1 +57 14 1 +57 15 1 +57 16 1 +57 17 1 +57 18 1 +57 19 1 +57 20 1 +57 21 1 +57 22 1 +57 23 1 +57 24 1 +57 25 1 +57 26 1 +57 27 1 +57 28 1 +57 29 1 +57 30 1 +57 31 1 +57 32 1 +57 33 1 +57 34 1 +57 35 1 +57 36 1 +57 37 1 +57 38 1 +57 39 1 +57 40 1 +57 41 1 +57 42 1 +57 43 1 +57 44 1 +57 45 1 +57 46 1 +57 47 1 +57 48 1 +57 49 1 +57 50 1 +57 51 1 +57 52 1 +57 53 1 +57 54 1 +57 55 1 +57 56 1 +57 57 1 +57 58 1 +57 59 1 +57 60 1 +57 61 1 +57 62 1 +57 63 1 +57 64 1 +57 65 1 +57 66 1 +57 67 1 +57 68 1 +57 69 1 +57 70 1 +57 71 1 +57 72 1 +57 73 1 +57 74 1 +57 75 1 +57 76 1 +57 77 1 +57 78 1 +57 79 1 +57 80 1 +57 81 1 +57 82 1 +57 83 1 +57 84 1 +57 85 1 +57 86 1 +57 87 1 +57 88 1 +57 89 1 +57 90 1 +57 91 1 +57 92 1 +57 93 1 +57 94 1 +57 95 1 +57 96 1 +57 97 1 +57 98 1 +57 99 1 +57 100 1 +58 0 1 +58 1 1 +58 2 1 +58 3 1 +58 4 1 +58 5 1 +58 6 1 +58 7 1 +58 8 1 +58 9 1 +58 10 1 +58 11 1 +58 12 1 +58 13 1 +58 14 1 +58 15 1 +58 16 1 +58 17 1 +58 18 1 +58 19 1 +58 20 1 +58 21 1 +58 22 1 +58 23 1 +58 24 1 +58 25 1 +58 26 1 +58 27 1 +58 28 1 +58 29 1 +58 30 1 +58 31 1 +58 32 1 +58 33 1 +58 34 1 +58 35 1 +58 36 1 +58 37 1 +58 38 1 +58 39 1 +58 40 1 +58 41 1 +58 42 1 +58 43 1 +58 44 1 +58 45 1 +58 46 1 +58 47 1 +58 48 1 +58 49 1 +58 50 1 +58 51 1 +58 52 1 +58 53 1 +58 54 1 +58 55 1 +58 56 1 +58 57 1 +58 58 1 +58 59 1 +58 60 1 +58 61 1 +58 62 1 +58 63 1 +58 64 1 +58 65 1 +58 66 1 +58 67 1 +58 68 1 +58 69 1 +58 70 1 +58 71 1 +58 72 1 +58 73 1 +58 74 1 +58 75 1 +58 76 1 +58 77 1 +58 78 1 +58 79 1 +58 80 1 +58 81 1 +58 82 1 +58 83 1 +58 84 1 +58 85 1 +58 86 1 +58 87 1 +58 88 1 +58 89 1 +58 90 1 +58 91 1 +58 92 1 +58 93 1 +58 94 1 +58 95 1 +58 96 1 +58 97 1 +58 98 1 +58 99 1 +58 100 1 +59 0 1 +59 1 1 +59 2 1 +59 3 1 +59 4 1 +59 5 1 +59 6 1 +59 7 1 +59 8 1 +59 9 1 +59 10 1 +59 11 1 +59 12 1 +59 13 1 +59 14 1 +59 15 1 +59 16 1 +59 17 1 +59 18 1 +59 19 1 +59 20 1 +59 21 1 +59 22 1 +59 23 1 +59 24 1 +59 25 1 +59 26 1 +59 27 1 +59 28 1 +59 29 1 +59 30 1 +59 31 1 +59 32 1 +59 33 1 +59 34 1 +59 35 1 +59 36 1 +59 37 1 +59 38 1 +59 39 1 +59 40 1 +59 41 1 +59 42 1 +59 43 1 +59 44 1 +59 45 1 +59 46 1 +59 47 1 +59 48 1 +59 49 1 +59 50 1 +59 51 1 +59 52 1 +59 53 1 +59 54 1 +59 55 1 +59 56 1 +59 57 1 +59 58 1 +59 59 1 +59 60 1 +59 61 1 +59 62 1 +59 63 1 +59 64 1 +59 65 1 +59 66 1 +59 67 1 +59 68 1 +59 69 1 +59 70 1 +59 71 1 +59 72 1 +59 73 1 +59 74 1 +59 75 1 +59 76 1 +59 77 1 +59 78 1 +59 79 1 +59 80 1 +59 81 1 +59 82 1 +59 83 1 +59 84 1 +59 85 1 +59 86 1 +59 87 1 +59 88 1 +59 89 1 +59 90 1 +59 91 1 +59 92 1 +59 93 1 +59 94 1 +59 95 1 +59 96 1 +59 97 1 +59 98 1 +59 99 1 +59 100 1 +60 0 1 +60 1 1 +60 2 1 +60 3 1 +60 4 1 +60 5 1 +60 6 1 +60 7 1 +60 8 1 +60 9 1 +60 10 1 +60 11 1 +60 12 1 +60 13 1 +60 14 1 +60 15 1 +60 16 1 +60 17 1 +60 18 1 +60 19 1 +60 20 1 +60 21 1 +60 22 1 +60 23 1 +60 24 1 +60 25 1 +60 26 1 +60 27 1 +60 28 1 +60 29 1 +60 30 1 +60 31 1 +60 32 1 +60 33 1 +60 34 1 +60 35 1 +60 36 1 +60 37 1 +60 38 1 +60 39 1 +60 40 1 +60 41 1 +60 42 1 +60 43 1 +60 44 1 +60 45 1 +60 46 1 +60 47 1 +60 48 1 +60 49 1 +60 50 1 +60 51 1 +60 52 1 +60 53 1 +60 54 1 +60 55 1 +60 56 1 +60 57 1 +60 58 1 +60 59 1 +60 60 1 +60 61 1 +60 62 1 +60 63 1 +60 64 1 +60 65 1 +60 66 1 +60 67 1 +60 68 1 +60 69 1 +60 70 1 +60 71 1 +60 72 1 +60 73 1 +60 74 1 +60 75 1 +60 76 1 +60 77 1 +60 78 1 +60 79 1 +60 80 1 +60 81 1 +60 82 1 +60 83 1 +60 84 1 +60 85 1 +60 86 1 +60 87 1 +60 88 1 +60 89 1 +60 90 1 +60 91 1 +60 92 1 +60 93 1 +60 94 1 +60 95 1 +60 96 1 +60 97 1 +60 98 1 +60 99 1 +60 100 1 +61 0 1 +61 1 1 +61 2 1 +61 3 1 +61 4 1 +61 5 1 +61 6 1 +61 7 1 +61 8 1 +61 9 1 +61 10 1 +61 11 1 +61 12 1 +61 13 1 +61 14 1 +61 15 1 +61 16 1 +61 17 1 +61 18 1 +61 19 1 +61 20 1 +61 21 1 +61 22 1 +61 23 1 +61 24 1 +61 25 1 +61 26 1 +61 27 1 +61 28 1 +61 29 1 +61 30 1 +61 31 1 +61 32 1 +61 33 1 +61 34 1 +61 35 1 +61 36 1 +61 37 1 +61 38 1 +61 39 1 +61 40 1 +61 41 1 +61 42 1 +61 43 1 +61 44 1 +61 45 1 +61 46 1 +61 47 1 +61 48 1 +61 49 1 +61 50 1 +61 51 1 +61 52 1 +61 53 1 +61 54 1 +61 55 1 +61 56 1 +61 57 1 +61 58 1 +61 59 1 +61 60 1 +61 61 1 +61 62 1 +61 63 1 +61 64 1 +61 65 1 +61 66 1 +61 67 1 +61 68 1 +61 69 1 +61 70 1 +61 71 1 +61 72 1 +61 73 1 +61 74 1 +61 75 1 +61 76 1 +61 77 1 +61 78 1 +61 79 1 +61 80 1 +61 81 1 +61 82 1 +61 83 1 +61 84 1 +61 85 1 +61 86 1 +61 87 1 +61 88 1 +61 89 1 +61 90 1 +61 91 1 +61 92 1 +61 93 1 +61 94 1 +61 95 1 +61 96 1 +61 97 1 +61 98 1 +61 99 1 +61 100 1 +62 0 1 +62 1 1 +62 2 1 +62 3 1 +62 4 1 +62 5 1 +62 6 1 +62 7 1 +62 8 1 +62 9 1 +62 10 1 +62 11 1 +62 12 1 +62 13 1 +62 14 1 +62 15 1 +62 16 1 +62 17 1 +62 18 1 +62 19 1 +62 20 1 +62 21 1 +62 22 1 +62 23 1 +62 24 1 +62 25 1 +62 26 1 +62 27 1 +62 28 1 +62 29 1 +62 30 1 +62 31 1 +62 32 1 +62 33 1 +62 34 1 +62 35 1 +62 36 1 +62 37 1 +62 38 1 +62 39 1 +62 40 1 +62 41 1 +62 42 1 +62 43 1 +62 44 1 +62 45 1 +62 46 1 +62 47 1 +62 48 1 +62 49 1 +62 50 1 +62 51 1 +62 52 1 +62 53 1 +62 54 1 +62 55 1 +62 56 1 +62 57 1 +62 58 1 +62 59 1 +62 60 1 +62 61 1 +62 62 1 +62 63 1 +62 64 1 +62 65 1 +62 66 1 +62 67 1 +62 68 1 +62 69 1 +62 70 1 +62 71 1 +62 72 1 +62 73 1 +62 74 1 +62 75 1 +62 76 1 +62 77 1 +62 78 1 +62 79 1 +62 80 1 +62 81 1 +62 82 1 +62 83 1 +62 84 1 +62 85 1 +62 86 1 +62 87 1 +62 88 1 +62 89 1 +62 90 1 +62 91 1 +62 92 1 +62 93 1 +62 94 1 +62 95 1 +62 96 1 +62 97 1 +62 98 1 +62 99 1 +62 100 1 +63 0 1 +63 1 1 +63 2 1 +63 3 1 +63 4 1 +63 5 1 +63 6 1 +63 7 1 +63 8 1 +63 9 1 +63 10 1 +63 11 1 +63 12 1 +63 13 1 +63 14 1 +63 15 1 +63 16 1 +63 17 1 +63 18 1 +63 19 1 +63 20 1 +63 21 1 +63 22 1 +63 23 1 +63 24 1 +63 25 1 +63 26 1 +63 27 1 +63 28 1 +63 29 1 +63 30 1 +63 31 1 +63 32 1 +63 33 1 +63 34 1 +63 35 1 +63 36 1 +63 37 1 +63 38 1 +63 39 1 +63 40 1 +63 41 1 +63 42 1 +63 43 1 +63 44 1 +63 45 1 +63 46 1 +63 47 1 +63 48 1 +63 49 1 +63 50 1 +63 51 1 +63 52 1 +63 53 1 +63 54 1 +63 55 1 +63 56 1 +63 57 1 +63 58 1 +63 59 1 +63 60 1 +63 61 1 +63 62 1 +63 63 1 +63 64 1 +63 65 1 +63 66 1 +63 67 1 +63 68 1 +63 69 1 +63 70 1 +63 71 1 +63 72 1 +63 73 1 +63 74 1 +63 75 1 +63 76 1 +63 77 1 +63 78 1 +63 79 1 +63 80 1 +63 81 1 +63 82 1 +63 83 1 +63 84 1 +63 85 1 +63 86 1 +63 87 1 +63 88 1 +63 89 1 +63 90 1 +63 91 1 +63 92 1 +63 93 1 +63 94 1 +63 95 1 +63 96 1 +63 97 1 +63 98 1 +63 99 1 +63 100 1 +64 0 1 +64 1 1 +64 2 1 +64 3 1 +64 4 1 +64 5 1 +64 6 1 +64 7 1 +64 8 1 +64 9 1 +64 10 1 +64 11 1 +64 12 1 +64 13 1 +64 14 1 +64 15 1 +64 16 1 +64 17 1 +64 18 1 +64 19 1 +64 20 1 +64 21 1 +64 22 1 +64 23 1 +64 24 1 +64 25 1 +64 26 1 +64 27 1 +64 28 1 +64 29 1 +64 30 1 +64 31 1 +64 32 1 +64 33 1 +64 34 1 +64 35 1 +64 36 1 +64 37 1 +64 38 1 +64 39 1 +64 40 1 +64 41 1 +64 42 1 +64 43 1 +64 44 1 +64 45 1 +64 46 1 +64 47 1 +64 48 1 +64 49 1 +64 50 1 +64 51 1 +64 52 1 +64 53 1 +64 54 1 +64 55 1 +64 56 1 +64 57 1 +64 58 1 +64 59 1 +64 60 1 +64 61 1 +64 62 1 +64 63 1 +64 64 1 +64 65 1 +64 66 1 +64 67 1 +64 68 1 +64 69 1 +64 70 1 +64 71 1 +64 72 1 +64 73 1 +64 74 1 +64 75 1 +64 76 1 +64 77 1 +64 78 1 +64 79 1 +64 80 1 +64 81 1 +64 82 1 +64 83 1 +64 84 1 +64 85 1 +64 86 1 +64 87 1 +64 88 1 +64 89 1 +64 90 1 +64 91 1 +64 92 1 +64 93 1 +64 94 1 +64 95 1 +64 96 1 +64 97 1 +64 98 1 +64 99 1 +64 100 1 +65 0 1 +65 1 1 +65 2 1 +65 3 1 +65 4 1 +65 5 1 +65 6 1 +65 7 1 +65 8 1 +65 9 1 +65 10 1 +65 11 1 +65 12 1 +65 13 1 +65 14 1 +65 15 1 +65 16 1 +65 17 1 +65 18 1 +65 19 1 +65 20 1 +65 21 1 +65 22 1 +65 23 1 +65 24 1 +65 25 1 +65 26 1 +65 27 1 +65 28 1 +65 29 1 +65 30 1 +65 31 1 +65 32 1 +65 33 1 +65 34 1 +65 35 1 +65 36 1 +65 37 1 +65 38 1 +65 39 1 +65 40 1 +65 41 1 +65 42 1 +65 43 1 +65 44 1 +65 45 1 +65 46 1 +65 47 1 +65 48 1 +65 49 1 +65 50 1 +65 51 1 +65 52 1 +65 53 1 +65 54 1 +65 55 1 +65 56 1 +65 57 1 +65 58 1 +65 59 1 +65 60 1 +65 61 1 +65 62 1 +65 63 1 +65 64 1 +65 65 1 +65 66 1 +65 67 1 +65 68 1 +65 69 1 +65 70 1 +65 71 1 +65 72 1 +65 73 1 +65 74 1 +65 75 1 +65 76 1 +65 77 1 +65 78 1 +65 79 1 +65 80 1 +65 81 1 +65 82 1 +65 83 1 +65 84 1 +65 85 1 +65 86 1 +65 87 1 +65 88 1 +65 89 1 +65 90 1 +65 91 1 +65 92 1 +65 93 1 +65 94 1 +65 95 1 +65 96 1 +65 97 1 +65 98 1 +65 99 1 +65 100 1 +66 0 1 +66 1 1 +66 2 1 +66 3 1 +66 4 1 +66 5 1 +66 6 1 +66 7 1 +66 8 1 +66 9 1 +66 10 1 +66 11 1 +66 12 1 +66 13 1 +66 14 1 +66 15 1 +66 16 1 +66 17 1 +66 18 1 +66 19 1 +66 20 1 +66 21 1 +66 22 1 +66 23 1 +66 24 1 +66 25 1 +66 26 1 +66 27 1 +66 28 1 +66 29 1 +66 30 1 +66 31 1 +66 32 1 +66 33 1 +66 34 1 +66 35 1 +66 36 1 +66 37 1 +66 38 1 +66 39 1 +66 40 1 +66 41 1 +66 42 1 +66 43 1 +66 44 1 +66 45 1 +66 46 1 +66 47 1 +66 48 1 +66 49 1 +66 50 1 +66 51 1 +66 52 1 +66 53 1 +66 54 1 +66 55 1 +66 56 1 +66 57 1 +66 58 1 +66 59 1 +66 60 1 +66 61 1 +66 62 1 +66 63 1 +66 64 1 +66 65 1 +66 66 1 +66 67 1 +66 68 1 +66 69 1 +66 70 1 +66 71 1 +66 72 1 +66 73 1 +66 74 1 +66 75 1 +66 76 1 +66 77 1 +66 78 1 +66 79 1 +66 80 1 +66 81 1 +66 82 1 +66 83 1 +66 84 1 +66 85 1 +66 86 1 +66 87 1 +66 88 1 +66 89 1 +66 90 1 +66 91 1 +66 92 1 +66 93 1 +66 94 1 +66 95 1 +66 96 1 +66 97 1 +66 98 1 +66 99 1 +66 100 1 +67 0 1 +67 1 1 +67 2 1 +67 3 1 +67 4 1 +67 5 1 +67 6 1 +67 7 1 +67 8 1 +67 9 1 +67 10 1 +67 11 1 +67 12 1 +67 13 1 +67 14 1 +67 15 1 +67 16 1 +67 17 1 +67 18 1 +67 19 1 +67 20 1 +67 21 1 +67 22 1 +67 23 1 +67 24 1 +67 25 1 +67 26 1 +67 27 1 +67 28 1 +67 29 1 +67 30 1 +67 31 1 +67 32 1 +67 33 1 +67 34 1 +67 35 1 +67 36 1 +67 37 1 +67 38 1 +67 39 1 +67 40 1 +67 41 1 +67 42 1 +67 43 1 +67 44 1 +67 45 1 +67 46 1 +67 47 1 +67 48 1 +67 49 1 +67 50 1 +67 51 1 +67 52 1 +67 53 1 +67 54 1 +67 55 1 +67 56 1 +67 57 1 +67 58 1 +67 59 1 +67 60 1 +67 61 1 +67 62 1 +67 63 1 +67 64 1 +67 65 1 +67 66 1 +67 67 1 +67 68 1 +67 69 1 +67 70 1 +67 71 1 +67 72 1 +67 73 1 +67 74 1 +67 75 1 +67 76 1 +67 77 1 +67 78 1 +67 79 1 +67 80 1 +67 81 1 +67 82 1 +67 83 1 +67 84 1 +67 85 1 +67 86 1 +67 87 1 +67 88 1 +67 89 1 +67 90 1 +67 91 1 +67 92 1 +67 93 1 +67 94 1 +67 95 1 +67 96 1 +67 97 1 +67 98 1 +67 99 1 +67 100 1 +68 0 1 +68 1 1 +68 2 1 +68 3 1 +68 4 1 +68 5 1 +68 6 1 +68 7 1 +68 8 1 +68 9 1 +68 10 1 +68 11 1 +68 12 1 +68 13 1 +68 14 1 +68 15 1 +68 16 1 +68 17 1 +68 18 1 +68 19 1 +68 20 1 +68 21 1 +68 22 1 +68 23 1 +68 24 1 +68 25 1 +68 26 1 +68 27 1 +68 28 1 +68 29 1 +68 30 1 +68 31 1 +68 32 1 +68 33 1 +68 34 1 +68 35 1 +68 36 1 +68 37 1 +68 38 1 +68 39 1 +68 40 1 +68 41 1 +68 42 1 +68 43 1 +68 44 1 +68 45 1 +68 46 1 +68 47 1 +68 48 1 +68 49 1 +68 50 1 +68 51 1 +68 52 1 +68 53 1 +68 54 1 +68 55 1 +68 56 1 +68 57 1 +68 58 1 +68 59 1 +68 60 1 +68 61 1 +68 62 1 +68 63 1 +68 64 1 +68 65 1 +68 66 1 +68 67 1 +68 68 1 +68 69 1 +68 70 1 +68 71 1 +68 72 1 +68 73 1 +68 74 1 +68 75 1 +68 76 1 +68 77 1 +68 78 1 +68 79 1 +68 80 1 +68 81 1 +68 82 1 +68 83 1 +68 84 1 +68 85 1 +68 86 1 +68 87 1 +68 88 1 +68 89 1 +68 90 1 +68 91 1 +68 92 1 +68 93 1 +68 94 1 +68 95 1 +68 96 1 +68 97 1 +68 98 1 +68 99 1 +68 100 1 +69 0 1 +69 1 1 +69 2 1 +69 3 1 +69 4 1 +69 5 1 +69 6 1 +69 7 1 +69 8 1 +69 9 1 +69 10 1 +69 11 1 +69 12 1 +69 13 1 +69 14 1 +69 15 1 +69 16 1 +69 17 1 +69 18 1 +69 19 1 +69 20 1 +69 21 1 +69 22 1 +69 23 1 +69 24 1 +69 25 1 +69 26 1 +69 27 1 +69 28 1 +69 29 1 +69 30 1 +69 31 1 +69 32 1 +69 33 1 +69 34 1 +69 35 1 +69 36 1 +69 37 1 +69 38 1 +69 39 1 +69 40 1 +69 41 1 +69 42 1 +69 43 1 +69 44 1 +69 45 1 +69 46 1 +69 47 1 +69 48 1 +69 49 1 +69 50 1 +69 51 1 +69 52 1 +69 53 1 +69 54 1 +69 55 1 +69 56 1 +69 57 1 +69 58 1 +69 59 1 +69 60 1 +69 61 1 +69 62 1 +69 63 1 +69 64 1 +69 65 1 +69 66 1 +69 67 1 +69 68 1 +69 69 1 +69 70 1 +69 71 1 +69 72 1 +69 73 1 +69 74 1 +69 75 1 +69 76 1 +69 77 1 +69 78 1 +69 79 1 +69 80 1 +69 81 1 +69 82 1 +69 83 1 +69 84 1 +69 85 1 +69 86 1 +69 87 1 +69 88 1 +69 89 1 +69 90 1 +69 91 1 +69 92 1 +69 93 1 +69 94 1 +69 95 1 +69 96 1 +69 97 1 +69 98 1 +69 99 1 +69 100 1 +70 0 1 +70 1 1 +70 2 1 +70 3 1 +70 4 1 +70 5 1 +70 6 1 +70 7 1 +70 8 1 +70 9 1 +70 10 1 +70 11 1 +70 12 1 +70 13 1 +70 14 1 +70 15 1 +70 16 1 +70 17 1 +70 18 1 +70 19 1 +70 20 1 +70 21 1 +70 22 1 +70 23 1 +70 24 1 +70 25 1 +70 26 1 +70 27 1 +70 28 1 +70 29 1 +70 30 1 +70 31 1 +70 32 1 +70 33 1 +70 34 1 +70 35 1 +70 36 1 +70 37 1 +70 38 1 +70 39 1 +70 40 1 +70 41 1 +70 42 1 +70 43 1 +70 44 1 +70 45 1 +70 46 1 +70 47 1 +70 48 1 +70 49 1 +70 50 1 +70 51 1 +70 52 1 +70 53 1 +70 54 1 +70 55 1 +70 56 1 +70 57 1 +70 58 1 +70 59 1 +70 60 1 +70 61 1 +70 62 1 +70 63 1 +70 64 1 +70 65 1 +70 66 1 +70 67 1 +70 68 1 +70 69 1 +70 70 1 +70 71 1 +70 72 1 +70 73 1 +70 74 1 +70 75 1 +70 76 1 +70 77 1 +70 78 1 +70 79 1 +70 80 1 +70 81 1 +70 82 1 +70 83 1 +70 84 1 +70 85 1 +70 86 1 +70 87 1 +70 88 1 +70 89 1 +70 90 1 +70 91 1 +70 92 1 +70 93 1 +70 94 1 +70 95 1 +70 96 1 +70 97 1 +70 98 1 +70 99 1 +70 100 1 +71 0 1 +71 1 1 +71 2 1 +71 3 1 +71 4 1 +71 5 1 +71 6 1 +71 7 1 +71 8 1 +71 9 1 +71 10 1 +71 11 1 +71 12 1 +71 13 1 +71 14 1 +71 15 1 +71 16 1 +71 17 1 +71 18 1 +71 19 1 +71 20 1 +71 21 1 +71 22 1 +71 23 1 +71 24 1 +71 25 1 +71 26 1 +71 27 1 +71 28 1 +71 29 1 +71 30 1 +71 31 1 +71 32 1 +71 33 1 +71 34 1 +71 35 1 +71 36 1 +71 37 1 +71 38 1 +71 39 1 +71 40 1 +71 41 1 +71 42 1 +71 43 1 +71 44 1 +71 45 1 +71 46 1 +71 47 1 +71 48 1 +71 49 1 +71 50 1 +71 51 1 +71 52 1 +71 53 1 +71 54 1 +71 55 1 +71 56 1 +71 57 1 +71 58 1 +71 59 1 +71 60 1 +71 61 1 +71 62 1 +71 63 1 +71 64 1 +71 65 1 +71 66 1 +71 67 1 +71 68 1 +71 69 1 +71 70 1 +71 71 1 +71 72 1 +71 73 1 +71 74 1 +71 75 1 +71 76 1 +71 77 1 +71 78 1 +71 79 1 +71 80 1 +71 81 1 +71 82 1 +71 83 1 +71 84 1 +71 85 1 +71 86 1 +71 87 1 +71 88 1 +71 89 1 +71 90 1 +71 91 1 +71 92 1 +71 93 1 +71 94 1 +71 95 1 +71 96 1 +71 97 1 +71 98 1 +71 99 1 +71 100 1 +72 0 1 +72 1 1 +72 2 1 +72 3 1 +72 4 1 +72 5 1 +72 6 1 +72 7 1 +72 8 1 +72 9 1 +72 10 1 +72 11 1 +72 12 1 +72 13 1 +72 14 1 +72 15 1 +72 16 1 +72 17 1 +72 18 1 +72 19 1 +72 20 1 +72 21 1 +72 22 1 +72 23 1 +72 24 1 +72 25 1 +72 26 1 +72 27 1 +72 28 1 +72 29 1 +72 30 1 +72 31 1 +72 32 1 +72 33 1 +72 34 1 +72 35 1 +72 36 1 +72 37 1 +72 38 1 +72 39 1 +72 40 1 +72 41 1 +72 42 1 +72 43 1 +72 44 1 +72 45 1 +72 46 1 +72 47 1 +72 48 1 +72 49 1 +72 50 1 +72 51 1 +72 52 1 +72 53 1 +72 54 1 +72 55 1 +72 56 1 +72 57 1 +72 58 1 +72 59 1 +72 60 1 +72 61 1 +72 62 1 +72 63 1 +72 64 1 +72 65 1 +72 66 1 +72 67 1 +72 68 1 +72 69 1 +72 70 1 +72 71 1 +72 72 1 +72 73 1 +72 74 1 +72 75 1 +72 76 1 +72 77 1 +72 78 1 +72 79 1 +72 80 1 +72 81 1 +72 82 1 +72 83 1 +72 84 1 +72 85 1 +72 86 1 +72 87 1 +72 88 1 +72 89 1 +72 90 1 +72 91 1 +72 92 1 +72 93 1 +72 94 1 +72 95 1 +72 96 1 +72 97 1 +72 98 1 +72 99 1 +72 100 1 +73 0 1 +73 1 1 +73 2 1 +73 3 1 +73 4 1 +73 5 1 +73 6 1 +73 7 1 +73 8 1 +73 9 1 +73 10 1 +73 11 1 +73 12 1 +73 13 1 +73 14 1 +73 15 1 +73 16 1 +73 17 1 +73 18 1 +73 19 1 +73 20 1 +73 21 1 +73 22 1 +73 23 1 +73 24 1 +73 25 1 +73 26 1 +73 27 1 +73 28 1 +73 29 1 +73 30 1 +73 31 1 +73 32 1 +73 33 1 +73 34 1 +73 35 1 +73 36 1 +73 37 1 +73 38 1 +73 39 1 +73 40 1 +73 41 1 +73 42 1 +73 43 1 +73 44 1 +73 45 1 +73 46 1 +73 47 1 +73 48 1 +73 49 1 +73 50 1 +73 51 1 +73 52 1 +73 53 1 +73 54 1 +73 55 1 +73 56 1 +73 57 1 +73 58 1 +73 59 1 +73 60 1 +73 61 1 +73 62 1 +73 63 1 +73 64 1 +73 65 1 +73 66 1 +73 67 1 +73 68 1 +73 69 1 +73 70 1 +73 71 1 +73 72 1 +73 73 1 +73 74 1 +73 75 1 +73 76 1 +73 77 1 +73 78 1 +73 79 1 +73 80 1 +73 81 1 +73 82 1 +73 83 1 +73 84 1 +73 85 1 +73 86 1 +73 87 1 +73 88 1 +73 89 1 +73 90 1 +73 91 1 +73 92 1 +73 93 1 +73 94 1 +73 95 1 +73 96 1 +73 97 1 +73 98 1 +73 99 1 +73 100 1 +74 0 1 +74 1 1 +74 2 1 +74 3 1 +74 4 1 +74 5 1 +74 6 1 +74 7 1 +74 8 1 +74 9 1 +74 10 1 +74 11 1 +74 12 1 +74 13 1 +74 14 1 +74 15 1 +74 16 1 +74 17 1 +74 18 1 +74 19 1 +74 20 1 +74 21 1 +74 22 1 +74 23 1 +74 24 1 +74 25 1 +74 26 1 +74 27 1 +74 28 1 +74 29 1 +74 30 1 +74 31 1 +74 32 1 +74 33 1 +74 34 1 +74 35 1 +74 36 1 +74 37 1 +74 38 1 +74 39 1 +74 40 1 +74 41 1 +74 42 1 +74 43 1 +74 44 1 +74 45 1 +74 46 1 +74 47 1 +74 48 1 +74 49 1 +74 50 1 +74 51 1 +74 52 1 +74 53 1 +74 54 1 +74 55 1 +74 56 1 +74 57 1 +74 58 1 +74 59 1 +74 60 1 +74 61 1 +74 62 1 +74 63 1 +74 64 1 +74 65 1 +74 66 1 +74 67 1 +74 68 1 +74 69 1 +74 70 1 +74 71 1 +74 72 1 +74 73 1 +74 74 1 +74 75 1 +74 76 1 +74 77 1 +74 78 1 +74 79 1 +74 80 1 +74 81 1 +74 82 1 +74 83 1 +74 84 1 +74 85 1 +74 86 1 +74 87 1 +74 88 1 +74 89 1 +74 90 1 +74 91 1 +74 92 1 +74 93 1 +74 94 1 +74 95 1 +74 96 1 +74 97 1 +74 98 1 +74 99 1 +74 100 1 +75 0 1 +75 1 1 +75 2 1 +75 3 1 +75 4 1 +75 5 1 +75 6 1 +75 7 1 +75 8 1 +75 9 1 +75 10 1 +75 11 1 +75 12 1 +75 13 1 +75 14 1 +75 15 1 +75 16 1 +75 17 1 +75 18 1 +75 19 1 +75 20 1 +75 21 1 +75 22 1 +75 23 1 +75 24 1 +75 25 1 +75 26 1 +75 27 1 +75 28 1 +75 29 1 +75 30 1 +75 31 1 +75 32 1 +75 33 1 +75 34 1 +75 35 1 +75 36 1 +75 37 1 +75 38 1 +75 39 1 +75 40 1 +75 41 1 +75 42 1 +75 43 1 +75 44 1 +75 45 1 +75 46 1 +75 47 1 +75 48 1 +75 49 1 +75 50 1 +75 51 1 +75 52 1 +75 53 1 +75 54 1 +75 55 1 +75 56 1 +75 57 1 +75 58 1 +75 59 1 +75 60 1 +75 61 1 +75 62 1 +75 63 1 +75 64 1 +75 65 1 +75 66 1 +75 67 1 +75 68 1 +75 69 1 +75 70 1 +75 71 1 +75 72 1 +75 73 1 +75 74 1 +75 75 1 +75 76 1 +75 77 1 +75 78 1 +75 79 1 +75 80 1 +75 81 1 +75 82 1 +75 83 1 +75 84 1 +75 85 1 +75 86 1 +75 87 1 +75 88 1 +75 89 1 +75 90 1 +75 91 1 +75 92 1 +75 93 1 +75 94 1 +75 95 1 +75 96 1 +75 97 1 +75 98 1 +75 99 1 +75 100 1 +76 0 1 +76 1 1 +76 2 1 +76 3 1 +76 4 1 +76 5 1 +76 6 1 +76 7 1 +76 8 1 +76 9 1 +76 10 1 +76 11 1 +76 12 1 +76 13 1 +76 14 1 +76 15 1 +76 16 1 +76 17 1 +76 18 1 +76 19 1 +76 20 1 +76 21 1 +76 22 1 +76 23 1 +76 24 1 +76 25 1 +76 26 1 +76 27 1 +76 28 1 +76 29 1 +76 30 1 +76 31 1 +76 32 1 +76 33 1 +76 34 1 +76 35 1 +76 36 1 +76 37 1 +76 38 1 +76 39 1 +76 40 1 +76 41 1 +76 42 1 +76 43 1 +76 44 1 +76 45 1 +76 46 1 +76 47 1 +76 48 1 +76 49 1 +76 50 1 +76 51 1 +76 52 1 +76 53 1 +76 54 1 +76 55 1 +76 56 1 +76 57 1 +76 58 1 +76 59 1 +76 60 1 +76 61 1 +76 62 1 +76 63 1 +76 64 1 +76 65 1 +76 66 1 +76 67 1 +76 68 1 +76 69 1 +76 70 1 +76 71 1 +76 72 1 +76 73 1 +76 74 1 +76 75 1 +76 76 1 +76 77 1 +76 78 1 +76 79 1 +76 80 1 +76 81 1 +76 82 1 +76 83 1 +76 84 1 +76 85 1 +76 86 1 +76 87 1 +76 88 1 +76 89 1 +76 90 1 +76 91 1 +76 92 1 +76 93 1 +76 94 1 +76 95 1 +76 96 1 +76 97 1 +76 98 1 +76 99 1 +76 100 1 +77 0 1 +77 1 1 +77 2 1 +77 3 1 +77 4 1 +77 5 1 +77 6 1 +77 7 1 +77 8 1 +77 9 1 +77 10 1 +77 11 1 +77 12 1 +77 13 1 +77 14 1 +77 15 1 +77 16 1 +77 17 1 +77 18 1 +77 19 1 +77 20 1 +77 21 1 +77 22 1 +77 23 1 +77 24 1 +77 25 1 +77 26 1 +77 27 1 +77 28 1 +77 29 1 +77 30 1 +77 31 1 +77 32 1 +77 33 1 +77 34 1 +77 35 1 +77 36 1 +77 37 1 +77 38 1 +77 39 1 +77 40 1 +77 41 1 +77 42 1 +77 43 1 +77 44 1 +77 45 1 +77 46 1 +77 47 1 +77 48 1 +77 49 1 +77 50 1 +77 51 1 +77 52 1 +77 53 1 +77 54 1 +77 55 1 +77 56 1 +77 57 1 +77 58 1 +77 59 1 +77 60 1 +77 61 1 +77 62 1 +77 63 1 +77 64 1 +77 65 1 +77 66 1 +77 67 1 +77 68 1 +77 69 1 +77 70 1 +77 71 1 +77 72 1 +77 73 1 +77 74 1 +77 75 1 +77 76 1 +77 77 1 +77 78 1 +77 79 1 +77 80 1 +77 81 1 +77 82 1 +77 83 1 +77 84 1 +77 85 1 +77 86 1 +77 87 1 +77 88 1 +77 89 1 +77 90 1 +77 91 1 +77 92 1 +77 93 1 +77 94 1 +77 95 1 +77 96 1 +77 97 1 +77 98 1 +77 99 1 +77 100 1 +78 0 1 +78 1 1 +78 2 1 +78 3 1 +78 4 1 +78 5 1 +78 6 1 +78 7 1 +78 8 1 +78 9 1 +78 10 1 +78 11 1 +78 12 1 +78 13 1 +78 14 1 +78 15 1 +78 16 1 +78 17 1 +78 18 1 +78 19 1 +78 20 1 +78 21 1 +78 22 1 +78 23 1 +78 24 1 +78 25 1 +78 26 1 +78 27 1 +78 28 1 +78 29 1 +78 30 1 +78 31 1 +78 32 1 +78 33 1 +78 34 1 +78 35 1 +78 36 1 +78 37 1 +78 38 1 +78 39 1 +78 40 1 +78 41 1 +78 42 1 +78 43 1 +78 44 1 +78 45 1 +78 46 1 +78 47 1 +78 48 1 +78 49 1 +78 50 1 +78 51 1 +78 52 1 +78 53 1 +78 54 1 +78 55 1 +78 56 1 +78 57 1 +78 58 1 +78 59 1 +78 60 1 +78 61 1 +78 62 1 +78 63 1 +78 64 1 +78 65 1 +78 66 1 +78 67 1 +78 68 1 +78 69 1 +78 70 1 +78 71 1 +78 72 1 +78 73 1 +78 74 1 +78 75 1 +78 76 1 +78 77 1 +78 78 1 +78 79 1 +78 80 1 +78 81 1 +78 82 1 +78 83 1 +78 84 1 +78 85 1 +78 86 1 +78 87 1 +78 88 1 +78 89 1 +78 90 1 +78 91 1 +78 92 1 +78 93 1 +78 94 1 +78 95 1 +78 96 1 +78 97 1 +78 98 1 +78 99 1 +78 100 1 +79 0 1 +79 1 1 +79 2 1 +79 3 1 +79 4 1 +79 5 1 +79 6 1 +79 7 1 +79 8 1 +79 9 1 +79 10 1 +79 11 1 +79 12 1 +79 13 1 +79 14 1 +79 15 1 +79 16 1 +79 17 1 +79 18 1 +79 19 1 +79 20 1 +79 21 1 +79 22 1 +79 23 1 +79 24 1 +79 25 1 +79 26 1 +79 27 1 +79 28 1 +79 29 1 +79 30 1 +79 31 1 +79 32 1 +79 33 1 +79 34 1 +79 35 1 +79 36 1 +79 37 1 +79 38 1 +79 39 1 +79 40 1 +79 41 1 +79 42 1 +79 43 1 +79 44 1 +79 45 1 +79 46 1 +79 47 1 +79 48 1 +79 49 1 +79 50 1 +79 51 1 +79 52 1 +79 53 1 +79 54 1 +79 55 1 +79 56 1 +79 57 1 +79 58 1 +79 59 1 +79 60 1 +79 61 1 +79 62 1 +79 63 1 +79 64 1 +79 65 1 +79 66 1 +79 67 1 +79 68 1 +79 69 1 +79 70 1 +79 71 1 +79 72 1 +79 73 1 +79 74 1 +79 75 1 +79 76 1 +79 77 1 +79 78 1 +79 79 1 +79 80 1 +79 81 1 +79 82 1 +79 83 1 +79 84 1 +79 85 1 +79 86 1 +79 87 1 +79 88 1 +79 89 1 +79 90 1 +79 91 1 +79 92 1 +79 93 1 +79 94 1 +79 95 1 +79 96 1 +79 97 1 +79 98 1 +79 99 1 +79 100 1 +80 0 1 +80 1 1 +80 2 1 +80 3 1 +80 4 1 +80 5 1 +80 6 1 +80 7 1 +80 8 1 +80 9 1 +80 10 1 +80 11 1 +80 12 1 +80 13 1 +80 14 1 +80 15 1 +80 16 1 +80 17 1 +80 18 1 +80 19 1 +80 20 1 +80 21 1 +80 22 1 +80 23 1 +80 24 1 +80 25 1 +80 26 1 +80 27 1 +80 28 1 +80 29 1 +80 30 1 +80 31 1 +80 32 1 +80 33 1 +80 34 1 +80 35 1 +80 36 1 +80 37 1 +80 38 1 +80 39 1 +80 40 1 +80 41 1 +80 42 1 +80 43 1 +80 44 1 +80 45 1 +80 46 1 +80 47 1 +80 48 1 +80 49 1 +80 50 1 +80 51 1 +80 52 1 +80 53 1 +80 54 1 +80 55 1 +80 56 1 +80 57 1 +80 58 1 +80 59 1 +80 60 1 +80 61 1 +80 62 1 +80 63 1 +80 64 1 +80 65 1 +80 66 1 +80 67 1 +80 68 1 +80 69 1 +80 70 1 +80 71 1 +80 72 1 +80 73 1 +80 74 1 +80 75 1 +80 76 1 +80 77 1 +80 78 1 +80 79 1 +80 80 1 +80 81 1 +80 82 1 +80 83 1 +80 84 1 +80 85 1 +80 86 1 +80 87 1 +80 88 1 +80 89 1 +80 90 1 +80 91 1 +80 92 1 +80 93 1 +80 94 1 +80 95 1 +80 96 1 +80 97 1 +80 98 1 +80 99 1 +80 100 1 +81 0 1 +81 1 1 +81 2 1 +81 3 1 +81 4 1 +81 5 1 +81 6 1 +81 7 1 +81 8 1 +81 9 1 +81 10 1 +81 11 1 +81 12 1 +81 13 1 +81 14 1 +81 15 1 +81 16 1 +81 17 1 +81 18 1 +81 19 1 +81 20 1 +81 21 1 +81 22 1 +81 23 1 +81 24 1 +81 25 1 +81 26 1 +81 27 1 +81 28 1 +81 29 1 +81 30 1 +81 31 1 +81 32 1 +81 33 1 +81 34 1 +81 35 1 +81 36 1 +81 37 1 +81 38 1 +81 39 1 +81 40 1 +81 41 1 +81 42 1 +81 43 1 +81 44 1 +81 45 1 +81 46 1 +81 47 1 +81 48 1 +81 49 1 +81 50 1 +81 51 1 +81 52 1 +81 53 1 +81 54 1 +81 55 1 +81 56 1 +81 57 1 +81 58 1 +81 59 1 +81 60 1 +81 61 1 +81 62 1 +81 63 1 +81 64 1 +81 65 1 +81 66 1 +81 67 1 +81 68 1 +81 69 1 +81 70 1 +81 71 1 +81 72 1 +81 73 1 +81 74 1 +81 75 1 +81 76 1 +81 77 1 +81 78 1 +81 79 1 +81 80 1 +81 81 1 +81 82 1 +81 83 1 +81 84 1 +81 85 1 +81 86 1 +81 87 1 +81 88 1 +81 89 1 +81 90 1 +81 91 1 +81 92 1 +81 93 1 +81 94 1 +81 95 1 +81 96 1 +81 97 1 +81 98 1 +81 99 1 +81 100 1 +82 0 1 +82 1 1 +82 2 1 +82 3 1 +82 4 1 +82 5 1 +82 6 1 +82 7 1 +82 8 1 +82 9 1 +82 10 1 +82 11 1 +82 12 1 +82 13 1 +82 14 1 +82 15 1 +82 16 1 +82 17 1 +82 18 1 +82 19 1 +82 20 1 +82 21 1 +82 22 1 +82 23 1 +82 24 1 +82 25 1 +82 26 1 +82 27 1 +82 28 1 +82 29 1 +82 30 1 +82 31 1 +82 32 1 +82 33 1 +82 34 1 +82 35 1 +82 36 1 +82 37 1 +82 38 1 +82 39 1 +82 40 1 +82 41 1 +82 42 1 +82 43 1 +82 44 1 +82 45 1 +82 46 1 +82 47 1 +82 48 1 +82 49 1 +82 50 1 +82 51 1 +82 52 1 +82 53 1 +82 54 1 +82 55 1 +82 56 1 +82 57 1 +82 58 1 +82 59 1 +82 60 1 +82 61 1 +82 62 1 +82 63 1 +82 64 1 +82 65 1 +82 66 1 +82 67 1 +82 68 1 +82 69 1 +82 70 1 +82 71 1 +82 72 1 +82 73 1 +82 74 1 +82 75 1 +82 76 1 +82 77 1 +82 78 1 +82 79 1 +82 80 1 +82 81 1 +82 82 1 +82 83 1 +82 84 1 +82 85 1 +82 86 1 +82 87 1 +82 88 1 +82 89 1 +82 90 1 +82 91 1 +82 92 1 +82 93 1 +82 94 1 +82 95 1 +82 96 1 +82 97 1 +82 98 1 +82 99 1 +82 100 1 +83 0 1 +83 1 1 +83 2 1 +83 3 1 +83 4 1 +83 5 1 +83 6 1 +83 7 1 +83 8 1 +83 9 1 +83 10 1 +83 11 1 +83 12 1 +83 13 1 +83 14 1 +83 15 1 +83 16 1 +83 17 1 +83 18 1 +83 19 1 +83 20 1 +83 21 1 +83 22 1 +83 23 1 +83 24 1 +83 25 1 +83 26 1 +83 27 1 +83 28 1 +83 29 1 +83 30 1 +83 31 1 +83 32 1 +83 33 1 +83 34 1 +83 35 1 +83 36 1 +83 37 1 +83 38 1 +83 39 1 +83 40 1 +83 41 1 +83 42 1 +83 43 1 +83 44 1 +83 45 1 +83 46 1 +83 47 1 +83 48 1 +83 49 1 +83 50 1 +83 51 1 +83 52 1 +83 53 1 +83 54 1 +83 55 1 +83 56 1 +83 57 1 +83 58 1 +83 59 1 +83 60 1 +83 61 1 +83 62 1 +83 63 1 +83 64 1 +83 65 1 +83 66 1 +83 67 1 +83 68 1 +83 69 1 +83 70 1 +83 71 1 +83 72 1 +83 73 1 +83 74 1 +83 75 1 +83 76 1 +83 77 1 +83 78 1 +83 79 1 +83 80 1 +83 81 1 +83 82 1 +83 83 1 +83 84 1 +83 85 1 +83 86 1 +83 87 1 +83 88 1 +83 89 1 +83 90 1 +83 91 1 +83 92 1 +83 93 1 +83 94 1 +83 95 1 +83 96 1 +83 97 1 +83 98 1 +83 99 1 +83 100 1 +84 0 1 +84 1 1 +84 2 1 +84 3 1 +84 4 1 +84 5 1 +84 6 1 +84 7 1 +84 8 1 +84 9 1 +84 10 1 +84 11 1 +84 12 1 +84 13 1 +84 14 1 +84 15 1 +84 16 1 +84 17 1 +84 18 1 +84 19 1 +84 20 1 +84 21 1 +84 22 1 +84 23 1 +84 24 1 +84 25 1 +84 26 1 +84 27 1 +84 28 1 +84 29 1 +84 30 1 +84 31 1 +84 32 1 +84 33 1 +84 34 1 +84 35 1 +84 36 1 +84 37 1 +84 38 1 +84 39 1 +84 40 1 +84 41 1 +84 42 1 +84 43 1 +84 44 1 +84 45 1 +84 46 1 +84 47 1 +84 48 1 +84 49 1 +84 50 1 +84 51 1 +84 52 1 +84 53 1 +84 54 1 +84 55 1 +84 56 1 +84 57 1 +84 58 1 +84 59 1 +84 60 1 +84 61 1 +84 62 1 +84 63 1 +84 64 1 +84 65 1 +84 66 1 +84 67 1 +84 68 1 +84 69 1 +84 70 1 +84 71 1 +84 72 1 +84 73 1 +84 74 1 +84 75 1 +84 76 1 +84 77 1 +84 78 1 +84 79 1 +84 80 1 +84 81 1 +84 82 1 +84 83 1 +84 84 1 +84 85 1 +84 86 1 +84 87 1 +84 88 1 +84 89 1 +84 90 1 +84 91 1 +84 92 1 +84 93 1 +84 94 1 +84 95 1 +84 96 1 +84 97 1 +84 98 1 +84 99 1 +84 100 1 +85 0 1 +85 1 1 +85 2 1 +85 3 1 +85 4 1 +85 5 1 +85 6 1 +85 7 1 +85 8 1 +85 9 1 +85 10 1 +85 11 1 +85 12 1 +85 13 1 +85 14 1 +85 15 1 +85 16 1 +85 17 1 +85 18 1 +85 19 1 +85 20 1 +85 21 1 +85 22 1 +85 23 1 +85 24 1 +85 25 1 +85 26 1 +85 27 1 +85 28 1 +85 29 1 +85 30 1 +85 31 1 +85 32 1 +85 33 1 +85 34 1 +85 35 1 +85 36 1 +85 37 1 +85 38 1 +85 39 1 +85 40 1 +85 41 1 +85 42 1 +85 43 1 +85 44 1 +85 45 1 +85 46 1 +85 47 1 +85 48 1 +85 49 1 +85 50 1 +85 51 1 +85 52 1 +85 53 1 +85 54 1 +85 55 1 +85 56 1 +85 57 1 +85 58 1 +85 59 1 +85 60 1 +85 61 1 +85 62 1 +85 63 1 +85 64 1 +85 65 1 +85 66 1 +85 67 1 +85 68 1 +85 69 1 +85 70 1 +85 71 1 +85 72 1 +85 73 1 +85 74 1 +85 75 1 +85 76 1 +85 77 1 +85 78 1 +85 79 1 +85 80 1 +85 81 1 +85 82 1 +85 83 1 +85 84 1 +85 85 1 +85 86 1 +85 87 1 +85 88 1 +85 89 1 +85 90 1 +85 91 1 +85 92 1 +85 93 1 +85 94 1 +85 95 1 +85 96 1 +85 97 1 +85 98 1 +85 99 1 +85 100 1 +86 0 1 +86 1 1 +86 2 1 +86 3 1 +86 4 1 +86 5 1 +86 6 1 +86 7 1 +86 8 1 +86 9 1 +86 10 1 +86 11 1 +86 12 1 +86 13 1 +86 14 1 +86 15 1 +86 16 1 +86 17 1 +86 18 1 +86 19 1 +86 20 1 +86 21 1 +86 22 1 +86 23 1 +86 24 1 +86 25 1 +86 26 1 +86 27 1 +86 28 1 +86 29 1 +86 30 1 +86 31 1 +86 32 1 +86 33 1 +86 34 1 +86 35 1 +86 36 1 +86 37 1 +86 38 1 +86 39 1 +86 40 1 +86 41 1 +86 42 1 +86 43 1 +86 44 1 +86 45 1 +86 46 1 +86 47 1 +86 48 1 +86 49 1 +86 50 1 +86 51 1 +86 52 1 +86 53 1 +86 54 1 +86 55 1 +86 56 1 +86 57 1 +86 58 1 +86 59 1 +86 60 1 +86 61 1 +86 62 1 +86 63 1 +86 64 1 +86 65 1 +86 66 1 +86 67 1 +86 68 1 +86 69 1 +86 70 1 +86 71 1 +86 72 1 +86 73 1 +86 74 1 +86 75 1 +86 76 1 +86 77 1 +86 78 1 +86 79 1 +86 80 1 +86 81 1 +86 82 1 +86 83 1 +86 84 1 +86 85 1 +86 86 1 +86 87 1 +86 88 1 +86 89 1 +86 90 1 +86 91 1 +86 92 1 +86 93 1 +86 94 1 +86 95 1 +86 96 1 +86 97 1 +86 98 1 +86 99 1 +86 100 1 +87 0 1 +87 1 1 +87 2 1 +87 3 1 +87 4 1 +87 5 1 +87 6 1 +87 7 1 +87 8 1 +87 9 1 +87 10 1 +87 11 1 +87 12 1 +87 13 1 +87 14 1 +87 15 1 +87 16 1 +87 17 1 +87 18 1 +87 19 1 +87 20 1 +87 21 1 +87 22 1 +87 23 1 +87 24 1 +87 25 1 +87 26 1 +87 27 1 +87 28 1 +87 29 1 +87 30 1 +87 31 1 +87 32 1 +87 33 1 +87 34 1 +87 35 1 +87 36 1 +87 37 1 +87 38 1 +87 39 1 +87 40 1 +87 41 1 +87 42 1 +87 43 1 +87 44 1 +87 45 1 +87 46 1 +87 47 1 +87 48 1 +87 49 1 +87 50 1 +87 51 1 +87 52 1 +87 53 1 +87 54 1 +87 55 1 +87 56 1 +87 57 1 +87 58 1 +87 59 1 +87 60 1 +87 61 1 +87 62 1 +87 63 1 +87 64 1 +87 65 1 +87 66 1 +87 67 1 +87 68 1 +87 69 1 +87 70 1 +87 71 1 +87 72 1 +87 73 1 +87 74 1 +87 75 1 +87 76 1 +87 77 1 +87 78 1 +87 79 1 +87 80 1 +87 81 1 +87 82 1 +87 83 1 +87 84 1 +87 85 1 +87 86 1 +87 87 1 +87 88 1 +87 89 1 +87 90 1 +87 91 1 +87 92 1 +87 93 1 +87 94 1 +87 95 1 +87 96 1 +87 97 1 +87 98 1 +87 99 1 +87 100 1 +88 0 1 +88 1 1 +88 2 1 +88 3 1 +88 4 1 +88 5 1 +88 6 1 +88 7 1 +88 8 1 +88 9 1 +88 10 1 +88 11 1 +88 12 1 +88 13 1 +88 14 1 +88 15 1 +88 16 1 +88 17 1 +88 18 1 +88 19 1 +88 20 1 +88 21 1 +88 22 1 +88 23 1 +88 24 1 +88 25 1 +88 26 1 +88 27 1 +88 28 1 +88 29 1 +88 30 1 +88 31 1 +88 32 1 +88 33 1 +88 34 1 +88 35 1 +88 36 1 +88 37 1 +88 38 1 +88 39 1 +88 40 1 +88 41 1 +88 42 1 +88 43 1 +88 44 1 +88 45 1 +88 46 1 +88 47 1 +88 48 1 +88 49 1 +88 50 1 +88 51 1 +88 52 1 +88 53 1 +88 54 1 +88 55 1 +88 56 1 +88 57 1 +88 58 1 +88 59 1 +88 60 1 +88 61 1 +88 62 1 +88 63 1 +88 64 1 +88 65 1 +88 66 1 +88 67 1 +88 68 1 +88 69 1 +88 70 1 +88 71 1 +88 72 1 +88 73 1 +88 74 1 +88 75 1 +88 76 1 +88 77 1 +88 78 1 +88 79 1 +88 80 1 +88 81 1 +88 82 1 +88 83 1 +88 84 1 +88 85 1 +88 86 1 +88 87 1 +88 88 1 +88 89 1 +88 90 1 +88 91 1 +88 92 1 +88 93 1 +88 94 1 +88 95 1 +88 96 1 +88 97 1 +88 98 1 +88 99 1 +88 100 1 +89 0 1 +89 1 1 +89 2 1 +89 3 1 +89 4 1 +89 5 1 +89 6 1 +89 7 1 +89 8 1 +89 9 1 +89 10 1 +89 11 1 +89 12 1 +89 13 1 +89 14 1 +89 15 1 +89 16 1 +89 17 1 +89 18 1 +89 19 1 +89 20 1 +89 21 1 +89 22 1 +89 23 1 +89 24 1 +89 25 1 +89 26 1 +89 27 1 +89 28 1 +89 29 1 +89 30 1 +89 31 1 +89 32 1 +89 33 1 +89 34 1 +89 35 1 +89 36 1 +89 37 1 +89 38 1 +89 39 1 +89 40 1 +89 41 1 +89 42 1 +89 43 1 +89 44 1 +89 45 1 +89 46 1 +89 47 1 +89 48 1 +89 49 1 +89 50 1 +89 51 1 +89 52 1 +89 53 1 +89 54 1 +89 55 1 +89 56 1 +89 57 1 +89 58 1 +89 59 1 +89 60 1 +89 61 1 +89 62 1 +89 63 1 +89 64 1 +89 65 1 +89 66 1 +89 67 1 +89 68 1 +89 69 1 +89 70 1 +89 71 1 +89 72 1 +89 73 1 +89 74 1 +89 75 1 +89 76 1 +89 77 1 +89 78 1 +89 79 1 +89 80 1 +89 81 1 +89 82 1 +89 83 1 +89 84 1 +89 85 1 +89 86 1 +89 87 1 +89 88 1 +89 89 1 +89 90 1 +89 91 1 +89 92 1 +89 93 1 +89 94 1 +89 95 1 +89 96 1 +89 97 1 +89 98 1 +89 99 1 +89 100 1 +90 0 1 +90 1 1 +90 2 1 +90 3 1 +90 4 1 +90 5 1 +90 6 1 +90 7 1 +90 8 1 +90 9 1 +90 10 1 +90 11 1 +90 12 1 +90 13 1 +90 14 1 +90 15 1 +90 16 1 +90 17 1 +90 18 1 +90 19 1 +90 20 1 +90 21 1 +90 22 1 +90 23 1 +90 24 1 +90 25 1 +90 26 1 +90 27 1 +90 28 1 +90 29 1 +90 30 1 +90 31 1 +90 32 1 +90 33 1 +90 34 1 +90 35 1 +90 36 1 +90 37 1 +90 38 1 +90 39 1 +90 40 1 +90 41 1 +90 42 1 +90 43 1 +90 44 1 +90 45 1 +90 46 1 +90 47 1 +90 48 1 +90 49 1 +90 50 1 +90 51 1 +90 52 1 +90 53 1 +90 54 1 +90 55 1 +90 56 1 +90 57 1 +90 58 1 +90 59 1 +90 60 1 +90 61 1 +90 62 1 +90 63 1 +90 64 1 +90 65 1 +90 66 1 +90 67 1 +90 68 1 +90 69 1 +90 70 1 +90 71 1 +90 72 1 +90 73 1 +90 74 1 +90 75 1 +90 76 1 +90 77 1 +90 78 1 +90 79 1 +90 80 1 +90 81 1 +90 82 1 +90 83 1 +90 84 1 +90 85 1 +90 86 1 +90 87 1 +90 88 1 +90 89 1 +90 90 1 +90 91 1 +90 92 1 +90 93 1 +90 94 1 +90 95 1 +90 96 1 +90 97 1 +90 98 1 +90 99 1 +90 100 1 +91 0 1 +91 1 1 +91 2 1 +91 3 1 +91 4 1 +91 5 1 +91 6 1 +91 7 1 +91 8 1 +91 9 1 +91 10 1 +91 11 1 +91 12 1 +91 13 1 +91 14 1 +91 15 1 +91 16 1 +91 17 1 +91 18 1 +91 19 1 +91 20 1 +91 21 1 +91 22 1 +91 23 1 +91 24 1 +91 25 1 +91 26 1 +91 27 1 +91 28 1 +91 29 1 +91 30 1 +91 31 1 +91 32 1 +91 33 1 +91 34 1 +91 35 1 +91 36 1 +91 37 1 +91 38 1 +91 39 1 +91 40 1 +91 41 1 +91 42 1 +91 43 1 +91 44 1 +91 45 1 +91 46 1 +91 47 1 +91 48 1 +91 49 1 +91 50 1 +91 51 1 +91 52 1 +91 53 1 +91 54 1 +91 55 1 +91 56 1 +91 57 1 +91 58 1 +91 59 1 +91 60 1 +91 61 1 +91 62 1 +91 63 1 +91 64 1 +91 65 1 +91 66 1 +91 67 1 +91 68 1 +91 69 1 +91 70 1 +91 71 1 +91 72 1 +91 73 1 +91 74 1 +91 75 1 +91 76 1 +91 77 1 +91 78 1 +91 79 1 +91 80 1 +91 81 1 +91 82 1 +91 83 1 +91 84 1 +91 85 1 +91 86 1 +91 87 1 +91 88 1 +91 89 1 +91 90 1 +91 91 1 +91 92 1 +91 93 1 +91 94 1 +91 95 1 +91 96 1 +91 97 1 +91 98 1 +91 99 1 +91 100 1 +92 0 1 +92 1 1 +92 2 1 +92 3 1 +92 4 1 +92 5 1 +92 6 1 +92 7 1 +92 8 1 +92 9 1 +92 10 1 +92 11 1 +92 12 1 +92 13 1 +92 14 1 +92 15 1 +92 16 1 +92 17 1 +92 18 1 +92 19 1 +92 20 1 +92 21 1 +92 22 1 +92 23 1 +92 24 1 +92 25 1 +92 26 1 +92 27 1 +92 28 1 +92 29 1 +92 30 1 +92 31 1 +92 32 1 +92 33 1 +92 34 1 +92 35 1 +92 36 1 +92 37 1 +92 38 1 +92 39 1 +92 40 1 +92 41 1 +92 42 1 +92 43 1 +92 44 1 +92 45 1 +92 46 1 +92 47 1 +92 48 1 +92 49 1 +92 50 1 +92 51 1 +92 52 1 +92 53 1 +92 54 1 +92 55 1 +92 56 1 +92 57 1 +92 58 1 +92 59 1 +92 60 1 +92 61 1 +92 62 1 +92 63 1 +92 64 1 +92 65 1 +92 66 1 +92 67 1 +92 68 1 +92 69 1 +92 70 1 +92 71 1 +92 72 1 +92 73 1 +92 74 1 +92 75 1 +92 76 1 +92 77 1 +92 78 1 +92 79 1 +92 80 1 +92 81 1 +92 82 1 +92 83 1 +92 84 1 +92 85 1 +92 86 1 +92 87 1 +92 88 1 +92 89 1 +92 90 1 +92 91 1 +92 92 1 +92 93 1 +92 94 1 +92 95 1 +92 96 1 +92 97 1 +92 98 1 +92 99 1 +92 100 1 +93 0 1 +93 1 1 +93 2 1 +93 3 1 +93 4 1 +93 5 1 +93 6 1 +93 7 1 +93 8 1 +93 9 1 +93 10 1 +93 11 1 +93 12 1 +93 13 1 +93 14 1 +93 15 1 +93 16 1 +93 17 1 +93 18 1 +93 19 1 +93 20 1 +93 21 1 +93 22 1 +93 23 1 +93 24 1 +93 25 1 +93 26 1 +93 27 1 +93 28 1 +93 29 1 +93 30 1 +93 31 1 +93 32 1 +93 33 1 +93 34 1 +93 35 1 +93 36 1 +93 37 1 +93 38 1 +93 39 1 +93 40 1 +93 41 1 +93 42 1 +93 43 1 +93 44 1 +93 45 1 +93 46 1 +93 47 1 +93 48 1 +93 49 1 +93 50 1 +93 51 1 +93 52 1 +93 53 1 +93 54 1 +93 55 1 +93 56 1 +93 57 1 +93 58 1 +93 59 1 +93 60 1 +93 61 1 +93 62 1 +93 63 1 +93 64 1 +93 65 1 +93 66 1 +93 67 1 +93 68 1 +93 69 1 +93 70 1 +93 71 1 +93 72 1 +93 73 1 +93 74 1 +93 75 1 +93 76 1 +93 77 1 +93 78 1 +93 79 1 +93 80 1 +93 81 1 +93 82 1 +93 83 1 +93 84 1 +93 85 1 +93 86 1 +93 87 1 +93 88 1 +93 89 1 +93 90 1 +93 91 1 +93 92 1 +93 93 1 +93 94 1 +93 95 1 +93 96 1 +93 97 1 +93 98 1 +93 99 1 +93 100 1 +94 0 1 +94 1 1 +94 2 1 +94 3 1 +94 4 1 +94 5 1 +94 6 1 +94 7 1 +94 8 1 +94 9 1 +94 10 1 +94 11 1 +94 12 1 +94 13 1 +94 14 1 +94 15 1 +94 16 1 +94 17 1 +94 18 1 +94 19 1 +94 20 1 +94 21 1 +94 22 1 +94 23 1 +94 24 1 +94 25 1 +94 26 1 +94 27 1 +94 28 1 +94 29 1 +94 30 1 +94 31 1 +94 32 1 +94 33 1 +94 34 1 +94 35 1 +94 36 1 +94 37 1 +94 38 1 +94 39 1 +94 40 1 +94 41 1 +94 42 1 +94 43 1 +94 44 1 +94 45 1 +94 46 1 +94 47 1 +94 48 1 +94 49 1 +94 50 1 +94 51 1 +94 52 1 +94 53 1 +94 54 1 +94 55 1 +94 56 1 +94 57 1 +94 58 1 +94 59 1 +94 60 1 +94 61 1 +94 62 1 +94 63 1 +94 64 1 +94 65 1 +94 66 1 +94 67 1 +94 68 1 +94 69 1 +94 70 1 +94 71 1 +94 72 1 +94 73 1 +94 74 1 +94 75 1 +94 76 1 +94 77 1 +94 78 1 +94 79 1 +94 80 1 +94 81 1 +94 82 1 +94 83 1 +94 84 1 +94 85 1 +94 86 1 +94 87 1 +94 88 1 +94 89 1 +94 90 1 +94 91 1 +94 92 1 +94 93 1 +94 94 1 +94 95 1 +94 96 1 +94 97 1 +94 98 1 +94 99 1 +94 100 1 +95 0 1 +95 1 1 +95 2 1 +95 3 1 +95 4 1 +95 5 1 +95 6 1 +95 7 1 +95 8 1 +95 9 1 +95 10 1 +95 11 1 +95 12 1 +95 13 1 +95 14 1 +95 15 1 +95 16 1 +95 17 1 +95 18 1 +95 19 1 +95 20 1 +95 21 1 +95 22 1 +95 23 1 +95 24 1 +95 25 1 +95 26 1 +95 27 1 +95 28 1 +95 29 1 +95 30 1 +95 31 1 +95 32 1 +95 33 1 +95 34 1 +95 35 1 +95 36 1 +95 37 1 +95 38 1 +95 39 1 +95 40 1 +95 41 1 +95 42 1 +95 43 1 +95 44 1 +95 45 1 +95 46 1 +95 47 1 +95 48 1 +95 49 1 +95 50 1 +95 51 1 +95 52 1 +95 53 1 +95 54 1 +95 55 1 +95 56 1 +95 57 1 +95 58 1 +95 59 1 +95 60 1 +95 61 1 +95 62 1 +95 63 1 +95 64 1 +95 65 1 +95 66 1 +95 67 1 +95 68 1 +95 69 1 +95 70 1 +95 71 1 +95 72 1 +95 73 1 +95 74 1 +95 75 1 +95 76 1 +95 77 1 +95 78 1 +95 79 1 +95 80 1 +95 81 1 +95 82 1 +95 83 1 +95 84 1 +95 85 1 +95 86 1 +95 87 1 +95 88 1 +95 89 1 +95 90 1 +95 91 1 +95 92 1 +95 93 1 +95 94 1 +95 95 1 +95 96 1 +95 97 1 +95 98 1 +95 99 1 +95 100 1 +96 0 1 +96 1 1 +96 2 1 +96 3 1 +96 4 1 +96 5 1 +96 6 1 +96 7 1 +96 8 1 +96 9 1 +96 10 1 +96 11 1 +96 12 1 +96 13 1 +96 14 1 +96 15 1 +96 16 1 +96 17 1 +96 18 1 +96 19 1 +96 20 1 +96 21 1 +96 22 1 +96 23 1 +96 24 1 +96 25 1 +96 26 1 +96 27 1 +96 28 1 +96 29 1 +96 30 1 +96 31 1 +96 32 1 +96 33 1 +96 34 1 +96 35 1 +96 36 1 +96 37 1 +96 38 1 +96 39 1 +96 40 1 +96 41 1 +96 42 1 +96 43 1 +96 44 1 +96 45 1 +96 46 1 +96 47 1 +96 48 1 +96 49 1 +96 50 1 +96 51 1 +96 52 1 +96 53 1 +96 54 1 +96 55 1 +96 56 1 +96 57 1 +96 58 1 +96 59 1 +96 60 1 +96 61 1 +96 62 1 +96 63 1 +96 64 1 +96 65 1 +96 66 1 +96 67 1 +96 68 1 +96 69 1 +96 70 1 +96 71 1 +96 72 1 +96 73 1 +96 74 1 +96 75 1 +96 76 1 +96 77 1 +96 78 1 +96 79 1 +96 80 1 +96 81 1 +96 82 1 +96 83 1 +96 84 1 +96 85 1 +96 86 1 +96 87 1 +96 88 1 +96 89 1 +96 90 1 +96 91 1 +96 92 1 +96 93 1 +96 94 1 +96 95 1 +96 96 1 +96 97 1 +96 98 1 +96 99 1 +96 100 1 +97 0 1 +97 1 1 +97 2 1 +97 3 1 +97 4 1 +97 5 1 +97 6 1 +97 7 1 +97 8 1 +97 9 1 +97 10 1 +97 11 1 +97 12 1 +97 13 1 +97 14 1 +97 15 1 +97 16 1 +97 17 1 +97 18 1 +97 19 1 +97 20 1 +97 21 1 +97 22 1 +97 23 1 +97 24 1 +97 25 1 +97 26 1 +97 27 1 +97 28 1 +97 29 1 +97 30 1 +97 31 1 +97 32 1 +97 33 1 +97 34 1 +97 35 1 +97 36 1 +97 37 1 +97 38 1 +97 39 1 +97 40 1 +97 41 1 +97 42 1 +97 43 1 +97 44 1 +97 45 1 +97 46 1 +97 47 1 +97 48 1 +97 49 1 +97 50 1 +97 51 1 +97 52 1 +97 53 1 +97 54 1 +97 55 1 +97 56 1 +97 57 1 +97 58 1 +97 59 1 +97 60 1 +97 61 1 +97 62 1 +97 63 1 +97 64 1 +97 65 1 +97 66 1 +97 67 1 +97 68 1 +97 69 1 +97 70 1 +97 71 1 +97 72 1 +97 73 1 +97 74 1 +97 75 1 +97 76 1 +97 77 1 +97 78 1 +97 79 1 +97 80 1 +97 81 1 +97 82 1 +97 83 1 +97 84 1 +97 85 1 +97 86 1 +97 87 1 +97 88 1 +97 89 1 +97 90 1 +97 91 1 +97 92 1 +97 93 1 +97 94 1 +97 95 1 +97 96 1 +97 97 1 +97 98 1 +97 99 1 +97 100 1 +98 0 1 +98 1 1 +98 2 1 +98 3 1 +98 4 1 +98 5 1 +98 6 1 +98 7 1 +98 8 1 +98 9 1 +98 10 1 +98 11 1 +98 12 1 +98 13 1 +98 14 1 +98 15 1 +98 16 1 +98 17 1 +98 18 1 +98 19 1 +98 20 1 +98 21 1 +98 22 1 +98 23 1 +98 24 1 +98 25 1 +98 26 1 +98 27 1 +98 28 1 +98 29 1 +98 30 1 +98 31 1 +98 32 1 +98 33 1 +98 34 1 +98 35 1 +98 36 1 +98 37 1 +98 38 1 +98 39 1 +98 40 1 +98 41 1 +98 42 1 +98 43 1 +98 44 1 +98 45 1 +98 46 1 +98 47 1 +98 48 1 +98 49 1 +98 50 1 +98 51 1 +98 52 1 +98 53 1 +98 54 1 +98 55 1 +98 56 1 +98 57 1 +98 58 1 +98 59 1 +98 60 1 +98 61 1 +98 62 1 +98 63 1 +98 64 1 +98 65 1 +98 66 1 +98 67 1 +98 68 1 +98 69 1 +98 70 1 +98 71 1 +98 72 1 +98 73 1 +98 74 1 +98 75 1 +98 76 1 +98 77 1 +98 78 1 +98 79 1 +98 80 1 +98 81 1 +98 82 1 +98 83 1 +98 84 1 +98 85 1 +98 86 1 +98 87 1 +98 88 1 +98 89 1 +98 90 1 +98 91 1 +98 92 1 +98 93 1 +98 94 1 +98 95 1 +98 96 1 +98 97 1 +98 98 1 +98 99 1 +98 100 1 +99 0 1 +99 1 1 +99 2 1 +99 3 1 +99 4 1 +99 5 1 +99 6 1 +99 7 1 +99 8 1 +99 9 1 +99 10 1 +99 11 1 +99 12 1 +99 13 1 +99 14 1 +99 15 1 +99 16 1 +99 17 1 +99 18 1 +99 19 1 +99 20 1 +99 21 1 +99 22 1 +99 23 1 +99 24 1 +99 25 1 +99 26 1 +99 27 1 +99 28 1 +99 29 1 +99 30 1 +99 31 1 +99 32 1 +99 33 1 +99 34 1 +99 35 1 +99 36 1 +99 37 1 +99 38 1 +99 39 1 +99 40 1 +99 41 1 +99 42 1 +99 43 1 +99 44 1 +99 45 1 +99 46 1 +99 47 1 +99 48 1 +99 49 1 +99 50 1 +99 51 1 +99 52 1 +99 53 1 +99 54 1 +99 55 1 +99 56 1 +99 57 1 +99 58 1 +99 59 1 +99 60 1 +99 61 1 +99 62 1 +99 63 1 +99 64 1 +99 65 1 +99 66 1 +99 67 1 +99 68 1 +99 69 1 +99 70 1 +99 71 1 +99 72 1 +99 73 1 +99 74 1 +99 75 1 +99 76 1 +99 77 1 +99 78 1 +99 79 1 +99 80 1 +99 81 1 +99 82 1 +99 83 1 +99 84 1 +99 85 1 +99 86 1 +99 87 1 +99 88 1 +99 89 1 +99 90 1 +99 91 1 +99 92 1 +99 93 1 +99 94 1 +99 95 1 +99 96 1 +99 97 1 +99 98 1 +99 99 1 +99 100 1 +100 0 1 +100 1 1 +100 2 1 +100 3 1 +100 4 1 +100 5 1 +100 6 1 +100 7 1 +100 8 1 +100 9 1 +100 10 1 +100 11 1 +100 12 1 +100 13 1 +100 14 1 +100 15 1 +100 16 1 +100 17 1 +100 18 1 +100 19 1 +100 20 1 +100 21 1 +100 22 1 +100 23 1 +100 24 1 +100 25 1 +100 26 1 +100 27 1 +100 28 1 +100 29 1 +100 30 1 +100 31 1 +100 32 1 +100 33 1 +100 34 1 +100 35 1 +100 36 1 +100 37 1 +100 38 1 +100 39 1 +100 40 1 +100 41 1 +100 42 1 +100 43 1 +100 44 1 +100 45 1 +100 46 1 +100 47 1 +100 48 1 +100 49 1 +100 50 1 +100 51 1 +100 52 1 +100 53 1 +100 54 1 +100 55 1 +100 56 1 +100 57 1 +100 58 1 +100 59 1 +100 60 1 +100 61 1 +100 62 1 +100 63 1 +100 64 1 +100 65 1 +100 66 1 +100 67 1 +100 68 1 +100 69 1 +100 70 1 +100 71 1 +100 72 1 +100 73 1 +100 74 1 +100 75 1 +100 76 1 +100 77 1 +100 78 1 +100 79 1 +100 80 1 +100 81 1 +100 82 1 +100 83 1 +100 84 1 +100 85 1 +100 86 1 +100 87 1 +100 88 1 +100 89 1 +100 90 1 +100 91 1 +100 92 1 +100 93 1 +100 94 1 +100 95 1 +100 96 1 +100 97 1 +100 98 1 +100 99 1 +100 100 1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable new file mode 100644 index 00000000..44ee42e2 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.srcsegmlentable @@ -0,0 +1 @@ +Uniform diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable new file mode 100644 index 00000000..80dd323c --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.trgsegmlentable @@ -0,0 +1 @@ +Geometric diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.ttable b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg.ttable new file mode 100644 index 00000000..e69de29b diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anji new file mode 100644 index 00000000..e69de29b diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji new file mode 100644 index 00000000..e69de29b diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor new file mode 100644 index 00000000..be586341 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.asifactor @@ -0,0 +1 @@ +0.3 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd new file mode 100644 index 00000000..d17ce6ae Binary files /dev/null and b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_alignd differ diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd new file mode 100644 index 00000000..67392c2b Binary files /dev/null and b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd differ diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 new file mode 100644 index 00000000..49d59571 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.hmm_p0 @@ -0,0 +1 @@ +0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor new file mode 100644 index 00000000..49d59571 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.lsifactor @@ -0,0 +1 @@ +0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo new file mode 100644 index 00000000..aa47d0d4 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.msinfo @@ -0,0 +1,2 @@ +0 +0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel new file mode 100644 index 00000000..1e5ffcdb --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.slmodel @@ -0,0 +1,3 @@ +Weighted incr. gaussian sentence length model... +numsents: 1 ; slensum: 1 ; tlensum: 1 +1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.src @@ -0,0 +1 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.srctrgc @@ -0,0 +1 @@ +1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb new file mode 100644 index 00000000..e04006a5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.svcb @@ -0,0 +1,4 @@ +2 0 +3 1 +0 NULL 0 +1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.trg @@ -0,0 +1 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb new file mode 100644 index 00000000..e04006a5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_invswm.tvcb @@ -0,0 +1,4 @@ +2 0 +3 1 +0 NULL 0 +1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anji new file mode 100644 index 00000000..e69de29b diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji new file mode 100644 index 00000000..e69de29b diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor new file mode 100644 index 00000000..be586341 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.asifactor @@ -0,0 +1 @@ +0.3 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd new file mode 100644 index 00000000..d17ce6ae Binary files /dev/null and b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_alignd differ diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd new file mode 100644 index 00000000..67392c2b Binary files /dev/null and b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_lexnd differ diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 new file mode 100644 index 00000000..49d59571 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.hmm_p0 @@ -0,0 +1 @@ +0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor new file mode 100644 index 00000000..49d59571 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.lsifactor @@ -0,0 +1 @@ +0.1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo new file mode 100644 index 00000000..aa47d0d4 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.msinfo @@ -0,0 +1,2 @@ +0 +0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel new file mode 100644 index 00000000..1e5ffcdb --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.slmodel @@ -0,0 +1,3 @@ +Weighted incr. gaussian sentence length model... +numsents: 1 ; slensum: 1 ; tlensum: 1 +1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.src @@ -0,0 +1 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.srctrgc @@ -0,0 +1 @@ +1 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb new file mode 100644 index 00000000..e04006a5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.svcb @@ -0,0 +1,4 @@ +2 0 +3 1 +0 NULL 0 +1 UNKNOWN_WORD 0 diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.trg @@ -0,0 +1 @@ + diff --git a/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb new file mode 100644 index 00000000..e04006a5 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/data/thot-new-model/tm/src_trg_swm.tvcb @@ -0,0 +1,4 @@ +2 0 +3 1 +0 NULL 0 +1 UNKNOWN_WORD 0 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj b/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj new file mode 100644 index 00000000..b8c398f9 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Serval.Machine.Shared.Tests.csproj @@ -0,0 +1,60 @@ + + + + net8.0 + Serval.Machine.Shared + enable + enable + false + true + true + true + $(NoWarn);CS1591;CS1573 + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + icu.net.dll.config + + + + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs new file mode 100644 index 00000000..3cd02524 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ClearMLServiceTests.cs @@ -0,0 +1,50 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class ClearMLServiceTests +{ + private const string ApiServer = "https://clearml.com"; + private const string AccessKey = "accessKey"; + private const string SecretKey = "secretKey"; + + [Test] + public async Task CreateTaskAsync() + { + var mockHttp = new MockHttpMessageHandler(); + mockHttp + .Expect(HttpMethod.Post, $"{ApiServer}/tasks.create") + .WithHeaders("Authorization", $"Bearer accessToken") + .WithPartialContent("\\u0027src_lang\\u0027: \\u0027spa_Latn\\u0027") + .WithPartialContent("\\u0027trg_lang\\u0027: \\u0027eng_Latn\\u0027") + .Respond("application/json", "{ \"data\": { \"id\": \"projectId\" } }"); + HttpClient httpClient = mockHttp.ToHttpClient(); + httpClient.BaseAddress = new Uri(ApiServer); + + var options = Substitute.For>(); + options.CurrentValue.Returns(new ClearMLOptions { AccessKey = AccessKey, SecretKey = SecretKey }); + var authService = Substitute.For(); + authService.GetAuthTokenAsync().Returns(Task.FromResult("accessToken")); + var env = new HostingEnvironment { EnvironmentName = Environments.Development }; + var httpClientFactory = Substitute.For(); + httpClientFactory.CreateClient("ClearML").Returns(httpClient); + var service = new ClearMLService(httpClientFactory, options, authService, env); + + string script = + "from machine.jobs.build_nmt_engine import run\n" + + "args = {\n" + + " 'model_type': 'huggingface',\n" + + " 'engine_id': 'engine1',\n" + + " 'build_id': 'build1',\n" + + " 'src_lang': 'spa_Latn',\n" + + " 'trg_lang': 'eng_Latn',\n" + + " 'max_steps': 20000,\n" + + " 'shared_file_uri': 's3://aqua-ml-data',\n" + + " 'clearml': True\n" + + "}\n" + + "run(args)\n"; + + string projectId = await service.CreateTaskAsync("build1", "project1", "dockerImage", script); + Assert.That(projectId, Is.EqualTo("projectId")); + mockHttp.VerifyNoOutstandingExpectation(); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs new file mode 100644 index 00000000..d9389a69 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockFactoryTests.cs @@ -0,0 +1,81 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class DistributedReaderWriterLockFactoryTests +{ + [Test] + public async Task InitAsync_ReleaseWriterLocks() + { + TestEnvironment env = new(); + env.Locks.Add( + new RWLock + { + Id = "resource1", + WriterLock = new() { Id = "lock1", HostId = "this_service" }, + ReaderLocks = [], + WriterQueue = [] + } + ); + + await env.Factory.InitAsync(); + + RWLock resource1 = env.Locks.Get("resource1"); + Assert.That(resource1.WriterLock, Is.Null); + } + + [Test] + public async Task InitAsync_ReleaseReaderLocks() + { + TestEnvironment env = new(); + env.Locks.Add( + new RWLock + { + Id = "resource1", + ReaderLocks = [new() { Id = "lock1", HostId = "this_service" }], + WriterQueue = [] + } + ); + + await env.Factory.InitAsync(); + + RWLock resource1 = env.Locks.Get("resource1"); + Assert.That(resource1.ReaderLocks, Is.Empty); + } + + [Test] + public async Task InitAsync_RemoveWaiters() + { + TestEnvironment env = new(); + env.Locks.Add( + new RWLock + { + Id = "resource1", + WriterLock = new() { Id = "lock1", HostId = "other_service" }, + ReaderLocks = [], + WriterQueue = [new() { Id = "lock2", HostId = "this_service" }] + } + ); + + await env.Factory.InitAsync(); + + RWLock resource1 = env.Locks.Get("resource1"); + Assert.That(resource1.WriterQueue, Is.Empty); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Locks = new MemoryRepository(); + ServiceOptions serviceOptions = new() { ServiceId = "this_service" }; + Factory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(serviceOptions), + Locks, + new ObjectIdGenerator() + ); + } + + public MemoryRepository Locks { get; } + public DistributedReaderWriterLockFactory Factory { get; } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs new file mode 100644 index 00000000..dae41b35 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/DistributedReaderWriterLockTests.cs @@ -0,0 +1,405 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class DistributedReaderWriterLockTests +{ + [Test] + public async Task ReaderLockAsync_NoLockAcquired() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + RWLock entity; + await using (await rwLock.ReaderLockAsync()) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task ReaderLockAsync_ReaderLockAcquired() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + RWLock entity; + await using (await rwLock.ReaderLockAsync()) + { + await using (await rwLock.ReaderLockAsync()) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task ReaderLockAsync_WriterLockAcquiredAndNotReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + await rwLock.WriterLockAsync(); + var task = rwLock.ReaderLockAsync(); + await AssertNeverCompletesAsync(task); + } + + [Test] + public async Task ReaderLockAsync_WriterLockAcquiredAndReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task; + await using (await rwLock.WriterLockAsync()) + { + task = rwLock.ReaderLockAsync(); + Assert.That(task.IsCompleted, Is.False); + } + + RWLock entity; + await using (await task) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task ReaderLockAsync_WriterLockAcquiredAndExpired() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + RWLock entity; + await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) + { + var task = rwLock.ReaderLockAsync(); + await Task.Delay(500); + await using (await task) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task ReaderLockAsync_Cancelled() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task; + await using (await rwLock.WriterLockAsync()) + { + var cts = new CancellationTokenSource(); + task = rwLock.ReaderLockAsync(cancellationToken: cts.Token); + cts.Cancel(); + Assert.CatchAsync(async () => await task); + } + + RWLock entity; + await using (await rwLock.ReaderLockAsync()) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task WriterLockAsync_NoLockAcquired() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + RWLock entity; + await using (await rwLock.WriterLockAsync()) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.False); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task WriterLockAsync_ReaderLockAcquiredAndNotReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + await rwLock.ReaderLockAsync(); + var task = rwLock.WriterLockAsync(); + await AssertNeverCompletesAsync(task); + } + + [Test] + public async Task WriterLockAsync_ReaderLockAcquiredAndReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task; + await using (await rwLock.ReaderLockAsync()) + { + task = rwLock.WriterLockAsync(); + Assert.That(task.IsCompleted, Is.False); + } + + RWLock entity; + await using (await task) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.False); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task WriterLockAsync_WriterLockAcquiredAndNeverReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + await rwLock.WriterLockAsync(); + var task = rwLock.WriterLockAsync(); + await AssertNeverCompletesAsync(task); + } + + [Test] + public async Task WriterLockAsync_WriterLockAcquiredAndReleased() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task; + await using (await rwLock.WriterLockAsync()) + { + task = rwLock.WriterLockAsync(); + Assert.That(task.IsCompleted, Is.False); + } + + RWLock entity; + await using (await task) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.False); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task WriterLockAsync_WriterLockTakesPriorityOverReaderLock() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task writeTask, + readTask; + await using (await rwLock.WriterLockAsync()) + { + readTask = rwLock.ReaderLockAsync(); + Assert.That(readTask.IsCompleted, Is.False); + writeTask = rwLock.WriterLockAsync(); + Assert.That(writeTask.IsCompleted, Is.False); + } + + await writeTask; + await AssertNeverCompletesAsync(readTask); + } + + [Test] + public async Task WriterLockAsync_FirstWriterLockHasPriority() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task1, + task2; + await using (await rwLock.WriterLockAsync()) + { + task1 = rwLock.WriterLockAsync(); + Assert.That(task1.IsCompleted, Is.False); + task2 = rwLock.WriterLockAsync(); + Assert.That(task2.IsCompleted, Is.False); + } + + await task1; + await AssertNeverCompletesAsync(task2); + } + + [Test] + public async Task WriterLockAsync_WriterLockAcquiredAndExpired() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + RWLock entity; + await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) + { + var task = rwLock.WriterLockAsync(); + await Task.Delay(500); + await using (await task) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.False); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + [Test] + public async Task WriterLockAsync_Cancelled() + { + var env = new TestEnvironment(); + IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); + + Task task; + await using (await rwLock.WriterLockAsync()) + { + var cts = new CancellationTokenSource(); + task = rwLock.WriterLockAsync(cancellationToken: cts.Token); + cts.Cancel(); + Assert.CatchAsync(async () => await task); + } + + RWLock entity; + await using (await rwLock.WriterLockAsync()) + { + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.False); + Assert.That(entity.IsAvailableForWriting(), Is.False); + }); + } + + entity = env.Locks.Get("test"); + Assert.Multiple(() => + { + Assert.That(entity.IsAvailableForReading(), Is.True); + Assert.That(entity.IsAvailableForWriting(), Is.True); + }); + } + + private static async Task AssertNeverCompletesAsync(Task task, int timeout = 100) + { + if (task.IsCompleted) + Assert.Fail("Task completed unexpectedly."); + Task completedTask = await Task.WhenAny(task, Task.Delay(timeout)).ConfigureAwait(false); + if (completedTask == task) + Assert.Fail("Task completed unexpectedly."); + var _ = task.ContinueWith(_ => Assert.Fail("Task completed unexpectedly."), TaskScheduler.Default); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Locks = new MemoryRepository(); + var idGenerator = new ObjectIdGenerator(); + var options = Substitute.For>(); + options.Value.Returns(new ServiceOptions { ServiceId = "host" }); + Factory = new DistributedReaderWriterLockFactory(options, Locks, idGenerator); + } + + public DistributedReaderWriterLockFactory Factory { get; } + public MemoryRepository Locks { get; } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs new file mode 100644 index 00000000..687bb477 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/InMemoryStorageTests.cs @@ -0,0 +1,91 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class InMemoryStorageTests +{ + [Test] + public async Task ExistsAsync() + { + using InMemoryStorage fs = new(); + using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + bool exists = await fs.ExistsAsync("file1"); + Assert.That(exists, Is.True); + } + + [Test] + public async Task OpenReadAsync() + { + using InMemoryStorage fs = new(); + string input; + using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) + { + input = "Hello"; + sw.WriteLine(input); + } + string? output; + using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) + { + output = sr.ReadLine(); + } + Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); + } + + [Test] + public async Task ListFilesAsync_Recurse() + { + using InMemoryStorage fs = new(); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); + Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); + } + + [Test] + public async Task ListFilesAsync_DoNotRecurse() + { + using InMemoryStorage fs = new(); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); + Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); + } + + [Test] + public async Task DeleteAsync() + { + using InMemoryStorage fs = new(); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + await fs.DeleteAsync("test", recurse: true); + var files = await fs.ListFilesAsync("test", recurse: true); + Assert.That(files, Is.Empty); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs new file mode 100644 index 00000000..008b13e0 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LanguageTagServiceTests.cs @@ -0,0 +1,69 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class LanguageTagServiceTests +{ + [Test] + [TestCase("es", "spa_Latn", Description = "Iso639_1Code")] + [TestCase("hne", "hne_Deva", Description = "Iso639_3Code")] + [TestCase("ks-Arab", "kas_Arab", Description = "ScriptCode")] + [TestCase("srp_Cyrl", "srp_Cyrl", Description = "InvalidLangTag")] + [TestCase("zh", "zho_Hans", Description = "ChineseNoScript")] + [TestCase("zh-Hant", "zho_Hant", Description = "ChineseScript")] + [TestCase("zh-TW", "zho_Hant", Description = "ChineseRegion")] + [TestCase("cmn", "zho_Hans", Description = "MandarinChineseNoScript")] + [TestCase("cmn-Hant", "zho_Hant", Description = "MandarinChineseScript")] + [TestCase("ms", "zsm_Latn", Description = "Macrolanguage")] + [TestCase("arb", "arb_Arab", Description = "Arabic")] + [TestCase("eng", "eng_Latn", Description = "InsteadOfISO639_1")] + [TestCase("eng-Latn", "eng_Latn", Description = "DashToUnderscore")] + [TestCase("kor", "kor_Hang", Description = "KoreanScript")] + [TestCase("kor_Kore", "kor_Hang", Description = "KoreanScriptCorrection")] + public void ConvertToFlores200CodeTest(string language, string internalCodeTruth) + { + if (!Sldr.IsInitialized) + Sldr.Initialize(); + new LanguageTagService().ConvertToFlores200Code(language, out string internalCode); + Assert.That(internalCode, Is.EqualTo(internalCodeTruth)); + } + + [Test] + [TestCase("en", "eng_Latn", true)] + [TestCase("ms", "zsm_Latn", true)] + [TestCase("cmn", "zho_Hans", true)] + [TestCase("xyz", "xyz", false)] + public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCode, bool nativeLanguageSupport) + { + if (!Sldr.IsInitialized) + Sldr.Initialize(); + bool isNative = new LanguageTagService().ConvertToFlores200Code(languageCode, out string internalCode); + Assert.Multiple(() => + { + Assert.That(internalCode, Is.EqualTo(resolvedLanguageCode)); + Assert.That(isNative, Is.EqualTo(nativeLanguageSupport)); + }); + } + + public class TestLanguageTagService : LanguageTagService + { + // Don't call Sldr initialize to call + protected override void InitializeSldrLanguageTags() + { + // remove langtags.json to force download + var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); + if (File.Exists(cachedAllTagsPath)) + File.Delete(cachedAllTagsPath); + Directory.CreateDirectory(Sldr.SldrCachePath); + } + } + + [Test] + public void BackupLangtagsJsonTest() + { + if (!Sldr.IsInitialized) + Sldr.Initialize(); + var service = new TestLanguageTagService(); + service.ConvertToFlores200Code("en", out string internalCode); + Assert.That(internalCode, Is.EqualTo("eng_Latn")); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs new file mode 100644 index 00000000..c1b7ff04 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/LocalStorageTests.cs @@ -0,0 +1,96 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class LocalStorageTests +{ + [Test] + public async Task ExistsAsync() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + bool exists = await fs.ExistsAsync("file1"); + Assert.That(exists, Is.True); + } + + [Test] + public async Task OpenReadAsync() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + string input; + using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) + { + input = "Hello"; + sw.WriteLine(input); + } + string? output; + using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) + { + output = sr.ReadLine(); + } + Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); + } + + [Test] + public async Task ListFilesAsync_Recurse() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); + Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); + } + + [Test] + public async Task ListFilesAsync_DoNotRecurse() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); + Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); + } + + [Test] + public async Task DeleteFileAsync() + { + using var tmpDir = new TempDirectory("test"); + using LocalStorage fs = new(tmpDir.Path); + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) + { + string input = "Hello"; + sw.WriteLine(input); + } + using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) + { + string input2 = "Hola"; + sw.WriteLine(input2); + } + await fs.DeleteAsync("test", recurse: true); + IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); + Assert.That(files, Is.Empty); + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs new file mode 100644 index 00000000..2a5e517b --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxDeliveryServiceTests.cs @@ -0,0 +1,214 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class MessageOutboxDeliveryServiceTests +{ + private const string OutboxId = "TestOutbox"; + private const string Method1 = "Method1"; + private const string Method2 = "Method2"; + + [Test] + public async Task ProcessMessagesAsync() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + await env.ProcessMessagesAsync(); + Received.InOrder(() => + { + env.Handler.HandleMessageAsync(Method2, "B", null, Arg.Any()); + env.Handler.HandleMessageAsync(Method1, "A", null, Arg.Any()); + env.Handler.HandleMessageAsync(Method2, "C", null, Arg.Any()); + }); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + } + + [Test] + public async Task ProcessMessagesAsync_Timeout() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + + // Timeout is long enough where the message attempt will be incremented, but not deleted. + env.EnableHandlerFailure(StatusCode.Internal); + await env.ProcessMessagesAsync(); + // Each group should try to send one message + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); + + // with now shorter timeout, the messages will be deleted. + // 4 start build attempts, and only one build completed attempt + env.Options.CurrentValue.Returns( + new MessageOutboxOptions { MessageExpirationTimeout = TimeSpan.FromMilliseconds(1) } + ); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env.Handler.Received(1) + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); + _ = env.Handler.Received(4) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Test] + public async Task ProcessMessagesAsync_UnavailableFailure() + { + var env = new TestEnvironment(); + env.AddStandardMessages(); + + env.EnableHandlerFailure(StatusCode.Unavailable); + await env.ProcessMessagesAsync(); + // Only the first group should be attempted - but not recorded as attempted + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(0)); + _ = env.Handler.Received(1) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + + env.Handler.ClearReceivedCalls(); + env.EnableHandlerFailure(StatusCode.Internal); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); + Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); + Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); + _ = env.Handler.Received(2) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + + env.Handler.ClearReceivedCalls(); + env.DisableHandlerFailure(); + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env.Handler.Received(1) + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); + _ = env.Handler.Received(2) + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Test] + public async Task ProcessMessagesAsync_File() + { + var env = new TestEnvironment(); + env.AddContentStreamMessages(); + + await env.ProcessMessagesAsync(); + Assert.That(env.Messages.Count, Is.EqualTo(0)); + _ = env.Handler.Received(1) + .HandleMessageAsync(Method1, "A", Arg.Is(s => s != null), Arg.Any()); + env.FileSystem.Received().DeleteFile(Path.Combine("outbox", "A")); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Outboxes = new MemoryRepository(); + Messages = new MemoryRepository(); + + Handler = Substitute.For(); + Handler.OutboxId.Returns(OutboxId); + FileSystem = Substitute.For(); + Options = Substitute.For>(); + Options.CurrentValue.Returns(new MessageOutboxOptions()); + + Service = new MessageOutboxDeliveryService( + Substitute.For(), + [Handler], + FileSystem, + Options, + Substitute.For>() + ); + } + + public MemoryRepository Outboxes { get; } + public MemoryRepository Messages { get; } + public MessageOutboxDeliveryService Service { get; } + public IOutboxMessageHandler Handler { get; } + public IOptionsMonitor Options { get; } + public IFileSystem FileSystem { get; } + + public Task ProcessMessagesAsync() + { + return Service.ProcessMessagesAsync(Messages); + } + + public void AddStandardMessages() + { + // messages out of order - will be fixed when retrieved + Messages.Add( + new OutboxMessage + { + Id = "A", + Index = 2, + Method = Method1, + GroupId = "A", + OutboxRef = OutboxId, + Content = "A", + HasContentStream = false + } + ); + Messages.Add( + new OutboxMessage + { + Id = "B", + Index = 1, + Method = Method2, + OutboxRef = OutboxId, + GroupId = "A", + Content = "B", + HasContentStream = false + } + ); + Messages.Add( + new OutboxMessage + { + Id = "C", + Index = 3, + Method = Method2, + OutboxRef = OutboxId, + GroupId = "B", + Content = "C", + HasContentStream = false + } + ); + } + + public void AddContentStreamMessages() + { + // messages out of order - will be fixed when retrieved + Messages.Add( + new OutboxMessage + { + Id = "A", + Index = 2, + Method = Method1, + GroupId = "A", + OutboxRef = OutboxId, + Content = "A", + HasContentStream = true + } + ); + FileSystem + .OpenRead(Path.Combine("outbox", "A")) + .Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes("Content"))); + } + + public void EnableHandlerFailure(StatusCode code) + { + Handler + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new RpcException(new Status(code, ""))); + Handler + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new RpcException(new Status(code, ""))); + } + + public void DisableHandlerFailure() + { + Handler + .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + Handler + .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs new file mode 100644 index 00000000..876568d9 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/MessageOutboxServiceTests.cs @@ -0,0 +1,93 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class MessageOutboxServiceTests +{ + private const string OutboxId = "TestOutbox"; + private const string Method = "TestMethod"; + + [Test] + public async Task EnqueueMessageAsync_NoContentStream() + { + TestEnvironment env = new(); + + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); + + Outbox outbox = env.Outboxes.Get(OutboxId); + Assert.That(outbox.CurrentIndex, Is.EqualTo(1)); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(1)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.False); + } + + [Test] + public async Task EnqueueMessageAsync_ExistingOutbox() + { + TestEnvironment env = new(); + env.Outboxes.Add(new Outbox { Id = OutboxId, CurrentIndex = 1 }); + + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); + + Outbox outbox = env.Outboxes.Get(OutboxId); + Assert.That(outbox.CurrentIndex, Is.EqualTo(2)); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(2)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.False); + } + + [Test] + public async Task EnqueueMessageAsync_HasContentStream() + { + TestEnvironment env = new(); + await using MemoryStream fileStream = new(); + env.FileSystem.OpenWrite(Path.Combine("outbox", "1")).Returns(fileStream); + + await using MemoryStream stream = new(Encoding.UTF8.GetBytes("content")); + await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content", stream); + + OutboxMessage message = env.Messages.Get("1"); + Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); + Assert.That(message.Method, Is.EqualTo(Method)); + Assert.That(message.Index, Is.EqualTo(1)); + Assert.That(message.Content, Is.EqualTo("content")); + Assert.That(message.HasContentStream, Is.True); + Assert.That(fileStream.ToArray(), Is.EqualTo(stream.ToArray())); + } + + [Test] + public void EnqueueMessageAsync_ContentTooLarge() + { + TestEnvironment env = new(); + env.Service.MaxDocumentSize = 5; + + Assert.ThrowsAsync(() => env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content")); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Outboxes = new MemoryRepository(); + Messages = new MemoryRepository(); + var idGenerator = Substitute.For(); + idGenerator.GenerateId().Returns("1"); + FileSystem = Substitute.For(); + var options = Substitute.For>(); + options.CurrentValue.Returns(new MessageOutboxOptions()); + Service = new MessageOutboxService(Outboxes, Messages, idGenerator, FileSystem, options); + } + + public MemoryRepository Outboxes { get; } + public MemoryRepository Messages { get; } + public IFileSystem FileSystem { get; } + public MessageOutboxService Service { get; } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs new file mode 100644 index 00000000..49923372 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ModelCleanupServiceTests.cs @@ -0,0 +1,104 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class ModelCleanupServiceTests +{ + private static readonly List ValidFiles = + [ + "models/engineId1_1.tar.gz", + "models/engineId2_2.tar.gz", + "models/engineId2_3.tar.gz" // only one build ahead - keep + ]; + private static readonly List InvalidFiles = + [ + "models/engineId2_1.tar.gz", // 1 build behind + "models/engineId2_4.tar.gz", // 2 builds ahead + "models/wrongId_1.tar.gz", + "models/engineId1_badBuildNumber.tar.gz", + "models/noBuildNumber.tar.gz", + "models/engineId1_1.differentExtension" + ]; + + [Test] + public async Task CheckModelsAsync_ValidFiles() + { + TestEnvironment env = new(); + await env.CreateFilesAsync(); + + Assert.That( + await env.SharedFileService.ListFilesAsync("models"), + Is.EquivalentTo(ValidFiles.Concat(InvalidFiles)) + ); + await env.CheckModelsAsync(); + // only valid files exist after running service + Assert.That(await env.SharedFileService.ListFilesAsync("models"), Is.EquivalentTo(ValidFiles)); + } + + private class TestEnvironment + { + private readonly MemoryRepository _engines; + + public TestEnvironment() + { + _engines = new MemoryRepository(); + _engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engineId1", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = true + } + ); + _engines.Add( + new TranslationEngine + { + Id = "engine2", + EngineId = "engineId2", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 2, + IsModelPersisted = true + } + ); + + SharedFileService = new SharedFileService(Substitute.For()); + + Service = new ModelCleanupService( + Substitute.For(), + SharedFileService, + Substitute.For>() + ); + } + + public ModelCleanupService Service { get; } + public ISharedFileService SharedFileService { get; } + + public async Task CreateFilesAsync() + { + foreach (string path in ValidFiles) + { + await WriteFileStubAsync(path, "content"); + } + foreach (string path in InvalidFiles) + { + await WriteFileStubAsync(path, "content"); + } + } + + public Task CheckModelsAsync() + { + return Service.CheckModelsAsync(_engines, CancellationToken.None); + } + + private async Task WriteFileStubAsync(string path, string content) + { + using StreamWriter streamWriter = new(await SharedFileService.OpenWriteAsync(path, CancellationToken.None)); + await streamWriter.WriteAsync(content); + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs new file mode 100644 index 00000000..439b8d7c --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtClearMLBuildJobFactoryTests.cs @@ -0,0 +1,123 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class NmtClearMLBuildJobFactoryTests +{ + [Test] + public async Task CreateJobScriptAsync_BuildOptions() + { + var env = new TestEnvironment(); + string script = await env.BuildJobFactory.CreateJobScriptAsync( + "engine1", + "build1", + "test_model", + BuildStage.Train, + buildOptions: "{ \"max_steps\": \"10\" }" + ); + Assert.That( + script, + Is.EqualTo( + @"from machine.jobs.build_nmt_engine import run +args = { + 'model_type': 'test_model', + 'engine_id': 'engine1', + 'build_id': 'build1', + 'src_lang': 'spa_Latn', + 'trg_lang': 'eng_Latn', + 'shared_file_uri': 's3://bucket', + 'shared_file_folder': 'folder1/folder2', + 'build_options': '''{ ""max_steps"": ""10"" }''', + 'clearml': True +} +run(args) +".ReplaceLineEndings("\n") + ) + ); + } + + [Test] + public async Task CreateJobScriptAsync_NoBuildOptions() + { + var env = new TestEnvironment(); + string script = await env.BuildJobFactory.CreateJobScriptAsync( + "engine1", + "build1", + "test_model", + BuildStage.Train + ); + Assert.That( + script, + Is.EqualTo( + @"from machine.jobs.build_nmt_engine import run +args = { + 'model_type': 'test_model', + 'engine_id': 'engine1', + 'build_id': 'build1', + 'src_lang': 'spa_Latn', + 'trg_lang': 'eng_Latn', + 'shared_file_uri': 's3://bucket', + 'shared_file_folder': 'folder1/folder2', + 'clearml': True +} +run(args) +".ReplaceLineEndings("\n") + ) + ); + } + + private class TestEnvironment + { + public ISharedFileService SharedFileService { get; } + public MemoryRepository Engines { get; } + public IOptionsMonitor Options { get; } + public ILanguageTagService LanguageTagService { get; } + public NmtClearMLBuildJobFactory BuildJobFactory { get; } + + public TestEnvironment() + { + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = false, + CurrentBuild = new() + { + BuildId = "build1", + JobId = "job1", + BuildJobRunner = BuildJobRunnerType.ClearML, + Stage = BuildStage.Train, + JobState = BuildJobState.Pending + } + } + ); + Options = Substitute.For>(); + Options.CurrentValue.Returns(new ClearMLOptions { }); + SharedFileService = Substitute.For(); + SharedFileService.GetBaseUri().Returns(new Uri("s3://bucket/folder1/folder2")); + LanguageTagService = Substitute.For(); + LanguageTagService.ConvertToFlores200Code("es", out string spa); + var anyStringArg = Arg.Any(); + LanguageTagService + .ConvertToFlores200Code("es", out anyStringArg) + .Returns(x => + { + x[1] = "spa_Latn"; + return true; + }); + LanguageTagService + .ConvertToFlores200Code("en", out anyStringArg) + .Returns(x => + { + x[1] = "eng_Latn"; + return true; + }); + BuildJobFactory = new NmtClearMLBuildJobFactory(SharedFileService, LanguageTagService, Engines); + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs new file mode 100644 index 00000000..5463e613 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/NmtEngineServiceTests.cs @@ -0,0 +1,324 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class NmtEngineServiceTests +{ + [Test] + public async Task StartBuildAsync() + { + using var env = new TestEnvironment(); + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildRevision, Is.EqualTo(1)); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.Multiple(() => + { + Assert.That(engine.CurrentBuild, Is.Null); + Assert.That(engine.BuildRevision, Is.EqualTo(2)); + Assert.That(engine.IsModelPersisted, Is.False); + }); + } + + [Test] + public async Task CancelBuildAsync_Building() + { + using var env = new TestEnvironment(); + env.UseInfiniteTrainJob(); + + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildRevision, Is.EqualTo(1)); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.WaitForBuildToStartAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + await env.Service.CancelBuildAsync("engine1"); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.CurrentBuild, Is.Null); + Assert.That(engine.BuildRevision, Is.EqualTo(1)); + } + + [Test] + public void CancelBuildAsync_NotBuilding() + { + using var env = new TestEnvironment(); + Assert.ThrowsAsync(() => env.Service.CancelBuildAsync("engine1")); + } + + [Test] + public async Task DeleteAsync_WhileBuilding() + { + using var env = new TestEnvironment(); + env.UseInfiniteTrainJob(); + + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildRevision, Is.EqualTo(1)); + await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); + await env.WaitForBuildToStartAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + await env.Service.DeleteAsync("engine1"); + // ensure that the train job has completed + await env.WaitForBuildToFinishAsync(); + Assert.That(env.Engines.Contains("engine1"), Is.False); + } + + private class TestEnvironment : DisposableBase + { + private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private Func _trainJobFunc; + private Task? _trainJobTask; + + public TestEnvironment() + { + if (!Sldr.IsInitialized) + Sldr.Initialize(offlineMode: true); + + _trainJobFunc = RunNormalTrainJob; + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = false + } + ); + _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync("engine1", Arg.Any()) + .Returns(Task.FromResult("project1")); + ClearMLService + .CreateTaskAsync( + "build1", + "project1", + Arg.Any(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult("job1")); + ClearMLService + .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) + .Do(_ => _trainJobTask = Task.Run(_trainJobFunc)); + ClearMLService + .When(x => x.StopTaskAsync("job1", Arg.Any())) + .Do(_ => _cancellationTokenSource.Cancel()); + SharedFileService = new SharedFileService(Substitute.For()); + var buildJobOptions = Substitute.For>(); + buildJobOptions.CurrentValue.Returns( + new BuildJobOptions + { + ClearML = + [ + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.Nmt, + ModelType = "huggingface", + DockerImage = "default", + Queue = "default" + }, + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.SmtTransfer, + ModelType = "thot", + DockerImage = "default", + Queue = "default" + } + ] + } + ); + BuildJobService = new BuildJobService( + [ + new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]), + new ClearMLBuildJobRunner( + ClearMLService, + [ + new NmtClearMLBuildJobFactory( + SharedFileService, + Substitute.For(), + Engines + ) + ], + buildJobOptions + ) + ], + Engines + ); + var clearMLOptions = Substitute.For>(); + clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); + ClearMLQueueService = new ClearMLMonitorService( + Substitute.For(), + ClearMLService, + SharedFileService, + clearMLOptions, + buildJobOptions, + Substitute.For>() + ); + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + public NmtEngineService Service { get; private set; } + public IClearMLQueueService ClearMLQueueService { get; } + public MemoryRepository Engines { get; } + public IPlatformService PlatformService { get; } + public IClearMLService ClearMLService { get; } + public ISharedFileService SharedFileService { get; } + public IBuildJobService BuildJobService { get; } + + public void StopServer() + { + _jobServer.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "nmt" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(50), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private NmtEngineService CreateService() + { + return new NmtEngineService( + PlatformService, + _lockFactory, + new MemoryDataAccessContext(), + Engines, + BuildJobService, + new LanguageTagService(), + ClearMLQueueService, + SharedFileService + ); + } + + public async Task WaitForBuildToFinishAsync() + { + await WaitForBuildState(e => e.CurrentBuild is null); + if (_trainJobTask is not null) + await _trainJobTask; + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => + e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage == BuildStage.Train + ); + } + + public void UseInfiniteTrainJob() + { + _trainJobFunc = RunInfiniteTrainJob; + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync(e => + e.EngineId == "engine1" + ); + while (true) + { + TranslationEngine? engine = subscription.Change.Entity; + if (engine is null || predicate(engine)) + break; + await subscription.WaitForChangeAsync(); + } + } + + private async Task RunNormalTrainJob() + { + await BuildJobService.BuildJobStartedAsync("engine1", "build1"); + + await using Stream stream = await SharedFileService.OpenWriteAsync("builds/build1/pretranslate.trg.json"); + + await BuildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + "engine1", + "build1", + BuildStage.Postprocess, + (0, 0.0) + ); + } + + private async Task RunInfiniteTrainJob() + { + await BuildJobService.BuildJobStartedAsync("engine1", "build1"); + + while (!_cancellationTokenSource.IsCancellationRequested) + await Task.Delay(50); + + await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); + } + + protected override void DisposeManagedResources() + { + _jobServer.Dispose(); + _cancellationTokenSource.Dispose(); + } + + private class EnvActivator(TestEnvironment env) : JobActivator + { + private readonly TestEnvironment _env = env; + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(NmtPreprocessBuildJob)) + { + return new NmtPreprocessBuildJob( + _env.PlatformService, + _env.Engines, + _env._lockFactory, + new MemoryDataAccessContext(), + Substitute.For>(), + _env.BuildJobService, + _env.SharedFileService, + Substitute.For(), + new LanguageTagService() + ); + } + if (jobType == typeof(PostprocessBuildJob)) + { + return new PostprocessBuildJob( + _env.PlatformService, + _env.Engines, + _env._lockFactory, + new MemoryDataAccessContext(), + _env.BuildJobService, + Substitute.For>(), + _env.SharedFileService + ); + } + return base.ActivateJob(jobType); + } + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs new file mode 100644 index 00000000..08b6d414 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -0,0 +1,581 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class PreprocessBuildJobTests +{ + [Test] + public async Task RunAsync_FilterOutEverything() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { }; + + await env.RunBuildJobAsync(corpus1); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(0)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task RunAsync_TrainOnAll() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = null }; + + await env.RunBuildJobAsync(corpus1); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(4)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task RunAsync_TrainOnTextIds() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = ["textId1"] }; + + await env.RunBuildJobAsync(corpus1); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(4)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task RunAsync_TrainAndPretranslateAll() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null, TrainOnTextIds = null }; + + await env.RunBuildJobAsync(corpus1); + + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); + } + + [Test] + public async Task RunAsync_PretranslateAll() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null }; + + await env.RunBuildJobAsync(corpus1); + + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); + } + + [Test] + public async Task RunAsync_PretranslateTextIds() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = ["textId1"], TrainOnTextIds = null }; + + await env.RunBuildJobAsync(corpus1); + + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); + } + + [Test] + public async Task RunAsync_EnableKeyTerms() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultParatextCorpus with { }; + + await env.RunBuildJobAsync(corpus1, useKeyTerms: true); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(0)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(1)); + }); + } + + [Test] + public async Task RunAsync_DisableKeyTerms() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultParatextCorpus with { }; + + await env.RunBuildJobAsync(corpus1, useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(0)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task RunAsync_PretranslateChapters() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultParatextCorpus with + { + PretranslateChapters = new Dictionary> + { + { + "1CH", + new HashSet { 12 } + } + } + }; + + await env.RunBuildJobAsync(corpus1); + + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); + } + + [Test] + public async Task RunAsync_TrainOnChapters() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultParatextCorpus with + { + TrainOnChapters = new Dictionary> + { + { + "MAT", + new HashSet { 1 } + } + } + }; + + await env.RunBuildJobAsync(corpus1, useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(5)); + Assert.That(src2Count, Is.EqualTo(0)); + Assert.That(trgCount, Is.EqualTo(0)); + Assert.That(termCount, Is.EqualTo(0)); + }); + } + + [Test] + public async Task RunAsync_MixedSource_Paratext() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultMixedSourceParatextCorpus with + { + TrainOnTextIds = null, + PretranslateTextIds = null + }; + + await env.RunBuildJobAsync(corpus1, useKeyTerms: false); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(4)); + Assert.That(src2Count, Is.EqualTo(12)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(12)); + } + + [Test] + public async Task RunAsync_MixedSource_Text() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultMixedSourceTextFileCorpus with + { + TrainOnTextIds = null, + PretranslateTextIds = null, + TrainOnChapters = null, + PretranslateChapters = null + }; + + await env.RunBuildJobAsync(corpus1); + + (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); + Assert.Multiple(() => + { + Assert.That(src1Count, Is.EqualTo(3)); + Assert.That(src2Count, Is.EqualTo(2)); + Assert.That(trgCount, Is.EqualTo(1)); + Assert.That(termCount, Is.EqualTo(0)); + }); + Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); + } + + [Test] + public void RunAsync_UnknownLanguageTagsNoData() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; + + Assert.ThrowsAsync(async () => + { + await env.RunBuildJobAsync(corpus1, engineId: "engine2"); + }); + } + + [Test] + public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() + { + using TestEnvironment env = new(); + Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; + + await env.RunBuildJobAsync(corpus1, engineId: "engine2", engineType: TranslationEngineType.SmtTransfer); + } + + private class TestEnvironment : DisposableBase + { + private static readonly string TestDataPath = Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "Services", + "data" + ); + + private readonly TempDirectory _tempDir; + + public ISharedFileService SharedFileService { get; } + public ICorpusService CorpusService { get; } + public IPlatformService PlatformService { get; } + public MemoryRepository Engines { get; } + public IDistributedReaderWriterLockFactory LockFactory { get; } + public IBuildJobService BuildJobService { get; } + public IClearMLService ClearMLService { get; } + public IOptionsMonitor BuildJobOptions { get; } + + public Corpus DefaultTextFileCorpus { get; } + public Corpus DefaultMixedSourceTextFileCorpus { get; } + public Corpus DefaultParatextCorpus { get; } + public Corpus DefaultMixedSourceParatextCorpus { get; } + + public TestEnvironment() + { + if (!Sldr.IsInitialized) + Sldr.Initialize(offlineMode: true); + + _tempDir = new TempDirectory("PreprocessBuildJobTests"); + + ZipParatextProject("pt-source1"); + ZipParatextProject("pt-source2"); + ZipParatextProject("pt-target1"); + + DefaultTextFileCorpus = new() + { + Id = "corpusId1", + SourceLanguage = "es", + TargetLanguage = "en", + PretranslateTextIds = [], + TrainOnTextIds = [], + SourceFiles = [TextFile("source1")], + TargetFiles = [TextFile("target1")] + }; + + DefaultMixedSourceTextFileCorpus = new() + { + Id = "corpusId1", + SourceLanguage = "es", + TargetLanguage = "en", + PretranslateTextIds = [], + TrainOnTextIds = [], + SourceFiles = [TextFile("source1"), TextFile("source2")], + TargetFiles = [TextFile("target1")] + }; + + DefaultParatextCorpus = new() + { + Id = "corpusId1", + SourceLanguage = "es", + TargetLanguage = "en", + PretranslateTextIds = [], + TrainOnTextIds = [], + SourceFiles = [ParatextFile("pt-source1")], + TargetFiles = [ParatextFile("pt-target1")] + }; + + DefaultMixedSourceParatextCorpus = new() + { + Id = "corpusId1", + SourceLanguage = "es", + TargetLanguage = "en", + PretranslateTextIds = [], + TrainOnTextIds = [], + SourceFiles = [ParatextFile("pt-source1"), ParatextFile("pt-source2")], + TargetFiles = [ParatextFile("pt-target1")] + }; + + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + Type = TranslationEngineType.Nmt, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = false, + CurrentBuild = new() + { + BuildId = "build1", + JobId = "job1", + JobState = BuildJobState.Pending, + BuildJobRunner = BuildJobRunnerType.Hangfire, + Stage = BuildStage.Preprocess + } + } + ); + Engines.Add( + new TranslationEngine + { + Id = "engine2", + EngineId = "engine2", + Type = TranslationEngineType.Nmt, + SourceLanguage = "xxx", + TargetLanguage = "zzz", + BuildRevision = 1, + IsModelPersisted = false, + CurrentBuild = new() + { + BuildId = "build1", + JobId = "job1", + JobState = BuildJobState.Pending, + BuildJobRunner = BuildJobRunnerType.Hangfire, + Stage = BuildStage.Preprocess + } + } + ); + Engines.Add( + new TranslationEngine + { + Id = "engine2", + EngineId = "engine2", + Type = TranslationEngineType.Nmt, + SourceLanguage = "xxx", + TargetLanguage = "zzz", + BuildRevision = 1, + IsModelPersisted = false, + CurrentBuild = new() + { + BuildId = "build1", + JobId = "job1", + JobState = BuildJobState.Pending, + BuildJobRunner = BuildJobRunnerType.Hangfire, + Stage = BuildStage.Preprocess + } + } + ); + CorpusService = new CorpusService(); + PlatformService = Substitute.For(); + LockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + BuildJobOptions = Substitute.For>(); + BuildJobOptions.CurrentValue.Returns( + new BuildJobOptions + { + ClearML = + [ + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.Nmt, + ModelType = "huggingface", + DockerImage = "default", + Queue = "default" + }, + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.SmtTransfer, + ModelType = "thot", + DockerImage = "default", + Queue = "default" + } + ] + } + ); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync("engine1", Arg.Any()) + .Returns(Task.FromResult("project1")); + ClearMLService + .GetProjectIdAsync("engine2", Arg.Any()) + .Returns(Task.FromResult("project1")); + ClearMLService + .GetProjectIdAsync("engine2", Arg.Any()) + .Returns(Task.FromResult("project1")); + ClearMLService + .CreateTaskAsync( + "build1", + "project1", + Arg.Any(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult("job1")); + SharedFileService = new SharedFileService(Substitute.For()); + BuildJobService = new BuildJobService( + [ + new HangfireBuildJobRunner( + Substitute.For(), + [new NmtHangfireBuildJobFactory()] + ), + new ClearMLBuildJobRunner( + ClearMLService, + [ + new NmtClearMLBuildJobFactory( + SharedFileService, + Substitute.For(), + Engines + ) + ], + BuildJobOptions + ) + ], + Engines + ); + } + + public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) + { + switch (engineType) + { + case TranslationEngineType.Nmt: + { + return new NmtPreprocessBuildJob( + PlatformService, + Engines, + LockFactory, + new MemoryDataAccessContext(), + Substitute.For>(), + BuildJobService, + SharedFileService, + CorpusService, + new LanguageTagService() + ) + { + Seed = 1234 + }; + } + case TranslationEngineType.SmtTransfer: + { + return new PreprocessBuildJob( + PlatformService, + Engines, + LockFactory, + new MemoryDataAccessContext(), + Substitute.For>(), + BuildJobService, + SharedFileService, + CorpusService + ) + { + Seed = 1234 + }; + } + default: + throw new InvalidOperationException("Unknown engine type."); + } + ; + } + + public Task RunBuildJobAsync( + Corpus corpus, + bool useKeyTerms = true, + string engineId = "engine1", + TranslationEngineType engineType = TranslationEngineType.Nmt + ) + { + return GetBuildJob(engineType) + .RunAsync(engineId, "build1", [corpus], useKeyTerms ? null : "{\"use_key_terms\":false}", default); + } + + public async Task<(int Source1Count, int Source2Count, int TargetCount, int TermCount)> GetTrainCountAsync() + { + using StreamReader srcReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.src.txt")); + using StreamReader trgReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.trg.txt")); + int src1Count = 0; + int src2Count = 0; + int trgCount = 0; + int termCount = 0; + string? srcLine; + string? trgLine; + while ( + (srcLine = await srcReader.ReadLineAsync()) is not null + && (trgLine = await trgReader.ReadLineAsync()) is not null + ) + { + srcLine = srcLine.Trim(); + trgLine = trgLine.Trim(); + if (srcLine.StartsWith("Source one")) + src1Count++; + else if (srcLine.StartsWith("Source two")) + src2Count++; + else if (srcLine.Length == 0) + trgCount++; + else + termCount++; + } + return (src1Count, src2Count, trgCount, termCount); + } + + public async Task GetPretranslateCountAsync() + { + using StreamReader reader = + new(await SharedFileService.OpenReadAsync("builds/build1/pretranslate.src.json")); + JsonArray? pretranslationJsonObject = JsonSerializer.Deserialize(await reader.ReadToEndAsync()); + return pretranslationJsonObject?.Count ?? 0; + } + + private void ZipParatextProject(string name) + { + ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), Path.Combine(_tempDir.Path, $"{name}.zip")); + } + + private CorpusFile ParatextFile(string name) + { + return new() + { + TextId = name, + Format = FileFormat.Paratext, + Location = Path.Combine(_tempDir.Path, $"{name}.zip") + }; + } + + private static CorpusFile TextFile(string name) + { + return new() + { + TextId = "textId1", + Format = FileFormat.Text, + Location = Path.Combine(TestDataPath, $"{name}.txt") + }; + } + + protected override void DisposeManagedResources() + { + _tempDir.Dispose(); + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs new file mode 100644 index 00000000..f3667838 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs @@ -0,0 +1,113 @@ +using Google.Protobuf.WellKnownTypes; +using Serval.Translation.V1; + +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class ServalPlatformOutboxMessageHandlerTests +{ + private static readonly JsonSerializerOptions JsonSerializerOptions = + new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + + [Test] + public async Task HandleMessageAsync_BuildStarted() + { + TestEnvironment env = new(); + + await env.Handler.HandleMessageAsync( + ServalPlatformOutboxConstants.BuildStarted, + JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), + null + ); + + _ = env.Client.Received(1).BuildStartedAsync(Arg.Is(x => x.BuildId == "C")); + } + + [Test] + public async Task HandleMessageAsync_InsertPretranslations() + { + TestEnvironment env = new(); + await using (MemoryStream stream = new()) + { + await JsonSerializer.SerializeAsync( + stream, + new[] + { + new Pretranslation + { + CorpusId = "corpus1", + TextId = "MAT", + Refs = ["MAT 1:1"], + Translation = "translation" + } + }, + JsonSerializerOptions + ); + stream.Seek(0, SeekOrigin.Begin); + await env.Handler.HandleMessageAsync( + ServalPlatformOutboxConstants.InsertPretranslations, + "engine1", + stream + ); + } + + _ = env.Client.Received(1).InsertPretranslations(); + _ = env.PretranslationWriter.Received(1) + .WriteAsync( + new InsertPretranslationRequest + { + EngineId = "engine1", + CorpusId = "corpus1", + TextId = "MAT", + Refs = { "MAT 1:1" }, + Translation = "translation" + }, + Arg.Any() + ); + } + + private class TestEnvironment + { + public TestEnvironment() + { + Client = Substitute.For(); + Client.BuildStartedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildCanceledAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildFaultedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client.BuildCompletedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); + Client + .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) + .Returns(CreateEmptyUnaryCall()); + PretranslationWriter = Substitute.For>(); + Client + .InsertPretranslations(cancellationToken: Arg.Any()) + .Returns( + TestCalls.AsyncClientStreamingCall( + PretranslationWriter, + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ) + ); + + Handler = new ServalPlatformOutboxMessageHandler(Client); + } + + public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } + public ServalPlatformOutboxMessageHandler Handler { get; } + public IClientStreamWriter PretranslationWriter { get; } + + private static AsyncUnaryCall CreateEmptyUnaryCall() + { + return new AsyncUnaryCall( + Task.FromResult(new Empty()), + Task.FromResult(new Metadata()), + () => Status.DefaultSuccess, + () => new Metadata(), + () => { } + ); + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs new file mode 100644 index 00000000..5517aabf --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/SmtTransferEngineServiceTests.cs @@ -0,0 +1,740 @@ +namespace Serval.Machine.Shared.Services; + +[TestFixture] +public class SmtTransferEngineServiceTests +{ + const string EngineId1 = "engine1"; + const string EngineId2 = "engine2"; + const string BuildId1 = "build1"; + const string CorpusId1 = "corpus1"; + + [Test] + public async Task CreateAsync() + { + using var env = new TestEnvironment(); + await env.Service.CreateAsync(EngineId2, "Engine 2", "es", "en"); + TranslationEngine? engine = await env.Engines.GetAsync(e => e.EngineId == EngineId2); + Assert.Multiple(() => + { + Assert.That(engine, Is.Not.Null); + Assert.That(engine?.EngineId, Is.EqualTo(EngineId2)); + Assert.That(engine?.BuildRevision, Is.EqualTo(0)); + Assert.That(engine?.IsModelPersisted, Is.True); + }); + string engineDir = Path.Combine("translation_engines", EngineId2); + _ = env.SmtModelFactory.Received().InitNewAsync(engineDir); + _ = env.TransferEngineFactory.Received().InitNewAsync(engineDir); + } + + [TestCase(BuildJobRunnerType.Hangfire)] + [TestCase(BuildJobRunnerType.ClearML)] + public async Task StartBuildAsync(BuildJobRunnerType trainJobRunnerType) + { + using var env = new TestEnvironment(trainJobRunnerType); + TranslationEngine engine = env.Engines.Get(EngineId1); + Assert.That(engine.BuildRevision, Is.EqualTo(1)); + // ensure that the SMT model was loaded before training + await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); + await env.Service.StartBuildAsync( + EngineId1, + BuildId1, + null, + [ + new Corpus() + { + Id = CorpusId1, + SourceLanguage = "es", + TargetLanguage = "en", + SourceFiles = [], + TargetFiles = [], + TrainOnTextIds = null, + PretranslateTextIds = null + } + ] + ); + await env.WaitForBuildToFinishAsync(); + _ = env.SmtBatchTrainer.Received() + .TrainAsync(Arg.Any>(), Arg.Any()); + _ = env.TruecaserTrainer.Received() + .TrainAsync(Arg.Any>(), Arg.Any()); + _ = env.SmtBatchTrainer.Received().SaveAsync(Arg.Any()); + _ = env.TruecaserTrainer.Received().SaveAsync(Arg.Any()); + engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Null); + Assert.That(engine.BuildRevision, Is.EqualTo(2)); + // check if SMT model was reloaded upon first use after training + env.SmtModel.ClearReceivedCalls(); + await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); + env.SmtModel.Received().Dispose(); + _ = env.SmtModel.DidNotReceive().SaveAsync(); + _ = env.Truecaser.DidNotReceive().SaveAsync(); + } + + [TestCase(BuildJobRunnerType.Hangfire)] + [TestCase(BuildJobRunnerType.ClearML)] + public async Task CancelBuildAsync_Building(BuildJobRunnerType trainJobRunnerType) + { + using var env = new TestEnvironment(trainJobRunnerType); + env.UseInfiniteTrainJob(); + + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.WaitForTrainingToStartAsync(); + TranslationEngine engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + await env.Service.CancelBuildAsync(EngineId1); + await env.WaitForBuildToFinishAsync(); + _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); + _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); + engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Null); + } + + [Test] + public void CancelBuildAsync_NotBuilding() + { + using var env = new TestEnvironment(); + Assert.ThrowsAsync(() => env.Service.CancelBuildAsync(EngineId1)); + } + + [Test] + public async Task StartBuildAsync_RestartUnfinishedBuild() + { + using var env = new TestEnvironment(BuildJobRunnerType.Hangfire); + env.UseInfiniteTrainJob(); + + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.WaitForTrainingToStartAsync(); + TranslationEngine engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + env.StopServer(); + await env.WaitForBuildToRestartAsync(); + engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Pending)); + _ = env.PlatformService.Received().BuildRestartingAsync(BuildId1); + env.SmtBatchTrainer.ClearSubstitute(ClearOptions.CallActions); + env.StartServer(); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Null); + } + + [TestCase(BuildJobRunnerType.Hangfire)] + [TestCase(BuildJobRunnerType.ClearML)] + public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerType) + { + using var env = new TestEnvironment(trainJobRunnerType); + env.UseInfiniteTrainJob(); + + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.WaitForTrainingToStartAsync(); + TranslationEngine engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + await env.Service.DeleteAsync(EngineId1); + await env.WaitForBuildToFinishAsync(); + await env.WaitForAllHangfireJobsToFinishAsync(); + _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); + _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); + Assert.That(env.Engines.Contains(EngineId1), Is.False); + } + + [TestCase(BuildJobRunnerType.Hangfire)] + [TestCase(BuildJobRunnerType.ClearML)] + public async Task TrainSegmentPairAsync(BuildJobRunnerType trainJobRunnerType) + { + using var env = new TestEnvironment(trainJobRunnerType); + env.UseInfiniteTrainJob(); + + await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); + await env.WaitForBuildToStartAsync(); + TranslationEngine engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Not.Null); + Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); + await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); + env.StopTraining(); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get(EngineId1); + Assert.That(engine.CurrentBuild, Is.Null); + Assert.That(engine.BuildRevision, Is.EqualTo(2)); + _ = env.SmtModel.Received(2).TrainSegmentAsync("esto es una prueba.", "this is a test.", true); + } + + [Test] + public async Task CommitAsync_LoadedInactive() + { + using var env = new TestEnvironment(); + await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); + await Task.Delay(10); + await env.CommitAsync(TimeSpan.Zero); + _ = env.SmtModel.Received().SaveAsync(); + Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.False); + } + + [Test] + public async Task CommitAsync_LoadedActive() + { + using var env = new TestEnvironment(); + await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); + await env.CommitAsync(TimeSpan.FromHours(1)); + _ = env.SmtModel.Received().SaveAsync(); + Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.True); + } + + [Test] + public async Task TranslateAsync() + { + using var env = new TestEnvironment(); + TranslationResult result = (await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."))[0]; + Assert.That(result.Translation, Is.EqualTo("this is a TEST.")); + } + + [Test] + public async Task GetWordGraphAsync() + { + using var env = new TestEnvironment(); + WordGraph result = await env.Service.GetWordGraphAsync(EngineId1, "esto es una prueba."); + Assert.That( + result.Arcs.Select(a => string.Join(' ', a.TargetTokens)), + Is.EqualTo(new[] { "this is", "a test", "." }) + ); + } + + private class TestEnvironment : DisposableBase + { + private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly ITruecaserFactory _truecaserFactory; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly BuildJobRunnerType _trainJobRunnerType; + private Task? _trainJobTask; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private bool _training = true; + + public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerType.ClearML) + { + _trainJobRunnerType = trainJobRunnerType; + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = EngineId1, + EngineId = EngineId1, + Type = TranslationEngineType.SmtTransfer, + SourceLanguage = "es", + TargetLanguage = "en", + BuildRevision = 1, + IsModelPersisted = false + } + ); + TrainSegmentPairs = new MemoryRepository(); + _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + SmtModel = Substitute.For(); + SmtBatchTrainer = Substitute.For(); + SmtBatchTrainer.Stats.Returns( + new TrainStats { TrainCorpusSize = 0, Metrics = { { "bleu", 0.0 }, { "perplexity", 0.0 } } } + ); + Truecaser = Substitute.For(); + TruecaserTrainer = Substitute.For(); + + SmtModelFactory = CreateSmtModelFactory(); + TransferEngineFactory = CreateTransferEngineFactory(); + _truecaserFactory = CreateTruecaserFactory(); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + SharedFileService = new SharedFileService(Substitute.For()); + var clearMLOptions = Substitute.For>(); + clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); + var buildJobOptions = Substitute.For>(); + buildJobOptions.CurrentValue.Returns( + new BuildJobOptions + { + ClearML = + [ + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.Nmt, + ModelType = "huggingface", + DockerImage = "default", + Queue = "default" + }, + new ClearMLBuildQueue() + { + TranslationEngineType = TranslationEngineType.SmtTransfer, + ModelType = "thot", + DockerImage = "default", + Queue = "default" + } + ] + } + ); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync("engine1", Arg.Any()) + .Returns(Task.FromResult("project1")); + ClearMLService + .CreateTaskAsync( + "build1", + "project1", + Arg.Any(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult("job1")); + ClearMLService + .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) + .Do(_ => _trainJobTask = Task.Run(RunTrainJob)); + ClearMLService + .When(x => x.StopTaskAsync("job1", Arg.Any())) + .Do(_ => _cancellationTokenSource.Cancel()); + ClearMLMonitorService = new ClearMLMonitorService( + Substitute.For(), + ClearMLService, + SharedFileService, + clearMLOptions, + buildJobOptions, + Substitute.For>() + ); + BuildJobService = new BuildJobService( + [ + new HangfireBuildJobRunner(_jobClient, [new SmtTransferHangfireBuildJobFactory()]), + new ClearMLBuildJobRunner( + ClearMLService, + [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], + buildJobOptions + ) + ], + Engines + ); + _jobServer = CreateJobServer(); + StateService = CreateStateService(); + Service = CreateService(); + } + + public SmtTransferEngineService Service { get; private set; } + public SmtTransferEngineStateService StateService { get; private set; } + public MemoryRepository Engines { get; } + public MemoryRepository TrainSegmentPairs { get; } + public ISmtModelFactory SmtModelFactory { get; } + public ITransferEngineFactory TransferEngineFactory { get; } + public ITrainer SmtBatchTrainer { get; } + public IInteractiveTranslationModel SmtModel { get; } + public ITruecaser Truecaser { get; } + public ITrainer TruecaserTrainer { get; } + public IPlatformService PlatformService { get; } + + public IClearMLService ClearMLService { get; } + public IClearMLQueueService ClearMLMonitorService { get; } + + public ISharedFileService SharedFileService { get; } + + public IBuildJobService BuildJobService { get; } + + public async Task CommitAsync(TimeSpan inactiveTimeout) + { + await StateService.CommitAsync(_lockFactory, Engines, inactiveTimeout); + } + + public void StopServer() + { + _jobServer.Dispose(); + StateService.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + StateService = CreateStateService(); + Service = CreateService(); + } + + public void UseInfiniteTrainJob() + { + SmtBatchTrainer.TrainAsync( + Arg.Any>(), + Arg.Do(cancellationToken => + { + while (_training) + { + cancellationToken.ThrowIfCancellationRequested(); + Thread.Sleep(100); + } + }) + ); + } + + public void StopTraining() + { + _training = false; + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "smt_transfer" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(50), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private SmtTransferEngineStateService CreateStateService() + { + var options = Substitute.For>(); + options.CurrentValue.Returns(new SmtTransferEngineOptions()); + return new SmtTransferEngineStateService( + SmtModelFactory, + TransferEngineFactory, + _truecaserFactory, + options + ); + } + + private SmtTransferEngineService CreateService() + { + return new SmtTransferEngineService( + _lockFactory, + PlatformService, + new MemoryDataAccessContext(), + Engines, + TrainSegmentPairs, + StateService, + BuildJobService, + ClearMLMonitorService + ); + } + + private ISmtModelFactory CreateSmtModelFactory() + { + ISmtModelFactory factory = Substitute.For(); + + var translationResult = new TranslationResult( + "this is a TEST.", + "esto es una prueba .".Split(), + "this is a TEST .".Split(), + [1.0, 1.0, 1.0, 1.0, 1.0], + [ + TranslationSources.Smt, + TranslationSources.Smt, + TranslationSources.Smt, + TranslationSources.Smt, + TranslationSources.Smt + ], + new WordAlignmentMatrix(5, 5) + { + [0, 0] = true, + [1, 1] = true, + [2, 2] = true, + [3, 3] = true, + [4, 4] = true + }, + [new Phrase(Range.Create(0, 5), 5)] + ); + SmtModel + .TranslateAsync(1, Arg.Any()) + .Returns(Task.FromResult>([translationResult])); + SmtModel + .GetWordGraphAsync(Arg.Any()) + .Returns( + Task.FromResult( + new WordGraph( + "esto es una prueba .".Split(), + new[] + { + new WordGraphArc( + 0, + 1, + 1.0, + "this is".Split(), + new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, + Range.Create(0, 2), + GetSources(2, false), + [1.0, 1.0] + ), + new WordGraphArc( + 1, + 2, + 1.0, + "a test".Split(), + new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, + Range.Create(2, 4), + GetSources(2, false), + [1.0, 1.0] + ), + new WordGraphArc( + 2, + 3, + 1.0, + ".".Split(), + new WordAlignmentMatrix(1, 1) { [0, 0] = true }, + Range.Create(4, 5), + GetSources(1, false), + [1.0] + ) + }, + [3] + ) + ) + ); + + factory + .CreateAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any>(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult(SmtModel)); + factory + .CreateTrainerAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult(SmtBatchTrainer)); + return factory; + } + + private static ITransferEngineFactory CreateTransferEngineFactory() + { + ITransferEngineFactory factory = Substitute.For(); + ITranslationEngine engine = Substitute.For(); + engine + .TranslateAsync(Arg.Any()) + .Returns( + Task.FromResult( + new TranslationResult( + "this is a TEST.", + "esto es una prueba .".Split(), + "this is a TEST .".Split(), + [1.0, 1.0, 1.0, 1.0, 1.0], + [ + TranslationSources.Transfer, + TranslationSources.Transfer, + TranslationSources.Transfer, + TranslationSources.Transfer, + TranslationSources.Transfer + ], + new WordAlignmentMatrix(5, 5) + { + [0, 0] = true, + [1, 1] = true, + [2, 2] = true, + [3, 3] = true, + [4, 4] = true + }, + [new Phrase(Range.Create(0, 5), 5)] + ) + ) + ); + factory + .CreateAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any>(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult(engine)); + return factory; + } + + private ITruecaserFactory CreateTruecaserFactory() + { + ITruecaserFactory factory = Substitute.For(); + factory.CreateAsync(Arg.Any()).Returns(Task.FromResult(Truecaser)); + factory + .CreateTrainerAsync( + Arg.Any(), + Arg.Any>(), + Arg.Any(), + Arg.Any() + ) + .Returns(Task.FromResult(TruecaserTrainer)); + return factory; + } + + private static TranslationSources[] GetSources(int count, bool isUnknown) + { + var sources = new TranslationSources[count]; + for (int i = 0; i < count; i++) + sources[i] = isUnknown ? TranslationSources.None : TranslationSources.Smt; + return sources; + } + + public async Task WaitForAllHangfireJobsToFinishAsync() + { + IMonitoringApi monitoringApi = _memoryStorage.GetMonitoringApi(); + while (monitoringApi.EnqueuedCount("smt_transfer") > 0 || monitoringApi.ProcessingCount() > 0) + await Task.Delay(50); + } + + public async Task WaitForBuildToFinishAsync() + { + await WaitForBuildState(e => e.CurrentBuild is null); + if (_trainJobTask is not null) + await _trainJobTask; + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Active); + } + + public Task WaitForTrainingToStartAsync() + { + return WaitForBuildState(e => + e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage is BuildStage.Train + ); + } + + public Task WaitForBuildToRestartAsync() + { + return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Pending); + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync(e => + e.EngineId == EngineId1 + ); + while (true) + { + TranslationEngine? engine = subscription.Change.Entity; + if (engine is null || predicate(engine)) + break; + await subscription.WaitForChangeAsync(); + } + } + + protected override void DisposeManagedResources() + { + StateService.Dispose(); + _jobServer.Dispose(); + } + + private async Task RunTrainJob() + { + try + { + await BuildJobService.BuildJobStartedAsync("engine1", "build1", _cancellationTokenSource.Token); + + string engineDir = Path.Combine("translation_engines", EngineId1); + await SmtModelFactory.InitNewAsync(engineDir, _cancellationTokenSource.Token); + ITextCorpus sourceCorpus = new DictionaryTextCorpus(); + ITextCorpus targetCorpus = new DictionaryTextCorpus(); + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + LatinWordTokenizer tokenizer = new(); + using ITrainer smtModelTrainer = await SmtModelFactory.CreateTrainerAsync( + engineDir, + tokenizer, + parallelCorpus, + _cancellationTokenSource.Token + ); + using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( + engineDir, + tokenizer, + targetCorpus, + _cancellationTokenSource.Token + ); + await smtModelTrainer.TrainAsync(null, _cancellationTokenSource.Token); + await truecaseTrainer.TrainAsync(cancellationToken: _cancellationTokenSource.Token); + + await smtModelTrainer.SaveAsync(_cancellationTokenSource.Token); + await truecaseTrainer.SaveAsync(_cancellationTokenSource.Token); + + await using Stream engineStream = await SharedFileService.OpenWriteAsync( + $"builds/{BuildId1}/model.tar.gz", + _cancellationTokenSource.Token + ); + + await using Stream targetStream = await SharedFileService.OpenWriteAsync( + $"builds/{BuildId1}/pretranslate.trg.json", + _cancellationTokenSource.Token + ); + + await BuildJobService.StartBuildJobAsync( + BuildJobRunnerType.Hangfire, + EngineId1, + BuildId1, + BuildStage.Postprocess, + data: (0, 0.0) + ); + } + catch (OperationCanceledException) + { + await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); + } + } + + private class EnvActivator(TestEnvironment env) : JobActivator + { + private readonly TestEnvironment _env = env; + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(PreprocessBuildJob)) + { + return new PreprocessBuildJob( + _env.PlatformService, + _env.Engines, + _env._lockFactory, + new MemoryDataAccessContext(), + Substitute.For>(), + _env.BuildJobService, + _env.SharedFileService, + Substitute.For() + ) + { + TrainJobRunnerType = _env._trainJobRunnerType + }; + } + if (jobType == typeof(SmtTransferPostprocessBuildJob)) + { + var options = Substitute.For>(); + options.CurrentValue.Returns(new SmtTransferEngineOptions()); + return new SmtTransferPostprocessBuildJob( + _env.PlatformService, + _env.Engines, + _env._lockFactory, + new MemoryDataAccessContext(), + _env.BuildJobService, + Substitute.For>(), + _env.SharedFileService, + _env.TrainSegmentPairs, + _env.SmtModelFactory, + _env._truecaserFactory, + options + ); + } + if (jobType == typeof(SmtTransferTrainBuildJob)) + { + return new SmtTransferTrainBuildJob( + _env.PlatformService, + _env.Engines, + _env._lockFactory, + new MemoryDataAccessContext(), + _env.BuildJobService, + Substitute.For>(), + _env.SharedFileService, + _env._truecaserFactory, + _env.SmtModelFactory, + _env.TransferEngineFactory + ); + } + return base.ActivateJob(jobType); + } + } + } +} diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM new file mode 100644 index 00000000..b8665290 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/04LEVTe1.SFM @@ -0,0 +1,8 @@ +\id LEV - Test +\h Leviticus +\mt Leviticus +\c 14 +\p +\v 55 Source one, chapter fourteen, verse fifty-five. +\v 55b Segment b. +\v 56 Source one, chapter fourteen, verse fifty-six. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM new file mode 100644 index 00000000..4eb8b5fd --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/131CHTe1.SFM @@ -0,0 +1,12 @@ +\id 1CH - Test +\h 1 Chronicles +\mt 1 Chronicles +\c 12 +\p +\v 1 Source one, chapter twelve, verse one. +\v 2 Source one, chapter twelve, verse two. +\v 3-7 Source one, chapter twelve, verses three through seven. +\v 8 Source one, chapter twelve, verse eight. +\c 13 +\p +\v 1 Source one, chapter thirteen, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM new file mode 100644 index 00000000..ccf166e2 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM @@ -0,0 +1,16 @@ +\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew +\c 1 +\p +\v 1 Source one, chapter one, verse one. +\v 2-3 Source one, chapter one, verse two and three. +\v 4 Source one, chapter one, verse four. +\v 5 Source one, chapter one, verse five. +\v 6 Source one, chapter one, verse six. +\v 7-9 Source one, chapter one, verse seven, eight, and nine. +\v 10 Source one, chapter one, verse ten. +\c 2 +\p +\v 1 Source one, chapter two, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM new file mode 100644 index 00000000..ff8aaf6e --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/42MRKTe1.SFM @@ -0,0 +1,4 @@ +\id MRK - Test +\h Mark +\mt Mark +\ip An introduction to Mark diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml new file mode 100644 index 00000000..c80caedf --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/Settings.xml @@ -0,0 +1,34 @@ + + usfm.sty + 4 + en::: + English + 8.0.100.76 + Test1 + 65001 + T + + NFC + Te1 + a7e0b3ce0200736062f9f810a444dbfbe64aca35 + Charis SIL + 12 + + + + 41MAT + + Tes.SFM + Major::BiblicalTerms.xml + F + F + F + Public + Standard:: + + 3 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 + + + \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml new file mode 100644 index 00000000..03e45020 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/TermRenderings.xml @@ -0,0 +1,9 @@ + + + Abraham + + + + + + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs new file mode 100644 index 00000000..9c1cd387 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/custom.vrs @@ -0,0 +1,31 @@ +# custom.vrs + +LEV 14:56 +ROM 14:26 +REV 12:17 +TOB 5:22 +TOB 10:12 +SIR 23:28 +ESG 1:22 +ESG 3:15 +ESG 5:14 +ESG 8:17 +ESG 10:14 +SIR 33:33 +SIR 41:24 +BAR 1:22 +4MA 7:25 +4MA 12:20 + +# deliberately missing verses +-ROM 16:26 +-ROM 16:27 +-3JN 1:15 +-S3Y 1:49 +-ESG 4:6 +-ESG 9:5 +-ESG 9:30 + +LEV 14:55 = LEV 14:55 +LEV 14:55 = LEV 14:56 +LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM new file mode 100644 index 00000000..a0edc38b --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/04LEVTe3.SFM @@ -0,0 +1,7 @@ +\id LEV - Test +\h Leviticus +\mt Leviticus +\c 14 +\p +\v 55 Source two, chapter fourteen, verse fifty-five. +\v 56 Source two, chapter fourteen, verse fifty-six. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM new file mode 100644 index 00000000..05cdff2c --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/131CHTe3.SFM @@ -0,0 +1,16 @@ +\id 1CH - Test +\h 1 Chronicles +\mt 1 Chronicles +\c 12 +\p +\v 1 Source two, chapter twelve, verse one. +\v 2 Source two, chapter twelve, verse two. +\v 3 Source two, chapter twelve, verse three. +\v 4 Source two, chapter twelve, verse four. +\v 5 Source two, chapter twelve, verse five. +\v 6 Source two, chapter twelve, verse six. +\v 7 Source two, chapter twelve, verse seven. +\v 8 Source two, chapter twelve, verse eight. +\c 13 +\p +\v 1 Source two, chapter thirteen, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM new file mode 100644 index 00000000..7208a72d --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/41MATTe3.SFM @@ -0,0 +1,19 @@ +\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew +\c 1 +\p +\v 1 Source two, chapter one, verse one. +\v 2 Source two, chapter one, verse two. +\v 3 Source two, chapter one, verse three. +\v 4 Source two, chapter one, verse four. +\v 5 Source two, chapter one, verse five. +\v 6 Source two, chapter one, verse six. +\v 7 Source two, chapter one, verse seven. +\v 8 Source two, chapter one, verse eight. +\v 9 Source two, chapter one, verse nine. +\v 10 Source two, chapter one, verse ten. +\c 2 +\p +\v 1 Source two, chapter two, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM new file mode 100644 index 00000000..22380983 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/42MRKTe3.SFM @@ -0,0 +1,7 @@ +\id MRK - Test +\h Mark +\mt Mark +\ip An introduction to Mark +\c 1 +\p +\v 1 Source two, chapter one, verse one. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml new file mode 100644 index 00000000..affce0ec --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/Settings.xml @@ -0,0 +1,34 @@ + + usfm.sty + 4 + en::: + English + 8.0.100.76 + Test3 + 65001 + T + + NFC + Te3 + a7e0b3ce0200736062f9f810a444dbfbe64aca35 + Charis SIL + 12 + + + + 41MAT + + Tes.SFM + Major::BiblicalTerms.xml + F + F + F + Public + Standard:: + + 3 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 + + + \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml new file mode 100644 index 00000000..03e45020 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/TermRenderings.xml @@ -0,0 +1,9 @@ + + + Abraham + + + + + + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs new file mode 100644 index 00000000..9c1cd387 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source2/custom.vrs @@ -0,0 +1,31 @@ +# custom.vrs + +LEV 14:56 +ROM 14:26 +REV 12:17 +TOB 5:22 +TOB 10:12 +SIR 23:28 +ESG 1:22 +ESG 3:15 +ESG 5:14 +ESG 8:17 +ESG 10:14 +SIR 33:33 +SIR 41:24 +BAR 1:22 +4MA 7:25 +4MA 12:20 + +# deliberately missing verses +-ROM 16:26 +-ROM 16:27 +-3JN 1:15 +-S3Y 1:49 +-ESG 4:6 +-ESG 9:5 +-ESG 9:30 + +LEV 14:55 = LEV 14:55 +LEV 14:55 = LEV 14:56 +LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM new file mode 100644 index 00000000..69f46250 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM @@ -0,0 +1,17 @@ +\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew +\c 1 +\p +\v 1 Target one, chapter one, verse one. +\v 2 Target one, chapter one, verse two. +\v 3 Target one, chapter one, verse three. +\v 4 +\v 5-6 Target one, chapter one, verse five and six. +\v 7-8 Target one, chapter one, verse seven and eight. +\v 9-10 Target one, chapter one, verse nine and ten. +\c 2 +\p +\v 1 Target one, chapter two, verse one. +\v 2 Target one, chapter two, verse two. diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM new file mode 100644 index 00000000..46000963 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/42MRKTe2.SFM @@ -0,0 +1,4 @@ +\id MRK - Test +\h Mark +\mt Mark +\ip An introduction to Mark \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml new file mode 100644 index 00000000..37d2772a --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/Settings.xml @@ -0,0 +1,33 @@ + + usfm.sty + 4 + en::: + English + 8.0.100.76 + Test2 + 65001 + T + + NFC + Te2 + a7e0b3ce0200736062f9f810a444dbfbe64aca35 + Charis SIL + 12 + + + + 41MAT + + Ten.SFM + F + F + F + Public + Standard:: + + 3 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 + + + \ No newline at end of file diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml new file mode 100644 index 00000000..03e45020 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/TermRenderings.xml @@ -0,0 +1,9 @@ + + + Abraham + + + + + + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs new file mode 100644 index 00000000..9c1cd387 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/custom.vrs @@ -0,0 +1,31 @@ +# custom.vrs + +LEV 14:56 +ROM 14:26 +REV 12:17 +TOB 5:22 +TOB 10:12 +SIR 23:28 +ESG 1:22 +ESG 3:15 +ESG 5:14 +ESG 8:17 +ESG 10:14 +SIR 33:33 +SIR 41:24 +BAR 1:22 +4MA 7:25 +4MA 12:20 + +# deliberately missing verses +-ROM 16:26 +-ROM 16:27 +-3JN 1:15 +-S3Y 1:49 +-ESG 4:6 +-ESG 9:5 +-ESG 9:30 + +LEV 14:55 = LEV 14:55 +LEV 14:55 = LEV 14:56 +LEV 14:56 = LEV 14:57 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt new file mode 100644 index 00000000..2aeb971c --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source1.txt @@ -0,0 +1,7 @@ +Source one, Line 1 +Source one, Line 2 + +Source one, Line 4 + +Source one, Line 6 + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt new file mode 100644 index 00000000..7f4a0669 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/source2.txt @@ -0,0 +1,7 @@ +Source two, Line 1 +Source two, Line 2 + +Source two, Line 4 +Source two, Line 5 +Source two, Line 6 + diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt new file mode 100644 index 00000000..816e9435 --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/target1.txt @@ -0,0 +1,7 @@ +Target one, Line 1 + + +Target one, Line 4 + + +Target one, Line 7 diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs new file mode 100644 index 00000000..4115517a --- /dev/null +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Usings.cs @@ -0,0 +1,28 @@ +global using System.IO.Compression; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Grpc.Core; +global using Grpc.Core.Testing; +global using Hangfire; +global using Hangfire.Storage; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Hosting.Internal; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using NSubstitute; +global using NSubstitute.ClearExtensions; +global using NSubstitute.ExceptionExtensions; +global using NSubstitute.ReceivedExtensions; +global using NUnit.Framework; +global using RichardSzalay.MockHttp; +global using Serval.Machine.Shared.Configuration; +global using Serval.Machine.Shared.Models; +global using SIL.DataAccess; +global using SIL.Machine.Annotations; +global using SIL.Machine.Corpora; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems;