Move Machine engine to Serval repo

.github/workflows/ci.yml

@@ -11,9 +11,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup .NET
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: 8.0.x
       - name: Start MongoDB

Serval.sln

@@ -64,6 +64,20 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{92805246-528
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{BA044B98-3136-4FDE-B90F-B0975758C07F}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Machine", "Machine", "{F6142E52-4B58-4D12-980F-B07D8AA932C2}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared", "src\Machine\src\Serval.Machine.Shared\Serval.Machine.Shared.csproj", "{090ECB69-464F-42C8-B92C-0808BE2802FA}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.EngineServer", "src\Machine\src\Serval.Machine.EngineServer\Serval.Machine.EngineServer.csproj", "{C02494FB-663E-4430-9F2D-41F1A740B271}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.JobServer", "src\Machine\src\Serval.Machine.JobServer\Serval.Machine.JobServer.csproj", "{BC766753-E560-4ADF-9923-C7A96076EA47}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serval.Machine.Shared.Tests", "src\Machine\test\Serval.Machine.Shared.Tests\Serval.Machine.Shared.Tests.csproj", "{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -134,6 +148,22 @@ Global
 		{0E220C65-AA88-450E-AFB2-844E49060B3F}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0E220C65-AA88-450E-AFB2-844E49060B3F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{090ECB69-464F-42C8-B92C-0808BE2802FA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{090ECB69-464F-42C8-B92C-0808BE2802FA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C02494FB-663E-4430-9F2D-41F1A740B271}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C02494FB-663E-4430-9F2D-41F1A740B271}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC766753-E560-4ADF-9923-C7A96076EA47}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC766753-E560-4ADF-9923-C7A96076EA47}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -160,6 +190,12 @@ Global
 		{3E753B99-7C31-42AC-B02E-012B802F58DB} = {6D20F76D-9A0E-44AC-8754-B4291C75D25B}
 		{92805246-5285-4F0A-9BF8-6EE4A027A41B} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF}
 		{BA044B98-3136-4FDE-B90F-B0975758C07F} = {33E6965E-5A58-4C6F-882E-F17C8E88A3FF}
+		{D808D2BE-ED26-4E60-A409-AE58F7C1CB8F} = {F6142E52-4B58-4D12-980F-B07D8AA932C2}
+		{40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB} = {F6142E52-4B58-4D12-980F-B07D8AA932C2}
+		{090ECB69-464F-42C8-B92C-0808BE2802FA} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}
+		{C02494FB-663E-4430-9F2D-41F1A740B271} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}
+		{BC766753-E560-4ADF-9923-C7A96076EA47} = {D808D2BE-ED26-4E60-A409-AE58F7C1CB8F}
+		{B0D23A55-AB09-4C2C-B309-F4BEB3BC968D} = {40C225C2-1EEF-4D1D-9D14-1CBB86C8A1CB}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370}

src/Machine/src/Serval.Machine.EngineServer/Program.cs

@@ -0,0 +1,39 @@
+using Hangfire;
+using OpenTelemetry.Trace;
+
+var builder = WebApplication.CreateBuilder(args);
+
+// Add services to the container.
+builder
+    .Services.AddMachine(builder.Configuration)
+    .AddBuildJobService()
+    .AddMongoDataAccess()
+    .AddMongoHangfireJobClient()
+    .AddServalTranslationEngineService()
+    .AddModelCleanupService()
+    .AddMessageOutboxDeliveryService()
+    .AddClearMLService();
+
+if (builder.Environment.IsDevelopment())
+{
+    builder
+        .Services.AddOpenTelemetry()
+        .WithTracing(builder =>
+        {
+            builder
+                .AddAspNetCoreInstrumentation()
+                .AddHttpClientInstrumentation()
+                .AddGrpcClientInstrumentation()
+                .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources")
+                .AddConsoleExporter();
+        });
+}
+
+var app = builder.Build();
+
+app.UseHttpsRedirection();
+
+app.MapServalTranslationEngineService();
+app.MapHangfireDashboard();
+
+app.Run();

src/Machine/src/Serval.Machine.EngineServer/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "profiles": {
+    "SIL.Machine.Serval.EngineServer": {
+      "commandName": "Project",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "https://localhost:9000"
+    }
+  }
+}

src/Machine/src/Serval.Machine.EngineServer/Serval.Machine.EngineServer.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+	<PropertyGroup>
+		<TargetFramework>net8.0</TargetFramework>
+		<Nullable>enable</Nullable>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<UserSecretsId>34e222a9-ef76-48f9-869e-338547f9bd25</UserSecretsId>
+		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+		<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+		<GenerateDocumentationFile>true</GenerateDocumentationFile>
+		<NoWarn>$(NoWarn);CS1591;CS1573</NoWarn>
+	</PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="icu.net" Version="2.9.0" />
+		<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.0" />
+		<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.0" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.8.0-beta.1" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\Serval.Machine.Shared\Serval.Machine.Shared.csproj" />
+	</ItemGroup>
+
+	<!-- Include icu.net.dll.config - which is only available after the package is built -->
+	<ItemGroup>
+		<ResolvedFileToPublish Include=".\bin\Release\net8.0\icu.net.dll.config">
+			<RelativePath>icu.net.dll.config</RelativePath>
+		</ResolvedFileToPublish>
+	</ItemGroup>
+
+</Project>

src/Machine/src/Serval.Machine.EngineServer/appsettings.Development.json

@@ -0,0 +1,21 @@
+{
+  "ConnectionStrings": {
+    "Hangfire": "mongodb://localhost:27017/machine_jobs",
+    "Mongo": "mongodb://localhost:27017/machine",
+    "Serval": "https://localhost:8444"
+  },
+  "ClearML": {
+    "MaxSteps": 1000,
+    "Project": "dev"
+  },
+  "SharedFile": {
+    "Uri": "s3://aqua-ml-data/dev/"
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning",
+      "System.Net.Http.HttpClient.Default": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.EngineServer/appsettings.Production.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.EngineServer/appsettings.Staging.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.EngineServer/appsettings.json

@@ -0,0 +1,43 @@
+{
+  "ConnectionStrings": {
+    "ClearML": "https://api.sil.hosted.allegro.ai"
+  },
+  "AllowedHosts": "*",
+  "Service": {
+    "ServiceId": "machine_engine"
+  },
+  "TranslationEngines": [
+    "SmtTransfer",
+    "Nmt"
+  ],
+  "BuildJob": {
+    "ClearML": [
+      {
+        "TranslationEngineType": "Nmt",
+        "ModelType": "huggingface",
+        "Queue": "jobs_backlog",
+        "DockerImage": "ghcr.io/sillsdev/machine.py:latest"
+      },
+      {
+        "TranslationEngineType": "SmtTransfer",
+        "ModelType": "thot",
+        "Queue": "cpu_only",
+        "DockerImage": "ghcr.io/sillsdev/machine.py:latest"
+      }
+    ]
+  },
+  "SmtTransferEngine": {
+    "EnginesDir": "/var/lib/machine/engines"
+  },
+  "ClearML": {
+    "BuildPollingEnabled": true
+  },
+  "MessageOutbox": {
+    "OutboxDir": "/var/lib/machine/outbox"
+  },
+  "Logging": {
+    "LogLevel": {
+      "System.Net.Http.HttpClient.Default": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.JobServer/Program.cs

@@ -0,0 +1,30 @@
+using OpenTelemetry.Trace;
+
+var builder = WebApplication.CreateBuilder(args);
+
+builder
+    .Services.AddMachine(builder.Configuration)
+    .AddBuildJobService()
+    .AddMongoDataAccess()
+    .AddMongoHangfireJobClient()
+    .AddHangfireJobServer()
+    .AddServalPlatformService()
+    .AddClearMLService();
+if (builder.Environment.IsDevelopment())
+{
+    builder
+        .Services.AddOpenTelemetry()
+        .WithTracing(builder =>
+        {
+            builder
+                .AddAspNetCoreInstrumentation()
+                .AddHttpClientInstrumentation()
+                .AddGrpcClientInstrumentation()
+                .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources")
+                .AddConsoleExporter();
+        });
+}
+
+var app = builder.Build();
+
+app.Run();

src/Machine/src/Serval.Machine.JobServer/Properties/launchSettings.json

@@ -0,0 +1,12 @@
+{
+  "profiles": {
+    "SIL.Machine.Serval.JobServer": {
+      "commandName": "Project",
+      "launchBrowser": false,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      },
+      "applicationUrl": "https://localhost:9100"
+    }
+  }
+}

src/Machine/src/Serval.Machine.JobServer/Serval.Machine.JobServer.csproj

@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+	<PropertyGroup>
+		<TargetFramework>net8.0</TargetFramework>
+		<Nullable>enable</Nullable>
+		<ImplicitUsings>enable</ImplicitUsings>
+		<UserSecretsId>aa9e7440-5a04-4de6-ba51-bab9ef4a62e1</UserSecretsId>
+		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
+		<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+		<GenerateDocumentationFile>true</GenerateDocumentationFile>
+		<NoWarn>$(NoWarn);CS1591;CS1573</NoWarn>
+	</PropertyGroup>
+
+	<Target BeforeTargets="Build" Name="EnforceAppSettingsName">
+	</Target>
+
+	<ItemGroup>
+		<PackageReference Include="icu.net" Version="2.9.0" />
+		<PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.8.0" />
+		<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.0" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.8.0-beta.1" />
+		<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\Serval.Machine.Shared\Serval.Machine.Shared.csproj" />
+	</ItemGroup>
+
+	<!-- Include icu.net.dll.config - which is only available after the package is built -->
+	<ItemGroup>
+		<ResolvedFileToPublish Include=".\bin\Release\net8.0\icu.net.dll.config">
+			<RelativePath>icu.net.dll.config</RelativePath>
+		</ResolvedFileToPublish>
+	</ItemGroup>
+
+</Project>

src/Machine/src/Serval.Machine.JobServer/appsettings.Development.json

@@ -0,0 +1,21 @@
+{
+  "ConnectionStrings": {
+    "Hangfire": "mongodb://localhost:27017/machine_jobs",
+    "Mongo": "mongodb://localhost:27017/machine",
+    "Serval": "https://localhost:8444"
+  },
+  "ClearML": {
+    "MaxSteps": 1000,
+    "Project": "dev"
+  },
+  "SharedFile": {
+    "Uri": "s3://aqua-ml-data/dev/"
+  },
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning",
+      "System.Net.Http.HttpClient.Default": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.JobServer/appsettings.Production.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

src/Machine/src/Serval.Machine.JobServer/appsettings.Staging.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}