Skip to content

Commit

Permalink
Create parent class and initializable interface
Browse files Browse the repository at this point in the history
  • Loading branch information
Enkidu93 committed Oct 16, 2024
1 parent 39de60f commit 8cd99a4
Show file tree
Hide file tree
Showing 10 changed files with 98 additions and 91 deletions.
7 changes: 7 additions & 0 deletions src/Serval/src/Serval.Shared/Models/IInitializableEntity.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace Serval.Shared.Models;

public interface IInitializableEntity : IEntity
{
bool? IsInitialized { get; set; }
DateTime? DateCreated { get; set; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -1297,7 +1297,7 @@ private Engine Map(TranslationEngineConfigDto source)
Owner = Owner,
Corpora = [],
IsModelPersisted = source.IsModelPersisted,
SuccessfullyCreated = false
IsInitialized = false
};
}

Expand All @@ -1310,7 +1310,7 @@ private static Build Map(Engine engine, TranslationBuildConfigDto source)
Pretranslate = Map(engine, source.Pretranslate),
TrainOn = Map(engine, source.TrainOn),
Options = Map(source.Options),
SuccessfullyStarted = false
IsInitialized = false
};
}

Expand Down
5 changes: 3 additions & 2 deletions src/Serval/src/Serval.Translation/Models/Build.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace Serval.Translation.Models;

public record Build : IEntity
public record Build : IInitializableEntity
{
public string Id { get; set; } = "";
public int Revision { get; set; } = 1;
Expand All @@ -15,5 +15,6 @@ public record Build : IEntity
public JobState State { get; init; } = JobState.Pending;
public DateTime? DateFinished { get; init; }
public IReadOnlyDictionary<string, object>? Options { get; init; }
public bool SuccessfullyStarted { get; init; }
public bool? IsInitialized { get; set; }
public DateTime? DateCreated { get; set; }
}
5 changes: 3 additions & 2 deletions src/Serval/src/Serval.Translation/Models/Engine.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace Serval.Translation.Models;

public record Engine : IOwnedEntity
public record Engine : IOwnedEntity, IInitializableEntity
{
public string Id { get; set; } = "";
public int Revision { get; set; } = 1;
Expand All @@ -16,5 +16,6 @@ public record Engine : IOwnedEntity
public int ModelRevision { get; init; }
public double Confidence { get; init; }
public int CorpusSize { get; init; }
public bool SuccessfullyCreated { get; init; }
public bool? IsInitialized { get; set; }
public DateTime? DateCreated { get; set; }
}
Original file line number Diff line number Diff line change
@@ -1,42 +1,7 @@
using Microsoft.Extensions.DependencyInjection;
using SIL.ServiceToolkit.Services;

namespace Serval.Translation.Services;

public class BuildCleanupService(
IServiceProvider services,
ILogger<BuildCleanupService> logger,
TimeSpan? timeout = null
) : RecurrentTask("Build Cleanup Service", services, RefreshPeriod, logger)
{
private readonly ILogger<BuildCleanupService> _logger = logger;
private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);

protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
{
_logger.LogInformation("Running build cleanup job");
var builds = scope.ServiceProvider.GetRequiredService<IRepository<Build>>();
await CheckBuildsAsync(builds, cancellationToken);
}

public async Task CheckBuildsAsync(IRepository<Build> builds, CancellationToken cancellationToken)
{
IReadOnlyList<Build> allBuilds = await builds.GetAllAsync(cancellationToken);
IEnumerable<Build> notStartedBuilds = allBuilds.Where(b => !b.SuccessfullyStarted);
await Task.Delay(_timeout, cancellationToken); //Make sure the builds are not midway through starting
foreach (
Build build in await builds.GetAllAsync(
b => notStartedBuilds.Select(c => c.Id).Contains(b.Id),
cancellationToken
)
)
{
if (!build.SuccessfullyStarted)
{
_logger.LogInformation("Deleting build {id} because it was never successfully started", build.Id);
await builds.DeleteAsync(build, cancellationToken);
}
}
}
}
) : UninitializedCleanupService<Build>(services, logger, timeout) { }
36 changes: 9 additions & 27 deletions src/Serval/src/Serval.Translation/Services/EngineCleanupService.cs
Original file line number Diff line number Diff line change
@@ -1,47 +1,29 @@
using Microsoft.Extensions.DependencyInjection;
using SIL.ServiceToolkit.Services;

namespace Serval.Translation.Services;

public class EngineCleanupService(
IServiceProvider services,
ILogger<EngineCleanupService> logger,
TimeSpan? timeout = null
) : RecurrentTask("Engine Cleanup Service", services, RefreshPeriod, logger)
) : UninitializedCleanupService<Engine>(services, logger, timeout)
{
private readonly ILogger<EngineCleanupService> _logger = logger;
private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);
public EngineService? EngineService { get; set; }

protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
{
_logger.LogInformation("Running engine cleanup job");
var engines = scope.ServiceProvider.GetRequiredService<IRepository<Engine>>();
var engineService = scope.ServiceProvider.GetRequiredService<EngineService>();
await CheckEnginesAsync(engines, engineService, cancellationToken);
EngineService = scope.ServiceProvider.GetRequiredService<EngineService>();
await base.DoWorkAsync(scope, cancellationToken);
}

public async Task CheckEnginesAsync(
protected override async Task DeleteEntityAsync(
IRepository<Engine> engines,
EngineService engineService,
Engine engine,
CancellationToken cancellationToken
)
{
IReadOnlyList<Engine> allEngines = await engines.GetAllAsync(cancellationToken);
IEnumerable<Engine> notCreatedEngines = allEngines.Where(e => !e.SuccessfullyCreated);
await Task.Delay(_timeout, cancellationToken); //Make sure the engines are not midway through being created
foreach (
Engine engine in await engines.GetAllAsync(
e => notCreatedEngines.Select(f => f.Id).Contains(e.Id),
cancellationToken
)
)
{
if (!engine.SuccessfullyCreated)
{
_logger.LogInformation("Deleting engine {id} because it was never successfully created", engine.Id);
await engineService.DeleteAsync(engine.Id, cancellationToken);
}
}
if (EngineService == null)
return;
await EngineService.DeleteAsync(engine.Id, cancellationToken);
}
}
23 changes: 10 additions & 13 deletions src/Serval/src/Serval.Translation/Services/EngineService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ await client.TrainSegmentPairAsync(

public override async Task<Engine> CreateAsync(Engine engine, CancellationToken cancellationToken = default)
{
bool updateIsModelPersisted = engine.IsModelPersisted is null;
try
{
engine.DateCreated = DateTime.UtcNow;
await Entities.InsertAsync(engine, cancellationToken);
TranslationEngineApi.TranslationEngineApiClient? client =
_grpcClientFactory.CreateClient<TranslationEngineApi.TranslationEngineApiClient>(engine.Type);
Expand All @@ -148,7 +148,11 @@ public override async Task<Engine> CreateAsync(Engine engine, CancellationToken
};
await Entities.UpdateAsync(
engine,
u => u.Set(e => e.SuccessfullyCreated, true),
u =>
{
u.Set(e => e.IsInitialized, true);
u.Set(e => e.IsModelPersisted, engine.IsModelPersisted);
},
cancellationToken: cancellationToken
);
}
Expand All @@ -169,14 +173,6 @@ await Entities.UpdateAsync(
await Entities.DeleteAsync(engine, CancellationToken.None);
throw;
}
if (updateIsModelPersisted)
{
await Entities.UpdateAsync(
engine,
u => u.Set(e => e.IsModelPersisted, engine.IsModelPersisted),
cancellationToken: cancellationToken
);
}
return engine;
}

Expand Down Expand Up @@ -221,6 +217,7 @@ private Dictionary<string, List<int>> GetChapters(string fileLocation, string sc

public async Task StartBuildAsync(Build build, CancellationToken cancellationToken = default)
{
build.DateCreated = DateTime.UtcNow;
Engine engine = await GetAsync(build.EngineRef, cancellationToken);
await _builds.InsertAsync(build, cancellationToken);

Expand Down Expand Up @@ -298,9 +295,9 @@ public async Task StartBuildAsync(Build build, CancellationToken cancellationTok
}
await client.StartBuildAsync(request, cancellationToken: cancellationToken);
await _builds.UpdateAsync(
build.Id,
u => u.Set(e => e.SuccessfullyStarted, true),
cancellationToken: cancellationToken
b => b.Id == build.Id,
u => u.Set(e => e.IsInitialized, true),
cancellationToken: CancellationToken.None
);
}
catch
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
using Microsoft.Extensions.DependencyInjection;
using SIL.ServiceToolkit.Services;

namespace Serval.Translation.Services;

public abstract class UninitializedCleanupService<T>(
IServiceProvider services,
ILogger<UninitializedCleanupService<T>> logger,
TimeSpan? timeout = null
) : RecurrentTask($"{typeof(T)} Cleanup Service", services, RefreshPeriod, logger)
where T : IInitializableEntity
{
private readonly ILogger<UninitializedCleanupService<T>> _logger = logger;
private readonly TimeSpan _timeout = timeout ?? TimeSpan.FromMinutes(2);
private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1);

protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken)
{
_logger.LogInformation("Running build cleanup job");
var entities = scope.ServiceProvider.GetRequiredService<IRepository<T>>();
await CheckEntitiesAsync(entities, cancellationToken);
}

public async Task CheckEntitiesAsync(IRepository<T> entities, CancellationToken cancellationToken)
{
IReadOnlyList<T> allEntities = await entities.GetAllAsync(cancellationToken);
var now = DateTime.UtcNow;
IEnumerable<T> uninitializedEntities = allEntities.Where(b =>
!(b.IsInitialized ?? true) && (now - (b.DateCreated ?? DateTime.UtcNow)) > _timeout
);
foreach (T entity in uninitializedEntities)
{
_logger.LogInformation(
"Deleting {type} {id} because it was never successfully started",
typeof(T),
entity.Id
);
await DeleteEntityAsync(entities, entity, cancellationToken);
}
}

protected virtual async Task DeleteEntityAsync(
IRepository<T> entities,
T entity,
CancellationToken cancellationToken
)
{
await entities.DeleteAsync(e => e.Id == entity.Id, cancellationToken);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,17 @@ public TestEnvironment()
{
Id = "build1",
EngineRef = "engine1",
SuccessfullyStarted = false
IsInitialized = false,
DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
}
);
Builds.Add(
new Build
{
Id = "build2",
EngineRef = "engine2",
SuccessfullyStarted = true
IsInitialized = true,
DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
}
);

Expand All @@ -48,7 +50,7 @@ public TestEnvironment()

public async Task CheckBuildsAsync()
{
await Service.CheckBuildsAsync(Builds, CancellationToken.None);
await Service.CheckEntitiesAsync(Builds, CancellationToken.None);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public TestEnvironment()
TargetLanguage = "es",
Type = "Nmt",
Owner = "client1",
SuccessfullyCreated = false
IsInitialized = false,
DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
}
);
Engines.Add(
Expand All @@ -43,7 +44,8 @@ public TestEnvironment()
TargetLanguage = "es",
Type = "Nmt",
Owner = "client1",
SuccessfullyCreated = true
IsInitialized = true,
DateCreated = DateTime.UtcNow.Subtract(TimeSpan.FromHours(10))
}
);

Expand All @@ -61,7 +63,7 @@ public TestEnvironment()
.CreateClient<TranslationEngineApi.TranslationEngineApiClient>("Nmt")
.Returns(translationServiceClient);

_engineService = new EngineService(
var engineService = new EngineService(
Engines,
new MemoryRepository<Build>(),
new MemoryRepository<Pretranslation>(),
Expand All @@ -72,15 +74,15 @@ public TestEnvironment()
new LoggerFactory(),
Substitute.For<IScriptureDataFileService>()
);

Service.EngineService = engineService;
}

public EngineCleanupService Service { get; }

private readonly EngineService _engineService;

public async Task CheckEnginesAsync()
{
await Service.CheckEnginesAsync(Engines, _engineService, CancellationToken.None);
await Service.CheckEntitiesAsync(Engines, CancellationToken.None);
}

private static AsyncUnaryCall<TResponse> CreateAsyncUnaryCall<TResponse>(TResponse response)
Expand Down

0 comments on commit 8cd99a4

Please sign in to comment.