-
-
Notifications
You must be signed in to change notification settings - Fork 15
/
PostprocessBuildJob.cs
84 lines (73 loc) · 3.02 KB
/
PostprocessBuildJob.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
namespace SIL.Machine.AspNetCore.Services;
public abstract class PostprocessBuildJob(
IPlatformService platformService,
IRepository<TranslationEngine> engines,
IDistributedReaderWriterLockFactory lockFactory,
IBuildJobService buildJobService,
ILogger<PostprocessBuildJob> logger,
ISharedFileService sharedFileService
) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, buildJobService, logger)
{
protected readonly ISharedFileService SharedFileService = sharedFileService;
public abstract bool GetPretranslationEnabled();
protected override async Task DoWorkAsync(
string engineId,
string buildId,
(int, double) data,
string? buildOptions,
IDistributedReaderWriterLock @lock,
CancellationToken cancellationToken
)
{
(int corpusSize, double confidence) = data;
if (GetPretranslationEnabled())
// The NMT job has successfully completed, so insert the generated pretranslations into the database.
await InsertPretranslationsAsync(engineId, buildId, cancellationToken);
await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None))
{
await PlatformService.BuildCompletedAsync(
buildId,
corpusSize,
Math.Round(confidence, 2, MidpointRounding.AwayFromZero),
CancellationToken.None
);
await BuildJobService.BuildJobFinishedAsync(engineId, buildId, buildComplete: true, CancellationToken.None);
}
Logger.LogInformation("Build completed ({0}).", buildId);
}
protected override async Task CleanupAsync(
string engineId,
string buildId,
(int, double) data,
IDistributedReaderWriterLock @lock,
JobCompletionStatus completionStatus
)
{
if (completionStatus is JobCompletionStatus.Restarting)
return;
try
{
if (completionStatus is not JobCompletionStatus.Faulted)
await SharedFileService.DeleteAsync($"builds/{buildId}/");
}
catch (Exception e)
{
Logger.LogWarning(e, "Unable to to delete job data for build {0}.", buildId);
}
}
private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken)
{
await using var targetPretranslateStream = await SharedFileService.OpenReadAsync(
$"builds/{buildId}/pretranslate.trg.json",
cancellationToken
);
IAsyncEnumerable<Pretranslation> pretranslations = JsonSerializer
.DeserializeAsyncEnumerable<Pretranslation>(
targetPretranslateStream,
new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase },
cancellationToken
)
.OfType<Pretranslation>();
await PlatformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken);
}
}