-
Notifications
You must be signed in to change notification settings - Fork 73
/
Program.cs
74 lines (60 loc) · 3.19 KB
/
Program.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
using Catalyst;
using Catalyst.Models;
using Mosaik.Core;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Linq;
using Version = Mosaik.Core.Version;
using P = Catalyst.PatternUnitPrototype;
using System.Text;
using Microsoft.Extensions.Logging;
namespace Catalyst.Samples.LanguageDetection
{
public static class Program
{
public static async Task Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
ApplicationLogging.SetLoggerFactory(LoggerFactory.Create(lb => lb.AddConsole()));
//Need to register the languages we want to use first
Catalyst.Models.English.Register();
//This example shows the two language detection models available on Catalyst.
//The first is derived from the Chrome former language detection code Compact Language Detector 2 (https://github.com/CLD2Owners/cld2)
//and the newer model is derived from Facebook's FastText language detection dataset (see: https://fasttext.cc/blog/2017/10/02/blog-post.html)
//Configures the model storage to use the local folder ./catalyst-models/
Storage.Current = new DiskStorage("catalyst-models");
var cld2LanguageDetector = await LanguageDetector.FromStoreAsync(Language.Any, Version.Latest, "");
var fastTextLanguageDetector = await FastTextLanguageDetector.FromStoreAsync(Language.Any, Version.Latest, "");
//We show bellow the detection on short and longer samples. You can expect lower precision on shorter texts, as there is less information for the model to work with
//It's also interesting to see the kind of mistakes these models make, such as detecting Welsh as Gaelic_Scottish_Gaelic
foreach (var (lang, text) in Data.ShortSamples)
{
var doc = new Document(text);
fastTextLanguageDetector.Process(doc);
var doc2 = new Document(text);
cld2LanguageDetector.Process(doc2);
Console.WriteLine(text);
Console.WriteLine($"Actual:\t{lang}\nFT:\t{doc.Language}\nCLD2\t{doc2.Language}");
Console.WriteLine();
}
foreach (var (lang, text) in Data.LongSamples)
{
var doc = new Document(text);
fastTextLanguageDetector.Process(doc);
var doc2 = new Document(text);
cld2LanguageDetector.Process(doc2);
Console.WriteLine(text);
Console.WriteLine($"Actual:\t{lang}\nFT:\t{doc.Language}\nCLD2\t{doc2.Language}");
Console.WriteLine();
}
// You can also access all predictions via the Predict method:
var allPredictions = fastTextLanguageDetector.Predict(new Document(Data.LongSamples[Language.Spanish]));
Console.WriteLine($"\n\nTop 10 predictions and scores for the Spanish sample:");
foreach (var kv in allPredictions.OrderByDescending(kv => kv.Value).Take(10))
{
Console.WriteLine($"{kv.Key.ToString().PadRight(40)}\tScore: {kv.Value:n2}");
}
}
}
}