-
-
Notifications
You must be signed in to change notification settings - Fork 14
/
TextFileText.cs
88 lines (83 loc) · 3.08 KB
/
TextFileText.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace SIL.Machine.Corpora
{
public class TextFileText : TextBase
{
public TextFileText(string id, string fileName)
: base(id, id)
{
FileName = fileName;
}
public string FileName { get; }
public override bool MissingRowsAllowed => false;
public override int Count(bool includeEmpty = true)
{
using (var reader = new StreamReader(FileName))
{
int count = 0;
string line;
while ((line = reader.ReadLine()) != null)
{
if (includeEmpty || line.Trim().Length > 0)
count++;
}
return count;
}
}
public override IEnumerable<TextRow> GetRows()
{
using (var reader = new StreamReader(FileName))
{
int lineNum = 1;
string line;
while ((line = reader.ReadLine()) != null)
{
string[] columns = line.Split('\t');
object rowRef;
var flags = TextRowFlags.SentenceStart;
if (columns.Length > 1)
{
string[] keys = columns[0].Trim().Split('_');
rowRef = new MultiKeyRef(Id, keys);
line = columns[1];
if (columns.Length == 3)
{
flags = TextRowFlags.None;
foreach (
string flagStr in columns[2]
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
.Select(f => f.Trim().ToLowerInvariant())
)
{
switch (flagStr)
{
case "sentence_start":
case "ss":
flags |= TextRowFlags.SentenceStart;
break;
case "in_range":
case "ir":
flags |= TextRowFlags.InRange;
break;
case "range_start":
case "rs":
flags |= TextRowFlags.InRange | TextRowFlags.RangeStart;
break;
}
}
}
}
else
{
rowRef = new MultiKeyRef(Id, lineNum);
}
yield return CreateRow(line, rowRef, flags);
lineNum++;
}
}
}
}
}