-
Notifications
You must be signed in to change notification settings - Fork 16
/
ClassicContentAnalyzer.cs
71 lines (61 loc) · 2.44 KB
/
ClassicContentAnalyzer.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
using OpenTextSummarizer.Interfaces;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace OpenTextSummarizer
{
/// <summary>
/// Analyzer class to determine important text units and score sentences
/// </summary>
internal class ClassicContentAnalyzer : IContentAnalyzer
{
public Dictionary m_Rules { get; set; }
public ClassicContentAnalyzer(Dictionary Rules)
{
m_Rules = Rules;
}
public List<TextUnitScore> GetImportantTextUnits(List<Sentence> Sentences)
{
var textUnitFrequencyGrader = new Dictionary<TextUnit, long>();
foreach (var tu in Sentences.SelectMany(s => s.TextUnits))
{
if (m_Rules.UnimportantWords.Contains(tu.FormattedValue))
{
continue;
}
if (textUnitFrequencyGrader.ContainsKey(tu))
{
textUnitFrequencyGrader[tu]++;
}
else
{
textUnitFrequencyGrader.Add(tu, 1);
}
}
return textUnitFrequencyGrader.OrderByDescending(kvp => kvp.Value).Select(kvp => new TextUnitScore() { ScoredTextUnit = kvp.Key, Score = kvp.Value }).ToList();
}
public List<SentenceScore> ScoreSentences(List<Sentence> Sentences, List<TextUnitScore> importantTextUnits)
{
var stemList = importantTextUnits.Select(tus => tus.ScoredTextUnit.Stem).Distinct().ToList();
var listSentenceScorer = new List<SentenceScore>();
foreach (var s in Sentences.Where(s => s.TextUnits.Count > 2))
{
var newSentenceScorer = new SentenceScore();
newSentenceScorer.ScoredSentence = s;
newSentenceScorer.Score = newSentenceScorer.ScoredSentence.TextUnits.Count(tu => stemList.Contains(tu.Stem));
if (s.TextUnits[0].RawValue.Contains("\n") && s.TextUnits[1].RawValue.Contains("\n"))
{
newSentenceScorer.Score *= 1.6;
}
listSentenceScorer.Add(newSentenceScorer);
}
// additional scoring
if (listSentenceScorer.Any())
{
listSentenceScorer.First().Score *= 2;
}
return listSentenceScorer.OrderByDescending(ss => ss.Score).ToList();
}
}
}