/
good_token.go
64 lines (56 loc) · 1.47 KB
/
good_token.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
// Copyright 2016 Mhd Sulhan <ms@kilabit.info>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package feature
import (
"github.com/shuLhan/tabula"
"github.com/shuLhan/tekstus"
)
// GoodToken count how many good token in inserted text.
type GoodToken Feature
var (
tokens = []string{
"=", "==", "===", "====", "=====",
":", "::", ":::", "::::", ":::::", "::::::",
"*", "**", "***", "****",
"#", "##", "###", "####",
";",
"''", "'''", "'''''",
"----",
"__FORCETOC__", "__TOC___", "__NOTOC__",
"<blockquote", "blockquote>",
"<div", "/div>",
"<code", "/code>",
"<syntaxhighlight", "/syntaxhighlight>",
"<small", "/small>",
"<big", "/big>",
"<pre", "/pre>",
"<nowiki", "/nowiki>",
"<sub", "/sub>",
"<sup", "/sup>",
"<math", "/math>",
"<ref", "/ref>",
"{{", "}}",
"[[", "]]",
"{{cite book", "{{cite web",
"{{Help:",
"~~~", "~~~~", "~~~~~",
"[[Special:", "[[media:", "[[Media:", "[[File:",
"[[Wikipedia", "[[Wiktionary:", "[[Category:",
"[http://",
"ISBN ", "#REDIRECT",
}
)
func init() {
Register(&GoodToken{}, tabula.TInteger, "good_token")
}
/*
Compute number of good token in inserted text.
*/
func (ftr *GoodToken) Compute(dataset tabula.DatasetInterface) {
col := dataset.GetColumnByName("additions")
for _, rec := range col.Records {
cnt := tekstus.StringCountTokens(rec.String(), tokens, false)
ftr.PushBack(&tabula.Record{V: int64(cnt)})
}
}