/
tokenize.go
93 lines (81 loc) · 2.72 KB
/
tokenize.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
package nlp
import (
"context"
language "cloud.google.com/go/language/apiv1"
"github.com/ndabAP/assocentity/v10/tokenize"
"google.golang.org/api/option"
languagepb "google.golang.org/genproto/googleapis/cloud/language/v1"
)
// Use map to be independent from library
var poSMapTypes = map[languagepb.PartOfSpeech_Tag]tokenize.PoS{
languagepb.PartOfSpeech_ADJ: tokenize.ADJ,
languagepb.PartOfSpeech_ADP: tokenize.ADP,
languagepb.PartOfSpeech_ADV: tokenize.ADV,
languagepb.PartOfSpeech_AFFIX: tokenize.AFFIX,
languagepb.PartOfSpeech_CONJ: tokenize.CONJ,
languagepb.PartOfSpeech_DET: tokenize.DET,
languagepb.PartOfSpeech_NOUN: tokenize.NOUN,
languagepb.PartOfSpeech_NUM: tokenize.NUM,
languagepb.PartOfSpeech_PRON: tokenize.PRON,
languagepb.PartOfSpeech_PRT: tokenize.PRT,
languagepb.PartOfSpeech_PUNCT: tokenize.PUNCT,
languagepb.PartOfSpeech_UNKNOWN: tokenize.UNKN,
languagepb.PartOfSpeech_VERB: tokenize.VERB,
languagepb.PartOfSpeech_X: tokenize.X,
}
// Lang defines the language used to examine the text. Both ISO and BCP-47
// language codes are accepted
type Lang string
// AutoLang tries to automatically recognize the language
var AutoLang Lang = "auto"
// NLPTokenizer tokenizes a text using Google NLP
type NLPTokenizer struct {
credsFilename string
lang Lang
}
// NewNLPTokenizer returns a new NLP tokenizer instance
func NewNLPTokenizer(credentialsFilename string, lang Lang) tokenize.Tokenizer {
return NLPTokenizer{
credsFilename: credentialsFilename,
lang: lang,
}
}
// Tokenize tokenizes a text
func (nlp NLPTokenizer) Tokenize(ctx context.Context, text string) ([]tokenize.Token, error) {
res, err := nlp.req(ctx, text)
if err != nil {
return []tokenize.Token{}, err
}
tokens := make([]tokenize.Token, 0)
for _, tok := range res.GetTokens() {
tokens = append(tokens, tokenize.Token{
PoS: poSMapTypes[tok.PartOfSpeech.Tag],
Text: tok.GetText().GetContent(),
})
}
return tokens, nil
}
// req sends a request to the Google server
func (nlp NLPTokenizer) req(ctx context.Context, text string) (*languagepb.AnnotateTextResponse, error) {
client, err := language.NewClient(ctx, option.WithCredentialsFile(nlp.credsFilename))
if err != nil {
return &languagepb.AnnotateTextResponse{}, err
}
doc := &languagepb.Document{
Source: &languagepb.Document_Content{
Content: text,
},
Type: languagepb.Document_PLAIN_TEXT,
}
// Set desired language if not auto
if nlp.lang != AutoLang {
doc.Language = string(nlp.lang)
}
return client.AnnotateText(ctx, &languagepb.AnnotateTextRequest{
Document: doc,
Features: &languagepb.AnnotateTextRequest_Features{
ExtractSyntax: true,
},
EncodingType: languagepb.EncodingType_UTF8,
})
}