Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cosme #200

Merged
merged 2 commits into from
Oct 6, 2020
Merged

Cosme #200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ddaaf6e
Rename sample dir
ikawaha Oct 6, 2020
14208ec
Cosme
ikawaha Oct 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions cmd/kagome/lattice/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ type option struct {
// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError // Call os.Exit(2).
// PanicOnError // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
func newOption(_ io.Writer, eh flag.ErrorHandling) (o *option) {
o = &option{
flagSet: flag.NewFlagSet(CommandName, eh),
}
Expand Down Expand Up @@ -68,7 +68,7 @@ func (o *option) parse(args []string) error {
return nil
}

//OptionCheck receives a slice of args and returns an error if it was not successfully parsed
// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
opt := newOption(ioutil.Discard, flag.ContinueOnError)
if err := opt.parse(args); err != nil {
Expand Down Expand Up @@ -117,14 +117,16 @@ func command(opt *option) error {
if err != nil {
return err
}
var out = os.Stdout
out := os.Stdout
if opt.output != "" {
var err error
out, err = os.OpenFile(opt.output, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666)
out, err = os.OpenFile(opt.output, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0o666)
if err != nil {
return err
}
defer out.Close()
defer func() {
_ = out.Close()
}()
}

mode := selectMode(opt.mode)
Expand All @@ -136,7 +138,6 @@ func command(opt *option) error {
if tok.Class == tokenizer.DUMMY {
fmt.Fprintf(ErrorWriter, "%s\n", tok.Surface)
} else {

fmt.Fprintf(ErrorWriter, "%s\t%v\n", tok.Surface, strings.Join(f, ","))
}
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/kagome/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ This must be specified by -X option during the go build. Such like:
defaultSubcommand = subcommands[0]
)

//Usage prints to stdout information about the tool
// Usage prints to stdout information about the tool
func Usage() {
fmt.Fprintf(errorWriter, "Japanese Morphological Analyzer -- github.com/ikawaha/kagome/v2\n")
fmt.Fprintf(errorWriter, "usage: %s <command>\n", filepath.Base(os.Args[0]))
Expand Down
2 changes: 1 addition & 1 deletion cmd/kagome/server/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func (o *option) parse(args []string) error {
return nil
}

//OptionCheck receives a slice of args and returns an error if it was not successfully parsed
// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
opt := newOption(flag.ContinueOnError)
if err := opt.parse(args); err != nil {
Expand Down
3 changes: 2 additions & 1 deletion cmd/kagome/server/demo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/ikawaha/kagome/v2/tokenizer"
)

//TokenizeDemoHandler represents the tokenizer demo server struct
// TokenizeDemoHandler represents the tokenizer demo server struct
type TokenizeDemoHandler struct {
tokenizer *tokenizer.Tokenizer
}
Expand Down Expand Up @@ -207,6 +207,7 @@ var graphHTML = `
</body>
</html>
`

var demoHTML = `
<!DOCTYPE html>
<html lang="ja">
Expand Down
8 changes: 5 additions & 3 deletions cmd/kagome/tokenize/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (o *option) parse(args []string) error {
return nil
}

//OptionCheck receives a slice of args and returns an error if it was not successfully parsed
// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
opt := newOption(ioutil.Discard, flag.ContinueOnError)
if err := opt.parse(args); err != nil {
Expand Down Expand Up @@ -136,14 +136,16 @@ func command(opt *option) error {
return err
}

var fp = os.Stdin
fp := os.Stdin
if opt.file != "" {
var err error
fp, err = os.Open(opt.file)
if err != nil {
return err
}
defer fp.Close()
defer func() {
_ = fp.Close()
}()
}
mode := selectMode(opt.mode)
scanner := bufio.NewScanner(fp)
Expand Down
1 change: 1 addition & 0 deletions filter/feature.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type filterEdge struct {
val Feature
to *filterNode
}

type filterNode struct {
fanout []*filterEdge
}
Expand Down
2 changes: 1 addition & 1 deletion filter/sentence_splitter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ func (s SentenceSplitter) isFollower(r rune) bool {
return false
}

// nolint: gocyclo
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[GolangCI-Lint] whyNoLint: include an explanation for nolint directive (view)

Rule Severity
gocritic:whyNoLint -

You can close this issue if no need to fix it. Learn more.

// ScanSentences is a split function for a Scanner that returns each sentence of text.
func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
Expand Down Expand Up @@ -120,5 +121,4 @@ func (s SentenceSplitter) ScanSentences(data []byte, atEOF bool) (advance int, t
}
// If we're at EOF, we have a final, non-terminated line. Return it.
return len(data), data[start:end], nil

}
0 _sample/userdict.txt → sample/userdict.txt
View file
Open in desktop
File renamed without changes.
4 changes: 2 additions & 2 deletions tokenizer/lattice/lattice.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const (
type TokenizeMode int

const (
//Normal Mode
// Normal Mode
Normal TokenizeMode = iota + 1
// Search Mode
Search
Expand Down Expand Up @@ -78,7 +78,7 @@ func (la *Lattice) addNode(pos, id, position, start int, class NodeClass, surfac
var m dict.Morph
switch class {
case DUMMY:
//use default cost
// use default cost
case KNOWN:
m = la.dic.Morphs[id]
case UNKNOWN:
Expand Down
2 changes: 1 addition & 1 deletion tokenizer/lattice/lattice_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func Test_LatticeBuild(t *testing.T) {
}

func Test_LatticeBuildWithUserDict(t *testing.T) {
const udictPath = "../../_sample/userdict.txt"
const udictPath = "../../sample/userdict.txt"

udic, err := dict.NewUserDict(udictPath)
if err != nil {
Expand Down
1 change: 0 additions & 1 deletion tokenizer/lattice/node_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
)

func Test_NodeClassString(t *testing.T) {

pairs := []struct {
in NodeClass
out string
Expand Down
3 changes: 1 addition & 2 deletions tokenizer/token_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
)

const (
userDictSample = "../_sample/userdict.txt"
userDictSample = "../sample/userdict.txt"
)

func Test_TokenClassString(t *testing.T) {
Expand Down Expand Up @@ -124,7 +124,6 @@ func Test_FeaturesUnknown(t *testing.T) {
t.Errorf("index >= len(want): expected empty feature and false, got %q, %v", f, ok)
}
})

}

func Test_FeatureAtUnknown(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion tokenizer/tokenizer_option_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

const (
testUserDictPath = "../_sample/userdict.txt"
testUserDictPath = "../sample/userdict.txt"
)

func TestTokenizer_Analyze_Nop(t *testing.T) {
Expand Down
5 changes: 0 additions & 5 deletions tokenizer/tokenizer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,6 @@ func Test_AnalyzeUnknown(t *testing.T) {
t.Errorf("got %v, expected %v", tok, expected[i])
}
}

}

func Test_TokenizeSpecialCase(t *testing.T) {
Expand Down Expand Up @@ -244,7 +243,6 @@ func Test_AnalyzeWithSearchMode(t *testing.T) {
t.Errorf("got %v, expected %v", tok, expected[i])
}
}

}

func Test_AnalyzeWithSearchModeUnknown(t *testing.T) {
Expand Down Expand Up @@ -304,7 +302,6 @@ func Test_AnalyzeWithExtendedModeEmpty(t *testing.T) {
t.Errorf("got %v, expected %v", tok, expected[i])
}
}

}

func Test_AnalyzeWithExtendedMode(t *testing.T) {
Expand Down Expand Up @@ -336,7 +333,6 @@ func Test_AnalyzeWithExtendedMode(t *testing.T) {
t.Errorf("got %v, expected %v", tok, expected[i])
}
}

}

func Test_AnalyzeWithExtendedModeUnknown(t *testing.T) {
Expand Down Expand Up @@ -369,7 +365,6 @@ func Test_AnalyzeWithExtendedModeUnknown(t *testing.T) {
t.Errorf("got %v, expected %v", tok, expected[i])
}
}

}

func Test_TokenizerDot(t *testing.T) {
Expand Down