Skip to content

Commit

Permalink
more useful logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpppk
mpppk committed Jul 16, 2019
1 parent 8c4a1c8 commit 4b38bdb
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 77 deletions.
86 changes: 40 additions & 46 deletions lib/iroha.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import (
"golang.org/x/sync/errgroup"
)

var logThresholdSec = 3.0
var goroutineThresholdSec = 1.0
var logDepthThreshold = 1
var minParallelDepth = 1
var maxParallelDepth = 2

type Iroha struct {
katakanaBitsMap KatakanaBitsMap
Expand All @@ -34,7 +35,7 @@ func (i *Iroha) PrintWordByKatakanaMap() {

func (i *Iroha) Search() (rowIndicesList [][]int, err error) {
katakanaBitsAndWordsList := i.katakana.ListSortedKatakanaBitsAndWords()
i.log = NewLog(katakanaBitsAndWordsList)
i.log = NewLog(katakanaBitsAndWordsList, logDepthThreshold, minParallelDepth)
wordsList, _, err := i.searchByBits(katakanaBitsAndWordsList, WordBits(0))
if err != nil {
return nil, err
Expand Down Expand Up @@ -99,25 +100,50 @@ func (i *Iroha) searchByBits(katakanaBitsAndWords []*KatakanaBitsAndWords, remai
return nil
}

goroutineMode := false
eg := errgroup.Group{}
wordListChan := make(chan []*Word, 100)
for cur, word := range katakanaAndWordBits.Words {
if goroutineMode {
gCur := cur
goroutineMode := depth >= minParallelDepth && depth <= maxParallelDepth

if goroutineMode {
eg := errgroup.Group{}
wordListChan := make(chan []*Word, 100)
for _, word := range katakanaAndWordBits.Words {
gWord := word
gDepth := depth
eg.Go(func() error {
measurer := NewTimeMeasurerAndStart()
if err := gf(gWord, wordListChan); err != nil {
return err
}
if t := measurer.GetElapsedTimeSec(); t > logThresholdSec {
i.log.PrintProgressLog(gDepth, gCur, t, "")
}
t := measurer.GetElapsedTimeSec()
i.log.PrintProgressLog(gDepth, t, "")
return nil
})
} else {
}
errChan := make(chan error)
go func() {
if err := eg.Wait(); err != nil {
errChan <- err
return
}
close(wordListChan)
}()

L:
for {
select {
case wordList, ok := <-wordListChan:
if !ok {
break L
}
irohaWordLists = append(irohaWordLists, wordList)
case err, ok := <-errChan:
if !ok {
return nil, false, fmt.Errorf("unexpected error channel closing")
}
return nil, false, err
}
}
} else {
for _, word := range katakanaAndWordBits.Words {
measurer := NewTimeMeasurerAndStart()
wordList, ok, err := f(word)
if err != nil {
Expand All @@ -127,39 +153,7 @@ func (i *Iroha) searchByBits(katakanaBitsAndWords []*KatakanaBitsAndWords, remai
irohaWordLists = append(irohaWordLists, wordList...)
}
t := measurer.GetElapsedTimeSec()
if t > logThresholdSec {
i.log.PrintProgressLog(depth, cur, t, "")
}
if t > goroutineThresholdSec {
goroutineMode = true
}

}

}

errChan := make(chan error)
go func() {
if err := eg.Wait(); err != nil {
errChan <- err
return
}
close(wordListChan)
}()

L:
for {
select {
case wordList, ok := <-wordListChan:
if !ok {
break L
}
irohaWordLists = append(irohaWordLists, wordList)
case err, ok := <-errChan:
if !ok {
return nil, false, fmt.Errorf("unexpected error channel closing")
}
return nil, false, err
i.log.PrintProgressLog(depth, t, "")
}
}

Expand Down
92 changes: 61 additions & 31 deletions lib/log.go
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,21 @@
package lib

import (
"fmt"
"log"
"runtime"
"strings"
)

type Log struct {
counts []int
curs []int
measurer *TimeMeasurer
counts []int
curs []int
measurer *TimeMeasurer
depthThreshold int
minParallelDepth int
}

func NewLog(katakanaBitsAndWordsList []*KatakanaBitsAndWords) *Log {
func NewLog(katakanaBitsAndWordsList []*KatakanaBitsAndWords, depthThreshold, maxParallelDepth int) *Log {
counts := make([]int, len(katakanaBitsAndWordsList), len(katakanaBitsAndWordsList))
for index, katakanaBitsAndWords := range katakanaBitsAndWordsList {
counts[index] = len(katakanaBitsAndWords.Words)
Expand All @@ -23,14 +27,24 @@ func NewLog(katakanaBitsAndWordsList []*KatakanaBitsAndWords) *Log {
log.Printf("counts: %.2v %d\n", counts, countsSum)

return &Log{
counts: counts,
curs: make([]int, len(counts), len(counts)),
measurer: NewTimeMeasurer(),
counts: counts,
curs: make([]int, len(counts), len(counts)),
measurer: NewTimeMeasurer(),
depthThreshold: depthThreshold,
minParallelDepth: maxParallelDepth,
}
}

func (l *Log) updateProgress(depth, cur int) {
l.curs[depth] = cur
func (l *Log) updateProgress(depth int) {
if depth > l.minParallelDepth {
return
}
l.curs[depth]++

if depth >= len(l.curs) {
return
}

for i := range l.curs {
if i <= depth {
continue
Expand All @@ -39,39 +53,55 @@ func (l *Log) updateProgress(depth, cur int) {
}
}

func (l *Log) getProgress(depth int) float64 {
max := float64(l.counts[0])
cur := float64(l.curs[0])
for index, count := range l.counts[1 : depth+1] {
max *= float64(count)
cur *= float64(l.curs[index])
func (l *Log) getProgress() float64 {
basePercent := 1.0
progress := 0.0
for index, count := range l.counts[:l.minParallelDepth+1] {
progress += float64(l.curs[index]) / float64(count) * basePercent * 100
basePercent *= 1.0 / float64(count)
}
return (cur / max) * 100
return progress
}

func (l *Log) PrintProgressLog(depth, current int, sec float64, msg string) {
l.updateProgress(depth, current)
func (l *Log) PrintProgressLog(depth int, sec float64, msg string) {
if depth > l.depthThreshold {
return
}
l.updateProgress(depth)
m := ""
if msg != "" {
m = " (" + msg + ")"
}
percents := make([]int, len(l.curs), len(l.curs))
for i, cur := range l.curs {
var logs []string
logs = append(logs, fmt.Sprintf("d:%02v", depth))
if depth < l.minParallelDepth+1 {
logs = append(logs, fmt.Sprintf("%04v/%04v", l.curs[depth], l.counts[depth]))
} else {
logs = append(logs, "----/----")
}
logs = append(logs, fmt.Sprintf("%05.2fsec", sec))
logs = append(logs, fmt.Sprintf("p:%02.6f", l.getProgress()))
logs = append(logs, fmt.Sprintf("g:%06d", runtime.NumGoroutine()))
logs = append(logs, fmt.Sprintf("%02v", l.getPercentSlice()))
logs = append(logs, m)
log.Printf(strings.Join(logs, " "))
}

func (l *Log) getPercentSlice() []int {
percentsLen := l.depthThreshold + 1
if percentsLen > l.minParallelDepth+1 {
percentsLen = l.minParallelDepth + 1
}
if percentsLen <= 0 {
return []int{}
}
percents := make([]int, percentsLen, percentsLen)
for i, cur := range l.curs[:percentsLen] {
if l.counts[i] == 0 {
percents[i] = 0
continue
}
percents[i] = int((float64(cur) / float64(l.counts[i])) * 100)
}
log.Printf("d:%02v %04v/%04v %05.2fsec %03.6f%s gr=%06d %02d%s",
depth,
current,
l.counts[depth],
sec,
l.getProgress(depth),
"%",
runtime.NumGoroutine(),
percents,
m,
)
return percents
}

0 comments on commit 4b38bdb

Please sign in to comment.