Make logging more verbose

#40
brawer · May 15, 2024 · 518c4b3 · 518c4b3
1 parent 468514d
commit 518c4b3
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 0 deletions.
diff --git a/cmd/qrank-builder/itemsignals.go b/cmd/qrank-builder/itemsignals.go
@@ -189,6 +189,7 @@ func buildItemSignals(ctx context.Context, pageviews []string, sites *map[string
 	config.NumWorkers = runtime.NumCPU()
 	sorter, outChan, errChan := extsort.New(sigChan, ItemSignalsFromBytes, ItemSignalsLess, config)
 	merger := NewLineMerger(scanners)
+	logger.Printf("BuildItemSignals(): merging signals from %d files; #0=PageSignalsScanner; rest=pageviews", len(scanners))
 	group, groupCtx := errgroup.WithContext(ctx)
 	group.Go(func() error {
 		joiner := itemSignalsJoiner{out: sigChan}

diff --git a/cmd/qrank-builder/linemerger.go b/cmd/qrank-builder/linemerger.go
@@ -32,6 +32,9 @@ func NewLineMerger(r []LineScanner) *LineMerger {
 			m.heap = append(m.heap, item)
 		}
 		if err := item.scanner.Err(); err != nil {
+			if logger != nil {
+				logger.Printf("LineMerger: scanner #%d failed to scan first line, err=%v", item.index, err)
+			}
 			m.err = err
 			return m
 		}
@@ -59,6 +62,9 @@ func (m *LineMerger) Advance() bool {
 	}
 	if err := item.scanner.Err(); err != nil {
 		m.err = err
+		if logger != nil {
+			logger.Printf("LineMerger: scanner #%d failed, err=%v", item.index, err)
+		}
 		return false
 	}
 	return len(m.heap) > 0

diff --git a/cmd/qrank-builder/pagesignals.go b/cmd/qrank-builder/pagesignals.go
@@ -372,14 +372,17 @@ func (s *pageSignalsScanner) Scan() bool {
 			}
 			s.err = s.scanner.Err()
 			if s.err != nil {
+				logger.Printf("PageSignalsScanner.Scan(): failed, domain=%s, err=%v", s.domains[s.curDomain], s.err)
 				break
 			}
 		}
 		s.curDomain += 1
 		if s.curDomain == len(s.domains) {
+			logger.Println("PageSignalsScanner.Scan(): finished last domain")
 			break
 		}
 
+		logger.Printf("PageSignalsScanner.Scan(): opening %s", s.paths[s.curDomain])
 		s.reader, s.err = NewS3Reader(context.Background(), "qrank", s.paths[s.curDomain], s.storage)
 		if s.err != nil {
 			break

diff --git a/cmd/qrank-builder/pagesignals_test.go b/cmd/qrank-builder/pagesignals_test.go
@@ -118,6 +118,7 @@ func TestStoredPageSignals(t *testing.T) {
 }
 
 func TestPageSignalsScanner(t *testing.T) {
+	logger = log.New(&bytes.Buffer{}, "", log.Lshortfile)
 	s3 := NewFakeS3()
 	storeFakePageSignals("enwiki-20111231", "1,Q111|7,Q777", s3, t)
 	storeFakePageSignals("rmwiki-20110203", "1,Q11|2,Q22|3,Q33", s3, t)