Downloader: correct logging when create .torrent files

downloader/util.go

@@ -18,15 +18,13 @@ package downloader
 
 import (
 	"context"
-	//nolint:gosec
 	"fmt"
 	"net"
 	"os"
 	"path/filepath"
 	"regexp"
 	"runtime"
 	"strconv"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -35,13 +33,13 @@ import (
 	"github.com/anacrolix/torrent/metainfo"
 	common2 "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/common/cmp"
+	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	dir2 "github.com/ledgerwatch/erigon-lib/common/dir"
 	"github.com/ledgerwatch/erigon-lib/downloader/downloadercfg"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/log/v3"
-
-	"golang.org/x/sync/semaphore"
+	"golang.org/x/sync/errgroup"
 )
 
 // `github.com/anacrolix/torrent` library spawning several goroutines and producing many requests for each tracker. So we limit amout of trackers by 7
@@ -145,6 +143,18 @@ func seedableSegmentFiles(dir string) ([]string, error) {
 var historyFileRegex = regexp.MustCompile("^([[:lower:]]+).([0-9]+)-([0-9]+).(.*)$")
 
 func seedableHistorySnapshots(dir, subDir string) ([]string, error) {
+	l, err := seedableSnapshotsBySubDir(dir, "history")
+	if err != nil {
+		return nil, err
+	}
+	l2, err := seedableSnapshotsBySubDir(dir, "warm")
+	if err != nil {
+		return nil, err
+	}
+	return append(l, l2...), nil
+}
+
+func seedableSnapshotsBySubDir(dir, subDir string) ([]string, error) {
 	historyDir := filepath.Join(dir, subDir)
 	dir2.MustExist(historyDir)
 	files, err := os.ReadDir(historyDir)
@@ -186,10 +196,19 @@ func seedableHistorySnapshots(dir, subDir string) ([]string, error) {
 }
 
 func buildTorrentIfNeed(ctx context.Context, fName, root string) (err error) {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
 	fPath := filepath.Join(root, fName)
 	if dir2.FileExist(fPath + ".torrent") {
 		return
 	}
+	if !dir2.FileExist(fPath) {
+		return
+	}
 	info := &metainfo.Info{PieceLength: downloadercfg.DefaultPieceSize, Name: fName}
 	if err := info.BuildFromFilePath(fPath); err != nil {
 		return fmt.Errorf("createTorrentFileFromSegment: %w", err)
@@ -226,42 +245,35 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err
 		return nil, err
 	}
 
-	errs := make(chan error, len(files)*2)
-	wg := &sync.WaitGroup{}
-	workers := cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 2
-	var sem = semaphore.NewWeighted(int64(workers))
-	i := atomic.Int32{}
-	for _, f := range files {
-		wg.Add(1)
-		if err := sem.Acquire(ctx, 1); err != nil {
-			return nil, err
-		}
-		go func(f string) {
-			defer i.Add(1)
-			defer sem.Release(1)
-			defer wg.Done()
-			if err := buildTorrentIfNeed(ctx, f, snapDir); err != nil {
-				errs <- err
-			}
+	g, ctx := errgroup.WithContext(ctx)
+	g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4)
+	var i atomic.Int32
 
-			select {
-			default:
-			case <-ctx.Done():
-				errs <- ctx.Err()
-			case <-logEvery.C:
-				log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files)))
+	for _, file := range files {
+		file := file
+		g.Go(func() error {
+			defer i.Add(1)
+			if err := buildTorrentIfNeed(ctx, file, snapDir); err != nil {
+				return err
 			}
-		}(f)
-	}
-	go func() {
-		wg.Wait()
-		close(errs)
-	}()
-	for err := range errs {
-		if err != nil {
-			return nil, err
+			return nil
+		})
+	}
+
+	var m runtime.MemStats
+Loop:
+	for int(i.Load()) < len(files) {
+		select {
+		case <-ctx.Done():
+			break Loop // g.Wait() will return right error
+		case <-logEvery.C:
+			dbg.ReadMemStats(&m)
+			log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
 		}
 	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
 	return files, nil
 }
 

tools/licenses_check.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+scriptDir=$(dirname "${BASH_SOURCE[0]}")
+scriptName=$(basename "${BASH_SOURCE[0]}")
+projectDir="$scriptDir/.."
+goLicensesVersion="v1.6.0"
+
+if [[ "$1" == "--install-deps" ]]
+then
+	go install "github.com/google/go-licenses@$goLicensesVersion"
+	exit
+fi
+
+if ! which go-licenses > /dev/null
+then
+	echo "go-licenses tool is not found, install it with:"
+	echo "    go install github.com/google/go-licenses@$goLicensesVersion"
+	exit
+fi
+
+# enable build tags to cover maximum .go files
+export GOFLAGS="-tags=gorules,linux,tools"
+
+output=$(find "$projectDir" -type 'd' -maxdepth 1 \
+    -not -name ".*" \
+    -not -name tools \
+    | xargs go-licenses report 2>&1 \
+    `# exceptions` \
+    | grep -v "erigon-lib has empty version"        `# self` \
+    | grep -v "golang.org/x/"                       `# a part of Go` \
+    | grep -v "crawshaw.io/sqlite"                  `# ISC` \
+    | grep -v "erigon-lib/sais"                     `# MIT` \
+    | grep -v "github.com/anacrolix/go-libutp"      `# MIT` \
+    | grep -v "github.com/anacrolix/mmsg"           `# MPL-2.0` \
+    | grep -v "github.com/anacrolix/multiless"      `# MPL-2.0` \
+    | grep -v "github.com/anacrolix/sync"           `# MPL-2.0` \
+    | grep -v "github.com/anacrolix/upnp"           `# MPL-2.0` \
+    | grep -v "github.com/go-llsqlite/adapter"      `# MPL-2.0` \
+    | grep -v "github.com/go-llsqlite/crawshaw"     `# ISC` \
+    | grep -v "github.com/consensys/gnark-crypto"   `# Apache-2.0` \
+    | grep -v "github.com/erigontech/mdbx-go"       `# Apache-2.0` \
+    | grep -v "github.com/ledgerwatch/secp256k1"    `# BSD-3-Clause` \
+    | grep -v "github.com/RoaringBitmap/roaring"    `# Apache-2.0` \
+    | grep -v "github.com/!roaring!bitmap/roaring"  `# Apache-2.0` \
+    | grep -v "pedersen_hash"                       `# Apache-2.0` \
+    `# approved licenses` \
+    | grep -Ev "Apache-2.0$" \
+    | grep -Ev "BSD-2-Clause$" \
+    | grep -Ev "BSD-3-Clause$" \
+    | grep -Ev "ISC$" \
+    | grep -Ev "MIT$" \
+    | grep -Ev "MPL-2.0$" \
+)
+
+if [[ -n "$output" ]]
+then
+	echo "ERROR: $scriptName has found issues!" 1>&2
+	echo "ERROR: If it is a false positive, add it to the exceptions list in the script:" 1>&2
+	echo "$output" 1>&2
+	exit 1
+fi