diff --git a/.gitmodules b/.gitmodules index e1a0db9182e..ae94b08f852 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,6 @@ [submodule "tests"] path = tests/testdata url = https://github.com/ethereum/tests -[submodule "cmd/downloader/trackers/trackerslist"] - path = cmd/downloader/trackers/trackerslist - url = https://github.com/ngosang/trackerslist.git - [submodule "libmdbx"] path = libmdbx url = https://github.com/torquem-ch/libmdbx.git diff --git a/cmd/downloader/trackers/embed.go b/cmd/downloader/trackers/embed.go index 694f2eee40e..bf179e756aa 100644 --- a/cmd/downloader/trackers/embed.go +++ b/cmd/downloader/trackers/embed.go @@ -2,29 +2,18 @@ package trackers import ( "bufio" - _ "embed" "strings" -) - -//go:embed trackerslist/trackers_best.txt -var best string -var Best = split(best) - -//go:embed trackerslist/trackers_all_https.txt -var https string -var Https = split(https) -//go:embed trackerslist/trackers_all_http.txt -var http string -var Http = split(http) - -//go:embed trackerslist/trackers_all_udp.txt -var udp string -var Udp = split(udp) + "github.com/ledgerwatch/trackerslist" +) -//go:embed trackerslist/trackers_all_ws.txt -var ws string -var Ws = split(ws) +var ( + Best = split(trackerslist.Best) + Https = split(trackerslist.Https) + Http = split(trackerslist.Http) + Udp = split(trackerslist.Udp) + Ws = split(trackerslist.Ws) +) func split(txt string) (lines []string) { sc := bufio.NewScanner(strings.NewReader(txt)) diff --git a/cmd/downloader/trackers/trackerslist b/cmd/downloader/trackers/trackerslist deleted file mode 160000 index 17f277f3762..00000000000 --- a/cmd/downloader/trackers/trackerslist +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 17f277f376286f5a99db386421897d5f82031f57 diff --git a/go.mod b/go.mod index 4ca64f2642a..47404e9aac6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,9 @@ go 1.18 require ( github.com/ledgerwatch/erigon-lib v0.0.0-20220719140506-af5355ee9286 github.com/ledgerwatch/erigon-snapshot v1.0.0 + github.com/ledgerwatch/log/v3 v3.4.1 + github.com/ledgerwatch/secp256k1 v1.0.0 + github.com/ledgerwatch/trackerslist v1.0.0 ) require ( @@ -41,8 +44,6 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/kevinburke/go-bindata v3.21.0+incompatible - github.com/ledgerwatch/log/v3 v3.4.1 - github.com/ledgerwatch/secp256k1 v1.0.0 github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6 github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml/v2 v2.0.2 diff --git a/go.sum b/go.sum index 120e46a2e14..22e34193ee1 100644 --- a/go.sum +++ b/go.sum @@ -398,6 +398,8 @@ github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= github.com/ledgerwatch/secp256k1 v1.0.0/go.mod h1:SPmqJFciiF/Q0mPt2jVs2dTr/1TZBTIA+kPMmKgBAak= +github.com/ledgerwatch/trackerslist v1.0.0 h1:6gnQu93WCTL4jPcdmc8UEmw56Cb8IFQHLGnevfIeLwo= +github.com/ledgerwatch/trackerslist v1.0.0/go.mod h1:pCC+eEw8izNcnBBiSwvIq8kKsxDLInAafSW275jqFrg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lispad/go-generics-tools v1.1.0 h1:mbSgcxdFVmpoyso1X/MJHXbSbSL3dD+qhRryyxk+/XY= diff --git a/turbo/snapshotsync/parallelcompress/decompress.go b/turbo/snapshotsync/parallelcompress/decompress.go deleted file mode 100644 index 481717b2ab9..00000000000 --- a/turbo/snapshotsync/parallelcompress/decompress.go +++ /dev/null @@ -1,62 +0,0 @@ -package parallelcompress - -import ( - "bufio" - "encoding/binary" - "fmt" - "os" - "time" - - "github.com/ledgerwatch/erigon-lib/compress" - "github.com/ledgerwatch/erigon-lib/etl" - "github.com/ledgerwatch/log/v3" -) - -func Decompress(logPrefix, segFilePath, datFilePath string) error { - d, err := compress.NewDecompressor(segFilePath) - if err != nil { - return err - } - defer d.Close() - logEvery := time.NewTicker(20 * time.Second) - defer logEvery.Stop() - var df *os.File - if df, err = os.Create(datFilePath); err != nil { - return err - } - dw := bufio.NewWriterSize(df, etl.BufIOSize) - var word = make([]byte, 0, 256) - numBuf := make([]byte, binary.MaxVarintLen64) - var decodeTime time.Duration - g := d.MakeGetter() - start := time.Now() - wc := 0 - for g.HasNext() { - word, _ = g.Next(word[:0]) - decodeTime += time.Since(start) - n := binary.PutUvarint(numBuf, uint64(len(word))) - if _, e := dw.Write(numBuf[:n]); e != nil { - return e - } - if len(word) > 0 { - if _, e := dw.Write(word); e != nil { - return e - } - } - wc++ - select { - default: - case <-logEvery.C: - log.Info(fmt.Sprintf("[%s] Decompress", logPrefix), "millions", wc/1_000_000) - } - start = time.Now() - } - log.Info(fmt.Sprintf("[%s] Average decoding time", logPrefix), "per word", time.Duration(int64(decodeTime)/int64(wc))) - if err = dw.Flush(); err != nil { - return err - } - if err = df.Close(); err != nil { - return err - } - return nil -}