Merge pull request #1 from ledgerwatch/devel

Upstream changes

.dockerignore

@@ -5,7 +5,5 @@
 **/*.dSYM
 
 build
-tests/testdata
-tests/.git
+tests
 cmd/prometheus
-

.github/CONTRIBUTING.md

@@ -4,10 +4,10 @@ Thank you for considering to help out with the source code! We welcome
 contributions from anyone on the internet, and are grateful for even the 
 smallest of fixes!
 
-If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+If you'd like to contribute to Erigon, please fork, fix, commit and send a 
 pull request for the maintainers to review and merge into the main code base. If
 you wish to submit more complex changes though, please check up with the core 
-devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+devs first on [Discord server](https://github.com/ledgerwatch/erigon#erigon-discord-server) to 
 ensure those changes are in line with the general philosophy of the project 
 and/or get some early feedback which can make both your efforts much lighter as
 well as our review and merge procedures quick and simple.
@@ -21,7 +21,7 @@ Please make sure your contributions adhere to our coding guidelines:
 (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
  * Code must be documented adhering to the official Go 
 [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
- * Pull requests need to be based on and opened against the `master` branch.
+ * Pull requests need to be based on and opened against the `devel` branch.
  * Commit messages should be prefixed with the package(s) they modify.
    * E.g. "eth, rpc: make trace configs optional"
 
@@ -30,11 +30,11 @@ Please make sure your contributions adhere to our coding guidelines:
 Before you submit a feature request, please check and make sure that it isn't 
 possible through some other means. The JavaScript-enabled console is a powerful 
 feature in the right hands. Please check our 
-[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+[Wiki page](https://github.com/ledgerwatch/erigon/wiki) for more info
 and help.
 
 ## Configuration, dependencies, and tests
 
-Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+Please see the [Readme](https://github.com/ledgerwatch/erigon#readme) file
 for more details on configuring your environment, managing project dependencies
 and testing procedures.

.github/workflows/ci.yml

@@ -3,73 +3,105 @@ on:
   push:
     branches:
       - devel
+      - alpha
+      - stable
   pull_request:
     branches:
       - devel
+      - alpha
+      - stable
 jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-16.04, ubuntu-20.04, macos-10.15 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
-          go-version: 1.16.x
-      - name: Install dependencies
-        run: |
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt update && sudo apt install build-essential
-          fi
-        shell: bash
-      - uses: actions/cache@v2
+          go-version: 1.18.x
+      - name: Install dependencies on Linux
+        if: runner.os == 'Linux'
+        run: sudo apt update && sudo apt install build-essential
+
+      - uses: actions/cache@v3
         with:
-          path: ~/go/pkg/mod
-          key: ${{ matrix.os }}-go-${{ hashFiles('**/go.sum') }}
-      - run: make all
+          path: |
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+            ~/go/pkg/mod
+          key: go-${{ matrix.os }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-${{ matrix.os }}-
+
+      - name: Build
+        run: make all
+
       - name: Reproducible build test
         run: |
+          make erigon
           shasum -a256 ./build/bin/erigon > erigon1.sha256
           make erigon
           shasum -a256 ./build/bin/erigon > erigon2.sha256
           if ! cmp -s erigon1.sha256 erigon2.sha256; then
-            echo >&2 "Reproducible build broken"; exit 1
+            echo >&2 "Reproducible build broken"; cat erigon1.sha256; cat erigon2.sha256; exit 1
           fi
+
       - name: Lint
-        if: matrix.os == 'ubuntu-20.04'
-        uses: golangci/golangci-lint-action@v2
+        if: runner.os == 'Linux'
+        uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.41
-          skip-go-installation: true
+          version: v1.46
           skip-pkg-cache: true
           skip-build-cache: true
-      - run: make test
 
-  win:
+      - name: Test
+        run: make test
+
+  tests-windows:
     strategy:
       matrix:
-        os: [ windows-2019 ]
+        os: [ windows-2022 ]
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: git submodule update --init --recursive --force
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
-          go-version: 1.16.x
-      - run: choco upgrade mingw cmake -y --no-progress
-      - name: Build
+          go-version: 1.18.x
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            C:\ProgramData\chocolatey\lib\mingw
+            C:\ProgramData\chocolatey\lib\cmake
+          key: chocolatey-${{ matrix.os }}
+      - name: Install dependencies
         run: |
-          .\wmake.ps1
-          make test
-        shell: powershell
+          choco upgrade mingw -y --no-progress --version 11.2.0.07112021
+          choco install cmake -y --no-progress --version 3.23.1
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~\AppData\Local\go-build
+            ~\go\pkg\mod
+          key: go-${{ matrix.os }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-${{ matrix.os }}-
+
+      - name: Build
+        run: .\wmake.ps1 all
+
+      - name: Test
+        run: .\wmake.ps1 test
 
   docker:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - run: git submodule update --init --recursive --force
-      - run: docker build .
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # fetch git tags for "git describe"
+      - run: make docker

.github/workflows/test-integration.yml

@@ -0,0 +1,72 @@
+name: Integration tests
+on:
+  push:
+    branches:
+      - devel
+      - alpha
+      - stable
+  schedule:
+    - cron: '20 16 * * *' # daily at 16:20 UTC
+jobs:
+  tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive --force
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+      - name: Install dependencies on Linux
+        if: runner.os == 'Linux'
+        run: sudo apt update && sudo apt install build-essential
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+            ~/go/pkg/mod
+          key: go-${{ matrix.os }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-${{ matrix.os }}-
+
+      - name: test-integration
+        run: make test-integration
+
+  tests-windows:
+    strategy:
+      matrix:
+        os: [ windows-2022 ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive --force
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            C:\ProgramData\chocolatey\lib\mingw
+            C:\ProgramData\chocolatey\lib\cmake
+          key: chocolatey-${{ matrix.os }}
+      - name: Install dependencies
+        run: |
+          choco upgrade mingw -y --no-progress --version 11.2.0.07112021
+          choco install cmake -y --no-progress --version 3.23.1
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~\AppData\Local\go-build
+            ~\go\pkg\mod
+          key: go-${{ matrix.os }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: go-${{ matrix.os }}-
+
+      - name: test-integration
+        run: .\wmake.ps1 test-integration

.gitignore

@@ -62,11 +62,16 @@ right_*.txt
 root_*.txt
 
 __pycache__
-docker-compose.dev.yml
 /build
 *.tmp
 
 /ethdb/*.fail
 
 libmdbx/build/*
-tests/testdata/*
+tests/testdata/*
+
+go.work
+
+/goerli
+
+docker-compose.*.yml

.gitmodules

@@ -1,6 +1,13 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
+[submodule "turbo/snapshotsync/snapshothashes/erigon-snapshots"]
+	path = turbo/snapshotsync/snapshothashes/erigon-snapshots
+	url = https://github.com/ledgerwatch/erigon-snapshot.git
+[submodule "cmd/downloader/trackers/trackerslist"]
+	path = cmd/downloader/trackers/trackerslist
+	url = https://github.com/ngosang/trackerslist.git
+
 [submodule "libmdbx"]
 	path = libmdbx
-	url = https://github.com/erthink/libmdbx
+	url = https://github.com/torquem-ch/libmdbx.git

.golangci.yml

@@ -7,30 +7,33 @@ linters:
     - gofmt
     - deadcode
     - errcheck
-    - gosimple
+#    - gosimple # 1.18
     - govet
     - ineffassign
-    - staticcheck
-    - structcheck
-    - unused
+#    - staticcheck # 1.18
+#    - structcheck # 1.18
+#    - unused # 1.18
     - varcheck
-    - gocritic
+#    - gocritic
+#    - bodyclose # 1.18
+#    - gosec
 #    - forcetypeassert
 
 linters-settings:
   gocritic:
     # Which checks should be enabled; can't be combined with 'disabled-checks';
     # See https://go-critic.github.io/overview#checks-overview
-    # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
+    # To check which checks are enabled run `GL_DEBUG=gocritic ./build/bin/golangci-lint run`
     # By default list of stable checks is used.
     enabled-checks:
       - ruleguard
       - truncateCmp
-    #      - defaultCaseOrder
+      # - defaultCaseOrder
 
     # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty
     disabled-checks:
       - regexpMust
+      - appendAssign
       #      - hugeParam
       - rangeValCopy
       - exitAfterDefer
@@ -42,17 +45,22 @@ linters-settings:
       - captLocal
       - commentFormatting
       - ifElseChain
-      - appendAssign
+      - importShadow
+      - paramTypeCombine
+      - builtinShadow
+      - typeUnparen
 
     # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks.
     # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags".
     enabled-tags:
       - performance
+      - diagnostic
+      - opinionated
     disabled-tags:
       - experimental
-    ruleguard:
-      rules: "rules.go"
     settings:
+      ruleguard:
+        rules: "rules.go"
       hugeParam:
         # size in bytes that makes the warning trigger (default 80)
         sizeThreshold: 1000
@@ -92,7 +100,19 @@ issues:
         - staticcheck
       text: "SA(1019|1029|5011)"
     # Exclude some linters from running on tests files.
-    - path: _test\.go
+    - path: test\.go
+      linters:
+        - gosec
+        - unused
+        - deadcode
+        - gocritic
+    - path: hack\.go
+      linters:
+        - gosec
+        - unused
+        - deadcode
+        - gocritic
+    - path: cmd/devp2p
       linters:
         - gosec
         - unused
@@ -102,3 +122,7 @@ issues:
       linters:
         - gosec
         - gocritic
+    - path: p2p/simulations
+      linters:
+        - gosec
+        - gocritic