Merge remote-tracking branch 'origin/master' into stable

.github/workflows/ci.yaml

@@ -38,11 +38,11 @@ jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 10
     strategy:
       matrix:
         go_version:
-          - ^1.17
+          - ^1.18
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -69,6 +69,38 @@ jobs:
         with:
           file: ./coverage.txt
 
+  fuzz:
+    name: Fuzzing
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.18
+
+      - name: Cache fuzz results
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/go-build/fuzz
+          key: ${{ runner.os }}-go-${{ hashFiles('**/*_fuzz_test.go', '**/*_fuzz_internal_test.go') }}
+          restore-keys: ${{ runner.os }}-go-
+
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+
+      - name: Run fuzzing
+        run: make -j4 fuzz
+
   lint:
     name: Lint
     runs-on: ubuntu-latest
@@ -79,10 +111,15 @@ jobs:
         with:
           submodules: recursive
 
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.18
+
       - name: Run linter
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.44.2
+          version: v1.45.0
 
   docker:
     name: Docker

.golangci.toml

@@ -9,4 +9,4 @@ format = "colored-line-number"
 
 [linters]
 enable-all = true
-disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"]
+disable = ["thelper", "ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"]

Dockerfile

@@ -1,7 +1,7 @@
 ###############################################################################
 # BUILD STAGE
 
-FROM golang:1.17-alpine AS build
+FROM golang:1.18-alpine AS build
 
 RUN set -x \
   && apk --no-cache --update add \

Makefile

@@ -2,12 +2,12 @@ ROOT_DIR     := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 IMAGE_NAME   := mtg
 APP_NAME     := $(IMAGE_NAME)
 
-GOLANGCI_LINT_VERSION := v1.44.2
+GOLANGCI_LINT_VERSION := v1.45.0
 
-VERSION_GO         := $(shell go version)
-VERSION_DATE       := $(shell date -Ru)
-VERSION_TAG        := $(shell git describe --tags --always)
-COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'"
+VERSION            := $(shell git describe --exact-match HEAD 2>/dev/null || git describe --tags --always)
+COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION)'"
+
+FUZZ_FLAGS := -fuzztime=120s
 
 GOBIN  := $(ROOT_DIR)/.bin
 GOTOOL := env "GOBIN=$(GOBIN)" "PATH=$(ROOT_DIR)/.bin:$(PATH)"
@@ -78,7 +78,7 @@ install-tools: install-tools-lint install-tools-godoc install-tools-gofumpt inst
 
 .PHONY: install-tools-lint
 install-tools-lint: .bin
-	@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh \
+	@curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \
 		| bash -s -- -b "$(GOBIN)" "$(GOLANGCI_LINT_VERSION)"
 
 .PHONY: install-tools-godoc
@@ -95,4 +95,27 @@ install-tools-goreleaser: .bin
 
 .PHONY: update-deps
 update-deps:
-	@go get -u && go mod tidy -go=1.17
+	@go get -u && go mod tidy -go=1.18
+
+.PHONY: fuzz
+fuzz: fuzz-ClientHello fuzz-ServerGenerateHandshakeFrame fuzz-ClientHandshake fuzz-ServerReceive fuzz-ServerSend
+
+.PHONY: fuzz-ClientHello
+fuzz-ClientHello:
+	@go test -fuzz=FuzzClientHello $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/faketls"
+
+.PHONY: fuzz-ServerGenerateHandshakeFrame
+fuzz-ServerGenerateHandshakeFrame:
+	@go test -fuzz=FuzzServerGenerateHandshakeFrame $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2"
+
+.PHONY: fuzz-ClientHandshake
+fuzz-ClientHandshake:
+	@go test -fuzz=FuzzClientHandshake $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2"
+
+.PHONY: fuzz-ServerReceive
+fuzz-ServerReceive:
+	@go test -fuzz=FuzzServerReceive $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2"
+
+.PHONY: fuzz-ServerSend
+fuzz-ServerSend:
+	@go test -fuzz=FuzzServerSend $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2"

README.md

@@ -317,12 +317,16 @@ Now you can create a systemd unit:
 ```console
 $ cat /etc/systemd/system/mtg.service
 [Unit]
-Description=mtg
+Description=mtg - MTProto proxy server
+Documentation=https://github.com/9seconds/mtg
+After=network.target
 
 [Service]
 ExecStart=/usr/local/bin/mtg run /etc/mtg.toml
 Restart=always
 RestartSec=3
+DynamicUser=true
+AmbientCapabilities=CAP_NET_BIND_SERVICE
 
 [Install]
 WantedBy=multi-user.target
@@ -388,7 +392,7 @@ Here goes a list of metrics with their types but without a prefix.
 | domain_fronting_traffic     | counter | `direction`                      | Count of bytes, transmitted to/from fronting domain.                                       |
 | domain_fronting             | counter | –                                | Count of domain fronting events.                                                           |
 | concurrency_limited         | counter | –                                | Count of events, when client connection was rejected due to concurrency limit.             |
-| ip_blocklisted              | counter | –                                | Count of events when client connection was rejected because IP was found in the blacklist. |
+| ip_blocklisted              | counter | `ip_list`                        | Count of events when client connection was rejected because IP was found in the blocklist. |
 | replay_attacks              | counter | –                                | Count of detected replay attacks.                                                          |
 
 Tag meaning:

go.mod

@@ -1,6 +1,6 @@
 module github.com/9seconds/mtg/v2
 
-go 1.17
+go 1.18
 
 require (
 	github.com/OneOfOne/xxhash v1.2.8
@@ -23,9 +23,9 @@ require (
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
-	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70
+	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
+	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
 	google.golang.org/protobuf v1.27.1 // indirect
 )
 

go.sum

@@ -291,8 +291,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU=
-golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38=
+golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -416,11 +416,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
-golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/cli/run_proxy.go

@@ -12,11 +12,13 @@ import (
 	"github.com/9seconds/mtg/v2/internal/config"
 	"github.com/9seconds/mtg/v2/internal/utils"
 	"github.com/9seconds/mtg/v2/ipblocklist"
+	"github.com/9seconds/mtg/v2/ipblocklist/files"
 	"github.com/9seconds/mtg/v2/logger"
 	"github.com/9seconds/mtg/v2/mtglib"
 	"github.com/9seconds/mtg/v2/network"
 	"github.com/9seconds/mtg/v2/stats"
 	"github.com/rs/zerolog"
+	"github.com/yl2chen/cidranger"
 )
 
 func makeLogger(conf *config.Config) mtglib.Logger {
@@ -89,7 +91,8 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache {
 func makeIPBlocklist(conf config.ListConfig,
 	logger mtglib.Logger,
 	ntw mtglib.Network,
-	updateCallback ipblocklist.FireholUpdateCallback) (mtglib.IPBlocklist, error) {
+	updateCallback ipblocklist.FireholUpdateCallback,
+) (mtglib.IPBlocklist, error) {
 	if !conf.Enabled.Get(false) {
 		return ipblocklist.NewNoop(), nil
 	}
@@ -105,7 +108,7 @@ func makeIPBlocklist(conf config.ListConfig,
 		}
 	}
 
-	firehol, err := ipblocklist.NewFirehol(logger.Named("ipblockist"),
+	blocklist, err := ipblocklist.NewFirehol(logger.Named("ipblockist"),
 		ntw,
 		conf.DownloadConcurrency.Get(1),
 		remoteURLs,
@@ -115,9 +118,44 @@ func makeIPBlocklist(conf config.ListConfig,
 		return nil, fmt.Errorf("incorrect parameters for firehol: %w", err)
 	}
 
-	go firehol.Run(conf.UpdateEach.Get(ipblocklist.DefaultFireholUpdateEach))
+	go blocklist.Run(conf.UpdateEach.Get(ipblocklist.DefaultFireholUpdateEach))
 
-	return firehol, nil
+	return blocklist, nil
+}
+
+func makeIPAllowlist(conf config.ListConfig,
+	logger mtglib.Logger,
+	ntw mtglib.Network,
+	updateCallback ipblocklist.FireholUpdateCallback,
+) (allowlist mtglib.IPBlocklist, err error) {
+	if !conf.Enabled.Get(false) {
+		allowlist, err = ipblocklist.NewFireholFromFiles(
+			logger.Named("ipblocklist"),
+			1,
+			[]files.File{
+				files.NewMem([]*net.IPNet{
+					cidranger.AllIPv4,
+					cidranger.AllIPv6,
+				}),
+			},
+			updateCallback,
+		)
+
+		go allowlist.Run(conf.UpdateEach.Get(ipblocklist.DefaultFireholUpdateEach))
+	} else {
+		allowlist, err = makeIPBlocklist(
+			conf,
+			logger,
+			ntw,
+			updateCallback,
+		)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("cannot build allowlist: %w", err)
+	}
+
+	return allowlist, nil
 }
 
 func makeEventStream(conf *config.Config, logger mtglib.Logger) (mtglib.EventStream, error) {
@@ -185,29 +223,24 @@ func runProxy(conf *config.Config, version string) error { // nolint: funlen
 		return fmt.Errorf("cannot build ip blocklist: %w", err)
 	}
 
-	var whitelist mtglib.IPBlocklist
-
-	if conf.Defense.Allowlist.Enabled.Get(false) {
-		whlist, err := makeIPBlocklist(
-			conf.Defense.Allowlist,
-			logger.Named("allowlist"),
-			ntw,
-			func(ctx context.Context, size int) {
-				eventStream.Send(ctx, mtglib.NewEventIPListSize(size, false))
-			})
-		if err != nil {
-			return fmt.Errorf("cannot build ip allowlist: %w", err)
-		}
-
-		whitelist = whlist
+	allowlist, err := makeIPAllowlist(
+		conf.Defense.Allowlist,
+		logger.Named("allowlist"),
+		ntw,
+		func(ctx context.Context, size int) {
+			eventStream.Send(ctx, mtglib.NewEventIPListSize(size, false))
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("cannot build ip allowlist: %w", err)
 	}
 
 	opts := mtglib.ProxyOpts{
 		Logger:          logger,
 		Network:         ntw,
 		AntiReplayCache: makeAntiReplayCache(conf),
 		IPBlocklist:     blocklist,
-		IPWhitelist:     whitelist,
+		IPAllowlist:     allowlist,
 		EventStream:     eventStream,
 
 		Secret:             conf.Secret,

internal/testlib/mtglib_network_mock.go

@@ -25,6 +25,7 @@ func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address st
 }
 
 func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context,
-	network, address string) (essentials.Conn, error)) *http.Client {
+	network, address string) (essentials.Conn, error),
+) *http.Client {
 	return m.Called(dialFunc).Get(0).(*http.Client) // nolint: forcetypeassert
 }

ipblocklist/files/mem.go

@@ -0,0 +1,37 @@
+package files
+
+import (
+	"context"
+	"io"
+	"net"
+	"strings"
+)
+
+type memFile struct {
+	data string
+}
+
+func (m memFile) Open(ctx context.Context) (io.ReadCloser, error) {
+	return io.NopCloser(strings.NewReader(m.data)), nil
+}
+
+func (m memFile) String() string {
+	return "mem"
+}
+
+func NewMem(networks []*net.IPNet) File {
+	builder := strings.Builder{}
+
+	if len(networks) > 0 {
+		builder.WriteString(networks[0].String())
+	}
+
+	for i := 1; i < len(networks); i++ {
+		builder.WriteString("\n")
+		builder.WriteString(networks[i].String())
+	}
+
+	return memFile{
+		data: builder.String(),
+	}
+}