Merge remote-tracking branch 'origin/master' into stable

.github/workflows/ci.yaml

@@ -42,7 +42,6 @@ jobs:
     strategy:
       matrix:
         go_version:
-          - ~1.16
           - ^1.17
     steps:
       - name: Checkout
@@ -83,7 +82,7 @@ jobs:
       - name: Run linter
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.43.0
+          version: v1.44.2
 
   docker:
     name: Docker

.golangci.toml

@@ -9,4 +9,4 @@ format = "colored-line-number"
 
 [linters]
 enable-all = true
-disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct"]
+disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"]

Makefile

@@ -2,7 +2,7 @@ ROOT_DIR     := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 IMAGE_NAME   := mtg
 APP_NAME     := $(IMAGE_NAME)
 
-GOLANGCI_LINT_VERSION := v1.43.0
+GOLANGCI_LINT_VERSION := v1.44.2
 
 VERSION_GO         := $(shell go version)
 VERSION_DATE       := $(shell date -Ru)

README.md

@@ -1,6 +1,6 @@
 # mtg
 
-Highly-opionated (ex-bullshit-free) MTPROTO proxy for
+Highly-opinionated (ex-bullshit-free) MTPROTO proxy for
 [Telegram](https://telegram.org/).
 
 [![CI](https://github.com/9seconds/mtg/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/9seconds/mtg/actions/workflows/ci.yaml)
@@ -224,6 +224,16 @@ $ mtg generate-secret --hex google.com
 ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d
 ```
 
+equivalent commands with docker:
+
+```console
+$ docker run --rm nineseconds/mtg:2 generate-secret google.com
+7ibaERuTSGPH1RdztfYnN4tnb29nbGUuY29t
+
+$ docker run --rm nineseconds/mtg:2 generate-secret --hex google.com
+ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d
+```
+
 This secret is a keystone for a proxy and your password for a client.
 You need to keep it secured.
 
@@ -324,7 +334,7 @@ $ sudo systemctl start mtg
 or you can run a docker image
 
 ```console
-docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --restart=unless-stopped nineseconds/mtg:2
+docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --name mtg-proxy --restart=unless-stopped nineseconds/mtg:2
 ```
 
 where _443_ is a host port (a port you want to connect to from a
@@ -353,6 +363,12 @@ $ mtg access /etc/mtg.toml
 }
 ```
 
+or if you are using docker:
+
+```console
+$ docker exec mtg-proxy /mtg access /config.toml
+```
+
 ## Metrics
 
 Out of the box, mtg works with
@@ -367,6 +383,7 @@ Here goes a list of metrics with their types but without a prefix.
 | client_connections          | gauge   | `ip_family`                      | Count of processing client connections.                                                    |
 | telegram_connections        | gauge   | `telegram_ip`, `dc`              | Count of connections to Telegram servers.                                                  |
 | domain_fronting_connections | gauge   | `ip_family`                      | Count of connections to fronting domain.                                                   |
+| iplist_size                 | gauge   | `ip_list`                        | A size of either allowlist or blocklist in use.                                            |
 | telegram_traffic            | counter | `telegram_ip`, `dc`, `direction` | Count of bytes, transmitted to/from Telegram.                                              |
 | domain_fronting_traffic     | counter | `direction`                      | Count of bytes, transmitted to/from fronting domain.                                       |
 | domain_fronting             | counter | –                                | Count of domain fronting events.                                                           |
@@ -382,3 +399,4 @@ Tag meaning:
 | dc          |                            | A number of the Telegram DC for a connection. |
 | telegram_ip |                            | IP address of the Telegram server.            |
 | direction   | `to_client`, `from_client` | A direction of the traffic flow.              |
+| ip_list     | `allowlist`, `blocklist`   | A type of the IP list.                        |

events/event_stream.go

@@ -102,6 +102,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob
 				observer.EventConcurrencyLimited(typedEvt)
 			case mtglib.EventReplayAttack:
 				observer.EventReplayAttack(typedEvt)
+			case mtglib.EventIPListSize:
+				observer.EventIPListSize(typedEvt)
 			}
 		}
 	}

events/event_stream_test.go

@@ -204,6 +204,27 @@ func (suite *EventStreamTestSuite) TestEventReplayAttack() {
 	time.Sleep(100 * time.Millisecond)
 }
 
+func (suite *EventStreamTestSuite) TestEventIPListSize() {
+	evt := mtglib.NewEventIPListSize(10, true)
+
+	for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} {
+		v.
+			On("EventIPListSize", mock.Anything).
+			Once().
+			Run(func(args mock.Arguments) {
+				caught, ok := args.Get(0).(mtglib.EventIPListSize)
+
+				suite.True(ok)
+				suite.Equal(evt.Timestamp(), caught.Timestamp())
+				suite.Equal(evt.Size, caught.Size)
+				suite.Equal(evt.IsBlockList, caught.IsBlockList)
+			})
+	}
+
+	suite.stream.Send(suite.ctx, evt)
+	time.Sleep(100 * time.Millisecond)
+}
+
 func (suite *EventStreamTestSuite) TearDownTest() {
 	suite.stream.Shutdown()
 	suite.ctxCancel()

events/init.go

@@ -53,6 +53,9 @@ type Observer interface {
 	// EventReplayAttack reacts on incoming mtglib.EventReplayAttack event.
 	EventReplayAttack(mtglib.EventReplayAttack)
 
+	// EventIPListSize reacts on incoming mtglib.EventIPListSize
+	EventIPListSize(mtglib.EventIPListSize)
+
 	// Shutdown stop observer. Default event stream guarantees:
 	//   1. If shutdown is executed, it is executed only once
 	//   2. Observer won't receieve any new message after this

events/init_test.go

@@ -41,6 +41,10 @@ func (o *ObserverMock) EventReplayAttack(evt mtglib.EventReplayAttack) {
 	o.Called(evt)
 }
 
+func (o *ObserverMock) EventIPListSize(evt mtglib.EventIPListSize) {
+	o.Called(evt)
+}
+
 func (o *ObserverMock) Shutdown() {
 	o.Called()
 }

events/multi_observer.go

@@ -130,6 +130,21 @@ func (m multiObserver) EventReplayAttack(evt mtglib.EventReplayAttack) {
 	wg.Wait()
 }
 
+func (m multiObserver) EventIPListSize(evt mtglib.EventIPListSize) {
+	wg := &sync.WaitGroup{}
+	wg.Add(len(m.observers))
+
+	for _, v := range m.observers {
+		go func(obs Observer) {
+			defer wg.Done()
+
+			obs.EventIPListSize(evt)
+		}(v)
+	}
+
+	wg.Wait()
+}
+
 func (m multiObserver) Shutdown() {
 	for _, v := range m.observers {
 		v.Shutdown()

events/noop.go

@@ -25,6 +25,7 @@ func (n noopObserver) EventFinish(_ mtglib.EventFinish)
 func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {}
 func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted)           {}
 func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack)             {}
+func (n noopObserver) EventIPListSize(_ mtglib.EventIPListSize)                 {}
 func (n noopObserver) Shutdown()                                                {}
 
 // NewNoopObserver creates an observer which discards each message.

events/noop_test.go

@@ -27,6 +27,7 @@ func (suite *NoopTestSuite) SetupSuite() {
 		"concurrency-limited": mtglib.NewEventConcurrencyLimited(),
 		"ip-blacklisted":      mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")),
 		"replay-attack":       mtglib.NewEventReplayAttack("connID"),
+		"ip-list-size":        mtglib.NewEventIPListSize(10, true),
 	}
 	suite.ctx = context.Background()
 }
@@ -65,6 +66,8 @@ func (suite *NoopTestSuite) TestObserver() {
 				observer.EventIPBlocklisted(typedEvt)
 			case mtglib.EventReplayAttack:
 				observer.EventReplayAttack(typedEvt)
+			case mtglib.EventIPListSize:
+				observer.EventIPListSize(typedEvt)
 			}
 		})
 	}

example.config.toml

@@ -170,7 +170,7 @@ download-concurrency = 2
 # You can provider links here (starts with https:// or http://) or
 # path to a local file, but in this case it should be absolute.
 urls = [
-    # "https://iplists.firehol.org/files/firehol_level1.netset",
+    "https://iplists.firehol.org/files/firehol_level1.netset",
     # "/local.file"
 ]
 # How often do we need to update a blocklist set.

go.mod

@@ -4,33 +4,35 @@ go 1.17
 
 require (
 	github.com/OneOfOne/xxhash v1.2.8
-	github.com/alecthomas/kong v0.2.19
-	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
+	github.com/alecthomas/kong v0.5.0
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gotd/td v0.34.0
 	github.com/jarcoal/httpmock v1.0.8
-	github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e
 	github.com/mccutchen/go-httpbin v1.1.1
-	github.com/panjf2000/ants/v2 v2.4.7
+	github.com/panjf2000/ants/v2 v2.4.8
 	github.com/pelletier/go-toml v1.9.4
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rs/zerolog v1.26.0
+	github.com/rs/zerolog v1.26.1
 	github.com/smira/go-statsd v1.3.2
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
-	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
+	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881
+	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
 	google.golang.org/protobuf v1.27.1 // indirect
 )
 
-require github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b
+require (
+	github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a
+	github.com/yl2chen/cidranger v1.0.2
+)
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -44,7 +46,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf // indirect
+	github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8 // indirect
 	github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect