diff --git a/.github/workflows/build-infra-dockers.yaml b/.github/workflows/build-infra-dockers.yaml index d785153a406..f899a3524ea 100644 --- a/.github/workflows/build-infra-dockers.yaml +++ b/.github/workflows/build-infra-dockers.yaml @@ -7,7 +7,7 @@ on: - infra-* env: - GO_VERSION: "~1.22.0" + GO_VERSION: "~1.22.3" CGO_ENABLED: "0" BUILD_USER: docker BUILD_HOST: github.syncthing.net diff --git a/.github/workflows/build-syncthing.yaml b/.github/workflows/build-syncthing.yaml index eba9709d789..1fa99dc07d9 100644 --- a/.github/workflows/build-syncthing.yaml +++ b/.github/workflows/build-syncthing.yaml @@ -12,7 +12,7 @@ env: # The go version to use for builds. We set check-latest to true when # installing, so we get the latest patch version that matches the # expression. - GO_VERSION: "~1.22.0" + GO_VERSION: "~1.22.3" # Optimize compatibility on the slow archictures. GO386: softfloat @@ -48,7 +48,7 @@ jobs: runner: ["windows-latest", "ubuntu-latest", "macos-latest"] # The oldest version in this list should match what we have in our go.mod. # Variables don't seem to be supported here, or we could have done something nice. - go: ["~1.21.7", "~1.22.0"] + go: ["~1.21.7", "~1.22.3"] runs-on: ${{ matrix.runner }} steps: - name: Set git to use LF diff --git a/Dockerfile.strelaypoolsrv b/Dockerfile.strelaypoolsrv index f7e2760e777..a0ad1fd6dde 100644 --- a/Dockerfile.strelaypoolsrv +++ b/Dockerfile.strelaypoolsrv @@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \ EXPOSE 8080 -RUN apk add --no-cache ca-certificates su-exec curl -ENV PUID=1000 PGID=1000 MAXMIND_KEY= - -RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv -USER 1000 - COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv -COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh -WORKDIR /var/strelaypoolsrv -ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"] +ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"] diff --git a/cmd/strelaypoolsrv/main.go b/cmd/strelaypoolsrv/main.go index fd3fd0d61f6..8724db93a92 100644 --- a/cmd/strelaypoolsrv/main.go +++ b/cmd/strelaypoolsrv/main.go @@ -21,12 +21,12 @@ import ( "time" lru "github.com/hashicorp/golang-lru/v2" - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto" "github.com/syncthing/syncthing/lib/assets" _ "github.com/syncthing/syncthing/lib/automaxprocs" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/relay/client" "github.com/syncthing/syncthing/lib/sync" @@ -102,11 +102,12 @@ var ( debug bool permRelaysFile string ipHeader string - geoipPath string proto string statsRefresh = time.Minute requestQueueLen = 64 requestProcessors = 8 + geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY") + geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) requests chan request @@ -132,34 +133,35 @@ func main() { flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays") flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays") flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.") - flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database") flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6") flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats") flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests") flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines") + flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database") flag.Parse() requests = make(chan request, requestQueueLen) + geoip := geoip.NewGeoLite2CityProvider(geoipAccountID, geoipLicenseKey, os.TempDir()) var listener net.Listener var err error if permRelaysFile != "" { - permanentRelays = loadRelays(permRelaysFile) + permanentRelays = loadRelays(permRelaysFile, geoip) } testCert = createTestCertificate() for i := 0; i < requestProcessors; i++ { - go requestProcessor() + go requestProcessor(geoip) } // Load relays from cache in the background. // Load them in a serial fashion to make sure any genuine requests // are not dropped. go func() { - for _, relay := range loadRelays(knownRelaysFile) { + for _, relay := range loadRelays(knownRelaysFile, geoip) { resultChan := make(chan result) requests <- request{relay, resultChan, nil} result := <-resultChan @@ -445,19 +447,19 @@ func handleRegister(w http.ResponseWriter, r *http.Request) { } } -func requestProcessor() { +func requestProcessor(geoip *geoip.Provider) { for request := range requests { if request.queueTimer != nil { request.queueTimer.ObserveDuration() } timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test")) - handleRelayTest(request) + handleRelayTest(request, geoip) timer.ObserveDuration() } } -func handleRelayTest(request request) { +func handleRelayTest(request request, geoip *geoip.Provider) { if debug { log.Println("Request for", request.relay) } @@ -470,7 +472,7 @@ func handleRelayTest(request request) { } stats := fetchStats(request.relay) - location := getLocation(request.relay.uri.Host) + location := getLocation(request.relay.uri.Host, geoip) mut.Lock() if stats != nil { @@ -543,7 +545,7 @@ func evict(relay *relay) func() { } } -func loadRelays(file string) []*relay { +func loadRelays(file string, geoip *geoip.Provider) []*relay { content, err := os.ReadFile(file) if err != nil { log.Println("Failed to load relays: " + err.Error()) @@ -567,7 +569,7 @@ func loadRelays(file string) []*relay { relays = append(relays, &relay{ URL: line, - Location: getLocation(uri.Host), + Location: getLocation(uri.Host, geoip), uri: uri, }) if debug { @@ -600,21 +602,16 @@ func createTestCertificate() tls.Certificate { return cert } -func getLocation(host string) location { +func getLocation(host string, geoip *geoip.Provider) location { timer := prometheus.NewTimer(locationLookupSeconds) defer timer.ObserveDuration() - db, err := geoip2.Open(geoipPath) - if err != nil { - return location{} - } - defer db.Close() addr, err := net.ResolveTCPAddr("tcp", host) if err != nil { return location{} } - city, err := db.City(addr.IP) + city, err := geoip.City(addr.IP) if err != nil { return location{} } diff --git a/cmd/ursrv/serve/serve.go b/cmd/ursrv/serve/serve.go index b7d51db3b51..dbc5044003c 100644 --- a/cmd/ursrv/serve/serve.go +++ b/cmd/ursrv/serve/serve.go @@ -17,6 +17,7 @@ import ( "log" "net" "net/http" + "os" "regexp" "sort" "strconv" @@ -26,20 +27,21 @@ import ( "unicode" _ "github.com/lib/pq" // PostgreSQL driver - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/text/cases" "golang.org/x/text/language" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/ur/contract" ) type CLI struct { - Debug bool `env:"UR_DEBUG"` - DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` - Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` - GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"` + Debug bool `env:"UR_DEBUG"` + DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` + Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` + GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"` + GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"` } //go:embed static @@ -190,9 +192,9 @@ func (cli *CLI) Run() error { } srv := &server{ - db: db, - debug: cli.Debug, - geoIPPath: cli.GeoIPPath, + db: db, + debug: cli.Debug, + geoip: geoip.NewGeoLite2CityProvider(cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir()), } http.HandleFunc("/", srv.rootHandler) http.HandleFunc("/newdata", srv.newDataHandler) @@ -213,9 +215,9 @@ func (cli *CLI) Run() error { } type server struct { - debug bool - db *sql.DB - geoIPPath string + debug bool + db *sql.DB + geoip *geoip.Provider cacheMut sync.Mutex cachedIndex []byte @@ -238,7 +240,7 @@ func (s *server) cacheRefresher() { } func (s *server) refreshCacheLocked() error { - rep := getReport(s.db, s.geoIPPath) + rep := getReport(s.db, s.geoip) buf := new(bytes.Buffer) err := tpl.Execute(buf, rep) if err != nil { @@ -492,15 +494,7 @@ type weightedLocation struct { Weight int `json:"weight"` } -func getReport(db *sql.DB, geoIPPath string) map[string]interface{} { - geoip, err := geoip2.Open(geoIPPath) - if err != nil { - log.Println("opening geoip db", err) - geoip = nil - } else { - defer geoip.Close() - } - +func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} { nodes := 0 countriesTotal := 0 var versions []string diff --git a/go.mod b/go.mod index d630181c02d..226734b7876 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/lib/pq v1.10.9 github.com/maruel/panicparse/v2 v2.3.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 + github.com/maxmind/geoipupdate/v6 v6.1.0 github.com/minio/sha256-simd v1.0.1 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 github.com/oschwald/geoip2-golang v1.9.0 @@ -52,6 +53,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect @@ -60,13 +62,13 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/onsi/ginkgo/v2 v2.17.1 // indirect github.com/onsi/gomega v1.31.1 // indirect diff --git a/go.sum b/go.sum index d0b7768497c..77d6cce00e7 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE= github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg= github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M= github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= +github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA= +github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= diff --git a/gui/default/assets/lang/lang-hi.json b/gui/default/assets/lang/lang-hi.json new file mode 100644 index 00000000000..2c63c085104 --- /dev/null +++ b/gui/default/assets/lang/lang-hi.json @@ -0,0 +1,2 @@ +{ +} diff --git a/lib/geoip/geoip.go b/lib/geoip/geoip.go new file mode 100644 index 00000000000..546c0bd6815 --- /dev/null +++ b/lib/geoip/geoip.go @@ -0,0 +1,121 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Package geoip provides an automatically updating MaxMind GeoIP2 database +// provider. +package geoip + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync" + "time" + + "github.com/maxmind/geoipupdate/v6/pkg/geoipupdate" + "github.com/oschwald/geoip2-golang" +) + +const maxDatabaseSize = 1 << 30 // 1 GiB, at the time of writing the database is about 95 MiB + +type Provider struct { + edition string + accountID int + licenseKey string + refreshInterval time.Duration + directory string + + mut sync.Mutex + db *geoip2.Reader + lastOpened time.Time +} + +// NewGeoLite2CityProvider returns a new GeoIP2 database provider for the +// GeoLite2-City database. The database will be stored in the given +// directory (which should exist) and refreshed every 7 days. +func NewGeoLite2CityProvider(accountID int, licenseKey string, directory string) *Provider { + return &Provider{ + edition: "GeoLite2-City", + accountID: accountID, + licenseKey: licenseKey, + refreshInterval: 7 * 24 * time.Hour, + directory: directory, + } +} + +func (p *Provider) City(ip net.IP) (*geoip2.City, error) { + p.mut.Lock() + + if p.db != nil && time.Since(p.lastOpened) > p.refreshInterval/2 { + p.db.Close() + p.db = nil + } + if p.db == nil { + var err error + p.db, err = p.open(context.Background()) + if err != nil { + p.mut.Unlock() + return nil, err + } + p.lastOpened = time.Now() + } + db := p.db + + p.mut.Unlock() + + return db.City(ip) +} + +// open returns a reader for the GeoIP2 database. If the database is not +// available locally, it will be downloaded. If the database is older than +// refreshInterval, it will be downloaded again. If the download fails, the +// existing database will be used. The returned reader must be closed by the +// caller in the normal manner. +func (p *Provider) open(ctx context.Context) (*geoip2.Reader, error) { + if p.licenseKey == "" { + return nil, errors.New("open: no license key set") + } + if p.edition == "" { + return nil, errors.New("open: no edition set") + } + + path := filepath.Join(p.directory, p.edition+".mmdb") + info, err := os.Stat(path) + if err != nil { + // No file exists, download it + err = p.download(ctx) + if err != nil { + return nil, fmt.Errorf("open: %w", err) + } + } else if time.Since(info.ModTime()) > p.refreshInterval { + // File is too old, attempt to download it. If it fails, use the old + // file. + _ = p.download(ctx) + } + + return geoip2.Open(path) +} + +func (p *Provider) download(ctx context.Context) error { + cfg := &geoipupdate.Config{ + URL: "https://updates.maxmind.com", + DatabaseDirectory: p.directory, + LockFile: filepath.Join(p.directory, "geoipupdate.lock"), + RetryFor: 5 * time.Minute, + Parallelism: 1, + AccountID: p.accountID, + LicenseKey: p.licenseKey, + EditionIDs: []string{p.edition}, + } + + if err := geoipupdate.NewClient(cfg).Run(ctx); err != nil { + return fmt.Errorf("download: %w", err) + } + return nil +} diff --git a/lib/geoip/geoip_test.go b/lib/geoip/geoip_test.go new file mode 100644 index 00000000000..f2098c6c090 --- /dev/null +++ b/lib/geoip/geoip_test.go @@ -0,0 +1,31 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package geoip + +import ( + "net" + "os" + "strconv" + "testing" +) + +func TestDownloadAndOpen(t *testing.T) { + acctID, _ := strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) + if acctID == 0 { + t.Skip("No account ID set") + } + license := os.Getenv("GEOIP_LICENSE_KEY") + if license == "" { + t.Skip("No license key set") + } + + p := NewGeoLite2CityProvider(acctID, license, t.TempDir()) + _, err := p.City(net.ParseIP("8.8.8.8")) + if err != nil { + t.Fatal(err) + } +} diff --git a/script/strelaypoolsrv-entrypoint.sh b/script/strelaypoolsrv-entrypoint.sh deleted file mode 100755 index 945c77d5b00..00000000000 --- a/script/strelaypoolsrv-entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -eu - -if [ "$MAXMIND_KEY" != "" ] ; then - curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_KEY}&suffix=tar.gz" \ - | tar --strip-components 1 -zxv -fi - -exec "$@"