Merge tag 'v1.13.1' into ios_try_syncthing_v1.13.1

v1.13.1

This release adds configuration options for min/max connections (see
https://docs.syncthing.net/advanced/option-connection-limits.html) and
moves the storage of pending devices/folders from the config to the
database (see
https://docs.syncthing.net/dev/rest.html#cluster-endpoints).

Bugfixes:

- syncthing#7005: panic: nil pointer dereference because (*db.Lowlevel)getMetaAndCheck() returns nil
- syncthing#7076: File not detected due to watching reporting events on old, deleted path
- syncthing#7165: Connections aren't actually closed when closing a protocol connection
- syncthing#7184: Spurious unexpected directory in untrusted folder
- syncthing#7197: In Recent Changes, the Device column is empty
- syncthing#7231: panic: deadlock detected at fmut

Enhancements:

- syncthing#7176: Active connections min/max setting
- syncthing#7178: Pending devices and folders should live in the database instead of configuration

Other issues:

- syncthing#6551: Test failure because of umask

AUTHORS

@@ -87,6 +87,7 @@ Dominik Heidler (asdil12) <dominik@heidler.eu>
 Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
 Elliot Huffman <thelich2@gmail.com>
 Emil Hessman (ceh) <emil@hessman.se>
+Eric Lesiuta <elesiuta@gmail.com>
 Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
 Evgeny Kuznetsov <evgeny@kuznetsov.md>
 Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
@@ -214,6 +215,7 @@ Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.
 Rahmi Pruitt <rjpruitt16@gmail.com>
 Richard Hartmann <RichiH@users.noreply.github.com>
 Robert Carosi (nov1n) <robert@carosi.nl>
+Roberto Santalla <roobre@users.noreply.github.com>
 Robin Schoonover <robin@cornhooves.org>
 Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
 Ross Smith II (rasa) <ross@smithii.com>

cmd/stevents/main.go

@@ -28,16 +28,21 @@ func main() {
 	log.SetFlags(0)
 
 	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
+	types := flag.String("types", "", "Filter for specific event types (comma-separated)")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 
 	if *apikey == "" {
 		log.Fatal("Must give -apikey argument")
 	}
+	var eventsArg string
+	if len(*types) > 0 {
+		eventsArg = "&events=" + *types
+	}
 
 	since := 0
 	for {
-		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d", *target, since), nil)
+		req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/rest/events?since=%d%s", *target, since, eventsArg), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stindex/dump.go

@@ -131,6 +131,23 @@ func dump(ldb backend.Backend) {
 				fmt.Printf(" V:%v\n", v)
 			}
 
+		case db.KeyTypePendingFolder:
+			device := binary.BigEndian.Uint32(key[1:])
+			folder := string(key[5:])
+			var of db.ObservedFolder
+			of.Unmarshal(it.Value())
+			fmt.Printf("[pendingFolder] D:%d F:%s V:%v\n", device, folder, of)
+
+		case db.KeyTypePendingDevice:
+			device := "<invalid>"
+			dev, err := protocol.DeviceIDFromBytes(key[1:])
+			if err == nil {
+				device = dev.String()
+			}
+			var od db.ObservedDevice
+			od.Unmarshal(it.Value())
+			fmt.Printf("[pendingDevice] D:%v V:%v\n", device, od)
+
 		default:
 			fmt.Printf("[??? %d]\n  %x\n  %x\n", key[0], key, it.Value())
 		}

cmd/strelaypoolsrv/main.go

@@ -6,9 +6,11 @@ import (
 	"compress/gzip"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"io"
 	"io/ioutil"
 	"log"
@@ -208,6 +210,7 @@ func main() {
 		tlsCfg := &tls.Config{
 			Certificates: []tls.Certificate{cert},
 			MinVersion:   tls.VersionTLS10, // No SSLv3
+			ClientAuth:   tls.RequestClientCert,
 			CipherSuites: []uint16{
 				// No RC4
 				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
@@ -338,6 +341,12 @@ func handleGetRequest(rw http.ResponseWriter, r *http.Request) {
 }
 
 func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var relayCert *x509.Certificate
+	if r.TLS != nil && len(r.TLS.PeerCertificates) > 0 {
+		relayCert = r.TLS.PeerCertificates[0]
+		log.Printf("Got TLS cert from relay server")
+	}
+
 	var newRelay relay
 	err := json.NewDecoder(r.Body).Decode(&newRelay)
 	r.Body.Close()
@@ -359,6 +368,16 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if relayCert != nil {
+		advertisedId := uri.Query().Get("id")
+		idFromCert := protocol.NewDeviceID(relayCert.Raw).String()
+		if advertisedId != idFromCert {
+			log.Println("Warning: Relay server requested to join with an ID different from the join request, rejecting")
+			http.Error(w, "mismatched advertised id and join request cert", http.StatusBadRequest)
+			return
+		}
+	}
+
 	host, port, err := net.SplitHostPort(uri.Host)
 	if err != nil {
 		if debug {
@@ -379,7 +398,7 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
 	if ip == nil || ip.IsUnspecified() {
 		uri.Host = net.JoinHostPort(rhost, port)
 		newRelay.URL = uri.String()
-	} else if host != rhost {
+	} else if host != rhost && relayCert == nil {
 		if debug {
 			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
 		}

cmd/strelaysrv/main.go

@@ -247,7 +247,7 @@ func main() {
 	for _, pool := range pools {
 		pool = strings.TrimSpace(pool)
 		if len(pool) > 0 {
-			go poolHandler(pool, uri, mapping)
+			go poolHandler(pool, uri, mapping, cert)
 		}
 	}
 

cmd/strelaysrv/pool.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"io/ioutil"
 	"log"
@@ -16,7 +17,7 @@ const (
 	httpStatusEnhanceYourCalm = 429
 )
 
-func poolHandler(pool string, uri *url.URL, mapping mapping) {
+func poolHandler(pool string, uri *url.URL, mapping mapping, ownCert tls.Certificate) {
 	if debug {
 		log.Println("Joining", pool)
 	}
@@ -31,7 +32,24 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
 			uriCopy.String(),
 		})
 
-		resp, err := httpClient.Post(pool, "application/json", &b)
+		poolUrl, err := url.Parse(pool)
+		if err != nil {
+			log.Printf("Could not parse pool url '%s': %v", pool, err)
+		}
+
+		client := http.DefaultClient
+		if poolUrl.Scheme == "https" {
+			// Sent our certificate in join request
+			client = &http.Client{
+				Transport: &http.Transport{
+					TLSClientConfig: &tls.Config{
+						Certificates: []tls.Certificate{ownCert},
+					},
+				},
+			}
+		}
+
+		resp, err := client.Post(pool, "application/json", &b)
 		if err != nil {
 			log.Printf("Error joining pool %v: HTTP request: %v", pool, err)
 			time.Sleep(time.Minute)

cmd/syncthing/main.go

@@ -39,10 +39,10 @@ import (
 	"github.com/syncthing/syncthing/lib/logger"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/svcutil"
 	"github.com/syncthing/syncthing/lib/syncthing"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
-	"github.com/syncthing/syncthing/lib/util"
 
 	"github.com/pkg/errors"
 )
@@ -325,7 +325,7 @@ func mainCmdline() int {
 	}
 	if err != nil {
 		l.Warnln("Command line options:", err)
-		return util.ExitError.AsInt()
+		return svcutil.ExitError.AsInt()
 	}
 
 	if options.logFile == "default" || options.logFile == "" {
@@ -362,7 +362,7 @@ func mainCmdline() int {
 		)
 		if err != nil {
 			l.Warnln("Error reading device ID:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 
 		fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
@@ -372,30 +372,30 @@ func mainCmdline() int {
 	if options.browserOnly {
 		if err := openGUI(protocol.EmptyDeviceID); err != nil {
 			l.Warnln("Failed to open web UI:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 		return 0
 	}
 
 	if options.generateDir != "" {
 		if err := generate(options.generateDir); err != nil {
 			l.Warnln("Failed to generate config and keys:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 		return 0
 	}
 
 	// Ensure that our home directory exists.
 	if err := EnsureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil {
 		l.Warnln("Failure on home directory:", err)
-		return util.ExitError.AsInt()
+		return svcutil.ExitError.AsInt()
 	}
 
 	if options.upgradeTo != "" {
 		err := upgrade.ToURL(options.upgradeTo)
 		if err != nil {
 			l.Warnln("Error while Upgrading:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 		l.Infoln("Upgraded from", options.upgradeTo)
 		return 0
@@ -426,13 +426,13 @@ func mainCmdline() int {
 			return exitCodeForUpgrade(err)
 		}
 		l.Infof("Upgraded to %q", release.Tag)
-		return util.ExitUpgrade.AsInt()
+		return svcutil.ExitUpgrade.AsInt()
 	}
 
 	if options.resetDatabase {
 		if err := resetDB(); err != nil {
 			l.Warnln("Resetting database:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 		l.Infoln("Successfully reset database - it will be rebuilt after next start.")
 		return 0
@@ -614,7 +614,7 @@ func SyncthingMain(runtimeOptions RuntimeOptions) int {
 	cfg, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, runtimeOptions.allowNewerConfig, noDefaultFolder)
 	if err != nil {
 		l.Warnln("Failed to initialize config:", err)
-		return util.ExitError.AsInt()
+		return svcutil.ExitError.AsInt()
 	}
 
 	// Candidate builds should auto upgrade. Make sure the option is set,
@@ -660,7 +660,7 @@ func SyncthingMain(runtimeOptions RuntimeOptions) int {
 			}
 		} else {
 			l.Infof("Upgraded to %q, exiting now.", release.Tag)
-			return util.ExitUpgrade.AsInt()
+			return svcutil.ExitUpgrade.AsInt()
 		}
 	}
 
@@ -685,7 +685,11 @@ func SyncthingMain(runtimeOptions RuntimeOptions) int {
 		appOpts.DBIndirectGCInterval = dur
 	}
 
-	app := syncthing.New(cfg, ldb, evLogger, cert, appOpts)
+	app, err := syncthing.New(cfg, ldb, evLogger, cert, appOpts)
+	if err != nil {
+		l.Warnln("Failed to start Syncthing:", err)
+		os.Exit(svcutil.ExitError.AsInt())
+	}
 
 	if autoUpgradePossible {
 		go autoUpgrade(cfg, app, evLogger)
@@ -701,18 +705,18 @@ func SyncthingMain(runtimeOptions RuntimeOptions) int {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			l.Warnln("Creating profile:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 		if err := pprof.StartCPUProfile(f); err != nil {
 			l.Warnln("Starting profile:", err)
-			return util.ExitError.AsInt()
+			return svcutil.ExitError.AsInt()
 		}
 	}
 
 	go standbyMonitor(app, cfg)
 
 	if err := app.Start(); err != nil {
-		return util.ExitError.AsInt()
+		return svcutil.ExitError.AsInt()
 	}
 
 	RunningApp = app
@@ -728,7 +732,7 @@ func SyncthingMain(runtimeOptions RuntimeOptions) int {
 
 	status := app.Wait()
 
-	if status == util.ExitError {
+	if status == svcutil.ExitError {
 		l.Warnln("Syncthing stopped with error:", app.Error())
 	}
 
@@ -751,7 +755,7 @@ func setupSignalHandling(app *syncthing.App) {
 	signal.Notify(restartSign, sigHup)
 	go func() {
 		<-restartSign
-		app.Stop(util.ExitRestart)
+		app.Stop(svcutil.ExitRestart)
 	}()
 
 	// Exit with "success" code (no restart) on INT/TERM
@@ -760,7 +764,7 @@ func setupSignalHandling(app *syncthing.App) {
 	signal.Notify(stopSign, os.Interrupt, sigTerm)
 	go func() {
 		<-stopSign
-		app.Stop(util.ExitSuccess)
+		app.Stop(svcutil.ExitSuccess)
 	}()
 }
 
@@ -797,7 +801,7 @@ func auditWriter(auditFile string) io.Writer {
 		fd, err = os.OpenFile(auditFile, auditFlags, 0600)
 		if err != nil {
 			l.Warnln("Audit:", err)
-			os.Exit(util.ExitError.AsInt())
+			os.Exit(svcutil.ExitError.AsInt())
 		}
 		auditDest = auditFile
 	}
@@ -847,7 +851,7 @@ func standbyMonitor(app *syncthing.App, cfg config.Wrapper) {
 			// things a moment to stabilize.
 			time.Sleep(restartDelay)
 
-			app.Stop(util.ExitRestart)
+			app.Stop(svcutil.ExitRestart)
 			return
 		}
 		now = time.Now()
@@ -917,7 +921,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger)
 		sub.Unsubscribe()
 		l.Warnf("Automatically upgraded to version %q. Restarting in 1 minute.", rel.Tag)
 		time.Sleep(time.Minute)
-		app.Stop(util.ExitUpgrade)
+		app.Stop(svcutil.ExitUpgrade)
 		return
 	}
 }
@@ -1005,13 +1009,13 @@ func setPauseState(cfg config.Wrapper, paused bool) {
 	}
 	if _, err := cfg.Replace(raw); err != nil {
 		l.Warnln("Cannot adjust paused state:", err)
-		os.Exit(util.ExitError.AsInt())
+		os.Exit(svcutil.ExitError.AsInt())
 	}
 }
 
 func exitCodeForUpgrade(err error) int {
 	if _, ok := err.(*errNoUpgrade); ok {
-		return util.ExitNoUpgradeAvailable.AsInt()
+		return svcutil.ExitNoUpgradeAvailable.AsInt()
 	}
-	return util.ExitError.AsInt()
+	return svcutil.ExitError.AsInt()
 }