server/camlistored: use Let's Encrypt

Or to be more precise, golang.org/x/crypto/acme/autocert

The default behaviour regarding HTTPS certificates changes as such:

1) If the high-level config does not specify a certificate, the
low-level config used to be generated with a default certificate path.
This is no longer the case.
2) If the low-level config does not specify a certificate, we used to
generate self-signed ones at the default path. This is no longer always
the case. We only do this if our hostname does not look like an FQDN,
otherwise we try Let's Encrypt.
3) As a result, if the high-level config does not specify a certificate,
and the hostname looks like an FQDN, it is no longer the case that we'll
generate a self-signed. Let's Encrypt will be tried instead.

To sum up, the new rules are:
If cert/key files are specified, and found, use them.
If cert/key files are specified, not found, and the default values,
generate them (self-signed CA used as a cert), and use them.
If cert/key files are not specified, use Let's Encrypt if we have an
FQDN, otherwise generate self-signed.

Regarding cert caching:

On non-GCE, store the autocert cache dir in
osutil.CamliConfigDir()/letsencrypt.cache
On GCE, store in /tmp/camli-letsencrypt.cache

Fixes #701
Fixes #859

Change-Id: Id78a9c6f113fa93e38d690033c10a749d1844ea6

app/publisher/main.go

@@ -219,8 +219,13 @@ func main() {
 	ws := webserver.New()
 	ws.Logger = logger
 	ws.Handle("/", ph)
+	// TODO(mpl): let's encrypt, etc.
+	// https://camlistore-review.googlesource.com/8647
 	if conf.HTTPSCert != "" && conf.HTTPSKey != "" {
-		ws.SetTLS(conf.HTTPSCert, conf.HTTPSKey)
+		ws.SetTLS(webserver.TLSSetup{
+			CertFile: conf.HTTPSCert,
+			KeyFile:  conf.HTTPSKey,
+		})
 	}
 	if err := ws.Listen(listenAddr); err != nil {
 		logger.Fatalf("Listen: %v", err)

pkg/netutil/netutil.go

@@ -163,3 +163,21 @@ func RandPort() (int, error) {
 func HasPort(s string) bool {
 	return strings.LastIndex(s, ":") > strings.LastIndex(s, "]")
 }
+
+// IsFQDN reports whether domain looks like a fully qualified domain name.
+func IsFQDN(domain string) bool {
+	// TODO(mpl): there's probably a regexp for all this...
+	if domain == "localhost" {
+		return false
+	}
+	if !strings.Contains(domain, ".") {
+		return false
+	}
+	if strings.Contains(domain, "/") {
+		return false
+	}
+	if net.ParseIP(domain) != nil {
+		return false
+	}
+	return true
+}

pkg/osutil/paths.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"camlistore.org/pkg/buildinfo"
+	"camlistore.org/pkg/env"
 	"go4.org/jsonconfig"
 )
 
@@ -215,6 +216,15 @@ func DefaultTLSKey() string {
 	return filepath.Join(CamliConfigDir(), "tls.key")
 }
 
+// DefaultLetsEncryptCache returns the path to the default Let's Encrypt cache
+// directory (or file, depending on the ACME implementation).
+func DefaultLetsEncryptCache() string {
+	if env.OnGCE() {
+		return "/tmp/camli-letsencrypt.cache"
+	}
+	return filepath.Join(CamliConfigDir(), "letsencrypt.cache")
+}
+
 // NewJSONConfigParser returns a jsonconfig.ConfigParser with its IncludeDirs
 // set with CamliConfigDir and the contents of CAMLI_INCLUDE_PATH.
 func NewJSONConfigParser() *jsonconfig.ConfigParser {

pkg/serverinit/genconfig.go

@@ -810,9 +810,6 @@ func (b *lowBuilder) build() (*Config, error) {
 		if conf.HTTPSCert != "" {
 			low["httpsCert"] = conf.HTTPSCert
 			low["httpsKey"] = conf.HTTPSKey
-		} else {
-			low["httpsCert"] = osutil.DefaultTLSCert()
-			low["httpsKey"] = osutil.DefaultTLSKey()
 		}
 	}
 

pkg/types/clientconfig/config.go

@@ -97,12 +97,7 @@ func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) {
 		}
 	}
 
-	param = "httpsCert"
-	httpsCert := serverConfig.OptionalString(param, "")
-	if https && httpsCert == "" {
-		return missingConfig(param)
-	}
-
+	httpsCert := serverConfig.OptionalString("httpsCert", "")
 	// TODO(mpl): See if we can detect that the cert is not self-signed,and in
 	// that case not add it to the trustedCerts
 	var trustedList []string
@@ -117,7 +112,6 @@ func GenerateClientConfig(serverConfig jsonconfig.Obj) (*Config, error) {
 		}
 		trustedList = []string{sig}
 	}
-
 	param = "prefixes"
 	prefixes := serverConfig.OptionalObject(param)
 	if len(prefixes) == 0 {

pkg/webserver/webserver.go

@@ -55,6 +55,7 @@ type Server struct {
 
 	enableTLS               bool
 	tlsCertFile, tlsKeyFile string
+	certManager             func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) // tlsCertFile takes precedence
 
 	mu   sync.Mutex
 	reqs int64
@@ -84,10 +85,21 @@ func (s *Server) fatalf(format string, v ...interface{}) {
 	log.Fatalf(format, v...)
 }
 
-func (s *Server) SetTLS(certFile, keyFile string) {
+// TLSSetup specifies how the server gets its TLS certificate.
+type TLSSetup struct {
+	// Certfile is the path to the TLS certificate file. It takes precedence over CertManager.
+	CertFile string
+	// KeyFile is the path to the TLS key file.
+	KeyFile string
+	// CertManager is the tls.GetCertificate of the tls Config. But CertFile takes precedence.
+	CertManager func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error)
+}
+
+func (s *Server) SetTLS(setup TLSSetup) {
 	s.enableTLS = true
-	s.tlsCertFile = certFile
-	s.tlsKeyFile = keyFile
+	s.certManager = setup.CertManager
+	s.tlsCertFile = setup.CertFile
+	s.tlsKeyFile = setup.KeyFile
 }
 
 func (s *Server) ListenURL() string {
@@ -171,20 +183,31 @@ func (s *Server) Listen(addr string) error {
 		s.printf("Starting to listen on %s\n", base)
 	}
 
-	if s.enableTLS {
+	doEnableTLS := func() error {
 		config := &tls.Config{
 			Rand:       rand.Reader,
 			Time:       time.Now,
 			NextProtos: []string{http2.NextProtoTLS, "http/1.1"},
 			MinVersion: tls.VersionTLS12,
 		}
-		config.Certificates = make([]tls.Certificate, 1)
+		if s.tlsCertFile == "" && s.certManager != nil {
+			config.GetCertificate = s.certManager
+			s.listener = tls.NewListener(s.listener, config)
+			return nil
+		}
 
+		config.Certificates = make([]tls.Certificate, 1)
 		config.Certificates[0], err = loadX509KeyPair(s.tlsCertFile, s.tlsKeyFile)
 		if err != nil {
 			return fmt.Errorf("Failed to load TLS cert: %v", err)
 		}
 		s.listener = tls.NewListener(s.listener, config)
+		return nil
+	}
+	if s.enableTLS {
+		if err := doEnableTLS(); err != nil {
+			return err
+		}
 	}
 
 	if doLog && strings.HasSuffix(base, ":0") {

server/camlistored/camlistored.go

@@ -84,6 +84,7 @@ import (
 	"go4.org/wkfs"
 
 	"cloud.google.com/go/logging"
+	"golang.org/x/crypto/acme/autocert"
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/option"
@@ -185,23 +186,10 @@ func loadConfig(arg string) (conf *serverinit.Config, isNewConfig bool, err erro
 	return
 }
 
-// 1) We do not want to force the user to buy a cert.
-// 2) We still want our client (camput) to be able to
-// verify the cert's authenticity.
-// 3) We want to avoid MITM attacks and warnings in
-// the browser.
-// Using a simple self-signed won't do because of 3),
-// as Chrome offers no way to set a self-signed as
-// trusted when importing it. (same on android).
-// We could have created a self-signed CA (that we
-// would import in the browsers) and create another
-// cert (signed by that CA) which would be the one
-// used in camlistore.
-// We're doing even simpler: create a self-signed
-// CA and directly use it as a self-signed cert
-// (and install it as a CA in the browsers).
-// 2) is satisfied by doing our own checks,
-// See pkg/client
+// If cert/key files are specified, and found, use them.
+// If cert/key files are specified, not found, and the default values, generate
+// them (self-signed CA used as a cert), and use them.
+// If cert/key files are not specified, use Let's Encrypt.
 func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string) {
 	cert, key := config.OptionalString("httpsCert", ""), config.OptionalString("httpsKey", "")
 	if !config.OptionalBool("https", true) {
@@ -229,8 +217,21 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string)
 			}
 		}
 	}
-	// Always generate new certificates if the config's httpsCert and httpsKey are empty.
 	if cert == "" && key == "" {
+		// Use Let's Encrypt if no files are specified, and we have a usable hostname.
+		if netutil.IsFQDN(hostname) {
+			m := autocert.Manager{
+				Prompt:     autocert.AcceptTOS,
+				HostPolicy: autocert.HostWhitelist(hostname),
+				Cache:      autocert.DirCache(osutil.DefaultLetsEncryptCache()),
+			}
+			log.Print("TLS enabled, with Let's Encrypt")
+			ws.SetTLS(webserver.TLSSetup{
+				CertManager: m.GetCertificate,
+			})
+			return
+		}
+		// Otherwise generate new certificates
 		sig, err := httputil.GenSelfTLSFiles(hostname, defCert, defKey)
 		if err != nil {
 			exitf("Could not generate self signed creds: %q", err)
@@ -248,7 +249,10 @@ func setupTLS(ws *webserver.Server, config *serverinit.Config, hostname string)
 		exitf("certificate error: %v", err)
 	}
 	log.Printf("TLS enabled, with SHA-256 certificate fingerprint: %v", sig)
-	ws.SetTLS(cert, key)
+	ws.SetTLS(webserver.TLSSetup{
+		CertFile: cert,
+		KeyFile:  key,
+	})
 }
 
 var osExit = os.Exit // testing hook