rpc_fetch_crl.go

// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package certconfig

import (
	"context"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	ds "go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/auth"

	"go.chromium.org/luci/tokenserver/api/admin/v1"
)

// List of OAuth scopes to use for token sent to CRL endpoint if config doesn't
// specify 'oauth_scopes' field.
var crlFetchDefaultScopes = []string{
	"https://www.googleapis.com/auth/userinfo.email",
}

// FetchCRLRPC implements CertificateAuthorities.FetchCRL RPC method.
type FetchCRLRPC struct {
}

// FetchCRL makes the server fetch a CRL for some CA.
func (r *FetchCRLRPC) FetchCRL(c context.Context, req *admin.FetchCRLRequest) (*admin.FetchCRLResponse, error) {
	// Grab a corresponding CA entity. It contains URL of CRL to fetch.
	ca := &CA{CN: req.Cn}
	switch err := ds.Get(c, ca); {
	case err == ds.ErrNoSuchEntity:
		return nil, status.Errorf(codes.NotFound, "no such CA %q", ca.CN)
	case err != nil:
		return nil, status.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Grab CRL URL from the CA config.
	cfg, err := ca.ParseConfig()
	if err != nil {
		return nil, status.Errorf(codes.Internal, "broken CA config in the datastore - %s", err)
	}
	if cfg.CrlUrl == "" {
		return nil, status.Errorf(codes.NotFound, "CA %q doesn't have CRL defined", ca.CN)
	}

	// Grab info about last processed CRL, if any.
	crl := &CRL{Parent: ds.KeyForObj(c, ca)}
	if err = ds.Get(c, crl); err != nil && err != ds.ErrNoSuchEntity {
		return nil, status.Errorf(codes.Internal, "datastore error - %s", err)
	}

	// Fetch latest CRL blob.
	logging.Infof(c, "Fetching CRL for %q from %s", ca.CN, cfg.CrlUrl)
	knownETag := crl.LastFetchETag
	if req.Force {
		knownETag = ""
	}
	fetchCtx, _ := clock.WithTimeout(c, time.Minute)
	crlDer, newEtag, err := fetchCRL(fetchCtx, cfg, knownETag)
	switch {
	case transient.Tag.In(err):
		return nil, status.Errorf(codes.Internal, "transient error when fetching CRL - %s", err)
	case err != nil:
		return nil, status.Errorf(codes.Unknown, "can't fetch CRL - %s", err)
	}

	// No changes?
	if knownETag != "" && knownETag == newEtag {
		logging.Infof(c, "No changes to CRL (etag is %s), skipping", knownETag)
	} else {
		logging.Infof(c, "Fetched CRL size is %d bytes, etag is %s", len(crlDer), newEtag)
		crl, err = validateAndStoreCRL(c, crlDer, newEtag, ca, crl)
		switch {
		case transient.Tag.In(err):
			return nil, status.Errorf(codes.Internal, "transient error when storing CRL - %s", err)
		case err != nil:
			return nil, status.Errorf(codes.Unknown, "bad CRL - %s", err)
		}
	}

	return &admin.FetchCRLResponse{CrlStatus: crl.GetStatusProto()}, nil
}

////////////////////////////////////////////////////////////////////////////////

// fetchCRL fetches a blob with pem or der-encoded CRL from the CRL endpoint.
//
// It knows how to use ETag headers to avoid fetching already known data.
// May return transient and fatal errors.
//
// It attempts to parse the fetched data as PEM first. If successful, returns
// decoded DER block. If the data doesn't look like a PEM block, returns it as
// is, assuming it is already in the DER form.
func fetchCRL(c context.Context, cfg *admin.CertificateAuthorityConfig, knownETag string) (blob []byte, etag string, err error) {
	// Pick auth or non-auth transport.
	var transport http.RoundTripper
	if cfg.UseOauth || len(cfg.OauthScopes) != 0 {
		var scopes []string
		if len(cfg.OauthScopes) != 0 {
			scopes = cfg.OauthScopes
		} else {
			scopes = crlFetchDefaultScopes
		}
		transport, err = auth.GetRPCTransport(c, auth.AsSelf, auth.WithScopes(scopes...))
	} else {
		transport, err = auth.GetRPCTransport(c, auth.NoAuth)
	}
	if err != nil {
		return nil, "", err
	}

	// Send the request with ETag related headers.
	req, err := http.NewRequest("GET", cfg.CrlUrl, nil)
	if err != nil {
		return nil, "", err
	}
	if knownETag != "" {
		req.Header.Set("If-None-Match", knownETag)
	}
	cl := http.Client{Transport: transport}
	resp, err := cl.Do(req)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	defer resp.Body.Close()

	// Nothing new?
	if resp.StatusCode == http.StatusNotModified {
		if knownETag == "" {
			return nil, "", errors.New("unexpected 304 status, no etag header was sent")
		}
		return nil, knownETag, nil
	}

	// Read the body in its entirety (plus new etag, if any).
	blob, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", transient.Tag.Apply(err)
	}
	etag = resp.Header.Get("ETag")

	// Transient error?
	if resp.StatusCode >= http.StatusInternalServerError {
		logging.Warningf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", errors.Reason("server replied with HTTP %d", resp.StatusCode).
			Tag(transient.Tag).Err()
	}

	// Something we don't support or expect?
	if resp.StatusCode != http.StatusOK {
		logging.Errorf(c, "GET %s - HTTP %d; %q", cfg.CrlUrl, resp.StatusCode, string(blob))
		return nil, "", fmt.Errorf("unexpected status HTTP %d", resp.StatusCode)
	}

	// Attempt to parse PEM. It's fine if it fails (block == nil), in that case we
	// assume the body is already in the DER form. If it's not really DER, it will
	// fail the subsequent validation step.
	block, rest := pem.Decode(blob)
	if block == nil {
		return blob, etag, nil
	}

	// It is a PEM, but maybe it's not a CRL, or it has multiple blocks. Reject
	// these with fatal errors.
	if block.Type != "X509 CRL" {
		logging.Errorf(c, "GET %s - bad CRL, expecting %q PEM, got %q", cfg.CrlUrl, "X509 CRL", block.Type)
		return nil, "", fmt.Errorf("got %q PEM, not a CRL PEM", block.Type)
	}
	if len(rest) != 0 {
		logging.Errorf(c, "GET %s - bad CRL, more than one PEM block", cfg.CrlUrl)
		return nil, "", fmt.Errorf("bad CRL, more than one PEM block")
	}

	// Have a valid X509 CRL PEM, return its decoded body.
	return block.Bytes, etag, nil
}

// validateAndStoreCRL handles incoming CRL blob fetched by 'fetchCRL'.
func validateAndStoreCRL(c context.Context, crlDer []byte, etag string, ca *CA, prev *CRL) (*CRL, error) {
	// Make sure it is signed by the CA.
	caCert, err := x509.ParseCertificate(ca.Cert)
	if err != nil {
		return nil, fmt.Errorf("cert in the datastore is broken - %s", err)
	}
	crl, err := x509.ParseDERCRL(crlDer)
	if err != nil {
		return nil, fmt.Errorf("not a valid x509 CRL - %s", err)
	}
	if err = caCert.CheckCRLSignature(crl); err != nil {
		return nil, fmt.Errorf("CRL is not signed by the CA - %s", err)
	}

	// The CRL is peachy. Update a sharded set of all revoked certs.
	logging.Infof(c, "CRL last updated %s", crl.TBSCertList.ThisUpdate)
	logging.Infof(c, "Found %d entries in the CRL", len(crl.TBSCertList.RevokedCertificates))
	if err = UpdateCRLSet(c, ca.CN, CRLShardCount, crl); err != nil {
		return nil, err
	}
	logging.Infof(c, "All CRL entries stored")

	// Update the CRL entity. Use EntityVersion to make sure we are not
	// overwriting someone else's changes.
	var updated *CRL
	err = ds.RunInTransaction(c, func(c context.Context) error {
		entity := *prev
		if err := ds.Get(c, &entity); err != nil && err != ds.ErrNoSuchEntity {
			return err
		}
		if entity.EntityVersion != prev.EntityVersion {
			return fmt.Errorf("CRL for %q was updated concurrently while we were working on it", ca.CN)
		}
		entity.EntityVersion++
		entity.LastUpdateTime = crl.TBSCertList.ThisUpdate.UTC()
		entity.LastFetchTime = clock.Now(c).UTC()
		entity.LastFetchETag = etag
		entity.RevokedCertsCount = len(crl.TBSCertList.RevokedCertificates)

		updated = &entity // used outside of this function
		toPut := []interface{}{updated}

		// Mark CA entity as ready for usage.
		curCA := CA{CN: ca.CN}
		switch err := ds.Get(c, &curCA); {
		case err == ds.ErrNoSuchEntity:
			return fmt.Errorf("CA entity for %q is unexpectedly gone", ca.CN)
		case err != nil:
			return err
		}
		if !curCA.Ready {
			logging.Infof(c, "CA %q is ready now", curCA.CN)
			curCA.Ready = true
			toPut = append(toPut, &curCA)
		}
		return ds.Put(c, toPut)
	}, nil)
	if err != nil {
		return nil, transient.Tag.Apply(err)
	}

	logging.Infof(c, "CRL for %q is updated, entity version is %d", ca.CN, updated.EntityVersion)
	return updated, nil
}