Improve renewal rate limiting

features/features.go

@@ -27,6 +27,7 @@ const (
 	RandomDirectoryEntry
 	IPv6First
 	DirectoryMeta
+	AllowRenewalFirstRL
 )
 
 // List of features and their default value, protected by fMu
@@ -45,6 +46,7 @@ var features = map[FeatureFlag]bool{
 	RandomDirectoryEntry:     false,
 	IPv6First:                false,
 	DirectoryMeta:            false,
+	AllowRenewalFirstRL:      false,
 }
 
 var fMu = new(sync.RWMutex)

sa/sa.go

@@ -442,19 +442,19 @@ func (ssa *SQLStorageAuthority) countCertificatesByExactName(domain string, earl
 }
 
 // countCertificates returns, for a single domain, the count of
-// certificates issued in the given time range for that domain using the
+// non-renewal certificate issuances in the given time range for that domain using the
 // provided query, assumed to be either `countCertificatesExactSelect` or
-// `countCertificatesSelect`.
+// `countCertificatesSelect`. If the `AllowRenewalFirstRL` feature flag is set,
+// renewals of certificates issued within the same window are considered "free"
+// and are not counted.
 //
 // The highest count this function can return is 10,000. If there are more
 // certificates than that matching one of the provided domain names, it will return
 // TooManyCertificatesError.
 func (ssa *SQLStorageAuthority) countCertificates(domain string, earliest, latest time.Time, query string) (int, error) {
 	var count int64
 	const max = 10000
-	var serials []struct {
-		Serial string
-	}
+	var serials []string
 	_, err := ssa.dbMap.Select(
 		&serials,
 		query,
@@ -467,16 +467,45 @@ func (ssa *SQLStorageAuthority) countCertificates(domain string, earliest, lates
 	if err == sql.ErrNoRows {
 		return 0, nil
 	} else if err != nil {
-		return -1, err
+		return 0, err
 	} else if count > max {
 		return max, TooManyCertificatesError(fmt.Sprintf("More than %d issuedName entries for %s.", max, domain))
 	}
-	serialMap := make(map[string]struct{}, len(serials))
-	for _, s := range serials {
-		serialMap[s.Serial] = struct{}{}
-	}
 
-	return len(serialMap), nil
+	// If the `AllowRenewalFirstRL` feature flag is enabled then do the work
+	// required to discount renewals
+	if features.Enabled(features.AllowRenewalFirstRL) {
+		// If there are no serials found, short circuit since there isn't subsequent
+		// work to do
+		if len(serials) == 0 {
+			return 0, nil
+		}
+
+		// Find all FQDN Set Hashes with the serials from the issuedNames table that
+		// were visible within our search window
+		fqdnSets, err := ssa.getFQDNSetsBySerials(serials)
+		if err != nil {
+			return 0, err
+		}
+
+		// Using those FQDN Set Hashes, we can then find all of the non-renewal
+		// issuances with a second query against the fqdnSets table using the set
+		// hashes we know about
+		nonRenewalIssuances, err := ssa.getNewIssuancesByFQDNSet(fqdnSets, earliest)
+		if err != nil {
+			return 0, err
+		}
+		return nonRenewalIssuances, nil
+	} else {
+		// Otherwise, use the preexisting behaviour and deduplicate by serials
+		// returning a count of unique serials qignoring any potential renewals
+		serialMap := make(map[string]struct{}, len(serials))
+		for _, s := range serials {
+			serialMap[s] = struct{}{}
+		}
+
+		return len(serialMap), nil
+	}
 }
 
 // GetCertificate takes a serial number and returns the corresponding
@@ -1062,6 +1091,116 @@ func (ssa *SQLStorageAuthority) CountFQDNSets(ctx context.Context, window time.D
 	return count, err
 }
 
+// setHash is a []byte representing the hash of an FQDN Set
+type setHash []byte
+
+// getFQDNSetsBySerials finds the setHashes corresponding to a set of
+// certificate serials. These serials can be used to check whether any
+// certificates have been issued for the same set of names previously.
+func (ssa *SQLStorageAuthority) getFQDNSetsBySerials(serials []string) ([]setHash, error) {
+	var fqdnSets []setHash
+
+	// It is unexpected that this function would be called with no serials
+	if len(serials) == 0 {
+		err := fmt.Errorf("getFQDNSetsBySerials called with no serials")
+		ssa.log.AuditErr(err.Error())
+		return nil, err
+	}
+
+	qmarks := make([]string, len(serials))
+	params := make([]interface{}, len(serials))
+	for i, serial := range serials {
+		params[i] = serial
+		qmarks[i] = "?"
+	}
+	query := "SELECT setHash FROM fqdnSets " +
+		"WHERE serial IN (" + strings.Join(qmarks, ",") + ")"
+	_, err := ssa.dbMap.Select(
+		&fqdnSets,
+		query,
+		params...)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// The serials existed when we found them in issuedNames, they should continue
+	// to exist here. Otherwise an internal consistency violation occured and
+	// needs to be audit logged
+	if err == sql.ErrNoRows {
+		err := fmt.Errorf("getFQDNSetsBySerials returned no rows - internal consistency violation")
+		ssa.log.AuditErr(err.Error())
+		return nil, err
+	}
+	return fqdnSets, nil
+}
+
+// getNewIssuancesByFQDNSet returns a count of new issuances (renewals are not
+// included) for a given slice of fqdnSets that occurred after the earliest
+// parameter.
+func (ssa *SQLStorageAuthority) getNewIssuancesByFQDNSet(fqdnSets []setHash, earliest time.Time) (int, error) {
+	var results []struct {
+		Serial  string
+		SetHash setHash
+		Issued  time.Time
+	}
+
+	qmarks := make([]string, len(fqdnSets))
+	params := make([]interface{}, len(fqdnSets))
+	for i, setHash := range fqdnSets {
+		// We have to cast the setHash back to []byte here since the sql package
+		// isn't able to convert `sa.setHash` for the parameter value itself
+		params[i] = []byte(setHash)
+		qmarks[i] = "?"
+	}
+
+	query := "SELECT serial, setHash, issued FROM fqdnSets " +
+		"WHERE setHash IN (" + strings.Join(qmarks, ",") + ") " +
+		"ORDER BY setHash, issued"
+
+	// First, find the serial, sethash and issued date from the fqdnSets table for
+	// the given fqdn set hashes
+	_, err := ssa.dbMap.Select(
+		&results,
+		query,
+		params...)
+	if err != nil && err != sql.ErrNoRows {
+		return -1, err
+	}
+
+	// If there are no results we have encountered a major error and
+	// should loudly complain
+	if err == sql.ErrNoRows || len(results) == 0 {
+		ssa.log.AuditErr(fmt.Sprintf("Found no results from fqdnSets for setHashes known to exist: %#v", fqdnSets))
+		return 0, err
+	}
+
+	processedSetHashes := make(map[string]bool)
+	issuanceCount := 0
+	// Loop through each set hash result, counting issuances per unique set hash
+	// that are within the window specified by the earliest parameter
+	for _, result := range results {
+		key := string(result.SetHash)
+		// Skip set hashes that we have already processed - we only care about the
+		// first issuance
+		if processedSetHashes[key] {
+			continue
+		}
+
+		// If the issued date is before our earliest cutoff then skip it
+		if result.Issued.Before(earliest) {
+			continue
+		}
+
+		// Otherwise note the issuance and mark the set hash as processed
+		issuanceCount++
+		processedSetHashes[key] = true
+	}
+
+	// Return the count of how many non-renewal issuances there were
+	return issuanceCount, nil
+}
+
 // FQDNSetExists returns a bool indicating if one or more FQDN sets |names|
 // exists in the database
 func (ssa *SQLStorageAuthority) FQDNSetExists(ctx context.Context, names []string) (bool, error) {