Merge pull request #640 from mozilla-services/fix-623-dedupe-monitor-…

…soft-notifications

Fix 623 dedupe monitor soft notifications

tools/autograph-monitor/README.md

@@ -7,16 +7,30 @@ correctly.
 
 Monitor runs standalone or as a lambda function.
 
-Three environment variables are required:
+It accepts two required environment variables:
 
-* AUTOGRAPH_URL is the address of the autograph endpoint
-* AUTOGRAPH_KEY is the monitoring API key
-* AUTOGRAPH_ENV optionally sets the root of the firefox pki to a pre-defined value
-    depending on the environment monitor is running in.
-    Acceptable values are "stage" and "prod".
-    When unset, this will use a default value for local development.
+* `AUTOGRAPH_URL` the address of the autograph endpoint
+* `AUTOGRAPH_KEY` the monitoring API key
 
-With these env vars set, running monitor is as easy as:
+And additional optional environment variables:
+
+* `AUTOGRAPH_ENV` sets the root of the Firefox PKI to a pre-defined
+  value depending on the environment monitor is running in.
+  Acceptable values are "stage" and "prod".  When unset, this will use
+  a default value for local development. The variables it uses can be
+  found in `constants.go`.
+
+* `AUTOGRAPH_ROOT_HASH` sets the root hash monitor to verify addon and
+  content signature against (as used in
+  `run-monitor-with-root-hash.sh`).
+
+* `AUTOGRAPH_PD_ROUTING_KEY` is an integration key for the pagerduty
+  events v2 API. When present the monitor will trigger and resolve
+  alerts for warnings like a content signature certificate expiring in
+  30 days.
+
+
+An example run looks like:
 
 ```bash
 AUTOGRAPH_URL=http://localhost:8000/ \
@@ -54,7 +68,3 @@ AUTOGRAPH_KEY=19zd4w3xirb5syjgdx8atq6g91m03bdsmzjifs2oddivswlu9qs \
 2019/04/09 09:41:13 Response 14 from signer "dummyrsapss" passes verification
 2019/04/09 09:41:13 All signature responses passed, monitoring OK
 ```
-
-The additional environmental variable `AUTOGRAPH_ROOT_HASH` sets the
-root hash monitor will use to verify addon and content signature (as
-used in `run-monitor-with-root-hash.sh`).

tools/autograph-monitor/contentsignaturepki.go

@@ -30,7 +30,7 @@ var ignoredCerts = map[string]bool{
 // to verify the sig. Otherwise, use the PublicKey contained in the response.
 //
 // If the signature passes, verify the chain of trust maps.
-func verifyContentSignature(rootHash string, response formats.SignatureResponse) error {
+func verifyContentSignature(notifier Notifier, rootHash string, response formats.SignatureResponse) error {
 	var (
 		key   *ecdsa.PublicKey
 		err   error
@@ -60,7 +60,7 @@ func verifyContentSignature(rootHash string, response formats.SignatureResponse)
 		return fmt.Errorf("Signature verification failed")
 	}
 	if certs != nil {
-		err = verifyCertChain(rootHash, certs)
+		err = verifyCertChain(notifier, rootHash, certs)
 		if err != nil {
 			// check if we should ignore this cert
 			if _, ok := ignoredCerts[certs[0].Subject.CommonName]; ok {
@@ -96,7 +96,7 @@ func parsePublicKeyFromB64(b64PubKey string) (pubkey *ecdsa.PublicKey, err error
 // When an AWS SNS topic is configured it also sends a soft
 // notification for each cert expiring in less than 30 days.
 //
-func verifyCertChain(rootHash string, certs []*x509.Certificate) error {
+func verifyCertChain(notifier Notifier, rootHash string, certs []*x509.Certificate) error {
 	for i, cert := range certs {
 		if (i + 1) == len(certs) {
 			err := verifyRoot(rootHash, cert)
@@ -115,12 +115,10 @@ func verifyCertChain(rootHash string, certs []*x509.Certificate) error {
 			log.Printf("Certificate %d %q has a valid signature from parent certificate %d %q",
 				i, cert.Subject.CommonName, i+1, certs[i+1].Subject.CommonName)
 		}
-		if time.Now().Add(30 * 24 * time.Hour).After(cert.NotAfter) {
+		if notifier != nil && time.Now().Add(30*24*time.Hour).After(cert.NotAfter) {
 			// cert expires in less than 30 days, this is a soft error. send an email.
-			err := sendSoftNotification(
-				fmt.Sprintf("%x", sha256.Sum256(cert.Raw)),
-				"Certificate %d %q expires in less than 30 days: notAfter=%s",
-				i, cert.Subject.CommonName, cert.NotAfter)
+			message := fmt.Sprintf("Certificate %d for %q expires in less than 30 days: notAfter=%s", i, cert.Subject.CommonName, cert.NotAfter)
+			err := notifier.Send(cert.Subject.CommonName, "warning", message)
 			if err != nil {
 				log.Printf("failed to send soft notification: %v", err)
 			}
@@ -133,8 +131,15 @@ func verifyCertChain(rootHash string, certs []*x509.Certificate) error {
 			return fmt.Errorf("Certificate %d %q is not yet valid: notBefore=%s",
 				i, cert.Subject.CommonName, cert.NotBefore)
 		}
-		log.Printf("Certificate %d %q is valid from %s to %s",
-			i, cert.Subject.CommonName, cert.NotBefore, cert.NotAfter)
+		message := fmt.Sprintf(fmt.Sprintf("Certificate %d %q is valid from %s to %s",
+			i, cert.Subject.CommonName, cert.NotBefore, cert.NotAfter))
+		log.Printf(message)
+		if notifier != nil {
+			err := notifier.Send(cert.Subject.CommonName, "info", message)
+			if err != nil {
+				log.Printf("failed to send soft notification: %v", err)
+			}
+		}
 	}
 	return nil
 }