Merge pull request #186 from 18F/stages/rc-2020-12-21

Release RC 13 to Production

.circleci/config.yml

@@ -2,7 +2,11 @@
 #
 # Check https://circleci.com/docs/2.0/language-ruby/ for more details
 #
-version: 2
+version: 2.1
+
+orbs:
+  slack: circleci/slack@3.4.2
+
 jobs:
   build:
     docker:
@@ -27,19 +31,7 @@ jobs:
 
     steps:
       - checkout
-
-      - restore-cache:
-          keys:
-            - v2-identity-pki-bundle-{{ checksum "Gemfile.lock" }}
-      - run:
-          name: Install dependencies
-          command: |
-            gem install bundler
-            bundle check || bundle install --deployment --jobs=4 --retry=3 --without deploy development doc production --path vendor/bundle
-      - save-cache:
-          key: v2-identity-pki-bundle-{{ checksum "Gemfile.lock" }}
-          paths:
-            - vendor/bundle
+      - install-deps
 
       - run:
           name: Install Code Climate Test Reporter
@@ -101,6 +93,62 @@ jobs:
           docker build -t logindotgov/pki:$CIRCLE_TAG -f Dockerfile .
           echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
           docker push logindotgov/pki:$CIRCLE_TAG
+
+  check-expiring-certs-config:
+    docker:
+      # Specify the Ruby version you desire here
+      - image: circleci/ruby:2.6-node-browsers
+        environment:
+          RAILS_ENV: test
+          CC_TEST_REPORTER_ID: c88a6f4af1fbf80e0fc9a5593ebff124b2f940645b1eacb5adb681522bbf650e
+          COVERAGE: true
+          # PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      - image: circleci/postgres:9.5-alpine
+        environment:
+          POSTGRES_USER: circleci
+
+      - image: redis:4.0.1
+
+    working_directory: ~/identity-pki
+
+    steps:
+      - checkout
+      - install-deps
+      - run:
+          name: Test Setup
+          command: |
+            cp config/application.yml.example config/application.yml
+            bundle exec rake db:setup --trace
+      - run:
+          name: Check for expiring certs
+          command: |
+            bundle exec rake certs:print_expiring
+
+      - slack/status:
+          fail_only: true
+          failure_message: ":piv-card::red_circle::scream: identity-pki has certs expiring within 30 days"
+          include_project_field: false
+
+commands:
+  install-deps:
+    steps:
+      - restore_cache:
+          keys:
+            - v2-identity-pki-bundle-{{ checksum "Gemfile.lock" }}
+      - run:
+          name: Install dependencies
+          command: |
+            gem install bundler
+            bundle check || bundle install --deployment --jobs=4 --retry=3 --without deploy development doc production --path vendor/bundle
+      - save_cache:
+          key: v2-identity-pki-bundle-{{ checksum "Gemfile.lock" }}
+          paths:
+            - vendor/bundle
+
 workflows:
   version: 2
   release:
@@ -114,4 +162,15 @@ workflows:
             - build
           filters:
             tags:
-              only: "/^[0-9]{4}-[0-9]{2}-[0-9]{2,}.*/"
+              only: "/^[0-9]{4}-[0-9]{2}-[0-9]{2,}.*/"
+  daily-30d-expiring-cert:
+    jobs:
+      - check-expiring-certs-config
+    triggers:
+      - schedule:
+          # Once a day at 12pm
+          cron: "0 12 * * *"
+          filters:
+            branches:
+              only:
+                - master

app/controllers/health/certs_controller.rb

@@ -0,0 +1,20 @@
+module Health
+  class CertsController < ApplicationController
+    newrelic_ignore_apdex
+
+    def index
+      deadline = params[:deadline].present? ? Time.zone.parse(params[:deadline]) : 30.days.from_now
+
+      result = health_checker.check_certs(deadline: deadline)
+
+      render json: result.as_json,
+             status: result.healthy? ? :ok : :service_unavailable
+    end
+
+    private
+
+    def health_checker
+      @health_checker ||= HealthChecker.new
+    end
+  end
+end

app/models/certificate.rb

@@ -44,8 +44,7 @@ def ==(other)
       signing_key_id == other.signing_key_id
   end
 
-  def expired?
-    now = Time.zone.now
+  def expired?(now = Time.zone.now)
     not_before > now || now > not_after # expiration bounds
   end
 
@@ -65,14 +64,6 @@ def validate_cert
   end
 
   def validate_untrusted_root
-    validate_untrusted_root_with_exceptions
-  rescue OpenSSL::OCSP::OCSPError
-    'ocsp_error'
-  rescue Timeout::Error
-    'timeout'
-  end
-
-  def validate_untrusted_root_with_exceptions
     if self_signed?
       'self-signed cert'
     elsif !signature_verified?

app/services/certificate_store.rb

@@ -23,6 +23,17 @@ def self.reset
     instance.reset
   end
 
+  # load all of the files in config/certs
+  def load_certs!(dir: Figaro.env.certificate_store_directory)
+    Dir.chdir(dir) do
+      Dir.glob(File.join('**', '*.pem')).each do |file|
+        next if file == 'all_certs_deploy.pem'
+
+        add_pem_file(file)
+      end
+    end
+  end
+
   def_delegators :@certificates, :[], :count, :empty?, :map
   def_delegators :certificates, :each, :select
   def_delegators CertificateStore,
@@ -72,6 +83,7 @@ def x509_certificate_chain(cert)
   def x509_certificate_chain_to_root(cert, cert_root_id)
     signing_key_id = cert.signing_key_id
     return [] unless signing_key_id
+
     @certificates.values_at(
       *@graph.dijkstra_shortest_path(Hash.new(1), signing_key_id, cert_root_id)
     )
@@ -128,10 +140,8 @@ def key_ids_signed_by(trusted)
   end
 
   def extract_certs(raw)
-    raw.split(END_CERTIFICATE).map(&method(:cert_from_pem)).compact.select(&:ca_capable?)
-  end
-
-  def cert_from_pem(pem)
-    Certificate.new(OpenSSL::X509::Certificate.new(pem + END_CERTIFICATE)) if pem.strip.present?
+    raw.split(END_CERTIFICATE).map do |pem|
+      Certificate.new(OpenSSL::X509::Certificate.new(pem + END_CERTIFICATE)) if pem.strip.present?
+    end.compact.select(&:ca_capable?)
   end
 end

app/services/health_checker.rb

@@ -0,0 +1,39 @@
+class HealthChecker
+  Result = Struct.new(:healthy, :info, keyword_init: true) do
+    alias_method :healthy?, :healthy
+  end
+
+  def initialize(certificates_store: CertificateStore.instance)
+    @certificates_store = certificates_store
+  end
+
+  # @param [Time] deadline
+  # @return [Result]
+  def check_certs(deadline:)
+    expiring_certs = certificates_store.select do |cert|
+      cert.expired?(deadline)
+    end
+
+    Result.new(
+      healthy: expiring_certs.empty?,
+      info: {
+        deadline: deadline,
+        expiring: expiring_certs.sort_by(&:not_after).map { |cert| cert_info(cert) },
+      }
+    )
+  end
+
+  private
+
+  attr_reader :certificates_store
+
+  # @param [Certificate] cert
+  def cert_info(cert)
+    {
+      expiration: cert.not_after,
+      subject: cert.subject.to_s,
+      issuer: cert.issuer.to_s,
+      key_id: cert.key_id,
+    }
+  end
+end

app/services/ocsp_service.rb

@@ -81,9 +81,12 @@ def handle_response(response, limit)
 
   def make_single_http_request(uri, request, retries = 1)
     make_single_http_request!(uri, request)
-  rescue Errno::ECONNRESET
+  rescue Errno::ECONNRESET, Timeout::Error => e
     retries -= 1
-    return if retries.negative?
+    if retries.negative?
+      log_ocsp_error(:ocsp_timeout, e)
+      return
+    end
     sleep(1)
     retry
   end
@@ -98,5 +101,18 @@ def make_single_http_request!(uri, request)
 
   def process_http_response_body(body)
     OpenSSL::OCSP::Response.new(body) if body.present?
+  rescue OpenSSL::OCSP::OCSPError => e
+    log_ocsp_error(:ocsp_response_error, e)
+    nil
+  end
+
+  def log_ocsp_error(error_type, exception)
+    info = {
+      error_type: error_type,
+      ocsp_url_for_subject: ocsp_url_for_subject,
+      key_id: subject.key_id,
+    }
+    Rails.logger.warn("OCSP error: #{info.to_json}")
+    NewRelic::Agent.notice_error(exception)
   end
 end

config/certs/C=US, O=Entrust, OU=Certification Authorities, OU=Entrust Managed Services Root CA 01.pem

@@ -1,42 +1,32 @@
 Subject: /C=US/O=Entrust/OU=Certification Authorities/OU=Entrust Managed Services Root CA
 Issuer: /C=US/O=U.S. Government/OU=FPKI/CN=Federal Common Policy CA
 -----BEGIN CERTIFICATE-----
-MIIGxDCCBaygAwIBAgICAgIwDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCVVMx
+MIIFKTCCBBGgAwIBAgICc0owDQYJKoZIhvcNAQELBQAwWTELMAkGA1UEBhMCVVMx
 GDAWBgNVBAoTD1UuUy4gR292ZXJubWVudDENMAsGA1UECxMERlBLSTEhMB8GA1UE
-AxMYRmVkZXJhbCBDb21tb24gUG9saWN5IENBMB4XDTEwMTIxNTIxMTIyNloXDTIw
-MTIxNTIxMTAyN1owbjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VudHJ1c3QxIjAg
+AxMYRmVkZXJhbCBDb21tb24gUG9saWN5IENBMB4XDTE5MDgxNDE1Mzk0NloXDTI5
+MDgxNDE1MzY0MlowbjELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB0VudHJ1c3QxIjAg
 BgNVBAsTGUNlcnRpZmljYXRpb24gQXV0aG9yaXRpZXMxKTAnBgNVBAsTIEVudHJ1
 c3QgTWFuYWdlZCBTZXJ2aWNlcyBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEAtntz0/7RJGVXZa6iSqs5aLys+CNBBpw/+O5oROQ7rSI4tnz2
-WPxiqPaC1ddAvKRWcb2+aUvYMJfRkdpss9xBH6nRZ5l/lwkQzfgTsOQtKlFv+rT2
-KVdUFXRikp/CAUDkVyiC7iPvi8kHgNK4aYgUbfGwxmwrS77n9csB3Nv5a1jjpMJQ
-M8CXdeQ+nZ2wUtiMIMMHJRVwrTn/259OdafLj//GO/rJolAM7E4PNm0O6Ydx06hz
-HkyxBj3jQa1dwrdxFzvIkwodVA9yVX/i0FVs8veomMZuFiBPsS+eH4ixACr4t/kQ
-k2llA6cpWChrKru8vXNHfJ/4O/NEiLVlKIC7pQIDAQABo4IDfzCCA3swDwYDVR0T
-AQH/BAUwAwEB/zBPBgNVHSAESDBGMAwGCmCGSAFlAwIBAwYwDAYKYIZIAWUDAgED
-BzAMBgpghkgBZQMCAQMIMAwGCmCGSAFlAwIBAw0wDAYKYIZIAWUDAgEDETCB6QYI
-KwYBBQUHAQEEgdwwgdkwPwYIKwYBBQUHMAKGM2h0dHA6Ly9odHRwLmZwa2kuZ292
-L2ZjcGNhL2NhQ2VydHNJc3N1ZWRUb2ZjcGNhLnA3YzCBlQYIKwYBBQUHMAKGgYhs
-ZGFwOi8vbGRhcC5mcGtpLmdvdi9jbj1GZWRlcmFsJTIwQ29tbW9uJTIwUG9saWN5
-JTIwQ0Esb3U9RlBLSSxvPVUuUy4lMjBHb3Zlcm5tZW50LGM9VVM/Y0FDZXJ0aWZp
-Y2F0ZTtiaW5hcnksY3Jvc3NDZXJ0aWZpY2F0ZVBhaXI7YmluYXJ5MIIBHgYIKwYB
-BQUHAQsEggEQMIIBDDBNBggrBgEFBQcwBYZBaHR0cDovL3Jvb3R3ZWIubWFuYWdl
-ZC5lbnRydXN0LmNvbS9TSUEvQ2VydHNJc3N1ZWRCeUVNU1Jvb3RDQS5wN2MwgboG
-CCsGAQUFBzAFhoGtbGRhcDovL3Jvb3RkaXIubWFuYWdlZC5lbnRydXN0LmNvbS9v
-dT1FbnRydXN0JTIwTWFuYWdlZCUyMFNlcnZpY2VzJTIwUm9vdCUyMENBLG91PUNl
-cnRpZmljYXRpb24lMjBBdXRob3JpdGllcyxvPUVudHJ1c3QsYz1VUz9jQUNlcnRp
-ZmljYXRlO2JpbmFyeSxjcm9zc0NlcnRpZmljYXRlUGFpcjtiaW5hcnkwDgYDVR0P
-AQH/BAQDAgEGMB8GA1UdIwQYMBaAFK0MenVc5fOYxHmYDqwo/Zf05wL8MIG4BgNV
-HR8EgbAwga0wKqAooCaGJGh0dHA6Ly9odHRwLmZwa2kuZ292L2ZjcGNhL2ZjcGNh
-LmNybDB/oH2ge4Z5bGRhcDovL2xkYXAuZnBraS5nb3YvY24lM2RGZWRlcmFsJTIw
-Q29tbW9uJTIwUG9saWN5JTIwQ0Esb3UlM2RGUEtJLG8lM2RVLlMuJTIwR292ZXJu
-bWVudCxjJTNkVVM/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdDAdBgNVHQ4EFgQU
-nGJmJp1xtqd1U2ThrLHHJTxEXQ0wDQYJKoZIhvcNAQELBQADggEBAF1VnwRwXqw8
-cXL1cTi+ooEKDlFQJG6ABGpk9wQMMLZNCWY2SlgTubCLsdmDWss/bjwkDZ32X5/S
-FniTYzOR/R/XQlCdVCrRNuEdTtB7z9GO7zVLusuc53T6yUz6btpK1Es5okLJP0tx
-uwoMKBysSTJ7THummuGaDkEOw5mdlygx0CgulNHrIViHsxDXIEsVpCDNMaMhSglO
-31oaS+gg+GOkFo6PmtOFlgLL4K6NO9Ujr31SJqxVB9KWPf1s5QiP7/VKkU7YCRh7
-WbdI4Bmaw7KVTyTU5aScGLe927fayO1onZ/4yRy04q+7QJmbcZZvw2D33xJmXlq/
-UGje3aBmkxM=
+AQ8AMIIBCgKCAQEA572gaoFb74+gsCeMrlon3dv5pjLJyU4nCO0QqiShzXK8Zqgw
+Na47z+KdF3w1ofeRxYsu0qg/6gzlQU5s1DblG8CeNsXXowjaYwDAMosDSR4HrsLt
+tr1C/4xxLkKejX4GQ01kpTHWMejtpioGMH3FqgK+E9Ga7hGU9rgy0CeVM2/LoJ3e
+kt36xdpndCEbUfe9yQIliEICbJbKhxcMebJKAOb6g8jyr0CzeKXnDqwVMUEn4RED
+sVxQgEzmQMryWdr/LBZckS40AEEhc4D1ojtssABvKrb9NzpGnSCPSDFXFY8N5C++
+CmA2OhZaZOHg//p85PExb4AVBmyZceIay1wezQIDAQABo4IB5DCCAeAwDwYDVR0T
+AQH/BAUwAwEB/zBPBggrBgEFBQcBAQRDMEEwPwYIKwYBBQUHMAKGM2h0dHA6Ly9o
+dHRwLmZwa2kuZ292L2ZjcGNhL2NhQ2VydHNJc3N1ZWRUb2ZjcGNhLnA3YzAPBgNV
+HSQECDAGgAEAgQEAMAoGA1UdNgQDAgEAMHkGA1UdIARyMHAwDAYKYIZIAWUDAgED
+BjAMBgpghkgBZQMCAQMHMAwGCmCGSAFlAwIBAwgwDAYKYIZIAWUDAgEDDTAMBgpg
+hkgBZQMCAQMRMAwGCmCGSAFlAwIBAycwDAYKYIZIAWUDAgEDKDAMBgpghkgBZQMC
+AQMpMF0GCCsGAQUFBwELBFEwTzBNBggrBgEFBQcwBYZBaHR0cDovL3Jvb3R3ZWIu
+bWFuYWdlZC5lbnRydXN0LmNvbS9TSUEvQ2VydHNJc3N1ZWRCeUVNU1Jvb3RDQS5w
+N2MwDgYDVR0PAQH/BAQDAgGGMB8GA1UdIwQYMBaAFK0MenVc5fOYxHmYDqwo/Zf0
+5wL8MDUGA1UdHwQuMCwwKqAooCaGJGh0dHA6Ly9odHRwLmZwa2kuZ292L2ZjcGNh
+L2ZjcGNhLmNybDAdBgNVHQ4EFgQUSVSRTGlEO8T4Aiz0+C0zVol1mBAwDQYJKoZI
+hvcNAQELBQADggEBAMX/TfukCGAdHdlIuDuBG3wg5+GIRzf5Vgt/gEl+dNR3BdVO
+FrA+yKdPwnV9A+HZtxwC6YrIgxHsD8iImvF6WCuDWwNl2mNg0AynC3FNfyJlzMCw
+kPbs2n4VqmcaP5hqVCiKVv+omQ7CwRM18ms4Ia0oHNFCaV3yvZb/QMFKUM3CaK0s
+qZNmmBAqf6+XVeha45kKNtI20HXhUBzGyvmo/3vNfzJTQIQMqV10QP5ectlFvlLv
+TjP+7mNJvuo3M5avGucbsNQLZrGsQMgIVcdhc4Juf3cklUNDJxAiyFbX3LEcP2SD
++6w/aYn9eB1GK8AqFv1dNfMK5dKBmrDRhMmxIqg=
 -----END CERTIFICATE-----
-