Move challenges to datagroup to handle SAN certs

.gitignore

@@ -1,3 +1,12 @@
+# project stuff
+!accounts/README.txt
+!certs/README.txt
+!wellknown/README.txt
+accounts/
+certs/
+wellknown/
+
+
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template
 # Byte-compiled / optimized / DLL files

README.md

@@ -14,34 +14,74 @@ out the mostly-retired [f5-sdk](https://github.com/f5networks/f5-common-python)
 Removed from this project altogether is the creation of client SSL profiles, as that is a separate function
 than certificate management and should have its own workflow.
 
+### [Tim Riker](https://rikers.org) added
+* run as non-root in a working directory
+* include full chain in .crt so no separate chain is needed
+* create or update certs/keys
+* create client-ssl profiles if missing
+* irule uses a datagroup to handle multiple challenges for multiple names in a certificate.
+
+## Getting Started
+
+Install dehydrated. On Debian based distros this probably works:
+
+```bash
+$ sudo apt install dehydrated
+```
+
+Install bigrest in python
+```bash
+$ pip install bigrest
+```
+Set CONTACT_EMAIL in config to your email.
+
+register with dehydrated
+```bash
+$ dehydrated -f config --register --accept-terms
+```
+Add your domains and aliases to domains.txt and try a request
+```bash
+$ dehydrated -f config -c --force --force-validation
+```
+
+
 ## Test Setup
 ```bash
-/etc/dehydrated/config # Dehydrated configuration file
-/etc/dehydrated/domains.txt # Domains to sign and generate certs for
-/etc/dehydrated/dehydrated # acme client
-/etc/dehydrated/challenge.irule # iRule configured and deployed to BIG-IP by the hook script
-/etc/dehydrated/hook_script.py # Python script called by dehydrated for special steps in the cert generation process
+config # Dehydrated configuration file (edit CONTACT_EMAIL)
+domains.txt # Domains to sign and generate certs for (add names and aliases)
+dehydrated # acme client (install)
+bigrest # install python library
+rule_le_challenge.iRule # iRule configured and deployed to BIG-IP by the hook script
+hook_script.py # Python script called by dehydrated for special steps in the cert generation process
+
 # Environment Variables
-export F5_HOST=x.x.x.x
+export F5_HOST=f.q.d.n
 export F5_USER=admin
 export F5_PASS=admin
+export F5_HTTP=vs_vip-name_HTTP
+export F5_HTTPS=vs_vip-name_HTTPS
 ```
 ## Usage
 
 ### Testing - Stage API
-./dehydrated -c --force --force-validation
+```bash
+$ dehydrated -f config -c --force --force-validation
+```
 
 ### Otherwise
-./dehydrated -c
+```bash
+$ dehydrated -f config -c
+```
 
 ## Expected Output
 
 ```bash
-# ./dehydrated -c --force --force-validation
-# INFO: Using main config file /etc/dehydrated/config
+$ dehydrated -f config -c --force --force-validation
+# INFO: Using main config file config
 Processing example.com
+ + Checking domain name(s) of existing cert... unchanged.
  + Checking expire date of existing cert...
- + Valid till Jun 20 02:03:26 2022 GMT (Longer than 30 days). Ignoring because renew was forced!
+ + Valid till Dec  7 17:08:55 2022 GMT (Longer than 30 days). Ignoring because renew was forced!
  + Signing domains...
  + Generating private key...
  + Generating signing request...
@@ -51,19 +91,24 @@ Processing example.com
  + A valid authorization has been found but will be ignored
  + 1 pending challenge(s)
  + Deploying challenge tokens...
- + (hook) Deploying Challenge
- + (hook) Challenge rule added to virtual.
+ + (hook) Deploying Challenge example.com
+ + (hook) irule rule_le_challenge added.
+ + (hook) datagroup dg_le_challenge added.
+ + (hook) Challenge rule added to virtual vs_example.com_HTTP.
+ + (hook) Challenge added to datagroup dg_le_challenge for example.com.
  + Responding to challenge for example.com authorization...
  + Challenge is valid!
  + Cleaning challenge tokens...
- + (hook) Cleaning Challenge
- + (hook) Challenge rule removed from virtual.
+ + (hook) Cleaning Challenge example.com
+ + (hook) Challenge rule rule_le_challenge removed from virtual vs_example.com_HTTP.
+ + (hook) irule rule_le_challenge removed.
+ + (hook) datagroup dg_le_challenge removed.
  + Requesting certificate...
  + Checking certificate...
  + Done!
  + Creating fullchain.pem...
- + (hook) Deploying Certs
- + (hook) Existing Cert/Key updated in transaction.
+ + (hook) Deploying Certs example.com
+ + (hook) Cert/Key example.com updated in transaction.
  + Done!
 ```
 ![Certs on BIG-IP](img/le_certs_bigip.png)

accounts/README.txt

@@ -0,0 +1 @@
+# dehydrated accounts are stored here

certs/README.txt

@@ -0,0 +1 @@
+# dehydrated certs are stored here

config

@@ -66,7 +66,7 @@ CHALLENGETYPE="http-01"
 
 # Output directory for challenge-tokens to be served by webserver or deployed in HOOK (default: /var/www/dehydrated)
 #WELLKNOWN="/var/www/dehydrated"
-WELLKNOWN="/etc/dehydrated/wellknowns"
+WELLKNOWN="wellknown"
 
 # Default keysize for private keys (default: 4096)
 #KEYSIZE="4096"
@@ -92,7 +92,7 @@ CURL_OPTS="--http1.1"
 # BASEDIR and WELLKNOWN variables are exported and can be used in an external program
 # default: <unset>
 #HOOK=
-HOOK=/etc/dehydrated/hook_script.py
+HOOK=${BASEDIR}/hook_script.py
 
 # Chain clean_challenge|deploy_challenge arguments together into one hook call per certificate (default: no)
 #HOOK_CHAIN="no"
@@ -111,7 +111,6 @@ HOOK=/etc/dehydrated/hook_script.py
 
 # E-mail to use during the registration (default: <unset>)
 #CONTACT_EMAIL=
-CONTACT_EMAIL="me@example.com"
 
 # Lockfile location, to prevent concurrent access (default: $BASEDIR/lock)
 #LOCKFILE="${BASEDIR}/lock"
@@ -135,4 +134,4 @@ CONTACT_EMAIL="me@example.com"
 #API=auto
 
 # Preferred issuer chain (default: <unset> -> uses default chain)
-#PREFERRED_CHAIN=
+#PREFERRED_CHAIN=

hook_script.py

@@ -9,118 +9,124 @@
 
 requests.packages.urllib3.disable_warnings()
 
-
 def get_credentials():
     return {'host': os.getenv('F5_HOST'), 'user': os.getenv('F5_USER'), 'pass': os.getenv('F5_PASS')}
 
-
 def instantiate_bigip(credentials):
     return BIGIP(credentials.get('host'), credentials.get('user'), credentials.get('pass'))
 
-
-def alter_template(token_info):
-    with open('challenge.irule', 'r') as f2:
-        new_challenge = ''
-        for line in f2:
-            new_line = line.rstrip()
-            new_line = new_line.replace('TOKENFILE', token_info[0])
-            new_line = new_line.replace('TOKENVALUE', token_info[1])
-            new_challenge += f'{new_line}\n'
-    return new_challenge
-
-
 def deploy_challenge(args):
-    new_irule = alter_template(args[1:])
     br = instantiate_bigip(get_credentials())
-    if br.exist('/mgmt/tm/ltm/rule/le_challenge_rule'):
-        rule = br.load('/mgmt/tm/ltm/rule/le_challenge_rule')
-        rule.properties['apiAnonymous'] = new_irule
-        br.save(rule)
-        vip = br.load('/mgmt/tm/ltm/virtual/external_http_test')
-        if vip.properties.get('rules') is None:
-            vip.properties['rules'] = ['le_challenge_rule']
-        else:
-            vip.properties['rules'].append('le_challenge_rule')
-        br.save(vip)
-    else:
-        ltm_rule_object = {'name': 'le_challenge_rule', '': new_irule}
-        br.create('/mgmt/tm/ltm/rule', ltm_rule_object)
-        if br.exist('/mgmt/tm/ltm/rule/le_challenge_rule'):
-            vip = br.load('/mgmt/tm/ltm/virtual/external_http_test')
-            vip.properties['rules'].append('le_challenge_rule')
+    f = open('rule_le_challenge.iRule')
+    irule = f.read()
+    f.close()
+    if not br.exist('/mgmt/tm/ltm/rule/rule_le_challenge'):
+        rule = {'name': 'rule_le_challenge', 'apiAnonymous': irule}
+        br.create('/mgmt/tm/ltm/rule', rule)
+        logger.info(' + (hook) irule rule_le_challenge added.')
+    if not br.exist('/mgmt/tm/ltm/data-group/internal/dg_le_challenge'):
+        dg = {'name': 'dg_le_challenge', 'type': 'string', 'records': [{'name':'name','data':'data'}]}
+        br.create('/mgmt/tm/ltm/data-group/internal', dg)
+        logger.info(' + (hook) datagroup dg_le_challenge added.')
+    if br.exist('/mgmt/tm/ltm/rule/rule_le_challenge'):
+        vip = br.load(f'/mgmt/tm/ltm/virtual/{f5_http}')
+        if '/Common/rule_le_challenge' not in vip.properties['rules']:
+            if vip.properties.get('rules') is None:
+                vip.properties['rules'] = ['rule_le_challenge']
+            elif not '/mgmt/tm/ltm/rule/rule_le_challenge' in vip.properties['rules']:
+                vip.properties['rules'].insert(0,'rule_le_challenge')
             br.save(vip)
-    logger.info(' + (hook) Challenge rule added to virtual.')
-
+            logger.info(f' + (hook) Challenge rule added to virtual {f5_http}.')
+    dg = br.load('/mgmt/tm/ltm/data-group/internal/dg_le_challenge')
+    dg.properties['records'].append({'name':args[1],'data':args[2]})
+    br.save(dg)
+    logger.info(f' + (hook) Challenge added to datagroup dg_le_challenge for {args[0]}.')
 
 def invalid_challenge(args):
     logger.info(f' + (hook) Invalid Challenge Args: {args}')
-
+    sys.exit(-1)
 
 def clean_challenge(args):
     br = instantiate_bigip(get_credentials())
-    vip = br.load('/mgmt/tm/ltm/virtual/external_http_test')
-    vip.properties['rules'].remove('/Common/le_challenge_rule')
-    br.save(vip)
-    logger.info(' + (hook) Challenge rule removed from virtual.')
-
+    vip = br.load(f'/mgmt/tm/ltm/virtual/{f5_http}')
+    if '/Common/rule_le_challenge' in vip.properties['rules']:
+        vip.properties['rules'].remove('/Common/rule_le_challenge')
+        br.save(vip)
+        logger.info(f' + (hook) Challenge rule rule_le_challenge removed from virtual {f5_http}.')
+    if br.exist('/mgmt/tm/ltm/rule/rule_le_challenge'):
+        br.delete('/mgmt/tm/ltm/rule/rule_le_challenge')
+        logger.info(f' + (hook) irule rule_le_challenge removed.')
+    if br.exist('/mgmt/tm/ltm/data-group/internal/dg_le_challenge'):
+        br.delete('/mgmt/tm/ltm/data-group/internal/dg_le_challenge')
+        logger.info(' + (hook) datagroup dg_le_challenge removed.')
 
 def deploy_cert(args):
     br = instantiate_bigip(get_credentials())
     br.upload('/mgmt/shared/file-transfer/uploads', args[1])
-    br.upload('/mgmt/shared/file-transfer/uploads', args[2])
-    br.upload('/mgmt/shared/file-transfer/uploads', args[4])
-    key_status = br.exist(f'/mgmt/tm/sys/file/ssl-key/le_auto_{args[0]}.key')
-    cert_status = br.exist(f'/mgmt/tm/sys/file/ssl-cert/le_auto_{args[0]}.crt')
-    chain_status = br.exist(f'/mgmt/tm/sys/file/ssl-cert/le_auto_chain.crt')
+    br.upload('/mgmt/shared/file-transfer/uploads', args[3])
+    key_status = br.exist(f'/mgmt/tm/sys/file/ssl-key/auto_le_{args[0]}.key')
+    cert_status = br.exist(f'/mgmt/tm/sys/file/ssl-cert/auto_le_{args[0]}.crt')
 
-    if key_status and cert_status and chain_status:
+    if key_status and cert_status:
         with br as transaction:
-            modkey = br.load(f'/mgmt/tm/sys/file/ssl-key/le_auto_{args[0]}.key')
+            modkey = br.load(f'/mgmt/tm/sys/file/ssl-key/auto_le_{args[0]}.key')
             modkey.properties['sourcePath'] = f'file:/var/config/rest/downloads/{args[1].split("/")[-1]}'
             br.save(modkey)
-            modcert = br.load(f'/mgmt/tm/sys/file/ssl-cert/le_auto_{args[0]}.crt')
-            modcert.properties['sourcePath'] = f'file:/var/config/rest/downloads/{args[2].split("/")[-1]}'
+            modcert = br.load(f'/mgmt/tm/sys/file/ssl-cert/auto_le_{args[0]}.crt')
+            modcert.properties['sourcePath'] = f'file:/var/config/rest/downloads/{args[3].split("/")[-1]}'
             br.save(modcert)
-            modchain = br.load(f'/mgmt/tm/sys/file/ssl-cert/le_auto_chain.crt')
-            modchain.properties['sourcePath'] = f'file:/var/config/rest/downloads/{args[4].split("/")[-1]}'
-            br.save(modchain)
-            logger.info(' + (hook) Existing Cert/Key updated in transaction.')
+            logger.info(f' + (hook) Cert/Key {args[0]} updated in transaction.')
     else:
-        keydata = {'name': f'le_auto_{args[0]}.key', 'sourcePath': f'file:/var/config/rest/downloads/{args[1].split("/")[-1]}'}
-        certdata = {'name': f'le_auto_{args[0]}.crt', 'sourcePath': f'file:/var/config/rest/downloads/{args[2].split("/")[-1]}'}
-        chaindata = {'name': f'le_auto_chain.crt', 'sourcePath': f'file:/var/config/rest/downloads/{args[4].split("/")[-1]}'}
+        keydata = {'name': f'auto_le_{args[0]}.key', 'sourcePath': f'file:/var/config/rest/downloads/{args[1].split("/")[-1]}'}
+        certdata = {'name': f'auto_le_{args[0]}.crt', 'sourcePath': f'file:/var/config/rest/downloads/{args[3].split("/")[-1]}'}
         br.create('/mgmt/tm/sys/file/ssl-key', keydata)
         br.create('/mgmt/tm/sys/file/ssl-cert', certdata)
-        br.create('/mgmt/tm/sys/file/ssl-cert', chaindata)
-        logger.info(' + (hook) New Certificate/Key/Chain created.')
-
+        logger.info(f' + (hook) Cert/Key {args[0]} created.')
+    if not br.exist(f'/mgmt/tm/ltm/profile/client-ssl/auto_le_{args[0]}'):
+        sslprof = {
+            'name' : f'auto_le_{args[0]}',
+            'defaultsFrom': '/Common/clientssl',
+            'certKeyChain': [{
+                'name': f'{args[0]}_0',
+                'cert': f'/Common/auto_le_{args[0]}.crt',
+                'key': f'/Common/auto_le_{args[0]}.key'
+            }]
+        }
+        logger.info(sslprof)
+        br.create('/mgmt/tm/ltm/profile/client-ssl', sslprof)
+        logger.info(f' + (hook) client-ssl profile created auto_le_{args[0]}.')
+    #profiles = br.load(f'/mgmt/tm/ltm/virtual/{f5_https}/profiles')
+    #logger.info(profiles)
 
 def unchanged_cert(args):
     logger.info(f' + (hook) No changes necessary.')
 
-
 if __name__ == '__main__':
     # Logging
     logger = logging.getLogger(__name__)
     logger.addHandler(logging.StreamHandler())
     logger.setLevel(logging.INFO)
 
+    # get virtualserver names from environment
+    f5_http = os.getenv('F5_HTTP')
+    f5_https = os.getenv('F5_HTTPS')
+
     if len(sys.argv) > 2:
         hook = sys.argv[1]
     else:
         hook = ''
     if hook == 'deploy_challenge':
-        logger.info(' + (hook) Deploying Challenge')
+        logger.info(f' + (hook) Deploying Challenge {sys.argv[2]}')
         deploy_challenge(sys.argv[2:])
     elif hook == 'invalid_challenge':
-        logger.info(' + (hook) Invalid Challenge')
+        logger.info(f' + (hook) Invalid Challenge {sys.argv[2]}')
         invalid_challenge(sys.argv[2:])
     elif hook == 'clean_challenge':
-        logger.info(' + (hook) Cleaning Challenge')
+        logger.info(f' + (hook) Cleaning Challenge {sys.argv[2]}')
         clean_challenge(sys.argv[2:])
     elif hook == 'deploy_cert':
-        logger.info(' + (hook) Deploying Certs')
+        logger.info(f' + (hook) Deploying Certs {sys.argv[2]}')
         deploy_cert(sys.argv[2:])
     elif hook == 'unchanged_cert':
-        logger.info(' + (hook) Unchanged Certs')
+        logger.info(f' + (hook) Unchanged Certs {sys.argv[2]}')
         unchanged_cert(sys.argv[2:])

rule_le_challenge.iRule

@@ -0,0 +1,17 @@
+# look up Let's Encrypt validation requests in dg_le_challenge and return if found
+# pass through if not found in dg
+# Tim Riker <Tim@Rikers.org>
+
+when HTTP_REQUEST {
+    if { [class exists dg_le_challenge] } {
+        if {"[HTTP::uri]" starts_with "/.well-known/acme-challenge/"} {
+            set log(lekey) [getfield "[HTTP::uri]" "/" 4]
+            set log(levalue) [class match -value -- $log(lekey) equals dg_le_challenge]
+            if { $log(levalue) ne "" } {
+                HTTP::respond 200 content "$log(levalue)\n" "Content-Type" "text/plain" "Connection" "close"
+                event disable
+                return
+            }
+        }
+    }
+}

wellknown/README.txt

@@ -0,0 +1 @@
+# dehydrated challenges are stored here