From bdf50c0465579efae3bef15aea5ce7d57347a073 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 29 Sep 2016 20:48:57 -0400 Subject: [PATCH 01/47] bringing in dehydrated to manage acme cert request/renews --- bin/dehydrated | 1116 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1116 insertions(+) create mode 100755 bin/dehydrated diff --git a/bin/dehydrated b/bin/dehydrated new file mode 100755 index 0000000..4cc2a66 --- /dev/null +++ b/bin/dehydrated @@ -0,0 +1,1116 @@ +#!/usr/bin/env bash + +# dehydrated by lukas2511 +# Source: https://github.com/lukas2511/dehydrated +# +# This script is licensed under The MIT License (see LICENSE for more information). + +set -e +set -u +set -o pipefail +[[ -n "${ZSH_VERSION:-}" ]] && set -o SH_WORD_SPLIT && set +o FUNCTION_ARGZERO +umask 077 # paranoid umask, we're creating private keys + +# Find directory in which this script is stored by traversing all symbolic links +SOURCE="${0}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +BASEDIR="${SCRIPTDIR}" + +# Create (identifiable) temporary files +_mktemp() { + # shellcheck disable=SC2068 + mktemp ${@:-} "${TMPDIR:-/tmp}/dehydrated-XXXXXX" +} + +# Check for script dependencies +check_dependencies() { + # just execute some dummy and/or version commands to see if required tools exist and are actually usable + openssl version > /dev/null 2>&1 || _exiterr "This script requires an openssl binary." + _sed "" < /dev/null > /dev/null 2>&1 || _exiterr "This script requires sed with support for extended (modern) regular expressions." + command -v grep > /dev/null 2>&1 || _exiterr "This script requires grep." + _mktemp -u > /dev/null 2>&1 || _exiterr "This script requires mktemp." + diff -u /dev/null /dev/null || _exiterr "This script requires diff." + + # curl returns with an error code in some ancient versions so we have to catch that + set +e + curl -V > /dev/null 2>&1 + retcode="$?" + set -e + if [[ ! "${retcode}" = "0" ]] && [[ ! "${retcode}" = "2" ]]; then + _exiterr "This script requires curl." + fi +} + +store_configvars() { + __KEY_ALGO="${KEY_ALGO}" + __OCSP_MUST_STAPLE="${OCSP_MUST_STAPLE}" + __PRIVATE_KEY_RENEW="${PRIVATE_KEY_RENEW}" + __KEYSIZE="${KEYSIZE}" + __CHALLENGETYPE="${CHALLENGETYPE}" + __HOOK="${HOOK}" + __WELLKNOWN="${WELLKNOWN}" + __HOOK_CHAIN="${HOOK_CHAIN}" + __OPENSSL_CNF="${OPENSSL_CNF}" + __RENEW_DAYS="${RENEW_DAYS}" + __IP_VERSION="${IP_VERSION}" +} + +reset_configvars() { + KEY_ALGO="${__KEY_ALGO}" + OCSP_MUST_STAPLE="${__OCSP_MUST_STAPLE}" + PRIVATE_KEY_RENEW="${__PRIVATE_KEY_RENEW}" + KEYSIZE="${__KEYSIZE}" + CHALLENGETYPE="${__CHALLENGETYPE}" + HOOK="${__HOOK}" + WELLKNOWN="${__WELLKNOWN}" + HOOK_CHAIN="${__HOOK_CHAIN}" + OPENSSL_CNF="${__OPENSSL_CNF}" + RENEW_DAYS="${__RENEW_DAYS}" + IP_VERSION="${__IP_VERSION}" +} + +# verify configuration values +verify_config() { + [[ "${CHALLENGETYPE}" =~ (http-01|dns-01) ]] || _exiterr "Unknown challenge type ${CHALLENGETYPE}... can not continue." + if [[ "${CHALLENGETYPE}" = "dns-01" ]] && [[ -z "${HOOK}" ]]; then + _exiterr "Challenge type dns-01 needs a hook script for deployment... can not continue." + fi + if [[ "${CHALLENGETYPE}" = "http-01" && ! -d "${WELLKNOWN}" ]]; then + _exiterr "WELLKNOWN directory doesn't exist, please create ${WELLKNOWN} and set appropriate permissions." + fi + [[ "${KEY_ALGO}" =~ ^(rsa|prime256v1|secp384r1)$ ]] || _exiterr "Unknown public key algorithm ${KEY_ALGO}... can not continue." + if [[ -n "${IP_VERSION}" ]]; then + [[ "${IP_VERSION}" = "4" || "${IP_VERSION}" = "6" ]] || _exiterr "Unknown IP version ${IP_VERSION}... can not continue." + fi +} + +# Setup default config values, search for and load configuration files +load_config() { + # Check for config in various locations + if [[ -z "${CONFIG:-}" ]]; then + for check_config in "/etc/dehydrated" "/usr/local/etc/dehydrated" "${PWD}" "${SCRIPTDIR}"; do + if [[ -f "${check_config}/config" ]]; then + BASEDIR="${check_config}" + CONFIG="${check_config}/config" + break + fi + done + fi + + # Default values + CA="https://acme-v01.api.letsencrypt.org/directory" + LICENSE="https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf" + CERTDIR= + ACCOUNTDIR= + CHALLENGETYPE="http-01" + CONFIG_D= + DOMAINS_D= + DOMAINS_TXT= + HOOK= + HOOK_CHAIN="no" + RENEW_DAYS="30" + KEYSIZE="4096" + WELLKNOWN= + PRIVATE_KEY_RENEW="yes" + KEY_ALGO=rsa + OPENSSL_CNF="$(openssl version -d | cut -d\" -f2)/openssl.cnf" + CONTACT_EMAIL= + LOCKFILE= + OCSP_MUST_STAPLE="no" + IP_VERSION= + + if [[ -z "${CONFIG:-}" ]]; then + echo "#" >&2 + echo "# !! WARNING !! No main config file found, using default config!" >&2 + echo "#" >&2 + elif [[ -f "${CONFIG}" ]]; then + echo "# INFO: Using main config file ${CONFIG}" + BASEDIR="$(dirname "${CONFIG}")" + # shellcheck disable=SC1090 + . "${CONFIG}" + else + _exiterr "Specified config file doesn't exist." + fi + + if [[ -n "${CONFIG_D}" ]]; then + if [[ ! -d "${CONFIG_D}" ]]; then + _exiterr "The path ${CONFIG_D} specified for CONFIG_D does not point to a directory." >&2 + fi + + for check_config_d in "${CONFIG_D}"/*.sh; do + if [[ ! -e "${check_config_d}" ]]; then + echo "# !! WARNING !! Extra configuration directory ${CONFIG_D} exists, but no configuration found in it." >&2 + break + elif [[ -f "${check_config_d}" ]] && [[ -r "${check_config_d}" ]]; then + echo "# INFO: Using additional config file ${check_config_d}" + # shellcheck disable=SC1090 + . "${check_config_d}" + else + _exiterr "Specified additional config ${check_config_d} is not readable or not a file at all." >&2 + fi + done + fi + + # Remove slash from end of BASEDIR. Mostly for cleaner outputs, doesn't change functionality. + BASEDIR="${BASEDIR%%/}" + + # Check BASEDIR and set default variables + [[ -d "${BASEDIR}" ]] || _exiterr "BASEDIR does not exist: ${BASEDIR}" + + CAHASH="$(echo "${CA}" | urlbase64)" + [[ -z "${ACCOUNTDIR}" ]] && ACCOUNTDIR="${BASEDIR}/accounts" + mkdir -p "${ACCOUNTDIR}/${CAHASH}" + [[ -f "${ACCOUNTDIR}/${CAHASH}/config" ]] && . "${ACCOUNTDIR}/${CAHASH}/config" + ACCOUNT_KEY="${ACCOUNTDIR}/${CAHASH}/account_key.pem" + ACCOUNT_KEY_JSON="${ACCOUNTDIR}/${CAHASH}/registration_info.json" + + if [[ -f "${BASEDIR}/private_key.pem" ]] && [[ ! -f "${ACCOUNT_KEY}" ]]; then + echo "! Moving private_key.pem to ${ACCOUNT_KEY}" + mv "${BASEDIR}/private_key.pem" "${ACCOUNT_KEY}" + fi + if [[ -f "${BASEDIR}/private_key.json" ]] && [[ ! -f "${ACCOUNT_KEY_JSON}" ]]; then + echo "! Moving private_key.json to ${ACCOUNT_KEY_JSON}" + mv "${BASEDIR}/private_key.json" "${ACCOUNT_KEY_JSON}" + fi + + [[ -z "${CERTDIR}" ]] && CERTDIR="${BASEDIR}/certs" + [[ -z "${DOMAINS_TXT}" ]] && DOMAINS_TXT="${BASEDIR}/domains.txt" + [[ -z "${WELLKNOWN}" ]] && WELLKNOWN="/var/www/dehydrated" + [[ -z "${LOCKFILE}" ]] && LOCKFILE="${BASEDIR}/lock" + [[ -n "${PARAM_NO_LOCK:-}" ]] && LOCKFILE="" + + [[ -n "${PARAM_HOOK:-}" ]] && HOOK="${PARAM_HOOK}" + [[ -n "${PARAM_CERTDIR:-}" ]] && CERTDIR="${PARAM_CERTDIR}" + [[ -n "${PARAM_CHALLENGETYPE:-}" ]] && CHALLENGETYPE="${PARAM_CHALLENGETYPE}" + [[ -n "${PARAM_KEY_ALGO:-}" ]] && KEY_ALGO="${PARAM_KEY_ALGO}" + [[ -n "${PARAM_OCSP_MUST_STAPLE:-}" ]] && OCSP_MUST_STAPLE="${PARAM_OCSP_MUST_STAPLE}" + [[ -n "${PARAM_IP_VERSION:-}" ]] && IP_VERSION="${PARAM_IP_VERSION}" + + verify_config + store_configvars +} + +# Initialize system +init_system() { + load_config + + # Lockfile handling (prevents concurrent access) + if [[ -n "${LOCKFILE}" ]]; then + LOCKDIR="$(dirname "${LOCKFILE}")" + [[ -w "${LOCKDIR}" ]] || _exiterr "Directory ${LOCKDIR} for LOCKFILE ${LOCKFILE} is not writable, aborting." + ( set -C; date > "${LOCKFILE}" ) 2>/dev/null || _exiterr "Lock file '${LOCKFILE}' present, aborting." + remove_lock() { rm -f "${LOCKFILE}"; } + trap 'remove_lock' EXIT + fi + + # Get CA URLs + CA_DIRECTORY="$(http_request get "${CA}")" + CA_NEW_CERT="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-cert)" && + CA_NEW_AUTHZ="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-authz)" && + CA_NEW_REG="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-reg)" && + # shellcheck disable=SC2015 + CA_REVOKE_CERT="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value revoke-cert)" || + _exiterr "Problem retrieving ACME/CA-URLs, check if your configured CA points to the directory entrypoint." + + # Export some environment variables to be used in hook script + export WELLKNOWN BASEDIR CERTDIR CONFIG + + # Checking for private key ... + register_new_key="no" + if [[ -n "${PARAM_ACCOUNT_KEY:-}" ]]; then + # a private key was specified from the command line so use it for this run + echo "Using private key ${PARAM_ACCOUNT_KEY} instead of account key" + ACCOUNT_KEY="${PARAM_ACCOUNT_KEY}" + ACCOUNT_KEY_JSON="${PARAM_ACCOUNT_KEY}.json" + else + # Check if private account key exists, if it doesn't exist yet generate a new one (rsa key) + if [[ ! -e "${ACCOUNT_KEY}" ]]; then + echo "+ Generating account key..." + _openssl genrsa -out "${ACCOUNT_KEY}" "${KEYSIZE}" + register_new_key="yes" + fi + fi + openssl rsa -in "${ACCOUNT_KEY}" -check 2>/dev/null > /dev/null || _exiterr "Account key is not valid, can not continue." + + # Get public components from private key and calculate thumbprint + pubExponent64="$(printf '%x' "$(openssl rsa -in "${ACCOUNT_KEY}" -noout -text | awk '/publicExponent/ {print $2}')" | hex2bin | urlbase64)" + pubMod64="$(openssl rsa -in "${ACCOUNT_KEY}" -noout -modulus | cut -d'=' -f2 | hex2bin | urlbase64)" + + thumbprint="$(printf '{"e":"%s","kty":"RSA","n":"%s"}' "${pubExponent64}" "${pubMod64}" | openssl dgst -sha256 -binary | urlbase64)" + + # If we generated a new private key in the step above we have to register it with the acme-server + if [[ "${register_new_key}" = "yes" ]]; then + echo "+ Registering account key with ACME server..." + [[ ! -z "${CA_NEW_REG}" ]] || _exiterr "Certificate authority doesn't allow registrations." + # If an email for the contact has been provided then adding it to the registration request + FAILED=false + if [[ -n "${CONTACT_EMAIL}" ]]; then + (signed_request "${CA_NEW_REG}" '{"resource": "new-reg", "contact":["mailto:'"${CONTACT_EMAIL}"'"], "agreement": "'"$LICENSE"'"}' > "${ACCOUNT_KEY_JSON}") || FAILED=true + else + (signed_request "${CA_NEW_REG}" '{"resource": "new-reg", "agreement": "'"$LICENSE"'"}' > "${ACCOUNT_KEY_JSON}") || FAILED=true + fi + if [[ "${FAILED}" = "true" ]]; then + echo + echo + echo "Error registering account key. See message above for more information." + rm "${ACCOUNT_KEY}" "${ACCOUNT_KEY_JSON}" + exit 1 + fi + fi + +} + +# Different sed version for different os types... +_sed() { + if [[ "${OSTYPE}" = "Linux" ]]; then + sed -r "${@}" + else + sed -E "${@}" + fi +} + +# Print error message and exit with error +_exiterr() { + echo "ERROR: ${1}" >&2 + exit 1 +} + +# Remove newlines and whitespace from json +clean_json() { + tr -d '\r\n' | _sed -e 's/ +/ /g' -e 's/\{ /{/g' -e 's/ \}/}/g' -e 's/\[ /[/g' -e 's/ \]/]/g' +} + +# Encode data as url-safe formatted base64 +urlbase64() { + # urlbase64: base64 encoded string with '+' replaced with '-' and '/' replaced with '_' + openssl base64 -e | tr -d '\n\r' | _sed -e 's:=*$::g' -e 'y:+/:-_:' +} + +# Convert hex string to binary data +hex2bin() { + # Remove spaces, add leading zero, escape as hex string and parse with printf + printf -- "$(cat | _sed -e 's/[[:space:]]//g' -e 's/^(.(.{2})*)$/0\1/' -e 's/(.{2})/\\x\1/g')" +} + +# Get string value from json dictionary +get_json_string_value() { + local filter + filter=$(printf 's/.*"%s": *"\([^"]*\)".*/\\1/p' "$1") + sed -n "${filter}" +} + +# OpenSSL writes to stderr/stdout even when there are no errors. So just +# display the output if the exit code was != 0 to simplify debugging. +_openssl() { + set +e + out="$(openssl "${@}" 2>&1)" + res=$? + set -e + if [[ ${res} -ne 0 ]]; then + echo " + ERROR: failed to run $* (Exitcode: ${res})" >&2 + echo >&2 + echo "Details:" >&2 + echo "${out}" >&2 + echo >&2 + exit ${res} + fi +} + +# Send http(s) request with specified method +http_request() { + tempcont="$(_mktemp)" + + if [[ -n "${IP_VERSION:-}" ]]; then + ip_version="-${IP_VERSION}" + fi + + set +e + if [[ "${1}" = "head" ]]; then + statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}" -I)" + curlret="${?}" + elif [[ "${1}" = "get" ]]; then + statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}")" + curlret="${?}" + elif [[ "${1}" = "post" ]]; then + statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}" -d "${3}")" + curlret="${?}" + else + set -e + _exiterr "Unknown request method: ${1}" + fi + set -e + + if [[ ! "${curlret}" = "0" ]]; then + _exiterr "Problem connecting to server (${1} for ${2}; curl returned with ${curlret})" + fi + + if [[ ! "${statuscode:0:1}" = "2" ]]; then + echo " + ERROR: An error occurred while sending ${1}-request to ${2} (Status ${statuscode})" >&2 + echo >&2 + echo "Details:" >&2 + cat "${tempcont}" >&2 + echo >&2 + echo >&2 + rm -f "${tempcont}" + + # Wait for hook script to clean the challenge if used + if [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && [[ -n "${challenge_token:+set}" ]]; then + "${HOOK}" "clean_challenge" '' "${challenge_token}" "${keyauth}" + fi + + # remove temporary domains.txt file if used + [[ -n "${PARAM_DOMAIN:-}" && -n "${DOMAINS_TXT:-}" ]] && rm "${DOMAINS_TXT}" + exit 1 + fi + + cat "${tempcont}" + rm -f "${tempcont}" +} + +# Send signed request +signed_request() { + # Encode payload as urlbase64 + payload64="$(printf '%s' "${2}" | urlbase64)" + + # Retrieve nonce from acme-server + nonce="$(http_request head "${CA}" | grep Replay-Nonce: | awk -F ': ' '{print $2}' | tr -d '\n\r')" + + # Build header with just our public key and algorithm information + header='{"alg": "RS256", "jwk": {"e": "'"${pubExponent64}"'", "kty": "RSA", "n": "'"${pubMod64}"'"}}' + + # Build another header which also contains the previously received nonce and encode it as urlbase64 + protected='{"alg": "RS256", "jwk": {"e": "'"${pubExponent64}"'", "kty": "RSA", "n": "'"${pubMod64}"'"}, "nonce": "'"${nonce}"'"}' + protected64="$(printf '%s' "${protected}" | urlbase64)" + + # Sign header with nonce and our payload with our private key and encode signature as urlbase64 + signed64="$(printf '%s' "${protected64}.${payload64}" | openssl dgst -sha256 -sign "${ACCOUNT_KEY}" | urlbase64)" + + # Send header + extended header + payload + signature to the acme-server + data='{"header": '"${header}"', "protected": "'"${protected64}"'", "payload": "'"${payload64}"'", "signature": "'"${signed64}"'"}' + + http_request post "${1}" "${data}" +} + +# Extracts all subject names from a CSR +# Outputs either the CN, or the SANs, one per line +extract_altnames() { + csr="${1}" # the CSR itself (not a file) + + if ! <<<"${csr}" openssl req -verify -noout 2>/dev/null; then + _exiterr "Certificate signing request isn't valid" + fi + + reqtext="$( <<<"${csr}" openssl req -noout -text )" + if <<<"${reqtext}" grep -q '^[[:space:]]*X509v3 Subject Alternative Name:[[:space:]]*$'; then + # SANs used, extract these + altnames="$( <<<"${reqtext}" grep -A1 '^[[:space:]]*X509v3 Subject Alternative Name:[[:space:]]*$' | tail -n1 )" + # split to one per line: + # shellcheck disable=SC1003 + altnames="$( <<<"${altnames}" _sed -e 's/^[[:space:]]*//; s/, /\'$'\n''/g' )" + # we can only get DNS: ones signed + if grep -qv '^DNS:' <<<"${altnames}"; then + _exiterr "Certificate signing request contains non-DNS Subject Alternative Names" + fi + # strip away the DNS: prefix + altnames="$( <<<"${altnames}" _sed -e 's/^DNS://' )" + echo "${altnames}" + + else + # No SANs, extract CN + altnames="$( <<<"${reqtext}" grep '^[[:space:]]*Subject:' | _sed -e 's/.* CN=([^ /,]*).*/\1/' )" + echo "${altnames}" + fi +} + +# Create certificate for domain(s) and outputs it FD 3 +sign_csr() { + csr="${1}" # the CSR itself (not a file) + + if { true >&3; } 2>/dev/null; then + : # fd 3 looks OK + else + _exiterr "sign_csr: FD 3 not open" + fi + + shift 1 || true + altnames="${*:-}" + if [ -z "${altnames}" ]; then + altnames="$( extract_altnames "${csr}" )" + fi + + if [[ -z "${CA_NEW_AUTHZ}" ]] || [[ -z "${CA_NEW_CERT}" ]]; then + _exiterr "Certificate authority doesn't allow certificate signing" + fi + + local idx=0 + if [[ -n "${ZSH_VERSION:-}" ]]; then + local -A challenge_uris challenge_tokens keyauths deploy_args + else + local -a challenge_uris challenge_tokens keyauths deploy_args + fi + + # Request challenges + for altname in ${altnames}; do + # Ask the acme-server for new challenge token and extract them from the resulting json block + echo " + Requesting challenge for ${altname}..." + response="$(signed_request "${CA_NEW_AUTHZ}" '{"resource": "new-authz", "identifier": {"type": "dns", "value": "'"${altname}"'"}}' | clean_json)" + + challenges="$(printf '%s\n' "${response}" | sed -n 's/.*\("challenges":[^\[]*\[[^]]*]\).*/\1/p')" + repl=$'\n''{' # fix syntax highlighting in Vim + challenge="$(printf "%s" "${challenges//\{/${repl}}" | grep \""${CHALLENGETYPE}"\")" + challenge_token="$(printf '%s' "${challenge}" | get_json_string_value token | _sed 's/[^A-Za-z0-9_\-]/_/g')" + challenge_uri="$(printf '%s' "${challenge}" | get_json_string_value uri)" + + if [[ -z "${challenge_token}" ]] || [[ -z "${challenge_uri}" ]]; then + _exiterr "Can't retrieve challenges (${response})" + fi + + # Challenge response consists of the challenge token and the thumbprint of our public certificate + keyauth="${challenge_token}.${thumbprint}" + + case "${CHALLENGETYPE}" in + "http-01") + # Store challenge response in well-known location and make world-readable (so that a webserver can access it) + printf '%s' "${keyauth}" > "${WELLKNOWN}/${challenge_token}" + chmod a+r "${WELLKNOWN}/${challenge_token}" + keyauth_hook="${keyauth}" + ;; + "dns-01") + # Generate DNS entry content for dns-01 validation + keyauth_hook="$(printf '%s' "${keyauth}" | openssl dgst -sha256 -binary | urlbase64)" + ;; + esac + + challenge_uris[${idx}]="${challenge_uri}" + keyauths[${idx}]="${keyauth}" + challenge_tokens[${idx}]="${challenge_token}" + # Note: assumes args will never have spaces! + deploy_args[${idx}]="${altname} ${challenge_token} ${keyauth_hook}" + idx=$((idx+1)) + done + + # Wait for hook script to deploy the challenges if used + # shellcheck disable=SC2068 + [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" = "yes" ]] && "${HOOK}" "deploy_challenge" ${deploy_args[@]} + + # Respond to challenges + idx=0 + for altname in ${altnames}; do + challenge_token="${challenge_tokens[${idx}]}" + keyauth="${keyauths[${idx}]}" + + # Wait for hook script to deploy the challenge if used + # shellcheck disable=SC2086 + [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && "${HOOK}" "deploy_challenge" ${deploy_args[${idx}]} + + # Ask the acme-server to verify our challenge and wait until it is no longer pending + echo " + Responding to challenge for ${altname}..." + result="$(signed_request "${challenge_uris[${idx}]}" '{"resource": "challenge", "keyAuthorization": "'"${keyauth}"'"}' | clean_json)" + + reqstatus="$(printf '%s\n' "${result}" | get_json_string_value status)" + + while [[ "${reqstatus}" = "pending" ]]; do + sleep 1 + result="$(http_request get "${challenge_uris[${idx}]}")" + reqstatus="$(printf '%s\n' "${result}" | get_json_string_value status)" + done + + [[ "${CHALLENGETYPE}" = "http-01" ]] && rm -f "${WELLKNOWN}/${challenge_token}" + + # Wait for hook script to clean the challenge if used + if [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && [[ -n "${challenge_token}" ]]; then + # shellcheck disable=SC2086 + "${HOOK}" "clean_challenge" ${deploy_args[${idx}]} + fi + idx=$((idx+1)) + + if [[ "${reqstatus}" = "valid" ]]; then + echo " + Challenge is valid!" + else + break + fi + done + + # Wait for hook script to clean the challenges if used + # shellcheck disable=SC2068 + [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" = "yes" ]] && "${HOOK}" "clean_challenge" ${deploy_args[@]} + + if [[ "${reqstatus}" != "valid" ]]; then + # Clean up any remaining challenge_tokens if we stopped early + if [[ "${CHALLENGETYPE}" = "http-01" ]]; then + while [ ${idx} -lt ${#challenge_tokens[@]} ]; do + rm -f "${WELLKNOWN}/${challenge_tokens[${idx}]}" + idx=$((idx+1)) + done + fi + + _exiterr "Challenge is invalid! (returned: ${reqstatus}) (result: ${result})" + fi + + # Finally request certificate from the acme-server and store it in cert-${timestamp}.pem and link from cert.pem + echo " + Requesting certificate..." + csr64="$( <<<"${csr}" openssl req -outform DER | urlbase64)" + crt64="$(signed_request "${CA_NEW_CERT}" '{"resource": "new-cert", "csr": "'"${csr64}"'"}' | openssl base64 -e)" + crt="$( printf -- '-----BEGIN CERTIFICATE-----\n%s\n-----END CERTIFICATE-----\n' "${crt64}" )" + + # Try to load the certificate to detect corruption + echo " + Checking certificate..." + _openssl x509 -text <<<"${crt}" + + echo "${crt}" >&3 + + unset challenge_token + echo " + Done!" +} + +# Create certificate for domain(s) +sign_domain() { + domain="${1}" + altnames="${*}" + timestamp="$(date +%s)" + + echo " + Signing domains..." + if [[ -z "${CA_NEW_AUTHZ}" ]] || [[ -z "${CA_NEW_CERT}" ]]; then + _exiterr "Certificate authority doesn't allow certificate signing" + fi + + # If there is no existing certificate directory => make it + if [[ ! -e "${CERTDIR}/${domain}" ]]; then + echo " + Creating new directory ${CERTDIR}/${domain} ..." + mkdir -p "${CERTDIR}/${domain}" || _exiterr "Unable to create directory ${CERTDIR}/${domain}" + fi + + privkey="privkey.pem" + # generate a new private key if we need or want one + if [[ ! -r "${CERTDIR}/${domain}/privkey.pem" ]] || [[ "${PRIVATE_KEY_RENEW}" = "yes" ]]; then + echo " + Generating private key..." + privkey="privkey-${timestamp}.pem" + case "${KEY_ALGO}" in + rsa) _openssl genrsa -out "${CERTDIR}/${domain}/privkey-${timestamp}.pem" "${KEYSIZE}";; + prime256v1|secp384r1) _openssl ecparam -genkey -name "${KEY_ALGO}" -out "${CERTDIR}/${domain}/privkey-${timestamp}.pem";; + esac + fi + + # Generate signing request config and the actual signing request + echo " + Generating signing request..." + SAN="" + for altname in ${altnames}; do + SAN+="DNS:${altname}, " + done + SAN="${SAN%%, }" + local tmp_openssl_cnf + tmp_openssl_cnf="$(_mktemp)" + cat "${OPENSSL_CNF}" > "${tmp_openssl_cnf}" + printf "[SAN]\nsubjectAltName=%s" "${SAN}" >> "${tmp_openssl_cnf}" + if [ "${OCSP_MUST_STAPLE}" = "yes" ]; then + printf "\n1.3.6.1.5.5.7.1.24=DER:30:03:02:01:05" >> "${tmp_openssl_cnf}" + fi + openssl req -new -sha256 -key "${CERTDIR}/${domain}/${privkey}" -out "${CERTDIR}/${domain}/cert-${timestamp}.csr" -subj "/CN=${domain}/" -reqexts SAN -config "${tmp_openssl_cnf}" + rm -f "${tmp_openssl_cnf}" + + crt_path="${CERTDIR}/${domain}/cert-${timestamp}.pem" + # shellcheck disable=SC2086 + sign_csr "$(< "${CERTDIR}/${domain}/cert-${timestamp}.csr" )" ${altnames} 3>"${crt_path}" + + # Create fullchain.pem + echo " + Creating fullchain.pem..." + cat "${crt_path}" > "${CERTDIR}/${domain}/fullchain-${timestamp}.pem" + http_request get "$(openssl x509 -in "${CERTDIR}/${domain}/cert-${timestamp}.pem" -noout -text | grep 'CA Issuers - URI:' | cut -d':' -f2-)" > "${CERTDIR}/${domain}/chain-${timestamp}.pem" + if ! grep -q "BEGIN CERTIFICATE" "${CERTDIR}/${domain}/chain-${timestamp}.pem"; then + openssl x509 -in "${CERTDIR}/${domain}/chain-${timestamp}.pem" -inform DER -out "${CERTDIR}/${domain}/chain-${timestamp}.pem" -outform PEM + fi + cat "${CERTDIR}/${domain}/chain-${timestamp}.pem" >> "${CERTDIR}/${domain}/fullchain-${timestamp}.pem" + + # Update symlinks + [[ "${privkey}" = "privkey.pem" ]] || ln -sf "privkey-${timestamp}.pem" "${CERTDIR}/${domain}/privkey.pem" + + ln -sf "chain-${timestamp}.pem" "${CERTDIR}/${domain}/chain.pem" + ln -sf "fullchain-${timestamp}.pem" "${CERTDIR}/${domain}/fullchain.pem" + ln -sf "cert-${timestamp}.csr" "${CERTDIR}/${domain}/cert.csr" + ln -sf "cert-${timestamp}.pem" "${CERTDIR}/${domain}/cert.pem" + + # Wait for hook script to clean the challenge and to deploy cert if used + export KEY_ALGO + [[ -n "${HOOK}" ]] && "${HOOK}" "deploy_cert" "${domain}" "${CERTDIR}/${domain}/privkey.pem" "${CERTDIR}/${domain}/cert.pem" "${CERTDIR}/${domain}/fullchain.pem" "${CERTDIR}/${domain}/chain.pem" "${timestamp}" + + unset challenge_token + echo " + Done!" +} + +# Usage: --cron (-c) +# Description: Sign/renew non-existant/changed/expiring certificates. +command_sign_domains() { + init_system + + if [[ -n "${PARAM_DOMAIN:-}" ]]; then + DOMAINS_TXT="$(_mktemp)" + printf -- "${PARAM_DOMAIN}" > "${DOMAINS_TXT}" + elif [[ -e "${DOMAINS_TXT}" ]]; then + if [[ ! -r "${DOMAINS_TXT}" ]]; then + _exiterr "domains.txt found but not readable" + fi + else + _exiterr "domains.txt not found and --domain not given" + fi + + # Generate certificates for all domains found in domains.txt. Check if existing certificate are about to expire + ORIGIFS="${IFS}" + IFS=$'\n' + for line in $(<"${DOMAINS_TXT}" tr -d '\r' | tr '[:upper:]' '[:lower:]' | _sed -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*$//g' -e 's/[[:space:]]+/ /g' | (grep -vE '^(#|$)' || true)); do + reset_configvars + IFS="${ORIGIFS}" + domain="$(printf '%s\n' "${line}" | cut -d' ' -f1)" + morenames="$(printf '%s\n' "${line}" | cut -s -d' ' -f2-)" + cert="${CERTDIR}/${domain}/cert.pem" + + force_renew="${PARAM_FORCE:-no}" + + if [[ -z "${morenames}" ]];then + echo "Processing ${domain}" + else + echo "Processing ${domain} with alternative names: ${morenames}" + fi + + # read cert config + # for now this loads the certificate specific config in a subshell and parses a diff of set variables. + # we could just source the config file but i decided to go this way to protect people from accidentally overriding + # variables used internally by this script itself. + if [[ -n "${DOMAINS_D}" ]]; then + certconfig="${DOMAINS_D}/${domain}" + else + certconfig="${CERTDIR}/${domain}/config" + fi + + if [ -f "${certconfig}" ]; then + echo " + Using certificate specific config file!" + ORIGIFS="${IFS}" + IFS=$'\n' + for cfgline in $( + beforevars="$(_mktemp)" + aftervars="$(_mktemp)" + set > "${beforevars}" + # shellcheck disable=SC1090 + . "${certconfig}" + set > "${aftervars}" + diff -u "${beforevars}" "${aftervars}" | grep -E '^\+[^+]' + rm "${beforevars}" + rm "${aftervars}" + ); do + config_var="$(echo "${cfgline:1}" | cut -d'=' -f1)" + config_value="$(echo "${cfgline:1}" | cut -d'=' -f2-)" + case "${config_var}" in + KEY_ALGO|OCSP_MUST_STAPLE|PRIVATE_KEY_RENEW|KEYSIZE|CHALLENGETYPE|HOOK|WELLKNOWN|HOOK_CHAIN|OPENSSL_CNF|RENEW_DAYS) + echo " + ${config_var} = ${config_value}" + declare -- "${config_var}=${config_value}" + ;; + _) ;; + *) echo " ! Setting ${config_var} on a per-certificate base is not (yet) supported" + esac + done + IFS="${ORIGIFS}" + fi + verify_config + + if [[ -e "${cert}" ]]; then + printf " + Checking domain name(s) of existing cert..." + + certnames="$(openssl x509 -in "${cert}" -text -noout | grep DNS: | _sed 's/DNS://g' | tr -d ' ' | tr ',' '\n' | sort -u | tr '\n' ' ' | _sed 's/ $//')" + givennames="$(echo "${domain}" "${morenames}"| tr ' ' '\n' | sort -u | tr '\n' ' ' | _sed 's/ $//' | _sed 's/^ //')" + + if [[ "${certnames}" = "${givennames}" ]]; then + echo " unchanged." + else + echo " changed!" + echo " + Domain name(s) are not matching!" + echo " + Names in old certificate: ${certnames}" + echo " + Configured names: ${givennames}" + echo " + Forcing renew." + force_renew="yes" + fi + fi + + if [[ -e "${cert}" ]]; then + echo " + Checking expire date of existing cert..." + valid="$(openssl x509 -enddate -noout -in "${cert}" | cut -d= -f2- )" + + printf " + Valid till %s " "${valid}" + if openssl x509 -checkend $((RENEW_DAYS * 86400)) -noout -in "${cert}"; then + printf "(Longer than %d days). " "${RENEW_DAYS}" + if [[ "${force_renew}" = "yes" ]]; then + echo "Ignoring because renew was forced!" + else + # Certificate-Names unchanged and cert is still valid + echo "Skipping renew!" + [[ -n "${HOOK}" ]] && "${HOOK}" "unchanged_cert" "${domain}" "${CERTDIR}/${domain}/privkey.pem" "${CERTDIR}/${domain}/cert.pem" "${CERTDIR}/${domain}/fullchain.pem" "${CERTDIR}/${domain}/chain.pem" + continue + fi + else + echo "(Less than ${RENEW_DAYS} days). Renewing!" + fi + fi + + # shellcheck disable=SC2086 + if [[ "${PARAM_KEEP_GOING:-}" = "yes" ]]; then + sign_domain ${line} & + wait $! || true + else + sign_domain ${line} + fi + done + + # remove temporary domains.txt file if used + [[ -n "${PARAM_DOMAIN:-}" ]] && rm -f "${DOMAINS_TXT}" + + exit 0 +} + +# Usage: --signcsr (-s) path/to/csr.pem +# Description: Sign a given CSR, output CRT on stdout (advanced usage) +command_sign_csr() { + # redirect stdout to stderr + # leave stdout over at fd 3 to output the cert + exec 3>&1 1>&2 + + init_system + + csrfile="${1}" + if [ ! -r "${csrfile}" ]; then + _exiterr "Could not read certificate signing request ${csrfile}" + fi + + # gen cert + certfile="$(_mktemp)" + sign_csr "$(< "${csrfile}" )" 3> "${certfile}" + + # print cert + echo "# CERT #" >&3 + cat "${certfile}" >&3 + echo >&3 + + # print chain + if [ -n "${PARAM_FULL_CHAIN:-}" ]; then + # get and convert ca cert + chainfile="$(_mktemp)" + http_request get "$(openssl x509 -in "${certfile}" -noout -text | grep 'CA Issuers - URI:' | cut -d':' -f2-)" > "${chainfile}" + + if ! grep -q "BEGIN CERTIFICATE" "${chainfile}"; then + openssl x509 -inform DER -in "${chainfile}" -outform PEM -out "${chainfile}" + fi + + echo "# CHAIN #" >&3 + cat "${chainfile}" >&3 + + rm "${chainfile}" + fi + + # cleanup + rm "${certfile}" + + exit 0 +} + +# Usage: --revoke (-r) path/to/cert.pem +# Description: Revoke specified certificate +command_revoke() { + init_system + + [[ -n "${CA_REVOKE_CERT}" ]] || _exiterr "Certificate authority doesn't allow certificate revocation." + + cert="${1}" + if [[ -L "${cert}" ]]; then + # follow symlink and use real certificate name (so we move the real file and not the symlink at the end) + local link_target + link_target="$(readlink -n "${cert}")" + if [[ "${link_target}" =~ ^/ ]]; then + cert="${link_target}" + else + cert="$(dirname "${cert}")/${link_target}" + fi + fi + [[ -f "${cert}" ]] || _exiterr "Could not find certificate ${cert}" + + echo "Revoking ${cert}" + + cert64="$(openssl x509 -in "${cert}" -inform PEM -outform DER | urlbase64)" + response="$(signed_request "${CA_REVOKE_CERT}" '{"resource": "revoke-cert", "certificate": "'"${cert64}"'"}' | clean_json)" + # if there is a problem with our revoke request _request (via signed_request) will report this and "exit 1" out + # so if we are here, it is safe to assume the request was successful + echo " + Done." + echo " + Renaming certificate to ${cert}-revoked" + mv -f "${cert}" "${cert}-revoked" +} + +# Usage: --cleanup (-gc) +# Description: Move unused certificate files to archive directory +command_cleanup() { + load_config + + # Create global archive directory if not existant + if [[ ! -e "${BASEDIR}/archive" ]]; then + mkdir "${BASEDIR}/archive" + fi + + # Loop over all certificate directories + for certdir in "${CERTDIR}/"*; do + # Skip if entry is not a folder + [[ -d "${certdir}" ]] || continue + + # Get certificate name + certname="$(basename "${certdir}")" + + # Create certitifaces archive directory if not existant + archivedir="${BASEDIR}/archive/${certname}" + if [[ ! -e "${archivedir}" ]]; then + mkdir "${archivedir}" + fi + + # Loop over file-types (certificates, keys, signing-requests, ...) + for filetype in cert.csr cert.pem chain.pem fullchain.pem privkey.pem; do + # Skip if symlink is broken + [[ -r "${certdir}/${filetype}" ]] || continue + + # Look up current file in use + current="$(basename "$(readlink "${certdir}/${filetype}")")" + + # Split filetype into name and extension + filebase="$(echo "${filetype}" | cut -d. -f1)" + fileext="$(echo "${filetype}" | cut -d. -f2)" + + # Loop over all files of this type + for file in "${certdir}/${filebase}-"*".${fileext}"; do + # Handle case where no files match the wildcard + [[ -f "${file}" ]] || break + + # Check if current file is in use, if unused move to archive directory + filename="$(basename "${file}")" + if [[ ! "${filename}" = "${current}" ]]; then + echo "Moving unused file to archive directory: ${certname}/${filename}" + mv "${certdir}/${filename}" "${archivedir}/${filename}" + fi + done + done + done + + exit 0 +} + +# Usage: --help (-h) +# Description: Show help text +command_help() { + printf "Usage: %s [-h] [command [argument]] [parameter [argument]] [parameter [argument]] ...\n\n" "${0}" + printf "Default command: help\n\n" + echo "Commands:" + grep -e '^[[:space:]]*# Usage:' -e '^[[:space:]]*# Description:' -e '^command_.*()[[:space:]]*{' "${0}" | while read -r usage; read -r description; read -r command; do + if [[ ! "${usage}" =~ Usage ]] || [[ ! "${description}" =~ Description ]] || [[ ! "${command}" =~ ^command_ ]]; then + _exiterr "Error generating help text." + fi + printf " %-32s %s\n" "${usage##"# Usage: "}" "${description##"# Description: "}" + done + printf -- "\nParameters:\n" + grep -E -e '^[[:space:]]*# PARAM_Usage:' -e '^[[:space:]]*# PARAM_Description:' "${0}" | while read -r usage; read -r description; do + if [[ ! "${usage}" =~ Usage ]] || [[ ! "${description}" =~ Description ]]; then + _exiterr "Error generating help text." + fi + printf " %-32s %s\n" "${usage##"# PARAM_Usage: "}" "${description##"# PARAM_Description: "}" + done +} + +# Usage: --env (-e) +# Description: Output configuration variables for use in other scripts +command_env() { + echo "# dehydrated configuration" + load_config + typeset -p CA LICENSE CERTDIR CHALLENGETYPE DOMAINS_D DOMAINS_TXT HOOK HOOK_CHAIN RENEW_DAYS ACCOUNT_KEY ACCOUNT_KEY_JSON KEYSIZE WELLKNOWN PRIVATE_KEY_RENEW OPENSSL_CNF CONTACT_EMAIL LOCKFILE +} + +# Main method (parses script arguments and calls command_* methods) +main() { + COMMAND="" + set_command() { + [[ -z "${COMMAND}" ]] || _exiterr "Only one command can be executed at a time. See help (-h) for more information." + COMMAND="${1}" + } + + check_parameters() { + if [[ -z "${1:-}" ]]; then + echo "The specified command requires additional parameters. See help:" >&2 + echo >&2 + command_help >&2 + exit 1 + elif [[ "${1:0:1}" = "-" ]]; then + _exiterr "Invalid argument: ${1}" + fi + } + + [[ -z "${@}" ]] && eval set -- "--help" + + while (( ${#} )); do + case "${1}" in + --help|-h) + command_help + exit 0 + ;; + + --env|-e) + set_command env + ;; + + --cron|-c) + set_command sign_domains + ;; + + --signcsr|-s) + shift 1 + set_command sign_csr + check_parameters "${1:-}" + PARAM_CSR="${1}" + ;; + + --revoke|-r) + shift 1 + set_command revoke + check_parameters "${1:-}" + PARAM_REVOKECERT="${1}" + ;; + + --cleanup|-gc) + set_command cleanup + ;; + + # PARAM_Usage: --full-chain (-fc) + # PARAM_Description: Print full chain when using --signcsr + --full-chain|-fc) + PARAM_FULL_CHAIN="1" + ;; + + # PARAM_Usage: --ipv4 (-4) + # PARAM_Description: Resolve names to IPv4 addresses only + --ipv4|-4) + PARAM_IP_VERSION="4" + ;; + + # PARAM_Usage: --ipv6 (-6) + # PARAM_Description: Resolve names to IPv6 addresses only + --ipv6|-6) + PARAM_IP_VERSION="6" + ;; + + # PARAM_Usage: --domain (-d) domain.tld + # PARAM_Description: Use specified domain name(s) instead of domains.txt entry (one certificate!) + --domain|-d) + shift 1 + check_parameters "${1:-}" + if [[ -z "${PARAM_DOMAIN:-}" ]]; then + PARAM_DOMAIN="${1}" + else + PARAM_DOMAIN="${PARAM_DOMAIN} ${1}" + fi + ;; + + # PARAM_Usage: --keep-going (-g) + # PARAM_Description: Keep going after encountering an error while creating/renewing multiple certificates in cron mode + --keep-going|-g) + PARAM_KEEP_GOING="yes" + ;; + + # PARAM_Usage: --force (-x) + # PARAM_Description: Force renew of certificate even if it is longer valid than value in RENEW_DAYS + --force|-x) + PARAM_FORCE="yes" + ;; + + # PARAM_Usage: --no-lock (-n) + # PARAM_Description: Don't use lockfile (potentially dangerous!) + --no-lock|-n) + PARAM_NO_LOCK="yes" + ;; + + # PARAM_Usage: --ocsp + # PARAM_Description: Sets option in CSR indicating OCSP stapling to be mandatory + --ocsp) + PARAM_OCSP_MUST_STAPLE="yes" + ;; + + # PARAM_Usage: --privkey (-p) path/to/key.pem + # PARAM_Description: Use specified private key instead of account key (useful for revocation) + --privkey|-p) + shift 1 + check_parameters "${1:-}" + PARAM_ACCOUNT_KEY="${1}" + ;; + + # PARAM_Usage: --config (-f) path/to/config + # PARAM_Description: Use specified config file + --config|-f) + shift 1 + check_parameters "${1:-}" + CONFIG="${1}" + ;; + + # PARAM_Usage: --hook (-k) path/to/hook.sh + # PARAM_Description: Use specified script for hooks + --hook|-k) + shift 1 + check_parameters "${1:-}" + PARAM_HOOK="${1}" + ;; + + # PARAM_Usage: --out (-o) certs/directory + # PARAM_Description: Output certificates into the specified directory + --out|-o) + shift 1 + check_parameters "${1:-}" + PARAM_CERTDIR="${1}" + ;; + + # PARAM_Usage: --challenge (-t) http-01|dns-01 + # PARAM_Description: Which challenge should be used? Currently http-01 and dns-01 are supported + --challenge|-t) + shift 1 + check_parameters "${1:-}" + PARAM_CHALLENGETYPE="${1}" + ;; + + # PARAM_Usage: --algo (-a) rsa|prime256v1|secp384r1 + # PARAM_Description: Which public key algorithm should be used? Supported: rsa, prime256v1 and secp384r1 + --algo|-a) + shift 1 + check_parameters "${1:-}" + PARAM_KEY_ALGO="${1}" + ;; + + *) + echo "Unknown parameter detected: ${1}" >&2 + echo >&2 + command_help >&2 + exit 1 + ;; + esac + + shift 1 + done + + case "${COMMAND}" in + env) command_env;; + sign_domains) command_sign_domains;; + sign_csr) command_sign_csr "${PARAM_CSR}";; + revoke) command_revoke "${PARAM_REVOKECERT}";; + cleanup) command_cleanup;; + *) command_help; exit 1;; + esac +} + +# Determine OS type +OSTYPE="$(uname)" + +# Check for missing dependencies +check_dependencies + +# Run script +main "${@:-}" From 9596d9b301733ea8f720a55bc8980db7f28b6325 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 29 Sep 2016 20:50:31 -0400 Subject: [PATCH 02/47] config and hooks for dehydrated, hooks will propogate challenge/cert data to consul --- etc/dehydrated/config | 87 ++++++++++++++++++++++++++++++++++++++++++ etc/dehydrated/hook.sh | 45 ++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 etc/dehydrated/config create mode 100755 etc/dehydrated/hook.sh diff --git a/etc/dehydrated/config b/etc/dehydrated/config new file mode 100644 index 0000000..b7277ba --- /dev/null +++ b/etc/dehydrated/config @@ -0,0 +1,87 @@ +######################################################## +# This is the main config file for dehydrated # +# # +# This file is looked for in the following locations: # +# $SCRIPTDIR/config (next to this script) # +# /usr/local/etc/dehydrated/config # +# /etc/dehydrated/config # +# ${PWD}/config (in current working-directory) # +# # +# Default values of this config are in comments # +######################################################## + +# Resolve names to addresses of IP version only. (curl) +# supported values: 4, 6 +# default: +#IP_VERSION= + +# TODO: Make CA configurable through env var? +# Path to certificate authority (default: https://acme-v01.api.letsencrypt.org/directory) +#CA="https://acme-v01.api.letsencrypt.org/directory" +CA="https://acme-staging.api.letsencrypt.org/directory" + +# Path to license agreement (default: https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf) +#LICENSE="https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf" + +# Which challenge should be used? Currently http-01 and dns-01 are supported +#CHALLENGETYPE="http-01" + +# Path to a directory containing additional config files, allowing to override +# the defaults found in the main configuration file. Additional config files +# in this directory needs to be named with a '.sh' ending. +# default: +#CONFIG_D= + +# Base directory for account key, generated certificates and list of domains (default: $SCRIPTDIR -- uses config directory if undefined) +#BASEDIR=$SCRIPTDIR + +# File containing the list of domains to request certificates for (default: $BASEDIR/domains.txt) +#DOMAINS_TXT="${BASEDIR}/domains.txt" + +# Output directory for generated certificates +#CERTDIR="${BASEDIR}/certs" + +# Directory for account keys and registration information +#ACCOUNTDIR="${BASEDIR}/accounts" + +# Output directory for challenge-tokens to be served by webserver or deployed in HOOK (default: /var/www/dehydrated) +#WELLKNOWN="/var/www/dehydrated" + +# Default keysize for private keys (default: 4096) +#KEYSIZE="4096" + +# Path to openssl config file (default: - tries to figure out system default) +#OPENSSL_CNF= + +# Program or function called in certain situations +# +# After generating the challenge-response, or after failed challenge (in this case altname is empty) +# Given arguments: clean_challenge|deploy_challenge altname token-filename token-content +# +# After successfully signing certificate +# Given arguments: deploy_cert domain path/to/privkey.pem path/to/cert.pem path/to/fullchain.pem +# +# BASEDIR and WELLKNOWN variables are exported and can be used in an external program +# default: +#HOOK= + +# Chain clean_challenge|deploy_challenge arguments together into one hook call per certificate (default: no) +#HOOK_CHAIN="no" + +# Minimum days before expiration to automatically renew certificate (default: 30) +#RENEW_DAYS="30" + +# Regenerate private keys instead of just signing new certificates on renewal (default: yes) +#PRIVATE_KEY_RENEW="yes" + +# Which public key algorithm should be used? Supported: rsa, prime256v1 and secp384r1 +#KEY_ALGO=rsa + +# E-mail to use during the registration (default: ) +#CONTACT_EMAIL= + +# Lockfile location, to prevent concurrent access (default: $BASEDIR/lock) +#LOCKFILE="${BASEDIR}/lock" + +# Option to add CSR-flag indicating OCSP stapling to be mandatory (default: no) +#OCSP_MUST_STAPLE="no" diff --git a/etc/dehydrated/hook.sh b/etc/dehydrated/hook.sh new file mode 100755 index 0000000..c64db15 --- /dev/null +++ b/etc/dehydrated/hook.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +CONSUL_ROOT="http://localhost:8500/v1/kv/nginx" +CHALLENGE_PATH="/.well-known/acme-challenge" + +function deploy_challenge { + local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" + + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-filename > /dev/null + curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-value > /dev/null + + # for each domain verify all nginx containers are responding with challenge before continuing + # TODO: this is currently only looking at localhost + printf " + Waiting for challenge to be deployed..." + while [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://localhost${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" ]; do + printf "." + sleep 2 + done + echo +} + +function clean_challenge { + local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" + + curl -sX DELETE ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-filename > /dev/null + curl -sX DELETE ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-value > /dev/null + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/last-token-filename > /dev/null +} + +function deploy_cert { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" + + curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/key > /dev/null + curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/cert > /dev/null + curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/chain > /dev/null + curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/fullchain > /dev/null + curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/timestamp > /dev/null + curl -sX PUT -d "$(date +%s)" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/touched > /dev/null +} + +function unchanged_cert { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" +} + +HANDLER=$1; shift; $HANDLER $@ From 974ddbd4a44228321b4e29146e8fdcc22a3b656b Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 29 Sep 2016 20:52:28 -0400 Subject: [PATCH 03/47] cannot generate dynamic filenames from consul-template, so hacking it --- bin/generate-token | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 bin/generate-token diff --git a/bin/generate-token b/bin/generate-token new file mode 100755 index 0000000..166b19e --- /dev/null +++ b/bin/generate-token @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +DOMAIN=$1 +IN="/var/ssl/acme" +OUT="/var/ssl/acme/challenge" +FILENAME="$(sed '1q;d' ${IN}/${DOMAIN}-token)" +VALUE="$(sed '2q;d' ${IN}/${DOMAIN}-token)" +LAST_FILENAME="$(sed '3q;d' ${IN}/${DOMAIN}-token)" + +if [ "${FILENAME}" ]; then + echo "${VALUE}" > ${OUT}/${FILENAME} +fi + +if [ "${LAST_FILENAME}" ]; then + rm -f ${OUT}/${LAST_FILENAME} +fi From 9b4d272278a44025c420fc4eae42159dc57ff15a Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 29 Sep 2016 20:53:19 -0400 Subject: [PATCH 04/47] respond to acme challenges if acme is enabled --- etc/nginx/nginx.conf.ctmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/nginx/nginx.conf.ctmpl b/etc/nginx/nginx.conf.ctmpl index eb1c9ba..91a0d7a 100644 --- a/etc/nginx/nginx.conf.ctmpl +++ b/etc/nginx/nginx.conf.ctmpl @@ -30,6 +30,7 @@ http { #gzip on; {{ $backend := env "BACKEND" }} + {{ $acme := env "ENABLE_ACME" }} {{ if service $backend }} upstream {{ $backend }} { # write the address:port pairs for each healthy backend instance @@ -48,6 +49,12 @@ http { allow 127.0.0.1; deny all; } + + {{ if $acme }} + location /.well-known/acme-challenge { + alias /var/www/acme/challenge; + }{{ end }} + {{ if service $backend }} location = /{{ $backend }} { return 301 /{{ $backend }}/; From 7b1cc7855922c4bc394a4e29c8afc344548be279 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 14:38:53 -0400 Subject: [PATCH 05/47] Respect CONSUL_AGENT and CONSUL env vars, simplify kv structure (1 domain for now), return from hook funcs early on failure, limit retries on verifying challenge token has been distributed --- etc/dehydrated/hook.sh | 59 +++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/etc/dehydrated/hook.sh b/etc/dehydrated/hook.sh index c64db15..6d775b3 100755 --- a/etc/dehydrated/hook.sh +++ b/etc/dehydrated/hook.sh @@ -1,18 +1,26 @@ #!/usr/bin/env bash - -CONSUL_ROOT="http://localhost:8500/v1/kv/nginx" +CONSUL_HOST_DEFAULT="localhost" +if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then + CONSUL_HOST_DEFAULT=${CONSUL} +fi +CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} +CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1/kv/nginx" CHALLENGE_PATH="/.well-known/acme-challenge" function deploy_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-filename > /dev/null - curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-value > /dev/null + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/acme/challenge/token-filename | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_ROOT}/acme/challenge/token-value | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - # for each domain verify all nginx containers are responding with challenge before continuing + # verify all nginx containers are responding with challenge before continuing # TODO: this is currently only looking at localhost + RETRIES=0 printf " + Waiting for challenge to be deployed..." - while [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://localhost${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" ]; do + while [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://localhost${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" -a $RETRIES -lt 6 ]; do + RETRIES=$((RETRIES+1)) printf "." sleep 2 done @@ -22,24 +30,45 @@ function deploy_challenge { function clean_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" - curl -sX DELETE ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-filename > /dev/null - curl -sX DELETE ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/token-value > /dev/null - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/challenge/last-token-filename > /dev/null + curl -sX DELETE ${CONSUL_ROOT}/acme/challenge/token-filename | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX DELETE ${CONSUL_ROOT}/acme/challenge/token-value | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/acme/challenge/last-token-filename | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) } function deploy_cert { local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" - curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/key > /dev/null - curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/cert > /dev/null - curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/chain > /dev/null - curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/fullchain > /dev/null - curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/timestamp > /dev/null - curl -sX PUT -d "$(date +%s)" ${CONSUL_ROOT}/domains/${DOMAIN}/acme/touched > /dev/null + curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_ROOT}/acme/key | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_ROOT}/acme/cert | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_ROOT}/acme/chain | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_ROOT}/acme/fullchain | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_ROOT}/acme/timestamp | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "$(date +%s)" ${CONSUL_ROOT}/acme/touched | log + test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) } function unchanged_cert { local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" } +function log { + if [ -n "$1" ]; then + IN="$1" + else + read IN + fi + + if [ "${IN}" != "true" ]; then + echo " - ${IN}" + fi +} + HANDLER=$1; shift; $HANDLER $@ From 1b01db8b49cc370e36bd2436138b50182e25699b Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:03:07 -0400 Subject: [PATCH 06/47] adding ssl cert and acme challenge token templates --- etc/acme/templates/cert.ctmpl | 1 + etc/acme/templates/chain.ctmpl | 1 + etc/acme/templates/challenge-token.ctmpl | 3 +++ etc/acme/templates/fullchain.ctmpl | 1 + etc/acme/templates/privkey.ctmpl | 1 + 5 files changed, 7 insertions(+) create mode 100644 etc/acme/templates/cert.ctmpl create mode 100644 etc/acme/templates/chain.ctmpl create mode 100644 etc/acme/templates/challenge-token.ctmpl create mode 100644 etc/acme/templates/fullchain.ctmpl create mode 100644 etc/acme/templates/privkey.ctmpl diff --git a/etc/acme/templates/cert.ctmpl b/etc/acme/templates/cert.ctmpl new file mode 100644 index 0000000..765aaf0 --- /dev/null +++ b/etc/acme/templates/cert.ctmpl @@ -0,0 +1 @@ +{{key "nginx/acme/cert"}} diff --git a/etc/acme/templates/chain.ctmpl b/etc/acme/templates/chain.ctmpl new file mode 100644 index 0000000..e0e0d94 --- /dev/null +++ b/etc/acme/templates/chain.ctmpl @@ -0,0 +1 @@ +{{key "nginx/acme/chain"}} diff --git a/etc/acme/templates/challenge-token.ctmpl b/etc/acme/templates/challenge-token.ctmpl new file mode 100644 index 0000000..895bf38 --- /dev/null +++ b/etc/acme/templates/challenge-token.ctmpl @@ -0,0 +1,3 @@ +{{key "nginx/acme/challenge/token-filename"}} +{{key "nginx/acme/challenge/token-value"}} +{{key "nginx/acme/challenge/last-token-filename"}} diff --git a/etc/acme/templates/fullchain.ctmpl b/etc/acme/templates/fullchain.ctmpl new file mode 100644 index 0000000..5db9394 --- /dev/null +++ b/etc/acme/templates/fullchain.ctmpl @@ -0,0 +1 @@ +{{key "nginx/acme/fullchain"}} diff --git a/etc/acme/templates/privkey.ctmpl b/etc/acme/templates/privkey.ctmpl new file mode 100644 index 0000000..704a1f8 --- /dev/null +++ b/etc/acme/templates/privkey.ctmpl @@ -0,0 +1 @@ +{{key "nginx/acme/key"}} From e73717e4ac052b3b9e2e69b86fca89e88c30e70b Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:04:04 -0400 Subject: [PATCH 07/47] corrected paths, removed consul host from config file to be specified at runtime --- etc/acme-watch.hcl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 etc/acme-watch.hcl diff --git a/etc/acme-watch.hcl b/etc/acme-watch.hcl new file mode 100644 index 0000000..44d13c2 --- /dev/null +++ b/etc/acme-watch.hcl @@ -0,0 +1,22 @@ +log_level = "err" +template { + source = "/etc/acme/templates/cert.ctmpl" + destination = "/var/www/ssl/cert.pem" +} +template { + source = "/etc/acme/templates/privkey.ctmpl" + destination = "/var/www/ssl/privkey.pem" +} +template { + source = "/etc/acme/templates/fullchain.ctmpl" + destination = "/var/www/ssl/fullchain.pem" +} +template { + source = "/etc/acme/templates/chain.ctmpl" + destination = "/var/www/ssl/chain.pem" +} +template { + source = "/etc/acme/templates/challenge-token.ctmpl" + destination = "/var/www/acme/challenge-token" + command = "/usr/local/bin/generate-token /var/www/acme/challenge-token /var/www/acme/challenge/" +} From ddf9fc584c96c7cf7c60e969b5205929b31ef8b4 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:04:25 -0400 Subject: [PATCH 08/47] moved acme watch file --- etc/{acme-watch.hcl => acme/watch.hcl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etc/{acme-watch.hcl => acme/watch.hcl} (100%) diff --git a/etc/acme-watch.hcl b/etc/acme/watch.hcl similarity index 100% rename from etc/acme-watch.hcl rename to etc/acme/watch.hcl From a59c152a89583374dc8efd8265bf3318035d1097 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:11:00 -0400 Subject: [PATCH 09/47] accept source and destination as args, remove source when done --- bin/generate-token | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bin/generate-token b/bin/generate-token index 166b19e..72c5137 100755 --- a/bin/generate-token +++ b/bin/generate-token @@ -1,10 +1,10 @@ #!/usr/bin/env bash -DOMAIN=$1 -IN="/var/ssl/acme" -OUT="/var/ssl/acme/challenge" -FILENAME="$(sed '1q;d' ${IN}/${DOMAIN}-token)" -VALUE="$(sed '2q;d' ${IN}/${DOMAIN}-token)" -LAST_FILENAME="$(sed '3q;d' ${IN}/${DOMAIN}-token)" + +IN=$1 +OUT=$2 +FILENAME="$(sed '1q;d' ${IN})" +VALUE="$(sed '2q;d' ${IN})" +LAST_FILENAME="$(sed '3q;d' ${IN})" if [ "${FILENAME}" ]; then echo "${VALUE}" > ${OUT}/${FILENAME} @@ -13,3 +13,5 @@ fi if [ "${LAST_FILENAME}" ]; then rm -f ${OUT}/${LAST_FILENAME} fi + +rm ${IN} From 6def3303855cdb5b1ebdfbcba1f218d7d622934e Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:11:24 -0400 Subject: [PATCH 10/47] Insure directories needed for acme/ssl operation are present --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 1f46676..76a662b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,8 @@ RUN export CONTAINERPILOT_CHECKSUM=ec9dbedaca9f4a7a50762f50768cbc42879c7208 \ # Add our configuration files and scripts COPY etc /etc COPY bin /usr/local/bin +RUN mkdir -p /var/www/ssl +RUN mkdir -p /var/www/acme/challenge CMD [ "/usr/local/bin/containerpilot", \ "nginx", \ From 9aa587a078adc07b4efa74a3bee1c14da2a48fbe Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:13:18 -0400 Subject: [PATCH 11/47] renamed env var to ACME_DOMAIN, supporting single domain for now --- etc/nginx/nginx.conf.ctmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/nginx/nginx.conf.ctmpl b/etc/nginx/nginx.conf.ctmpl index 91a0d7a..51add7b 100644 --- a/etc/nginx/nginx.conf.ctmpl +++ b/etc/nginx/nginx.conf.ctmpl @@ -30,7 +30,7 @@ http { #gzip on; {{ $backend := env "BACKEND" }} - {{ $acme := env "ENABLE_ACME" }} + {{ $acme_domain := env "ACME_DOMAIN" }} {{ if service $backend }} upstream {{ $backend }} { # write the address:port pairs for each healthy backend instance @@ -50,7 +50,7 @@ http { deny all; } - {{ if $acme }} + {{ if $acme_domain }} location /.well-known/acme-challenge { alias /var/www/acme/challenge; }{{ end }} From f113253d106622e7271194e56bfd76c7e91216a0 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 15:13:35 -0400 Subject: [PATCH 12/47] do not need trailing / --- etc/acme/watch.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/acme/watch.hcl b/etc/acme/watch.hcl index 44d13c2..a334608 100644 --- a/etc/acme/watch.hcl +++ b/etc/acme/watch.hcl @@ -18,5 +18,5 @@ template { template { source = "/etc/acme/templates/challenge-token.ctmpl" destination = "/var/www/acme/challenge-token" - command = "/usr/local/bin/generate-token /var/www/acme/challenge-token /var/www/acme/challenge/" + command = "/usr/local/bin/generate-token /var/www/acme/challenge-token /var/www/acme/challenge" } From a67a5676126872307da5e542cbe09a0c02a00f59 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 16:05:20 -0400 Subject: [PATCH 13/47] Added script for managing Consul sessions (for use with leader lock) --- Dockerfile | 5 +++++ bin/renew-consul-session | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100755 bin/renew-consul-session diff --git a/Dockerfile b/Dockerfile index 76a662b..77d9d7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,8 +46,13 @@ RUN export CONTAINERPILOT_CHECKSUM=ec9dbedaca9f4a7a50762f50768cbc42879c7208 \ # Add our configuration files and scripts COPY etc /etc COPY bin /usr/local/bin + +# SSL certs written here RUN mkdir -p /var/www/ssl +# ACME challenge tokens written here RUN mkdir -p /var/www/acme/challenge +# Consul session data written here +RUN mkdir -p /var/consul CMD [ "/usr/local/bin/containerpilot", \ "nginx", \ diff --git a/bin/renew-consul-session b/bin/renew-consul-session new file mode 100755 index 0000000..75424ba --- /dev/null +++ b/bin/renew-consul-session @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +CONSUL_HOST_DEFAULT="localhost" +if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then + CONSUL_HOST_DEFAULT=${CONSUL} +fi +CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} + +SESSION_DIR_DEFAULT="/var/consul" +SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} +SESSION_FILE=${SESSION_DIR}/session + +function getSession () { + if [ -f $SESSION_FILE ]; then + echo $(cat ${SESSION_DIR}/session) + else + echo "" + fi +} + +function renewSession () { + local SID="$(getSession)" + printf "Renewing Consul session ${SID}... " + local STATUS=$(curl -s -o /dev/null -X PUT -w '%{http_code}' http://$CONSUL_HOST:8500/v1/session/renew/${SID}) + if [ "${STATUS}" = "200" ]; then + echo "complete." + else + echo "failed." + createSession + fi +} + +function createSession () { + printf "Creating Consul session... " + local SID=$(curl -sX PUT -d '{"LockDelay":"0s","Name":"acme-lock","Behavior":"release","TTL":"600s"}' http://127.0.0.1:8500/v1/session/create | awk -F '"' '{print $4}') + rc=$?; if [[ $rc != 0 ]]; then + echo "failed." + else + echo $SID + echo $SID > $SESSION_FILE + fi +} + +if [ -f $SESSION_FILE ]; then + renewSession +else + createSession +fi + From ae56880f145a8b1f8056bda287b34674b93d3e57 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 17:06:53 -0400 Subject: [PATCH 14/47] Added script for managing leader aquisition (determines container responsible for renewing certs) --- bin/acquire-acme-leader | 21 +++++++++++++++++++++ bin/renew-consul-session | 6 +++--- 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100755 bin/acquire-acme-leader diff --git a/bin/acquire-acme-leader b/bin/acquire-acme-leader new file mode 100755 index 0000000..34f5754 --- /dev/null +++ b/bin/acquire-acme-leader @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +CONSUL_HOST_DEFAULT="localhost" +if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then + CONSUL_HOST_DEFAULT=${CONSUL} +fi +CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} +CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1/kv/nginx" + +SESSION_DIR_DEFAULT="/var/consul" +SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} +SESSION_FILE=${SESSION_DIR}/session + +SID=$(cat $SESSION_FILE) + +STATUS=$(curl -sX PUT -d "$(hostname)" -o /dev/null -w '%{http_code}' "${CONSUL_ROOT}/acme/leader?acquire=${SID}") +if [ "${STATUS}" = "200" ]; then + echo "ACME leader claimed" +else + echo "Failed to claim ACME leader" + exit 1 +fi diff --git a/bin/renew-consul-session b/bin/renew-consul-session index 75424ba..12f1205 100755 --- a/bin/renew-consul-session +++ b/bin/renew-consul-session @@ -22,9 +22,9 @@ function renewSession () { printf "Renewing Consul session ${SID}... " local STATUS=$(curl -s -o /dev/null -X PUT -w '%{http_code}' http://$CONSUL_HOST:8500/v1/session/renew/${SID}) if [ "${STATUS}" = "200" ]; then - echo "complete." + echo "complete" else - echo "failed." + echo "failed" createSession fi } @@ -33,7 +33,7 @@ function createSession () { printf "Creating Consul session... " local SID=$(curl -sX PUT -d '{"LockDelay":"0s","Name":"acme-lock","Behavior":"release","TTL":"600s"}' http://127.0.0.1:8500/v1/session/create | awk -F '"' '{print $4}') rc=$?; if [[ $rc != 0 ]]; then - echo "failed." + echo "failed" else echo $SID echo $SID > $SESSION_FILE From 4d6719097c81bb33022e21f406a5eb1a17fc2678 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 17:08:28 -0400 Subject: [PATCH 15/47] Added tasks for renewing consol session, claiming leader, renewing/cleaning certificates (if leader). Also added a co-process responsible for writing acme challenge tokens as well as certs provided by the leader (leader uses this too) --- etc/containerpilot.json | 49 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index d1e68c2..db61615 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -37,6 +37,13 @@ "-retry-max", "10", "-retry-interval", "10s"], "restarts": "unlimited" + }{{ end }} + {{ if .CONSUL_AGENT and .ACME_DOMAIN }},{{ end }} + {{ if .ACME_DOMAIN }} + { + "command": ["/usr/local/bin/consul-template", + "-config", "/etc/acme/watch.hcl", + "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500"] }{{ end }}], "telemetry": { "port": 9090, @@ -56,5 +63,45 @@ "check": ["/usr/local/bin/sensor.sh", "connections_load"] } ] - } + }, + "tasks": [{{ if .ACME_DOMAIN }} + { + "name": "renew-session-and-leader", + "command": ["/usr/local/bin/renew-consul-session", + "&&", + "/usr/local/bin/acquire-acme-leader", + "||", + "exit 0"], + "frequency": "150s", + "timeout": "10s" + }, + { + "name": "renew-leader-and-certs", + "command": ["/usr/local/bin/acquire-acme-leader", + "&&", + "cd /var/www/acme", + "&&", + "/usr/local/bin/dehydrated", + "--cron", + "--domain", "{{ .ACME_DOMAIN }}", + "--hook", "/etc/dehydrated/hook.sh", + "--config", "/etc/dehydrated/config"], + "frequency": "12h", + "timeout": "10m" + }, + { + "name": "clean-unused-certs", + "command": ["/usr/local/bin/acquire-acme-leader", + "&&", + "cd /var/www/acme", + "&&", + "/usr/local/bin/dehydrated", + "--cleanup", + "--domain", "{{ .ACME_DOMAIN }}", + "--hook", "/etc/dehydrated/hook.sh", + "--config", "/etc/dehydrated/config"], + "frequency": "24h", + "timeout": "10m" + }{{ end }} + ] } From 893e1c97746b24b9ed9d7b0b8436e7d76e8d7bd8 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 17:20:23 -0400 Subject: [PATCH 16/47] Support toggling ACME/LetsEncrypt between staging/production endpoints (default to staging so as not to exceed limits in prod) --- docker-compose.yml | 1 + etc/containerpilot.json | 4 +- etc/dehydrated/config | 87 -------------------------------- etc/dehydrated/config.production | 1 + etc/dehydrated/config.staging | 1 + local-compose.yml | 1 + 6 files changed, 6 insertions(+), 89 deletions(-) delete mode 100644 etc/dehydrated/config create mode 100644 etc/dehydrated/config.production create mode 100644 etc/dehydrated/config.staging diff --git a/docker-compose.yml b/docker-compose.yml index fd7d3d7..dac2d85 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ nginx: environment: - BACKEND=example - CONSUL_AGENT=1 + - ACME_ENV=staging ports: - 80 - 9090 # so we can see telemetry diff --git a/etc/containerpilot.json b/etc/containerpilot.json index db61615..490923d 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -85,7 +85,7 @@ "--cron", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config"], + "--config", "/etc/dehydrated/config.{{ .ACME_ENV }}"], "frequency": "12h", "timeout": "10m" }, @@ -99,7 +99,7 @@ "--cleanup", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config"], + "--config", "/etc/dehydrated/config.{{ .ACME_ENV }}"], "frequency": "24h", "timeout": "10m" }{{ end }} diff --git a/etc/dehydrated/config b/etc/dehydrated/config deleted file mode 100644 index b7277ba..0000000 --- a/etc/dehydrated/config +++ /dev/null @@ -1,87 +0,0 @@ -######################################################## -# This is the main config file for dehydrated # -# # -# This file is looked for in the following locations: # -# $SCRIPTDIR/config (next to this script) # -# /usr/local/etc/dehydrated/config # -# /etc/dehydrated/config # -# ${PWD}/config (in current working-directory) # -# # -# Default values of this config are in comments # -######################################################## - -# Resolve names to addresses of IP version only. (curl) -# supported values: 4, 6 -# default: -#IP_VERSION= - -# TODO: Make CA configurable through env var? -# Path to certificate authority (default: https://acme-v01.api.letsencrypt.org/directory) -#CA="https://acme-v01.api.letsencrypt.org/directory" -CA="https://acme-staging.api.letsencrypt.org/directory" - -# Path to license agreement (default: https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf) -#LICENSE="https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf" - -# Which challenge should be used? Currently http-01 and dns-01 are supported -#CHALLENGETYPE="http-01" - -# Path to a directory containing additional config files, allowing to override -# the defaults found in the main configuration file. Additional config files -# in this directory needs to be named with a '.sh' ending. -# default: -#CONFIG_D= - -# Base directory for account key, generated certificates and list of domains (default: $SCRIPTDIR -- uses config directory if undefined) -#BASEDIR=$SCRIPTDIR - -# File containing the list of domains to request certificates for (default: $BASEDIR/domains.txt) -#DOMAINS_TXT="${BASEDIR}/domains.txt" - -# Output directory for generated certificates -#CERTDIR="${BASEDIR}/certs" - -# Directory for account keys and registration information -#ACCOUNTDIR="${BASEDIR}/accounts" - -# Output directory for challenge-tokens to be served by webserver or deployed in HOOK (default: /var/www/dehydrated) -#WELLKNOWN="/var/www/dehydrated" - -# Default keysize for private keys (default: 4096) -#KEYSIZE="4096" - -# Path to openssl config file (default: - tries to figure out system default) -#OPENSSL_CNF= - -# Program or function called in certain situations -# -# After generating the challenge-response, or after failed challenge (in this case altname is empty) -# Given arguments: clean_challenge|deploy_challenge altname token-filename token-content -# -# After successfully signing certificate -# Given arguments: deploy_cert domain path/to/privkey.pem path/to/cert.pem path/to/fullchain.pem -# -# BASEDIR and WELLKNOWN variables are exported and can be used in an external program -# default: -#HOOK= - -# Chain clean_challenge|deploy_challenge arguments together into one hook call per certificate (default: no) -#HOOK_CHAIN="no" - -# Minimum days before expiration to automatically renew certificate (default: 30) -#RENEW_DAYS="30" - -# Regenerate private keys instead of just signing new certificates on renewal (default: yes) -#PRIVATE_KEY_RENEW="yes" - -# Which public key algorithm should be used? Supported: rsa, prime256v1 and secp384r1 -#KEY_ALGO=rsa - -# E-mail to use during the registration (default: ) -#CONTACT_EMAIL= - -# Lockfile location, to prevent concurrent access (default: $BASEDIR/lock) -#LOCKFILE="${BASEDIR}/lock" - -# Option to add CSR-flag indicating OCSP stapling to be mandatory (default: no) -#OCSP_MUST_STAPLE="no" diff --git a/etc/dehydrated/config.production b/etc/dehydrated/config.production new file mode 100644 index 0000000..955e35a --- /dev/null +++ b/etc/dehydrated/config.production @@ -0,0 +1 @@ +CA="https://acme-v01.api.letsencrypt.org/directory" diff --git a/etc/dehydrated/config.staging b/etc/dehydrated/config.staging new file mode 100644 index 0000000..5165b54 --- /dev/null +++ b/etc/dehydrated/config.staging @@ -0,0 +1 @@ +CA="https://acme-staging.api.letsencrypt.org/directory" diff --git a/local-compose.yml b/local-compose.yml index a0ae00f..a644c6b 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -7,6 +7,7 @@ nginx: environment: - CONSUL=consul - CONSUL_AGENT=1 + - ACME_ENV=staging links: - consul:consul ports: From 35cedcd74af0fb13204a4da6bfc12004ce194f4e Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 30 Sep 2016 17:32:50 -0400 Subject: [PATCH 17/47] fetch dehydrated at docker build time --- Dockerfile | 7 + bin/dehydrated | 1116 ------------------------------------------------ 2 files changed, 7 insertions(+), 1116 deletions(-) delete mode 100755 bin/dehydrated diff --git a/Dockerfile b/Dockerfile index 77d9d7e..97b2619 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,13 @@ RUN export CONTAINERPILOT_CHECKSUM=ec9dbedaca9f4a7a50762f50768cbc42879c7208 \ && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \ && rm /tmp/containerpilot.tar.gz +# Add Dehydrated +RUN export DEHYDRATED_VERSION=v0.3.1 \ + && curl --retry 8 --fail -Lso /tmp/dehydrated.zip "https://github.com/lukas2511/dehydrated/archive/${DEHYDRATED_VERSION}.tar.gz" \ + && tar xzf /tmp/dehydrated.tar.gz -C /tmp + && mv /tmp/dehydrated-0.3.1/dehydrated /usr/local/bin \ + && rm -rf /tmp/dehydrated-0.3.1 + # Add our configuration files and scripts COPY etc /etc COPY bin /usr/local/bin diff --git a/bin/dehydrated b/bin/dehydrated deleted file mode 100755 index 4cc2a66..0000000 --- a/bin/dehydrated +++ /dev/null @@ -1,1116 +0,0 @@ -#!/usr/bin/env bash - -# dehydrated by lukas2511 -# Source: https://github.com/lukas2511/dehydrated -# -# This script is licensed under The MIT License (see LICENSE for more information). - -set -e -set -u -set -o pipefail -[[ -n "${ZSH_VERSION:-}" ]] && set -o SH_WORD_SPLIT && set +o FUNCTION_ARGZERO -umask 077 # paranoid umask, we're creating private keys - -# Find directory in which this script is stored by traversing all symbolic links -SOURCE="${0}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -BASEDIR="${SCRIPTDIR}" - -# Create (identifiable) temporary files -_mktemp() { - # shellcheck disable=SC2068 - mktemp ${@:-} "${TMPDIR:-/tmp}/dehydrated-XXXXXX" -} - -# Check for script dependencies -check_dependencies() { - # just execute some dummy and/or version commands to see if required tools exist and are actually usable - openssl version > /dev/null 2>&1 || _exiterr "This script requires an openssl binary." - _sed "" < /dev/null > /dev/null 2>&1 || _exiterr "This script requires sed with support for extended (modern) regular expressions." - command -v grep > /dev/null 2>&1 || _exiterr "This script requires grep." - _mktemp -u > /dev/null 2>&1 || _exiterr "This script requires mktemp." - diff -u /dev/null /dev/null || _exiterr "This script requires diff." - - # curl returns with an error code in some ancient versions so we have to catch that - set +e - curl -V > /dev/null 2>&1 - retcode="$?" - set -e - if [[ ! "${retcode}" = "0" ]] && [[ ! "${retcode}" = "2" ]]; then - _exiterr "This script requires curl." - fi -} - -store_configvars() { - __KEY_ALGO="${KEY_ALGO}" - __OCSP_MUST_STAPLE="${OCSP_MUST_STAPLE}" - __PRIVATE_KEY_RENEW="${PRIVATE_KEY_RENEW}" - __KEYSIZE="${KEYSIZE}" - __CHALLENGETYPE="${CHALLENGETYPE}" - __HOOK="${HOOK}" - __WELLKNOWN="${WELLKNOWN}" - __HOOK_CHAIN="${HOOK_CHAIN}" - __OPENSSL_CNF="${OPENSSL_CNF}" - __RENEW_DAYS="${RENEW_DAYS}" - __IP_VERSION="${IP_VERSION}" -} - -reset_configvars() { - KEY_ALGO="${__KEY_ALGO}" - OCSP_MUST_STAPLE="${__OCSP_MUST_STAPLE}" - PRIVATE_KEY_RENEW="${__PRIVATE_KEY_RENEW}" - KEYSIZE="${__KEYSIZE}" - CHALLENGETYPE="${__CHALLENGETYPE}" - HOOK="${__HOOK}" - WELLKNOWN="${__WELLKNOWN}" - HOOK_CHAIN="${__HOOK_CHAIN}" - OPENSSL_CNF="${__OPENSSL_CNF}" - RENEW_DAYS="${__RENEW_DAYS}" - IP_VERSION="${__IP_VERSION}" -} - -# verify configuration values -verify_config() { - [[ "${CHALLENGETYPE}" =~ (http-01|dns-01) ]] || _exiterr "Unknown challenge type ${CHALLENGETYPE}... can not continue." - if [[ "${CHALLENGETYPE}" = "dns-01" ]] && [[ -z "${HOOK}" ]]; then - _exiterr "Challenge type dns-01 needs a hook script for deployment... can not continue." - fi - if [[ "${CHALLENGETYPE}" = "http-01" && ! -d "${WELLKNOWN}" ]]; then - _exiterr "WELLKNOWN directory doesn't exist, please create ${WELLKNOWN} and set appropriate permissions." - fi - [[ "${KEY_ALGO}" =~ ^(rsa|prime256v1|secp384r1)$ ]] || _exiterr "Unknown public key algorithm ${KEY_ALGO}... can not continue." - if [[ -n "${IP_VERSION}" ]]; then - [[ "${IP_VERSION}" = "4" || "${IP_VERSION}" = "6" ]] || _exiterr "Unknown IP version ${IP_VERSION}... can not continue." - fi -} - -# Setup default config values, search for and load configuration files -load_config() { - # Check for config in various locations - if [[ -z "${CONFIG:-}" ]]; then - for check_config in "/etc/dehydrated" "/usr/local/etc/dehydrated" "${PWD}" "${SCRIPTDIR}"; do - if [[ -f "${check_config}/config" ]]; then - BASEDIR="${check_config}" - CONFIG="${check_config}/config" - break - fi - done - fi - - # Default values - CA="https://acme-v01.api.letsencrypt.org/directory" - LICENSE="https://letsencrypt.org/documents/LE-SA-v1.1.1-August-1-2016.pdf" - CERTDIR= - ACCOUNTDIR= - CHALLENGETYPE="http-01" - CONFIG_D= - DOMAINS_D= - DOMAINS_TXT= - HOOK= - HOOK_CHAIN="no" - RENEW_DAYS="30" - KEYSIZE="4096" - WELLKNOWN= - PRIVATE_KEY_RENEW="yes" - KEY_ALGO=rsa - OPENSSL_CNF="$(openssl version -d | cut -d\" -f2)/openssl.cnf" - CONTACT_EMAIL= - LOCKFILE= - OCSP_MUST_STAPLE="no" - IP_VERSION= - - if [[ -z "${CONFIG:-}" ]]; then - echo "#" >&2 - echo "# !! WARNING !! No main config file found, using default config!" >&2 - echo "#" >&2 - elif [[ -f "${CONFIG}" ]]; then - echo "# INFO: Using main config file ${CONFIG}" - BASEDIR="$(dirname "${CONFIG}")" - # shellcheck disable=SC1090 - . "${CONFIG}" - else - _exiterr "Specified config file doesn't exist." - fi - - if [[ -n "${CONFIG_D}" ]]; then - if [[ ! -d "${CONFIG_D}" ]]; then - _exiterr "The path ${CONFIG_D} specified for CONFIG_D does not point to a directory." >&2 - fi - - for check_config_d in "${CONFIG_D}"/*.sh; do - if [[ ! -e "${check_config_d}" ]]; then - echo "# !! WARNING !! Extra configuration directory ${CONFIG_D} exists, but no configuration found in it." >&2 - break - elif [[ -f "${check_config_d}" ]] && [[ -r "${check_config_d}" ]]; then - echo "# INFO: Using additional config file ${check_config_d}" - # shellcheck disable=SC1090 - . "${check_config_d}" - else - _exiterr "Specified additional config ${check_config_d} is not readable or not a file at all." >&2 - fi - done - fi - - # Remove slash from end of BASEDIR. Mostly for cleaner outputs, doesn't change functionality. - BASEDIR="${BASEDIR%%/}" - - # Check BASEDIR and set default variables - [[ -d "${BASEDIR}" ]] || _exiterr "BASEDIR does not exist: ${BASEDIR}" - - CAHASH="$(echo "${CA}" | urlbase64)" - [[ -z "${ACCOUNTDIR}" ]] && ACCOUNTDIR="${BASEDIR}/accounts" - mkdir -p "${ACCOUNTDIR}/${CAHASH}" - [[ -f "${ACCOUNTDIR}/${CAHASH}/config" ]] && . "${ACCOUNTDIR}/${CAHASH}/config" - ACCOUNT_KEY="${ACCOUNTDIR}/${CAHASH}/account_key.pem" - ACCOUNT_KEY_JSON="${ACCOUNTDIR}/${CAHASH}/registration_info.json" - - if [[ -f "${BASEDIR}/private_key.pem" ]] && [[ ! -f "${ACCOUNT_KEY}" ]]; then - echo "! Moving private_key.pem to ${ACCOUNT_KEY}" - mv "${BASEDIR}/private_key.pem" "${ACCOUNT_KEY}" - fi - if [[ -f "${BASEDIR}/private_key.json" ]] && [[ ! -f "${ACCOUNT_KEY_JSON}" ]]; then - echo "! Moving private_key.json to ${ACCOUNT_KEY_JSON}" - mv "${BASEDIR}/private_key.json" "${ACCOUNT_KEY_JSON}" - fi - - [[ -z "${CERTDIR}" ]] && CERTDIR="${BASEDIR}/certs" - [[ -z "${DOMAINS_TXT}" ]] && DOMAINS_TXT="${BASEDIR}/domains.txt" - [[ -z "${WELLKNOWN}" ]] && WELLKNOWN="/var/www/dehydrated" - [[ -z "${LOCKFILE}" ]] && LOCKFILE="${BASEDIR}/lock" - [[ -n "${PARAM_NO_LOCK:-}" ]] && LOCKFILE="" - - [[ -n "${PARAM_HOOK:-}" ]] && HOOK="${PARAM_HOOK}" - [[ -n "${PARAM_CERTDIR:-}" ]] && CERTDIR="${PARAM_CERTDIR}" - [[ -n "${PARAM_CHALLENGETYPE:-}" ]] && CHALLENGETYPE="${PARAM_CHALLENGETYPE}" - [[ -n "${PARAM_KEY_ALGO:-}" ]] && KEY_ALGO="${PARAM_KEY_ALGO}" - [[ -n "${PARAM_OCSP_MUST_STAPLE:-}" ]] && OCSP_MUST_STAPLE="${PARAM_OCSP_MUST_STAPLE}" - [[ -n "${PARAM_IP_VERSION:-}" ]] && IP_VERSION="${PARAM_IP_VERSION}" - - verify_config - store_configvars -} - -# Initialize system -init_system() { - load_config - - # Lockfile handling (prevents concurrent access) - if [[ -n "${LOCKFILE}" ]]; then - LOCKDIR="$(dirname "${LOCKFILE}")" - [[ -w "${LOCKDIR}" ]] || _exiterr "Directory ${LOCKDIR} for LOCKFILE ${LOCKFILE} is not writable, aborting." - ( set -C; date > "${LOCKFILE}" ) 2>/dev/null || _exiterr "Lock file '${LOCKFILE}' present, aborting." - remove_lock() { rm -f "${LOCKFILE}"; } - trap 'remove_lock' EXIT - fi - - # Get CA URLs - CA_DIRECTORY="$(http_request get "${CA}")" - CA_NEW_CERT="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-cert)" && - CA_NEW_AUTHZ="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-authz)" && - CA_NEW_REG="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value new-reg)" && - # shellcheck disable=SC2015 - CA_REVOKE_CERT="$(printf "%s" "${CA_DIRECTORY}" | get_json_string_value revoke-cert)" || - _exiterr "Problem retrieving ACME/CA-URLs, check if your configured CA points to the directory entrypoint." - - # Export some environment variables to be used in hook script - export WELLKNOWN BASEDIR CERTDIR CONFIG - - # Checking for private key ... - register_new_key="no" - if [[ -n "${PARAM_ACCOUNT_KEY:-}" ]]; then - # a private key was specified from the command line so use it for this run - echo "Using private key ${PARAM_ACCOUNT_KEY} instead of account key" - ACCOUNT_KEY="${PARAM_ACCOUNT_KEY}" - ACCOUNT_KEY_JSON="${PARAM_ACCOUNT_KEY}.json" - else - # Check if private account key exists, if it doesn't exist yet generate a new one (rsa key) - if [[ ! -e "${ACCOUNT_KEY}" ]]; then - echo "+ Generating account key..." - _openssl genrsa -out "${ACCOUNT_KEY}" "${KEYSIZE}" - register_new_key="yes" - fi - fi - openssl rsa -in "${ACCOUNT_KEY}" -check 2>/dev/null > /dev/null || _exiterr "Account key is not valid, can not continue." - - # Get public components from private key and calculate thumbprint - pubExponent64="$(printf '%x' "$(openssl rsa -in "${ACCOUNT_KEY}" -noout -text | awk '/publicExponent/ {print $2}')" | hex2bin | urlbase64)" - pubMod64="$(openssl rsa -in "${ACCOUNT_KEY}" -noout -modulus | cut -d'=' -f2 | hex2bin | urlbase64)" - - thumbprint="$(printf '{"e":"%s","kty":"RSA","n":"%s"}' "${pubExponent64}" "${pubMod64}" | openssl dgst -sha256 -binary | urlbase64)" - - # If we generated a new private key in the step above we have to register it with the acme-server - if [[ "${register_new_key}" = "yes" ]]; then - echo "+ Registering account key with ACME server..." - [[ ! -z "${CA_NEW_REG}" ]] || _exiterr "Certificate authority doesn't allow registrations." - # If an email for the contact has been provided then adding it to the registration request - FAILED=false - if [[ -n "${CONTACT_EMAIL}" ]]; then - (signed_request "${CA_NEW_REG}" '{"resource": "new-reg", "contact":["mailto:'"${CONTACT_EMAIL}"'"], "agreement": "'"$LICENSE"'"}' > "${ACCOUNT_KEY_JSON}") || FAILED=true - else - (signed_request "${CA_NEW_REG}" '{"resource": "new-reg", "agreement": "'"$LICENSE"'"}' > "${ACCOUNT_KEY_JSON}") || FAILED=true - fi - if [[ "${FAILED}" = "true" ]]; then - echo - echo - echo "Error registering account key. See message above for more information." - rm "${ACCOUNT_KEY}" "${ACCOUNT_KEY_JSON}" - exit 1 - fi - fi - -} - -# Different sed version for different os types... -_sed() { - if [[ "${OSTYPE}" = "Linux" ]]; then - sed -r "${@}" - else - sed -E "${@}" - fi -} - -# Print error message and exit with error -_exiterr() { - echo "ERROR: ${1}" >&2 - exit 1 -} - -# Remove newlines and whitespace from json -clean_json() { - tr -d '\r\n' | _sed -e 's/ +/ /g' -e 's/\{ /{/g' -e 's/ \}/}/g' -e 's/\[ /[/g' -e 's/ \]/]/g' -} - -# Encode data as url-safe formatted base64 -urlbase64() { - # urlbase64: base64 encoded string with '+' replaced with '-' and '/' replaced with '_' - openssl base64 -e | tr -d '\n\r' | _sed -e 's:=*$::g' -e 'y:+/:-_:' -} - -# Convert hex string to binary data -hex2bin() { - # Remove spaces, add leading zero, escape as hex string and parse with printf - printf -- "$(cat | _sed -e 's/[[:space:]]//g' -e 's/^(.(.{2})*)$/0\1/' -e 's/(.{2})/\\x\1/g')" -} - -# Get string value from json dictionary -get_json_string_value() { - local filter - filter=$(printf 's/.*"%s": *"\([^"]*\)".*/\\1/p' "$1") - sed -n "${filter}" -} - -# OpenSSL writes to stderr/stdout even when there are no errors. So just -# display the output if the exit code was != 0 to simplify debugging. -_openssl() { - set +e - out="$(openssl "${@}" 2>&1)" - res=$? - set -e - if [[ ${res} -ne 0 ]]; then - echo " + ERROR: failed to run $* (Exitcode: ${res})" >&2 - echo >&2 - echo "Details:" >&2 - echo "${out}" >&2 - echo >&2 - exit ${res} - fi -} - -# Send http(s) request with specified method -http_request() { - tempcont="$(_mktemp)" - - if [[ -n "${IP_VERSION:-}" ]]; then - ip_version="-${IP_VERSION}" - fi - - set +e - if [[ "${1}" = "head" ]]; then - statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}" -I)" - curlret="${?}" - elif [[ "${1}" = "get" ]]; then - statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}")" - curlret="${?}" - elif [[ "${1}" = "post" ]]; then - statuscode="$(curl ${ip_version:-} -s -w "%{http_code}" -o "${tempcont}" "${2}" -d "${3}")" - curlret="${?}" - else - set -e - _exiterr "Unknown request method: ${1}" - fi - set -e - - if [[ ! "${curlret}" = "0" ]]; then - _exiterr "Problem connecting to server (${1} for ${2}; curl returned with ${curlret})" - fi - - if [[ ! "${statuscode:0:1}" = "2" ]]; then - echo " + ERROR: An error occurred while sending ${1}-request to ${2} (Status ${statuscode})" >&2 - echo >&2 - echo "Details:" >&2 - cat "${tempcont}" >&2 - echo >&2 - echo >&2 - rm -f "${tempcont}" - - # Wait for hook script to clean the challenge if used - if [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && [[ -n "${challenge_token:+set}" ]]; then - "${HOOK}" "clean_challenge" '' "${challenge_token}" "${keyauth}" - fi - - # remove temporary domains.txt file if used - [[ -n "${PARAM_DOMAIN:-}" && -n "${DOMAINS_TXT:-}" ]] && rm "${DOMAINS_TXT}" - exit 1 - fi - - cat "${tempcont}" - rm -f "${tempcont}" -} - -# Send signed request -signed_request() { - # Encode payload as urlbase64 - payload64="$(printf '%s' "${2}" | urlbase64)" - - # Retrieve nonce from acme-server - nonce="$(http_request head "${CA}" | grep Replay-Nonce: | awk -F ': ' '{print $2}' | tr -d '\n\r')" - - # Build header with just our public key and algorithm information - header='{"alg": "RS256", "jwk": {"e": "'"${pubExponent64}"'", "kty": "RSA", "n": "'"${pubMod64}"'"}}' - - # Build another header which also contains the previously received nonce and encode it as urlbase64 - protected='{"alg": "RS256", "jwk": {"e": "'"${pubExponent64}"'", "kty": "RSA", "n": "'"${pubMod64}"'"}, "nonce": "'"${nonce}"'"}' - protected64="$(printf '%s' "${protected}" | urlbase64)" - - # Sign header with nonce and our payload with our private key and encode signature as urlbase64 - signed64="$(printf '%s' "${protected64}.${payload64}" | openssl dgst -sha256 -sign "${ACCOUNT_KEY}" | urlbase64)" - - # Send header + extended header + payload + signature to the acme-server - data='{"header": '"${header}"', "protected": "'"${protected64}"'", "payload": "'"${payload64}"'", "signature": "'"${signed64}"'"}' - - http_request post "${1}" "${data}" -} - -# Extracts all subject names from a CSR -# Outputs either the CN, or the SANs, one per line -extract_altnames() { - csr="${1}" # the CSR itself (not a file) - - if ! <<<"${csr}" openssl req -verify -noout 2>/dev/null; then - _exiterr "Certificate signing request isn't valid" - fi - - reqtext="$( <<<"${csr}" openssl req -noout -text )" - if <<<"${reqtext}" grep -q '^[[:space:]]*X509v3 Subject Alternative Name:[[:space:]]*$'; then - # SANs used, extract these - altnames="$( <<<"${reqtext}" grep -A1 '^[[:space:]]*X509v3 Subject Alternative Name:[[:space:]]*$' | tail -n1 )" - # split to one per line: - # shellcheck disable=SC1003 - altnames="$( <<<"${altnames}" _sed -e 's/^[[:space:]]*//; s/, /\'$'\n''/g' )" - # we can only get DNS: ones signed - if grep -qv '^DNS:' <<<"${altnames}"; then - _exiterr "Certificate signing request contains non-DNS Subject Alternative Names" - fi - # strip away the DNS: prefix - altnames="$( <<<"${altnames}" _sed -e 's/^DNS://' )" - echo "${altnames}" - - else - # No SANs, extract CN - altnames="$( <<<"${reqtext}" grep '^[[:space:]]*Subject:' | _sed -e 's/.* CN=([^ /,]*).*/\1/' )" - echo "${altnames}" - fi -} - -# Create certificate for domain(s) and outputs it FD 3 -sign_csr() { - csr="${1}" # the CSR itself (not a file) - - if { true >&3; } 2>/dev/null; then - : # fd 3 looks OK - else - _exiterr "sign_csr: FD 3 not open" - fi - - shift 1 || true - altnames="${*:-}" - if [ -z "${altnames}" ]; then - altnames="$( extract_altnames "${csr}" )" - fi - - if [[ -z "${CA_NEW_AUTHZ}" ]] || [[ -z "${CA_NEW_CERT}" ]]; then - _exiterr "Certificate authority doesn't allow certificate signing" - fi - - local idx=0 - if [[ -n "${ZSH_VERSION:-}" ]]; then - local -A challenge_uris challenge_tokens keyauths deploy_args - else - local -a challenge_uris challenge_tokens keyauths deploy_args - fi - - # Request challenges - for altname in ${altnames}; do - # Ask the acme-server for new challenge token and extract them from the resulting json block - echo " + Requesting challenge for ${altname}..." - response="$(signed_request "${CA_NEW_AUTHZ}" '{"resource": "new-authz", "identifier": {"type": "dns", "value": "'"${altname}"'"}}' | clean_json)" - - challenges="$(printf '%s\n' "${response}" | sed -n 's/.*\("challenges":[^\[]*\[[^]]*]\).*/\1/p')" - repl=$'\n''{' # fix syntax highlighting in Vim - challenge="$(printf "%s" "${challenges//\{/${repl}}" | grep \""${CHALLENGETYPE}"\")" - challenge_token="$(printf '%s' "${challenge}" | get_json_string_value token | _sed 's/[^A-Za-z0-9_\-]/_/g')" - challenge_uri="$(printf '%s' "${challenge}" | get_json_string_value uri)" - - if [[ -z "${challenge_token}" ]] || [[ -z "${challenge_uri}" ]]; then - _exiterr "Can't retrieve challenges (${response})" - fi - - # Challenge response consists of the challenge token and the thumbprint of our public certificate - keyauth="${challenge_token}.${thumbprint}" - - case "${CHALLENGETYPE}" in - "http-01") - # Store challenge response in well-known location and make world-readable (so that a webserver can access it) - printf '%s' "${keyauth}" > "${WELLKNOWN}/${challenge_token}" - chmod a+r "${WELLKNOWN}/${challenge_token}" - keyauth_hook="${keyauth}" - ;; - "dns-01") - # Generate DNS entry content for dns-01 validation - keyauth_hook="$(printf '%s' "${keyauth}" | openssl dgst -sha256 -binary | urlbase64)" - ;; - esac - - challenge_uris[${idx}]="${challenge_uri}" - keyauths[${idx}]="${keyauth}" - challenge_tokens[${idx}]="${challenge_token}" - # Note: assumes args will never have spaces! - deploy_args[${idx}]="${altname} ${challenge_token} ${keyauth_hook}" - idx=$((idx+1)) - done - - # Wait for hook script to deploy the challenges if used - # shellcheck disable=SC2068 - [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" = "yes" ]] && "${HOOK}" "deploy_challenge" ${deploy_args[@]} - - # Respond to challenges - idx=0 - for altname in ${altnames}; do - challenge_token="${challenge_tokens[${idx}]}" - keyauth="${keyauths[${idx}]}" - - # Wait for hook script to deploy the challenge if used - # shellcheck disable=SC2086 - [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && "${HOOK}" "deploy_challenge" ${deploy_args[${idx}]} - - # Ask the acme-server to verify our challenge and wait until it is no longer pending - echo " + Responding to challenge for ${altname}..." - result="$(signed_request "${challenge_uris[${idx}]}" '{"resource": "challenge", "keyAuthorization": "'"${keyauth}"'"}' | clean_json)" - - reqstatus="$(printf '%s\n' "${result}" | get_json_string_value status)" - - while [[ "${reqstatus}" = "pending" ]]; do - sleep 1 - result="$(http_request get "${challenge_uris[${idx}]}")" - reqstatus="$(printf '%s\n' "${result}" | get_json_string_value status)" - done - - [[ "${CHALLENGETYPE}" = "http-01" ]] && rm -f "${WELLKNOWN}/${challenge_token}" - - # Wait for hook script to clean the challenge if used - if [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" != "yes" ]] && [[ -n "${challenge_token}" ]]; then - # shellcheck disable=SC2086 - "${HOOK}" "clean_challenge" ${deploy_args[${idx}]} - fi - idx=$((idx+1)) - - if [[ "${reqstatus}" = "valid" ]]; then - echo " + Challenge is valid!" - else - break - fi - done - - # Wait for hook script to clean the challenges if used - # shellcheck disable=SC2068 - [[ -n "${HOOK}" ]] && [[ "${HOOK_CHAIN}" = "yes" ]] && "${HOOK}" "clean_challenge" ${deploy_args[@]} - - if [[ "${reqstatus}" != "valid" ]]; then - # Clean up any remaining challenge_tokens if we stopped early - if [[ "${CHALLENGETYPE}" = "http-01" ]]; then - while [ ${idx} -lt ${#challenge_tokens[@]} ]; do - rm -f "${WELLKNOWN}/${challenge_tokens[${idx}]}" - idx=$((idx+1)) - done - fi - - _exiterr "Challenge is invalid! (returned: ${reqstatus}) (result: ${result})" - fi - - # Finally request certificate from the acme-server and store it in cert-${timestamp}.pem and link from cert.pem - echo " + Requesting certificate..." - csr64="$( <<<"${csr}" openssl req -outform DER | urlbase64)" - crt64="$(signed_request "${CA_NEW_CERT}" '{"resource": "new-cert", "csr": "'"${csr64}"'"}' | openssl base64 -e)" - crt="$( printf -- '-----BEGIN CERTIFICATE-----\n%s\n-----END CERTIFICATE-----\n' "${crt64}" )" - - # Try to load the certificate to detect corruption - echo " + Checking certificate..." - _openssl x509 -text <<<"${crt}" - - echo "${crt}" >&3 - - unset challenge_token - echo " + Done!" -} - -# Create certificate for domain(s) -sign_domain() { - domain="${1}" - altnames="${*}" - timestamp="$(date +%s)" - - echo " + Signing domains..." - if [[ -z "${CA_NEW_AUTHZ}" ]] || [[ -z "${CA_NEW_CERT}" ]]; then - _exiterr "Certificate authority doesn't allow certificate signing" - fi - - # If there is no existing certificate directory => make it - if [[ ! -e "${CERTDIR}/${domain}" ]]; then - echo " + Creating new directory ${CERTDIR}/${domain} ..." - mkdir -p "${CERTDIR}/${domain}" || _exiterr "Unable to create directory ${CERTDIR}/${domain}" - fi - - privkey="privkey.pem" - # generate a new private key if we need or want one - if [[ ! -r "${CERTDIR}/${domain}/privkey.pem" ]] || [[ "${PRIVATE_KEY_RENEW}" = "yes" ]]; then - echo " + Generating private key..." - privkey="privkey-${timestamp}.pem" - case "${KEY_ALGO}" in - rsa) _openssl genrsa -out "${CERTDIR}/${domain}/privkey-${timestamp}.pem" "${KEYSIZE}";; - prime256v1|secp384r1) _openssl ecparam -genkey -name "${KEY_ALGO}" -out "${CERTDIR}/${domain}/privkey-${timestamp}.pem";; - esac - fi - - # Generate signing request config and the actual signing request - echo " + Generating signing request..." - SAN="" - for altname in ${altnames}; do - SAN+="DNS:${altname}, " - done - SAN="${SAN%%, }" - local tmp_openssl_cnf - tmp_openssl_cnf="$(_mktemp)" - cat "${OPENSSL_CNF}" > "${tmp_openssl_cnf}" - printf "[SAN]\nsubjectAltName=%s" "${SAN}" >> "${tmp_openssl_cnf}" - if [ "${OCSP_MUST_STAPLE}" = "yes" ]; then - printf "\n1.3.6.1.5.5.7.1.24=DER:30:03:02:01:05" >> "${tmp_openssl_cnf}" - fi - openssl req -new -sha256 -key "${CERTDIR}/${domain}/${privkey}" -out "${CERTDIR}/${domain}/cert-${timestamp}.csr" -subj "/CN=${domain}/" -reqexts SAN -config "${tmp_openssl_cnf}" - rm -f "${tmp_openssl_cnf}" - - crt_path="${CERTDIR}/${domain}/cert-${timestamp}.pem" - # shellcheck disable=SC2086 - sign_csr "$(< "${CERTDIR}/${domain}/cert-${timestamp}.csr" )" ${altnames} 3>"${crt_path}" - - # Create fullchain.pem - echo " + Creating fullchain.pem..." - cat "${crt_path}" > "${CERTDIR}/${domain}/fullchain-${timestamp}.pem" - http_request get "$(openssl x509 -in "${CERTDIR}/${domain}/cert-${timestamp}.pem" -noout -text | grep 'CA Issuers - URI:' | cut -d':' -f2-)" > "${CERTDIR}/${domain}/chain-${timestamp}.pem" - if ! grep -q "BEGIN CERTIFICATE" "${CERTDIR}/${domain}/chain-${timestamp}.pem"; then - openssl x509 -in "${CERTDIR}/${domain}/chain-${timestamp}.pem" -inform DER -out "${CERTDIR}/${domain}/chain-${timestamp}.pem" -outform PEM - fi - cat "${CERTDIR}/${domain}/chain-${timestamp}.pem" >> "${CERTDIR}/${domain}/fullchain-${timestamp}.pem" - - # Update symlinks - [[ "${privkey}" = "privkey.pem" ]] || ln -sf "privkey-${timestamp}.pem" "${CERTDIR}/${domain}/privkey.pem" - - ln -sf "chain-${timestamp}.pem" "${CERTDIR}/${domain}/chain.pem" - ln -sf "fullchain-${timestamp}.pem" "${CERTDIR}/${domain}/fullchain.pem" - ln -sf "cert-${timestamp}.csr" "${CERTDIR}/${domain}/cert.csr" - ln -sf "cert-${timestamp}.pem" "${CERTDIR}/${domain}/cert.pem" - - # Wait for hook script to clean the challenge and to deploy cert if used - export KEY_ALGO - [[ -n "${HOOK}" ]] && "${HOOK}" "deploy_cert" "${domain}" "${CERTDIR}/${domain}/privkey.pem" "${CERTDIR}/${domain}/cert.pem" "${CERTDIR}/${domain}/fullchain.pem" "${CERTDIR}/${domain}/chain.pem" "${timestamp}" - - unset challenge_token - echo " + Done!" -} - -# Usage: --cron (-c) -# Description: Sign/renew non-existant/changed/expiring certificates. -command_sign_domains() { - init_system - - if [[ -n "${PARAM_DOMAIN:-}" ]]; then - DOMAINS_TXT="$(_mktemp)" - printf -- "${PARAM_DOMAIN}" > "${DOMAINS_TXT}" - elif [[ -e "${DOMAINS_TXT}" ]]; then - if [[ ! -r "${DOMAINS_TXT}" ]]; then - _exiterr "domains.txt found but not readable" - fi - else - _exiterr "domains.txt not found and --domain not given" - fi - - # Generate certificates for all domains found in domains.txt. Check if existing certificate are about to expire - ORIGIFS="${IFS}" - IFS=$'\n' - for line in $(<"${DOMAINS_TXT}" tr -d '\r' | tr '[:upper:]' '[:lower:]' | _sed -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*$//g' -e 's/[[:space:]]+/ /g' | (grep -vE '^(#|$)' || true)); do - reset_configvars - IFS="${ORIGIFS}" - domain="$(printf '%s\n' "${line}" | cut -d' ' -f1)" - morenames="$(printf '%s\n' "${line}" | cut -s -d' ' -f2-)" - cert="${CERTDIR}/${domain}/cert.pem" - - force_renew="${PARAM_FORCE:-no}" - - if [[ -z "${morenames}" ]];then - echo "Processing ${domain}" - else - echo "Processing ${domain} with alternative names: ${morenames}" - fi - - # read cert config - # for now this loads the certificate specific config in a subshell and parses a diff of set variables. - # we could just source the config file but i decided to go this way to protect people from accidentally overriding - # variables used internally by this script itself. - if [[ -n "${DOMAINS_D}" ]]; then - certconfig="${DOMAINS_D}/${domain}" - else - certconfig="${CERTDIR}/${domain}/config" - fi - - if [ -f "${certconfig}" ]; then - echo " + Using certificate specific config file!" - ORIGIFS="${IFS}" - IFS=$'\n' - for cfgline in $( - beforevars="$(_mktemp)" - aftervars="$(_mktemp)" - set > "${beforevars}" - # shellcheck disable=SC1090 - . "${certconfig}" - set > "${aftervars}" - diff -u "${beforevars}" "${aftervars}" | grep -E '^\+[^+]' - rm "${beforevars}" - rm "${aftervars}" - ); do - config_var="$(echo "${cfgline:1}" | cut -d'=' -f1)" - config_value="$(echo "${cfgline:1}" | cut -d'=' -f2-)" - case "${config_var}" in - KEY_ALGO|OCSP_MUST_STAPLE|PRIVATE_KEY_RENEW|KEYSIZE|CHALLENGETYPE|HOOK|WELLKNOWN|HOOK_CHAIN|OPENSSL_CNF|RENEW_DAYS) - echo " + ${config_var} = ${config_value}" - declare -- "${config_var}=${config_value}" - ;; - _) ;; - *) echo " ! Setting ${config_var} on a per-certificate base is not (yet) supported" - esac - done - IFS="${ORIGIFS}" - fi - verify_config - - if [[ -e "${cert}" ]]; then - printf " + Checking domain name(s) of existing cert..." - - certnames="$(openssl x509 -in "${cert}" -text -noout | grep DNS: | _sed 's/DNS://g' | tr -d ' ' | tr ',' '\n' | sort -u | tr '\n' ' ' | _sed 's/ $//')" - givennames="$(echo "${domain}" "${morenames}"| tr ' ' '\n' | sort -u | tr '\n' ' ' | _sed 's/ $//' | _sed 's/^ //')" - - if [[ "${certnames}" = "${givennames}" ]]; then - echo " unchanged." - else - echo " changed!" - echo " + Domain name(s) are not matching!" - echo " + Names in old certificate: ${certnames}" - echo " + Configured names: ${givennames}" - echo " + Forcing renew." - force_renew="yes" - fi - fi - - if [[ -e "${cert}" ]]; then - echo " + Checking expire date of existing cert..." - valid="$(openssl x509 -enddate -noout -in "${cert}" | cut -d= -f2- )" - - printf " + Valid till %s " "${valid}" - if openssl x509 -checkend $((RENEW_DAYS * 86400)) -noout -in "${cert}"; then - printf "(Longer than %d days). " "${RENEW_DAYS}" - if [[ "${force_renew}" = "yes" ]]; then - echo "Ignoring because renew was forced!" - else - # Certificate-Names unchanged and cert is still valid - echo "Skipping renew!" - [[ -n "${HOOK}" ]] && "${HOOK}" "unchanged_cert" "${domain}" "${CERTDIR}/${domain}/privkey.pem" "${CERTDIR}/${domain}/cert.pem" "${CERTDIR}/${domain}/fullchain.pem" "${CERTDIR}/${domain}/chain.pem" - continue - fi - else - echo "(Less than ${RENEW_DAYS} days). Renewing!" - fi - fi - - # shellcheck disable=SC2086 - if [[ "${PARAM_KEEP_GOING:-}" = "yes" ]]; then - sign_domain ${line} & - wait $! || true - else - sign_domain ${line} - fi - done - - # remove temporary domains.txt file if used - [[ -n "${PARAM_DOMAIN:-}" ]] && rm -f "${DOMAINS_TXT}" - - exit 0 -} - -# Usage: --signcsr (-s) path/to/csr.pem -# Description: Sign a given CSR, output CRT on stdout (advanced usage) -command_sign_csr() { - # redirect stdout to stderr - # leave stdout over at fd 3 to output the cert - exec 3>&1 1>&2 - - init_system - - csrfile="${1}" - if [ ! -r "${csrfile}" ]; then - _exiterr "Could not read certificate signing request ${csrfile}" - fi - - # gen cert - certfile="$(_mktemp)" - sign_csr "$(< "${csrfile}" )" 3> "${certfile}" - - # print cert - echo "# CERT #" >&3 - cat "${certfile}" >&3 - echo >&3 - - # print chain - if [ -n "${PARAM_FULL_CHAIN:-}" ]; then - # get and convert ca cert - chainfile="$(_mktemp)" - http_request get "$(openssl x509 -in "${certfile}" -noout -text | grep 'CA Issuers - URI:' | cut -d':' -f2-)" > "${chainfile}" - - if ! grep -q "BEGIN CERTIFICATE" "${chainfile}"; then - openssl x509 -inform DER -in "${chainfile}" -outform PEM -out "${chainfile}" - fi - - echo "# CHAIN #" >&3 - cat "${chainfile}" >&3 - - rm "${chainfile}" - fi - - # cleanup - rm "${certfile}" - - exit 0 -} - -# Usage: --revoke (-r) path/to/cert.pem -# Description: Revoke specified certificate -command_revoke() { - init_system - - [[ -n "${CA_REVOKE_CERT}" ]] || _exiterr "Certificate authority doesn't allow certificate revocation." - - cert="${1}" - if [[ -L "${cert}" ]]; then - # follow symlink and use real certificate name (so we move the real file and not the symlink at the end) - local link_target - link_target="$(readlink -n "${cert}")" - if [[ "${link_target}" =~ ^/ ]]; then - cert="${link_target}" - else - cert="$(dirname "${cert}")/${link_target}" - fi - fi - [[ -f "${cert}" ]] || _exiterr "Could not find certificate ${cert}" - - echo "Revoking ${cert}" - - cert64="$(openssl x509 -in "${cert}" -inform PEM -outform DER | urlbase64)" - response="$(signed_request "${CA_REVOKE_CERT}" '{"resource": "revoke-cert", "certificate": "'"${cert64}"'"}' | clean_json)" - # if there is a problem with our revoke request _request (via signed_request) will report this and "exit 1" out - # so if we are here, it is safe to assume the request was successful - echo " + Done." - echo " + Renaming certificate to ${cert}-revoked" - mv -f "${cert}" "${cert}-revoked" -} - -# Usage: --cleanup (-gc) -# Description: Move unused certificate files to archive directory -command_cleanup() { - load_config - - # Create global archive directory if not existant - if [[ ! -e "${BASEDIR}/archive" ]]; then - mkdir "${BASEDIR}/archive" - fi - - # Loop over all certificate directories - for certdir in "${CERTDIR}/"*; do - # Skip if entry is not a folder - [[ -d "${certdir}" ]] || continue - - # Get certificate name - certname="$(basename "${certdir}")" - - # Create certitifaces archive directory if not existant - archivedir="${BASEDIR}/archive/${certname}" - if [[ ! -e "${archivedir}" ]]; then - mkdir "${archivedir}" - fi - - # Loop over file-types (certificates, keys, signing-requests, ...) - for filetype in cert.csr cert.pem chain.pem fullchain.pem privkey.pem; do - # Skip if symlink is broken - [[ -r "${certdir}/${filetype}" ]] || continue - - # Look up current file in use - current="$(basename "$(readlink "${certdir}/${filetype}")")" - - # Split filetype into name and extension - filebase="$(echo "${filetype}" | cut -d. -f1)" - fileext="$(echo "${filetype}" | cut -d. -f2)" - - # Loop over all files of this type - for file in "${certdir}/${filebase}-"*".${fileext}"; do - # Handle case where no files match the wildcard - [[ -f "${file}" ]] || break - - # Check if current file is in use, if unused move to archive directory - filename="$(basename "${file}")" - if [[ ! "${filename}" = "${current}" ]]; then - echo "Moving unused file to archive directory: ${certname}/${filename}" - mv "${certdir}/${filename}" "${archivedir}/${filename}" - fi - done - done - done - - exit 0 -} - -# Usage: --help (-h) -# Description: Show help text -command_help() { - printf "Usage: %s [-h] [command [argument]] [parameter [argument]] [parameter [argument]] ...\n\n" "${0}" - printf "Default command: help\n\n" - echo "Commands:" - grep -e '^[[:space:]]*# Usage:' -e '^[[:space:]]*# Description:' -e '^command_.*()[[:space:]]*{' "${0}" | while read -r usage; read -r description; read -r command; do - if [[ ! "${usage}" =~ Usage ]] || [[ ! "${description}" =~ Description ]] || [[ ! "${command}" =~ ^command_ ]]; then - _exiterr "Error generating help text." - fi - printf " %-32s %s\n" "${usage##"# Usage: "}" "${description##"# Description: "}" - done - printf -- "\nParameters:\n" - grep -E -e '^[[:space:]]*# PARAM_Usage:' -e '^[[:space:]]*# PARAM_Description:' "${0}" | while read -r usage; read -r description; do - if [[ ! "${usage}" =~ Usage ]] || [[ ! "${description}" =~ Description ]]; then - _exiterr "Error generating help text." - fi - printf " %-32s %s\n" "${usage##"# PARAM_Usage: "}" "${description##"# PARAM_Description: "}" - done -} - -# Usage: --env (-e) -# Description: Output configuration variables for use in other scripts -command_env() { - echo "# dehydrated configuration" - load_config - typeset -p CA LICENSE CERTDIR CHALLENGETYPE DOMAINS_D DOMAINS_TXT HOOK HOOK_CHAIN RENEW_DAYS ACCOUNT_KEY ACCOUNT_KEY_JSON KEYSIZE WELLKNOWN PRIVATE_KEY_RENEW OPENSSL_CNF CONTACT_EMAIL LOCKFILE -} - -# Main method (parses script arguments and calls command_* methods) -main() { - COMMAND="" - set_command() { - [[ -z "${COMMAND}" ]] || _exiterr "Only one command can be executed at a time. See help (-h) for more information." - COMMAND="${1}" - } - - check_parameters() { - if [[ -z "${1:-}" ]]; then - echo "The specified command requires additional parameters. See help:" >&2 - echo >&2 - command_help >&2 - exit 1 - elif [[ "${1:0:1}" = "-" ]]; then - _exiterr "Invalid argument: ${1}" - fi - } - - [[ -z "${@}" ]] && eval set -- "--help" - - while (( ${#} )); do - case "${1}" in - --help|-h) - command_help - exit 0 - ;; - - --env|-e) - set_command env - ;; - - --cron|-c) - set_command sign_domains - ;; - - --signcsr|-s) - shift 1 - set_command sign_csr - check_parameters "${1:-}" - PARAM_CSR="${1}" - ;; - - --revoke|-r) - shift 1 - set_command revoke - check_parameters "${1:-}" - PARAM_REVOKECERT="${1}" - ;; - - --cleanup|-gc) - set_command cleanup - ;; - - # PARAM_Usage: --full-chain (-fc) - # PARAM_Description: Print full chain when using --signcsr - --full-chain|-fc) - PARAM_FULL_CHAIN="1" - ;; - - # PARAM_Usage: --ipv4 (-4) - # PARAM_Description: Resolve names to IPv4 addresses only - --ipv4|-4) - PARAM_IP_VERSION="4" - ;; - - # PARAM_Usage: --ipv6 (-6) - # PARAM_Description: Resolve names to IPv6 addresses only - --ipv6|-6) - PARAM_IP_VERSION="6" - ;; - - # PARAM_Usage: --domain (-d) domain.tld - # PARAM_Description: Use specified domain name(s) instead of domains.txt entry (one certificate!) - --domain|-d) - shift 1 - check_parameters "${1:-}" - if [[ -z "${PARAM_DOMAIN:-}" ]]; then - PARAM_DOMAIN="${1}" - else - PARAM_DOMAIN="${PARAM_DOMAIN} ${1}" - fi - ;; - - # PARAM_Usage: --keep-going (-g) - # PARAM_Description: Keep going after encountering an error while creating/renewing multiple certificates in cron mode - --keep-going|-g) - PARAM_KEEP_GOING="yes" - ;; - - # PARAM_Usage: --force (-x) - # PARAM_Description: Force renew of certificate even if it is longer valid than value in RENEW_DAYS - --force|-x) - PARAM_FORCE="yes" - ;; - - # PARAM_Usage: --no-lock (-n) - # PARAM_Description: Don't use lockfile (potentially dangerous!) - --no-lock|-n) - PARAM_NO_LOCK="yes" - ;; - - # PARAM_Usage: --ocsp - # PARAM_Description: Sets option in CSR indicating OCSP stapling to be mandatory - --ocsp) - PARAM_OCSP_MUST_STAPLE="yes" - ;; - - # PARAM_Usage: --privkey (-p) path/to/key.pem - # PARAM_Description: Use specified private key instead of account key (useful for revocation) - --privkey|-p) - shift 1 - check_parameters "${1:-}" - PARAM_ACCOUNT_KEY="${1}" - ;; - - # PARAM_Usage: --config (-f) path/to/config - # PARAM_Description: Use specified config file - --config|-f) - shift 1 - check_parameters "${1:-}" - CONFIG="${1}" - ;; - - # PARAM_Usage: --hook (-k) path/to/hook.sh - # PARAM_Description: Use specified script for hooks - --hook|-k) - shift 1 - check_parameters "${1:-}" - PARAM_HOOK="${1}" - ;; - - # PARAM_Usage: --out (-o) certs/directory - # PARAM_Description: Output certificates into the specified directory - --out|-o) - shift 1 - check_parameters "${1:-}" - PARAM_CERTDIR="${1}" - ;; - - # PARAM_Usage: --challenge (-t) http-01|dns-01 - # PARAM_Description: Which challenge should be used? Currently http-01 and dns-01 are supported - --challenge|-t) - shift 1 - check_parameters "${1:-}" - PARAM_CHALLENGETYPE="${1}" - ;; - - # PARAM_Usage: --algo (-a) rsa|prime256v1|secp384r1 - # PARAM_Description: Which public key algorithm should be used? Supported: rsa, prime256v1 and secp384r1 - --algo|-a) - shift 1 - check_parameters "${1:-}" - PARAM_KEY_ALGO="${1}" - ;; - - *) - echo "Unknown parameter detected: ${1}" >&2 - echo >&2 - command_help >&2 - exit 1 - ;; - esac - - shift 1 - done - - case "${COMMAND}" in - env) command_env;; - sign_domains) command_sign_domains;; - sign_csr) command_sign_csr "${PARAM_CSR}";; - revoke) command_revoke "${PARAM_REVOKECERT}";; - cleanup) command_cleanup;; - *) command_help; exit 1;; - esac -} - -# Determine OS type -OSTYPE="$(uname)" - -# Check for missing dependencies -check_dependencies - -# Run script -main "${@:-}" From 3039fc385c062b3efee12613ae0ce57b115dc82b Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 13:19:30 -0400 Subject: [PATCH 18/47] If ACME_ENV not set, default to staging. --- etc/containerpilot.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index 490923d..09de64a 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -85,7 +85,7 @@ "--cron", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config.{{ .ACME_ENV }}"], + "--config", "/etc/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], "frequency": "12h", "timeout": "10m" }, @@ -99,7 +99,7 @@ "--cleanup", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config.{{ .ACME_ENV }}"], + "--config", "/etc/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], "frequency": "24h", "timeout": "10m" }{{ end }} From 562ee7e04b0f31d3455a1ac4c48b138012103f50 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 20:39:48 -0400 Subject: [PATCH 19/47] consolidated acme related tasks into bin/acme. Moved etc/dehydrated to etc/acme --- bin/acme | 135 ++++++++++++++++++++ bin/acquire-acme-leader | 21 --- bin/generate-token | 17 --- bin/renew-consul-session | 48 ------- etc/{ => acme}/dehydrated/config.production | 0 etc/{ => acme}/dehydrated/config.staging | 0 etc/{ => acme}/dehydrated/hook.sh | 0 etc/acme/watch.hcl | 2 +- etc/containerpilot.json | 32 ++--- 9 files changed, 145 insertions(+), 110 deletions(-) create mode 100755 bin/acme delete mode 100755 bin/acquire-acme-leader delete mode 100755 bin/generate-token delete mode 100755 bin/renew-consul-session rename etc/{ => acme}/dehydrated/config.production (100%) rename etc/{ => acme}/dehydrated/config.staging (100%) rename etc/{ => acme}/dehydrated/hook.sh (100%) diff --git a/bin/acme b/bin/acme new file mode 100755 index 0000000..bfbbac1 --- /dev/null +++ b/bin/acme @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +pushd `dirname $0` > /dev/null +SCRIPTPATH=`pwd -P` +popd > /dev/null + +CONSUL_HOST_DEFAULT="localhost" +if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then + CONSUL_HOST_DEFAULT=${CONSUL} +fi +CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} +CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1" +CONSUL_KEY_ROOT="${CONSUL_ROOT}/kv/nginx" + +SESSION_DIR_DEFAULT="/var/consul" +SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} +SESSION_FILE=${SESSION_DIR}/session + +function getConsulSession () { + if [ -f $SESSION_FILE ]; then + SID=$(cat ${SESSION_DIR}/session) + local STATUS=$(curl -s ${CONSUL_ROOT}/session/info/${SID}) + if [ "${STATUS}" != "[]" ]; then + echo $SID + return 0 + else + return 1 + fi + else + return 1 + fi +} + +function renewConsulSession () { + local SID="$(getConsulSession)" + rc=$? + if [ $rc -ne 0 ]; then + createConsulSession + return $? + else + printf "Renewing Consul session ${SID}... " + local STATUS=$(curl -s -o /dev/null -X PUT -w '%{http_code}' ${CONSUL_ROOT}/session/renew/${SID}) + if [ "${STATUS}" = "200" ]; then + echo "complete" + return 0 + else + echo "failed" + createConsulSession + return $? + fi + fi +} + +function createConsulSession () { + printf "Creating Consul session... " + local SID=$(curl -sX PUT -d '{"LockDelay":"0s","Name":"acme-lock","Behavior":"release","TTL":"600s"}' ${CONSUL_ROOT}/session/create | awk -F '"' '{print $4}') + rc=$? + if [[ $rc -ne 0 ]]; then + echo "failed" + return 1 + else + echo $SID + echo $SID > $SESSION_FILE + return 0 + fi +} + +function acquireLeader () { + local SID="$(getConsulSession)" + STATUS=$(curl -sX PUT -d "$(hostname)" "${CONSUL_KEY_ROOT}/acme/leader?acquire=${SID}") + + if [ "${STATUS}" = "true" ]; then + echo "ACME leader claimed" + return 0 + else + echo "Failed to claim ACME leader" + return 1 + fi +} + +function generateChallengeToken () { + IN=$1 + OUT=$2 + FILENAME="$(sed '1q;d' ${IN})" + VALUE="$(sed '2q;d' ${IN})" + LAST_FILENAME="$(sed '3q;d' ${IN})" + + if [ "${FILENAME}" ]; then + echo "${VALUE}" > ${OUT}/${FILENAME} + fi + + if [ "${LAST_FILENAME}" ]; then + rm -f ${OUT}/${LAST_FILENAME} + fi + + rm ${IN} +} + +case "$1" in + get-consul-session) + getConsulSession + ;; + create-consul-session) + createConsulSession + ;; + renew-consul-session) + renewConsulSession + ;; + acquire-leader) + acquireLeader + ;; + checkin) + renewConsulSession && + ( acquireLeader || exit 0 ) + ;; + renew-certs) + shift + renewConsulSession && + acquireLeader && + ${SCRIPTPATH}/dehydrated --cron "$@" + ;; + clean-certs) + shift + renewConsulSession && + acquireLeader && + ${SCRIPTPATH}/dehydrated --cleanup "$@" + ;; + generate-challenge-token) + generateChallengeToken $2 $3 + ;; + *) + echo $"Usage: $0 [ {get,create,renew}-consul-session | acquire-leader | checkin | renew-certs | clean-certs | generate-challenge-token ]" + exit 1 + ;; +esac diff --git a/bin/acquire-acme-leader b/bin/acquire-acme-leader deleted file mode 100755 index 34f5754..0000000 --- a/bin/acquire-acme-leader +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -CONSUL_HOST_DEFAULT="localhost" -if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then - CONSUL_HOST_DEFAULT=${CONSUL} -fi -CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} -CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1/kv/nginx" - -SESSION_DIR_DEFAULT="/var/consul" -SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} -SESSION_FILE=${SESSION_DIR}/session - -SID=$(cat $SESSION_FILE) - -STATUS=$(curl -sX PUT -d "$(hostname)" -o /dev/null -w '%{http_code}' "${CONSUL_ROOT}/acme/leader?acquire=${SID}") -if [ "${STATUS}" = "200" ]; then - echo "ACME leader claimed" -else - echo "Failed to claim ACME leader" - exit 1 -fi diff --git a/bin/generate-token b/bin/generate-token deleted file mode 100755 index 72c5137..0000000 --- a/bin/generate-token +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -IN=$1 -OUT=$2 -FILENAME="$(sed '1q;d' ${IN})" -VALUE="$(sed '2q;d' ${IN})" -LAST_FILENAME="$(sed '3q;d' ${IN})" - -if [ "${FILENAME}" ]; then - echo "${VALUE}" > ${OUT}/${FILENAME} -fi - -if [ "${LAST_FILENAME}" ]; then - rm -f ${OUT}/${LAST_FILENAME} -fi - -rm ${IN} diff --git a/bin/renew-consul-session b/bin/renew-consul-session deleted file mode 100755 index 12f1205..0000000 --- a/bin/renew-consul-session +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -CONSUL_HOST_DEFAULT="localhost" -if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then - CONSUL_HOST_DEFAULT=${CONSUL} -fi -CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} - -SESSION_DIR_DEFAULT="/var/consul" -SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} -SESSION_FILE=${SESSION_DIR}/session - -function getSession () { - if [ -f $SESSION_FILE ]; then - echo $(cat ${SESSION_DIR}/session) - else - echo "" - fi -} - -function renewSession () { - local SID="$(getSession)" - printf "Renewing Consul session ${SID}... " - local STATUS=$(curl -s -o /dev/null -X PUT -w '%{http_code}' http://$CONSUL_HOST:8500/v1/session/renew/${SID}) - if [ "${STATUS}" = "200" ]; then - echo "complete" - else - echo "failed" - createSession - fi -} - -function createSession () { - printf "Creating Consul session... " - local SID=$(curl -sX PUT -d '{"LockDelay":"0s","Name":"acme-lock","Behavior":"release","TTL":"600s"}' http://127.0.0.1:8500/v1/session/create | awk -F '"' '{print $4}') - rc=$?; if [[ $rc != 0 ]]; then - echo "failed" - else - echo $SID - echo $SID > $SESSION_FILE - fi -} - -if [ -f $SESSION_FILE ]; then - renewSession -else - createSession -fi - diff --git a/etc/dehydrated/config.production b/etc/acme/dehydrated/config.production similarity index 100% rename from etc/dehydrated/config.production rename to etc/acme/dehydrated/config.production diff --git a/etc/dehydrated/config.staging b/etc/acme/dehydrated/config.staging similarity index 100% rename from etc/dehydrated/config.staging rename to etc/acme/dehydrated/config.staging diff --git a/etc/dehydrated/hook.sh b/etc/acme/dehydrated/hook.sh similarity index 100% rename from etc/dehydrated/hook.sh rename to etc/acme/dehydrated/hook.sh diff --git a/etc/acme/watch.hcl b/etc/acme/watch.hcl index a334608..c935561 100644 --- a/etc/acme/watch.hcl +++ b/etc/acme/watch.hcl @@ -18,5 +18,5 @@ template { template { source = "/etc/acme/templates/challenge-token.ctmpl" destination = "/var/www/acme/challenge-token" - command = "/usr/local/bin/generate-token /var/www/acme/challenge-token /var/www/acme/challenge" + command = "/usr/local/bin/acme generate-challenge-token /var/www/acme/challenge-token /var/www/acme/challenge" } diff --git a/etc/containerpilot.json b/etc/containerpilot.json index 09de64a..319f6a1 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -66,40 +66,26 @@ }, "tasks": [{{ if .ACME_DOMAIN }} { - "name": "renew-session-and-leader", - "command": ["/usr/local/bin/renew-consul-session", - "&&", - "/usr/local/bin/acquire-acme-leader", - "||", - "exit 0"], + "name": "acme-checkin", + "command": [ "/usr/local/bin/acme", "checkin" ], "frequency": "150s", "timeout": "10s" }, { - "name": "renew-leader-and-certs", - "command": ["/usr/local/bin/acquire-acme-leader", - "&&", - "cd /var/www/acme", - "&&", - "/usr/local/bin/dehydrated", - "--cron", + "name": "acme-renew-certs", + "command": [ "/usr/local/bin/acme", "renew-certs", "--domain", "{{ .ACME_DOMAIN }}", - "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], + "--hook", "/etc/acme/dehydrated/hook.sh", + "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], "frequency": "12h", "timeout": "10m" }, { "name": "clean-unused-certs", - "command": ["/usr/local/bin/acquire-acme-leader", - "&&", - "cd /var/www/acme", - "&&", - "/usr/local/bin/dehydrated", - "--cleanup", + "command": ["/usr/local/bin/acme", "clean-certs", "--domain", "{{ .ACME_DOMAIN }}", - "--hook", "/etc/dehydrated/hook.sh", - "--config", "/etc/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], + "--hook", "/etc/acme/dehydrated/hook.sh", + "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], "frequency": "24h", "timeout": "10m" }{{ end }} From c5e3485a1ccb274384b9158207df883be3a216af Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 21:40:12 -0400 Subject: [PATCH 20/47] Corrected template formatting errors --- etc/containerpilot.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index 319f6a1..20012f6 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -38,7 +38,7 @@ "-retry-interval", "10s"], "restarts": "unlimited" }{{ end }} - {{ if .CONSUL_AGENT and .ACME_DOMAIN }},{{ end }} + {{ if and .CONSUL_AGENT .ACME_DOMAIN }},{{ end }} {{ if .ACME_DOMAIN }} { "command": ["/usr/local/bin/consul-template", @@ -76,7 +76,7 @@ "command": [ "/usr/local/bin/acme", "renew-certs", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/acme/dehydrated/hook.sh", - "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], + "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], "frequency": "12h", "timeout": "10m" }, @@ -85,7 +85,7 @@ "command": ["/usr/local/bin/acme", "clean-certs", "--domain", "{{ .ACME_DOMAIN }}", "--hook", "/etc/acme/dehydrated/hook.sh", - "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ \"staging\" }}{{ end }}"], + "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], "frequency": "24h", "timeout": "10m" }{{ end }} From 42f92ef0a33da9c586a5a90c783d0d88d47032af Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 21:41:36 -0400 Subject: [PATCH 21/47] Fixed typo. Added jq. --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97b2619..19e4fd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,11 +45,16 @@ RUN export CONTAINERPILOT_CHECKSUM=ec9dbedaca9f4a7a50762f50768cbc42879c7208 \ # Add Dehydrated RUN export DEHYDRATED_VERSION=v0.3.1 \ - && curl --retry 8 --fail -Lso /tmp/dehydrated.zip "https://github.com/lukas2511/dehydrated/archive/${DEHYDRATED_VERSION}.tar.gz" \ - && tar xzf /tmp/dehydrated.tar.gz -C /tmp + && curl --retry 8 --fail -Lso /tmp/dehydrated.tar.gz "https://github.com/lukas2511/dehydrated/archive/${DEHYDRATED_VERSION}.tar.gz" \ + && tar xzf /tmp/dehydrated.tar.gz -C /tmp \ && mv /tmp/dehydrated-0.3.1/dehydrated /usr/local/bin \ && rm -rf /tmp/dehydrated-0.3.1 +# Add jq +RUN export JQ_VERSION=1.5 + curl --retry 8 --fail -Lso /usr/local/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" \ + && chmod a+x /usr/local/bin/jq + # Add our configuration files and scripts COPY etc /etc COPY bin /usr/local/bin From 9f7e18a019fbbd983c2414a6f40ae13bd77cc90d Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 22:08:56 -0400 Subject: [PATCH 22/47] Corrected commands to add jq --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19e4fd9..b0c033c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,8 +51,8 @@ RUN export DEHYDRATED_VERSION=v0.3.1 \ && rm -rf /tmp/dehydrated-0.3.1 # Add jq -RUN export JQ_VERSION=1.5 - curl --retry 8 --fail -Lso /usr/local/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" \ +RUN export JQ_VERSION=1.5 \ + && curl --retry 8 --fail -Lso /usr/local/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64" \ && chmod a+x /usr/local/bin/jq # Add our configuration files and scripts From b65eafca9e847cc4e5929a86a7289ac18bae9c33 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 22:10:05 -0400 Subject: [PATCH 23/47] Verify challenge token is written to all nginx containers before accepting ACME challenge --- etc/acme/dehydrated/hook.sh | 43 ++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/etc/acme/dehydrated/hook.sh b/etc/acme/dehydrated/hook.sh index 6d775b3..2a9e3c8 100755 --- a/etc/acme/dehydrated/hook.sh +++ b/etc/acme/dehydrated/hook.sh @@ -4,23 +4,36 @@ if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then CONSUL_HOST_DEFAULT=${CONSUL} fi CONSUL_HOST=${CONSUL_HOST:-$CONSUL_HOST_DEFAULT} -CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1/kv/nginx" +CONSUL_ROOT="http://${CONSUL_HOST}:8500/v1" +CONSUL_KEY_ROOT="${CONSUL_ROOT}/kv/nginx" CHALLENGE_PATH="/.well-known/acme-challenge" function deploy_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" + local TOKEN_DEPLOY_RETRY_LIMIT=6 - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/acme/challenge/token-filename | log + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_ROOT}/acme/challenge/token-value | log + curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) # verify all nginx containers are responding with challenge before continuing - # TODO: this is currently only looking at localhost - RETRIES=0 + local NGINX_INSTANCES=$(curl -s "${CONSUL_ROOT}/catalog/service/nginx" | jq -Mr '.[].ServiceAddress') + local NGINX_INSTANCE_COUNT=$(echo "${NGINX_INSTANCES}" | wc -l) + local RETRIES=0 + local MATCHING=0 printf " + Waiting for challenge to be deployed..." - while [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://localhost${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" -a $RETRIES -lt 6 ]; do + while [ $RETRIES -lt $TOKEN_DEPLOY_RETRY_LIMIT ]; do + for NGINX_INSTANCE_HOST in $NGINX_INSTANCES; do + if [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://${NGINX_INSTANCE_HOST}${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" ]; then + MATCHING=$((MATCHING+1)) + fi + done + if [ $MATCHING -eq $NGINX_INSTANCE_COUNT ]; then + break + fi RETRIES=$((RETRIES+1)) + MATCHING=0 printf "." sleep 2 done @@ -30,28 +43,28 @@ function deploy_challenge { function clean_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" - curl -sX DELETE ${CONSUL_ROOT}/acme/challenge/token-filename | log + curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX DELETE ${CONSUL_ROOT}/acme/challenge/token-value | log + curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_ROOT}/acme/challenge/last-token-filename | log + curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/last-token-filename | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) } function deploy_cert { local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" - curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_ROOT}/acme/key | log + curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_KEY_ROOT}/acme/key | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_ROOT}/acme/cert | log + curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_KEY_ROOT}/acme/cert | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_ROOT}/acme/chain | log + curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_KEY_ROOT}/acme/chain | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_ROOT}/acme/fullchain | log + curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_KEY_ROOT}/acme/fullchain | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_ROOT}/acme/timestamp | log + curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_KEY_ROOT}/acme/timestamp | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(date +%s)" ${CONSUL_ROOT}/acme/touched | log + curl -sX PUT -d "$(date +%s)" ${CONSUL_KEY_ROOT}/acme/touched | log test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) } From b48b811502734816f49dfe2290cc8ba933a27554 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 22:10:28 -0400 Subject: [PATCH 24/47] No limit to consul-template restarts. Formatting fixes. --- etc/containerpilot.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index 20012f6..7ea3cfc 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -41,9 +41,10 @@ {{ if and .CONSUL_AGENT .ACME_DOMAIN }},{{ end }} {{ if .ACME_DOMAIN }} { - "command": ["/usr/local/bin/consul-template", - "-config", "/etc/acme/watch.hcl", - "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500"] + "command": ["/usr/local/bin/consul-template", + "-config", "/etc/acme/watch.hcl", + "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500"], + "restarts": "unlimited" }{{ end }}], "telemetry": { "port": 9090, @@ -74,9 +75,9 @@ { "name": "acme-renew-certs", "command": [ "/usr/local/bin/acme", "renew-certs", - "--domain", "{{ .ACME_DOMAIN }}", - "--hook", "/etc/acme/dehydrated/hook.sh", - "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], + "--domain", "{{ .ACME_DOMAIN }}", + "--hook", "/etc/acme/dehydrated/hook.sh", + "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], "frequency": "12h", "timeout": "10m" }, From f8f384a9559366d4e0ecc585373fc87f08887536 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Mon, 3 Oct 2016 22:18:20 -0400 Subject: [PATCH 25/47] Defined dehydrated WELLKNOWN --- etc/acme/dehydrated/config.production | 1 + etc/acme/dehydrated/config.staging | 1 + 2 files changed, 2 insertions(+) diff --git a/etc/acme/dehydrated/config.production b/etc/acme/dehydrated/config.production index 955e35a..9e7a77c 100644 --- a/etc/acme/dehydrated/config.production +++ b/etc/acme/dehydrated/config.production @@ -1 +1,2 @@ CA="https://acme-v01.api.letsencrypt.org/directory" +WELLKNOWN="/var/www/acme/challenge" diff --git a/etc/acme/dehydrated/config.staging b/etc/acme/dehydrated/config.staging index 5165b54..a6d20b5 100644 --- a/etc/acme/dehydrated/config.staging +++ b/etc/acme/dehydrated/config.staging @@ -1 +1,2 @@ CA="https://acme-staging.api.letsencrypt.org/directory" +WELLKNOWN="/var/www/acme/challenge" From 7122b63c5be1f2ed5d87e64e75ae229e0b4f8ce5 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 17:16:01 -0400 Subject: [PATCH 26/47] Moved ENV VAR lookups into acme script to declutter containerpilot.json. Add init command to be used by health check to insure certs are installed before ssl health check is a success. This may change in the future, see: https://github.com/joyent/containerpilot/issues/227 --- bin/acme | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/acme b/bin/acme index bfbbac1..0947997 100755 --- a/bin/acme +++ b/bin/acme @@ -16,6 +16,10 @@ SESSION_DIR_DEFAULT="/var/consul" SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} SESSION_FILE=${SESSION_DIR}/session +CERT_DIR="/var/www/ssl" + +ACME_ENV=${ACME_ENV:-staging} + function getConsulSession () { if [ -f $SESSION_FILE ]; then SID=$(cat ${SESSION_DIR}/session) @@ -113,23 +117,33 @@ case "$1" in renewConsulSession && ( acquireLeader || exit 0 ) ;; + init) + if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem -a "$(cat ${CERT_DIR}/fullchain.pem)" != "" -a "$(cat ${CERT_DIR}/privkey.pem)" != "" ]; then + exit 0 + fi + shift + renewConsulSession && + acquireLeader && + ${SCRIPTPATH}/dehydrated --cron --domain ${ACME_DOMAIN} --hook /etc/acme/dehydrated/hook.sh --config /etc/acme/dehydrated/config.${ACME_ENV} && + ${SCRIPTPATH}/reload.sh onChange + ;; renew-certs) shift renewConsulSession && acquireLeader && - ${SCRIPTPATH}/dehydrated --cron "$@" + ${SCRIPTPATH}/dehydrated --cron --domain ${ACME_DOMAIN} --hook /etc/acme/dehydrated/hook.sh --config /etc/acme/dehydrated/config.${ACME_ENV} ;; clean-certs) shift renewConsulSession && acquireLeader && - ${SCRIPTPATH}/dehydrated --cleanup "$@" + ${SCRIPTPATH}/dehydrated --cleanup --domain ${ACME_DOMAIN} --hook /etc/acme/dehydrated/hook.sh --config /etc/acme/dehydrated/config.${ACME_ENV} ;; generate-challenge-token) generateChallengeToken $2 $3 ;; *) - echo $"Usage: $0 [ {get,create,renew}-consul-session | acquire-leader | checkin | renew-certs | clean-certs | generate-challenge-token ]" + echo $"Usage: $0 [ {get,create,renew}-consul-session | acquire-leader | init | checkin | renew-certs | clean-certs | generate-challenge-token ]" exit 1 ;; esac From 079f977115a056e3bc81b3554b7d5b8de1e1db5a Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 17:22:20 -0400 Subject: [PATCH 27/47] Upgraded consul to 0.7.0. Adjusted options. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index dac2d85..e545781 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ example: # Start with a single host which will bootstrap the cluster. # In production we'll want to use an HA cluster. consul: - image: progrium/consul:latest + image: consul:v0.7.0 restart: always mem_limit: 128m expose: @@ -49,4 +49,4 @@ consul: - 127.0.0.1 labels: - triton.cns.services=consul - command: -server -bootstrap -ui-dir /ui + command: agent -server -client=0.0.0.0 -bootstrap -ui From 56a39e1ac3547ce3168ba3682200abae58e4d1ab Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:06:02 -0400 Subject: [PATCH 28/47] Insure all nginx containers have challenge token before accepting challenge --- etc/acme/dehydrated/hook.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/etc/acme/dehydrated/hook.sh b/etc/acme/dehydrated/hook.sh index 2a9e3c8..823ec3d 100755 --- a/etc/acme/dehydrated/hook.sh +++ b/etc/acme/dehydrated/hook.sh @@ -23,17 +23,15 @@ function deploy_challenge { local RETRIES=0 local MATCHING=0 printf " + Waiting for challenge to be deployed..." - while [ $RETRIES -lt $TOKEN_DEPLOY_RETRY_LIMIT ]; do + while [ $RETRIES -lt $TOKEN_DEPLOY_RETRY_LIMIT -a $MATCHING -lt $NGINX_INSTANCE_COUNT ]; do + echo "MATCHING: $MATCHING, COUNT: $NGINX_INSTANCE_COUNT" + MATCHING=0 for NGINX_INSTANCE_HOST in $NGINX_INSTANCES; do - if [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://${NGINX_INSTANCE_HOST}${CHALLENGE_PATH}/${TOKEN_FILENAME})" != "${TOKEN_VALUE}" ]; then + if [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://${NGINX_INSTANCE_HOST}${CHALLENGE_PATH}/${TOKEN_FILENAME})" = "${TOKEN_VALUE}" ]; then MATCHING=$((MATCHING+1)) fi done - if [ $MATCHING -eq $NGINX_INSTANCE_COUNT ]; then - break - fi RETRIES=$((RETRIES+1)) - MATCHING=0 printf "." sleep 2 done From f1dfdfbca57285a8e0955ef14360f129e14d3209 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:06:36 -0400 Subject: [PATCH 29/47] check that key exists before writing it --- etc/acme/templates/cert.ctmpl | 2 +- etc/acme/templates/chain.ctmpl | 2 +- etc/acme/templates/challenge-token.ctmpl | 6 +++--- etc/acme/templates/fullchain.ctmpl | 2 +- etc/acme/templates/privkey.ctmpl | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/acme/templates/cert.ctmpl b/etc/acme/templates/cert.ctmpl index 765aaf0..46d94d2 100644 --- a/etc/acme/templates/cert.ctmpl +++ b/etc/acme/templates/cert.ctmpl @@ -1 +1 @@ -{{key "nginx/acme/cert"}} +{{if key "nginx/acme/cert"}}{{key "nginx/acme/cert"}}{{end}} diff --git a/etc/acme/templates/chain.ctmpl b/etc/acme/templates/chain.ctmpl index e0e0d94..c914597 100644 --- a/etc/acme/templates/chain.ctmpl +++ b/etc/acme/templates/chain.ctmpl @@ -1 +1 @@ -{{key "nginx/acme/chain"}} +{{if key "nginx/acme/chain"}}{{key "nginx/acme/chain"}}{{end}} diff --git a/etc/acme/templates/challenge-token.ctmpl b/etc/acme/templates/challenge-token.ctmpl index 895bf38..902c5ce 100644 --- a/etc/acme/templates/challenge-token.ctmpl +++ b/etc/acme/templates/challenge-token.ctmpl @@ -1,3 +1,3 @@ -{{key "nginx/acme/challenge/token-filename"}} -{{key "nginx/acme/challenge/token-value"}} -{{key "nginx/acme/challenge/last-token-filename"}} +{{if key "nginx/acme/challenge/token-filename"}}{{key "nginx/acme/challenge/token-filename"}}{{end}} +{{if key "nginx/acme/challenge/token-value"}}{{key "nginx/acme/challenge/token-value"}}{{end}} +{{if key "nginx/acme/challenge/last-token-filename"}}{{key "nginx/acme/challenge/last-token-filename"}}{{end}} diff --git a/etc/acme/templates/fullchain.ctmpl b/etc/acme/templates/fullchain.ctmpl index 5db9394..3a785ff 100644 --- a/etc/acme/templates/fullchain.ctmpl +++ b/etc/acme/templates/fullchain.ctmpl @@ -1 +1 @@ -{{key "nginx/acme/fullchain"}} +{{if key "nginx/acme/fullchain"}}{{key "nginx/acme/fullchain"}}{{end}} diff --git a/etc/acme/templates/privkey.ctmpl b/etc/acme/templates/privkey.ctmpl index 704a1f8..0a4a20b 100644 --- a/etc/acme/templates/privkey.ctmpl +++ b/etc/acme/templates/privkey.ctmpl @@ -1 +1 @@ -{{key "nginx/acme/key"}} +{{if key "nginx/acme/key"}}{{key "nginx/acme/key"}}{{end}} From 99141c46d1b5aab3a6ea209ee340a842d2f45854 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:07:55 -0400 Subject: [PATCH 30/47] Add SSL interface. Increasedd acme-checkin frequency from 2.5 mins to 5, --- etc/containerpilot.json | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index 7ea3cfc..d304d31 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -18,7 +18,15 @@ "poll": 10, "ttl": 25, "interfaces": ["eth1", "eth0"] - } + }{{ if .ACME_DOMAIN }}, + { + "name": "nginx-public-ssl", + "port": 443, + "health": "/usr/local/bin/acme init && /usr/bin/curl --insecure --fail --silent --show-error --output /dev/null https://localhost/nginx-health", + "poll": 10, + "ttl": 25, + "interfaces": ["eth1", "eth0"] + }{{ end }} ], "backends": [ { @@ -69,7 +77,7 @@ { "name": "acme-checkin", "command": [ "/usr/local/bin/acme", "checkin" ], - "frequency": "150s", + "frequency": "5m", "timeout": "10s" }, { From 0b2a71e6b960fe933a20c590f2c0dd94b420e0a5 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:09:33 -0400 Subject: [PATCH 31/47] Handle dehydrated flags in acme script --- etc/containerpilot.json | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index d304d31..a880321 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -82,19 +82,13 @@ }, { "name": "acme-renew-certs", - "command": [ "/usr/local/bin/acme", "renew-certs", - "--domain", "{{ .ACME_DOMAIN }}", - "--hook", "/etc/acme/dehydrated/hook.sh", - "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], + "command": [ "/usr/local/bin/acme", "renew-certs" ], "frequency": "12h", "timeout": "10m" }, { "name": "clean-unused-certs", - "command": ["/usr/local/bin/acme", "clean-certs", - "--domain", "{{ .ACME_DOMAIN }}", - "--hook", "/etc/acme/dehydrated/hook.sh", - "--config", "/etc/acme/dehydrated/config.{{ if .ACME_ENV }}{{ .ACME_ENV }}{{ else }}{{ `"staging"` }}{{ end }}"], + "command": ["/usr/local/bin/acme", "clean-certs" ], "frequency": "24h", "timeout": "10m" }{{ end }} From 04feeb30ce3f90f0908f549fc74d34235afcadd5 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:11:10 -0400 Subject: [PATCH 32/47] Add SSL directives to Nginx config if ACME_DOMAIN env var set --- etc/nginx/nginx.conf.ctmpl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/etc/nginx/nginx.conf.ctmpl b/etc/nginx/nginx.conf.ctmpl index 51add7b..290a5cb 100644 --- a/etc/nginx/nginx.conf.ctmpl +++ b/etc/nginx/nginx.conf.ctmpl @@ -53,7 +53,21 @@ http { {{ if $acme_domain }} location /.well-known/acme-challenge { alias /var/www/acme/challenge; - }{{ end }} + } + {{ if key "nginx/acme/fullchain" }} + listen 443 ssl; + ssl_certificate /var/www/ssl/fullchain.pem; + ssl_certificate_key /var/www/ssl/privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:50m; + ssl_stapling on; + ssl_stapling_verify on; + add_header Strict-Transport-Security max-age=15768000; + {{ end }} + {{ end }} {{ if service $backend }} location = /{{ $backend }} { From 6aac1012ced21413e4d037258cbf80681c16ed85 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:18:24 -0400 Subject: [PATCH 33/47] Remove debugging messages --- etc/acme/dehydrated/hook.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/acme/dehydrated/hook.sh b/etc/acme/dehydrated/hook.sh index 823ec3d..372dbb6 100755 --- a/etc/acme/dehydrated/hook.sh +++ b/etc/acme/dehydrated/hook.sh @@ -24,7 +24,6 @@ function deploy_challenge { local MATCHING=0 printf " + Waiting for challenge to be deployed..." while [ $RETRIES -lt $TOKEN_DEPLOY_RETRY_LIMIT -a $MATCHING -lt $NGINX_INSTANCE_COUNT ]; do - echo "MATCHING: $MATCHING, COUNT: $NGINX_INSTANCE_COUNT" MATCHING=0 for NGINX_INSTANCE_HOST in $NGINX_INSTANCES; do if [ "$(curl -s --header \"HOST: ${DOMAIN}\" http://${NGINX_INSTANCE_HOST}${CHALLENGE_PATH}/${TOKEN_FILENAME})" = "${TOKEN_VALUE}" ]; then From b51807e9e1c9bf1f3f98161a4ce401416b480eba Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Tue, 4 Oct 2016 18:19:12 -0400 Subject: [PATCH 34/47] Upgrade consul agent to 0.7.0 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0c033c..19f24ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ RUN apt-get update \ # Install Consul # Releases at https://releases.hashicorp.com/consul -RUN export CONSUL_VERSION=0.6.4 \ - && export CONSUL_CHECKSUM=abdf0e1856292468e2c9971420d73b805e93888e006c76324ae39416edcf0627 \ +RUN export CONSUL_VERSION=0.7.0 \ + && export CONSUL_CHECKSUM=b350591af10d7d23514ebaa0565638539900cdb3aaa048f077217c4c46653dd8 \ && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \ && echo "${CONSUL_CHECKSUM} /tmp/consul.zip" | sha256sum -c \ && unzip /tmp/consul -d /usr/local/bin \ From 55b893f5dca367b3ba5d37897a0cb57e6de33677 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 10:21:07 -0400 Subject: [PATCH 35/47] Seperate template for ssl enabled nginx for startup reasons --- etc/nginx/nginx-ssl.conf.ctmpl | 80 ++++++++++++++++++++++++++++++++++ etc/nginx/nginx.conf.ctmpl | 13 ------ 2 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 etc/nginx/nginx-ssl.conf.ctmpl diff --git a/etc/nginx/nginx-ssl.conf.ctmpl b/etc/nginx/nginx-ssl.conf.ctmpl new file mode 100644 index 0000000..d3458a6 --- /dev/null +++ b/etc/nginx/nginx-ssl.conf.ctmpl @@ -0,0 +1,80 @@ +# This is an example Nginx configuration template file. +# Adjust the values below as required for your application. + +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + {{ $backend := env "BACKEND" }} + {{ $acme_domain := env "ACME_DOMAIN" }} + {{ if service $backend }} + upstream {{ $backend }} { + # write the address:port pairs for each healthy backend instance + {{range service $backend }} + server {{.Address}}:{{.Port}}; + {{end}} + least_conn; + }{{ end }} + + server { + listen 80; + server_name _; + + location /nginx-health { + stub_status; + allow 127.0.0.1; + deny all; + } + + {{ if $acme_domain }} + location /.well-known/acme-challenge { + alias /var/www/acme/challenge; + } + {{ end }} + listen 443 ssl; + ssl_certificate /var/www/ssl/fullchain.pem; + ssl_certificate_key /var/www/ssl/privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:50m; + ssl_stapling on; + ssl_stapling_verify on; + add_header Strict-Transport-Security max-age=15768000; + + {{ if service $backend }} + location = /{{ $backend }} { + return 301 /{{ $backend }}/; + } + + location /{{ $backend }} { + proxy_pass http://{{ $backend }}; + proxy_redirect off; + }{{ end }} + } +} diff --git a/etc/nginx/nginx.conf.ctmpl b/etc/nginx/nginx.conf.ctmpl index 290a5cb..ddf65c2 100644 --- a/etc/nginx/nginx.conf.ctmpl +++ b/etc/nginx/nginx.conf.ctmpl @@ -54,19 +54,6 @@ http { location /.well-known/acme-challenge { alias /var/www/acme/challenge; } - {{ if key "nginx/acme/fullchain" }} - listen 443 ssl; - ssl_certificate /var/www/ssl/fullchain.pem; - ssl_certificate_key /var/www/ssl/privkey.pem; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_prefer_server_ciphers on; - ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; - ssl_session_timeout 1d; - ssl_session_cache shared:SSL:50m; - ssl_stapling on; - ssl_stapling_verify on; - add_header Strict-Transport-Security max-age=15768000; - {{ end }} {{ end }} {{ if service $backend }} From c1bf4abd5baed873e9e4f0511fd005b5918131a1 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 10:22:18 -0400 Subject: [PATCH 36/47] Reload will choose whether to use ssl template or non-ssl template based on existance of keys --- bin/reload.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/reload.sh b/bin/reload.sh index 9ef8bff..abeb12e 100755 --- a/bin/reload.sh +++ b/bin/reload.sh @@ -2,6 +2,7 @@ SERVICE_NAME=${SERVICE_NAME:-nginx} CONSUL=${CONSUL:-consul} +CERT_DIR="/var/www/ssl" # Render Nginx configuration template using values from Consul, # but do not reload because Nginx has't started yet @@ -16,11 +17,15 @@ preStart() { # Render Nginx configuration template using values from Consul, # then gracefully reload Nginx onChange() { + local TEMPLATE="nginx.conf.ctmpl" + if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem -a "$(cat ${CERT_DIR}/fullchain.pem)" != "" -a "$(cat ${CERT_DIR}/privkey.pem)" != "" ]; then + TEMPLATE="nginx-ssl.conf.ctmpl" + fi consul-template \ -once \ -dedup \ -consul ${CONSUL}:8500 \ - -template "/etc/nginx/nginx.conf.ctmpl:/etc/nginx/nginx.conf:nginx -s reload" + -template "/etc/nginx/${TEMPLATE}:/etc/nginx/nginx.conf:nginx -s reload" } help() { From 0ac4ba6e064cb94fda2efb71f12273825940d003 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 10:23:18 -0400 Subject: [PATCH 37/47] Perform reload as part of consul-template key writes instead of acme init, for startup reasons --- bin/acme | 3 +-- etc/acme/watch.hcl | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/acme b/bin/acme index 0947997..45a9327 100755 --- a/bin/acme +++ b/bin/acme @@ -124,8 +124,7 @@ case "$1" in shift renewConsulSession && acquireLeader && - ${SCRIPTPATH}/dehydrated --cron --domain ${ACME_DOMAIN} --hook /etc/acme/dehydrated/hook.sh --config /etc/acme/dehydrated/config.${ACME_ENV} && - ${SCRIPTPATH}/reload.sh onChange + ${SCRIPTPATH}/dehydrated --cron --domain ${ACME_DOMAIN} --hook /etc/acme/dehydrated/hook.sh --config /etc/acme/dehydrated/config.${ACME_ENV} ;; renew-certs) shift diff --git a/etc/acme/watch.hcl b/etc/acme/watch.hcl index c935561..0bbf9d9 100644 --- a/etc/acme/watch.hcl +++ b/etc/acme/watch.hcl @@ -6,10 +6,12 @@ template { template { source = "/etc/acme/templates/privkey.ctmpl" destination = "/var/www/ssl/privkey.pem" + command = "/usr/local/bin/reload.sh" } template { source = "/etc/acme/templates/fullchain.ctmpl" destination = "/var/www/ssl/fullchain.pem" + command = "/usr/local/bin/reload.sh" } template { source = "/etc/acme/templates/chain.ctmpl" From 0d2b2c83cdc30645802c3fac4a423425d45d8806 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 10:24:16 -0400 Subject: [PATCH 38/47] Expose port 443. Correct mem_limit. --- docker-compose.yml | 1 + local-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e545781..d9d553a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,7 @@ nginx: - ACME_ENV=staging ports: - 80 + - 443 - 9090 # so we can see telemetry labels: - triton.cns.services=nginx diff --git a/local-compose.yml b/local-compose.yml index a644c6b..211450f 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -3,7 +3,7 @@ nginx: file: docker-compose.yml service: nginx build: . - mem_limit: 128g + mem_limit: 128m environment: - CONSUL=consul - CONSUL_AGENT=1 @@ -12,6 +12,7 @@ nginx: - consul:consul ports: - 80:80 + - 443:443 - 9090:9090 # telemetry endpoint example: From 0019d513ae905923344b4a7f2bf9b0eee518937b Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 10:35:14 -0400 Subject: [PATCH 39/47] Mention ACME support in README --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 70ed823..ba0a6db 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ Autopilot Pattern Nginx -========== +======================= *A re-usable Nginx base image implemented according to the [Autopilot Pattern](http://autopilotpattern.io/) for automatic discovery and configuration.* @@ -39,3 +39,40 @@ You can open the demo app that Nginx is proxying by opening a browser to the Ngi ```bash open "http://$(triton ip nginx_nginx_1)/example" ``` + +### Configuring LetsEncrypt (ACME) + +Setting the `ACME_DOMAIN` environment variable will enable LetsEncrypt within +the image. The image will automatically acquire certificates for the given +domain, and renew them over time. If you scale to multiple instances of Nginx, +they will elect a leader who will be responsible for renewing the certificates. +Any challenge response tokens as well as acquired certificates will be +replicated to all Nginx instances. + +By default, this process will use the LetsEncrypt staging endpoint, so as not to +impact your api limits. When ready for production, you must also set the +`ACME_ENV` environment variable to `production`. + +You must ensure the domain resolves to your Nginx containers so that they can +respond to the ACME http challenges. + +Example excerpt from `docker-compose.yml` with LetsEncrypt enabled: + +```yaml +nginx: + image: autopilotpattern/nginx + restart: always + mem_limit: 512m + env_file: _env + environment: + - BACKEND=example + - CONSUL_AGENT=1 + - ACME_ENV=staging + - ACME_DOMAIN=example.com + ports: + - 80 + - 443 + - 9090 + labels: + - triton.cns.services=nginx +``` From 3416757fee9d61b203c401219fe33225f8a8ee2d Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 15:43:09 -0400 Subject: [PATCH 40/47] Clean up logs my omitting sucesful (200) health check requests from nginx logging. --- etc/nginx/nginx-ssl.conf.ctmpl | 6 ++++++ etc/nginx/nginx.conf.ctmpl | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/etc/nginx/nginx-ssl.conf.ctmpl b/etc/nginx/nginx-ssl.conf.ctmpl index d3458a6..a85a880 100644 --- a/etc/nginx/nginx-ssl.conf.ctmpl +++ b/etc/nginx/nginx-ssl.conf.ctmpl @@ -16,6 +16,11 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; + map $status $isError { + ~^2 0; + default 1; + } + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; @@ -48,6 +53,7 @@ http { stub_status; allow 127.0.0.1; deny all; + access_log /var/log/nginx/access.log main if=$isError; } {{ if $acme_domain }} diff --git a/etc/nginx/nginx.conf.ctmpl b/etc/nginx/nginx.conf.ctmpl index ddf65c2..54add5d 100644 --- a/etc/nginx/nginx.conf.ctmpl +++ b/etc/nginx/nginx.conf.ctmpl @@ -11,11 +11,15 @@ events { worker_connections 1024; } - http { include /etc/nginx/mime.types; default_type application/octet-stream; + map $status $isError { + ~^2 0; + default 1; + } + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; @@ -48,6 +52,7 @@ http { stub_status; allow 127.0.0.1; deny all; + access_log /var/log/nginx/access.log main if=$isError; } {{ if $acme_domain }} From 6025b0f2dcb0b0b1517f28a072cba437881d1893 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Thu, 6 Oct 2016 16:14:54 -0400 Subject: [PATCH 41/47] Updated example backend to Consul v0.7.0 --- example-backend/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/example-backend/Dockerfile b/example-backend/Dockerfile index 0d70898..55a0b7f 100755 --- a/example-backend/Dockerfile +++ b/example-backend/Dockerfile @@ -12,9 +12,10 @@ COPY package.json /opt/example/ RUN cd /opt/example && npm install # Add Consul from https://releases.hashicorp.com/consul -RUN export CHECKSUM=abdf0e1856292468e2c9971420d73b805e93888e006c76324ae39416edcf0627 \ - && curl -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip" \ - && echo "${CHECKSUM} /tmp/consul.zip" | sha256sum -c \ +RUN export CONSUL_VERSION=0.7.0 \ + && export CONSUL_CHECKSUM=b350591af10d7d23514ebaa0565638539900cdb3aaa048f077217c4c46653dd8 \ + && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \ + && echo "${CONSUL_CHECKSUM} /tmp/consul.zip" | sha256sum -c \ && unzip /tmp/consul -d /usr/local/bin \ && rm /tmp/consul.zip \ && mkdir /config From 770924232f4ea7558cbcab7cda4710d7940ef439 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 10:39:56 -0400 Subject: [PATCH 42/47] Un-handwrapped text --- README.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ba0a6db..e16ef00 100644 --- a/README.md +++ b/README.md @@ -42,19 +42,11 @@ open "http://$(triton ip nginx_nginx_1)/example" ### Configuring LetsEncrypt (ACME) -Setting the `ACME_DOMAIN` environment variable will enable LetsEncrypt within -the image. The image will automatically acquire certificates for the given -domain, and renew them over time. If you scale to multiple instances of Nginx, -they will elect a leader who will be responsible for renewing the certificates. -Any challenge response tokens as well as acquired certificates will be -replicated to all Nginx instances. - -By default, this process will use the LetsEncrypt staging endpoint, so as not to -impact your api limits. When ready for production, you must also set the -`ACME_ENV` environment variable to `production`. - -You must ensure the domain resolves to your Nginx containers so that they can -respond to the ACME http challenges. +Setting the `ACME_DOMAIN` environment variable will enable LetsEncrypt within the image. The image will automatically acquire certificates for the given domain, and renew them over time. If you scale to multiple instances of Nginx, they will elect a leader who will be responsible for renewing the certificates. Any challenge response tokens as well as acquired certificates will be replicated to all Nginx instances. + +By default, this process will use the LetsEncrypt staging endpoint, so as not to impact your api limits. When ready for production, you must also set the `ACME_ENV` environment variable to `production`. + +You must ensure the domain resolves to your Nginx containers so that they can respond to the ACME http challenges. Example excerpt from `docker-compose.yml` with LetsEncrypt enabled: From 5b1c69a07d868de543f06c76e6ae9ff0c0aba7ba Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 14:20:41 -0400 Subject: [PATCH 43/47] Link to document on using your own domain with Triton CNS --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e16ef00..0a66ccf 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Setting the `ACME_DOMAIN` environment variable will enable LetsEncrypt within th By default, this process will use the LetsEncrypt staging endpoint, so as not to impact your api limits. When ready for production, you must also set the `ACME_ENV` environment variable to `production`. -You must ensure the domain resolves to your Nginx containers so that they can respond to the ACME http challenges. +You must ensure the domain resolves to your Nginx containers so that they can respond to the ACME http challenges. Triton users may [refer to this document](https://docs.joyent.com/public-cloud/network/cns/faq#can-i-use-my-own-domain-name-with-triton-cns) for more information on how to insure your domain resolves to your Triton containers. Example excerpt from `docker-compose.yml` with LetsEncrypt enabled: From cdfaf0fc3ec6123c84a02074695ee863c79c36aa Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 14:21:46 -0400 Subject: [PATCH 44/47] Improve readability of code by using pipefail --- etc/acme/dehydrated/hook.sh | 42 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/etc/acme/dehydrated/hook.sh b/etc/acme/dehydrated/hook.sh index 372dbb6..1f147e7 100755 --- a/etc/acme/dehydrated/hook.sh +++ b/etc/acme/dehydrated/hook.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -o pipefail + CONSUL_HOST_DEFAULT="localhost" if [ "${CONSUL_AGENT}" = "" -a "${CONSUL}" != "" ]; then CONSUL_HOST_DEFAULT=${CONSUL} @@ -12,10 +14,9 @@ function deploy_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" local TOKEN_DEPLOY_RETRY_LIMIT=6 - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + (curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log \ + && curl -sX PUT -d "${TOKEN_VALUE}" ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log \ + ) || (log "${FUNCNAME} failed"; return 1) # verify all nginx containers are responding with challenge before continuing local NGINX_INSTANCES=$(curl -s "${CONSUL_ROOT}/catalog/service/nginx" | jq -Mr '.[].ServiceAddress') @@ -40,29 +41,26 @@ function deploy_challenge { function clean_challenge { local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}" - curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/last-token-filename | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-filename | log \ + && curl -sX DELETE ${CONSUL_KEY_ROOT}/acme/challenge/token-value | log \ + && curl -sX PUT -d "${TOKEN_FILENAME}" ${CONSUL_KEY_ROOT}/acme/challenge/last-token-filename | log \ + && return 0 + log "${FUNCNAME} failed" + return 1 } function deploy_cert { local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" - curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_KEY_ROOT}/acme/key | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_KEY_ROOT}/acme/cert | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_KEY_ROOT}/acme/chain | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_KEY_ROOT}/acme/fullchain | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_KEY_ROOT}/acme/timestamp | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) - curl -sX PUT -d "$(date +%s)" ${CONSUL_KEY_ROOT}/acme/touched | log - test ${PIPESTATUS[0]} -eq 0 || (log "${FUNCNAME} failed" && return) + curl -sX PUT -d "$(cat ${KEYFILE})" ${CONSUL_KEY_ROOT}/acme/key | log \ + && curl -sX PUT -d "$(cat ${CERTFILE})" ${CONSUL_KEY_ROOT}/acme/cert | log \ + && curl -sX PUT -d "$(cat ${CHAINFILE})" ${CONSUL_KEY_ROOT}/acme/chain | log \ + && curl -sX PUT -d "$(cat ${FULLCHAINFILE})" ${CONSUL_KEY_ROOT}/acme/fullchain | log \ + && curl -sX PUT -d "${TIMESTAMP}" ${CONSUL_KEY_ROOT}/acme/timestamp | log \ + && curl -sX PUT -d "$(date +%s)" ${CONSUL_KEY_ROOT}/acme/touched | log \ + && return 0 + log "${FUNCNAME} failed" + return 1 } function unchanged_cert { From 8febef63641630eb4d50ad827820f2c54532435e Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 14:39:04 -0400 Subject: [PATCH 45/47] Redirect http to https (except for health check) when SSL is enabled --- etc/nginx/nginx-ssl.conf.ctmpl | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/etc/nginx/nginx-ssl.conf.ctmpl b/etc/nginx/nginx-ssl.conf.ctmpl index a85a880..4f14528 100644 --- a/etc/nginx/nginx-ssl.conf.ctmpl +++ b/etc/nginx/nginx-ssl.conf.ctmpl @@ -46,7 +46,23 @@ http { }{{ end }} server { - listen 80; + listen 80; + server_name _; + + location / { + return 301 https://$host$request_uri; + } + + location /nginx-health { + stub_status; + allow 127.0.0.1; + deny all; + access_log /var/log/nginx/access.log main if=$isError; + } + } + + server { + listen 443 ssl; server_name _; location /nginx-health { @@ -56,12 +72,10 @@ http { access_log /var/log/nginx/access.log main if=$isError; } - {{ if $acme_domain }} location /.well-known/acme-challenge { alias /var/www/acme/challenge; } - {{ end }} - listen 443 ssl; + ssl_certificate /var/www/ssl/fullchain.pem; ssl_certificate_key /var/www/ssl/privkey.pem; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; From f4f21b23fceaa4417f4354dd75f8a927847fb194 Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 15:01:48 -0400 Subject: [PATCH 46/47] Include host header in ssl health check --- etc/containerpilot.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/containerpilot.json b/etc/containerpilot.json index a880321..8f21a19 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -22,7 +22,7 @@ { "name": "nginx-public-ssl", "port": 443, - "health": "/usr/local/bin/acme init && /usr/bin/curl --insecure --fail --silent --show-error --output /dev/null https://localhost/nginx-health", + "health": "/usr/local/bin/acme init && /usr/bin/curl --insecure --fail --silent --show-error --output /dev/null --header \"HOST: {{ .ACME_DOMAIN }}\" https://localhost/nginx-health", "poll": 10, "ttl": 25, "interfaces": ["eth1", "eth0"] From 894f7e7a4aa1ca1b3e749910636676ae5016047d Mon Sep 17 00:00:00 2001 From: Jason Pincin Date: Fri, 7 Oct 2016 15:32:13 -0400 Subject: [PATCH 47/47] Write certs/keys to temporary location before copying them to final location. This prevents empty cert/key files from being written and simplifies logic pertaining to their existence --- Dockerfile | 4 +++- bin/acme | 18 +++++++++++++++++- bin/reload.sh | 2 +- etc/acme/watch.hcl | 12 ++++++------ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19f24ef..57366b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,8 +59,10 @@ RUN export JQ_VERSION=1.5 \ COPY etc /etc COPY bin /usr/local/bin -# SSL certs written here +# Usable SSL certs written here RUN mkdir -p /var/www/ssl +# Temporary/work space for keys +RUN mkdir -p /var/www/acme/ssl # ACME challenge tokens written here RUN mkdir -p /var/www/acme/challenge # Consul session data written here diff --git a/bin/acme b/bin/acme index 45a9327..fe9bbc6 100755 --- a/bin/acme +++ b/bin/acme @@ -16,6 +16,7 @@ SESSION_DIR_DEFAULT="/var/consul" SESSION_DIR=${SESSION_DIR:-$SESSION_DIR_DEFAULT} SESSION_FILE=${SESSION_DIR}/session +TEMP_CERT_DIR="/var/www/acme/ssl" CERT_DIR="/var/www/ssl" ACME_ENV=${ACME_ENV:-staging} @@ -100,6 +101,18 @@ function generateChallengeToken () { rm ${IN} } +function updateKeys () { + local TEMP_FULLCHAIN="${TEMP_CERT_DIR}/fullchain.pem" + local TEMP_PRIVKEY="${TEMP_CERT_DIR}/privkey.pem" + local FULLCHAIN="${CERT_DIR}/fullchain.pem" + local PRIVKEY="${CERT_DIR}/privkey.pem" + if [ -f ${TEMP_FULLCHAIN} -a -f ${TEMP_PRIVKEY} -a "$(cat ${TEMP_FULLCHAIN})" != "" -a "$(cat ${TEMP_PRIVKEY})" != "" ]; then + cp -f $TEMP_FULLCHAIN $FULLCHAIN + cp -f $TEMP_PRIVKEY $PRIVKEY + $SCRIPTPATH/reload.sh + fi +} + case "$1" in get-consul-session) getConsulSession @@ -118,7 +131,7 @@ case "$1" in ( acquireLeader || exit 0 ) ;; init) - if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem -a "$(cat ${CERT_DIR}/fullchain.pem)" != "" -a "$(cat ${CERT_DIR}/privkey.pem)" != "" ]; then + if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem ]; then exit 0 fi shift @@ -141,6 +154,9 @@ case "$1" in generate-challenge-token) generateChallengeToken $2 $3 ;; + update-keys) + updateKeys + ;; *) echo $"Usage: $0 [ {get,create,renew}-consul-session | acquire-leader | init | checkin | renew-certs | clean-certs | generate-challenge-token ]" exit 1 diff --git a/bin/reload.sh b/bin/reload.sh index abeb12e..2b033c9 100755 --- a/bin/reload.sh +++ b/bin/reload.sh @@ -18,7 +18,7 @@ preStart() { # then gracefully reload Nginx onChange() { local TEMPLATE="nginx.conf.ctmpl" - if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem -a "$(cat ${CERT_DIR}/fullchain.pem)" != "" -a "$(cat ${CERT_DIR}/privkey.pem)" != "" ]; then + if [ -f ${CERT_DIR}/fullchain.pem -a -f ${CERT_DIR}/privkey.pem ]; then TEMPLATE="nginx-ssl.conf.ctmpl" fi consul-template \ diff --git a/etc/acme/watch.hcl b/etc/acme/watch.hcl index 0bbf9d9..7b09653 100644 --- a/etc/acme/watch.hcl +++ b/etc/acme/watch.hcl @@ -1,21 +1,21 @@ log_level = "err" template { source = "/etc/acme/templates/cert.ctmpl" - destination = "/var/www/ssl/cert.pem" + destination = "/var/www/acme/ssl/cert.pem" } template { source = "/etc/acme/templates/privkey.ctmpl" - destination = "/var/www/ssl/privkey.pem" - command = "/usr/local/bin/reload.sh" + destination = "/var/www/acme/ssl/privkey.pem" + command = "/usr/local/bin/acme update-keys" } template { source = "/etc/acme/templates/fullchain.ctmpl" - destination = "/var/www/ssl/fullchain.pem" - command = "/usr/local/bin/reload.sh" + destination = "/var/www/acme/ssl/fullchain.pem" + command = "/usr/local/bin/acme update-keys" } template { source = "/etc/acme/templates/chain.ctmpl" - destination = "/var/www/ssl/chain.pem" + destination = "/var/www/acme/ssl/chain.pem" } template { source = "/etc/acme/templates/challenge-token.ctmpl"