Merge branch 'main' into fix_vault_plugin

CHANGELOG.rst

@@ -8,6 +8,8 @@ Added ability for new versions of LEMUR_TOKEN_SECRET via the LEMUR_TOKEN_SECRETS
 migration and rotation of the secret.
 Added ENTRUST_INFER_EKU config property which attempts to computes the appropriate EKU value from the csr (default False).
 Added DIGICERT_CIS_USE_CSR_FIELDS to control the `use_csr_fields` create certificate API field (default False).
+Added AWS ACM source plugin. This plugin retreives all certificates for an account and a region.
+Added AWS ACM destination plugin. This plugin uploads a certificate to AWS ACM.
 
 
 

lemur/plugins/lemur_aws/acm.py

@@ -0,0 +1,181 @@
+"""
+.. module: lemur.plugins.lemur_aws.acm
+    :platform: Unix
+    :synopsis: Contains helper functions for interactive with AWS ACM Apis.
+    :copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Pinmarva <pinmarva@gmail.com>
+"""
+import botocore
+
+from retrying import retry
+from sentry_sdk import capture_exception
+
+from lemur.extensions import metrics
+from lemur.plugins.lemur_aws.sts import sts_client
+
+
+def retry_throttled(exception):
+    """
+    Determines if this exception is due to throttling
+    :param exception:
+    :return:
+    """
+    if isinstance(exception, botocore.exceptions.ClientError):
+        if exception.response["Error"]["Code"] == "NoSuchEntity":
+            return False
+
+        # No need to retry deletion requests if there is a DeleteConflict error.
+        # This error indicates that the certificate is still attached to an entity
+        # and cannot be deleted.
+        if exception.response["Error"]["Code"] == "DeleteConflict":
+            return False
+
+    metrics.send("acm_retry", "counter", 1, metric_tags={"exception": str(exception)})
+    return True
+
+
+def get_id_from_arn(arn):
+    """
+    Extract the certificate name from an arn.
+
+    examples:
+    'arn:aws:acm:us-west-2:123456789012:certificate/1aa111a1-1a11-1111-11aa-a11aa111aa11' '1aa111a1-1a11-1111-11aa-a11aa111aa11'
+
+    :param arn: ACM TLS certificate arn
+    :return: id of the certificate as uploaded to AWS
+    """
+    return arn.split("/")[-1]
+
+
+@sts_client("acm")
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=25)
+def upload_cert(name, body, private_key, cert_chain=None, **kwargs):
+    """
+    Upload a certificate to ACM AWS
+
+    :param body:
+    :param private_key:
+    :param cert_chain:
+    :param path:
+    :return:
+    """
+    assert isinstance(private_key, str)
+    client = kwargs.pop("client")
+
+    metrics.send("upload_acm_cert", "counter", 1, metric_tags={"name": name})
+    try:
+        if cert_chain:
+            return client.import_certificate(
+                Certificate=str(body),
+                PrivateKey=str(private_key),
+                CertificateChain=str(cert_chain),
+            )
+        else:
+            return client.import_certificate(
+                Certificate=str(body),
+                PrivateKey=str(private_key),
+            )
+    except botocore.exceptions.ClientError as e:
+        if e.response["Error"]["Code"] != "EntityAlreadyExists":
+            raise e
+
+
+@sts_client("acm")
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=25)
+def delete_cert(cert_arn, **kwargs):
+    """
+    Delete a certificate from ACM AWS
+
+    :param cert_arn:
+    :return:
+    """
+    client = kwargs.pop("client")
+    metrics.send("delete_acm_cert", "counter", 1, metric_tags={"cert_arn": cert_arn})
+    try:
+        client.delete_certificate(CertificateArn=cert_arn)
+    except botocore.exceptions.ClientError as e:
+        if e.response["Error"]["Code"] != "NoSuchEntity":
+            raise e
+
+
+@sts_client("acm")
+def get_certificate(name, **kwargs):
+    """
+    Retrieves an acm SSL certificate.
+
+    :return:
+    """
+    return _get_certificate(name, **kwargs)
+
+
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=25)
+def _get_certificate(arn, **kwargs):
+    metrics.send("get_acm_certificate", "counter", 1, metric_tags={"arn": arn})
+    client = kwargs.pop("client")
+    try:
+        return client.get_certificate(CertificateArn=arn)
+    except client.exceptions.NoSuchEntityException:
+        capture_exception()
+        return None
+
+
+@sts_client("acm")
+def get_certificates(**kwargs):
+    """
+    Fetches one page of acm certificate objects for a given account.
+    :param kwargs:
+    :return:
+    """
+    return _get_certificates(**kwargs)
+
+
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=25)
+def _get_certificates(**kwargs):
+    metrics.send("get_acm_certificates", "counter", 1)
+    return kwargs.pop("client").list_certificates(
+        **kwargs,
+        CertificateStatuses=[
+            'ISSUED'
+        ]
+    )
+
+
+@sts_client("acm")
+def get_all_certificates(**kwargs):
+    """
+    Use STS to fetch all of the ACM SSL certificates from a given account
+    :param restrict_path: If provided, only return certificates with a matching Path value.
+    """
+    certificates = []
+    account_number = kwargs.get("account_number")
+    metrics.send(
+        "get_all_acm_certificates",
+        "counter",
+        1,
+        metric_tags={"account_number": account_number},
+    )
+
+    while True:
+        response = _get_certificates(**kwargs)
+        metadata = response["CertificateSummaryList"]
+
+        for m in metadata:
+            certificate = _get_certificate(
+                m["CertificateArn"],
+                client=kwargs["client"]
+            )
+
+            if certificate is None:
+                continue
+
+            certificate.update(
+                name=m["DomainName"],
+                external_id=m["CertificateArn"]
+            )
+            certificates.append(certificate)
+
+        if not response.get("Marker"):
+            return certificates
+        else:
+            kwargs.update(dict(Marker=response["Marker"]))

lemur/plugins/lemur_aws/plugin.py

@@ -42,7 +42,7 @@
 from lemur.extensions import metrics
 from lemur.plugins import lemur_aws as aws, ExpirationNotificationPlugin
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
-from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns, cloudfront
+from lemur.plugins.lemur_aws import iam, s3, elb, ec2, sns, cloudfront, acm
 
 
 def get_region_from_dns(dns):
@@ -750,3 +750,85 @@ def send(self, notification_type, message, excluded_targets, options, **kwargs):
 
         current_app.logger.info(f"Publishing {notification_type} notification to topic {topic_arn}")
         sns.publish(topic_arn, message, notification_type, options, region_name=self.get_option("region", options))
+
+
+class AWSACMSourcePlugin(SourcePlugin):
+    title = "AWS-ACM"
+    slug = "aws-acm-source"
+    description = "Discovers all ACM TLS certificates in an AWS account"
+    version = aws.VERSION
+
+    author_url = "https://github.com/netflix/lemur"
+
+    options = [
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "required": True,
+            "validation": check_validation("^[0-9]{12,12}$"),
+            "helpMessage": "Must be a valid AWS account number!",
+        },
+        {
+            "name": "regions",
+            "type": "str",
+            "helpMessage": "Comma separated list of regions to search in, if no region is specified we look in all regions.",
+        },
+    ]
+
+    def get_certificates(self, options, **kwargs):
+        cert_data = acm.get_all_certificates(
+            account_number=self.get_option("accountNumber", options)
+        )
+
+        return [
+            dict(
+                body=c["Certificate"],
+                chain=c.get("CertificateChain"),
+                name=c["name"],
+                external_id=c["external_id"],
+            )
+            for c in cert_data
+        ]
+
+
+class ACMDestinationPlugin(DestinationPlugin):
+    title = "AWS-ACM"
+    slug = "aws-acm-dest"
+    description = "Allow the uploading of certificates to Amazon ACM"
+    version = aws.VERSION
+
+    author_url = "https://github.com/Netflix/lemur"
+
+    options = [
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "required": True,
+            "validation": check_validation("[0-9]{12}"),
+            "helpMessage": "A valid AWS account number with permission to access ACM",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+    ]
+
+    def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        try:
+            acm.upload_cert(
+                name,
+                body,
+                private_key,
+                cert_chain=cert_chain,
+                account_number=self.get_option("accountNumber", options),
+            )
+        except ClientError:
+            capture_exception()
+
+    def clean(self, certificate, options, **kwargs):
+        account_number = self.get_option("accountNumber", options)
+        acm.delete_cert(certificate["external_id"], account_number=account_number)

lemur/plugins/lemur_aws/tests/test_acm.py

@@ -0,0 +1,54 @@
+from moto import mock_acm, mock_sts
+from lemur.common.utils import check_validation
+from lemur.tests.vectors import ROOTCA_CERT_STR, INTERMEDIATE_CERT_STR, SAN_CERT_STR, SAN_CERT_KEY
+
+
+def test_acm_source_certificates(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get("aws-acm-source")
+    assert p
+
+
+def test_acm_dest_certificates(app):
+    from lemur.plugins.base import plugins
+
+    p = plugins.get("aws-acm-dest")
+    assert p
+
+
+@mock_sts()
+@mock_acm()
+def test_acm_plugin(app):
+    from lemur.plugins.base import plugins
+
+    cert_name = "testCert"
+    options = [
+        {
+            "name": "accountNumber",
+            "type": "str",
+            "required": True,
+            "validation": check_validation("[0-9]{12}"),
+            "helpMessage": "A valid AWS account number with permission to access ACM",
+        },
+        {
+            "name": "region",
+            "type": "str",
+            "default": "us-east-1",
+            "required": False,
+            "helpMessage": "Region bucket exists",
+            "available": ["us-east-1", "us-west-2", "eu-west-1"],
+        },
+    ]
+
+    dp = plugins.get("aws-acm-dest")
+    sp = plugins.get("aws-acm-source")
+
+    chain = ROOTCA_CERT_STR + INTERMEDIATE_CERT_STR
+    dp.upload(cert_name, SAN_CERT_STR, SAN_CERT_KEY, chain, options)
+
+    certs = sp.get_certificates(options)
+    assert len(certs) == 1
+
+    dp.clean(certs[0], options)
+    assert len(sp.get_certificates(options)) == 0

requirements-dev.txt

@@ -621,7 +621,7 @@ pyyaml==6.0.1
     #   responses
 readme-renderer==42.0
     # via twine
-redis==4.6.0
+redis==5.0.1
     # via
     #   -r requirements-tests.txt
     #   celery

requirements-docs.txt

@@ -643,7 +643,7 @@ pyyaml==6.0.1
     #   jsonschema-path
     #   moto
     #   responses
-redis==4.6.0
+redis==5.0.1
     # via
     #   -r requirements-docs.in
     #   -r requirements-tests.txt

requirements-tests.txt

@@ -506,7 +506,7 @@ pyyaml==6.0.1
     #   jsonschema-path
     #   moto
     #   responses
-redis==4.6.0
+redis==5.0.1
     # via
     #   -r requirements.txt
     #   celery

requirements.txt

@@ -45,9 +45,7 @@ botocore==1.31.84
     #   boto3
     #   s3transfer
 celery[redis]==5.3.5
-    # via
-    #   -r requirements.in
-    #   celery
+    # via -r requirements.in
 certbot==2.7.2
     # via -r requirements.in
 certifi==2023.7.22
@@ -265,7 +263,7 @@ pyyaml==6.0.1
     # via
     #   -r requirements.in
     #   cloudflare
-redis==4.6.0
+redis==5.0.1
     # via
     #   -r requirements.in
     #   celery

setup.py

@@ -135,7 +135,9 @@ def run(self):
             'acme_issuer = lemur.plugins.lemur_acme.plugin:ACMEIssuerPlugin',
             'acme_http_issuer = lemur.plugins.lemur_acme.plugin:ACMEHttpIssuerPlugin',
             'aws_destination = lemur.plugins.lemur_aws.plugin:AWSDestinationPlugin',
+            'aws_acm_destination = lemur.plugins.lemur_aws.plugin:ACMDestinationPlugin',
             'aws_source = lemur.plugins.lemur_aws.plugin:AWSSourcePlugin',
+            'aws_acm_source = lemur.plugins.lemur_aws.plugin:AWSACMSourcePlugin',
             'aws_s3 = lemur.plugins.lemur_aws.plugin:S3DestinationPlugin',
             'aws_sns = lemur.plugins.lemur_aws.plugin:SNSNotificationPlugin',
             'email_notification = lemur.plugins.lemur_email.plugin:EmailNotificationPlugin',