Merge branch 'master' into entrust_source

Makefile

@@ -115,10 +115,10 @@ endif
 	@echo "--> Updating Python requirements"
 	pip install --upgrade pip
 	pip install --upgrade pip-tools
-	pip-compile --output-file requirements.txt requirements.in -U --no-index
+	pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
-	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
+	pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
-	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
+	pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
-	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
+	pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
 	@echo "--> Done updating Python requirements"
 	@echo "--> Removing python-ldap from requirements-docs.txt"
 	grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

lemur/certificates/service.py

@@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput
 from lemur.common.utils import generate_private_key, truthiness
 from lemur.destinations.models import Destination
 from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
 from lemur.extensions import metrics, sentry, signals
 from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
@@ -797,3 +798,61 @@ def reissue_certificate(certificate, replace=None, user=None):
     new_cert = create(**primitives)
 
     return new_cert
+
+
+def is_attached_to_endpoint(certificate_name, endpoint_name):
+    """
+    Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
+    This method talks to elb and finds the real time information.
+    :param certificate_name:
+    :param endpoint_name:
+    :return: True if certificate is attached to the given endpoint, False otherwise
+    """
+    endpoint = endpoint_service.get_by_name(endpoint_name)
+    attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
+    return certificate_name in attached_certificates
+
+
+def remove_from_destination(certificate, destination):
+    """
+    Remove the certificate from given destination if clean() is implemented
+    :param certificate:
+    :param destination:
+    :return:
+    """
+    plugin = plugins.get(destination.plugin_name)
+    if not hasattr(plugin, "clean"):
+        info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
+        current_app.logger.warning(info_text)
+    else:
+        plugin.clean(certificate=certificate, options=destination.options)
+
+
+def cleanup_after_revoke(certificate):
+    """
+    Perform the needed cleanup for a revoked certificate. This includes -
+    1. Disabling notification
+    2. Disabling auto-rotation
+    3. Update certificate status to 'revoked'
+    4. Remove from AWS
+    :param certificate: Certificate object to modify and update in DB
+    :return: None
+    """
+    certificate.notify = False
+    certificate.rotation = False
+    certificate.status = 'revoked'
+
+    error_message = ""
+
+    for destination in list(certificate.destinations):
+        try:
+            remove_from_destination(certificate, destination)
+            certificate.destinations.remove(destination)
+        except Exception as e:
+            # This cleanup is the best-effort since certificate is already revoked at this point.
+            # We will capture the exception and move on to the next destination
+            sentry.captureException()
+            error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+    database.update(certificate)
+    return error_message

lemur/certificates/views.py

@@ -19,6 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
 
 from lemur.certificates import service
 from lemur.certificates.models import Certificate
+from lemur.extensions import sentry
 from lemur.plugins.base import plugins
 from lemur.certificates.schemas import (
     certificate_input_schema,
@@ -888,8 +889,24 @@ class Certificates(AuthenticatedResource):
         if cert.owner != data["owner"]:
             service.cleanup_owner_roles_notification(cert.owner, data)
 
+        error_message = ""
+        # if destination is removed, cleanup the certificate from AWS
+        for destination in cert.destinations:
+            if destination not in data["destinations"]:
+                try:
+                    service.remove_from_destination(cert, destination)
+                except Exception as e:
+                    sentry.captureException()
+                    # Add the removed destination back
+                    data["destinations"].append(destination)
+                    error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
+
+        # go ahead with DB update
         cert = service.update(certificate_id, **data)
         log_service.create(g.current_user, "update_cert", certificate=cert)
+
+        if error_message:
+            return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
         return cert
 
     @validate_schema(certificate_edit_input_schema, certificate_output_schema)
@@ -1433,16 +1450,24 @@ class CertificateRevoke(AuthenticatedResource):
             return dict(message="Cannot revoke certificate. No external id found."), 400
 
         if cert.endpoints:
-            return (
+            for endpoint in cert.endpoints:
-                dict(
+                if service.is_attached_to_endpoint(cert.name, endpoint.name):
-                    message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
+                    return (
-                ),
+                        dict(
-                403,
+                            message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
-            )
+                        ),
+                        403,
+                    )
 
         plugin = plugins.get(cert.authority.plugin_name)
         plugin.revoke_certificate(cert, data)
+
         log_service.create(g.current_user, "revoke_cert", certificate=cert)
+
+        # Perform cleanup after revoke
+        error_message = service.cleanup_after_revoke(cert)
+        if error_message:
+            return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
         return dict(id=cert.id)
 
 

lemur/plugins/lemur_aws/elb.py

@@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
         raise
 
 
+@sts_client("elbv2")
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5)
+def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs):
+    """
+    Get a load balancer ARN from an endpoint.
+    :param endpoint_name:
+    :return:
+    """
+    try:
+        client = kwargs.pop("client")
+        elbs = client.describe_load_balancers(Names=[endpoint_name])
+        if "LoadBalancers" in elbs and elbs["LoadBalancers"]:
+            return elbs["LoadBalancers"][0]["LoadBalancerArn"]
+
+    except Exception as e:  # noqa
+        metrics.send(
+            "get_load_balancer_arn_from_endpoint",
+            "counter",
+            1,
+            metric_tags={
+                "error": str(e),
+                "endpoint_name": endpoint_name,
+            },
+        )
+        sentry.captureException(
+            extra={
+                "endpoint_name": str(endpoint_name),
+            }
+        )
+        raise
+
+
 @sts_client("elb")
 @retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
 def get_elbs(**kwargs):

lemur/plugins/lemur_aws/plugin.py

@@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin):
             )
         return None
 
+    def get_endpoint_certificate_names(self, endpoint):
+        options = endpoint.source.options
+        account_number = self.get_option("accountNumber", options)
+        region = get_region_from_dns(endpoint.dnsname)
+        certificate_names = []
+
+        if endpoint.type == "elb":
+            elb_details = elb.get_elbs(account_number=account_number,
+                                    region=region,
+                                    LoadBalancerNames=[endpoint.name],)
+
+            for lb_description in elb_details["LoadBalancerDescriptions"]:
+                for listener_description in lb_description["ListenerDescriptions"]:
+                    listener = listener_description.get("Listener")
+                    if not listener.get("SSLCertificateId"):
+                        continue
+
+                    certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId")))
+        elif endpoint.type == "elbv2":
+            listeners = elb.describe_listeners_v2(
+                account_number=account_number,
+                region=region,
+                LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name,
+                                                                        account_number=account_number,
+                                                                        region=region),
+            )
+            for listener in listeners["Listeners"]:
+                if not listener.get("Certificates"):
+                    continue
+
+                for certificate in listener["Certificates"]:
+                    certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"]))
+
+        return certificate_names
+
 
 class AWSDestinationPlugin(DestinationPlugin):
     title = "AWS"
@@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin):
     def deploy(self, elb_name, account, region, certificate):
         pass
 
+    def clean(self, certificate, options, **kwargs):
+        account_number = self.get_option("accountNumber", options)
+        iam.delete_cert(certificate.name, account_number=account_number)
+
 
 class S3DestinationPlugin(ExportDestinationPlugin):
     title = "AWS-S3"

lemur/plugins/lemur_aws/tests/test_elb.py

@@ -1,5 +1,5 @@
 import boto3
-from moto import mock_sts, mock_elb
+from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam
 
 
 @mock_sts()
@@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials):
 
     elbs = get_all_elbs(account_number="123456789012", region="us-east-1")
     assert elbs
+
+
+@mock_sts()
+@mock_ec2
+@mock_elbv2()
+@mock_iam
+def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials):
+    from lemur.plugins.lemur_aws import iam, elb
+    endpoint_name = "example-lbv2"
+    account_number = "123456789012"
+    region_ue1 = "us-east-1"
+
+    client = boto3.client("elbv2", region_name="us-east-1")
+    ec2 = boto3.resource("ec2", region_name="us-east-1")
+
+    # Create VPC
+    vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24")
+
+    # Create LB (elbv2) in above VPC
+    assert create_load_balancer(client, ec2, vpc.id, endpoint_name)
+    # Create target group
+    target_group_arn = create_target_group(client, vpc.id)
+    assert target_group_arn
+
+    # Test get_load_balancer_arn_from_endpoint
+    lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name,
+                                                     account_number=account_number,
+                                                     region=region_ue1)
+    assert lb_arn
+
+    # Test describe_listeners_v2
+    listeners = elb.describe_listeners_v2(account_number=account_number,
+                                          region=region_ue1,
+                                          LoadBalancerArn=lb_arn)
+    assert listeners
+    assert not listeners["Listeners"]
+
+    # Upload cert
+    response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2",
+                               account_number=account_number)
+    assert response
+    cert_arn = response["ServerCertificateMetadata"]["Arn"]
+    assert cert_arn
+
+    # Create https listener using above cert
+    listeners = client.create_listener(
+        LoadBalancerArn=lb_arn,
+        Protocol="HTTPS",
+        Port=443,
+        Certificates=[{"CertificateArn": cert_arn}],
+        DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+    )
+    assert listeners
+    listener_arn = listeners["Listeners"][0]["ListenerArn"]
+    assert listener_arn
+
+    assert listeners["Listeners"]
+    for listener in listeners["Listeners"]:
+        if listener["Port"] == 443:
+            assert listener["Certificates"]
+            assert cert_arn == listener["Certificates"][0]["CertificateArn"]
+
+    # Test get_listener_arn_from_endpoint
+    assert listener_arn == elb.get_listener_arn_from_endpoint(
+        endpoint_name,
+        443,
+        account_number=account_number,
+        region=region_ue1,
+    )
+
+
+@mock_sts()
+@mock_elb()
+def test_get_all_elbs_v2():
+    from lemur.plugins.lemur_aws.elb import get_all_elbs_v2
+
+    elbs = get_all_elbs_v2(account_number="123456789012",
+                           region="us-east-1")
+    assert elbs
+
+
+def create_load_balancer(client, ec2, vpc_id, endpoint_name):
+    subnet1 = ec2.create_subnet(
+        VpcId=vpc_id,
+        CidrBlock="172.28.7.192/26",
+        AvailabilityZone="us-east-1a"
+    )
+
+    return client.create_load_balancer(
+        Name=endpoint_name,
+        Subnets=[
+            subnet1.id,
+        ],
+    )
+
+
+def create_target_group(client, vpc_id):
+    response = client.create_target_group(
+        Name="a-target",
+        Protocol="HTTPS",
+        Port=443,
+        VpcId=vpc_id,
+    )
+    return response.get("TargetGroups")[0]["TargetGroupArn"]

lemur/static/app/angular/certificates/certificate/options.tpl.html

@@ -33,7 +33,7 @@
         <div class="col-sm-10">
           <select class="form-control" ng-model="certificate.keyType"
                   ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']"
-                  ng-init="certificate.keyType = 'RSA2048'"></select>
+                  ng-init="certificate.keyType = 'ECCPRIME256V1'"></select>
         </div>
       </div>
       <div class="form-group">

lemur/static/app/angular/certificates/certificate/revoke.tpl.html

@@ -26,9 +26,8 @@
     </div>
   </form>
   <div ng-if="certificate.endpoints.length">
-    <p><strong>Certificate cannot be revoked, it is associated with the following endpoints. Disassociate this
+    <p><strong>Certificate might be associated with the following endpoints. Disassociate this
-      certificate
+      certificate before revoking or continue if you've already done so.</strong></p>
-      before revoking.</strong></p>
     <ul class="list-group">
       <li class="list-group-item" ng-repeat="endpoint in certificate.endpoints">
         <span class="pull-right"><label class="label label-default">{{ endpoint.type }}</label></span>

requirements-dev.txt

@@ -24,7 +24,7 @@ keyring==21.2.0           # via twine
 mccabe==0.6.1             # via flake8
 nodeenv==1.5.0            # via -r requirements-dev.in, pre-commit
 pkginfo==1.5.0.1          # via twine
-pre-commit==2.9.0         # via -r requirements-dev.in
+pre-commit==2.9.2         # via -r requirements-dev.in
 pycodestyle==2.6.0        # via flake8
 pycparser==2.20           # via cffi
 pyflakes==2.2.0           # via flake8

requirements-docs.txt

@@ -17,8 +17,8 @@ bcrypt==3.1.7             # via -r requirements.txt, flask-bcrypt, paramiko
 beautifulsoup4==4.9.1     # via -r requirements.txt, cloudflare
 billiard==3.6.3.0         # via -r requirements.txt, celery
 blinker==1.4              # via -r requirements.txt, flask-mail, flask-principal, raven
-boto3==1.16.24            # via -r requirements.txt
+boto3==1.16.25            # via -r requirements.txt
-botocore==1.19.24         # via -r requirements.txt, boto3, s3transfer
+botocore==1.19.25         # via -r requirements.txt, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.txt
 certifi==2020.11.8        # via -r requirements.txt, requests
 certsrv==2.1.1            # via -r requirements.txt
@@ -73,7 +73,7 @@ pygments==2.6.1           # via sphinx
 pyjks==20.0.0             # via -r requirements.txt
 pyjwt==1.7.1              # via -r requirements.txt
 pynacl==1.3.0             # via -r requirements.txt, paramiko
-pyopenssl==19.1.0         # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
+pyopenssl==20.0.0         # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
 pyparsing==2.4.7          # via packaging
 pyrfc3339==1.1            # via -r requirements.txt, acme
 python-dateutil==2.8.1    # via -r requirements.txt, alembic, arrow, botocore

requirements-tests.txt

@@ -10,9 +10,9 @@ aws-sam-translator==1.22.0  # via cfn-lint
 aws-xray-sdk==2.5.0       # via moto
 bandit==1.6.2             # via -r requirements-tests.in
 black==20.8b1             # via -r requirements-tests.in
-boto3==1.16.24            # via aws-sam-translator, moto
+boto3==1.16.25            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.19.24         # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.19.25         # via aws-xray-sdk, boto3, moto, s3transfer
 certifi==2020.11.8        # via requests
 cffi==1.14.0              # via cryptography
 cfn-lint==0.29.5          # via moto
@@ -25,7 +25,7 @@ docker==4.2.0             # via moto
 ecdsa==0.14.1             # via moto, python-jose, sshpubkeys
 factory-boy==3.1.0        # via -r requirements-tests.in
 faker==4.17.1             # via -r requirements-tests.in, factory-boy
-fakeredis==1.4.4          # via -r requirements-tests.in
+fakeredis==1.4.5          # via -r requirements-tests.in
 flask==1.1.2              # via pytest-flask
 freezegun==1.0.0          # via -r requirements-tests.in
 future==0.18.2            # via aws-xray-sdk

requirements.txt

@@ -15,8 +15,8 @@ bcrypt==3.1.7             # via flask-bcrypt, paramiko
 beautifulsoup4==4.9.1     # via cloudflare
 billiard==3.6.3.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.16.24            # via -r requirements.in
+boto3==1.16.25            # via -r requirements.in
-botocore==1.19.24         # via -r requirements.in, boto3, s3transfer
+botocore==1.19.25         # via -r requirements.in, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.in
 certifi==2020.11.8        # via -r requirements.in, requests
 certsrv==2.1.1            # via -r requirements.in
@@ -67,7 +67,7 @@ pycryptodomex==3.9.7      # via pyjks
 pyjks==20.0.0             # via -r requirements.in
 pyjwt==1.7.1              # via -r requirements.in
 pynacl==1.3.0             # via paramiko
-pyopenssl==19.1.0         # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
+pyopenssl==20.0.0         # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
 pyrfc3339==1.1            # via acme
 python-dateutil==2.8.1    # via alembic, arrow, botocore
 python-editor==1.0.4      # via alembic