Merge pull request #1487 from castrapel/cancel_pending_cert_failures

Cancel pending cert failures

lemur/notifications/messaging.py

@@ -24,6 +24,7 @@ from lemur.common.utils import windowed_query
 
 from lemur.certificates.schemas import certificate_notification_output_schema
 from lemur.certificates.models import Certificate
+from lemur.pending_certificates.schemas import pending_certificate_output_schema
 
 from lemur.plugins import plugins
 from lemur.plugins.utils import get_plugin_option
@@ -172,6 +173,44 @@ def send_rotation_notification(certificate, notification_plugin=None):
         return True
 
 
+def send_pending_failure_notification(pending_cert, notify_owner=True, notify_security=True, notification_plugin=None):
+    """
+    Sends a report to certificate owners when their pending certificate failed to be created.
+
+    :param pending_cert:
+    :param notification_plugin:
+    :return:
+    """
+    status = FAILURE_METRIC_STATUS
+
+    if not notification_plugin:
+        notification_plugin = plugins.get(
+            current_app.config.get('LEMUR_DEFAULT_NOTIFICATION_PLUGIN', 'email-notification')
+        )
+
+    data = pending_certificate_output_schema.dump(pending_cert).data
+    data["security_email"] = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
+
+    if notify_owner:
+        try:
+            notification_plugin.send('failed', data, [data['owner']], pending_cert)
+            status = SUCCESS_METRIC_STATUS
+        except Exception as e:
+            sentry.captureException()
+
+    if notify_security:
+        try:
+            notification_plugin.send('failed', data, data["security_email"], pending_cert)
+            status = SUCCESS_METRIC_STATUS
+        except Exception as e:
+            sentry.captureException()
+
+    metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': 'rotation'})
+
+    if status == SUCCESS_METRIC_STATUS:
+        return True
+
+
 def needs_notification(certificate):
     """
     Determine if notifications for a given certificate should

lemur/pending_certificates/cli.py

@@ -4,9 +4,15 @@
 .. moduleauthor:: James Chuong <jchuong@instartlogic.com>
 .. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 """
+
+import copy
+import sys
+
+from flask import current_app
 from flask_script import Manager
 
 from lemur.authorities.service import get as get_authority
+from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.users import service as user_service
@@ -56,6 +62,10 @@ def fetch_all_acme():
     for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any
     certificates.
     """
+
+    log_data = {
+        "function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
+    }
     pending_certs = pending_certificate_service.get_pending_certs('all')
     user = user_service.get_by_username('lemur')
     new = 0
@@ -88,7 +98,27 @@ def fetch_all_acme():
             new += 1
         else:
             pending_certificate_service.increment_attempt(pending_cert)
+            pending_certificate_service.update(
+                cert.get("pending_cert").id,
+                status=str(cert.get("last_error"))[0:128]
+            )
             failed += 1
+            error_log = copy.deepcopy(log_data)
+            error_log["message"] = "Pending certificate creation failure"
+            error_log["pending_cert_id"] = pending_cert.id
+            error_log["last_error"] = cert.get("last_error")
+            error_log["cn"] = pending_cert.cn
+
+            if pending_cert.number_attempts > 4:
+                error_log["message"] = "Deleting pending certificate"
+                send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
+                pending_certificate_service.delete_by_id(pending_cert.id)
+            current_app.logger.error(error_log)
+    log_data["message"] = "Complete"
+    log_data["new"] = new
+    log_data["failed"] = failed
+    log_data["wrong_issuer"] = wrong_issuer
+    current_app.logger.debug(log_data)
     print(
         "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
             new=new,

lemur/plugins/lemur_acme/plugin.py

@@ -300,11 +300,12 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     "order": order,
                     "dns_provider_options": dns_provider_options,
                 })
-            except (ClientError, ValueError, Exception):
+            except (ClientError, ValueError, Exception) as e:
                 current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
                 certs.append({
                     "cert": False,
                     "pending_cert": pending_cert,
+                    "last_error": e,
                 })
 
         for entry in pending:

lemur/plugins/lemur_email/templates/failed.html

@@ -0,0 +1,161 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+        "http://www.w3.org/TR/html4/loose.dtd">
+<html lang="en">
+<head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta name="viewport" content="initial-scale=1.0">    <!-- So that mobile webkit will display zoomed in -->
+    <meta name="format-detection" content="telephone=no"> <!-- disable auto telephone linking in iOS -->
+
+    <title>Lemur</title>
+</head>
+
+<div style="margin:0;padding:0" bgcolor="#FFFFFF">
+    <table width="100%" height="100%" style="min-width:348px" border="0" cellspacing="0" cellpadding="0">
+        <tbody>
+        <tr height="32px"></tr>
+        <tr align="center">
+            <td width="32px"></td>
+            <td>
+                <table border="0" cellspacing="0" cellpadding="0" style="max-width:600px">
+                    <tbody>
+                    <tr>
+                        <td>
+                            <table width="100%" border="0" cellspacing="0" cellpadding="0">
+                                <tbody>
+                                <tr>
+                                    <td align="left" style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:35px;color:#727272; line-height:1.5">
+                                        Lemur
+                                    </td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr height="16"></tr>
+                    <tr>
+                        <td>
+                            <table bgcolor="#F44336" width="100%" border="0" cellspacing="0" cellpadding="0"
+                                   style="min-width:332px;max-width:600px;border:1px solid #e0e0e0;border-bottom:0;border-top-left-radius:3px;border-top-right-radius:3px">
+                                <tbody>
+                                <tr>
+                                    <td height="72px" colspan="3"></td>
+                                </tr>
+                                <tr>
+                                    <td width="32px"></td>
+                                    <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:24px;color:#ffffff;line-height:1.25">
+                                       Your certificate request has failed!
+                                    </td>
+                                    <td width="32px"></td>
+                                </tr>
+                                <tr>
+                                    <td height="18px" colspan="3"></td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr>
+                        <td>
+                            <table width="100%" border="0" cellspacing="0" cellpadding="0"
+                                   style="min-width:332px;max-width:600px;border:1px solid #f0f0f0;border-bottom:1px solid #c0c0c0;border-top:0;border-bottom-left-radius:3px;border-bottom-right-radius:3px">
+                                <tbody>
+                                <tr height="16px">
+                                    <td width="32px" rowspan="3"></td>
+                                    <td></td>
+                                    <td width="32px" rowspan="3"></td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <table style="min-width:300px" border="0" cellspacing="0" cellpadding="0">
+                                            <tbody>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    Hi,
+                                                    <br>This is a Lemur certificate failure notice. We were unable to create or rotate your certificate. Please retry your request. The reason for the failure is listed below.
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    <table border="0" cellspacing="0" cellpadding="0"
+                                                           style="margin-top:48px;margin-bottom:48px">
+                                                        <tbody>
+                                                            <tr valign="middle">
+                                                                <td width="32px"></td>
+                                                                <td width="16px"></td>
+                                                                <td style="line-height:1.2">
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:20px;color:#202020">{{ message.certificates.name }}</span>
+                                                                    <br>
+                                                                    <span style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#727272">
+                                                                        <br>{{ message.certificates.owner }}
+                                                                        <br>{{ message.certificates.status }}
+                                                                    </span>
+                                                                </td>
+                                                            </tr>
+                                                        </tbody>
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    If you are having any trouble, please reach out to {{ ", ".join(message.certificates.security_email) }}.</span>
+                                                </td>
+                                            </tr>
+                                            <tr>
+                                            </tr>
+                                            <tr>
+                                                <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
+                                                    <br>Best,<br><span class="il">Lemur</span>
+                                                </td>
+                                            </tr>
+                                            <tr height="16px"></tr>
+                                            <tr>
+                                                <td>
+                                                    <table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:12px;color:#b9b9b9;line-height:1.5">
+                                                        <tbody>
+                                                        <tr>
+                                                            <td>*All times are in UTC<br></td>
+                                                        </tr>
+                                                        </tbody>
+                                                    </table>
+                                                </td>
+                                            </tr>
+                                            </tbody>
+                                        </table>
+                                    </td>
+                                </tr>
+                                <tr height="32px"></tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    <tr height="16"></tr>
+                    <tr>
+                        <td style="max-width:600px;font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#bcbcbc;line-height:1.5"></td>
+                    </tr>
+                    <tr>
+                        <td>
+                            <table style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:10px;color:#666666;line-height:18px;padding-bottom:10px">
+                                <tbody>
+                                <tr>
+                                    <td>You received this mandatory email announcement to update you about
+                                        important changes to your <span class="il">TLS certificate</span>.
+                                    </td>
+                                </tr>
+                                <tr>
+                                    <td>
+                                        <div style="direction:ltr;text-align:left">© 2016 <span class="il">Lemur</span></div>
+                                    </td>
+                                </tr>
+                                </tbody>
+                            </table>
+                        </td>
+                    </tr>
+                    </tbody>
+                </table>
+            </td>
+            <td width="32px"></td>
+        </tr>
+        <tr height="32px"></tr>
+        </tbody>
+    </table>
+</div>

lemur/static/app/angular/pending_certificates/view/view.tpl.html

@@ -80,6 +80,12 @@
                           {{ pendingCertificate.numberAttempts }}
                         </span>
                     </li>
+                  <li class="list-group-item">
+                      <strong>Latest Status</strong>
+                        <span class="pull-right">
+                          {{ pendingCertificate.status }}
+                        </span>
+                    </li>
                     <li class="list-group-item">
                       <strong>Date Created</strong>
                         <span class="pull-right">

requirements-dev.txt

@@ -10,7 +10,7 @@ certifi==2018.4.16        # via requests
 cfgv==1.1.0               # via pre-commit
 chardet==3.0.4            # via requests
 flake8==3.5.0
-identify==1.1.3           # via pre-commit
+identify==1.1.4           # via pre-commit
 idna==2.7                 # via requests
 invoke==1.1.0
 mccabe==0.6.1             # via flake8
@@ -24,7 +24,7 @@ requests-toolbelt==0.8.0  # via twine
 requests==2.19.1          # via requests-toolbelt, twine
 six==1.11.0               # via cfgv, pre-commit
 toml==0.9.4               # via pre-commit
-tqdm==4.23.4              # via twine
+tqdm==4.24.0              # via twine
 twine==1.11.0
 urllib3==1.23             # via requests
 virtualenv==16.0.0        # via pre-commit

requirements-docs.txt

@@ -15,8 +15,8 @@ asyncpool==1.0
 babel==2.6.0              # via sphinx
 bcrypt==3.1.4
 blinker==1.4
-boto3==1.7.61
+boto3==1.7.62
-botocore==1.10.61
+botocore==1.10.62
 certifi==2018.4.16
 cffi==1.11.5
 chardet==3.0.4
@@ -52,7 +52,7 @@ markupsafe==1.0
 marshmallow-sqlalchemy==0.14.0
 marshmallow==2.15.3
 mock==2.0.0
-ndg-httpsclient==0.5.0
+ndg-httpsclient==0.5.1
 packaging==17.1           # via sphinx
 paramiko==2.4.1
 pbr==4.1.1
@@ -78,7 +78,7 @@ retrying==1.3.3
 s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1    # via sphinx
-sphinx-rtd-theme==0.4.0
+sphinx-rtd-theme==0.4.1
 sphinx==1.7.6
 sphinxcontrib-httpdomain==1.7.0
 sphinxcontrib-websupport==1.1.0  # via sphinx

requirements-tests.txt

@@ -8,9 +8,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.1.5       # via pytest
 attrs==18.1.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.7.62             # via moto
+boto3==1.7.65             # via moto
 boto==2.49.0              # via moto
-botocore==1.10.62         # via boto3, moto, s3transfer
+botocore==1.10.65         # via boto3, moto, s3transfer
 certifi==2018.4.16        # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -36,7 +36,7 @@ mock==2.0.0               # via moto
 more-itertools==4.2.0     # via pytest
 moto==1.3.3
 nose==1.3.7
-pbr==4.1.1                # via mock
+pbr==4.2.0                # via mock
 pluggy==0.6.0             # via pytest
 py==1.5.4                 # via pytest
 pyaml==17.12.1            # via moto

requirements.txt

@@ -13,8 +13,8 @@ asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.4             # via flask-bcrypt, paramiko
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.7.62
+boto3==1.7.65
-botocore==1.10.62         # via boto3, s3transfer
+botocore==1.10.65         # via boto3, s3transfer
 certifi==2018.4.16
 cffi==1.11.5              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
@@ -51,11 +51,11 @@ marshmallow==2.15.3
 mock==2.0.0               # via acme
 ndg-httpsclient==0.5.1
 paramiko==2.4.1
-pbr==4.1.1                # via mock
+pbr==4.2.0                # via mock
 pem==18.1.0
 psycopg2==2.7.5
 pyasn1-modules==0.2.2     # via python-ldap
-pyasn1==0.4.3             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
+pyasn1==0.4.4             # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
 pycparser==2.18           # via cffi
 pyjwt==1.6.4
 pynacl==1.2.1             # via paramiko