Fix issue with automatically renewing acme certificates

CHANGELOG.rst

@@ -6,6 +6,7 @@ Changelog
 ~~~~~~~~~~~~~~
 
 This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
+The linux_dst plugin will also be deprecated and removed.
 
 The pending_dns_authorizations and dns_providers tables were created. New columns
 were added to the certificates and pending_certificates tables, (For the DNS provider ID), and authorities (For options).

lemur/certificates/cli.py

@@ -154,6 +154,7 @@ def request_reissue(certificate, commit):
 
     except Exception as e:
         sentry.captureException()
+        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
         print(
             "[!] Failed to reissue certificates. Reason: {}".format(
                 e
@@ -245,6 +246,7 @@ def reissue(old_certificate_name, commit):
         print("[+] Done!")
     except Exception as e:
         sentry.captureException()
+        current_app.logger.exception("Error reissuing certificate.", exc_info=True)
         print(
             "[!] Failed to reissue certificates. Reason: {}".format(
                 e

lemur/certificates/models.py

@@ -178,6 +178,8 @@ class Certificate(db.Model):
         self.signing_algorithm = defaults.signing_algorithm(cert)
         self.bits = defaults.bitstrength(cert)
         self.external_id = kwargs.get('external_id')
+        self.authority_id = kwargs.get('authority_id')
+        self.dns_provider_id = kwargs.get('dns_provider_id')
 
         for domain in defaults.domains(cert):
             self.domains.append(Domain(name=domain))

lemur/certificates/schemas.py

@@ -70,6 +70,7 @@ class CertificateInputSchema(CertificateCreationSchema):
     replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
     dns_provider = fields.Nested(DnsProviderSchema, missing={}, required=False, allow_none=True)
+    dns_provider_id = fields.Integer(required=False, allow_none=True)
 
     csr = fields.String(validate=validators.csr)
 
@@ -186,6 +187,7 @@ class CertificateOutputSchema(LemurOutputSchema):
     description = fields.String()
     issuer = fields.String()
     name = fields.String()
+    dns_provider_id = fields.Integer(required=False, allow_none=True)
 
     rotation = fields.Boolean()
 

lemur/common/defaults.py

@@ -178,7 +178,7 @@ def serial(cert):
     :param cert:
     :return: serial number
     """
-    return cert.serial
+    return cert.serial_number
 
 
 def san(cert):

lemur/pending_certificates/cli.py

@@ -6,6 +6,7 @@
 """
 from flask_script import Manager
 
+from lemur.authorities.service import get as get_authority
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.users import service as user_service
@@ -56,14 +57,27 @@ def fetch(ids):
 @manager.command
 def fetch_all_acme():
     """
-    Attempt to get full certificates for each pending certificate listed for ACME.
+    Attempt to get full certificates for each pending certificate listed with the acme-issuer. This is more efficient
+    for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any
+    certificates.
     """
     pending_certs = pending_certificate_service.get_pending_certs('all')
     user = user_service.get_by_username('lemur')
     new = 0
     failed = 0
+    wrong_issuer = 0
+    acme_certs = []
+
+    # We only care about certs using the acme-issuer plugin
+    for cert in pending_certs:
+        cert_authority = get_authority(cert.authority_id)
+        if cert_authority.plugin_name == 'acme-issuer':
+            acme_certs.append(cert)
+        else:
+            wrong_issuer += 1
+
     authority = plugins.get("acme-issuer")
-    resolved_certs = authority.get_ordered_certificates(pending_certs)
+    resolved_certs = authority.get_ordered_certificates(acme_certs)
 
     for cert in resolved_certs:
         real_cert = cert.get("cert")
@@ -81,8 +95,9 @@ def fetch_all_acme():
             pending_certificate_service.increment_attempt(pending_cert)
             failed += 1
     print(
-        "[+] Certificates: New: {new} Failed: {failed}".format(
+        "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
             new=new,
             failed=failed,
+            wrong_issuer=wrong_issuer
         )
     )

lemur/pending_certificates/models.py

@@ -97,6 +97,6 @@ class PendingCertificate(db.Model):
         self.rotation = kwargs.get('rotation')
         self.rotation_policy = kwargs.get('rotation_policy')
         try:
-            self.dns_provider_id = kwargs.get('dns_provider', {}).get("id")
-        except AttributeError:
-            self.dns_provider_id = None
+            self.dns_provider_id = kwargs.get('dns_provider')["id"]
+        except (AttributeError, KeyError, TypeError):
+            self.dns_provider_id = kwargs.get('dns_provider_id')

lemur/plugins/lemur_acme/dyn.py

@@ -2,6 +2,7 @@ import time
 
 import dns.exception
 import dns.resolver
+from dyn.tm.errors import DynectCreateError
 from dyn.tm.session import DynectSession
 from dyn.tm.zones import Node, Zone
 from flask import current_app
@@ -54,7 +55,11 @@ def create_txt_record(domain, token, account_number):
     node_name = '.'.join(domain.split('.')[:-zone_parts])
     fqdn = "{0}.{1}".format(node_name, zone_name)
     zone = Zone(zone_name)
-    zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
+    try:
+        zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
+    except DynectCreateError:
+        delete_txt_record(None, None, domain, token)
+        zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
     node = zone.get_node(node_name)
     zone.publish()
     current_app.logger.debug("TXT record created: {0}".format(fqdn))

lemur/plugins/lemur_acme/plugin.py

@@ -17,6 +17,7 @@ import OpenSSL.crypto
 import josepy as jose
 from acme import challenges, messages
 from acme.client import Client
+from acme.messages import Error as AcmeError
 from acme.errors import PollError
 from botocore.exceptions import ClientError
 from flask import current_app
@@ -273,7 +274,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     "authorizations": authorizations,
                     "pending_cert": pending_cert,
                 })
-            except (ClientError, ValueError):
+            except (ClientError, ValueError, Exception):
                 current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
                 certs.append({
                     "cert": False,
@@ -281,14 +282,13 @@ class ACMEIssuerPlugin(IssuerPlugin):
                 })
 
         for entry in pending:
-            entry["authorizations"] = finalize_authorizations(
-                entry["acme_client"],
-                entry["account_number"],
-                entry["dns_provider_type"],
-                entry["authorizations"]
-            )
-
             try:
+                entry["authorizations"] = finalize_authorizations(
+                    entry["acme_client"],
+                    entry["account_number"],
+                    entry["dns_provider_type"],
+                    entry["authorizations"]
+                )
                 pem_certificate, pem_certificate_chain = request_certificate(
                     entry["acme_client"],
                     entry["authorizations"],
@@ -304,7 +304,7 @@ class ACMEIssuerPlugin(IssuerPlugin):
                     "cert": cert,
                     "pending_cert": entry["pending_cert"],
                 })
-            except PollError:
+            except (PollError, AcmeError):
                 current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
                 certs.append({
                     "cert": False,
@@ -325,8 +325,12 @@ class ACMEIssuerPlugin(IssuerPlugin):
         acme_client, registration = setup_acme_client(authority)
         dns_provider_d = issuer_options.get('dns_provider')
         if not dns_provider_d:
-            raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
-        dns_provider = dns_provider_service.get(dns_provider_d.get("id"))
+            try:
+                dns_provider = dns_provider_service.get(issuer_options['dns_provider_id'])
+            except KeyError:
+                raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
+        else:
+            dns_provider = dns_provider_service.get(dns_provider_d.get("id"))
         credentials = json.loads(dns_provider.credentials)
 
         current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type))

lemur/tests/test_certificates.py

@@ -63,7 +63,7 @@ def test_get_certificate_primitives(certificate):
 
     with freeze_time(datetime.date(year=2016, month=10, day=30)):
         primitives = get_certificate_primitives(certificate)
-        assert len(primitives) == 24
+        assert len(primitives) == 25
 
 
 def test_certificate_edit_schema(session):