Merge branch 'master' into feature/travis-python38

docs/administration.rst

@@ -362,6 +362,9 @@ Whenever a pending ACME certificate fails to be issued, Lemur will send a notifi
 and security team (as specified by the ``LEMUR_SECURITY_TEAM_EMAIL`` configuration parameter). This email is not sent if
 the pending certificate had notifications disabled.
 
+Lemur will attempt 3x times to resolve a pending certificate.
+This can at times result into 3 duplicate certificates, if all certificate attempts get resolved.
+
 **Certificate rotation**
 
 Whenever a cert is rotated, Lemur will send a notification via email to the certificate owner. This notification is
@@ -820,6 +823,20 @@ ACME Plugin
         Enables delegated DNS domain validation using CNAMES.  When enabled, Lemur will attempt to follow CNAME records to authoritative DNS servers when creating DNS-01 challenges.
 
 
+The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
+account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
+
+
+.. data:: ACME_PRIVATE_KEY
+    :noindex:
+
+            This is the private key, the account was registered with (in JWK format)
+
+.. data:: ACME_REGR
+    :noindex:
+
+            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
+
 Active Directory Certificate Services Plugin
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1336,23 +1353,6 @@ The following configuration properties are required to use the PowerDNS ACME Plu
 
             File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
 
-ACME Plugin
-~~~~~~~~~~~~
-
-The following configration properties are optional for the ACME plugin to use. They allow reusing an existing ACME
-account. See :ref:`Using a pre-existing ACME account <AcmeAccountReuse>` for more details.
-
-
-.. data:: ACME_PRIVATE_KEY
-    :noindex:
-
-            This is the private key, the account was registered with (in JWK format)
-
-.. data:: ACME_REGR
-    :noindex:
-
-            This is the registration for the ACME account, the most important part is the uri attribute (in JSON)
-
 .. _CommandLineInterface:
 
 Command Line Interface

lemur/certificates/cli.py

@@ -261,10 +261,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
                 log_data["endpoint"] = endpoint.dnsname
                 log_data["certificate"] = endpoint.certificate.replaced[0].name
-                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
                 print(
                     f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
                 )
+                request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
                 current_app.logger.info(log_data)
 
         status = SUCCESS_METRIC_STATUS
@@ -427,7 +427,7 @@ def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, mes
                     1,
                     metric_tags={
                         "status": FAILURE_METRIC_STATUS,
-                        "new_certificate_name": str(endpoint.certificate.replaced[0].name),
+                        "new_certificate_name": str(log_data["certificate"]),
                         "endpoint_name": str(endpoint.dnsname),
                         "message": str(message),
                         "region": str(region),

lemur/certificates/service.py

@@ -388,6 +388,7 @@ def create(**kwargs):
         cert = Certificate(**kwargs)
         kwargs["creator"].certificates.append(cert)
     else:
+        # ACME path
         cert = PendingCertificate(**kwargs)
         kwargs["creator"].pending_certificates.append(cert)
 

lemur/common/celery.py

@@ -20,6 +20,7 @@ from flask import current_app
 from lemur.authorities.service import get as get_authority
 from lemur.certificates import cli as cli_certificate
 from lemur.common.redis import RedisHandler
+from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
 from lemur.destinations import service as destinations_service
 from lemur.dns_providers import cli as cli_dns_providers
 from lemur.endpoints import cli as cli_endpoints
@@ -273,7 +274,8 @@ def fetch_acme_cert(id):
         real_cert = cert.get("cert")
         # It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
         pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
-        if not pending_cert:
+        if not pending_cert or pending_cert.resolved:
+            # pending_cert is cleared or it was resolved by another process
             log_data[
                 "message"
             ] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
@@ -301,7 +303,7 @@ def fetch_acme_cert(id):
             error_log["last_error"] = cert.get("last_error")
             error_log["cn"] = pending_cert.cn
 
-            if pending_cert.number_attempts > 4:
+            if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
                 error_log["message"] = "Deleting pending certificate"
                 send_pending_failure_notification(
                     pending_cert, notify_owner=pending_cert.notify

lemur/constants.py

@@ -12,6 +12,9 @@ NONSTANDARD_NAMING_TEMPLATE = "{issuer}-{not_before}-{not_after}"
 SUCCESS_METRIC_STATUS = "success"
 FAILURE_METRIC_STATUS = "failure"
 
+# when ACME attempts to resolve a certificate try in total 3 times
+ACME_ADDITIONAL_ATTEMPTS = 2
+
 CERTIFICATE_KEY_TYPES = [
     "RSA2048",
     "RSA4096",

lemur/pending_certificates/cli.py

@@ -12,10 +12,12 @@ from flask import current_app
 from flask_script import Manager
 
 from lemur.authorities.service import get as get_authority
+from lemur.constants import ACME_ADDITIONAL_ATTEMPTS
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 
+
 manager = Manager(usage="Handles pending certificate related tasks.")
 
 
@@ -107,7 +109,7 @@ def fetch_all_acme():
             error_log["last_error"] = cert.get("last_error")
             error_log["cn"] = pending_cert.cn
 
-            if pending_cert.number_attempts > 4:
+            if pending_cert.number_attempts > ACME_ADDITIONAL_ATTEMPTS:
                 error_log["message"] = "Marking pending certificate as resolved"
                 send_pending_failure_notification(
                     pending_cert, notify_owner=pending_cert.notify

requirements-dev.txt

@@ -48,7 +48,7 @@ nodeenv==1.5.0
     #   pre-commit
 pkginfo==1.5.0.1
     # via twine
-pre-commit==2.9.3
+pre-commit==2.10.0
     # via -r requirements-dev.in
 pycodestyle==2.6.0
     # via flake8

requirements-docs.txt

@@ -4,32 +4,63 @@
 #
 #    pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
 #
-alabaster==0.7.12         # via sphinx
+alabaster==0.7.12
-babel==2.8.0              # via sphinx
+    # via sphinx
-certifi==2020.12.5        # via requests
+babel==2.8.0
-chardet==3.0.4            # via requests
+    # via sphinx
-docutils==0.15.2          # via sphinx
+certifi==2020.12.5
-idna==2.9                 # via requests
+    # via requests
-imagesize==1.2.0          # via sphinx
+chardet==3.0.4
-jinja2==2.11.2            # via sphinx
+    # via requests
-markupsafe==1.1.1         # via jinja2
+docutils==0.15.2
-packaging==20.3           # via sphinx
+    # via sphinx
-pygments==2.6.1           # via sphinx
+idna==2.9
-pyparsing==2.4.7          # via packaging
+    # via requests
-pytz==2019.3              # via babel
+imagesize==1.2.0
-requests==2.25.1          # via sphinx
+    # via sphinx
-six==1.15.0               # via packaging, sphinxcontrib-httpdomain
+jinja2==2.11.3
-snowballstemmer==2.0.0    # via sphinx
+    # via sphinx
-sphinx-rtd-theme==0.5.1   # via -r requirements-docs.in
+markupsafe==1.1.1
-sphinx==3.4.3             # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
+    # via jinja2
-sphinxcontrib-applehelp==1.0.2  # via sphinx
+packaging==20.3
-sphinxcontrib-devhelp==1.0.2  # via sphinx
+    # via sphinx
-sphinxcontrib-htmlhelp==1.0.3  # via sphinx
+pygments==2.6.1
-sphinxcontrib-httpdomain==1.7.0  # via -r requirements-docs.in
+    # via sphinx
-sphinxcontrib-jsmath==1.0.1  # via sphinx
+pyparsing==2.4.7
-sphinxcontrib-qthelp==1.0.3  # via sphinx
+    # via packaging
-sphinxcontrib-serializinghtml==1.1.4  # via sphinx
+pytz==2019.3
-urllib3==1.25.8           # via requests
+    # via babel
+requests==2.25.1
+    # via sphinx
+six==1.15.0
+    # via
+    #   packaging
+    #   sphinxcontrib-httpdomain
+snowballstemmer==2.0.0
+    # via sphinx
+sphinx-rtd-theme==0.5.1
+    # via -r requirements-docs.in
+sphinx==3.4.3
+    # via
+    #   -r requirements-docs.in
+    #   sphinx-rtd-theme
+    #   sphinxcontrib-httpdomain
+sphinxcontrib-applehelp==1.0.2
+    # via sphinx
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==1.0.3
+    # via sphinx
+sphinxcontrib-httpdomain==1.7.0
+    # via -r requirements-docs.in
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.4
+    # via sphinx
+urllib3==1.25.8
+    # via requests
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements-tests.txt

@@ -18,13 +18,13 @@ bandit==1.7.0
     # via -r requirements-tests.in
 black==20.8b1
     # via -r requirements-tests.in
-boto3==1.16.60
+boto3==1.16.63
     # via
     #   aws-sam-translator
     #   moto
 boto==2.49.0
     # via moto
-botocore==1.19.60
+botocore==1.19.63
     # via
     #   aws-xray-sdk
     #   boto3
@@ -42,7 +42,7 @@ click==7.1.2
     # via
     #   black
     #   flask
-coverage==5.3.1
+coverage==5.4
     # via -r requirements-tests.in
 cryptography==3.3.1
     # via
@@ -86,7 +86,7 @@ iniconfig==1.0.1
     # via pytest
 itsdangerous==1.1.0
     # via flask
-jinja2==2.11.2
+jinja2==2.11.3
     # via
     #   flask
     #   moto
@@ -148,7 +148,7 @@ pytest-flask==1.1.0
     # via -r requirements-tests.in
 pytest-mock==3.5.1
     # via -r requirements-tests.in
-pytest==6.2.1
+pytest==6.2.2
     # via
     #   -r requirements-tests.in
     #   pytest-flask

requirements.txt

@@ -31,9 +31,9 @@ blinker==1.4
     #   flask-mail
     #   flask-principal
     #   raven
-boto3==1.16.60
+boto3==1.16.63
     # via -r requirements.in
-botocore==1.19.60
+botocore==1.19.63
     # via
     #   -r requirements.in
     #   boto3
@@ -117,7 +117,7 @@ itsdangerous==1.1.0
     # via flask
 javaobj-py3==0.4.0.1
     # via pyjks
-jinja2==2.11.2
+jinja2==2.11.3
     # via
     #   -r requirements.in
     #   flask