Merge branch 'master' into master

Dockerfile

@@ -4,6 +4,7 @@ RUN apt-get install -y make software-properties-common curl
 RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
 RUN apt-get update
 RUN apt-get install -y npm libldap2-dev libsasl2-dev libldap2-dev libssl-dev
+RUN pip install pip==20.0.2
 RUN pip install -U setuptools
 RUN pip install coveralls bandit
 WORKDIR /app

README.rst

@@ -22,7 +22,7 @@ Lemur
 Lemur manages TLS certificate creation. While not able to issue certificates itself, Lemur acts as a broker between CAs
 and environments providing a central portal for developers to issue TLS certificates with 'sane' defaults.
 
-It works on CPython 3.5. We deploy on Ubuntu and develop on OS X.
+It works on Python 3.7. We deploy on Ubuntu and develop on OS X.
 
 
 Project resources

bower.json

@@ -11,12 +11,12 @@
     "angular": "1.4.9",
     "json3": "~3.3",
     "es5-shim": "~4.5.0",
-    "bootstrap": "~3.3.6",
     "angular-bootstrap": "~1.1.1",
     "angular-animate": "~1.4.9",
     "restangular": "~1.5.1",
     "ng-table": "~0.8.3",
     "moment": "~2.11.1",
+    "bootstrap": "~3.4.1",
     "angular-loading-bar": "~0.8.0",
     "angular-moment": "~0.10.3",
     "moment-range": "~2.1.0",
@@ -24,7 +24,7 @@
     "angularjs-toaster": "~1.0.0",
     "angular-chart.js": "~0.8.8",
     "ngletteravatar": "~4.0.0",
-    "bootswatch": "~3.3.6",
+    "bootswatch": "3.4.1+1",
     "fontawesome": "~4.5.0",
     "satellizer": "~0.13.4",
     "angular-ui-router": "~0.2.15",

docker/Dockerfile

@@ -3,6 +3,8 @@ FROM alpine:3.8
 ARG VERSION
 ENV VERSION master
 
+ARG URLCONTEXT
+
 ENV uid 1337
 ENV gid 1337
 ENV user lemur
@@ -22,6 +24,7 @@ RUN addgroup -S ${group} -g ${gid} && \
                 gcc \
                 autoconf \
                 automake \
+                libtool \
                 make \
                 nasm  \
                 zlib-dev \
@@ -42,7 +45,7 @@ WORKDIR /opt/lemur
 RUN npm install --unsafe-perm && \
     pip3 install -e . && \
     node_modules/.bin/gulp build && \
-    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
     apk del build-dependencies
 
 COPY entrypoint /

docker/Dockerfile-src

@@ -0,0 +1,67 @@
+FROM alpine:3.8
+
+ARG VERSION
+ENV VERSION master
+
+ARG URLCONTEXT
+
+ENV uid 1337
+ENV gid 1337
+ENV user lemur
+ENV group lemur
+
+RUN addgroup -S ${group} -g ${gid} && \
+    adduser -D -S ${user} -G ${group} -u ${uid} && \
+    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk --update add --virtual build-dependencies \
+                git \
+                tar \
+                curl \
+                python3-dev \
+                npm \
+                bash \
+                musl-dev \
+                gcc \
+                autoconf \
+                automake \
+                libtool \
+                make \
+                nasm  \
+                zlib-dev \
+                postgresql-dev \
+                libressl-dev  \
+                libffi-dev \
+                cyrus-sasl-dev \
+                openldap-dev && \
+    pip3 install --upgrade pip && \
+    pip3 install --upgrade setuptools && \
+    mkdir -p /home/lemur/.lemur/ && \
+    mkdir -p /run/nginx/ /etc/nginx/ssl/
+
+COPY ./ /opt/lemur    
+WORKDIR /opt/lemur
+
+RUN chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \
+    npm install --unsafe-perm && \
+    pip3 install -e . && \
+    node_modules/.bin/gulp build && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
+    apk del build-dependencies
+
+COPY docker/entrypoint /
+COPY docker/src/lemur.conf.py /home/lemur/.lemur/lemur.conf.py
+COPY docker/supervisor.conf /
+COPY docker/nginx/default.conf /etc/nginx/conf.d/
+COPY docker/nginx/default-ssl.conf /etc/nginx/conf.d/
+
+RUN chmod +x /entrypoint
+WORKDIR /
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=30s \  
+ CMD curl --fail http://localhost:80/api/1/healthcheck | grep -q ok || exit 1
+
+USER root
+
+ENTRYPOINT ["/entrypoint"]
+
+CMD ["/usr/bin/supervisord","-c","supervisor.conf"]

docker/entrypoint

@@ -36,7 +36,7 @@ fi
 # fi
 
 echo " # Running init"
-su lemur -s /bin/bash -c "cd /opt/lemur/lemur; python3 /opt/lemur/lemur/manage.py init -p ${LEMUR_ADMIN_PASSWORD}"
+su lemur -s /bin/bash -c "cd /opt/lemur/lemur; lemur init -p ${LEMUR_ADMIN_PASSWORD}"
 echo " # Done"
 
 # echo "Creating user"
@@ -47,11 +47,13 @@ echo " # Done"
 cron_notify="${CRON_NOTIFY:-"0 22 * * *"}"
 cron_sync="${CRON_SYNC:-"*/15 * * * *"}"
 cron_revoked="${CRON_CHECK_REVOKED:-"0 22 * * *"}"
+cron_reissue="${CRON_REISSUE:-"0 23 * * *"}"
 
 echo " # Populating crontab"
-echo "${cron_notify} lemur python3 /opt/lemur/lemur/manage.py notify expirations" > /etc/crontabs/lemur_notify
+echo "${cron_notify} lemur notify expirations" > /etc/crontabs/lemur
-echo "${cron_sync} lemur python3 /opt/lemur/lemur/manage.py source sync -s all" > /etc/crontabs/lemur_sync
+echo "${cron_sync} lemur source sync -s all" >> /etc/crontabs/lemur
-echo "${cron_revoked} lemur python3 /opt/lemur/lemur/manage.py certificate check_revoked" > /etc/crontabs/lemur_revoked
+echo "${cron_revoked} lemur certificate check_revoked" >> /etc/crontabs/lemur
+echo "${cron_reissue} lemur certificate reissue -c" >> /etc/crontabs/lemur
 echo " # Done"
 
 exec "$@"

docker/src/lemur.conf.py

@@ -16,12 +16,16 @@ LEMUR_WHITELISTED_DOMAINS = []
 LEMUR_EMAIL = ''
 LEMUR_SECURITY_TEAM_EMAIL = []
 
+ALLOW_CERT_DELETION = os.environ.get('ALLOW_CERT_DELETION') == "True"
 
-LEMUR_DEFAULT_COUNTRY = repr(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
-LEMUR_DEFAULT_STATE = repr(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE',''))
-LEMUR_DEFAULT_LOCATION = repr(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
-LEMUR_DEFAULT_ORGANIZATION = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
-LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = repr(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+
+LEMUR_DEFAULT_ISSUER_PLUGIN = str(os.environ.get('LEMUR_DEFAULT_ISSUER_PLUGIN',''))
+LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY',''))
 
 ACTIVE_PROVIDERS = []
 

docker/supervisor.conf

@@ -7,7 +7,7 @@ pidfile = /tmp/supervisord.pid
 
 [program:lemur]
 environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
-command=/usr/bin/python3 manage.py start -b 0.0.0.0:8000
+command=lemur start -b 0.0.0.0:8000
 user=lemur
 directory=/opt/lemur/lemur
 stdout_logfile=/dev/stdout
@@ -24,6 +24,7 @@ stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
 [program:cron]
+environment=LEMUR_CONF=/home/lemur/.lemur/lemur.conf.py
 command=/usr/sbin/crond -f
 user=root
 stdout_logfile=/dev/stdout

docs/administration.rst

@@ -66,7 +66,7 @@ Basic Configuration
 
 
 .. data:: SQLALCHEMY_POOL_SIZE
-:noindex:
+    :noindex:
 
             The default connection pool size is 5 for sqlalchemy managed connections.   Depending on the number of Lemur instances,
             please specify per instance connection pool size.  Below is an example to set connection pool size to 10.
@@ -80,7 +80,7 @@ Basic Configuration
 This is an optional setting but important to review and set for optimal database connection usage and for overall database performance.
 
 .. data:: SQLALCHEMY_MAX_OVERFLOW
-:noindex:
+    :noindex:
 
         This setting allows to create connections in addition to specified number of connections in pool size.   By default, sqlalchemy
         allows 10 connections to create in addition to the pool size.  This is also an optional setting.  If `SQLALCHEMY_POOL_SIZE` and
@@ -155,6 +155,22 @@ Specifying the `SQLALCHEMY_MAX_OVERFLOW` to 0 will enforce limit to not create c
 
         LEMUR_ENCRYPTION_KEYS = ['1YeftooSbxCiX2zo8m1lXtpvQjy27smZcUUaGmffhMY=', 'LAfQt6yrkLqOK5lwpvQcT4jf2zdeTQJV1uYeh9coT5s=']
 
+.. data:: PUBLIC_CA_AUTHORITY_NAMES
+    :noindex:
+        A list of public issuers which would be checked against to determine whether limit of max validity of 397 days
+        should be applied to the certificate. Configure public CA authority names in this list to enforce validity check.
+        This is an optional setting. Using this will allow the sanity check as mentioned. The name check is a case-insensitive
+        string comparision.
+
+.. data:: PUBLIC_CA_MAX_VALIDITY_DAYS
+    :noindex:
+        Use this config to override the limit of 397 days of validity for certificates issued by public issuers configured
+        using PUBLIC_CA_AUTHORITY_NAMES. Below example overrides the default validity of 397 days and sets it to 365 days.
+
+    ::
+
+        PUBLIC_CA_MAX_VALIDITY_DAYS = 365
+
 
 .. data:: DEBUG_DUMP
     :noindex:
@@ -729,10 +745,16 @@ The following configuration properties are required to use the Digicert issuer p
             This is the root to be used for your CA chain
 
 
-.. data:: DIGICERT_DEFAULT_VALIDITY
+.. data:: DIGICERT_DEFAULT_VALIDITY_DAYS
     :noindex:
 
-            This is the default validity (in years), if no end date is specified. (Default: 1)
+            This is the default validity (in days), if no end date is specified. (Default: 397)
+
+
+.. data:: DIGICERT_MAX_VALIDITY_DAYS
+    :noindex:
+
+            This is the maximum validity (in days). (Default: value of DIGICERT_DEFAULT_VALIDITY_DAYS)
 
 
 .. data:: DIGICERT_PRIVATE
@@ -1008,6 +1030,18 @@ The following configuration properties are required to use the PowerDNS ACME Plu
 
             This is the number of times DNS Verification should be attempted (i.e. 20)
 
+
+.. data:: ACME_POWERDNS_VERIFY
+    :noindex:
+
+            This configures how TLS certificates on the PowerDNS API target are validated.  The PowerDNS Plugin depends on the PyPi requests library, which supports the following options for the verify parameter:
+
+            True: Verifies the TLS certificate was issued by a known publicly-trusted CA. (Default)
+
+            False: Disables certificate validation (Not Recommended)
+
+            File/Dir path to CA Bundle: Verifies the TLS certificate was issued by a Certificate Authority in the provided CA bundle.
+
 .. _CommandLineInterface:
 
 Command Line Interface

docs/production/index.rst

@@ -390,6 +390,10 @@ Here are the Celery configuration variables that should be set::
     CELERY_IMPORTS = ('lemur.common.celery')
     CELERY_TIMEZONE = 'UTC'
 
+Do not forget to import crontab module in your configuration file::
+
+    from celery.task.schedules import crontab
+
 You must start a single Celery scheduler instance and one or more worker instances in order to handle incoming tasks.
 The scheduler can be started with::
 
@@ -447,3 +451,53 @@ LetsEncrypt flow to function. However, Lemur will attempt to automatically deter
 possible. To enable this functionality, periodically (or through Cron/Celery) run `lemur dns_providers get_all_zones`.
 This command will traverse all DNS providers, determine which zones they control, and upload this list of zones to
 Lemur's database (in the dns_providers table). Alternatively, you can manually input this data.
+
+
+LetsEncrypt: pinning to cross-signed ICA
+----------------------------------------
+
+Let's Encrypt has been using a `cross-signed <https://letsencrypt.org/certificates/>`_ intermediate CA by DST Root CA X3,
+which is included in many older devices' TrustStore.
+
+
+Let's Encrypt is `transitioning <https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html>`_ to use
+the intermediate CA issued by their own root (ISRG X1) starting from September 29th 2020.
+This is in preparation of concluding the initial bootstrapping of their CA, by having it cross-signed by an older CA.
+
+
+Lemur can temporarily pin to the cross-signed intermediate CA (same public/private key pair as the ICA signed by ISRG X1).
+This will prolong support for incompatible devices.
+
+The following must be added to the config file to activate the pinning (the pinning will be removed by September 2021)::
+
+    # remove or update after Mar 17 16:40:46 2021 GMT
+    IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE = "17/03/21"
+    IDENTRUST_CROSS_SIGNED_LE_ICA = """
+    -----BEGIN CERTIFICATE-----
+    MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+    MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+    DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+    SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+    GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+    AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+    q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+    SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+    Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+    a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+    /PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+    AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+    CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+    bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+    c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+    VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+    ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+    MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+    Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+    AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+    uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+    wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+    X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+    PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+    KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+    -----END CERTIFICATE-----
+    """

lemur/auth/views.py

@@ -127,6 +127,10 @@ def retrieve_user(user_api_url, access_token):
 
     # retrieve information about the current user.
     r = requests.get(user_api_url, params=user_params, headers=headers)
+    # Some IDPs, like "Keycloak", require a POST instead of a GET
+    if r.status_code == 400:
+        r = requests.post(user_api_url, data=user_params, headers=headers)
+
     profile = r.json()
 
     user = user_service.get_by_email(profile["email"])
@@ -434,7 +438,7 @@ class OAuth2(Resource):
             verify_cert=verify_cert,
         )
 
-        jwks_url = current_app.config.get("PING_JWKS_URL")
+        jwks_url = current_app.config.get("OAUTH2_JWKS_URL")
         error_code = validate_id_token(id_token, args["clientId"], jwks_url)
         if error_code:
             return error_code

lemur/authorizations/models.py

@@ -25,7 +25,7 @@ class Authorization(db.Model):
         return plugins.get(self.plugin_name)
 
     def __repr__(self):
-        return "Authorization(id={id})".format(label=self.id)
+        return "Authorization(id={id})".format(id=self.id)
 
     def __init__(self, account_number, domains, dns_provider_type, options=None):
         self.account_number = account_number

lemur/certificates/cli.py

@@ -5,39 +5,36 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import sys
 import multiprocessing
-from tabulate import tabulate
+import sys
-from sqlalchemy import or_
-
 from flask import current_app
-
-from flask_script import Manager
 from flask_principal import Identity, identity_changed
-
+from flask_script import Manager
+from sqlalchemy import or_
+from tabulate import tabulate
 
 from lemur import database
-from lemur.extensions import sentry
-from lemur.extensions import metrics
-from lemur.plugins.base import plugins
-from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
-from lemur.deployment import service as deployment_service
-from lemur.endpoints import service as endpoint_service
-from lemur.notifications.messaging import send_rotation_notification
-from lemur.domains.models import Domain
 from lemur.authorities.models import Authority
-from lemur.certificates.schemas import CertificateOutputSchema
+from lemur.authorities.service import get as authorities_get_by_id
 from lemur.certificates.models import Certificate
+from lemur.certificates.schemas import CertificateOutputSchema
 from lemur.certificates.service import (
     reissue_certificate,
     get_certificate_primitives,
     get_all_pending_reissue,
     get_by_name,
-    get_all_certs,
+    get_all_valid_certs,
     get,
+    get_all_certs_attached_to_endpoint_without_autorotate,
 )
-
 from lemur.certificates.verify import verify_string
+from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
+from lemur.deployment import service as deployment_service
+from lemur.domains.models import Domain
+from lemur.endpoints import service as endpoint_service
+from lemur.extensions import sentry, metrics
+from lemur.notifications.messaging import send_rotation_notification
+from lemur.plugins.base import plugins
 
 manager = Manager(usage="Handles all certificate related tasks.")
 
@@ -213,6 +210,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
     status = FAILURE_METRIC_STATUS
 
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+    }
+
     try:
         old_cert = validate_certificate(old_certificate_name)
         new_cert = validate_certificate(new_certificate_name)
@@ -222,26 +223,43 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
+            log_data["message"] = "Rotating endpoint"
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
+            current_app.logger.info(log_data)
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
 
+            log_data["message"] = "Rotating all endpoints"
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
             for endpoint in old_cert.endpoints:
                 print(f"[+] Rotating {endpoint.name}")
+                log_data["endpoint"] = endpoint.dnsname
                 request_rotation(endpoint, new_cert, message, commit)
+                current_app.logger.info(log_data)
 
         else:
             print("[+] Rotating all endpoints that have new certificates available")
+            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
+                log_data["endpoint"] = endpoint.dnsname
                 if len(endpoint.certificate.replaced) == 1:
                     print(
                         f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
                     )
+                    log_data["certificate"] = endpoint.certificate.replaced[0].name
                     request_rotation(
                         endpoint, endpoint.certificate.replaced[0], message, commit
                     )
+                    current_app.logger.info(log_data)
+
                 else:
+                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                    print(log_data)
                     metrics.send(
                         "endpoint_rotation",
                         "counter",
@@ -289,6 +307,178 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
     )
 
 
+def request_rotation_region(endpoint, new_cert, message, commit, log_data, region):
+    if region in endpoint.dnsname:
+        log_data["message"] = "Rotating endpoint in region"
+        request_rotation(endpoint, new_cert, message, commit)
+    else:
+        log_data["message"] = "Skipping rotation, region mismatch"
+
+    print(log_data)
+    current_app.logger.info(log_data)
+
+
+@manager.option(
+    "-e",
+    "--endpoint",
+    dest="endpoint_name",
+    help="Name of the endpoint you wish to rotate.",
+)
+@manager.option(
+    "-n",
+    "--new-certificate",
+    dest="new_certificate_name",
+    help="Name of the certificate you wish to rotate to.",
+)
+@manager.option(
+    "-o",
+    "--old-certificate",
+    dest="old_certificate_name",
+    help="Name of the certificate you wish to rotate.",
+)
+@manager.option(
+    "-a",
+    "--notify",
+    dest="message",
+    action="store_true",
+    help="Send a rotation notification to the certificates owner.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
+@manager.option(
+    "-r",
+    "--region",
+    dest="region",
+    required=True,
+    help="Region in which to rotate the endpoint.",
+)
+def rotate_region(endpoint_name, new_certificate_name, old_certificate_name, message, commit, region):
+    """
+    Rotates an endpoint in a defined region it if it has not already been replaced. If it has
+    been replaced, will use the replacement certificate for the rotation.
+    :param old_certificate_name: Name of the certificate you wish to rotate.
+    :param new_certificate_name: Name of the certificate you wish to rotate to.
+    :param endpoint_name: Name of the endpoint you wish to rotate.
+    :param message: Send a rotation notification to the certificates owner.
+    :param commit: Persist changes.
+    :param region: Region in which to rotate the endpoint.
+    """
+    if commit:
+        print("[!] Running in COMMIT mode.")
+
+    print("[+] Starting endpoint rotation.")
+    status = FAILURE_METRIC_STATUS
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "region": region,
+    }
+
+    try:
+        old_cert = validate_certificate(old_certificate_name)
+        new_cert = validate_certificate(new_certificate_name)
+        endpoint = validate_endpoint(endpoint_name)
+
+        if endpoint and new_cert:
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
+            request_rotation_region(endpoint, new_cert, message, commit, log_data, region)
+
+        elif old_cert and new_cert:
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
+            print(log_data)
+            current_app.logger.info(log_data)
+            for endpoint in old_cert.endpoints:
+                log_data["endpoint"] = endpoint.dnsname
+                request_rotation_region(endpoint, new_cert, message, commit, log_data, region)
+
+        else:
+            log_data["message"] = "Rotating all endpoints that have new certificates available"
+            print(log_data)
+            current_app.logger.info(log_data)
+            all_pending_rotation_endpoints = endpoint_service.get_all_pending_rotation()
+            for endpoint in all_pending_rotation_endpoints:
+                log_data["endpoint"] = endpoint.dnsname
+                if region not in endpoint.dnsname:
+                    log_data["message"] = "Skipping rotation, region mismatch"
+                    print(log_data)
+                    current_app.logger.info(log_data)
+                    metrics.send(
+                        "endpoint_rotation_region_skipped",
+                        "counter",
+                        1,
+                        metric_tags={
+                            "region": region,
+                            "old_certificate_name": str(old_cert),
+                            "new_certificate_name": str(endpoint.certificate.replaced[0].name),
+                            "endpoint_name": str(endpoint.dnsname),
+                        },
+                    )
+
+                if len(endpoint.certificate.replaced) == 1:
+                    log_data["certificate"] = endpoint.certificate.replaced[0].name
+                    log_data["message"] = "Rotating all endpoints in region"
+                    print(log_data)
+                    current_app.logger.info(log_data)
+                    request_rotation(endpoint, endpoint.certificate.replaced[0], message, commit)
+                    status = SUCCESS_METRIC_STATUS
+                else:
+                    status = FAILURE_METRIC_STATUS
+                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                    print(log_data)
+                    current_app.logger.info(log_data)
+
+                metrics.send(
+                    "endpoint_rotation_region",
+                    "counter",
+                    1,
+                    metric_tags={
+                        "status": FAILURE_METRIC_STATUS,
+                        "old_certificate_name": str(old_cert),
+                        "new_certificate_name": str(endpoint.certificate.replaced[0].name),
+                        "endpoint_name": str(endpoint.dnsname),
+                        "message": str(message),
+                        "region": str(region),
+                    },
+                )
+        status = SUCCESS_METRIC_STATUS
+        print("[+] Done!")
+
+    except Exception as e:
+        sentry.captureException(
+            extra={
+                "old_certificate_name": str(old_certificate_name),
+                "new_certificate_name": str(new_certificate_name),
+                "endpoint": str(endpoint_name),
+                "message": str(message),
+                "region": str(region),
+            }
+        )
+
+    metrics.send(
+        "endpoint_rotation_region_job",
+        "counter",
+        1,
+        metric_tags={
+            "status": status,
+            "old_certificate_name": str(old_certificate_name),
+            "new_certificate_name": str(new_certificate_name),
+            "endpoint_name": str(endpoint_name),
+            "message": str(message),
+            "endpoint": str(globals().get("endpoint")),
+            "region": str(region),
+        },
+    )
+
+
 @manager.option(
     "-o",
     "--old-certificate",
@@ -467,7 +657,14 @@ def check_revoked():
     encounters an issue with verification it marks the certificate status
     as `unknown`.
     """
-    for cert in get_all_certs():
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Checking for revoked Certificates"
+    }
+
+    certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
+    for cert in certs:
         try:
             if cert.chain:
                 status = verify_string(cert.body, cert.chain)
@@ -476,9 +673,65 @@ def check_revoked():
 
             cert.status = "valid" if status else "revoked"
 
+            if cert.status == "revoked":
+                log_data["valid"] = cert.status
+                log_data["certificate_name"] = cert.name
+                log_data["certificate_id"] = cert.id
+                metrics.send(
+                    "certificate_revoked",
+                    "counter",
+                    1,
+                    metric_tags={"status": log_data["valid"],
+                                 "certificate_name": log_data["certificate_name"],
+                                 "certificate_id": log_data["certificate_id"]},
+                )
+                current_app.logger.info(log_data)
+
         except Exception as e:
             sentry.captureException()
             current_app.logger.exception(e)
             cert.status = "unknown"
 
         database.update(cert)
+
+
+@manager.command
+def automatically_enable_autorotate():
+    """
+    This function automatically enables auto-rotation for unexpired certificates that are
+    attached to an endpoint but do not have autorotate enabled.
+
+    WARNING: This will overwrite the Auto-rotate toggle!
+    """
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Enabling auto-rotate for certificate"
+    }
+
+    permitted_authorities = current_app.config.get("ENABLE_AUTO_ROTATE_AUTHORITY", [])
+
+    eligible_certs = get_all_certs_attached_to_endpoint_without_autorotate()
+    for cert in eligible_certs:
+
+        if cert.authority_id not in permitted_authorities:
+            continue
+
+        log_data["certificate"] = cert.name
+        log_data["certificate_id"] = cert.id
+        log_data["authority_id"] = cert.authority_id
+        log_data["authority_name"] = authorities_get_by_id(cert.authority_id).name
+        if cert.destinations:
+            log_data["destination_names"] = ', '.join([d.label for d in cert.destinations])
+        else:
+            log_data["destination_names"] = "NONE"
+        current_app.logger.info(log_data)
+        metrics.send("automatically_enable_autorotate",
+                     "counter", 1,
+                     metric_tags={"certificate": log_data["certificate"],
+                                  "certificate_id": log_data["certificate_id"],
+                                  "authority_id": log_data["authority_id"],
+                                  "authority_name": log_data["authority_name"],
+                                  "destination_names": log_data["destination_names"]
+                                  })
+        cert.rotation = True
+        database.update(cert)

lemur/certificates/models.py

@@ -321,7 +321,8 @@ class Certificate(db.Model):
 
     @hybrid_property
     def expired(self):
-        if self.not_after <= arrow.utcnow():
+        # can't compare offset-naive and offset-aware datetimes
+        if arrow.Arrow.fromdatetime(self.not_after) <= arrow.utcnow():
             return True
 
     @expired.expression
@@ -445,6 +446,9 @@ def update_destinations(target, value, initiator):
     """
     destination_plugin = plugins.get(value.plugin_name)
     status = FAILURE_METRIC_STATUS
+
+    if target.expired:
+        return
     try:
         if target.private_key or not destination_plugin.requires_key:
             destination_plugin.upload(

lemur/certificates/schemas.py

@@ -146,7 +146,8 @@ class CertificateInputSchema(CertificateCreationSchema):
                 data["extensions"]["subAltNames"] = {"names": []}
             elif not data["extensions"]["subAltNames"].get("names"):
                 data["extensions"]["subAltNames"]["names"] = []
-            data["extensions"]["subAltNames"]["names"] += csr_sans
+
+            data["extensions"]["subAltNames"]["names"] = csr_sans
         return missing.convert_validity_years(data)
 
 

lemur/certificates/service.py

@@ -20,6 +20,7 @@ from lemur.common.utils import generate_private_key, truthiness
 from lemur.destinations.models import Destination
 from lemur.domains.models import Domain
 from lemur.extensions import metrics, sentry, signals
+from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
 from lemur.plugins.base import plugins
@@ -102,13 +103,35 @@ def get_all_certs():
     return Certificate.query.all()
 
 
-def get_all_pending_cleaning(source):
+def get_all_valid_certs(authority_plugin_name):
     """
-    Retrieves all certificates that are available for cleaning.
+    Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
+    ignored if no authority_plugin_name provided.
+
+    Note that depending on the DB size retrieving all certificates might an expensive operation
 
-    :param source:
     :return:
     """
+    if authority_plugin_name:
+        return (
+            Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
+                Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Authority.plugin_name.in_(authority_plugin_name)).all()
+        )
+    else:
+        return (
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
+        )
+
+
+def get_all_pending_cleaning_expired(source):
+    """
+    Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not
+    attached to any endpoints.
+
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
     return (
         Certificate.query.filter(Certificate.sources.any(id=source.id))
         .filter(not_(Certificate.endpoints.any()))
@@ -117,6 +140,58 @@ def get_all_pending_cleaning(source):
     )
 
 
+def get_all_certs_attached_to_endpoint_without_autorotate():
+    """
+        Retrieves all certificates that are attached to an endpoint, but that do not have autorotate enabled.
+
+        :return: list of certificates attached to an endpoint without autorotate
+        """
+    return (
+        Certificate.query.filter(Certificate.endpoints.any())
+        .filter(Certificate.rotation == False)
+        .filter(Certificate.not_after >= arrow.now())
+        .filter(not_(Certificate.replaced.any()))
+        .all()  # noqa
+    )
+
+
+def get_all_pending_cleaning_expiring_in_days(source, days_to_expire):
+    """
+    Retrieves all certificates that are available for cleaning, not attached to endpoint,
+    and within X days from expiration.
+
+    :param days_to_expire: defines how many days till the certificate is expired
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    expiration_window = arrow.now().shift(days=+days_to_expire).format("YYYY-MM-DD")
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.not_after < expiration_window)
+        .all()
+    )
+
+
+def get_all_pending_cleaning_issued_since_days(source, days_since_issuance):
+    """
+    Retrieves all certificates that are available for cleaning: not attached to endpoint, and X days since issuance.
+
+    :param days_since_issuance: defines how many days since the certificate is issued
+    :param source: the source to search for certificates
+    :return: list of pending certificates
+    """
+    not_in_use_window = (
+        arrow.now().shift(days=-days_since_issuance).format("YYYY-MM-DD")
+    )
+    return (
+        Certificate.query.filter(Certificate.sources.any(id=source.id))
+        .filter(not_(Certificate.endpoints.any()))
+        .filter(Certificate.date_created > not_in_use_window)
+        .all()
+    )
+
+
 def get_all_pending_reissue():
     """
     Retrieves all certificates that need to be rotated.
@@ -331,9 +406,11 @@ def render(args):
 
     show_expired = args.pop("showExpired")
     if show_expired != 1:
-        one_month_old = arrow.now()\
+        one_month_old = (
-            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))\
+            arrow.now()
+            .shift(months=current_app.config.get("HIDE_EXPIRED_CERTS_AFTER_MONTHS", -1))
             .format("YYYY-MM-DD")
+        )
         query = query.filter(Certificate.not_after > one_month_old)
 
     time_range = args.pop("time_range")
@@ -379,8 +456,8 @@ def render(args):
         elif "cn" in terms:
             query = query.filter(
                 or_(
-                    Certificate.cn.ilike(term),
+                    func.lower(Certificate.cn).like(term.lower()),
-                    Certificate.domains.any(Domain.name.ilike(term)),
+                    Certificate.id.in_(like_domain_query(term)),
                 )
             )
         elif "id" in terms:
@@ -388,9 +465,9 @@ def render(args):
         elif "name" in terms:
             query = query.filter(
                 or_(
-                    Certificate.name.ilike(term),
+                    func.lower(Certificate.name).like(term.lower()),
-                    Certificate.domains.any(Domain.name.ilike(term)),
+                    Certificate.id.in_(like_domain_query(term)),
-                    Certificate.cn.ilike(term),
+                    func.lower(Certificate.cn).like(term.lower()),
                 )
             )
         elif "fixedName" in terms:
@@ -435,6 +512,14 @@ def render(args):
     return result
 
 
+def like_domain_query(term):
+    domain_query = database.session_query(Domain.id)
+    domain_query = domain_query.filter(func.lower(Domain.name).like(term.lower()))
+    assoc_query = database.session_query(certificate_associations.c.certificate_id)
+    assoc_query = assoc_query.filter(certificate_associations.c.domain_id.in_(domain_query))
+    return assoc_query
+
+
 def query_name(certificate_name, args):
     """
     Helper function that queries for a certificate by name

lemur/certificates/verify.py

@@ -8,6 +8,7 @@
 import requests
 import subprocess
 from flask import current_app
+from lemur.extensions import sentry
 from requests.exceptions import ConnectionError, InvalidSchema
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -152,10 +153,19 @@ def verify(cert_path, issuer_chain_path):
 
     # OCSP is our main source of truth, in a lot of cases CRLs
     # have been deprecated and are no longer updated
+    verify_result = None
+    try:
         verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception(e)
 
     if verify_result is None:
+        try:
             verify_result = crl_verify(cert, cert_path)
+        except Exception as e:
+            sentry.captureException()
+            current_app.logger.exception(e)
 
     if verify_result is None:
         current_app.logger.debug("Failed to verify {}".format(cert.serial_number))

lemur/common/celery.py

@@ -10,27 +10,27 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
 import copy
 import sys
 import time
-from datetime import datetime, timezone, timedelta
-
 from celery import Celery
+from celery.app.task import Context
 from celery.exceptions import SoftTimeLimitExceeded
+from celery.signals import task_failure, task_received, task_revoked, task_success
+from datetime import datetime, timezone, timedelta
 from flask import current_app
 
 from lemur.authorities.service import get as get_authority
+from lemur.certificates import cli as cli_certificate
 from lemur.common.redis import RedisHandler
 from lemur.destinations import service as destinations_service
+from lemur.dns_providers import cli as cli_dns_providers
+from lemur.endpoints import cli as cli_endpoints
 from lemur.extensions import metrics, sentry
 from lemur.factory import create_app
+from lemur.notifications import cli as cli_notification
 from lemur.notifications.messaging import send_pending_failure_notification
 from lemur.pending_certificates import service as pending_certificate_service
 from lemur.plugins.base import plugins
 from lemur.sources.cli import clean, sync, validate_sources
 from lemur.sources.service import add_aws_destination_to_sources
-from lemur.certificates import cli as cli_certificate
-from lemur.dns_providers import cli as cli_dns_providers
-from lemur.notifications import cli as cli_notification
-from lemur.endpoints import cli as cli_endpoints
-
 
 if current_app:
     flask_app = current_app
@@ -67,7 +67,7 @@ def is_task_active(fun, task_id, args):
     from celery.task.control import inspect
 
     if not args:
-        args = '()'  # empty args
+        args = "()"  # empty args
 
     i = inspect()
     active_tasks = i.active()
@@ -80,6 +80,37 @@ def is_task_active(fun, task_id, args):
     return False
 
 
+def get_celery_request_tags(**kwargs):
+    request = kwargs.get("request")
+    sender_hostname = "unknown"
+    sender = kwargs.get("sender")
+    if sender:
+        try:
+            sender_hostname = sender.hostname
+        except AttributeError:
+            sender_hostname = vars(sender.request).get("origin", "unknown")
+    if request and not isinstance(
+        request, Context
+    ):  # unlike others, task_revoked sends a Context for `request`
+        task_name = request.name
+        task_id = request.id
+        receiver_hostname = request.hostname
+    else:
+        task_name = sender.name
+        task_id = sender.request.id
+        receiver_hostname = sender.request.hostname
+
+    tags = {
+        "task_name": task_name,
+        "task_id": task_id,
+        "sender_hostname": sender_hostname,
+        "receiver_hostname": receiver_hostname,
+    }
+    if kwargs.get("exception"):
+        tags["error"] = repr(kwargs["exception"])
+    return tags
+
+
 @celery.task()
 def report_celery_last_success_metrics():
     """
@@ -89,7 +120,6 @@ def report_celery_last_success_metrics():
     report_celery_last_success_metrics should be ran periodically to emit metrics on when a task was last successful.
     Admins can then alert when tasks are not ran when intended. Admins should also alert when no metrics are emitted
     from this function.
-
     """
     function = f"{__name__}.{sys._getframe().f_code.co_name}"
     task_id = None
@@ -108,15 +138,91 @@ def report_celery_last_success_metrics():
         return
 
     current_time = int(time.time())
-    schedule = current_app.config.get('CELERYBEAT_SCHEDULE')
+    schedule = current_app.config.get("CELERYBEAT_SCHEDULE")
     for _, t in schedule.items():
         task = t.get("task")
         last_success = int(red.get(f"{task}.last_success") or 0)
-        metrics.send(f"{task}.time_since_last_success", 'gauge', current_time - last_success)
+        metrics.send(
+            f"{task}.time_since_last_success", "gauge", current_time - last_success
+        )
     red.set(
         f"{function}.last_success", int(time.time())
     )  # Alert if this metric is not seen
-    metrics.send(f"{function}.success", 'counter', 1)
+    metrics.send(f"{function}.success", "counter", 1)
+
+
+@task_received.connect
+def report_number_pending_tasks(**kwargs):
+    """
+    Report the number of pending tasks to our metrics broker every time a task is published. This metric can be used
+    for autoscaling workers.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-received
+    """
+    with flask_app.app_context():
+        metrics.send(
+            "celery.new_pending_task",
+            "TIMER",
+            1,
+            metric_tags=get_celery_request_tags(**kwargs),
+        )
+
+
+@task_success.connect
+def report_successful_task(**kwargs):
+    """
+    Report a generic success metric as tasks to our metrics broker every time a task finished correctly.
+    This metric can be used for autoscaling workers.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-success
+    """
+    with flask_app.app_context():
+        tags = get_celery_request_tags(**kwargs)
+        red.set(f"{tags['task_name']}.last_success", int(time.time()))
+        metrics.send("celery.successful_task", "TIMER", 1, metric_tags=tags)
+
+
+@task_failure.connect
+def report_failed_task(**kwargs):
+    """
+    Report a generic failure metric as tasks to our metrics broker every time a task fails.
+    This metric can be used for alerting.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-failure
+    """
+    with flask_app.app_context():
+        log_data = {
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+            "Message": "Celery Task Failure",
+        }
+
+        # Add traceback if exception info is in the kwargs
+        einfo = kwargs.get("einfo")
+        if einfo:
+            log_data["traceback"] = einfo.traceback
+
+        error_tags = get_celery_request_tags(**kwargs)
+
+        log_data.update(error_tags)
+        current_app.logger.error(log_data)
+        metrics.send("celery.failed_task", "TIMER", 1, metric_tags=error_tags)
+
+
+@task_revoked.connect
+def report_revoked_task(**kwargs):
+    """
+    Report a generic failure metric as tasks to our metrics broker every time a task is revoked.
+    This metric can be used for alerting.
+    https://docs.celeryproject.org/en/latest/userguide/signals.html#task-revoked
+    """
+    with flask_app.app_context():
+        log_data = {
+            "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+            "Message": "Celery Task Revoked",
+        }
+
+        error_tags = get_celery_request_tags(**kwargs)
+
+        log_data.update(error_tags)
+        current_app.logger.error(log_data)
+        metrics.send("celery.revoked_task", "TIMER", 1, metric_tags=error_tags)
 
 
 @celery.task(soft_time_limit=600)
@@ -217,15 +323,15 @@ def fetch_acme_cert(id):
     log_data["failed"] = failed
     log_data["wrong_issuer"] = wrong_issuer
     current_app.logger.debug(log_data)
-    metrics.send(f"{function}.resolved", 'gauge', new)
+    metrics.send(f"{function}.resolved", "gauge", new)
-    metrics.send(f"{function}.failed", 'gauge', failed)
+    metrics.send(f"{function}.failed", "gauge", failed)
-    metrics.send(f"{function}.wrong_issuer", 'gauge', wrong_issuer)
+    metrics.send(f"{function}.wrong_issuer", "gauge", wrong_issuer)
     print(
         "[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
             new=new, failed=failed, wrong_issuer=wrong_issuer
         )
     )
-    red.set(f'{function}.last_success', int(time.time()))
+    return log_data
 
 
 @celery.task()
@@ -262,8 +368,8 @@ def fetch_all_pending_acme_certs():
                 current_app.logger.debug(log_data)
                 fetch_acme_cert.delay(cert.id)
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task()
@@ -296,8 +402,8 @@ def remove_old_acme_certs():
             current_app.logger.debug(log_data)
             pending_certificate_service.delete(cert)
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task()
@@ -328,11 +434,11 @@ def clean_all_sources():
         current_app.logger.debug(log_data)
         clean_source.delay(source.label)
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
-@celery.task(soft_time_limit=600)
+@celery.task(soft_time_limit=3600)
 def clean_source(source):
     """
     This celery task will clean the specified source. This is a destructive operation that will delete unused
@@ -366,6 +472,7 @@ def clean_source(source):
         current_app.logger.error(log_data)
         sentry.captureException()
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
+    return log_data
 
 
 @celery.task()
@@ -395,8 +502,8 @@ def sync_all_sources():
         current_app.logger.debug(log_data)
         sync_source.delay(source.label)
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=7200)
@@ -428,19 +535,23 @@ def sync_source(source):
     current_app.logger.debug(log_data)
     try:
         sync([source])
-        metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
+        metrics.send(
+            f"{function}.success", "counter", 1, metric_tags={"source": source}
+        )
     except SoftTimeLimitExceeded:
         log_data["message"] = "Error syncing source: Time limit exceeded."
         current_app.logger.error(log_data)
         sentry.captureException()
-        metrics.send("sync_source_timeout", "counter", 1, metric_tags={"source": source})
+        metrics.send(
+            "sync_source_timeout", "counter", 1, metric_tags={"source": source}
+        )
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
         return
 
     log_data["message"] = "Done syncing source"
     current_app.logger.debug(log_data)
-    metrics.send(f"{function}.success", 'counter', 1, metric_tags={"source": source})
+    metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source})
-    red.set(f'{function}.last_success', int(time.time()))
+    return log_data
 
 
 @celery.task()
@@ -477,8 +588,8 @@ def sync_source_destination():
 
     log_data["message"] = "completed Syncing AWS destinations and sources"
     current_app.logger.debug(log_data)
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=3600)
@@ -515,12 +626,13 @@ def certificate_reissue():
 
     log_data["message"] = "reissuance completed"
     current_app.logger.debug(log_data)
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=3600)
-def certificate_rotate():
+def certificate_rotate(**kwargs):
+
     """
     This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert
     :return:
@@ -530,11 +642,11 @@ def certificate_rotate():
     if celery.current_task:
         task_id = celery.current_task.request.id
 
+    region = kwargs.get("region")
     log_data = {
         "function": function,
         "message": "rotating certificates",
         "task_id": task_id,
-
     }
 
     if task_id and is_task_active(function, task_id, None):
@@ -544,6 +656,10 @@ def certificate_rotate():
 
     current_app.logger.debug(log_data)
     try:
+        if region:
+            log_data["region"] = region
+            cli_certificate.rotate_region(None, None, None, None, True, region)
+        else:
             cli_certificate.rotate(None, None, None, None, True)
     except SoftTimeLimitExceeded:
         log_data["message"] = "Certificate rotate: Time limit exceeded."
@@ -554,8 +670,8 @@ def certificate_rotate():
 
     log_data["message"] = "rotation completed"
     current_app.logger.debug(log_data)
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=3600)
@@ -590,8 +706,8 @@ def endpoints_expire():
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
         return
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=600)
@@ -626,8 +742,8 @@ def get_all_zones():
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
         return
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=3600)
@@ -662,8 +778,8 @@ def check_revoked():
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
         return
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
 
 
 @celery.task(soft_time_limit=3600)
@@ -690,7 +806,9 @@ def notify_expirations():
 
     current_app.logger.debug(log_data)
     try:
-        cli_notification.expirations(current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", []))
+        cli_notification.expirations(
+            current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])
+        )
     except SoftTimeLimitExceeded:
         log_data["message"] = "Notify expiring Time limit exceeded."
         current_app.logger.error(log_data)
@@ -698,5 +816,29 @@ def notify_expirations():
         metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
         return
 
-    red.set(f'{function}.last_success', int(time.time()))
+    metrics.send(f"{function}.success", "counter", 1)
-    metrics.send(f"{function}.success", 'counter', 1)
+    return log_data
+
+
+@celery.task(soft_time_limit=3600)
+def enable_autorotate_for_certs_attached_to_endpoint():
+    """
+    This celery task automatically enables autorotation for unexpired certificates that are
+    attached to an endpoint but do not have autorotate enabled.
+    :return:
+    """
+    function = f"{__name__}.{sys._getframe().f_code.co_name}"
+    task_id = None
+    if celery.current_task:
+        task_id = celery.current_task.request.id
+
+    log_data = {
+        "function": function,
+        "task_id": task_id,
+        "message": "Enabling autorotate to eligible certificates",
+    }
+    current_app.logger.debug(log_data)
+
+    cli_certificate.automatically_enable_autorotate()
+    metrics.send(f"{function}.success", "counter", 1)
+    return log_data

lemur/common/defaults.py

@@ -2,6 +2,7 @@ import re
 import unicodedata
 
 from cryptography import x509
+from cryptography.hazmat.primitives.serialization import Encoding
 from flask import current_app
 
 from lemur.common.utils import is_selfsigned
@@ -71,12 +72,20 @@ def common_name(cert):
     :return: Common name or None
     """
     try:
-        return cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)[
+        subject_oid = cert.subject.get_attributes_for_oid(x509.OID_COMMON_NAME)
-            0
+        if len(subject_oid) > 0:
-        ].value.strip()
+            return subject_oid[0].value.strip()
+        return None
     except Exception as e:
         sentry.captureException()
-        current_app.logger.error("Unable to get common name! {0}".format(e))
+        current_app.logger.error(
+            {
+                "message": "Unable to get common name",
+                "error": e,
+                "public_key": cert.public_bytes(Encoding.PEM).decode("utf-8")
+            },
+            exc_info=True
+        )
 
 
 def organization(cert):

lemur/common/validators.py

@@ -99,8 +99,12 @@ def csr(data):
         raise ValidationError("CSR presented is not valid.")
 
     # Validate common name and SubjectAltNames
+    try:
         for name in request.subject.get_attributes_for_oid(NameOID.COMMON_NAME):
             common_name(name.value)
+    except ValueError as err:
+        current_app.logger.info("Error parsing Subject from CSR: %s", err)
+        raise ValidationError("Invalid Subject value in supplied CSR")
 
     try:
         alt_names = request.extensions.get_extension_for_class(
@@ -148,6 +152,18 @@ def dates(data):
                         data["authority"].authority_certificate.not_after
                     )
                 )
+            # Allow no more than PUBLIC_CA_MAX_VALIDITY_DAYS (Default: 397) days of validity
+            # for certs issued by public CA
+            # The list of public issuers can be managed through a config named PUBLIC_CA
+            public_CA = current_app.config.get("PUBLIC_CA_AUTHORITY_NAMES", [])
+            if data["authority"].name.lower() in [ca.lower() for ca in public_CA]:
+                max_validity_days = current_app.config.get("PUBLIC_CA_MAX_VALIDITY_DAYS", 397)
+                if (
+                    (data.get("validity_end").date() - data.get("validity_start").date()).days
+                    > max_validity_days
+                ):
+                    raise ValidationError("Certificate cannot be valid for more than " +
+                                          str(max_validity_days) + " days")
 
     return data
 

lemur/dns_providers/util.py

@@ -31,11 +31,11 @@ class DNSResolveError(DNSError):
 
 def is_valid_domain(domain):
     """Checks if a domain is syntactically valid and returns a bool"""
-    if len(domain) > 253:
-        return False
     if domain[-1] == ".":
         domain = domain[:-1]
-    fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(?<!-)\.?)+(?:[a-zA-Z]{2,})$)", re.IGNORECASE)
+    if len(domain) > 253:
+        return False
+    fqdn_re = re.compile("(?=^.{1,63}$)(^(?:[a-z0-9_](?:-*[a-z0-9_])+)$|^[a-z0-9]$)", re.IGNORECASE)
     return all(fqdn_re.match(d) for d in domain.split("."))
 
 

lemur/migrations/versions/8323a5ea723a_.py

@@ -0,0 +1,50 @@
+"""Add lowercase index for certificate name and cn and also for domain name
+
+Revision ID: 8323a5ea723a
+Revises: b33c838cb669
+Create Date: 2020-01-10 10:51:44.776052
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '8323a5ea723a'
+down_revision = 'b33c838cb669'
+
+from alembic import op
+from sqlalchemy import text
+
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_index(
+        "ix_certificates_cn_lower",
+        "certificates",
+        [text("lower(cn)")],
+        unique=False,
+        postgresql_ops={"lower(cn)": "gin_trgm_ops"},
+        postgresql_using="gin",
+    )
+    op.create_index(
+        "ix_certificates_name_lower",
+        "certificates",
+        [text("lower(name)")],
+        unique=False,
+        postgresql_ops={"lower(name)": "gin_trgm_ops"},
+        postgresql_using="gin",
+    )
+    op.create_index(
+        "ix_domains_name_lower",
+        "domains",
+        [text("lower(name)")],
+        unique=False,
+        postgresql_ops={"lower(name)": "gin_trgm_ops"},
+        postgresql_using="gin",
+    )
+
+
+
+def downgrade():
+    op.drop_index("ix_certificates_cn_lower", table_name="certificates")
+    op.drop_index("ix_certificates_name_lower", table_name="certificates")
+    op.drop_index("ix_domains_name_lower", table_name="domains")

lemur/migrations/versions/ee827d1e1974_.py

@@ -45,6 +45,6 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_index("ix_domains_name", table_name="domains")
+    op.drop_index("ix_domains_name_gin", table_name="domains")
     op.drop_index("ix_certificates_name", table_name="certificates")
     op.drop_index("ix_certificates_cn", table_name="certificates")

lemur/plugins/lemur_acme/plugin.py

@@ -54,18 +54,30 @@ class AcmeHandler(object):
             current_app.logger.error(f"Unable to fetch DNS Providers: {e}")
             self.all_dns_providers = []
 
-    def find_dns_challenge(self, host, authorizations):
+    def get_dns_challenges(self, host, authorizations):
+        """Get dns challenges for provided domain"""
+
+        domain_to_validate, is_wildcard = self.strip_wildcard(host)
         dns_challenges = []
         for authz in authorizations:
-            if not authz.body.identifier.value.lower() == host.lower():
+            if not authz.body.identifier.value.lower() == domain_to_validate.lower():
+                continue
+            if is_wildcard and not authz.body.wildcard:
+                continue
+            if not is_wildcard and authz.body.wildcard:
                 continue
             for combo in authz.body.challenges:
                 if isinstance(combo.chall, challenges.DNS01):
                     dns_challenges.append(combo)
+
         return dns_challenges
 
-    def maybe_remove_wildcard(self, host):
+    def strip_wildcard(self, host):
-        return host.replace("*.", "")
+        """Removes the leading *. and returns Host and whether it was removed or not (True/False)"""
+        prefix = "*."
+        if host.startswith(prefix):
+            return host[len(prefix):], True
+        return host, False
 
     def maybe_add_extension(self, host, dns_provider_options):
         if dns_provider_options and dns_provider_options.get(
@@ -86,9 +98,8 @@ class AcmeHandler(object):
         current_app.logger.debug("Starting DNS challenge for {0}".format(host))
 
         change_ids = []
-
+        dns_challenges = self.get_dns_challenges(host, order.authorizations)
-        host_to_validate = self.maybe_remove_wildcard(host)
+        host_to_validate, _ = self.strip_wildcard(host)
-        dns_challenges = self.find_dns_challenge(host_to_validate, order.authorizations)
         host_to_validate = self.maybe_add_extension(
             host_to_validate, dns_provider_options
         )
@@ -172,7 +183,7 @@ class AcmeHandler(object):
 
         except (AcmeError, TimeoutError):
             sentry.captureException(extra={"order_url": str(order.uri)})
-            metrics.send("request_certificate_error", "counter", 1)
+            metrics.send("request_certificate_error", "counter", 1, metric_tags={"uri": order.uri})
             current_app.logger.error(
                 f"Unable to resolve Acme order: {order.uri}", exc_info=True
             )
@@ -183,12 +194,23 @@ class AcmeHandler(object):
             else:
                 raise
 
+        metrics.send("request_certificate_success", "counter", 1, metric_tags={"uri": order.uri})
+        current_app.logger.info(
+            f"Successfully resolved Acme order: {order.uri}", exc_info=True
+        )
+
         pem_certificate = OpenSSL.crypto.dump_certificate(
             OpenSSL.crypto.FILETYPE_PEM,
             OpenSSL.crypto.load_certificate(
                 OpenSSL.crypto.FILETYPE_PEM, orderr.fullchain_pem
             ),
         ).decode()
+
+        if current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA", False) \
+                and datetime.datetime.now() < datetime.datetime.strptime(
+                    current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA_EXPIRATION_DATE", "17/03/21"), '%d/%m/%y'):
+            pem_certificate_chain = current_app.config.get("IDENTRUST_CROSS_SIGNED_LE_ICA")
+        else:
             pem_certificate_chain = orderr.fullchain_pem[
                 len(pem_certificate) :  # noqa
             ].lstrip()
@@ -320,7 +342,7 @@ class AcmeHandler(object):
                     )
                     dns_provider_options = json.loads(dns_provider.credentials)
                     account_number = dns_provider_options.get("account_id")
-                    host_to_validate = self.maybe_remove_wildcard(authz_record.host)
+                    host_to_validate, _ = self.strip_wildcard(authz_record.host)
                     host_to_validate = self.maybe_add_extension(
                         host_to_validate, dns_provider_options
                     )
@@ -352,7 +374,7 @@ class AcmeHandler(object):
                 dns_provider_options = json.loads(dns_provider.credentials)
                 account_number = dns_provider_options.get("account_id")
                 dns_challenges = authz_record.dns_challenge
-                host_to_validate = self.maybe_remove_wildcard(authz_record.host)
+                host_to_validate, _ = self.strip_wildcard(authz_record.host)
                 host_to_validate = self.maybe_add_extension(
                     host_to_validate, dns_provider_options
                 )

lemur/plugins/lemur_acme/powerdns.py

@@ -1,11 +1,10 @@
-import time
-import requests
 import json
 import sys
+import time
 
 import lemur.common.utils as utils
 import lemur.dns_providers.util as dnsutil
-
+import requests
 from flask import current_app
 from lemur.extensions import metrics, sentry
 
@@ -17,7 +16,9 @@ REQUIRED_VARIABLES = [
 
 
 class Zone:
-    """ This class implements a PowerDNS zone in JSON. """
+    """
+    This class implements a PowerDNS zone in JSON.
+    """
 
     def __init__(self, _data):
         self._data = _data
@@ -39,7 +40,9 @@ class Zone:
 
 
 class Record:
-    """ This class implements a PowerDNS record. """
+    """
+    This class implements a PowerDNS record.
+    """
 
     def __init__(self, _data):
         self._data = _data
@@ -49,20 +52,30 @@ class Record:
         return self._data["name"]
 
     @property
-    def disabled(self):
+    def type(self):
-        return self._data["disabled"]
+        return self._data["type"]
+
+    @property
+    def ttl(self):
+        return self._data["ttl"]
 
     @property
     def content(self):
         return self._data["content"]
 
     @property
-    def ttl(self):
+    def disabled(self):
-        return self._data["ttl"]
+        return self._data["disabled"]
 
 
 def get_zones(account_number):
-    """Retrieve authoritative zones from the PowerDNS API and return a list"""
+    """
+    Retrieve authoritative zones from the PowerDNS API and return a list of zones
+
+    :param account_number:
+    :raise: Exception
+    :return: list of Zone Objects
+    """
     _check_conf()
     server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
     path = f"/api/v1/servers/{server_id}/zones"
@@ -90,44 +103,41 @@ def get_zones(account_number):
 
 
 def create_txt_record(domain, token, account_number):
-    """ Create a TXT record for the given domain and token and return a change_id tuple """
+    """
+    Create a TXT record for the given domain and token and return a change_id tuple
+
+    :param domain: FQDN
+    :param token: challenge value
+    :param account_number:
+    :return: tuple of domain/token
+    """
     _check_conf()
-    zone_name = _get_zone_name(domain, account_number)
+
-    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
-    zone_id = zone_name + "."
-    domain_id = domain + "."
-    path = f"/api/v1/servers/{server_id}/zones/{zone_id}"
-    payload = {
-        "rrsets": [
-            {
-                "name": domain_id,
-                "type": "TXT",
-                "ttl": 300,
-                "changetype": "REPLACE",
-                "records": [
-                    {
-                        "content": f"\"{token}\"",
-                        "disabled": False
-                    }
-                ],
-                "comments": []
-            }
-        ]
-    }
     function = sys._getframe().f_code.co_name
     log_data = {
         "function": function,
         "fqdn": domain,
         "token": token,
     }
+
+    # Create new record
+    domain_id = domain + "."
+    records = [Record({'name': domain_id, 'content': f"\"{token}\"", 'disabled': False})]
+
+    # Get current records
+    cur_records = _get_txt_records(domain)
+    for record in cur_records:
+        if record.content != token:
+            records.append(record)
+
     try:
-        _patch(path, payload)
+        _patch_txt_records(domain, account_number, records)
-        log_data["message"] = "TXT record successfully created"
+        log_data["message"] = "TXT record(s) successfully created"
         current_app.logger.debug(log_data)
     except Exception as e:
         sentry.captureException()
         log_data["Exception"] = e
-        log_data["message"] = "Unable to create TXT record"
+        log_data["message"] = "Unable to create TXT record(s)"
         current_app.logger.debug(log_data)
 
     change_id = (domain, token)
@@ -136,8 +146,11 @@ def create_txt_record(domain, token, account_number):
 
 def wait_for_dns_change(change_id, account_number=None):
     """
-    Checks the authoritative DNS Server to see if changes have propagated to DNS
+    Checks the authoritative DNS Server to see if changes have propagated.
-    Retries and waits until successful.
+
+    :param change_id: tuple of domain/token
+    :param account_number:
+    :return:
     """
     _check_conf()
     domain, token = change_id
@@ -171,8 +184,61 @@ def wait_for_dns_change(change_id, account_number=None):
 
 
 def delete_txt_record(change_id, account_number, domain, token):
-    """ Delete the TXT record for the given domain and token """
+    """
+    Delete the TXT record for the given domain and token
+
+    :param change_id: tuple of domain/token
+    :param account_number:
+    :param domain: FQDN
+    :param token: challenge to delete
+    :return:
+    """
     _check_conf()
+
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function,
+        "fqdn": domain,
+        "token": token,
+    }
+
+    """
+    Get existing TXT records matching the domain from DNS
+    The token to be deleted should already exist
+    There may be other records with different tokens as well
+    """
+    cur_records = _get_txt_records(domain)
+    found = False
+    new_records = []
+    for record in cur_records:
+        if record.content == f"\"{token}\"":
+            found = True
+        else:
+            new_records.append(record)
+
+    # Since the matching token is not in DNS, there is nothing to delete
+    if not found:
+        log_data["message"] = "Unable to delete TXT record: Token not found in existing TXT records"
+        current_app.logger.debug(log_data)
+        return
+
+    # The record to delete has been found AND there are other tokens set on the same domain
+    # Since we only want to delete one token value from the RRSet, we need to use the Patch command to
+    # overwrite the current RRSet with the existing records.
+    elif new_records:
+        try:
+            _patch_txt_records(domain, account_number, new_records)
+            log_data["message"] = "TXT record successfully deleted"
+            current_app.logger.debug(log_data)
+        except Exception as e:
+            sentry.captureException()
+            log_data["Exception"] = e
+            log_data["message"] = "Unable to delete TXT record: patching exception"
+            current_app.logger.debug(log_data)
+
+    # The record to delete has been found AND there are no other token values set on the same domain
+    # Use the Delete command to delete the whole RRSet.
+    else:
         zone_name = _get_zone_name(domain, account_number)
         server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
         zone_id = zone_name + "."
@@ -213,11 +279,20 @@ def delete_txt_record(change_id, account_number, domain, token):
 
 
 def _check_conf():
+    """
+    Verifies required configuration variables are set
+
+    :return:
+    """
     utils.validate_conf(current_app, REQUIRED_VARIABLES)
 
 
 def _generate_header():
-    """Generate a PowerDNS API header and return it as a dictionary"""
+    """
+    Generate a PowerDNS API header and return it as a dictionary
+
+    :return: Dict of header parameters
+    """
     api_key_name = current_app.config.get("ACME_POWERDNS_APIKEYNAME")
     api_key = current_app.config.get("ACME_POWERDNS_APIKEY")
     headers = {api_key_name: api_key}
@@ -225,7 +300,13 @@ def _generate_header():
 
 
 def _get_zone_name(domain, account_number):
-    """Get most specific matching zone for the given domain and return as a String"""
+    """
+    Get most specific matching zone for the given domain and return as a String
+
+    :param domain: FQDN
+    :param account_number:
+    :return: FQDN of domain
+    """
     zones = get_zones(account_number)
     zone_name = ""
     for z in zones:
@@ -243,25 +324,113 @@ def _get_zone_name(domain, account_number):
     return zone_name
 
 
+def _get_txt_records(domain):
+    """
+    Retrieve TXT records for a given domain and return list of Record Objects
+
+    :param domain: FQDN
+    :return: list of Record objects
+    """
+    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
+
+    path = f"/api/v1/servers/{server_id}/search-data?q={domain}&max=100&object_type=record"
+    function = sys._getframe().f_code.co_name
+    log_data = {
+        "function": function
+    }
+    try:
+        records = _get(path)
+        log_data["message"] = "Retrieved TXT Records Successfully"
+        current_app.logger.debug(log_data)
+
+    except Exception as e:
+        sentry.captureException()
+        log_data["Exception"] = e
+        log_data["message"] = "Failed to Retrieve TXT Records"
+        current_app.logger.debug(log_data)
+        return []
+
+    txt_records = []
+    for record in records:
+        cur_record = Record(record)
+        txt_records.append(cur_record)
+    return txt_records
+
+
 def _get(path, params=None):
-    """ Execute a GET request on the given URL (base_uri + path) and return response as JSON object """
+    """
+    Execute a GET request on the given URL (base_uri + path) and return response as JSON object
+
+    :param path: Relative URL path
+    :param params: additional parameters
+    :return: json response
+    """
     base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN")
+    verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True)
     resp = requests.get(
         f"{base_uri}{path}",
         headers=_generate_header(),
         params=params,
-        verify=True,
+        verify=verify_value
     )
     resp.raise_for_status()
     return resp.json()
 
 
+def _patch_txt_records(domain, account_number, records):
+    """
+    Send Patch request to PowerDNS Server
+
+    :param domain: FQDN
+    :param account_number:
+    :param records: List of Record objects
+    :return:
+    """
+    domain_id = domain + "."
+
+    # Create records
+    txt_records = []
+    for record in records:
+        txt_records.append(
+            {'content': record.content, 'disabled': record.disabled}
+        )
+
+    # Create RRSet
+    payload = {
+        "rrsets": [
+            {
+                "name": domain_id,
+                "type": "TXT",
+                "ttl": 300,
+                "changetype": "REPLACE",
+                "records": txt_records,
+                "comments": []
+            }
+        ]
+    }
+
+    # Create Txt Records
+    server_id = current_app.config.get("ACME_POWERDNS_SERVERID", "localhost")
+    zone_name = _get_zone_name(domain, account_number)
+    zone_id = zone_name + "."
+    path = f"/api/v1/servers/{server_id}/zones/{zone_id}"
+    _patch(path, payload)
+
+
 def _patch(path, payload):
-    """ Execute a Patch request on the given URL (base_uri + path) with given payload """
+    """
+    Execute a Patch request on the given URL (base_uri + path) with given payload
+
+    :param path:
+    :param payload:
+    :return:
+    """
     base_uri = current_app.config.get("ACME_POWERDNS_DOMAIN")
+    verify_value = current_app.config.get("ACME_POWERDNS_VERIFY", True)
     resp = requests.patch(
         f"{base_uri}{path}",
         data=json.dumps(payload),
-        headers=_generate_header()
+        headers=_generate_header(),
+        verify=verify_value
     )
     resp.raise_for_status()

lemur/plugins/lemur_acme/route53.py

@@ -35,6 +35,7 @@ def get_zones(client=None):
     zones = []
     for page in paginator.paginate():
         for zone in page["HostedZones"]:
+            if not zone["Config"]["PrivateZone"]:
                 zones.append(
                     zone["Name"][:-1]
                 )  # We need [:-1] to strip out the trailing dot.

lemur/plugins/lemur_acme/tests/test_acme.py

@@ -1,11 +1,9 @@
 import unittest
+from unittest.mock import patch, Mock
 
 from cryptography.x509 import DNSName
-from requests.models import Response
+from lemur.plugins.lemur_acme import plugin
-
+from mock import MagicMock
-from mock import MagicMock, Mock, patch
-
-from lemur.plugins.lemur_acme import plugin, ultradns
 
 
 class TestAcme(unittest.TestCase):
@@ -23,11 +21,12 @@ class TestAcme(unittest.TestCase):
         }
 
     @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
-    def test_find_dns_challenge(self, mock_len):
+    def test_get_dns_challenges(self, mock_len):
         assert mock_len
 
         from acme import challenges
 
+        host = "example.com"
         c = challenges.DNS01()
 
         mock_authz = Mock()
@@ -35,9 +34,18 @@ class TestAcme(unittest.TestCase):
         mock_entry = Mock()
         mock_entry.chall = c
         mock_authz.body.resolved_combinations.append(mock_entry)
-        result = yield self.acme.find_dns_challenge(mock_authz)
+        result = yield self.acme.get_dns_challenges(host, mock_authz)
         self.assertEqual(result, mock_entry)
 
+    def test_strip_wildcard(self):
+        expected = ("example.com", False)
+        result = self.acme.strip_wildcard("example.com")
+        self.assertEqual(expected, result)
+
+        expected = ("example.com", True)
+        result = self.acme.strip_wildcard("*.example.com")
+        self.assertEqual(expected, result)
+
     def test_authz_record(self):
         a = plugin.AuthorizationRecord("host", "authz", "challenge", "id")
         self.assertEqual(type(a), plugin.AuthorizationRecord)
@@ -45,9 +53,9 @@ class TestAcme(unittest.TestCase):
     @patch("acme.client.Client")
     @patch("lemur.plugins.lemur_acme.plugin.current_app")
     @patch("lemur.plugins.lemur_acme.plugin.len", return_value=1)
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
     def test_start_dns_challenge(
-        self, mock_find_dns_challenge, mock_len, mock_app, mock_acme
+            self, mock_get_dns_challenges, mock_len, mock_app, mock_acme
     ):
         assert mock_len
         mock_order = Mock()
@@ -65,7 +73,7 @@ class TestAcme(unittest.TestCase):
         mock_dns_provider.create_txt_record = Mock(return_value=1)
 
         values = [mock_entry]
-        iterable = mock_find_dns_challenge.return_value
+        iterable = mock_get_dns_challenges.return_value
         iterator = iter(values)
         iterable.__iter__.return_value = iterator
         result = self.acme.start_dns_challenge(
@@ -127,12 +135,12 @@ class TestAcme(unittest.TestCase):
     @patch("acme.client.Client")
     @patch("OpenSSL.crypto", return_value="mock_cert")
     @patch("josepy.util.ComparableX509")
-    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.find_dns_challenge")
+    @patch("lemur.plugins.lemur_acme.plugin.AcmeHandler.get_dns_challenges")
     @patch("lemur.plugins.lemur_acme.plugin.current_app")
     def test_request_certificate(
             self,
             mock_current_app,
-        mock_find_dns_challenge,
+            mock_get_dns_challenges,
             mock_jose,
             mock_crypto,
             mock_acme,
@@ -148,6 +156,7 @@ class TestAcme(unittest.TestCase):
         mock_acme.fetch_chain = Mock(return_value="mock_chain")
         mock_crypto.dump_certificate = Mock(return_value=b"chain")
         mock_order = Mock()
+        mock_current_app.config = {}
         self.acme.request_certificate(mock_acme, [], mock_order)
 
     def test_setup_acme_client_fail(self):
@@ -172,7 +181,7 @@ class TestAcme(unittest.TestCase):
         assert result_client
         assert result_registration
 
-    @patch("lemur.plugins.lemur_acme.plugin.current_app")
+    @patch('lemur.plugins.lemur_acme.plugin.current_app')
     def test_get_domains_single(self, mock_current_app):
         options = {"common_name": "test.netflix.net"}
         result = self.acme.get_domains(options)
@@ -378,121 +387,3 @@ class TestAcme(unittest.TestCase):
         mock_request_certificate.return_value = ("pem_certificate", "chain")
         result = provider.create_certificate(csr, issuer_options)
         assert result
-
-    @patch("lemur.plugins.lemur_acme.ultradns.requests")
-    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    def test_ultradns_get_token(self, mock_current_app, mock_requests):
-        # ret_val = json.dumps({"access_token": "access"})
-        the_response = Response()
-        the_response._content = b'{"access_token": "access"}'
-        mock_requests.post = Mock(return_value=the_response)
-        mock_current_app.config.get = Mock(return_value="Test")
-        result = ultradns.get_ultradns_token()
-        self.assertTrue(len(result) > 0)
-
-    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    def test_ultradns_create_txt_record(self, mock_current_app):
-        domain = "_acme_challenge.test.example.com"
-        zone = "test.example.com"
-        token = "ABCDEFGHIJ"
-        account_number = "1234567890"
-        change_id = (domain, token)
-        ultradns.get_zone_name = Mock(return_value=zone)
-        mock_current_app.logger.debug = Mock()
-        ultradns._post = Mock()
-        log_data = {
-            "function": "create_txt_record",
-            "fqdn": domain,
-            "token": token,
-            "message": "TXT record created"
-        }
-        result = ultradns.create_txt_record(domain, token, account_number)
-        mock_current_app.logger.debug.assert_called_with(log_data)
-        self.assertEqual(result, change_id)
-
-    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    @patch("lemur.extensions.metrics")
-    def test_ultradns_delete_txt_record(self, mock_metrics, mock_current_app):
-        domain = "_acme_challenge.test.example.com"
-        zone = "test.example.com"
-        token = "ABCDEFGHIJ"
-        account_number = "1234567890"
-        change_id = (domain, token)
-        mock_current_app.logger.debug = Mock()
-        ultradns.get_zone_name = Mock(return_value=zone)
-        ultradns._post = Mock()
-        ultradns._get = Mock()
-        ultradns._get.return_value = {'zoneName': 'test.example.com.com',
-                'rrSets': [{'ownerName': '_acme-challenge.test.example.com.',
-                            'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}],
-                'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100},
-                'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}}
-        ultradns._delete = Mock()
-        mock_metrics.send = Mock()
-        ultradns.delete_txt_record(change_id, account_number, domain, token)
-        mock_current_app.logger.debug.assert_not_called()
-        mock_metrics.send.assert_not_called()
-
-    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
-    @patch("lemur.extensions.metrics")
-    def test_ultradns_wait_for_dns_change(self, mock_metrics, mock_current_app):
-        ultradns._has_dns_propagated = Mock(return_value=True)
-        nameserver = "1.1.1.1"
-        ultradns.get_authoritative_nameserver = Mock(return_value=nameserver)
-        mock_metrics.send = Mock()
-        domain = "_acme-challenge.test.example.com"
-        token = "ABCDEFGHIJ"
-        change_id = (domain, token)
-        mock_current_app.logger.debug = Mock()
-        ultradns.wait_for_dns_change(change_id)
-        # mock_metrics.send.assert_not_called()
-        log_data = {
-            "function": "wait_for_dns_change",
-            "fqdn": domain,
-            "status": True,
-            "message": "Record status on Public DNS"
-        }
-        mock_current_app.logger.debug.assert_called_with(log_data)
-
-    def test_ultradns_get_zone_name(self):
-        zones = ['example.com', 'test.example.com']
-        zone = "test.example.com"
-        domain = "_acme-challenge.test.example.com"
-        account_number = "1234567890"
-        ultradns.get_zones = Mock(return_value=zones)
-        result = ultradns.get_zone_name(domain, account_number)
-        self.assertEqual(result, zone)
-
-    def test_ultradns_get_zones(self):
-        account_number = "1234567890"
-        path = "a/b/c"
-        zones = ['example.com', 'test.example.com']
-        paginate_response = [{
-            'properties': {
-                'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
-                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                'lastModifiedDateTime': '2017-06-14T06:45Z'},
-            'registrarInfo': {
-                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
-            'inherit': 'ALL'}, {
-            'properties': {
-                'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
-                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                'lastModifiedDateTime': '2017-06-14T06:45Z'},
-            'registrarInfo': {
-                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
-            'inherit': 'ALL'}, {
-            'properties': {
-                'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY',
-                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
-                'lastModifiedDateTime': '2017-06-14T06:45Z'},
-            'registrarInfo': {
-                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
-                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
-            'inherit': 'ALL'}]
-        ultradns._paginate = Mock(path, "zones")
-        ultradns._paginate.side_effect = [[paginate_response]]
-        result = ultradns.get_zones(account_number)
-        self.assertEqual(result, zones)

lemur/plugins/lemur_acme/tests/test_powerdns.py

@@ -1,5 +1,5 @@
 import unittest
-from mock import Mock, patch
+from unittest.mock import patch, Mock
 from lemur.plugins.lemur_acme import plugin, powerdns
 
 
@@ -48,13 +48,14 @@ class TestPowerdns(unittest.TestCase):
         self.assertEqual(result, zone)
 
     @patch("lemur.plugins.lemur_acme.powerdns.current_app")
-    def test_create_txt_record(self, mock_current_app):
+    def test_create_txt_record_write_only(self, mock_current_app):
         domain = "_acme_challenge.test.example.com"
         zone = "test.example.com"
         token = "ABCDEFGHIJ"
         account_number = "1234567890"
         change_id = (domain, token)
         powerdns._check_conf = Mock()
+        powerdns._get_txt_records = Mock(return_value=[])
         powerdns._get_zone_name = Mock(return_value=zone)
         mock_current_app.logger.debug = Mock()
         mock_current_app.config.get = Mock(return_value="localhost")
@@ -63,24 +64,74 @@ class TestPowerdns(unittest.TestCase):
             "function": "create_txt_record",
             "fqdn": domain,
             "token": token,
-            "message": "TXT record successfully created"
+            "message": "TXT record(s) successfully created"
         }
         result = powerdns.create_txt_record(domain, token, account_number)
         mock_current_app.logger.debug.assert_called_with(log_data)
         self.assertEqual(result, change_id)
 
+    @patch("lemur.plugins.lemur_acme.powerdns.current_app")
+    def test_create_txt_record_append(self, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        powerdns._check_conf = Mock()
+        cur_token = "123456"
+        cur_records = [powerdns.Record({'name': domain, 'content': f"\"{cur_token}\"", 'disabled': False})]
+        powerdns._get_txt_records = Mock(return_value=cur_records)
+        powerdns._get_zone_name = Mock(return_value=zone)
+        mock_current_app.logger.debug = Mock()
+        mock_current_app.config.get = Mock(return_value="localhost")
+        powerdns._patch = Mock()
+        log_data = {
+            "function": "create_txt_record",
+            "fqdn": domain,
+            "token": token,
+            "message": "TXT record(s) successfully created"
+        }
+        expected_path = "/api/v1/servers/localhost/zones/test.example.com."
+        expected_payload = {
+            "rrsets": [
+                {
+                    "name": domain + ".",
+                    "type": "TXT",
+                    "ttl": 300,
+                    "changetype": "REPLACE",
+                    "records": [
+                        {
+                            "content": f"\"{token}\"",
+                            "disabled": False
+                        },
+                        {
+                            "content": f"\"{cur_token}\"",
+                            "disabled": False
+                        }
+                    ],
+                    "comments": []
+                }
+            ]
+        }
+
+        result = powerdns.create_txt_record(domain, token, account_number)
+        mock_current_app.logger.debug.assert_called_with(log_data)
+        powerdns._patch.assert_called_with(expected_path, expected_payload)
+        self.assertEqual(result, change_id)
+
     @patch("lemur.plugins.lemur_acme.powerdns.dnsutil")
     @patch("lemur.plugins.lemur_acme.powerdns.current_app")
     @patch("lemur.extensions.metrics")
     @patch("time.sleep")
     def test_wait_for_dns_change(self, mock_sleep, mock_metrics, mock_current_app, mock_dnsutil):
         domain = "_acme-challenge.test.example.com"
-        token = "ABCDEFG"
+        token1 = "ABCDEFG"
+        token2 = "HIJKLMN"
         zone_name = "test.example.com"
         nameserver = "1.1.1.1"
-        change_id = (domain, token)
+        change_id = (domain, token1)
         powerdns._check_conf = Mock()
-        mock_records = (token,)
+        mock_records = (token2, token1)
         mock_current_app.config.get = Mock(return_value=1)
         powerdns._get_zone_name = Mock(return_value=zone_name)
         mock_dnsutil.get_authoritative_nameserver = Mock(return_value=nameserver)
@@ -114,7 +165,7 @@ class TestPowerdns(unittest.TestCase):
             "function": "delete_txt_record",
             "fqdn": domain,
             "token": token,
-            "message": "TXT record successfully deleted"
+            "message": "Unable to delete TXT record: Token not found in existing TXT records"
         }
         powerdns.delete_txt_record(change_id, account_number, domain, token)
         mock_current_app.logger.debug.assert_called_with(log_data)

lemur/plugins/lemur_acme/tests/test_ultradns.py

@@ -0,0 +1,138 @@
+import unittest
+from unittest.mock import patch, Mock
+
+from lemur.plugins.lemur_acme import plugin, ultradns
+from requests.models import Response
+
+
+class TestUltradns(unittest.TestCase):
+    @patch("lemur.plugins.lemur_acme.plugin.dns_provider_service")
+    def setUp(self, mock_dns_provider_service):
+        self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
+        self.acme = plugin.AcmeHandler()
+        mock_dns_provider = Mock()
+        mock_dns_provider.name = "cloudflare"
+        mock_dns_provider.credentials = "{}"
+        mock_dns_provider.provider_type = "cloudflare"
+        self.acme.dns_providers_for_domain = {
+            "www.test.com": [mock_dns_provider],
+            "test.fakedomain.net": [mock_dns_provider],
+        }
+
+    @patch("lemur.plugins.lemur_acme.ultradns.requests")
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    def test_ultradns_get_token(self, mock_current_app, mock_requests):
+        # ret_val = json.dumps({"access_token": "access"})
+        the_response = Response()
+        the_response._content = b'{"access_token": "access"}'
+        mock_requests.post = Mock(return_value=the_response)
+        mock_current_app.config.get = Mock(return_value="Test")
+        result = ultradns.get_ultradns_token()
+        self.assertTrue(len(result) > 0)
+
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    def test_ultradns_create_txt_record(self, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        ultradns.get_zone_name = Mock(return_value=zone)
+        mock_current_app.logger.debug = Mock()
+        ultradns._post = Mock()
+        log_data = {
+            "function": "create_txt_record",
+            "fqdn": domain,
+            "token": token,
+            "message": "TXT record created"
+        }
+        result = ultradns.create_txt_record(domain, token, account_number)
+        mock_current_app.logger.debug.assert_called_with(log_data)
+        self.assertEqual(result, change_id)
+
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    @patch("lemur.extensions.metrics")
+    def test_ultradns_delete_txt_record(self, mock_metrics, mock_current_app):
+        domain = "_acme_challenge.test.example.com"
+        zone = "test.example.com"
+        token = "ABCDEFGHIJ"
+        account_number = "1234567890"
+        change_id = (domain, token)
+        mock_current_app.logger.debug = Mock()
+        ultradns.get_zone_name = Mock(return_value=zone)
+        ultradns._post = Mock()
+        ultradns._get = Mock()
+        ultradns._get.return_value = {'zoneName': 'test.example.com.com',
+                                      'rrSets': [{'ownerName': '_acme-challenge.test.example.com.',
+                                                  'rrtype': 'TXT (16)', 'ttl': 5, 'rdata': ['ABCDEFGHIJ']}],
+                                      'queryInfo': {'sort': 'OWNER', 'reverse': False, 'limit': 100},
+                                      'resultInfo': {'totalCount': 1, 'offset': 0, 'returnedCount': 1}}
+        ultradns._delete = Mock()
+        mock_metrics.send = Mock()
+        ultradns.delete_txt_record(change_id, account_number, domain, token)
+        mock_current_app.logger.debug.assert_not_called()
+        mock_metrics.send.assert_not_called()
+
+    @patch("lemur.plugins.lemur_acme.ultradns.current_app")
+    @patch("lemur.extensions.metrics")
+    def test_ultradns_wait_for_dns_change(self, mock_metrics, mock_current_app):
+        ultradns._has_dns_propagated = Mock(return_value=True)
+        nameserver = "1.1.1.1"
+        ultradns.get_authoritative_nameserver = Mock(return_value=nameserver)
+        mock_metrics.send = Mock()
+        domain = "_acme-challenge.test.example.com"
+        token = "ABCDEFGHIJ"
+        change_id = (domain, token)
+        mock_current_app.logger.debug = Mock()
+        ultradns.wait_for_dns_change(change_id)
+        # mock_metrics.send.assert_not_called()
+        log_data = {
+            "function": "wait_for_dns_change",
+            "fqdn": domain,
+            "status": True,
+            "message": "Record status on Public DNS"
+        }
+        mock_current_app.logger.debug.assert_called_with(log_data)
+
+    def test_ultradns_get_zone_name(self):
+        zones = ['example.com', 'test.example.com']
+        zone = "test.example.com"
+        domain = "_acme-challenge.test.example.com"
+        account_number = "1234567890"
+        ultradns.get_zones = Mock(return_value=zones)
+        result = ultradns.get_zone_name(domain, account_number)
+        self.assertEqual(result, zone)
+
+    def test_ultradns_get_zones(self):
+        account_number = "1234567890"
+        path = "a/b/c"
+        zones = ['example.com', 'test.example.com']
+        paginate_response = [{
+            'properties': {
+                'name': 'example.com.', 'accountName': 'example', 'type': 'PRIMARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}, {
+            'properties': {
+                'name': 'test.example.com.', 'accountName': 'example', 'type': 'PRIMARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}, {
+            'properties': {
+                'name': 'example2.com.', 'accountName': 'example', 'type': 'SECONDARY',
+                'dnssecStatus': 'UNSIGNED', 'status': 'ACTIVE', 'resourceRecordCount': 9,
+                'lastModifiedDateTime': '2017-06-14T06:45Z'},
+            'registrarInfo': {
+                'nameServers': {'missing': ['example.ultradns.com.', 'example.ultradns.net.',
+                                            'example.ultradns.biz.', 'example.ultradns.org.']}},
+            'inherit': 'ALL'}]
+        ultradns._paginate = Mock(path, "zones")
+        ultradns._paginate.side_effect = [[paginate_response]]
+        result = ultradns.get_zones(account_number)
+        self.assertEqual(result, zones)

lemur/plugins/lemur_aws/iam.py

@@ -24,6 +24,12 @@ def retry_throttled(exception):
         if exception.response["Error"]["Code"] == "NoSuchEntity":
             return False
 
+        # No need to retry deletion requests if there is a DeleteConflict error.
+        # This error indicates that the certificate is still attached to an entity
+        # and cannot be deleted.
+        if exception.response["Error"]["Code"] == "DeleteConflict":
+            return False
+
     metrics.send("iam_retry", "counter", 1, metric_tags={"exception": str(exception)})
     return True
 

lemur/plugins/lemur_aws/plugin.py

@@ -216,22 +216,24 @@ class AWSSourcePlugin(SourcePlugin):
 
         for region in regions:
             elbs = elb.get_all_elbs(account_number=account_number, region=region)
-            current_app.logger.info(
+            current_app.logger.info({
-                "Describing classic load balancers in {0}-{1}".format(
+                "message": "Describing classic load balancers",
-                    account_number, region
+                "account_number": account_number,
-                )
+                "region": region,
-            )
+                "number_of_load_balancers": len(elbs)
+            })
 
             for e in elbs:
                 endpoints.extend(get_elb_endpoints(account_number, region, e))
 
             # fetch advanced ELBs
             elbs_v2 = elb.get_all_elbs_v2(account_number=account_number, region=region)
-            current_app.logger.info(
+            current_app.logger.info({
-                "Describing advanced load balancers in {0}-{1}".format(
+                "message": "Describing advanced load balancers",
-                    account_number, region
+                "account_number": account_number,
-                )
+                "region": region,
-            )
+                "number_of_load_balancers": len(elbs_v2)
+            })
 
             for e in elbs_v2:
                 endpoints.extend(get_elb_endpoints_v2(account_number, region, e))
@@ -325,6 +327,7 @@ class AWSDestinationPlugin(DestinationPlugin):
     ]
 
     def upload(self, name, body, private_key, cert_chain, options, **kwargs):
+        try:
             iam.upload_cert(
                 name,
                 body,
@@ -333,6 +336,8 @@ class AWSDestinationPlugin(DestinationPlugin):
                 cert_chain=cert_chain,
                 account_number=self.get_option("accountNumber", options),
             )
+        except ClientError:
+            sentry.captureException()
 
     def deploy(self, elb_name, account, region, certificate):
         pass

lemur/plugins/lemur_cryptography/plugin.py

@@ -24,6 +24,11 @@ from lemur.certificates.service import create_csr
 def build_certificate_authority(options):
     options["certificate_authority"] = True
     csr, private_key = create_csr(**options)
+
+    if options.get("parent"):
+        # Intermediate Cert Issuance
+        cert_pem, chain_cert_pem = issue_certificate(csr, options, None)
+    else:
         cert_pem, chain_cert_pem = issue_certificate(csr, options, private_key)
 
     return cert_pem, private_key, chain_cert_pem

lemur/plugins/lemur_cryptography/tests/test_cryptography.py

@@ -25,6 +25,31 @@ def test_build_certificate_authority():
     assert chain_cert_pem == ""
 
 
+def test_build_intermediate_certificate_authority(authority):
+    from lemur.plugins.lemur_cryptography.plugin import build_certificate_authority
+
+    options = {
+        "key_type": "RSA2048",
+        "country": "US",
+        "state": "CA",
+        "location": "Example place",
+        "organization": "Example, Inc.",
+        "organizational_unit": "Example Unit",
+        "common_name": "Example INTERMEDIATE",
+        "validity_start": arrow.get("2016-12-01").datetime,
+        "validity_end": arrow.get("2016-12-02").datetime,
+        "first_serial": 1,
+        "serial_number": 1,
+        "owner": "owner@example.com",
+        "parent": authority
+    }
+    cert_pem, private_key_pem, chain_cert_pem = build_certificate_authority(options)
+
+    assert cert_pem
+    assert private_key_pem
+    assert chain_cert_pem == authority.authority_certificate.body
+
+
 def test_issue_certificate(authority):
     from lemur.tests.vectors import CSR_STR
     from lemur.plugins.lemur_cryptography.plugin import issue_certificate

lemur/plugins/lemur_digicert/plugin.py

@@ -14,21 +14,17 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import json
+
 import arrow
-import requests
-
 import pem
-from retrying import retry
+import requests
-
-from flask import current_app
-
 from cryptography import x509
-
+from flask import current_app
-from lemur.extensions import metrics
 from lemur.common.utils import validate_conf
-from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+from lemur.extensions import metrics
-
 from lemur.plugins import lemur_digicert as digicert
+from lemur.plugins.bases import IssuerPlugin, SourcePlugin
+from retrying import retry
 
 
 def log_status_code(r, *args, **kwargs):
@@ -64,24 +60,35 @@ def signature_hash(signing_algorithm):
     raise Exception("Unsupported signing algorithm.")
 
 
-def determine_validity_years(end_date):
+def determine_validity_years(years):
-    """Given an end date determine how many years into the future that date is.
+    """
+    Considering maximum allowed certificate validity period of 397 days, this method should not return
+    more than 1 year of validity. Thus changing it to always return 1.
+    Lemur will change this method in future to handle validity in months (determine_validity_months)
+    instead of years. This will allow flexibility to handle short-lived certificates.
+
+    :param years:
+    :return: 1
+    """
+    return 1
+
+
+def determine_end_date(end_date):
+    """
+    Determine appropriate end date
 
     :param end_date:
-    :return: str validity in years
+    :return: validity_end
     """
-    now = arrow.utcnow()
+    default_days = current_app.config.get("DIGICERT_DEFAULT_VALIDITY_DAYS", 397)
+    max_validity_end = arrow.utcnow().shift(days=current_app.config.get("DIGICERT_MAX_VALIDITY_DAYS", default_days))
 
-    if end_date < now.shift(years=+1):
+    if not end_date:
-        return 1
+        end_date = arrow.utcnow().shift(days=default_days)
-    elif end_date < now.shift(years=+2):
-        return 2
-    elif end_date < now.shift(years=+3):
-        return 3
 
-    raise Exception(
+    if end_date > max_validity_end:
-        "DigiCert issued certificates cannot exceed three" " years in validity"
+        end_date = max_validity_end
-    )
+    return end_date
 
 
 def get_additional_names(options):
@@ -107,12 +114,6 @@ def map_fields(options, csr):
     :param csr:
     :return: dict or valid DigiCert options
     """
-    if not options.get("validity_years"):
-        if not options.get("validity_end"):
-            options["validity_years"] = current_app.config.get(
-                "DIGICERT_DEFAULT_VALIDITY", 1
-            )
-
     data = dict(
         certificate={
             "common_name": options["common_name"],
@@ -125,9 +126,11 @@ def map_fields(options, csr):
     data["certificate"]["dns_names"] = get_additional_names(options)
 
     if options.get("validity_years"):
-        data["validity_years"] = options["validity_years"]
+        data["validity_years"] = determine_validity_years(options.get("validity_years"))
+    elif options.get("validity_end"):
+        data["custom_expiration_date"] = determine_end_date(options.get("validity_end")).format("YYYY-MM-DD")
     else:
-        data["custom_expiration_date"] = options["validity_end"].format("YYYY-MM-DD")
+        data["validity_years"] = determine_validity_years(0)
 
     if current_app.config.get("DIGICERT_PRIVATE", False):
         if "product" in data:
@@ -144,18 +147,15 @@ def map_cis_fields(options, csr):
 
     :param options:
     :param csr:
-    :return:
+    :return: data
     """
-    if not options.get("validity_years"):
+
-        if not options.get("validity_end"):
+    if options.get("validity_years"):
-            options["validity_end"] = arrow.utcnow().shift(
+        validity_end = determine_end_date(arrow.utcnow().shift(years=options["validity_years"]))
-                years=current_app.config.get("DIGICERT_DEFAULT_VALIDITY", 1)
+    elif options.get("validity_end"):
-            )
+        validity_end = determine_end_date(options.get("validity_end"))
-        options["validity_years"] = determine_validity_years(options["validity_end"])
     else:
-        options["validity_end"] = arrow.utcnow().shift(
+        validity_end = determine_end_date(False)
-            years=options["validity_years"]
-        )
 
     data = {
         "profile_name": current_app.config.get("DIGICERT_CIS_PROFILE_NAMES", {}).get(options['authority'].name),
@@ -164,7 +164,7 @@ def map_cis_fields(options, csr):
         "csr": csr,
         "signature_hash": signature_hash(options.get("signing_algorithm")),
         "validity": {
-            "valid_to": options["validity_end"].format("YYYY-MM-DDTHH:MM") + "Z"
+            "valid_to": validity_end.format("YYYY-MM-DDTHH:MM") + "Z"
         },
         "organization": {
             "name": options["organization"],
@@ -173,7 +173,8 @@ def map_cis_fields(options, csr):
     }
     #  possibility to default to a SIGNING_ALGORITHM for a given profile
     if current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name):
-        data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(options['authority'].name)
+        data["signature_hash"] = current_app.config.get("DIGICERT_CIS_SIGNING_ALGORITHMS", {}).get(
+            options['authority'].name)
 
     return data
 

lemur/plugins/lemur_digicert/tests/test_digicert.py

@@ -1,20 +1,78 @@
-import pytest
-import arrow
 import json
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
+import arrow
+import pytest
+from cryptography import x509
 from freezegun import freeze_time
-
+from lemur.plugins.lemur_digicert import plugin
 from lemur.tests.vectors import CSR_STR
 
-from cryptography import x509
+
+def config_mock(*args):
+    values = {
+        "DIGICERT_ORG_ID": 111111,
+        "DIGICERT_PRIVATE": False,
+        "DIGICERT_DEFAULT_SIGNING_ALGORITHM": "sha256",
+        "DIGICERT_CIS_PROFILE_NAMES": {"digicert": 'digicert'},
+        "DIGICERT_CIS_SIGNING_ALGORITHMS": {"digicert": 'digicert'},
+    }
+    return values[args[0]]
 
 
-def test_map_fields_with_validity_end_and_start(app):
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
-    from lemur.plugins.lemur_digicert.plugin import map_fields
+def test_determine_validity_years(mock_current_app):
+    assert plugin.determine_validity_years(1) == 1
+    assert plugin.determine_validity_years(0) == 1
+    assert plugin.determine_validity_years(3) == 1
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_determine_end_date(mock_current_app):
+    mock_current_app.config.get = Mock(return_value=397)  # 397 days validity
+    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        assert arrow.get(2017, 12, 5) == plugin.determine_end_date(0)  # 397 days from (2016, 11, 3)
+        assert arrow.get(2017, 12, 5) == plugin.determine_end_date(arrow.get(2017, 12, 5))
+        assert arrow.get(2017, 12, 5) == plugin.determine_end_date(arrow.get(2020, 5, 7))
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_fields_with_validity_years(mock_current_app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
 
         names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+        options = {
+            "common_name": "example.com",
+            "owner": "bob@example.com",
+            "description": "test certificate",
+            "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+            "validity_years": 1
+        }
+        expected = {
+            "certificate": {
+                "csr": CSR_STR,
+                "common_name": "example.com",
+                "dns_names": names,
+                "signature_hash": "sha256",
+            },
+            "organization": {"id": 111111},
+            "validity_years": 1,
+        }
+        assert expected == plugin.map_fields(options, CSR_STR)
 
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_fields_with_validity_end_and_start(mock_current_app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
+
+        names = [u"one.example.com", u"two.example.com", u"three.example.com"]
         options = {
             "common_name": "example.com",
             "owner": "bob@example.com",
@@ -24,9 +82,7 @@ def test_map_fields_with_validity_end_and_start(app):
             "validity_start": arrow.get(2016, 10, 30),
         }
 
-    data = map_fields(options, CSR_STR)
+        expected = {
-
-    assert data == {
             "certificate": {
                 "csr": CSR_STR,
                 "common_name": "example.com",
@@ -37,66 +93,18 @@ def test_map_fields_with_validity_end_and_start(app):
             "custom_expiration_date": arrow.get(2017, 5, 7).format("YYYY-MM-DD"),
         }
 
+        assert expected == plugin.map_fields(options, CSR_STR)
 
-def test_map_fields_with_validity_years(app):
+
-    from lemur.plugins.lemur_digicert.plugin import map_fields
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_map_cis_fields_with_validity_years(mock_current_app, authority):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    plugin.determine_end_date = Mock(return_value=arrow.get(2018, 11, 3))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
 
         names = [u"one.example.com", u"two.example.com", u"three.example.com"]
-
-    options = {
-        "common_name": "example.com",
-        "owner": "bob@example.com",
-        "description": "test certificate",
-        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
-        "validity_years": 2,
-        "validity_end": arrow.get(2017, 10, 30),
-    }
-
-    data = map_fields(options, CSR_STR)
-
-    assert data == {
-        "certificate": {
-            "csr": CSR_STR,
-            "common_name": "example.com",
-            "dns_names": names,
-            "signature_hash": "sha256",
-        },
-        "organization": {"id": 111111},
-        "validity_years": 2,
-    }
-
-
-def test_map_cis_fields(app, authority):
-    from lemur.plugins.lemur_digicert.plugin import map_cis_fields
-
-    names = [u"one.example.com", u"two.example.com", u"three.example.com"]
-
-    options = {
-        "common_name": "example.com",
-        "owner": "bob@example.com",
-        "description": "test certificate",
-        "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
-        "organization": "Example, Inc.",
-        "organizational_unit": "Example Org",
-        "validity_end": arrow.get(2017, 5, 7),
-        "validity_start": arrow.get(2016, 10, 30),
-        "authority": authority,
-    }
-
-    data = map_cis_fields(options, CSR_STR)
-
-    assert data == {
-        "common_name": "example.com",
-        "csr": CSR_STR,
-        "additional_dns_names": names,
-        "signature_hash": "sha256",
-        "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
-        "validity": {
-            "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
-        },
-        "profile_name": None,
-    }
-
         options = {
             "common_name": "example.com",
             "owner": "bob@example.com",
@@ -108,10 +116,7 @@ def test_map_cis_fields(app, authority):
             "authority": authority,
         }
 
-    with freeze_time(time_to_freeze=arrow.get(2016, 11, 3).datetime):
+        expected = {
-        data = map_cis_fields(options, CSR_STR)
-
-        assert data == {
             "common_name": "example.com",
             "csr": CSR_STR,
             "additional_dns_names": names,
@@ -123,17 +128,55 @@ def test_map_cis_fields(app, authority):
             "profile_name": None,
         }
 
+        assert expected == plugin.map_cis_fields(options, CSR_STR)
 
-def test_signature_hash(app):
-    from lemur.plugins.lemur_digicert.plugin import signature_hash
 
-    assert signature_hash(None) == "sha256"
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
-    assert signature_hash("sha256WithRSA") == "sha256"
+def test_map_cis_fields_with_validity_end_and_start(mock_current_app, app, authority):
-    assert signature_hash("sha384WithRSA") == "sha384"
+    mock_current_app.config.get = Mock(side_effect=config_mock)
-    assert signature_hash("sha512WithRSA") == "sha512"
+    plugin.determine_end_date = Mock(return_value=arrow.get(2017, 5, 7))
+
+    with patch('lemur.plugins.lemur_digicert.plugin.signature_hash') as mock_signature_hash:
+        mock_signature_hash.return_value = "sha256"
+
+        names = [u"one.example.com", u"two.example.com", u"three.example.com"]
+        options = {
+            "common_name": "example.com",
+            "owner": "bob@example.com",
+            "description": "test certificate",
+            "extensions": {"sub_alt_names": {"names": [x509.DNSName(x) for x in names]}},
+            "organization": "Example, Inc.",
+            "organizational_unit": "Example Org",
+            "validity_end": arrow.get(2017, 5, 7),
+            "validity_start": arrow.get(2016, 10, 30),
+            "authority": authority
+        }
+
+        expected = {
+            "common_name": "example.com",
+            "csr": CSR_STR,
+            "additional_dns_names": names,
+            "signature_hash": "sha256",
+            "organization": {"name": "Example, Inc.", "units": ["Example Org"]},
+            "validity": {
+                "valid_to": arrow.get(2017, 5, 7).format("YYYY-MM-DDTHH:MM") + "Z"
+            },
+            "profile_name": None,
+        }
+
+        assert expected == plugin.map_cis_fields(options, CSR_STR)
+
+
+@patch("lemur.plugins.lemur_digicert.plugin.current_app")
+def test_signature_hash(mock_current_app, app):
+    mock_current_app.config.get = Mock(side_effect=config_mock)
+    assert plugin.signature_hash(None) == "sha256"
+    assert plugin.signature_hash("sha256WithRSA") == "sha256"
+    assert plugin.signature_hash("sha384WithRSA") == "sha384"
+    assert plugin.signature_hash("sha512WithRSA") == "sha512"
 
     with pytest.raises(Exception):
-        signature_hash("sdfdsf")
+        plugin.signature_hash("sdfdsf")
 
 
 def test_issuer_plugin_create_certificate(

lemur/plugins/lemur_email/templates/expiration.html

@@ -75,7 +75,8 @@
                                             </tr>
                                             <tr>
                                                 <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
-                                                    <br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used.
+                                                    <br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used,
+                                                    and disable notifications via the Notify toggle in Lemur, if applicable.
                                                     <table border="0" cellspacing="0" cellpadding="0"
                                                            style="margin-top:48px;margin-bottom:48px">
                                                         <tbody>

lemur/plugins/lemur_vault_dest/plugin.py

@@ -14,7 +14,7 @@ import re
 import hvac
 from flask import current_app
 
-from lemur.common.defaults import common_name
+from lemur.common.defaults import common_name, country, state, location, organizational_unit, organization
 from lemur.common.utils import parse_certificate
 from lemur.plugins.bases import DestinationPlugin
 from lemur.plugins.bases import SourcePlugin
@@ -58,7 +58,7 @@ class VaultSourcePlugin(SourcePlugin):
             "helpMessage": "Authentication method to use",
         },
         {
-            "name": "tokenFile/VaultRole",
+            "name": "tokenFileOrVaultRole",
             "type": "str",
             "required": True,
             "validation": "^([a-zA-Z0-9/._-]+/?)+$",
@@ -94,7 +94,7 @@ class VaultSourcePlugin(SourcePlugin):
         body = ""
         url = self.get_option("vaultUrl", options)
         auth_method = self.get_option("authenticationMethod", options)
-        auth_key = self.get_option("tokenFile/vaultRole", options)
+        auth_key = self.get_option("tokenFileOrVaultRole", options)
         mount = self.get_option("vaultMount", options)
         path = self.get_option("vaultPath", options)
         obj_name = self.get_option("objectName", options)
@@ -185,7 +185,7 @@ class VaultDestinationPlugin(DestinationPlugin):
             "helpMessage": "Authentication method to use",
         },
         {
-            "name": "tokenFile/VaultRole",
+            "name": "tokenFileOrVaultRole",
             "type": "str",
             "required": True,
             "validation": "^([a-zA-Z0-9/._-]+/?)+$",
@@ -202,15 +202,15 @@ class VaultDestinationPlugin(DestinationPlugin):
             "name": "vaultPath",
             "type": "str",
             "required": True,
-            "validation": "^([a-zA-Z0-9._-]+/?)+$",
+            "validation": "^(([a-zA-Z0-9._-]+|{(CN|OU|O|L|S|C)})+/?)+$",
-            "helpMessage": "Must be a valid Vault secrets path",
+            "helpMessage": "Must be a valid Vault secrets path. Support vars: {CN|OU|O|L|S|C}",
         },
         {
             "name": "objectName",
             "type": "str",
             "required": False,
-            "validation": "[0-9a-zA-Z.:_-]+",
+            "validation": "^([0-9a-zA-Z.:_-]+|{(CN|OU|O|L|S|C)})+$",
-            "helpMessage": "Name to bundle certs under, if blank use cn",
+            "helpMessage": "Name to bundle certs under, if blank use {CN}. Support vars: {CN|OU|O|L|S|C}",
         },
         {
             "name": "bundleChain",
@@ -241,11 +241,12 @@ class VaultDestinationPlugin(DestinationPlugin):
         :param cert_chain:
         :return:
         """
-        cname = common_name(parse_certificate(body))
+        cert = parse_certificate(body)
+        cname = common_name(cert)
 
         url = self.get_option("vaultUrl", options)
         auth_method = self.get_option("authenticationMethod", options)
-        auth_key = self.get_option("tokenFile/vaultRole", options)
+        auth_key = self.get_option("tokenFileOrVaultRole", options)
         mount = self.get_option("vaultMount", options)
         path = self.get_option("vaultPath", options)
         bundle = self.get_option("bundleChain", options)
@@ -285,10 +286,27 @@ class VaultDestinationPlugin(DestinationPlugin):
 
         client.secrets.kv.default_kv_version = api_version
 
-        if obj_name:
+        t_path = path.format(
-            path = "{0}/{1}".format(path, obj_name)
+            CN=cname,
-        else:
+            OU=organizational_unit(cert),
-            path = "{0}/{1}".format(path, cname)
+            O=organization(cert),  # noqa: E741
+            L=location(cert),
+            S=state(cert),
+            C=country(cert)
+        )
+        if not obj_name:
+            obj_name = '{CN}'
+
+        f_obj_name = obj_name.format(
+            CN=cname,
+            OU=organizational_unit(cert),
+            O=organization(cert),  # noqa: E741
+            L=location(cert),
+            S=state(cert),
+            C=country(cert)
+        )
+
+        path = "{0}/{1}".format(t_path, f_obj_name)
 
         secret = get_secret(client, mount, path)
         secret["data"][cname] = {}

lemur/sources/cli.py

@@ -54,6 +54,24 @@ def validate_sources(source_strings):
     return sources
 
 
+def execute_clean(plugin, certificate, source):
+    try:
+        plugin.clean(certificate, source.options)
+        certificate.sources.remove(source)
+
+        # If we want to remove the source from the certificate, we also need to clear any equivalent destinations to
+        # prevent Lemur from re-uploading the certificate.
+        for destination in certificate.destinations:
+            if destination.label == source.label:
+                certificate.destinations.remove(destination)
+
+        certificate_service.database.update(certificate)
+        return SUCCESS_METRIC_STATUS
+    except Exception as e:
+        current_app.logger.exception(e)
+        sentry.captureException()
+
+
 @manager.option(
     "-s",
     "--sources",
@@ -132,11 +150,9 @@ def clean(source_strings, commit):
         s = plugins.get(source.plugin_name)
 
         if not hasattr(s, "clean"):
-            print(
+            info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'"
-                "Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
+            current_app.logger.warning(info_text)
-                    source.label
+            print(info_text)
-                )
-            )
             continue
 
         start_time = time.time()
@@ -144,35 +160,147 @@ def clean(source_strings, commit):
         print("[+] Staring to clean source: {label}!\n".format(label=source.label))
 
         cleaned = 0
-        for certificate in certificate_service.get_all_pending_cleaning(source):
+        certificates = certificate_service.get_all_pending_cleaning_expired(source)
+        for certificate in certificates:
             status = FAILURE_METRIC_STATUS
             if commit:
-                try:
+                status = execute_clean(s, certificate, source)
-                    s.clean(certificate, source.options)
-                    certificate.sources.remove(source)
-                    certificate_service.database.update(certificate)
-                    status = SUCCESS_METRIC_STATUS
-                except Exception as e:
-                    current_app.logger.exception(e)
-                    sentry.captureException()
 
             metrics.send(
-                "clean",
+                "certificate_clean",
                 "counter",
                 1,
-                metric_tags={"source": source.label, "status": status},
+                metric_tags={"status": status, "source": source.label, "certificate": certificate.name},
             )
-
+            current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning")
-            current_app.logger.warning(
-                "Removed {0} from source {1} during cleaning".format(
-                    certificate.name, source.label
-                )
-            )
-
             cleaned += 1
 
-        print(
+        info_text = f"[+] Finished cleaning source: {source.label}. " \
-            "[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
+                    f"Removed {cleaned} certificates from source. " \
-                label=source.label, time=(time.time() - start_time), cleaned=cleaned
+                    f"Run Time: {(time.time() - start_time)}\n"
+        print(info_text)
+        current_app.logger.warning(info_text)
+
+
+@manager.option(
+    "-s",
+    "--sources",
+    dest="source_strings",
+    action="append",
+    help="Sources to operate on.",
+)
+@manager.option(
+    "-d",
+    "--days",
+    dest="days_to_expire",
+    type=int,
+    action="store",
+    required=True,
+    help="The expiry range within days.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
+def clean_unused_and_expiring_within_days(source_strings, days_to_expire, commit):
+    sources = validate_sources(source_strings)
+    for source in sources:
+        s = plugins.get(source.plugin_name)
+
+        if not hasattr(s, "clean"):
+            info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'"
+            current_app.logger.warning(info_text)
+            print(info_text)
+            continue
+
+        start_time = time.time()
+
+        print("[+] Staring to clean source: {label}!\n".format(label=source.label))
+
+        cleaned = 0
+        certificates = certificate_service.get_all_pending_cleaning_expiring_in_days(source, days_to_expire)
+        for certificate in certificates:
+            status = FAILURE_METRIC_STATUS
+            if commit:
+                status = execute_clean(s, certificate, source)
+
+            metrics.send(
+                "certificate_clean",
+                "counter",
+                1,
+                metric_tags={"status": status, "source": source.label, "certificate": certificate.name},
             )
+            current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning")
+            cleaned += 1
+
+        info_text = f"[+] Finished cleaning source: {source.label}. " \
+                    f"Removed {cleaned} certificates from source. " \
+                    f"Run Time: {(time.time() - start_time)}\n"
+        print(info_text)
+        current_app.logger.warning(info_text)
+
+
+@manager.option(
+    "-s",
+    "--sources",
+    dest="source_strings",
+    action="append",
+    help="Sources to operate on.",
+)
+@manager.option(
+    "-d",
+    "--days",
+    dest="days_since_issuance",
+    type=int,
+    action="store",
+    required=True,
+    help="Days since issuance.",
+)
+@manager.option(
+    "-c",
+    "--commit",
+    dest="commit",
+    action="store_true",
+    default=False,
+    help="Persist changes.",
+)
+def clean_unused_and_issued_since_days(source_strings, days_since_issuance, commit):
+    sources = validate_sources(source_strings)
+    for source in sources:
+        s = plugins.get(source.plugin_name)
+
+        if not hasattr(s, "clean"):
+            info_text = f"Cannot clean source: {source.label}, source plugin does not implement 'clean()'"
+            current_app.logger.warning(info_text)
+            print(info_text)
+            continue
+
+        start_time = time.time()
+
+        print("[+] Staring to clean source: {label}!\n".format(label=source.label))
+
+        cleaned = 0
+        certificates = certificate_service.get_all_pending_cleaning_issued_since_days(source, days_since_issuance)
+        for certificate in certificates:
+            status = FAILURE_METRIC_STATUS
+            if commit:
+                status = execute_clean(s, certificate, source)
+
+            metrics.send(
+                "certificate_clean",
+                "counter",
+                1,
+                metric_tags={"status": status, "source": source.label, "certificate": certificate.name},
             )
+            current_app.logger.warning(f"Removed {certificate.name} from source {source.label} during cleaning")
+            cleaned += 1
+
+        info_text = f"[+] Finished cleaning source: {source.label}. " \
+                    f"Removed {cleaned} certificates from source. " \
+                    f"Run Time: {(time.time() - start_time)}\n"
+        print(info_text)
+        current_app.logger.warning(info_text)

lemur/sources/service.py

@@ -123,15 +123,19 @@ def sync_endpoints(source):
                                               "acct": s.get_option("accountNumber", source.options)})
 
         if not endpoint["certificate"]:
-            current_app.logger.error(
+            current_app.logger.error({
-                "Certificate Not Found. Name: {0} Endpoint: {1}".format(
+                "message": "Certificate Not Found",
-                    certificate_name, endpoint["name"]
+                "certificate_name": certificate_name,
-                )
+                "endpoint_name": endpoint["name"],
-            )
+                "dns_name": endpoint.get("dnsname"),
+                "account": s.get_option("accountNumber", source.options),
+            })
+
             metrics.send("endpoint.certificate.not.found",
                          "counter", 1,
                          metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
-                                      "acct": s.get_option("accountNumber", source.options)})
+                                      "acct": s.get_option("accountNumber", source.options),
+                                      "dnsname": endpoint.get("dnsname")})
             continue
 
         policy = endpoint.pop("policy")
@@ -193,6 +197,11 @@ def sync_certificates(source, user):
     s = plugins.get(source.plugin_name)
     certificates = s.get_certificates(source.options)
 
+    # emitting the count of certificates on the source
+    metrics.send("sync_certificates_count",
+                 "gauge", len(certificates),
+                 metric_tags={"source": source.label})
+
     for certificate in certificates:
         exists, updated_by_hash = find_cert(certificate)
 

lemur/static/app/angular/certificates/certificate/certificate.js

@@ -212,12 +212,18 @@ angular.module('lemur')
   })
 
 .controller('CertificateCloneController', function ($scope, $uibModalInstance, CertificateApi, CertificateService, DestinationService, AuthorityService, AuthorityApi, PluginService, MomentService, WizardHandler, LemurRestangular, NotificationService, toaster, editId) {
+  $scope.certificate = LemurRestangular.restangularizeElement(null, {}, 'certificates');
   CertificateApi.get(editId).then(function (certificate) {
     $scope.certificate = certificate;
+    // prepare the certificate for cloning
     $scope.certificate.name = ''; // we should prefer the generated name
     $scope.certificate.csr = null;  // should not clone CSR in case other settings are changed in clone
     $scope.certificate.validityStart = null;
     $scope.certificate.validityEnd = null;
+    $scope.certificate.keyType = 'RSA2048'; // default algo to show during clone
+    $scope.certificate.description = 'Cloning from cert ID ' + editId;
+    $scope.certificate.replacedBy = []; // should not clone 'replaced by' info
+    $scope.certificate.removeReplaces(); // should not clone 'replacement cert' info
     CertificateService.getDefaults($scope.certificate);
   });
 

lemur/static/app/angular/certificates/certificate/tracking.tpl.html

@@ -140,7 +140,6 @@
         <select ng-model="certificate.validityYears" class="form-control">
           <option value="">-</option>
           <option value="1">1 year</option>
-          <option value="2">2 years</option>
         </select>
       </div>
       <span style="padding-top: 15px" class="text-center col-sm-1">

lemur/tests/test_certificates.py

@@ -9,7 +9,8 @@ from cryptography import x509
 from cryptography.hazmat.backends import default_backend
 from marshmallow import ValidationError
 from freezegun import freeze_time
-from mock import patch
+# from mock import patch
+from unittest.mock import patch
 
 from lemur.certificates.service import create_csr
 from lemur.certificates.views import *  # noqa
@@ -906,12 +907,12 @@ def test_certificate_get_body(client):
     assert response_body["serial"] == "211983098819107449768450703123665283596"
     assert response_body["serialHex"] == "9F7A75B39DAE4C3F9524C68B06DA6A0C"
     assert response_body["distinguishedName"] == (
-        "CN=LemurTrust Unittests Class 1 CA 2018,"
+        "L=Earth,"
-        "O=LemurTrust Enterprises Ltd,"
-        "OU=Unittesting Operations Center,"
-        "C=EE,"
         "ST=N/A,"
-        "L=Earth"
+        "C=EE,"
+        "OU=Unittesting Operations Center,"
+        "O=LemurTrust Enterprises Ltd,"
+        "CN=LemurTrust Unittests Class 1 CA 2018"
     )
 
 

lemur/tests/test_dns_providers.py

@@ -4,9 +4,20 @@ from lemur.dns_providers import util as dnsutil
 
 class TestDNSProvider(unittest.TestCase):
     def test_is_valid_domain(self):
-        self.assertTrue(dnsutil.is_valid_domain("example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example.com'))
-        self.assertTrue(dnsutil.is_valid_domain("foo.bar.org"))
+        self.assertTrue(dnsutil.is_valid_domain('foo.bar.org'))
-        self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('exam--ple.io'))
-        self.assertFalse(dnsutil.is_valid_domain("e/xample.com"))
+        self.assertTrue(dnsutil.is_valid_domain('a.example.com'))
-        self.assertFalse(dnsutil.is_valid_domain("exam\ple.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example.io'))
-        self.assertFalse(dnsutil.is_valid_domain("*.example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example-of-under-63-character-domain-label-length-limit-1234567.com'))
+        self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com'))
+        self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('e/xample.com'))
+        self.assertFalse(dnsutil.is_valid_domain('exam\ple.com'))
+        self.assertFalse(dnsutil.is_valid_domain('<example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('*.example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('-example.io'))
+        self.assertFalse(dnsutil.is_valid_domain('example-.io'))
+        self.assertFalse(dnsutil.is_valid_domain('example..io'))
+        self.assertFalse(dnsutil.is_valid_domain('exa mple.io'))
+        self.assertFalse(dnsutil.is_valid_domain('-'))

package.json

@@ -6,7 +6,7 @@
     "url": "git://github.com/netflix/lemur.git"
   },
   "dependencies": {
-    "bower": "^1.8.2",
+    "bower": "^1.8.8",
     "browser-sync": "^2.26.7",
     "del": "^2.2.2",
     "gulp-autoprefixer": "^3.1.1",
@@ -17,10 +17,10 @@
     "gulp-flatten": "^0.3.1",
     "gulp-foreach": "0.1.0",
     "gulp-if": "^2.0.2",
-    "gulp-imagemin": "^3.1.1",
+    "gulp-imagemin": "^7.1.0",
     "gulp-inject": "~4.1.0",
     "gulp-jshint": "^2.0.4",
-    "gulp-less": "^3.0.3",
+    "gulp-less": "^4.0.1",
     "gulp-load-plugins": "^1.4.0",
     "gulp-minify-css": "^1.2.4",
     "gulp-minify-html": "~1.0.6",
@@ -29,7 +29,7 @@
     "gulp-notify": "^2.2.0",
     "gulp-plumber": "^1.1.0",
     "gulp-print": "^2.0.1",
-    "gulp-protractor": "3.0.0",
+    "gulp-protractor": "^4.1.1",
     "gulp-replace": "~0.5.3",
     "gulp-replace-task": "~0.11.0",
     "gulp-rev": "^7.1.2",
@@ -41,7 +41,7 @@
     "gulp-util": "^3.0.1",
     "http-proxy": "~1.16.2",
     "jshint-stylish": "^2.2.1",
-    "karma": "~1.3.0",
+    "karma": "^4.4.1",
     "karma-jasmine": "^1.1.0",
     "main-bower-files": "^2.13.1",
     "merge-stream": "^1.0.1",
@@ -60,7 +60,7 @@
   },
   "devDependencies": {
     "gulp": "^3.9.1",
-    "jshint": "^2.8.0",
+    "jshint": "^2.11.0",
     "karma-chrome-launcher": "^2.0.0"
   }
 }

requirements-dev.txt

@@ -4,38 +4,44 @@
 #
 #    pip-compile --no-index --output-file=requirements-dev.txt requirements-dev.in
 #
-aspy.yaml==1.3.0          # via pre-commit
+appdirs==1.4.3            # via virtualenv
-bleach==3.1.0             # via readme-renderer
+bleach==3.1.4             # via readme-renderer
-certifi==2019.11.28       # via requests
+certifi==2020.6.20        # via requests
-cfgv==2.0.1               # via pre-commit
+cffi==1.14.0              # via cryptography
+cfgv==3.1.0               # via pre-commit
 chardet==3.0.4            # via requests
-docutils==0.15.2          # via readme-renderer
+colorama==0.4.3           # via twine
-flake8==3.5.0
+cryptography==3.0         # via secretstorage
-identify==1.4.9           # via pre-commit
+distlib==0.3.0            # via virtualenv
-idna==2.8                 # via requests
+docutils==0.16            # via readme-renderer
-importlib-metadata==1.3.0  # via keyring, pre-commit, twine
+filelock==3.0.12          # via virtualenv
-invoke==1.3.0
+flake8==3.5.0             # via -r requirements-dev.in
-keyring==21.0.0           # via twine
+identify==1.4.14          # via pre-commit
+idna==2.9                 # via requests
+invoke==1.4.1             # via -r requirements-dev.in
+jeepney==0.4.3            # via keyring, secretstorage
+keyring==21.2.0           # via twine
 mccabe==0.6.1             # via flake8
-more-itertools==8.0.2     # via zipp
+nodeenv==1.4.0            # via -r requirements-dev.in, pre-commit
-nodeenv==1.3.3
 pkginfo==1.5.0.1          # via twine
-pre-commit==1.21.0
+pre-commit==2.6.0         # via -r requirements-dev.in
 pycodestyle==2.3.1        # via flake8
+pycparser==2.20           # via cffi
 pyflakes==1.6.0           # via flake8
-pygments==2.5.2           # via readme-renderer
+pygments==2.6.1           # via readme-renderer
-pyyaml==5.2
+pyyaml==5.3.1             # via -r requirements-dev.in, pre-commit
-readme-renderer==24.0     # via twine
+readme-renderer==25.0     # via twine
 requests-toolbelt==0.9.1  # via twine
-requests==2.22.0          # via requests-toolbelt, twine
+requests==2.24.0          # via requests-toolbelt, twine
-six==1.13.0               # via bleach, cfgv, pre-commit, readme-renderer
+rfc3986==1.4.0            # via twine
+secretstorage==3.1.2      # via keyring
+six==1.15.0               # via bleach, cryptography, readme-renderer, virtualenv
 toml==0.10.0              # via pre-commit
-tqdm==4.41.1              # via twine
+tqdm==4.45.0              # via twine
-twine==3.1.1
+twine==3.2.0              # via -r requirements-dev.in
-urllib3==1.25.7           # via requests
+urllib3==1.25.8           # via requests
-virtualenv==16.7.9        # via pre-commit
+virtualenv==20.0.17       # via pre-commit
 webencodings==0.5.1       # via bleach
-zipp==0.6.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements-docs.txt

@@ -4,111 +4,110 @@
 #
 #    pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
 #
-acme==1.0.0
+acme==1.6.0               # via -r requirements.txt
 alabaster==0.7.12         # via sphinx
-alembic-autogenerate-enums==0.0.2
+alembic-autogenerate-enums==0.0.2  # via -r requirements.txt
-alembic==1.3.2
+alembic==1.4.2            # via -r requirements.txt, flask-migrate
-amqp==2.5.2
+amqp==2.5.2               # via -r requirements.txt, kombu
-aniso8601==8.0.0
+aniso8601==8.0.0          # via -r requirements.txt, flask-restful
-arrow==0.15.5
+arrow==0.15.8             # via -r requirements.txt
-asyncpool==1.0
+asyncpool==1.0            # via -r requirements.txt
 babel==2.8.0              # via sphinx
-bcrypt==3.1.7
+bcrypt==3.1.7             # via -r requirements.txt, flask-bcrypt, paramiko
-billiard==3.6.1.0
+beautifulsoup4==4.9.1     # via -r requirements.txt, cloudflare
-blinker==1.4
+billiard==3.6.3.0         # via -r requirements.txt, celery
-boto3==1.10.46
+blinker==1.4              # via -r requirements.txt, flask-mail, flask-principal, raven
-botocore==1.13.46
+boto3==1.14.33            # via -r requirements.txt
-celery[redis]==4.4.0
+botocore==1.17.33         # via -r requirements.txt, boto3, s3transfer
-certifi==2019.11.28
+celery[redis]==4.4.2      # via -r requirements.txt
-certsrv==2.1.1
+certifi==2020.6.20        # via -r requirements.txt, requests
-cffi==1.13.2
+certsrv==2.1.1            # via -r requirements.txt
-chardet==3.0.4
+cffi==1.14.0              # via -r requirements.txt, bcrypt, cryptography, pynacl
-click==7.0
+chardet==3.0.4            # via -r requirements.txt, requests
-cloudflare==2.3.1
+click==7.1.1              # via -r requirements.txt, flask
-cryptography==2.8
+cloudflare==2.8.9         # via -r requirements.txt
-dnspython3==1.15.0
+cryptography==3.0         # via -r requirements.txt, acme, josepy, paramiko, pyopenssl, requests
-dnspython==1.15.0
+dnspython3==1.15.0        # via -r requirements.txt
-docutils==0.15.2
+dnspython==1.15.0         # via -r requirements.txt, dnspython3
-dyn==1.8.1
+docutils==0.15.2          # via -r requirements.txt, botocore, sphinx
-flask-bcrypt==0.7.1
+dyn==1.8.1                # via -r requirements.txt
-flask-cors==3.0.8
+flask-bcrypt==0.7.1       # via -r requirements.txt
-flask-mail==0.9.1
+flask-cors==3.0.8         # via -r requirements.txt
-flask-migrate==2.5.2
+flask-mail==0.9.1         # via -r requirements.txt
-flask-principal==0.4.0
+flask-migrate==2.5.3      # via -r requirements.txt
-flask-replicated==1.3
+flask-principal==0.4.0    # via -r requirements.txt
-flask-restful==0.3.7
+flask-replicated==1.4     # via -r requirements.txt
-flask-script==2.0.6
+flask-restful==0.3.8      # via -r requirements.txt
-flask-sqlalchemy==2.4.1
+flask-script==2.0.6       # via -r requirements.txt
-flask==1.1.1
+flask-sqlalchemy==2.4.4   # via -r requirements.txt, flask-migrate
-future==0.18.2
+flask==1.1.2              # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
-gunicorn==20.0.4
+future==0.18.2            # via -r requirements.txt
-hvac==0.9.6
+gunicorn==20.0.4          # via -r requirements.txt
-idna==2.8
+hvac==0.10.5              # via -r requirements.txt
+idna==2.9                 # via -r requirements.txt, requests
 imagesize==1.2.0          # via sphinx
-importlib-metadata==1.3.0
+inflection==0.5.0         # via -r requirements.txt
-inflection==0.3.1
+itsdangerous==1.1.0       # via -r requirements.txt, flask
-itsdangerous==1.1.0
+javaobj-py3==0.4.0.1      # via -r requirements.txt, pyjks
-javaobj-py3==0.4.0.1
+jinja2==2.11.2            # via -r requirements.txt, flask, sphinx
-jinja2==2.10.3
+jmespath==0.9.5           # via -r requirements.txt, boto3, botocore
-jmespath==0.9.4
+josepy==1.3.0             # via -r requirements.txt, acme
-josepy==1.2.0
+jsonlines==1.2.0          # via -r requirements.txt, cloudflare
-jsonlines==1.2.0
+kombu==4.6.8              # via -r requirements.txt, celery
-kombu==4.6.7
+lockfile==0.12.2          # via -r requirements.txt
-lockfile==0.12.2
+logmatic-python==0.1.7    # via -r requirements.txt
-logmatic-python==0.1.7
+mako==1.1.2               # via -r requirements.txt, alembic
-mako==1.1.0
+markupsafe==1.1.1         # via -r requirements.txt, jinja2, mako
-markupsafe==1.1.1
+marshmallow-sqlalchemy==0.23.1  # via -r requirements.txt
-marshmallow-sqlalchemy==0.21.0
+marshmallow==2.20.4       # via -r requirements.txt, marshmallow-sqlalchemy
-marshmallow==2.20.4
+ndg-httpsclient==0.5.1    # via -r requirements.txt
-mock==3.0.5
+packaging==20.3           # via sphinx
-more-itertools==8.0.2
+paramiko==2.7.1           # via -r requirements.txt
-ndg-httpsclient==0.5.1
+pem==20.1.0               # via -r requirements.txt
-packaging==19.2           # via sphinx
+psycopg2==2.8.5           # via -r requirements.txt
-paramiko==2.7.1
+pyasn1-modules==0.2.8     # via -r requirements.txt, pyjks, python-ldap
-pem==19.3.0
+pyasn1==0.4.8             # via -r requirements.txt, ndg-httpsclient, pyasn1-modules, pyjks, python-ldap
-psycopg2==2.8.4
+pycparser==2.20           # via -r requirements.txt, cffi
-pyasn1-modules==0.2.7
+pycryptodomex==3.9.7      # via -r requirements.txt, pyjks
-pyasn1==0.4.8
+pygments==2.6.1           # via sphinx
-pycparser==2.19
+pyjks==20.0.0             # via -r requirements.txt
-pycryptodomex==3.9.4
+pyjwt==1.7.1              # via -r requirements.txt
-pygments==2.5.2           # via sphinx
+pynacl==1.3.0             # via -r requirements.txt, paramiko
-pyjks==19.0.0
+pyopenssl==19.1.0         # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
-pyjwt==1.7.1
+pyparsing==2.4.7          # via packaging
-pynacl==1.3.0
+pyrfc3339==1.1            # via -r requirements.txt, acme
-pyopenssl==19.1.0
+python-dateutil==2.8.1    # via -r requirements.txt, alembic, arrow, botocore
-pyparsing==2.4.6          # via packaging
+python-editor==1.0.4      # via -r requirements.txt, alembic
-pyrfc3339==1.1
+python-json-logger==0.1.11  # via -r requirements.txt, logmatic-python
-python-dateutil==2.8.1
+python-ldap==3.3.1        # via -r requirements.txt
-python-editor==1.0.4
+pytz==2019.3              # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339
-python-json-logger==0.1.11
+pyyaml==5.3.1             # via -r requirements.txt, cloudflare
-pytz==2019.3
+raven[flask]==6.10.0      # via -r requirements.txt
-pyyaml==5.2
+redis==3.5.3              # via -r requirements.txt, celery
-raven[flask]==6.10.0
+requests-toolbelt==0.9.1  # via -r requirements.txt, acme
-redis==3.3.11
+requests[security]==2.24.0  # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
-requests-toolbelt==0.9.1
+retrying==1.3.3           # via -r requirements.txt
-requests[security]==2.22.0
+s3transfer==0.3.3         # via -r requirements.txt, boto3
-retrying==1.3.3
+six==1.15.0               # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils
-s3transfer==0.2.1
-six==1.13.0
 snowballstemmer==2.0.0    # via sphinx
-sphinx-rtd-theme==0.4.3
+soupsieve==2.0.1          # via -r requirements.txt, beautifulsoup4
-sphinx==2.3.1
+sphinx-rtd-theme==0.5.0   # via -r requirements-docs.in
-sphinxcontrib-applehelp==1.0.1  # via sphinx
+sphinx==3.2.0             # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
-sphinxcontrib-devhelp==1.0.1  # via sphinx
+sphinxcontrib-applehelp==1.0.2  # via sphinx
-sphinxcontrib-htmlhelp==1.0.2  # via sphinx
+sphinxcontrib-devhelp==1.0.2  # via sphinx
-sphinxcontrib-httpdomain==1.7.0
+sphinxcontrib-htmlhelp==1.0.3  # via sphinx
+sphinxcontrib-httpdomain==1.7.0  # via -r requirements-docs.in
 sphinxcontrib-jsmath==1.0.1  # via sphinx
-sphinxcontrib-qthelp==1.0.2  # via sphinx
+sphinxcontrib-qthelp==1.0.3  # via sphinx
-sphinxcontrib-serializinghtml==1.1.3  # via sphinx
+sphinxcontrib-serializinghtml==1.1.4  # via sphinx
-sqlalchemy-utils==0.36.1
+sqlalchemy-utils==0.36.8  # via -r requirements.txt
-sqlalchemy==1.3.12
+sqlalchemy==1.3.16        # via -r requirements.txt, alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-tabulate==0.8.6
+tabulate==0.8.7           # via -r requirements.txt
-twofish==0.3.0
+twofish==0.3.0            # via -r requirements.txt, pyjks
-urllib3==1.25.7
+urllib3==1.25.8           # via -r requirements.txt, botocore, requests
-vine==1.3.0
+vine==1.3.0               # via -r requirements.txt, amqp, celery
-werkzeug==0.16.0
+werkzeug==1.0.1           # via -r requirements.txt, flask
-xmltodict==0.12.0
+xmltodict==0.12.0         # via -r requirements.txt
-zipp==0.6.0
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements-tests.txt

@@ -6,85 +6,87 @@
 #
 appdirs==1.4.3            # via black
 attrs==19.3.0             # via black, jsonschema, pytest
-aws-sam-translator==1.19.1  # via cfn-lint
+aws-sam-translator==1.22.0  # via cfn-lint
-aws-xray-sdk==2.4.3       # via moto
+aws-xray-sdk==2.5.0       # via moto
-bandit==1.6.2
+bandit==1.6.2             # via -r requirements-tests.in
-black==19.10b0
+black==19.10b0            # via -r requirements-tests.in
-boto3==1.10.46            # via aws-sam-translator, moto
+boto3==1.14.33            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.13.46         # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.17.33         # via aws-xray-sdk, boto3, moto, s3transfer
-certifi==2019.11.28       # via requests
+certifi==2020.6.20        # via requests
-cffi==1.13.2              # via cryptography
+cffi==1.14.0              # via cryptography
-cfn-lint==0.26.2          # via moto
+cfn-lint==0.29.5          # via moto
 chardet==3.0.4            # via requests
-click==7.0                # via black, flask
+click==7.1.1              # via black, flask
-coverage==5.0.1
+coverage==5.2.1           # via -r requirements-tests.in
-cryptography==2.8         # via moto, sshpubkeys
+cryptography==3.0         # via moto, sshpubkeys
-docker==4.1.0             # via moto
+decorator==4.4.2          # via networkx
+docker==4.2.0             # via moto
 docutils==0.15.2          # via botocore
 ecdsa==0.15               # via python-jose, sshpubkeys
-factory-boy==2.12.0
+factory-boy==2.12.0       # via -r requirements-tests.in
-faker==3.0.0
+faker==4.1.1              # via -r requirements-tests.in, factory-boy
-fakeredis==1.1.0
+fakeredis==1.4.1          # via -r requirements-tests.in
-flask==1.1.1              # via pytest-flask
+flask==1.1.2              # via pytest-flask
-freezegun==0.3.12
+freezegun==0.3.15         # via -r requirements-tests.in
 future==0.18.2            # via aws-xray-sdk
-gitdb2==2.0.6             # via gitpython
+gitdb==4.0.4              # via gitpython
-gitpython==3.0.5          # via bandit
+gitpython==3.1.1          # via bandit
 idna==2.8                 # via moto, requests
-importlib-metadata==1.3.0  # via jsonschema, pluggy, pytest
+importlib-metadata==1.6.0  # via jsonpickle
+iniconfig==1.0.1          # via pytest
 itsdangerous==1.1.0       # via flask
-jinja2==2.10.3            # via flask, moto
+jinja2==2.11.2            # via flask, moto
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.5           # via boto3, botocore
 jsondiff==1.1.2           # via moto
-jsonpatch==1.24           # via cfn-lint
+jsonpatch==1.25           # via cfn-lint
-jsonpickle==1.2           # via aws-xray-sdk
+jsonpickle==1.4           # via aws-xray-sdk
 jsonpointer==2.0          # via jsonpatch
 jsonschema==3.2.0         # via aws-sam-translator, cfn-lint
 markupsafe==1.1.1         # via jinja2
-mock==3.0.5               # via moto
+mock==4.0.2               # via moto
-more-itertools==8.0.2     # via pytest, zipp
+more-itertools==8.2.0     # via pytest
-moto==1.3.14
+moto==1.3.14              # via -r requirements-tests.in
-nose==1.3.7
+networkx==2.4             # via cfn-lint
-packaging==19.2           # via pytest
+nose==1.3.7               # via -r requirements-tests.in
-pathspec==0.7.0           # via black
+packaging==20.3           # via pytest
-pbr==5.4.4                # via stevedore
+pathspec==0.8.0           # via black
+pbr==5.4.5                # via stevedore
 pluggy==0.13.1            # via pytest
-py==1.8.1                 # via pytest
+py==1.9.0                 # via pytest
 pyasn1==0.4.8             # via python-jose, rsa
-pycparser==2.19           # via cffi
+pycparser==2.20           # via cffi
-pyflakes==2.1.1
+pyflakes==2.2.0           # via -r requirements-tests.in
-pyparsing==2.4.6          # via packaging
+pyparsing==2.4.7          # via packaging
-pyrsistent==0.15.6        # via jsonschema
+pyrsistent==0.16.0        # via jsonschema
-pytest-flask==0.15.0
+pytest-flask==1.0.0       # via -r requirements-tests.in
-pytest-mock==1.13.0
+pytest-mock==3.2.0        # via -r requirements-tests.in
-pytest==5.3.2
+pytest==6.0.1             # via -r requirements-tests.in, pytest-flask, pytest-mock
 python-dateutil==2.8.1    # via botocore, faker, freezegun, moto
 python-jose==3.1.0        # via moto
 pytz==2019.3              # via moto
-pyyaml==5.2
+pyyaml==5.3.1             # via -r requirements-tests.in, bandit, cfn-lint, moto
-redis==3.3.11             # via fakeredis
+redis==3.5.3              # via fakeredis
-regex==2019.12.20         # via black
+regex==2020.4.4           # via black
-requests-mock==1.7.0
+requests-mock==1.8.0      # via -r requirements-tests.in
-requests==2.22.0          # via docker, moto, requests-mock, responses
+requests==2.24.0          # via docker, moto, requests-mock, responses
-responses==0.10.9         # via moto
+responses==0.10.12        # via moto
 rsa==4.0                  # via python-jose
-s3transfer==0.2.1         # via boto3
+s3transfer==0.3.3         # via boto3
-six==1.13.0               # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, faker, fakeredis, freezegun, jsonschema, mock, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client
+six==1.15.0               # via aws-sam-translator, bandit, cfn-lint, cryptography, docker, ecdsa, fakeredis, freezegun, jsonschema, moto, packaging, pyrsistent, python-dateutil, python-jose, requests-mock, responses, stevedore, websocket-client
-smmap2==2.0.5             # via gitdb2
+smmap==3.0.2              # via gitdb
 sortedcontainers==2.1.0   # via fakeredis
 sshpubkeys==3.1.0         # via moto
-stevedore==1.31.0         # via bandit
+stevedore==1.32.0         # via bandit
 text-unidecode==1.3       # via faker
-toml==0.10.0              # via black
+toml==0.10.0              # via black, pytest
-typed-ast==1.4.0          # via black
+typed-ast==1.4.1          # via black
-urllib3==1.25.7           # via botocore, requests
+urllib3==1.25.8           # via botocore, requests
-wcwidth==0.1.8            # via pytest
 websocket-client==0.57.0  # via docker
-werkzeug==0.16.0          # via flask, moto, pytest-flask
+werkzeug==1.0.1           # via flask, moto, pytest-flask
-wrapt==1.11.2             # via aws-xray-sdk
+wrapt==1.12.1             # via aws-xray-sdk
 xmltodict==0.12.0         # via moto
-zipp==0.6.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements.txt

@@ -4,96 +4,94 @@
 #
 #    pip-compile --no-index --output-file=requirements.txt requirements.in
 #
-acme==1.0.0
+acme==1.6.0               # via -r requirements.in
-alembic-autogenerate-enums==0.0.2
+alembic-autogenerate-enums==0.0.2  # via -r requirements.in
-alembic==1.3.2            # via flask-migrate
+alembic==1.4.2            # via flask-migrate
 amqp==2.5.2               # via kombu
 aniso8601==8.0.0          # via flask-restful
-arrow==0.15.5
+arrow==0.15.8             # via -r requirements.in
-asyncpool==1.0
+asyncpool==1.0            # via -r requirements.in
 bcrypt==3.1.7             # via flask-bcrypt, paramiko
-billiard==3.6.1.0         # via celery
+beautifulsoup4==4.9.1     # via cloudflare
+billiard==3.6.3.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.10.46
+boto3==1.14.33            # via -r requirements.in
-botocore==1.13.46
+botocore==1.17.33         # via -r requirements.in, boto3, s3transfer
-celery[redis]==4.4.0
+celery[redis]==4.4.2      # via -r requirements.in
-certifi==2019.11.28
+certifi==2020.6.20        # via -r requirements.in, requests
-certsrv==2.1.1
+certsrv==2.1.1            # via -r requirements.in
-cffi==1.13.2              # via bcrypt, cryptography, pynacl
+cffi==1.14.0              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
-click==7.0                # via flask
+click==7.1.1              # via flask
-cloudflare==2.3.1
+cloudflare==2.8.9         # via -r requirements.in
-cryptography==2.8
+cryptography==3.0         # via -r requirements.in, acme, josepy, paramiko, pyopenssl, requests
-dnspython3==1.15.0
+dnspython3==1.15.0        # via -r requirements.in
 dnspython==1.15.0         # via dnspython3
 docutils==0.15.2          # via botocore
-dyn==1.8.1
+dyn==1.8.1                # via -r requirements.in
-flask-bcrypt==0.7.1
+flask-bcrypt==0.7.1       # via -r requirements.in
-flask-cors==3.0.8
+flask-cors==3.0.8         # via -r requirements.in
-flask-mail==0.9.1
+flask-mail==0.9.1         # via -r requirements.in
-flask-migrate==2.5.2
+flask-migrate==2.5.3      # via -r requirements.in
-flask-principal==0.4.0
+flask-principal==0.4.0    # via -r requirements.in
-flask-replicated==1.3
+flask-replicated==1.4     # via -r requirements.in
-flask-restful==0.3.7
+flask-restful==0.3.8      # via -r requirements.in
-flask-script==2.0.6
+flask-script==2.0.6       # via -r requirements.in
-flask-sqlalchemy==2.4.1
+flask-sqlalchemy==2.4.4   # via -r requirements.in, flask-migrate
-flask==1.1.1
+flask==1.1.2              # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
-future==0.18.2
+future==0.18.2            # via -r requirements.in
-gunicorn==20.0.4
+gunicorn==20.0.4          # via -r requirements.in
-hvac==0.9.6
+hvac==0.10.5              # via -r requirements.in
-idna==2.8                 # via requests
+idna==2.9                 # via requests
-importlib-metadata==1.3.0  # via kombu
+inflection==0.5.0         # via -r requirements.in
-inflection==0.3.1
 itsdangerous==1.1.0       # via flask
 javaobj-py3==0.4.0.1      # via pyjks
-jinja2==2.10.3
+jinja2==2.11.2            # via -r requirements.in, flask
-jmespath==0.9.4           # via boto3, botocore
+jmespath==0.9.5           # via boto3, botocore
-josepy==1.2.0             # via acme
+josepy==1.3.0             # via acme
 jsonlines==1.2.0          # via cloudflare
-kombu==4.6.7              # via celery
+kombu==4.6.8              # via celery
-lockfile==0.12.2
+lockfile==0.12.2          # via -r requirements.in
-logmatic-python==0.1.7
+logmatic-python==0.1.7    # via -r requirements.in
-mako==1.1.0               # via alembic
+mako==1.1.2               # via alembic
 markupsafe==1.1.1         # via jinja2, mako
-marshmallow-sqlalchemy==0.21.0
+marshmallow-sqlalchemy==0.23.1  # via -r requirements.in
-marshmallow==2.20.4
+marshmallow==2.20.4       # via -r requirements.in, marshmallow-sqlalchemy
-mock==3.0.5               # via acme
+ndg-httpsclient==0.5.1    # via -r requirements.in
-more-itertools==8.0.2     # via zipp
+paramiko==2.7.1           # via -r requirements.in
-ndg-httpsclient==0.5.1
+pem==20.1.0               # via -r requirements.in
-paramiko==2.7.1
+psycopg2==2.8.5           # via -r requirements.in
-pem==19.3.0
+pyasn1-modules==0.2.8     # via pyjks, python-ldap
-psycopg2==2.8.4
-pyasn1-modules==0.2.7     # via pyjks, python-ldap
 pyasn1==0.4.8             # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap
-pycparser==2.19           # via cffi
+pycparser==2.20           # via cffi
-pycryptodomex==3.9.4      # via pyjks
+pycryptodomex==3.9.7      # via pyjks
-pyjks==19.0.0
+pyjks==20.0.0             # via -r requirements.in
-pyjwt==1.7.1
+pyjwt==1.7.1              # via -r requirements.in
 pynacl==1.3.0             # via paramiko
-pyopenssl==19.1.0
+pyopenssl==19.1.0         # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
 pyrfc3339==1.1            # via acme
 python-dateutil==2.8.1    # via alembic, arrow, botocore
 python-editor==1.0.4      # via alembic
 python-json-logger==0.1.11  # via logmatic-python
-python-ldap==3.2.0
+python-ldap==3.3.1        # via -r requirements.in
 pytz==2019.3              # via acme, celery, flask-restful, pyrfc3339
-pyyaml==5.2
+pyyaml==5.3.1             # via -r requirements.in, cloudflare
-raven[flask]==6.10.0
+raven[flask]==6.10.0      # via -r requirements.in
-redis==3.3.11
+redis==3.5.3              # via -r requirements.in, celery
 requests-toolbelt==0.9.1  # via acme
-requests[security]==2.22.0
+requests[security]==2.24.0  # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
-retrying==1.3.3
+retrying==1.3.3           # via -r requirements.in
-s3transfer==0.2.1         # via boto3
+s3transfer==0.3.3         # via boto3
-six==1.13.0
+six==1.15.0               # via -r requirements.in, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, pynacl, pyopenssl, python-dateutil, retrying, sqlalchemy-utils
-sqlalchemy-utils==0.36.1
+soupsieve==2.0.1          # via beautifulsoup4
-sqlalchemy==1.3.12        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy-utils==0.36.8  # via -r requirements.in
-tabulate==0.8.6
+sqlalchemy==1.3.16        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+tabulate==0.8.7           # via -r requirements.in
 twofish==0.3.0            # via pyjks
-urllib3==1.25.7           # via botocore, requests
+urllib3==1.25.8           # via botocore, requests
 vine==1.3.0               # via amqp, celery
-werkzeug==0.16.0          # via flask
+werkzeug==1.0.1           # via flask
-xmltodict==0.12.0
+xmltodict==0.12.0         # via -r requirements.in
-zipp==0.6.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

setup.py

@@ -45,16 +45,20 @@ with open(os.path.join(ROOT, 'lemur', '__about__.py')) as f:
     exec(f.read(), about)  # nosec: about file is benign
 
 install_requires_g = parse_requirements("requirements.txt", session=PipSession())
-install_requires = [str(ir.req) for ir in install_requires_g]
-
 tests_require_g = parse_requirements("requirements-tests.txt", session=PipSession())
-tests_require = [str(ir.req) for ir in tests_require_g]
-
 docs_require_g = parse_requirements("requirements-docs.txt", session=PipSession())
-docs_require = [str(ir.req) for ir in docs_require_g]
-
 dev_requires_g = parse_requirements("requirements-dev.txt", session=PipSession())
-dev_requires = [str(ir.req) for ir in dev_requires_g]
+
+if tuple(map(int, pip.__version__.split('.'))) >= (20, 1):
+    install_requires = [str(ir.requirement) for ir in install_requires_g]
+    tests_require = [str(ir.requirement) for ir in tests_require_g]
+    docs_require = [str(ir.requirement) for ir in docs_require_g]
+    dev_requires = [str(ir.requirement) for ir in dev_requires_g]
+else:
+    install_requires = [str(ir.req) for ir in install_requires_g]
+    tests_require = [str(ir.req) for ir in tests_require_g]
+    docs_require = [str(ir.req) for ir in docs_require_g]
+    dev_requires = [str(ir.req) for ir in dev_requires_g]
 
 
 class SmartInstall(install):