Merge branch 'master' into ilabun/optimize-certificates-sql

.gitignore

@@ -14,7 +14,6 @@
 MANIFEST
 test.conf
 pip-log.txt
-package-lock.json
 /htmlcov
 /cover
 /build

docker/Dockerfile

@@ -3,6 +3,8 @@ FROM alpine:3.8
 ARG VERSION
 ENV VERSION master
 
+ARG URLCONTEXT
+
 ENV uid 1337
 ENV gid 1337
 ENV user lemur
@@ -22,6 +24,7 @@ RUN addgroup -S ${group} -g ${gid} && \
                 gcc \
                 autoconf \
                 automake \
+                libtool \
                 make \
                 nasm  \
                 zlib-dev \
@@ -42,7 +45,7 @@ WORKDIR /opt/lemur
 RUN npm install --unsafe-perm && \
     pip3 install -e . && \
     node_modules/.bin/gulp build && \
-    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
     apk del build-dependencies
 
 COPY entrypoint /

docker/Dockerfile-src

@@ -3,6 +3,8 @@ FROM alpine:3.8
 ARG VERSION
 ENV VERSION master
 
+ARG URLCONTEXT
+
 ENV uid 1337
 ENV gid 1337
 ENV user lemur
@@ -22,6 +24,7 @@ RUN addgroup -S ${group} -g ${gid} && \
                 gcc \
                 autoconf \
                 automake \
+                libtool \
                 make \
                 nasm  \
                 zlib-dev \
@@ -42,7 +45,7 @@ RUN chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \
     npm install --unsafe-perm && \
     pip3 install -e . && \
     node_modules/.bin/gulp build && \
-    node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
+    node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
     apk del build-dependencies
 
 COPY docker/entrypoint /

lemur/certificates/cli.py

@@ -23,7 +23,7 @@ from lemur.certificates.service import (
     get_certificate_primitives,
     get_all_pending_reissue,
     get_by_name,
-    get_all_certs,
+    get_all_valid_certs,
     get,
     get_all_certs_attached_to_endpoint_without_autorotate,
 )
@@ -210,6 +210,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
 
     status = FAILURE_METRIC_STATUS
 
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+    }
+
     try:
         old_cert = validate_certificate(old_certificate_name)
         new_cert = validate_certificate(new_certificate_name)
@@ -219,26 +223,43 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
             print(
                 f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
             )
+            log_data["message"] = "Rotating endpoint"
+            log_data["endpoint"] = endpoint.dnsname
+            log_data["certificate"] = new_cert.name
             request_rotation(endpoint, new_cert, message, commit)
+            current_app.logger.info(log_data)
 
         elif old_cert and new_cert:
             print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
 
+            log_data["message"] = "Rotating all endpoints"
+            log_data["certificate"] = new_cert.name
+            log_data["certificate_old"] = old_cert.name
+            log_data["message"] = "Rotating endpoint from old to new cert"
             for endpoint in old_cert.endpoints:
                 print(f"[+] Rotating {endpoint.name}")
+                log_data["endpoint"] = endpoint.dnsname
                 request_rotation(endpoint, new_cert, message, commit)
+                current_app.logger.info(log_data)
 
         else:
             print("[+] Rotating all endpoints that have new certificates available")
+            log_data["message"] = "Rotating all endpoints that have new certificates available"
             for endpoint in endpoint_service.get_all_pending_rotation():
+                log_data["endpoint"] = endpoint.dnsname
                 if len(endpoint.certificate.replaced) == 1:
                     print(
                         f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
                     )
+                    log_data["certificate"] = endpoint.certificate.replaced[0].name
                     request_rotation(
                         endpoint, endpoint.certificate.replaced[0], message, commit
                     )
+                    current_app.logger.info(log_data)
+
                 else:
+                    log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
+                    print(log_data)
                     metrics.send(
                         "endpoint_rotation",
                         "counter",
@@ -636,7 +657,14 @@ def check_revoked():
     encounters an issue with verification it marks the certificate status
     as `unknown`.
     """
-    for cert in get_all_certs():
+
+    log_data = {
+        "function": f"{__name__}.{sys._getframe().f_code.co_name}",
+        "message": "Checking for revoked Certificates"
+    }
+
+    certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
+    for cert in certs:
         try:
             if cert.chain:
                 status = verify_string(cert.body, cert.chain)
@@ -645,6 +673,20 @@ def check_revoked():
 
             cert.status = "valid" if status else "revoked"
 
+            if cert.status == "revoked":
+                log_data["valid"] = cert.status
+                log_data["certificate_name"] = cert.name
+                log_data["certificate_id"] = cert.id
+                metrics.send(
+                    "certificate_revoked",
+                    "counter",
+                    1,
+                    metric_tags={"status": log_data["valid"],
+                                 "certificate_name": log_data["certificate_name"],
+                                 "certificate_id": log_data["certificate_id"]},
+                )
+                current_app.logger.info(log_data)
+
         except Exception as e:
             sentry.captureException()
             current_app.logger.exception(e)

lemur/certificates/service.py

@@ -103,6 +103,27 @@ def get_all_certs():
     return Certificate.query.all()
 
 
+def get_all_valid_certs(authority_plugin_name):
+    """
+    Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
+    ignored if no authority_plugin_name provided.
+
+    Note that depending on the DB size retrieving all certificates might an expensive operation
+
+    :return:
+    """
+    if authority_plugin_name:
+        return (
+            Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
+                Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
+                Authority.plugin_name.in_(authority_plugin_name)).all()
+        )
+    else:
+        return (
+            Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
+        )
+
+
 def get_all_pending_cleaning_expired(source):
     """
     Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not

lemur/certificates/verify.py

@@ -8,6 +8,7 @@
 import requests
 import subprocess
 from flask import current_app
+from lemur.extensions import sentry
 from requests.exceptions import ConnectionError, InvalidSchema
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
@@ -152,10 +153,18 @@ def verify(cert_path, issuer_chain_path):
 
     # OCSP is our main source of truth, in a lot of cases CRLs
     # have been deprecated and are no longer updated
+    try:
         verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
+    except Exception as e:
+        sentry.captureException()
+        current_app.logger.exception(e)
 
     if verify_result is None:
+        try:
             verify_result = crl_verify(cert, cert_path)
+        except Exception as e:
+            sentry.captureException()
+            current_app.logger.exception(e)
 
     if verify_result is None:
         current_app.logger.debug("Failed to verify {}".format(cert.serial_number))

lemur/dns_providers/util.py

@@ -31,11 +31,11 @@ class DNSResolveError(DNSError):
 
 def is_valid_domain(domain):
     """Checks if a domain is syntactically valid and returns a bool"""
-    if len(domain) > 253:
-        return False
     if domain[-1] == ".":
         domain = domain[:-1]
-    fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(?<!-)\.?)+(?:[a-zA-Z]{2,})$)", re.IGNORECASE)
+    if len(domain) > 253:
+        return False
+    fqdn_re = re.compile("(?=^.{1,63}$)(^(?:[a-z0-9_](?:-*[a-z0-9_])+)$|^[a-z0-9]$)", re.IGNORECASE)
     return all(fqdn_re.match(d) for d in domain.split("."))
 
 

lemur/plugins/lemur_email/templates/expiration.html

@@ -75,7 +75,8 @@
                                             </tr>
                                             <tr>
                                                 <td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
-                                                    <br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used.
+                                                    <br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used,
+                                                    and disable notifications via the Notify toggle in Lemur, if applicable.
                                                     <table border="0" cellspacing="0" cellpadding="0"
                                                            style="margin-top:48px;margin-bottom:48px">
                                                         <tbody>

lemur/tests/test_dns_providers.py

@@ -4,9 +4,20 @@ from lemur.dns_providers import util as dnsutil
 
 class TestDNSProvider(unittest.TestCase):
     def test_is_valid_domain(self):
-        self.assertTrue(dnsutil.is_valid_domain("example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example.com'))
-        self.assertTrue(dnsutil.is_valid_domain("foo.bar.org"))
+        self.assertTrue(dnsutil.is_valid_domain('foo.bar.org'))
-        self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('exam--ple.io'))
-        self.assertFalse(dnsutil.is_valid_domain("e/xample.com"))
+        self.assertTrue(dnsutil.is_valid_domain('a.example.com'))
-        self.assertFalse(dnsutil.is_valid_domain("exam\ple.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example.io'))
-        self.assertFalse(dnsutil.is_valid_domain("*.example.com"))
+        self.assertTrue(dnsutil.is_valid_domain('example-of-under-63-character-domain-label-length-limit-1234567.com'))
+        self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com'))
+        self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('e/xample.com'))
+        self.assertFalse(dnsutil.is_valid_domain('exam\ple.com'))
+        self.assertFalse(dnsutil.is_valid_domain('<example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('*.example.com'))
+        self.assertFalse(dnsutil.is_valid_domain('-example.io'))
+        self.assertFalse(dnsutil.is_valid_domain('example-.io'))
+        self.assertFalse(dnsutil.is_valid_domain('example..io'))
+        self.assertFalse(dnsutil.is_valid_domain('exa mple.io'))
+        self.assertFalse(dnsutil.is_valid_domain('-'))

requirements-dev.txt

@@ -6,7 +6,7 @@
 #
 appdirs==1.4.3            # via virtualenv
 bleach==3.1.4             # via readme-renderer
-certifi==2020.4.5.1       # via requests
+certifi==2020.4.5.2       # via requests
 cffi==1.14.0              # via cryptography
 cfgv==3.1.0               # via pre-commit
 chardet==3.0.4            # via requests
@@ -21,7 +21,7 @@ invoke==1.4.1             # via -r requirements-dev.in
 jeepney==0.4.3            # via keyring, secretstorage
 keyring==21.2.0           # via twine
 mccabe==0.6.1             # via flake8
-nodeenv==1.3.5            # via -r requirements-dev.in, pre-commit
+nodeenv==1.4.0            # via -r requirements-dev.in, pre-commit
 pkginfo==1.5.0.1          # via twine
 pre-commit==2.4.0         # via -r requirements-dev.in
 pycodestyle==2.3.1        # via flake8

requirements-docs.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
 #
-acme==1.4.0               # via -r requirements.txt
+acme==1.5.0               # via -r requirements.txt
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2  # via -r requirements.txt
 alembic==1.4.2            # via -r requirements.txt, flask-migrate
@@ -16,10 +16,10 @@ babel==2.8.0              # via sphinx
 bcrypt==3.1.7             # via -r requirements.txt, flask-bcrypt, paramiko
 billiard==3.6.3.0         # via -r requirements.txt, celery
 blinker==1.4              # via -r requirements.txt, flask-mail, flask-principal, raven
-boto3==1.13.18            # via -r requirements.txt
+boto3==1.13.19            # via -r requirements.txt
-botocore==1.16.18         # via -r requirements.txt, boto3, s3transfer
+botocore==1.16.25         # via -r requirements.txt, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.txt
-certifi==2020.4.5.1       # via -r requirements.txt, requests
+certifi==2020.4.5.2       # via -r requirements.txt, requests
 certsrv==2.1.1            # via -r requirements.txt
 cffi==1.14.0              # via -r requirements.txt, bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements.txt, requests
@@ -38,14 +38,14 @@ flask-principal==0.4.0    # via -r requirements.txt
 flask-replicated==1.3     # via -r requirements.txt
 flask-restful==0.3.8      # via -r requirements.txt
 flask-script==2.0.6       # via -r requirements.txt
-flask-sqlalchemy==2.4.1   # via -r requirements.txt, flask-migrate
+flask-sqlalchemy==2.4.3   # via -r requirements.txt, flask-migrate
 flask==1.1.2              # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
 future==0.18.2            # via -r requirements.txt, cloudflare
 gunicorn==20.0.4          # via -r requirements.txt
 hvac==0.10.3              # via -r requirements.txt
 idna==2.9                 # via -r requirements.txt, requests
 imagesize==1.2.0          # via sphinx
-inflection==0.4.0         # via -r requirements.txt
+inflection==0.5.0         # via -r requirements.txt
 itsdangerous==1.1.0       # via -r requirements.txt, flask
 javaobj-py3==0.4.0.1      # via -r requirements.txt, pyjks
 jinja2==2.11.2            # via -r requirements.txt, flask, sphinx
@@ -57,7 +57,7 @@ lockfile==0.12.2          # via -r requirements.txt
 logmatic-python==0.1.7    # via -r requirements.txt
 mako==1.1.2               # via -r requirements.txt, alembic
 markupsafe==1.1.1         # via -r requirements.txt, jinja2, mako
-marshmallow-sqlalchemy==0.23.0  # via -r requirements.txt
+marshmallow-sqlalchemy==0.23.1  # via -r requirements.txt
 marshmallow==2.20.4       # via -r requirements.txt, marshmallow-sqlalchemy
 ndg-httpsclient==0.5.1    # via -r requirements.txt
 packaging==20.3           # via sphinx
@@ -82,7 +82,7 @@ python-ldap==3.2.0        # via -r requirements.txt
 pytz==2019.3              # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339
 pyyaml==5.3.1             # via -r requirements.txt, cloudflare
 raven[flask]==6.10.0      # via -r requirements.txt
-redis==3.5.2              # via -r requirements.txt, celery
+redis==3.5.3              # via -r requirements.txt, celery
 requests-toolbelt==0.9.1  # via -r requirements.txt, acme
 requests[security]==2.23.0  # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
 retrying==1.3.3           # via -r requirements.txt
@@ -90,7 +90,7 @@ s3transfer==0.3.3         # via -r requirements.txt, boto3
 six==1.15.0               # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils
 snowballstemmer==2.0.0    # via sphinx
 sphinx-rtd-theme==0.4.3   # via -r requirements-docs.in
-sphinx==3.0.3             # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
+sphinx==3.1.0             # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
 sphinxcontrib-applehelp==1.0.2  # via sphinx
 sphinxcontrib-devhelp==1.0.2  # via sphinx
 sphinxcontrib-htmlhelp==1.0.3  # via sphinx

requirements-tests.txt

@@ -10,10 +10,10 @@ aws-sam-translator==1.22.0  # via cfn-lint
 aws-xray-sdk==2.5.0       # via moto
 bandit==1.6.2             # via -r requirements-tests.in
 black==19.10b0            # via -r requirements-tests.in
-boto3==1.13.18            # via aws-sam-translator, moto
+boto3==1.13.19            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.16.18         # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.16.25         # via aws-xray-sdk, boto3, moto, s3transfer
-certifi==2020.4.5.1       # via requests
+certifi==2020.4.5.2       # via requests
 cffi==1.14.0              # via cryptography
 cfn-lint==0.29.5          # via moto
 chardet==3.0.4            # via requests
@@ -59,13 +59,13 @@ pyflakes==2.2.0           # via -r requirements-tests.in
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.16.0        # via jsonschema
 pytest-flask==1.0.0       # via -r requirements-tests.in
-pytest-mock==3.1.0        # via -r requirements-tests.in
+pytest-mock==3.1.1        # via -r requirements-tests.in
-pytest==5.4.2             # via -r requirements-tests.in, pytest-flask, pytest-mock
+pytest==5.4.3             # via -r requirements-tests.in, pytest-flask, pytest-mock
 python-dateutil==2.8.1    # via botocore, faker, freezegun, moto
 python-jose==3.1.0        # via moto
 pytz==2019.3              # via moto
 pyyaml==5.3.1             # via -r requirements-tests.in, bandit, cfn-lint, moto
-redis==3.5.2              # via fakeredis
+redis==3.5.3              # via fakeredis
 regex==2020.4.4           # via black
 requests-mock==1.8.0      # via -r requirements-tests.in
 requests==2.23.0          # via docker, moto, requests-mock, responses

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --no-index --output-file=requirements.txt requirements.in
 #
-acme==1.4.0               # via -r requirements.in
+acme==1.5.0               # via -r requirements.in
 alembic-autogenerate-enums==0.0.2  # via -r requirements.in
 alembic==1.4.2            # via flask-migrate
 amqp==2.5.2               # via kombu
@@ -14,10 +14,10 @@ asyncpool==1.0            # via -r requirements.in
 bcrypt==3.1.7             # via flask-bcrypt, paramiko
 billiard==3.6.3.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.13.18            # via -r requirements.in
+boto3==1.13.19            # via -r requirements.in
-botocore==1.16.18         # via -r requirements.in, boto3, s3transfer
+botocore==1.16.25         # via -r requirements.in, boto3, s3transfer
 celery[redis]==4.4.2      # via -r requirements.in
-certifi==2020.4.5.1       # via -r requirements.in, requests
+certifi==2020.4.5.2       # via -r requirements.in, requests
 certsrv==2.1.1            # via -r requirements.in
 cffi==1.14.0              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
@@ -36,13 +36,13 @@ flask-principal==0.4.0    # via -r requirements.in
 flask-replicated==1.3     # via -r requirements.in
 flask-restful==0.3.8      # via -r requirements.in
 flask-script==2.0.6       # via -r requirements.in
-flask-sqlalchemy==2.4.1   # via -r requirements.in, flask-migrate
+flask-sqlalchemy==2.4.3   # via -r requirements.in, flask-migrate
 flask==1.1.2              # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
 future==0.18.2            # via -r requirements.in, cloudflare
 gunicorn==20.0.4          # via -r requirements.in
 hvac==0.10.3              # via -r requirements.in
 idna==2.9                 # via requests
-inflection==0.4.0         # via -r requirements.in
+inflection==0.5.0         # via -r requirements.in
 itsdangerous==1.1.0       # via flask
 javaobj-py3==0.4.0.1      # via pyjks
 jinja2==2.11.2            # via -r requirements.in, flask
@@ -54,7 +54,7 @@ lockfile==0.12.2          # via -r requirements.in
 logmatic-python==0.1.7    # via -r requirements.in
 mako==1.1.2               # via alembic
 markupsafe==1.1.1         # via jinja2, mako
-marshmallow-sqlalchemy==0.23.0  # via -r requirements.in
+marshmallow-sqlalchemy==0.23.1  # via -r requirements.in
 marshmallow==2.20.4       # via -r requirements.in, marshmallow-sqlalchemy
 ndg-httpsclient==0.5.1    # via -r requirements.in
 paramiko==2.7.1           # via -r requirements.in
@@ -76,7 +76,7 @@ python-ldap==3.2.0        # via -r requirements.in
 pytz==2019.3              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==5.3.1             # via -r requirements.in, cloudflare
 raven[flask]==6.10.0      # via -r requirements.in
-redis==3.5.2              # via -r requirements.in, celery
+redis==3.5.3              # via -r requirements.in, celery
 requests-toolbelt==0.9.1  # via acme
 requests[security]==2.23.0  # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
 retrying==1.3.3           # via -r requirements.in