Merge branch 'master' into master

lemur/plugins/lemur_acme/plugin.py

@@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry
 from lemur.plugins import lemur_acme as acme
 from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns
+from retrying import retry
 
 
 class AuthorizationRecord(object):
@@ -197,6 +198,7 @@ class AcmeHandler(object):
         )
         return pem_certificate, pem_certificate_chain
 
+    @retry(stop_max_attempt_number=5, wait_fixed=5000)
     def setup_acme_client(self, authority):
         if not authority.options:
             raise InvalidAuthority("Invalid authority. Options not set")

lemur/plugins/lemur_aws/iam.py

@@ -10,7 +10,7 @@ import botocore
 
 from retrying import retry
 
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.plugins.lemur_aws.sts import sts_client
 
 
@@ -122,9 +122,11 @@ def get_certificate(name, **kwargs):
     """
     client = kwargs.pop("client")
     metrics.send("get_certificate", "counter", 1, metric_tags={"name": name})
-    return client.get_server_certificate(ServerCertificateName=name)[
+    try:
-        "ServerCertificate"
+        return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"]
-    ]
+    except client.exceptions.NoSuchEntityException:
+        sentry.captureException()
+        return None
 
 
 @sts_client("iam")

lemur/plugins/lemur_aws/plugin.py

@@ -32,7 +32,9 @@
 .. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
 .. moduleauthor:: Harm Weites <harm@weites.com>
 """
+from acme.errors import ClientError
 from flask import current_app
+from lemur.extensions import sentry, metrics
 
 from lemur.plugins import lemur_aws as aws
 from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
@@ -271,6 +273,29 @@ class AWSSourcePlugin(SourcePlugin):
         account_number = self.get_option("accountNumber", options)
         iam.delete_cert(certificate.name, account_number=account_number)
 
+    def get_certificate_by_name(self, certificate_name, options):
+        account_number = self.get_option("accountNumber", options)
+        # certificate name may contain path, in which case we remove it
+        if "/" in certificate_name:
+            certificate_name = certificate_name.split('/')[-1]
+        try:
+            cert = iam.get_certificate(certificate_name, account_number=account_number)
+            if cert:
+                return dict(
+                    body=cert["CertificateBody"],
+                    chain=cert.get("CertificateChain"),
+                    name=cert["ServerCertificateMetadata"]["ServerCertificateName"],
+                )
+        except ClientError:
+            current_app.logger.warning(
+                "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name))
+            sentry.captureException()
+            metrics.send(
+                "get_elb_certificate_failed", "counter", 1,
+                metric_tags={"certificate_name": certificate_name, "account_number": account_number}
+            )
+        return None
+
 
 class AWSDestinationPlugin(DestinationPlugin):
     title = "AWS"

lemur/sources/service.py

@@ -15,7 +15,7 @@ from lemur.sources.models import Source
 from lemur.certificates.models import Certificate
 from lemur.certificates import service as certificate_service
 from lemur.endpoints import service as endpoint_service
-from lemur.extensions import metrics
+from lemur.extensions import metrics, sentry
 from lemur.destinations import service as destination_service
 
 from lemur.certificates.schemas import CertificateUploadInputSchema
@@ -66,7 +66,7 @@ def sync_update_destination(certificate, source):
 
 
 def sync_endpoints(source):
-    new, updated = 0, 0
+    new, updated, updated_by_hash = 0, 0, 0
     current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
     s = plugins.get(source.plugin_name)
 
@@ -89,6 +89,39 @@ def sync_endpoints(source):
 
         endpoint["certificate"] = certificate_service.get_by_name(certificate_name)
 
+        # if get cert by name failed, we attempt a search via serial number and hash comparison
+        # and link the endpoint certificate to Lemur certificate
+        if not endpoint["certificate"]:
+            certificate_attached_to_endpoint = None
+            try:
+                certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options)
+            except NotImplementedError:
+                current_app.logger.warning(
+                    "Unable to describe server certificate for endpoints in source {0}:"
+                    " plugin has not implemented 'get_certificate_by_name'".format(
+                        source.label
+                    )
+                )
+                sentry.captureException()
+
+            if certificate_attached_to_endpoint:
+                lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint)
+                updated_by_hash += updated_by_hash_tmp
+
+                if lemur_matching_cert:
+                    endpoint["certificate"] = lemur_matching_cert[0]
+
+                if len(lemur_matching_cert) > 1:
+                    current_app.logger.error(
+                        "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format(
+                            len(lemur_matching_cert), certificate_name, endpoint["name"]
+                        )
+                    )
+                    metrics.send("endpoint.certificate.conflict",
+                                 "gauge", len(lemur_matching_cert),
+                                 metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
+                                              "acct": s.get_option("accountNumber", source.options)})
+
         if not endpoint["certificate"]:
             current_app.logger.error(
                 "Certificate Not Found. Name: {0} Endpoint: {1}".format(
@@ -97,7 +130,8 @@ def sync_endpoints(source):
             )
             metrics.send("endpoint.certificate.not.found",
                          "counter", 1,
-                         metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)})
+                         metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
+                                      "acct": s.get_option("accountNumber", source.options)})
             continue
 
         policy = endpoint.pop("policy")
@@ -122,18 +156,11 @@ def sync_endpoints(source):
             endpoint_service.update(exists.id, **endpoint)
             updated += 1
 
-    return new, updated
+    return new, updated, updated_by_hash
 
 
-# TODO this is very slow as we don't batch update certificates
+def find_cert(certificate):
-def sync_certificates(source, user):
+    updated_by_hash = 0
-    new, updated = 0, 0
-
-    current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
-    s = plugins.get(source.plugin_name)
-    certificates = s.get_certificates(source.options)
-
-    for certificate in certificates:
     exists = False
 
     if certificate.get("search", None):
@@ -152,12 +179,27 @@ def sync_certificates(source, user):
         cert = parse_certificate(certificate["body"])
         matching_serials = certificate_service.get_by_serial(serial(cert))
         exists = find_matching_certificates_by_hash(cert, matching_serials)
+        updated_by_hash += 1
+
+    exists = [x for x in exists if x]
+    return exists, updated_by_hash
+
+
+# TODO this is very slow as we don't batch update certificates
+def sync_certificates(source, user):
+    new, updated, updated_by_hash = 0, 0, 0
+
+    current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
+    s = plugins.get(source.plugin_name)
+    certificates = s.get_certificates(source.options)
+
+    for certificate in certificates:
+        exists, updated_by_hash = find_cert(certificate)
 
         if not certificate.get("owner"):
             certificate["owner"] = user.email
 
         certificate["creator"] = user
-        exists = [x for x in exists if x]
 
         if not exists:
             certificate_create(certificate, source)
@@ -172,12 +214,20 @@ def sync_certificates(source, user):
                 certificate_update(e, source)
                 updated += 1
 
-    return new, updated
+    return new, updated, updated_by_hash
 
 
 def sync(source, user):
-    new_certs, updated_certs = sync_certificates(source, user)
+    new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user)
-    new_endpoints, updated_endpoints = sync_endpoints(source)
+    new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source)
+
+    metrics.send("sync.updated_certs_by_hash",
+                 "gauge", updated_certs_by_hash,
+                 metric_tags={"source": source.label})
+
+    metrics.send("sync.updated_endpoints_by_hash",
+                 "gauge", updated_endpoints_by_hash,
+                 metric_tags={"source": source.label})
 
     source.last_run = arrow.utcnow()
     database.update(source)

requirements-dev.txt

@@ -30,11 +30,11 @@ requests==2.22.0          # via requests-toolbelt, twine
 six==1.12.0               # via bleach, cfgv, pre-commit, readme-renderer
 toml==0.10.0              # via pre-commit
 tqdm==4.36.1              # via twine
-twine==1.15.0
+twine==2.0.0
-urllib3==1.25.5           # via requests
+urllib3==1.25.6           # via requests
 virtualenv==16.7.5        # via pre-commit
 webencodings==0.5.1       # via bleach
 zipp==0.6.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.2.0        # via twine
+# setuptools==41.4.0        # via twine

requirements-docs.txt

@@ -4,25 +4,25 @@
 #
 #    pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
 #
-acme==0.38.0
+acme==0.39.0
 alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
-alembic==1.2.0
+alembic==1.2.1
-amqp==2.5.1
+amqp==2.5.2
 aniso8601==8.0.0
 arrow==0.15.2
-asn1crypto==0.24.0
+asn1crypto==1.1.0
 asyncpool==1.0
 babel==2.7.0              # via sphinx
 bcrypt==3.1.7
 billiard==3.6.1.0
 blinker==1.4
-boto3==1.9.232
+boto3==1.9.250
-botocore==1.12.232
+botocore==1.12.250
 celery[redis]==4.3.0
 certifi==2019.9.11
 certsrv==2.1.1
-cffi==1.12.3
+cffi==1.13.0
 chardet==3.0.4
 click==7.0
 cloudflare==2.3.0
@@ -39,9 +39,9 @@ flask-principal==0.4.0
 flask-replicated==1.3
 flask-restful==0.3.7
 flask-script==2.0.6
-flask-sqlalchemy==2.4.0
+flask-sqlalchemy==2.4.1
 flask==1.1.1
-future==0.17.1
+future==0.18.0
 gunicorn==19.9.0
 hvac==0.9.5
 idna==2.8
@@ -49,7 +49,7 @@ imagesize==1.1.0          # via sphinx
 inflection==0.3.1
 itsdangerous==1.1.0
 javaobj-py3==0.3.0
-jinja2==2.10.1
+jinja2==2.10.3
 jmespath==0.9.4
 josepy==1.2.0
 jsonlines==1.2.0
@@ -66,7 +66,7 @@ packaging==19.2           # via sphinx
 paramiko==2.6.0
 pem==19.2.0
 psycopg2==2.8.3
-pyasn1-modules==0.2.6
+pyasn1-modules==0.2.7
 pyasn1==0.4.7
 pycparser==2.19
 pycryptodomex==3.9.0
@@ -80,16 +80,16 @@ pyrfc3339==1.1
 python-dateutil==2.8.0
 python-editor==1.0.4
 python-json-logger==0.1.11
-pytz==2019.2
+pytz==2019.3
 pyyaml==5.1.2
 raven[flask]==6.10.0
-redis==3.3.8
+redis==3.3.11
 requests-toolbelt==0.9.1
 requests[security]==2.22.0
 retrying==1.3.3
 s3transfer==0.2.1
 six==1.12.0
-snowballstemmer==1.9.1    # via sphinx
+snowballstemmer==2.0.0    # via sphinx
 sphinx-rtd-theme==0.4.3
 sphinx==2.2.0
 sphinxcontrib-applehelp==1.0.1  # via sphinx
@@ -100,13 +100,13 @@ sphinxcontrib-jsmath==1.0.1  # via sphinx
 sphinxcontrib-qthelp==1.0.2  # via sphinx
 sphinxcontrib-serializinghtml==1.1.3  # via sphinx
 sqlalchemy-utils==0.34.2
-sqlalchemy==1.3.8
+sqlalchemy==1.3.10
-tabulate==0.8.3
+tabulate==0.8.5
 twofish==0.3.0
-urllib3==1.25.5
+urllib3==1.25.6
 vine==1.3.0
 werkzeug==0.16.0
 xmltodict==0.12.0
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.2.0        # via acme, josepy, sphinx
+# setuptools==41.4.0        # via acme, josepy, sphinx

requirements-tests.txt

@@ -5,45 +5,45 @@
 #    pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in
 #
 appdirs==1.4.3            # via black
-asn1crypto==0.24.0        # via cryptography
+asn1crypto==1.1.0         # via cryptography
 atomicwrites==1.3.0       # via pytest
-attrs==19.1.0             # via black, jsonschema, pytest
+attrs==19.3.0             # via black, jsonschema, pytest
-aws-sam-translator==1.14.0  # via cfn-lint
+aws-sam-translator==1.15.1  # via cfn-lint
 aws-xray-sdk==2.4.2       # via moto
 bandit==1.6.2
 black==19.3b0
-boto3==1.9.232            # via aws-sam-translator, moto
+boto3==1.9.250            # via aws-sam-translator, moto
 boto==2.49.0              # via moto
-botocore==1.12.232        # via aws-xray-sdk, boto3, moto, s3transfer
+botocore==1.12.250        # via aws-xray-sdk, boto3, moto, s3transfer
 certifi==2019.9.11        # via requests
-cffi==1.12.3              # via cryptography
+cffi==1.13.0              # via cryptography
-cfn-lint==0.24.1          # via moto
+cfn-lint==0.24.4          # via moto
 chardet==3.0.4            # via requests
 click==7.0                # via black, flask
 coverage==4.5.4
 cryptography==2.7         # via moto, sshpubkeys
 datetime==4.3             # via moto
-docker==4.0.2             # via moto
+docker==4.1.0             # via moto
 docutils==0.15.2          # via botocore
-ecdsa==0.13.2             # via python-jose, sshpubkeys
+ecdsa==0.13.3             # via python-jose, sshpubkeys
 factory-boy==2.12.0
-faker==2.0.2
+faker==2.0.3
 fakeredis==1.0.5
 flask==1.1.1              # via pytest-flask
 freezegun==0.3.12
-future==0.17.1            # via aws-xray-sdk, python-jose
+future==0.18.0            # via aws-xray-sdk, python-jose
-gitdb2==2.0.5             # via gitpython
+gitdb2==2.0.6             # via gitpython
-gitpython==3.0.2          # via bandit
+gitpython==3.0.3          # via bandit
 idna==2.8                 # via moto, requests
-importlib-metadata==0.23  # via pluggy, pytest
+importlib-metadata==0.23  # via jsonschema, pluggy, pytest
 itsdangerous==1.1.0       # via flask
-jinja2==2.10.1            # via flask, moto
+jinja2==2.10.3            # via flask, moto
 jmespath==0.9.4           # via boto3, botocore
 jsondiff==1.1.2           # via moto
 jsonpatch==1.24           # via cfn-lint
 jsonpickle==1.2           # via aws-xray-sdk
 jsonpointer==2.0          # via jsonpatch
-jsonschema==3.0.2         # via aws-sam-translator, cfn-lint
+jsonschema==3.1.1         # via aws-sam-translator, cfn-lint
 markupsafe==1.1.1         # via jinja2
 mock==3.0.5               # via moto
 more-itertools==7.2.0     # via pytest, zipp
@@ -59,13 +59,13 @@ pyflakes==2.1.1
 pyparsing==2.4.2          # via packaging
 pyrsistent==0.15.4        # via jsonschema
 pytest-flask==0.15.0
-pytest-mock==1.10.4
+pytest-mock==1.11.1
-pytest==5.1.2
+pytest==5.2.1
 python-dateutil==2.8.0    # via botocore, faker, freezegun, moto
 python-jose==3.0.1        # via moto
-pytz==2019.2              # via datetime, moto
+pytz==2019.3              # via datetime, moto
 pyyaml==5.1.2
-redis==3.3.8              # via fakeredis
+redis==3.3.11             # via fakeredis
 requests-mock==1.7.0
 requests==2.22.0          # via docker, moto, requests-mock, responses
 responses==0.10.6         # via moto
@@ -78,7 +78,7 @@ sshpubkeys==3.1.0         # via moto
 stevedore==1.31.0         # via bandit
 text-unidecode==1.3       # via faker
 toml==0.10.0              # via black
-urllib3==1.25.5           # via botocore, requests
+urllib3==1.25.6           # via botocore, requests
 wcwidth==0.1.7            # via pytest
 websocket-client==0.56.0  # via docker
 werkzeug==0.16.0          # via flask, moto, pytest-flask
@@ -88,4 +88,4 @@ zipp==0.6.0               # via importlib-metadata
 zope.interface==4.6.0     # via datetime
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.2.0        # via cfn-lint, jsonschema, zope.interface
+# setuptools==41.4.0        # via cfn-lint, jsonschema, zope.interface

requirements.txt

@@ -4,23 +4,23 @@
 #
 #    pip-compile --no-index --output-file=requirements.txt requirements.in
 #
-acme==0.38.0
+acme==0.39.0
 alembic-autogenerate-enums==0.0.2
-alembic==1.2.0            # via flask-migrate
+alembic==1.2.1            # via flask-migrate
-amqp==2.5.1               # via kombu
+amqp==2.5.2               # via kombu
 aniso8601==8.0.0          # via flask-restful
 arrow==0.15.2
-asn1crypto==0.24.0        # via cryptography
+asn1crypto==1.1.0         # via cryptography
 asyncpool==1.0
 bcrypt==3.1.7             # via flask-bcrypt, paramiko
 billiard==3.6.1.0         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.232
+boto3==1.9.250
-botocore==1.12.232
+botocore==1.12.250
 celery[redis]==4.3.0
 certifi==2019.9.11
 certsrv==2.1.1
-cffi==1.12.3              # via bcrypt, cryptography, pynacl
+cffi==1.13.0              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 click==7.0                # via flask
 cloudflare==2.3.0
@@ -37,16 +37,16 @@ flask-principal==0.4.0
 flask-replicated==1.3
 flask-restful==0.3.7
 flask-script==2.0.6
-flask-sqlalchemy==2.4.0
+flask-sqlalchemy==2.4.1
 flask==1.1.1
-future==0.17.1
+future==0.18.0
 gunicorn==19.9.0
 hvac==0.9.5
 idna==2.8                 # via requests
 inflection==0.3.1
 itsdangerous==1.1.0       # via flask
 javaobj-py3==0.3.0        # via pyjks
-jinja2==2.10.1
+jinja2==2.10.3
 jmespath==0.9.4           # via boto3, botocore
 josepy==1.2.0             # via acme
 jsonlines==1.2.0          # via cloudflare
@@ -62,7 +62,7 @@ ndg-httpsclient==0.5.1
 paramiko==2.6.0
 pem==19.2.0
 psycopg2==2.8.3
-pyasn1-modules==0.2.6     # via pyjks, python-ldap
+pyasn1-modules==0.2.7     # via pyjks, python-ldap
 pyasn1==0.4.7             # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap
 pycparser==2.19           # via cffi
 pycryptodomex==3.9.0      # via pyjks
@@ -75,23 +75,23 @@ python-dateutil==2.8.0    # via alembic, arrow, botocore
 python-editor==1.0.4      # via alembic
 python-json-logger==0.1.11  # via logmatic-python
 python-ldap==3.2.0
-pytz==2019.2              # via acme, celery, flask-restful, pyrfc3339
+pytz==2019.3              # via acme, celery, flask-restful, pyrfc3339
 pyyaml==5.1.2
 raven[flask]==6.10.0
-redis==3.3.8
+redis==3.3.11
 requests-toolbelt==0.9.1  # via acme
 requests[security]==2.22.0
 retrying==1.3.3
 s3transfer==0.2.1         # via boto3
 six==1.12.0
 sqlalchemy-utils==0.34.2
-sqlalchemy==1.3.8         # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
+sqlalchemy==1.3.10        # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
-tabulate==0.8.3
+tabulate==0.8.5
 twofish==0.3.0            # via pyjks
-urllib3==1.25.5           # via botocore, requests
+urllib3==1.25.6           # via botocore, requests
 vine==1.3.0               # via amqp, celery
 werkzeug==0.16.0          # via flask
 xmltodict==0.12.0
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.2.0        # via acme, josepy
+# setuptools==41.4.0        # via acme, josepy