diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f31ffdcb..e38870d8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns +from retrying import retry class AuthorizationRecord(object): @@ -197,6 +198,7 @@ class AcmeHandler(object): ) return pem_certificate, pem_certificate_chain + @retry(stop_max_attempt_number=5, wait_fixed=5000) def setup_acme_client(self, authority): if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 67c35262..13590ddd 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -10,7 +10,7 @@ import botocore from retrying import retry -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.lemur_aws.sts import sts_client @@ -122,9 +122,11 @@ def get_certificate(name, **kwargs): """ client = kwargs.pop("client") metrics.send("get_certificate", "counter", 1, metric_tags={"name": name}) - return client.get_server_certificate(ServerCertificateName=name)[ - "ServerCertificate" - ] + try: + return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"] + except client.exceptions.NoSuchEntityException: + sentry.captureException() + return None @sts_client("iam") diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index cf6c8643..98b01672 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,7 +32,9 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ +from acme.errors import ClientError from flask import current_app +from lemur.extensions import sentry, metrics from lemur.plugins import lemur_aws as aws from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin @@ -271,6 +273,29 @@ class AWSSourcePlugin(SourcePlugin): account_number = self.get_option("accountNumber", options) iam.delete_cert(certificate.name, account_number=account_number) + def get_certificate_by_name(self, certificate_name, options): + account_number = self.get_option("accountNumber", options) + # certificate name may contain path, in which case we remove it + if "/" in certificate_name: + certificate_name = certificate_name.split('/')[-1] + try: + cert = iam.get_certificate(certificate_name, account_number=account_number) + if cert: + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) + except ClientError: + current_app.logger.warning( + "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + sentry.captureException() + metrics.send( + "get_elb_certificate_failed", "counter", 1, + metric_tags={"certificate_name": certificate_name, "account_number": account_number} + ) + return None + class AWSDestinationPlugin(DestinationPlugin): title = "AWS" diff --git a/lemur/sources/service.py b/lemur/sources/service.py index d5bd7426..f69f70f5 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -15,7 +15,7 @@ from lemur.sources.models import Source from lemur.certificates.models import Certificate from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema @@ -66,7 +66,7 @@ def sync_update_destination(certificate, source): def sync_endpoints(source): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving endpoints from {0}".format(source.label)) s = plugins.get(source.plugin_name) @@ -89,6 +89,39 @@ def sync_endpoints(source): endpoint["certificate"] = certificate_service.get_by_name(certificate_name) + # if get cert by name failed, we attempt a search via serial number and hash comparison + # and link the endpoint certificate to Lemur certificate + if not endpoint["certificate"]: + certificate_attached_to_endpoint = None + try: + certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options) + except NotImplementedError: + current_app.logger.warning( + "Unable to describe server certificate for endpoints in source {0}:" + " plugin has not implemented 'get_certificate_by_name'".format( + source.label + ) + ) + sentry.captureException() + + if certificate_attached_to_endpoint: + lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint) + updated_by_hash += updated_by_hash_tmp + + if lemur_matching_cert: + endpoint["certificate"] = lemur_matching_cert[0] + + if len(lemur_matching_cert) > 1: + current_app.logger.error( + "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format( + len(lemur_matching_cert), certificate_name, endpoint["name"] + ) + ) + metrics.send("endpoint.certificate.conflict", + "gauge", len(lemur_matching_cert), + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) + if not endpoint["certificate"]: current_app.logger.error( "Certificate Not Found. Name: {0} Endpoint: {1}".format( @@ -97,7 +130,8 @@ def sync_endpoints(source): ) metrics.send("endpoint.certificate.not.found", "counter", 1, - metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) continue policy = endpoint.pop("policy") @@ -122,42 +156,50 @@ def sync_endpoints(source): endpoint_service.update(exists.id, **endpoint) updated += 1 - return new, updated + return new, updated, updated_by_hash + + +def find_cert(certificate): + updated_by_hash = 0 + exists = False + + if certificate.get("search", None): + conditions = certificate.pop("search") + exists = certificate_service.get_by_attributes(conditions) + + if not exists and certificate.get("name"): + result = certificate_service.get_by_name(certificate["name"]) + if result: + exists = [result] + + if not exists and certificate.get("serial"): + exists = certificate_service.get_by_serial(certificate["serial"]) + + if not exists: + cert = parse_certificate(certificate["body"]) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) + updated_by_hash += 1 + + exists = [x for x in exists if x] + return exists, updated_by_hash # TODO this is very slow as we don't batch update certificates def sync_certificates(source, user): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving certificates from {0}".format(source.label)) s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) for certificate in certificates: - exists = False - - if certificate.get("search", None): - conditions = certificate.pop("search") - exists = certificate_service.get_by_attributes(conditions) - - if not exists and certificate.get("name"): - result = certificate_service.get_by_name(certificate["name"]) - if result: - exists = [result] - - if not exists and certificate.get("serial"): - exists = certificate_service.get_by_serial(certificate["serial"]) - - if not exists: - cert = parse_certificate(certificate["body"]) - matching_serials = certificate_service.get_by_serial(serial(cert)) - exists = find_matching_certificates_by_hash(cert, matching_serials) + exists, updated_by_hash = find_cert(certificate) if not certificate.get("owner"): certificate["owner"] = user.email certificate["creator"] = user - exists = [x for x in exists if x] if not exists: certificate_create(certificate, source) @@ -172,12 +214,20 @@ def sync_certificates(source, user): certificate_update(e, source) updated += 1 - return new, updated + return new, updated, updated_by_hash def sync(source, user): - new_certs, updated_certs = sync_certificates(source, user) - new_endpoints, updated_endpoints = sync_endpoints(source) + new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user) + new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source) + + metrics.send("sync.updated_certs_by_hash", + "gauge", updated_certs_by_hash, + metric_tags={"source": source.label}) + + metrics.send("sync.updated_endpoints_by_hash", + "gauge", updated_endpoints_by_hash, + metric_tags={"source": source.label}) source.last_run = arrow.utcnow() database.update(source) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6dff5655..4e940357 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -30,11 +30,11 @@ requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.36.1 # via twine -twine==1.15.0 -urllib3==1.25.5 # via requests +twine==2.0.0 +urllib3==1.25.6 # via requests virtualenv==16.7.5 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via twine +# setuptools==41.4.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 05cfb49c..260c8608 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,25 +4,25 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.38.0 +acme==0.39.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 -amqp==2.5.1 +alembic==1.2.1 +amqp==2.5.2 aniso8601==8.0.0 arrow==0.15.2 -asn1crypto==0.24.0 +asn1crypto==1.1.0 asyncpool==1.0 babel==2.7.0 # via sphinx bcrypt==3.1.7 billiard==3.6.1.0 blinker==1.4 -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 +cffi==1.13.0 chardet==3.0.4 click==7.0 cloudflare==2.3.0 @@ -39,9 +39,9 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 @@ -49,7 +49,7 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 javaobj-py3==0.3.0 -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 josepy==1.2.0 jsonlines==1.2.0 @@ -66,7 +66,7 @@ packaging==19.2 # via sphinx paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 +pyasn1-modules==0.2.7 pyasn1==0.4.7 pycparser==2.19 pycryptodomex==3.9.0 @@ -80,16 +80,16 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 -pytz==2019.2 +pytz==2019.3 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.9.1 # via sphinx +snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 sphinx==2.2.0 sphinxcontrib-applehelp==1.0.1 # via sphinx @@ -100,13 +100,13 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 -tabulate==0.8.3 +sqlalchemy==1.3.10 +tabulate==0.8.5 twofish==0.3.0 -urllib3==1.25.5 +urllib3==1.25.6 vine==1.3.0 werkzeug==0.16.0 xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy, sphinx +# setuptools==41.4.0 # via acme, josepy, sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 29d272a0..e6dc53c5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,45 +5,45 @@ # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # appdirs==1.4.3 # via black -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, jsonschema, pytest -aws-sam-translator==1.14.0 # via cfn-lint +attrs==19.3.0 # via black, jsonschema, pytest +aws-sam-translator==1.15.1 # via cfn-lint aws-xray-sdk==2.4.2 # via moto bandit==1.6.2 black==19.3b0 -boto3==1.9.232 # via aws-sam-translator, moto +boto3==1.9.250 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.232 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.12.250 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2019.9.11 # via requests -cffi==1.12.3 # via cryptography -cfn-lint==0.24.1 # via moto +cffi==1.13.0 # via cryptography +cfn-lint==0.24.4 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask coverage==4.5.4 cryptography==2.7 # via moto, sshpubkeys datetime==4.3 # via moto -docker==4.0.2 # via moto +docker==4.1.0 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.2 # via python-jose, sshpubkeys +ecdsa==0.13.3 # via python-jose, sshpubkeys factory-boy==2.12.0 -faker==2.0.2 +faker==2.0.3 fakeredis==1.0.5 flask==1.1.1 # via pytest-flask freezegun==0.3.12 -future==0.17.1 # via aws-xray-sdk, python-jose -gitdb2==2.0.5 # via gitpython -gitpython==3.0.2 # via bandit +future==0.18.0 # via aws-xray-sdk, python-jose +gitdb2==2.0.6 # via gitpython +gitpython==3.0.3 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.23 # via pluggy, pytest +importlib-metadata==0.23 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.1 # via flask, moto +jinja2==2.10.3 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.24 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==3.0.2 # via aws-sam-translator, cfn-lint +jsonschema==3.1.1 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto more-itertools==7.2.0 # via pytest, zipp @@ -59,13 +59,13 @@ pyflakes==2.1.1 pyparsing==2.4.2 # via packaging pyrsistent==0.15.4 # via jsonschema pytest-flask==0.15.0 -pytest-mock==1.10.4 -pytest==5.1.2 +pytest-mock==1.11.1 +pytest==5.2.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.2 # via datetime, moto +pytz==2019.3 # via datetime, moto pyyaml==5.1.2 -redis==3.3.8 # via fakeredis +redis==3.3.11 # via fakeredis requests-mock==1.7.0 requests==2.22.0 # via docker, moto, requests-mock, responses responses==0.10.6 # via moto @@ -78,7 +78,7 @@ sshpubkeys==3.1.0 # via moto stevedore==1.31.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.16.0 # via flask, moto, pytest-flask @@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface +# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.txt b/requirements.txt index db7e46a7..305fe7e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.38.0 +acme==0.39.0 alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 # via flask-migrate -amqp==2.5.1 # via kombu +alembic==1.2.1 # via flask-migrate +amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful arrow==0.15.2 -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography asyncpool==1.0 bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.1.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 # via bcrypt, cryptography, pynacl +cffi==1.13.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.3.0 @@ -37,16 +37,16 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 # via boto3, botocore josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare @@ -62,7 +62,7 @@ ndg-httpsclient==0.5.1 paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 # via pyjks, python-ldap +pyasn1-modules==0.2.7 # via pyjks, python-ldap pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi pycryptodomex==3.9.0 # via pyjks @@ -75,23 +75,23 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 -pytz==2019.2 # via acme, celery, flask-restful, pyrfc3339 +pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 # via boto3 six==1.12.0 sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.3 +sqlalchemy==1.3.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.5 twofish==0.3.0 # via pyjks -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.16.0 # via flask xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy +# setuptools==41.4.0 # via acme, josepy