From 6f96a8f5b0ccd6aa16d1a3a606bb4c25f4c7ab46 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Tue, 15 Oct 2019 15:47:21 -0700 Subject: [PATCH 01/11] updating requirements --- requirements-dev.txt | 6 +++--- requirements-docs.txt | 36 +++++++++++++++++----------------- requirements-tests.txt | 44 +++++++++++++++++++++--------------------- requirements.txt | 34 ++++++++++++++++---------------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6dff5655..4e940357 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -30,11 +30,11 @@ requests==2.22.0 # via requests-toolbelt, twine six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer toml==0.10.0 # via pre-commit tqdm==4.36.1 # via twine -twine==1.15.0 -urllib3==1.25.5 # via requests +twine==2.0.0 +urllib3==1.25.6 # via requests virtualenv==16.7.5 # via pre-commit webencodings==0.5.1 # via bleach zipp==0.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via twine +# setuptools==41.4.0 # via twine diff --git a/requirements-docs.txt b/requirements-docs.txt index 05cfb49c..260c8608 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,25 +4,25 @@ # # pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in # -acme==0.38.0 +acme==0.39.0 alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 -amqp==2.5.1 +alembic==1.2.1 +amqp==2.5.2 aniso8601==8.0.0 arrow==0.15.2 -asn1crypto==0.24.0 +asn1crypto==1.1.0 asyncpool==1.0 babel==2.7.0 # via sphinx bcrypt==3.1.7 billiard==3.6.1.0 blinker==1.4 -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 +cffi==1.13.0 chardet==3.0.4 click==7.0 cloudflare==2.3.0 @@ -39,9 +39,9 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 @@ -49,7 +49,7 @@ imagesize==1.1.0 # via sphinx inflection==0.3.1 itsdangerous==1.1.0 javaobj-py3==0.3.0 -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 josepy==1.2.0 jsonlines==1.2.0 @@ -66,7 +66,7 @@ packaging==19.2 # via sphinx paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 +pyasn1-modules==0.2.7 pyasn1==0.4.7 pycparser==2.19 pycryptodomex==3.9.0 @@ -80,16 +80,16 @@ pyrfc3339==1.1 python-dateutil==2.8.0 python-editor==1.0.4 python-json-logger==0.1.11 -pytz==2019.2 +pytz==2019.3 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 six==1.12.0 -snowballstemmer==1.9.1 # via sphinx +snowballstemmer==2.0.0 # via sphinx sphinx-rtd-theme==0.4.3 sphinx==2.2.0 sphinxcontrib-applehelp==1.0.1 # via sphinx @@ -100,13 +100,13 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-qthelp==1.0.2 # via sphinx sphinxcontrib-serializinghtml==1.1.3 # via sphinx sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 -tabulate==0.8.3 +sqlalchemy==1.3.10 +tabulate==0.8.5 twofish==0.3.0 -urllib3==1.25.5 +urllib3==1.25.6 vine==1.3.0 werkzeug==0.16.0 xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy, sphinx +# setuptools==41.4.0 # via acme, josepy, sphinx diff --git a/requirements-tests.txt b/requirements-tests.txt index 29d272a0..e6dc53c5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,45 +5,45 @@ # pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in # appdirs==1.4.3 # via black -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography atomicwrites==1.3.0 # via pytest -attrs==19.1.0 # via black, jsonschema, pytest -aws-sam-translator==1.14.0 # via cfn-lint +attrs==19.3.0 # via black, jsonschema, pytest +aws-sam-translator==1.15.1 # via cfn-lint aws-xray-sdk==2.4.2 # via moto bandit==1.6.2 black==19.3b0 -boto3==1.9.232 # via aws-sam-translator, moto +boto3==1.9.250 # via aws-sam-translator, moto boto==2.49.0 # via moto -botocore==1.12.232 # via aws-xray-sdk, boto3, moto, s3transfer +botocore==1.12.250 # via aws-xray-sdk, boto3, moto, s3transfer certifi==2019.9.11 # via requests -cffi==1.12.3 # via cryptography -cfn-lint==0.24.1 # via moto +cffi==1.13.0 # via cryptography +cfn-lint==0.24.4 # via moto chardet==3.0.4 # via requests click==7.0 # via black, flask coverage==4.5.4 cryptography==2.7 # via moto, sshpubkeys datetime==4.3 # via moto -docker==4.0.2 # via moto +docker==4.1.0 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.2 # via python-jose, sshpubkeys +ecdsa==0.13.3 # via python-jose, sshpubkeys factory-boy==2.12.0 -faker==2.0.2 +faker==2.0.3 fakeredis==1.0.5 flask==1.1.1 # via pytest-flask freezegun==0.3.12 -future==0.17.1 # via aws-xray-sdk, python-jose -gitdb2==2.0.5 # via gitpython -gitpython==3.0.2 # via bandit +future==0.18.0 # via aws-xray-sdk, python-jose +gitdb2==2.0.6 # via gitpython +gitpython==3.0.3 # via bandit idna==2.8 # via moto, requests -importlib-metadata==0.23 # via pluggy, pytest +importlib-metadata==0.23 # via jsonschema, pluggy, pytest itsdangerous==1.1.0 # via flask -jinja2==2.10.1 # via flask, moto +jinja2==2.10.3 # via flask, moto jmespath==0.9.4 # via boto3, botocore jsondiff==1.1.2 # via moto jsonpatch==1.24 # via cfn-lint jsonpickle==1.2 # via aws-xray-sdk jsonpointer==2.0 # via jsonpatch -jsonschema==3.0.2 # via aws-sam-translator, cfn-lint +jsonschema==3.1.1 # via aws-sam-translator, cfn-lint markupsafe==1.1.1 # via jinja2 mock==3.0.5 # via moto more-itertools==7.2.0 # via pytest, zipp @@ -59,13 +59,13 @@ pyflakes==2.1.1 pyparsing==2.4.2 # via packaging pyrsistent==0.15.4 # via jsonschema pytest-flask==0.15.0 -pytest-mock==1.10.4 -pytest==5.1.2 +pytest-mock==1.11.1 +pytest==5.2.1 python-dateutil==2.8.0 # via botocore, faker, freezegun, moto python-jose==3.0.1 # via moto -pytz==2019.2 # via datetime, moto +pytz==2019.3 # via datetime, moto pyyaml==5.1.2 -redis==3.3.8 # via fakeredis +redis==3.3.11 # via fakeredis requests-mock==1.7.0 requests==2.22.0 # via docker, moto, requests-mock, responses responses==0.10.6 # via moto @@ -78,7 +78,7 @@ sshpubkeys==3.1.0 # via moto stevedore==1.31.0 # via bandit text-unidecode==1.3 # via faker toml==0.10.0 # via black -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests wcwidth==0.1.7 # via pytest websocket-client==0.56.0 # via docker werkzeug==0.16.0 # via flask, moto, pytest-flask @@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface +# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface diff --git a/requirements.txt b/requirements.txt index db7e46a7..305fe7e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,23 +4,23 @@ # # pip-compile --no-index --output-file=requirements.txt requirements.in # -acme==0.38.0 +acme==0.39.0 alembic-autogenerate-enums==0.0.2 -alembic==1.2.0 # via flask-migrate -amqp==2.5.1 # via kombu +alembic==1.2.1 # via flask-migrate +amqp==2.5.2 # via kombu aniso8601==8.0.0 # via flask-restful arrow==0.15.2 -asn1crypto==0.24.0 # via cryptography +asn1crypto==1.1.0 # via cryptography asyncpool==1.0 bcrypt==3.1.7 # via flask-bcrypt, paramiko billiard==3.6.1.0 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.232 -botocore==1.12.232 +boto3==1.9.250 +botocore==1.12.250 celery[redis]==4.3.0 certifi==2019.9.11 certsrv==2.1.1 -cffi==1.12.3 # via bcrypt, cryptography, pynacl +cffi==1.13.0 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests click==7.0 # via flask cloudflare==2.3.0 @@ -37,16 +37,16 @@ flask-principal==0.4.0 flask-replicated==1.3 flask-restful==0.3.7 flask-script==2.0.6 -flask-sqlalchemy==2.4.0 +flask-sqlalchemy==2.4.1 flask==1.1.1 -future==0.17.1 +future==0.18.0 gunicorn==19.9.0 hvac==0.9.5 idna==2.8 # via requests inflection==0.3.1 itsdangerous==1.1.0 # via flask javaobj-py3==0.3.0 # via pyjks -jinja2==2.10.1 +jinja2==2.10.3 jmespath==0.9.4 # via boto3, botocore josepy==1.2.0 # via acme jsonlines==1.2.0 # via cloudflare @@ -62,7 +62,7 @@ ndg-httpsclient==0.5.1 paramiko==2.6.0 pem==19.2.0 psycopg2==2.8.3 -pyasn1-modules==0.2.6 # via pyjks, python-ldap +pyasn1-modules==0.2.7 # via pyjks, python-ldap pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap pycparser==2.19 # via cffi pycryptodomex==3.9.0 # via pyjks @@ -75,23 +75,23 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore python-editor==1.0.4 # via alembic python-json-logger==0.1.11 # via logmatic-python python-ldap==3.2.0 -pytz==2019.2 # via acme, celery, flask-restful, pyrfc3339 +pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339 pyyaml==5.1.2 raven[flask]==6.10.0 -redis==3.3.8 +redis==3.3.11 requests-toolbelt==0.9.1 # via acme requests[security]==2.22.0 retrying==1.3.3 s3transfer==0.2.1 # via boto3 six==1.12.0 sqlalchemy-utils==0.34.2 -sqlalchemy==1.3.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils -tabulate==0.8.3 +sqlalchemy==1.3.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils +tabulate==0.8.5 twofish==0.3.0 # via pyjks -urllib3==1.25.5 # via botocore, requests +urllib3==1.25.6 # via botocore, requests vine==1.3.0 # via amqp, celery werkzeug==0.16.0 # via flask xmltodict==0.12.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via acme, josepy +# setuptools==41.4.0 # via acme, josepy From a076497cf0c8bb33be81ade490e7bb5b258eec5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2019 22:49:30 +0000 Subject: [PATCH 02/11] Bump ecdsa from 0.13.2 to 0.13.3 Bumps [ecdsa](https://github.com/warner/python-ecdsa) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/warner/python-ecdsa/releases) - [Changelog](https://github.com/warner/python-ecdsa/blob/master/NEWS) - [Commits](https://github.com/warner/python-ecdsa/compare/python-ecdsa-0.13.2...python-ecdsa-0.13.3) Signed-off-by: dependabot[bot] --- requirements-tests.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 29d272a0..8f646bc0 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -25,7 +25,7 @@ cryptography==2.7 # via moto, sshpubkeys datetime==4.3 # via moto docker==4.0.2 # via moto docutils==0.15.2 # via botocore -ecdsa==0.13.2 # via python-jose, sshpubkeys +ecdsa==0.13.3 # via python-jose, sshpubkeys factory-boy==2.12.0 faker==2.0.2 fakeredis==1.0.5 @@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata zope.interface==4.6.0 # via datetime # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface +# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface From b5ab87877b34e760de6c1d0abef166301bd9618a Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Thu, 17 Oct 2019 10:16:33 -0700 Subject: [PATCH 03/11] adding retry to acme setup client, since it can experience timeouts or other types of Connection Errors --- lemur/plugins/lemur_acme/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index f31ffdcb..e38870d8 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry from lemur.plugins import lemur_acme as acme from lemur.plugins.bases import IssuerPlugin from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns +from retrying import retry class AuthorizationRecord(object): @@ -197,6 +198,7 @@ class AcmeHandler(object): ) return pem_certificate, pem_certificate_chain + @retry(stop_max_attempt_number=5, wait_fixed=5000) def setup_acme_client(self, authority): if not authority.options: raise InvalidAuthority("Invalid authority. Options not set") From 10b600424efbabcdbe2727e1a94d3ba15778ae71 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:45:32 -0700 Subject: [PATCH 04/11] refactoring searching for cert --- lemur/sources/service.py | 47 +++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index d5bd7426..070e1a47 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -124,40 +124,47 @@ def sync_endpoints(source): return new, updated +def find_cert(certificate): + updated_by_hash = 0 + exists = False + + if certificate.get("search", None): + conditions = certificate.pop("search") + exists = certificate_service.get_by_attributes(conditions) + + if not exists and certificate.get("name"): + result = certificate_service.get_by_name(certificate["name"]) + if result: + exists = [result] + + if not exists and certificate.get("serial"): + exists = certificate_service.get_by_serial(certificate["serial"]) + + if not exists: + cert = parse_certificate(certificate["body"]) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) + updated_by_hash += 1 + + exists = [x for x in exists if x] + return exists, updated_by_hash + # TODO this is very slow as we don't batch update certificates def sync_certificates(source, user): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving certificates from {0}".format(source.label)) s = plugins.get(source.plugin_name) certificates = s.get_certificates(source.options) for certificate in certificates: - exists = False - - if certificate.get("search", None): - conditions = certificate.pop("search") - exists = certificate_service.get_by_attributes(conditions) - - if not exists and certificate.get("name"): - result = certificate_service.get_by_name(certificate["name"]) - if result: - exists = [result] - - if not exists and certificate.get("serial"): - exists = certificate_service.get_by_serial(certificate["serial"]) - - if not exists: - cert = parse_certificate(certificate["body"]) - matching_serials = certificate_service.get_by_serial(serial(cert)) - exists = find_matching_certificates_by_hash(cert, matching_serials) + exists, updated_by_hash = find_cert(certificate) if not certificate.get("owner"): certificate["owner"] = user.email certificate["creator"] = user - exists = [x for x in exists if x] if not exists: certificate_create(certificate, source) From d43e859c34ca61caca375485a5c0a912655d5474 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:46:01 -0700 Subject: [PATCH 05/11] describing the cert for each endpoint, for better cert search --- lemur/plugins/lemur_aws/plugin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index cf6c8643..a03e92a8 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -32,7 +32,9 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Harm Weites """ +from acme.errors import ClientError from flask import current_app +from lemur.extensions import sentry, metrics from lemur.plugins import lemur_aws as aws from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin @@ -110,6 +112,8 @@ def get_elb_endpoints(account_number, region, elb_dict): listener["Listener"]["SSLCertificateId"] ), ) + endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], + account_number=account_number) if listener["PolicyNames"]: policy = elb.describe_load_balancer_policies( @@ -127,6 +131,28 @@ def get_elb_endpoints(account_number, region, elb_dict): return endpoints +def get_elb_certificate_by_name(certificate_name, account_number): + # certificate name may contain path, in which case we remove it + if "/" in certificate_name: + certificate_name = certificate_name.split('/')[1] + try: + cert = iam.get_certificate(certificate_name, account_number=account_number) + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) + except ClientError: + current_app.logger.warning( + "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + sentry.captureException() + metrics.send( + "get_elb_certificate_failed", "counter", 1, + metric_tags={"certificate_name": certificate_name, "account_number": account_number} + ) + return None + + def get_elb_endpoints_v2(account_number, region, elb_dict): """ Retrieves endpoint information from elbv2 response data. @@ -153,6 +179,8 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): port=listener["Port"], certificate_name=iam.get_name_from_arn(certificate["CertificateArn"]), ) + endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], + account_number=account_number) if listener["SslPolicy"]: policy = elb.describe_ssl_policies_v2( From f075c5af3d7e6c8d5353186770b3b7bc05453b50 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 08:48:11 -0700 Subject: [PATCH 06/11] in case no cert match via name-search, search via the cert itself (serial number, hash comparison) --- lemur/sources/service.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 070e1a47..23f2af72 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -66,7 +66,7 @@ def sync_update_destination(certificate, source): def sync_endpoints(source): - new, updated = 0, 0 + new, updated, updated_by_hash = 0, 0, 0 current_app.logger.debug("Retrieving endpoints from {0}".format(source.label)) s = plugins.get(source.plugin_name) @@ -89,6 +89,29 @@ def sync_endpoints(source): endpoint["certificate"] = certificate_service.get_by_name(certificate_name) + # if get cert by name failed, we attempt a search via serial number and hash comparison + # and link the endpoint certificate to Lemur certificate + if not endpoint["certificate"]: + certificate_attached_to_endpoint = endpoint.pop("certificate") + if certificate_attached_to_endpoint: + lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint) + updated_by_hash += updated_by_hash_tmp + + if lemur_matching_cert: + endpoint["certificate"] = lemur_matching_cert[0] + + if len(lemur_matching_cert) > 1: + current_app.logger.error( + "Too Many Certificates Found. Name: {0} Endpoint: {1}".format( + certificate_name, endpoint["name"] + ) + ) + metrics.send("endpoint.certificate.conflict", + "counter", 1, + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) + + # this indicates the we were not able to describe the endpoint cert if not endpoint["certificate"]: current_app.logger.error( "Certificate Not Found. Name: {0} Endpoint: {1}".format( @@ -97,7 +120,8 @@ def sync_endpoints(source): ) metrics.send("endpoint.certificate.not.found", "counter", 1, - metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) + metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], + "acct": s.get_option("accountNumber", source.options)}) continue policy = endpoint.pop("policy") @@ -122,7 +146,8 @@ def sync_endpoints(source): endpoint_service.update(exists.id, **endpoint) updated += 1 - return new, updated + return new, updated, updated_by_hash + def find_cert(certificate): updated_by_hash = 0 @@ -159,7 +184,7 @@ def sync_certificates(source, user): certificates = s.get_certificates(source.options) for certificate in certificates: - exists, updated_by_hash = find_cert(certificate) + exists, updated_by_hash = find_cert(certificate) if not certificate.get("owner"): certificate["owner"] = user.email @@ -179,12 +204,12 @@ def sync_certificates(source, user): certificate_update(e, source) updated += 1 - return new, updated + return new, updated, updated_by_hash def sync(source, user): - new_certs, updated_certs = sync_certificates(source, user) - new_endpoints, updated_endpoints = sync_endpoints(source) + new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user) + new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source) source.last_run = arrow.utcnow() database.update(source) From 8aea257e6abb3f2d940ebf230fa81075c2425547 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 09:24:49 -0700 Subject: [PATCH 07/11] optimizing the call to describe cert to only the few certs with the naming issue --- lemur/plugins/lemur_aws/plugin.py | 48 ++++++++++++++----------------- lemur/sources/service.py | 16 +++++++++-- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index a03e92a8..46c65c4f 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -112,8 +112,6 @@ def get_elb_endpoints(account_number, region, elb_dict): listener["Listener"]["SSLCertificateId"] ), ) - endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], - account_number=account_number) if listener["PolicyNames"]: policy = elb.describe_load_balancer_policies( @@ -131,28 +129,6 @@ def get_elb_endpoints(account_number, region, elb_dict): return endpoints -def get_elb_certificate_by_name(certificate_name, account_number): - # certificate name may contain path, in which case we remove it - if "/" in certificate_name: - certificate_name = certificate_name.split('/')[1] - try: - cert = iam.get_certificate(certificate_name, account_number=account_number) - return dict( - body=cert["CertificateBody"], - chain=cert.get("CertificateChain"), - name=cert["ServerCertificateMetadata"]["ServerCertificateName"], - ) - except ClientError: - current_app.logger.warning( - "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) - sentry.captureException() - metrics.send( - "get_elb_certificate_failed", "counter", 1, - metric_tags={"certificate_name": certificate_name, "account_number": account_number} - ) - return None - - def get_elb_endpoints_v2(account_number, region, elb_dict): """ Retrieves endpoint information from elbv2 response data. @@ -179,8 +155,6 @@ def get_elb_endpoints_v2(account_number, region, elb_dict): port=listener["Port"], certificate_name=iam.get_name_from_arn(certificate["CertificateArn"]), ) - endpoint["certificate"] = get_elb_certificate_by_name(certificate_name=endpoint["certificate_name"], - account_number=account_number) if listener["SslPolicy"]: policy = elb.describe_ssl_policies_v2( @@ -299,6 +273,28 @@ class AWSSourcePlugin(SourcePlugin): account_number = self.get_option("accountNumber", options) iam.delete_cert(certificate.name, account_number=account_number) + def get_certificate_by_name(self, certificate_name, options): + account_number = self.get_option("accountNumber", options) + # certificate name may contain path, in which case we remove it + if "/" in certificate_name: + certificate_name = certificate_name.split('/')[1] + try: + cert = iam.get_certificate(certificate_name, account_number=account_number) + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) + except ClientError: + current_app.logger.warning( + "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) + sentry.captureException() + metrics.send( + "get_elb_certificate_failed", "counter", 1, + metric_tags={"certificate_name": certificate_name, "account_number": account_number} + ) + return None + class AWSDestinationPlugin(DestinationPlugin): title = "AWS" diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 23f2af72..498adfeb 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -15,7 +15,7 @@ from lemur.sources.models import Source from lemur.certificates.models import Certificate from lemur.certificates import service as certificate_service from lemur.endpoints import service as endpoint_service -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema @@ -92,7 +92,18 @@ def sync_endpoints(source): # if get cert by name failed, we attempt a search via serial number and hash comparison # and link the endpoint certificate to Lemur certificate if not endpoint["certificate"]: - certificate_attached_to_endpoint = endpoint.pop("certificate") + certificate_attached_to_endpoint = None + try: + certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options) + except NotImplementedError: + current_app.logger.warning( + "Unable to describe server certificate for endpoints in source {0}:" + " plugin has not implemented 'get_certificate_by_name'".format( + source.label + ) + ) + sentry.captureException() + if certificate_attached_to_endpoint: lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint) updated_by_hash += updated_by_hash_tmp @@ -111,7 +122,6 @@ def sync_endpoints(source): metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) - # this indicates the we were not able to describe the endpoint cert if not endpoint["certificate"]: current_app.logger.error( "Certificate Not Found. Name: {0} Endpoint: {1}".format( From 1768aad9e2ee95ed28ecfa9837b7db3597ff8551 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 10:17:58 -0700 Subject: [PATCH 08/11] capturing no such entity exception. --- lemur/plugins/lemur_aws/iam.py | 10 ++++++---- lemur/plugins/lemur_aws/plugin.py | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index 67c35262..13590ddd 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -10,7 +10,7 @@ import botocore from retrying import retry -from lemur.extensions import metrics +from lemur.extensions import metrics, sentry from lemur.plugins.lemur_aws.sts import sts_client @@ -122,9 +122,11 @@ def get_certificate(name, **kwargs): """ client = kwargs.pop("client") metrics.send("get_certificate", "counter", 1, metric_tags={"name": name}) - return client.get_server_certificate(ServerCertificateName=name)[ - "ServerCertificate" - ] + try: + return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"] + except client.exceptions.NoSuchEntityException: + sentry.captureException() + return None @sts_client("iam") diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 46c65c4f..86cd7912 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -280,11 +280,12 @@ class AWSSourcePlugin(SourcePlugin): certificate_name = certificate_name.split('/')[1] try: cert = iam.get_certificate(certificate_name, account_number=account_number) - return dict( - body=cert["CertificateBody"], - chain=cert.get("CertificateChain"), - name=cert["ServerCertificateMetadata"]["ServerCertificateName"], - ) + if cert: + return dict( + body=cert["CertificateBody"], + chain=cert.get("CertificateChain"), + name=cert["ServerCertificateMetadata"]["ServerCertificateName"], + ) except ClientError: current_app.logger.warning( "get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name)) From 9037f8843072ed3ab1695d4bca681d38e01f46de Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:02:41 -0700 Subject: [PATCH 09/11] just in case the path varies --- lemur/plugins/lemur_aws/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index 86cd7912..98b01672 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -277,7 +277,7 @@ class AWSSourcePlugin(SourcePlugin): account_number = self.get_option("accountNumber", options) # certificate name may contain path, in which case we remove it if "/" in certificate_name: - certificate_name = certificate_name.split('/')[1] + certificate_name = certificate_name.split('/')[-1] try: cert = iam.get_certificate(certificate_name, account_number=account_number) if cert: From 14e13b512e70f1819f88964291744ab690417aff Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:03:28 -0700 Subject: [PATCH 10/11] providing a count for conflicts --- lemur/sources/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 498adfeb..8ba4ea0d 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -113,12 +113,12 @@ def sync_endpoints(source): if len(lemur_matching_cert) > 1: current_app.logger.error( - "Too Many Certificates Found. Name: {0} Endpoint: {1}".format( - certificate_name, endpoint["name"] + "Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format( + len(lemur_matching_cert), certificate_name, endpoint["name"] ) ) metrics.send("endpoint.certificate.conflict", - "counter", 1, + "gauge", len(lemur_matching_cert), metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)}) From 06f4aed6939f8b7081b30002f705a5be5d2cdc62 Mon Sep 17 00:00:00 2001 From: Hossein Shafagh Date: Fri, 18 Oct 2019 11:20:52 -0700 Subject: [PATCH 11/11] keeping track of certs found by hash --- lemur/sources/service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 8ba4ea0d..f69f70f5 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -221,6 +221,14 @@ def sync(source, user): new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user) new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source) + metrics.send("sync.updated_certs_by_hash", + "gauge", updated_certs_by_hash, + metric_tags={"source": source.label}) + + metrics.send("sync.updated_endpoints_by_hash", + "gauge", updated_endpoints_by_hash, + metric_tags={"source": source.label}) + source.last_run = arrow.utcnow() database.update(source)