diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index c9a2fa24..1b203260 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -20,7 +20,6 @@ from lemur.common.utils import generate_private_key, truthiness from lemur.destinations.models import Destination from lemur.domains.models import Domain from lemur.extensions import metrics, sentry, signals -from lemur.models import certificate_associations from lemur.notifications.models import Notification from lemur.pending_certificates.models import PendingCertificate from lemur.plugins.base import plugins @@ -307,7 +306,7 @@ def render(args): if filt: terms = filt.split(';') - term = '%{0}%'.format(terms[1]) + term = '{0}%'.format(terms[1]) # Exact matches for quotes. Only applies to name, issuer, and cn if terms[1].startswith('"') and terms[1].endswith('"'): term = terms[1][1:-1] @@ -341,13 +340,13 @@ def render(args): elif 'id' in terms: query = query.filter(Certificate.id == cast(terms[1], Integer)) elif 'name' in terms: - query = query.outerjoin(certificate_associations).outerjoin(Domain).filter( + query = query.filter( or_( Certificate.name.ilike(term), - Domain.name.ilike(term), + Certificate.domains.any(Domain.name.ilike(term)), Certificate.cn.ilike(term), ) - ).group_by(Certificate.id) + ) else: query = database.filter(query, Certificate, terms) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 62e59d69..0504c958 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -12,6 +12,7 @@ import string import sqlalchemy from cryptography import x509 from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import rsa, ec from cryptography.hazmat.primitives.serialization import load_pem_private_key from flask_restful.reqparse import RequestParser @@ -226,3 +227,13 @@ def truthiness(s): """If input string resembles something truthy then return True, else False.""" return s.lower() in ('true', 'yes', 'on', 't', '1') + + +def find_matching_certificates_by_hash(cert, matching_certs): + """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs), + determine if any of the certificate hashes match and return the matches.""" + matching = [] + for c in matching_certs: + if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()): + matching.append(c) + return matching diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 4c4ce97f..b4391dd8 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): """ Get a listener ARN from an endpoint. @@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs(**kwargs): """ Fetches one page elb objects for a given account and region. @@ -123,7 +123,7 @@ def get_elbs(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_elbs_v2(**kwargs): """ Fetches one page of elb objects for a given account and region. @@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_listeners_v2(**kwargs): """ Fetches one page of listener objects for a given elb arn. @@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_ssl_policies_v2(policy_names, **kwargs): """ Fetching all policies currently associated with an ELB. @@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def describe_load_balancer_types(policies, **kwargs): """ Describe the policies with policy details. @@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs): @sts_client('elb') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate(name, port, certificate_id, **kwargs): """ Attaches a certificate to a listener, throws exception @@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs): @sts_client('elbv2') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def attach_certificate_v2(listener_arn, port, certificates, **kwargs): """ Attaches a certificate to a listener, throws exception diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index b2a07798..7010c909 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): """ Upload a certificate to AWS @@ -95,7 +95,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def delete_cert(cert_name, **kwargs): """ Delete a certificate from AWS @@ -112,7 +112,7 @@ def delete_cert(cert_name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificate(name, **kwargs): """ Retrieves an SSL certificate. @@ -126,7 +126,7 @@ def get_certificate(name, **kwargs): @sts_client('iam') -@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100) +@retry(retry_on_exception=retry_throttled, wait_fixed=2000) def get_certificates(**kwargs): """ Fetches one page of certificate objects for a given account. diff --git a/lemur/plugins/lemur_aws/sts.py b/lemur/plugins/lemur_aws/sts.py index 001ea2c8..6253ad7a 100644 --- a/lemur/plugins/lemur_aws/sts.py +++ b/lemur/plugins/lemur_aws/sts.py @@ -9,14 +9,22 @@ from functools import wraps import boto3 +from botocore.config import Config from flask import current_app +config = Config( + retries=dict( + max_attempts=20 + ) +) + + def sts_client(service, service_type='client'): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): - sts = boto3.client('sts') + sts = boto3.client('sts', config=config) arn = 'arn:aws:iam::{0}:role/{1}'.format( kwargs.pop('account_number'), current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur') @@ -31,7 +39,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['client'] = client elif service_type == 'resource': @@ -40,7 +49,8 @@ def sts_client(service, service_type='client'): region_name=kwargs.pop('region', 'us-east-1'), aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] + aws_session_token=role['Credentials']['SessionToken'], + config=config ) kwargs['resource'] = resource return f(*args, **kwargs) diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 227f1bce..55d2ee62 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service from lemur.destinations import service as destination_service from lemur.certificates.schemas import CertificateUploadInputSchema -from lemur.common.utils import parse_certificate +from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate from lemur.common.defaults import serial from lemur.plugins.base import plugins @@ -126,7 +126,8 @@ def sync_certificates(source, user): if not exists: cert = parse_certificate(certificate['body']) - exists = certificate_service.get_by_serial(serial(cert)) + matching_serials = certificate_service.get_by_serial(serial(cert)) + exists = find_matching_certificates_by_hash(cert, matching_serials) if not certificate.get('owner'): certificate['owner'] = user.email diff --git a/requirements-dev.txt b/requirements-dev.txt index e9e47ed5..21156588 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,7 @@ # pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in # aspy.yaml==1.1.1 # via pre-commit -bleach==3.0.2 # via readme-renderer -cached-property==1.5.1 # via pre-commit +bleach==3.1.0 # via readme-renderer certifi==2018.11.29 # via requests cfgv==1.4.0 # via pre-commit chardet==3.0.4 # via requests @@ -19,8 +18,8 @@ importlib-resources==1.0.2 # via pre-commit invoke==1.2.0 mccabe==0.6.1 # via flake8 nodeenv==1.3.3 -pkginfo==1.5.0 # via twine -pre-commit==1.13.0 +pkginfo==1.5.0.1 # via twine +pre-commit==1.14.0 pycodestyle==2.3.1 # via flake8 pyflakes==1.6.0 # via flake8 pygments==2.3.1 # via readme-renderer diff --git a/requirements-docs.txt b/requirements-docs.txt index 19ebb0ea..f3182456 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -9,7 +9,7 @@ alabaster==0.7.12 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==1.0.5 amqp==2.3.2 -aniso8601==4.0.1 +aniso8601==4.1.0 arrow==0.13.0 asn1crypto==0.24.0 asyncpool==1.0 @@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx bcrypt==3.1.5 billiard==3.5.0.5 blinker==1.4 -boto3==1.9.75 -botocore==1.12.75 +boto3==1.9.76 +botocore==1.12.76 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 diff --git a/requirements-tests.txt b/requirements-tests.txt index a11de6ec..490d74d1 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,9 +8,9 @@ asn1crypto==0.24.0 # via cryptography atomicwrites==1.2.1 # via pytest attrs==18.2.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.9.75 # via moto +boto3==1.9.76 # via moto boto==2.49.0 # via moto -botocore==1.12.75 # via boto3, moto, s3transfer +botocore==1.12.76 # via boto3, moto, s3transfer certifi==2018.11.29 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -38,7 +38,7 @@ more-itertools==5.0.0 # via pytest moto==1.3.7 nose==1.3.7 pbr==5.1.1 # via mock -pluggy==0.8.0 # via pytest +pluggy==0.8.1 # via pytest py==1.7.0 # via pytest pyaml==18.11.0 # via moto pycparser==2.19 # via cffi @@ -60,5 +60,5 @@ text-unidecode==1.2 # via faker urllib3==1.24.1 # via botocore, requests websocket-client==0.54.0 # via docker werkzeug==0.14.1 # via flask, moto, pytest-flask -wrapt==1.10.11 # via aws-xray-sdk +wrapt==1.11.0 # via aws-xray-sdk xmltodict==0.11.0 # via moto diff --git a/requirements.txt b/requirements.txt index 59871284..bc72db0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,15 +8,15 @@ acme==0.30.0 alembic-autogenerate-enums==0.0.2 alembic==1.0.5 # via flask-migrate amqp==2.3.2 # via kombu -aniso8601==4.0.1 # via flask-restful +aniso8601==4.1.0 # via flask-restful arrow==0.13.0 asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.5 # via flask-bcrypt, paramiko billiard==3.5.0.5 # via celery blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.9.75 -botocore==1.12.75 +boto3==1.9.76 +botocore==1.12.76 celery[redis]==4.2.1 certifi==2018.11.29 cffi==1.11.5 # via bcrypt, cryptography, pynacl