Merge branch 'master' of github.com:Netflix/lemur

lemur/certificates/service.py

@@ -20,7 +20,6 @@ from lemur.common.utils import generate_private_key, truthiness
 from lemur.destinations.models import Destination
 from lemur.domains.models import Domain
 from lemur.extensions import metrics, sentry, signals
-from lemur.models import certificate_associations
 from lemur.notifications.models import Notification
 from lemur.pending_certificates.models import PendingCertificate
 from lemur.plugins.base import plugins
@@ -307,7 +306,7 @@ def render(args):
 
     if filt:
         terms = filt.split(';')
-        term = '%{0}%'.format(terms[1])
+        term = '{0}%'.format(terms[1])
         # Exact matches for quotes. Only applies to name, issuer, and cn
         if terms[1].startswith('"') and terms[1].endswith('"'):
             term = terms[1][1:-1]
@@ -341,13 +340,13 @@ def render(args):
         elif 'id' in terms:
             query = query.filter(Certificate.id == cast(terms[1], Integer))
         elif 'name' in terms:
-            query = query.outerjoin(certificate_associations).outerjoin(Domain).filter(
+            query = query.filter(
                 or_(
                     Certificate.name.ilike(term),
-                    Domain.name.ilike(term),
+                    Certificate.domains.any(Domain.name.ilike(term)),
                     Certificate.cn.ilike(term),
                 )
-            ).group_by(Certificate.id)
+            )
         else:
             query = database.filter(query, Certificate, terms)
 

lemur/common/utils.py

@@ -12,6 +12,7 @@ import string
 import sqlalchemy
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import rsa, ec
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from flask_restful.reqparse import RequestParser
@@ -226,3 +227,13 @@ def truthiness(s):
     """If input string resembles something truthy then return True, else False."""
 
     return s.lower() in ('true', 'yes', 'on', 't', '1')
+
+
+def find_matching_certificates_by_hash(cert, matching_certs):
+    """Given a Cryptography-formatted certificate cert, and Lemur-formatted certificates (matching_certs),
+    determine if any of the certificate hashes match and return the matches."""
+    matching = []
+    for c in matching_certs:
+        if parse_certificate(c.body).fingerprint(hashes.SHA256()) == cert.fingerprint(hashes.SHA256()):
+            matching.append(c)
+    return matching

lemur/plugins/lemur_aws/elb.py

@@ -95,7 +95,7 @@ def get_all_elbs_v2(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
     """
     Get a listener ARN from an endpoint.
@@ -113,7 +113,7 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_elbs(**kwargs):
     """
     Fetches one page elb objects for a given account and region.
@@ -123,7 +123,7 @@ def get_elbs(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_elbs_v2(**kwargs):
     """
     Fetches one page of elb objects for a given account and region.
@@ -136,7 +136,7 @@ def get_elbs_v2(**kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_listeners_v2(**kwargs):
     """
     Fetches one page of listener objects for a given elb arn.
@@ -149,7 +149,7 @@ def describe_listeners_v2(**kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -161,7 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_ssl_policies_v2(policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -173,7 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def describe_load_balancer_types(policies, **kwargs):
     """
     Describe the policies with policy details.
@@ -185,7 +185,7 @@ def describe_load_balancer_types(policies, **kwargs):
 
 
 @sts_client('elb')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def attach_certificate(name, port, certificate_id, **kwargs):
     """
     Attaches a certificate to a listener, throws exception
@@ -205,7 +205,7 @@ def attach_certificate(name, port, certificate_id, **kwargs):
 
 
 @sts_client('elbv2')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def attach_certificate_v2(listener_arn, port, certificates, **kwargs):
     """
     Attaches a certificate to a listener, throws exception

lemur/plugins/lemur_aws/iam.py

@@ -52,7 +52,7 @@ def create_arn_from_cert(account_number, region, certificate_name):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
     """
     Upload a certificate to AWS
@@ -95,7 +95,7 @@ def upload_cert(name, body, private_key, path, cert_chain=None, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def delete_cert(cert_name, **kwargs):
     """
     Delete a certificate from AWS
@@ -112,7 +112,7 @@ def delete_cert(cert_name, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_certificate(name, **kwargs):
     """
     Retrieves an SSL certificate.
@@ -126,7 +126,7 @@ def get_certificate(name, **kwargs):
 
 
 @sts_client('iam')
-@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=100)
+@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
 def get_certificates(**kwargs):
     """
     Fetches one page of certificate objects for a given account.

lemur/plugins/lemur_aws/sts.py

@@ -9,14 +9,22 @@ from functools import wraps
 
 import boto3
 
+from botocore.config import Config
 from flask import current_app
 
 
+config = Config(
+    retries=dict(
+        max_attempts=20
+    )
+)
+
+
 def sts_client(service, service_type='client'):
     def decorator(f):
         @wraps(f)
         def decorated_function(*args, **kwargs):
-            sts = boto3.client('sts')
+            sts = boto3.client('sts', config=config)
             arn = 'arn:aws:iam::{0}:role/{1}'.format(
                 kwargs.pop('account_number'),
                 current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
@@ -31,7 +39,8 @@ def sts_client(service, service_type='client'):
                     region_name=kwargs.pop('region', 'us-east-1'),
                     aws_access_key_id=role['Credentials']['AccessKeyId'],
                     aws_secret_access_key=role['Credentials']['SecretAccessKey'],
-                    aws_session_token=role['Credentials']['SessionToken']
+                    aws_session_token=role['Credentials']['SessionToken'],
+                    config=config
                 )
                 kwargs['client'] = client
             elif service_type == 'resource':
@@ -40,7 +49,8 @@ def sts_client(service, service_type='client'):
                     region_name=kwargs.pop('region', 'us-east-1'),
                     aws_access_key_id=role['Credentials']['AccessKeyId'],
                     aws_secret_access_key=role['Credentials']['SecretAccessKey'],
-                    aws_session_token=role['Credentials']['SessionToken']
+                    aws_session_token=role['Credentials']['SessionToken'],
+                    config=config
                 )
                 kwargs['resource'] = resource
             return f(*args, **kwargs)

lemur/sources/service.py

@@ -17,7 +17,7 @@ from lemur.endpoints import service as endpoint_service
 from lemur.destinations import service as destination_service
 
 from lemur.certificates.schemas import CertificateUploadInputSchema
-from lemur.common.utils import parse_certificate
+from lemur.common.utils import find_matching_certificates_by_hash, parse_certificate
 from lemur.common.defaults import serial
 
 from lemur.plugins.base import plugins
@@ -126,7 +126,8 @@ def sync_certificates(source, user):
 
         if not exists:
             cert = parse_certificate(certificate['body'])
-            exists = certificate_service.get_by_serial(serial(cert))
+            matching_serials = certificate_service.get_by_serial(serial(cert))
+            exists = find_matching_certificates_by_hash(cert, matching_serials)
 
         if not certificate.get('owner'):
             certificate['owner'] = user.email

requirements-dev.txt

@@ -5,8 +5,7 @@
 #    pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
 #
 aspy.yaml==1.1.1          # via pre-commit
-bleach==3.0.2             # via readme-renderer
+bleach==3.1.0             # via readme-renderer
-cached-property==1.5.1    # via pre-commit
 certifi==2018.11.29       # via requests
 cfgv==1.4.0               # via pre-commit
 chardet==3.0.4            # via requests
@@ -19,8 +18,8 @@ importlib-resources==1.0.2  # via pre-commit
 invoke==1.2.0
 mccabe==0.6.1             # via flake8
 nodeenv==1.3.3
-pkginfo==1.5.0            # via twine
+pkginfo==1.5.0.1          # via twine
-pre-commit==1.13.0
+pre-commit==1.14.0
 pycodestyle==2.3.1        # via flake8
 pyflakes==1.6.0           # via flake8
 pygments==2.3.1           # via readme-renderer

requirements-docs.txt

@@ -9,7 +9,7 @@ alabaster==0.7.12         # via sphinx
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.5
 amqp==2.3.2
-aniso8601==4.0.1
+aniso8601==4.1.0
 arrow==0.13.0
 asn1crypto==0.24.0
 asyncpool==1.0
@@ -17,8 +17,8 @@ babel==2.6.0              # via sphinx
 bcrypt==3.1.5
 billiard==3.5.0.5
 blinker==1.4
-boto3==1.9.75
+boto3==1.9.76
-botocore==1.12.75
+botocore==1.12.76
 celery[redis]==4.2.1
 certifi==2018.11.29
 cffi==1.11.5

requirements-tests.txt

@@ -8,9 +8,9 @@ asn1crypto==0.24.0        # via cryptography
 atomicwrites==1.2.1       # via pytest
 attrs==18.2.0             # via pytest
 aws-xray-sdk==0.95        # via moto
-boto3==1.9.75             # via moto
+boto3==1.9.76             # via moto
 boto==2.49.0              # via moto
-botocore==1.12.75         # via boto3, moto, s3transfer
+botocore==1.12.76         # via boto3, moto, s3transfer
 certifi==2018.11.29       # via requests
 cffi==1.11.5              # via cryptography
 chardet==3.0.4            # via requests
@@ -38,7 +38,7 @@ more-itertools==5.0.0     # via pytest
 moto==1.3.7
 nose==1.3.7
 pbr==5.1.1                # via mock
-pluggy==0.8.0             # via pytest
+pluggy==0.8.1             # via pytest
 py==1.7.0                 # via pytest
 pyaml==18.11.0            # via moto
 pycparser==2.19           # via cffi
@@ -60,5 +60,5 @@ text-unidecode==1.2       # via faker
 urllib3==1.24.1           # via botocore, requests
 websocket-client==0.54.0  # via docker
 werkzeug==0.14.1          # via flask, moto, pytest-flask
-wrapt==1.10.11            # via aws-xray-sdk
+wrapt==1.11.0             # via aws-xray-sdk
 xmltodict==0.11.0         # via moto

requirements.txt

@@ -8,15 +8,15 @@ acme==0.30.0
 alembic-autogenerate-enums==0.0.2
 alembic==1.0.5            # via flask-migrate
 amqp==2.3.2               # via kombu
-aniso8601==4.0.1          # via flask-restful
+aniso8601==4.1.0          # via flask-restful
 arrow==0.13.0
 asn1crypto==0.24.0        # via cryptography
 asyncpool==1.0
 bcrypt==3.1.5             # via flask-bcrypt, paramiko
 billiard==3.5.0.5         # via celery
 blinker==1.4              # via flask-mail, flask-principal, raven
-boto3==1.9.75
+boto3==1.9.76
-botocore==1.12.75
+botocore==1.12.76
 celery[redis]==4.2.1
 certifi==2018.11.29
 cffi==1.11.5              # via bcrypt, cryptography, pynacl