Fixing elb sync issues. (#641)

* Fixing elb sync issues.

* Fixing de-duplications of names.

lemur/certificates/models.py

@@ -36,14 +36,40 @@ from lemur.models import certificate_associations, certificate_source_associatio
 from lemur.domains.models import Domain
 
 
+def get_sequence(name):
+    if '-' not in name:
+        return name, None
+
+    parts = name.split('-')
+    end = parts.pop(-1)
+    root = '-'.join(parts)
+
+    if len(end) == 8:
+        return root + '-' + end, None
+
+    try:
+        end = int(end)
+    except ValueError:
+        end = None
+
+    return root, end
+
+
 def get_or_increase_name(name):
     name = '-'.join(name.strip().split(' '))
-    count = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).count()
+    certificates = Certificate.query.filter(Certificate.name.ilike('{0}%'.format(name))).all()
 
-    if count >= 1:
-        return name + '-' + str(count)
+    if not certificates:
+        return name
 
-    return name
+    ends = [0]
+    root, end = get_sequence(name)
+    for cert in certificates:
+        root, end = get_sequence(cert.name)
+        if end:
+            ends.append(end)
+
+    return '{0}-{1}'.format(root, max(ends) + 1)
 
 
 class Certificate(db.Model):
@@ -228,7 +254,7 @@ class Certificate(db.Model):
         return "Certificate(name={name})".format(name=self.name)
 
 
-@event.listens_for(Certificate.destinations, 'append', retval=True)
+@event.listens_for(Certificate.destinations, 'append')
 def update_destinations(target, value, initiator):
     """
     Attempt to upload certificate to the new destination
@@ -241,12 +267,11 @@ def update_destinations(target, value, initiator):
     destination_plugin = plugins.get(value.plugin_name)
 
     try:
-        destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
-        return value
+        if target.private_key:
+            destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
     except Exception as e:
         current_app.logger.exception(e)
         metrics.send('destination_upload_failure', 'counter', 1, metric_tags={'certificate': target.name, 'destination': value.label})
-        return None
 
 
 @event.listens_for(Certificate.replaces, 'append')

lemur/plugins/lemur_aws/elb.py

@@ -28,7 +28,7 @@ def retry_throttled(exception):
         if exception.response['Error']['Code'] == 'CertificateNotFound':
             return False
 
-    metrics.send('ec2_retry', 'counter', 1)
+    metrics.send('elb_retry', 'counter', 1)
     return True
 
 
@@ -149,6 +149,7 @@ def describe_listeners_v2(**kwargs):
 
 
 @sts_client('elb')
+@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
 def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -160,6 +161,7 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
 
 
 @sts_client('elbv2')
+@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
 def describe_ssl_policies_v2(policy_names, **kwargs):
     """
     Fetching all policies currently associated with an ELB.
@@ -171,6 +173,7 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
 
 
 @sts_client('elb')
+@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
 def describe_load_balancer_types(policies, **kwargs):
     """
     Describe the policies with policy details.

lemur/sources/service.py

@@ -72,15 +72,15 @@ def sync_endpoints(source):
     for endpoint in endpoints:
         exists = endpoint_service.get_by_dnsname(endpoint['dnsname'])
 
-        cert = certificate_service.get_by_name(endpoint['certificate_name'])
+        certificate_name = endpoint.pop('certificate_name')
 
-        if not cert:
+        endpoint['certificate'] = certificate_service.get_by_name(certificate_name)
+
+        if not endpoint['certificate']:
             current_app.logger.error(
-                "Certificate Not Found. Name: {0} Endpoint: {1}".format(endpoint['certificate_name'], endpoint['name']))
+                "Certificate Not Found. Name: {0} Endpoint: {1}".format(certificate_name, endpoint['name']))
             continue
 
-        endpoint['certificate'] = cert
-
         policy = endpoint.pop('policy')
 
         policy_ciphers = []

lemur/tests/test_certificates.py

@@ -13,6 +13,19 @@ from lemur.tests.vectors import VALID_ADMIN_HEADER_TOKEN, VALID_USER_HEADER_TOKE
     INTERNAL_VALID_LONG_STR, INTERNAL_VALID_SAN_STR, PRIVATE_KEY_STR
 
 
+def test_get_or_increase_name(session, certificate):
+    from lemur.certificates.models import get_or_increase_name
+
+    assert get_or_increase_name('test name') == 'test-name'
+    assert get_or_increase_name(certificate.name) == '{0}-1'.format(certificate.name)
+
+    certificate.name = 'test-cert-11111111'
+    assert get_or_increase_name(certificate.name) == 'test-cert-11111111-1'
+
+    certificate.name = 'test-cert-11111111-1'
+    assert get_or_increase_name('test-cert-11111111-1') == 'test-cert-11111111-2'
+
+
 def test_get_certificate_primitives(certificate):
     from lemur.certificates.service import get_certificate_primitives