Improving endpoint rotation logic (#545)

lemur/endpoints/service.py

@@ -94,6 +94,8 @@ def update(endpoint_id, **kwargs):
 
     endpoint.policy = kwargs['policy']
     endpoint.certificate = kwargs['certificate']
+    endpoint.source = kwargs['source']
+    metrics.send('endpoint_added', 'counter', 1)
     database.update(endpoint)
     return endpoint
 
@@ -104,7 +106,7 @@ def rotate_certificate(endpoint, new_cert):
         endpoint.source.plugin.update_endpoint(endpoint, new_cert)
         endpoint.certificate = new_cert
     except Exception as e:
-        metrics.send('rotate_failure', 'counter', 1, tags={'endpoint': endpoint.name})
+        metrics.send('rotate_failure', 'counter', 1, metric_tags={'endpoint': endpoint.name})
         current_app.logger.exception(e)
         raise e
 

lemur/manage.py

@@ -518,81 +518,32 @@ def print_certificate_details(details):
     )
 
 
-class RotateCertificate(Command):
+certificate_manager = Manager(usage="Handles all certificate related tasks.")
-    """
-    Rotates certificate on all endpoints managed by Lemur.
-    """
-    option_list = (
-        Option('-n', '--cert-name', dest='new_cert_name'),
-        Option('-o', '--old-cert-name', dest='old_cert_name', required=True),
-        Option('-c', '--commit', dest='commit', action='store_true', default=False)
-    )
 
-    def run(self, new_cert_name, old_cert_name, commit):
-        from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
-        from lemur.endpoints.service import rotate_certificate
 
-        old_cert = get_by_name(old_cert_name)
+@certificate_manager.command
+def rotate(new_certificate_name, old_certificate_name, commit=False):
+    from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
+    from lemur.endpoints.service import rotate_certificate
 
-        if not old_cert:
+    old_cert = get_by_name(old_certificate_name)
-            sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name))
+
+    if not old_cert:
+        sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
+        sys.exit(1)
+
+    if new_certificate_name:
+        new_cert = get_by_name(new_certificate_name)
+
+        if not new_cert:
+            sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
             sys.exit(1)
 
-        if new_cert_name:
+    if commit:
-            new_cert = get_by_name(new_cert_name)
+        sys.stdout.write("[!] Running in COMMIT mode.\n")
 
-            if not new_cert:
+    if not new_certificate_name:
-                sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name))
+        sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_certificate_name))
-                sys.exit(1)
-
-        if commit:
-            sys.stdout.write("[!] Running in COMMIT mode.\n")
-
-        if not new_cert_name:
-            sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_cert_name))
-
-            details = get_certificate_primitives(old_cert)
-            print_certificate_details(details)
-
-            if commit:
-                new_cert = reissue_certificate(old_cert, replace=True)
-                sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name))
-
-            sys.stdout.write("[+] Done! \n")
-
-        if len(old_cert.endpoints) > 0:
-            for endpoint in old_cert.endpoints:
-                sys.stdout.write("[+] Certificate found deployed onto {0}\n ".format(endpoint.name))
-                sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n ".format(old_cert_name, new_cert.name))
-
-                if commit:
-                    rotate_certificate(endpoint, new_cert_name)
-
-                sys.stdout.write("[+] Done! \n")
-        else:
-            sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n")
-
-
-class ReissueCertificate(Command):
-    """
-    Reissues a certificate based on a given certificate.
-    """
-    option_list = (
-        Option('-o', '--old-cert-name', dest='old_cert_name', required=True),
-        Option('-c', '--commit', dest='commit', action='store_true', default=False)
-    )
-
-    def run(self, old_cert_name, commit):
-        from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
-
-        old_cert = get_by_name(old_cert_name)
-
-        if not old_cert:
-            sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name))
-            sys.exit(1)
-
-        if commit:
-            sys.stdout.write("[!] Running in COMMIT mode.\n")
 
         details = get_certificate_primitives(old_cert)
         print_certificate_details(details)
@@ -603,6 +554,48 @@ class ReissueCertificate(Command):
 
         sys.stdout.write("[+] Done! \n")
 
+    if len(old_cert.endpoints) > 0:
+        for endpoint in old_cert.endpoints:
+            sys.stdout.write(
+                "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}\n".format(
+                    name=endpoint.name,
+                    dnsname=endpoint.dnsname,
+                    port=endpoint.port,
+                    type=endpoint.type
+                )
+            )
+            sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n".format(old_certificate_name, new_cert.name))
+
+            if commit:
+                rotate_certificate(endpoint, new_cert)
+
+            sys.stdout.write("[+] Done! \n")
+    else:
+        sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n")
+
+
+@certificate_manager.command
+def reissue(old_certificate_name, commit=False):
+    from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives
+
+    old_cert = get_by_name(old_certificate_name)
+
+    if not old_cert:
+        sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name))
+        sys.exit(1)
+
+    if commit:
+        sys.stdout.write("[!] Running in COMMIT mode.\n")
+
+    details = get_certificate_primitives(old_cert)
+    print_certificate_details(details)
+
+    if commit:
+        new_cert = reissue_certificate(old_cert, replace=True)
+        sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name))
+
+    sys.stdout.write("[+] Done! \n")
+
 
 @manager.command
 def publish_verisign_units():
@@ -743,46 +736,37 @@ class Report(Command):
         sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n")
 
 
-class Sources(Command):
+source_manager = Manager(usage="Handles all source related tasks.")
-    """
-    Defines a set of actions to take against Lemur's sources.
-    """
-    option_list = (
-        Option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.', required=True),
-        Option('-a', '--action', choices=['sync', 'clean'], dest='action', help='Action to take on source.', required=True)
-    )
 
-    def run(self, source_strings, action):
-        sources = []
-        if not source_strings:
-            table = []
-            for source in source_service.get_all():
-                table.append([source.label, source.active, source.description])
 
-            sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description']))
+def validate_sources(source_strings):
-            sys.exit(1)
+    sources = []
+    if not source_strings:
+        table = []
+        for source in source_service.get_all():
+            table.append([source.label, source.active, source.description])
 
-        elif 'all' in source_strings:
+        sys.stdout.write("No source specified choose from below:\n")
-            sources = source_service.get_all()
+        sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description']))
+        sys.exit(1)
 
-        else:
+    if 'all' in source_strings:
-            for source_str in source_strings:
+        sources = source_service.get_all()
-                source = source_service.get_by_label(source_str)
 
-                if not source:
+    for source_str in source_strings:
-                    sys.stderr.write("Unable to find specified source with label: {0}".format(source_str))
+        source = source_service.get_by_label(source_str)
 
-                sources.append(source)
+        if not source:
+            sys.stderr.write("Unable to find specified source with label: {0}".format(source_str))
 
-        for source in sources:
+        sources.append(source)
-            if action == 'sync':
+    return sources
-                self.sync(source)
 
-            if action == 'clean':
-                self.clean(source)
 
-    @staticmethod
+@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
-    def sync(source):
+def sync(source_strings):
+    source_objs = validate_sources(source_strings)
+    for source in source_objs:
         start_time = time.time()
         sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label))
 
@@ -805,8 +789,11 @@ class Sources(Command):
 
             metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label})
 
-    @staticmethod
+
-    def clean(source):
+@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
+def clean(source_strings):
+    source_objs = validate_sources(source_strings)
+    for source in source_objs:
         start_time = time.time()
         sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label))
         source_service.clean(source)
@@ -828,10 +815,9 @@ def main():
     manager.add_command("create_user", CreateUser())
     manager.add_command("reset_password", ResetPassword())
     manager.add_command("create_role", CreateRole())
-    manager.add_command("sources", Sources())
+    manager.add_command("source", source_manager)
     manager.add_command("report", Report())
-    manager.add_command("rotate_certificate", RotateCertificate())
+    manager.add_command("certificate", certificate_manager)
-    manager.add_command("reissue_certificate", ReissueCertificate())
     manager.run()
 
 

lemur/plugins/lemur_aws/elb.py

@@ -11,7 +11,7 @@ from flask import current_app
 from retrying import retry
 
 from lemur.exceptions import InvalidListener
-from lemur.plugins.lemur_aws.sts import sts_client, assume_service
+from lemur.plugins.lemur_aws.sts import sts_client
 
 
 def retry_throttled(exception):
@@ -104,15 +104,13 @@ def describe_load_balancer_types(policies, **kwargs):
 
 
 @sts_client('elb')
-def attach_certificate(account_number, region, name, port, certificate_id):
+def attach_certificate(name, port, certificate_id, **kwargs):
     """
     Attaches a certificate to a listener, throws exception
     if certificate specified does not exist in a particular account.
 
-    :param account_number:
-    :param region:
     :param name:
     :param port:
     :param certificate_id:
     """
-    return assume_service(account_number, 'elb', region).set_lb_listener_SSL_certificate(name, port, certificate_id)
+    return kwargs['client'].set_load_balancer_listener_ssl_certificate(LoadBalancerName=name, LoadBalancerPort=port, SSLCertificateId=certificate_id)

lemur/plugins/lemur_aws/iam.py

@@ -88,8 +88,7 @@ def create_arn_from_cert(account_number, region, certificate_name):
     :param certificate_name:
     :return:
     """
-    return "arn:aws:iam:{region}:{account_number}:{certificate_name}".format(
+    return "arn:aws:iam::{account_number}:server-certificate/{certificate_name}".format(
-        region=region,
         account_number=account_number,
         certificate_name=certificate_name)
 

lemur/plugins/lemur_aws/plugin.py

@@ -40,6 +40,10 @@ from lemur.plugins.lemur_aws import iam, s3, elb, ec2
 from lemur.plugins import lemur_aws as aws
 
 
+def get_region_from_dns(dns):
+    return dns.split('.')[-4]
+
+
 class AWSDestinationPlugin(DestinationPlugin):
     title = 'AWS'
     slug = 'aws-destination'
@@ -149,20 +153,21 @@ class AWSSourcePlugin(SourcePlugin):
                     )
 
                     if listener['PolicyNames']:
-                        policy = e.describe_load_balancer_policies(e['LoadBalancerName'], listener['PolicyNames'], account_number=account_number, region=region)
+                        policy = elb.describe_load_balancer_policies(e['LoadBalancerName'], listener['PolicyNames'], account_number=account_number, region=region)
                         endpoint['policy'] = format_elb_cipher_policy(policy)
 
                     endpoints.append(endpoint)
 
         return endpoints
 
-    def update_endpoint(self, options, endpoint, certificate):
+    def update_endpoint(self, endpoint, certificate):
+        options = endpoint.source.options
         account_number = self.get_option('accountNumber', options)
-        regions = self.get_option('regions', options)
 
-        for region in regions:
+        # relies on the fact that region is included in DNS name
-            arn = iam.create_arn_from_cert(account_number, region, certificate.name)
+        region = get_region_from_dns(endpoint.dnsname)
-            elb.attach_certificate(account_number, region, certificate.name, endpoint.port, arn)
+        arn = iam.create_arn_from_cert(account_number, region, certificate.name)
+        elb.attach_certificate(endpoint.name, endpoint.port, arn, account_number=account_number, region=region)
 
     def clean(self, options, **kwargs):
         account_number = self.get_option('accountNumber', options)

lemur/plugins/lemur_digicert/plugin.py

@@ -259,6 +259,11 @@ class DigiCertIssuerPlugin(IssuerPlugin):
     def __init__(self, *args, **kwargs):
         """Initialize the issuer with the appropriate details."""
         required_vars = [
+            'DIGICERT_API_KEY',
+            'DIGICERT_URL',
+            'DIGICERT_ORG_ID',
+            'DIGICERT_ROOT',
+            'DIGICERT_INTERMEDIATE'
         ]
 
         validate_conf(current_app, required_vars)

lemur/sources/models.py

@@ -6,11 +6,12 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from sqlalchemy.orm import relationship
-from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean
+from sqlalchemy import Column, Integer, String, Text, Boolean
 from sqlalchemy_utils import JSONType
 from lemur.database import db
 
 from lemur.plugins.base import plugins
+from sqlalchemy_utils import ArrowType
 
 
 class Source(db.Model):
@@ -21,7 +22,7 @@ class Source(db.Model):
     description = Column(Text())
     plugin_name = Column(String(32))
     active = Column(Boolean, default=True)
-    last_run = Column(DateTime)
+    last_run = Column(ArrowType)
     endpoints = relationship("Endpoint", back_populates="source")
 
     @property

lemur/sources/service.py

@@ -141,6 +141,7 @@ def sync_endpoints(source):
 
         policy['ciphers'] = policy_ciphers
         endpoint['policy'] = endpoint_service.get_or_create_policy(**policy)
+        endpoint['source'] = source
 
         if not exists:
             endpoint_service.create(**endpoint)