From e1bbf9d80c3ae66d4da9a93a39d59999bd674942 Mon Sep 17 00:00:00 2001 From: kevgliss Date: Wed, 30 Nov 2016 15:11:17 -0800 Subject: [PATCH] Improving endpoint rotation logic (#545) --- lemur/endpoints/service.py | 4 +- lemur/manage.py | 196 ++++++++++++------------- lemur/plugins/lemur_aws/elb.py | 8 +- lemur/plugins/lemur_aws/iam.py | 3 +- lemur/plugins/lemur_aws/plugin.py | 17 ++- lemur/plugins/lemur_digicert/plugin.py | 5 + lemur/sources/models.py | 5 +- lemur/sources/service.py | 1 + 8 files changed, 118 insertions(+), 121 deletions(-) diff --git a/lemur/endpoints/service.py b/lemur/endpoints/service.py index d6363619..8178a103 100644 --- a/lemur/endpoints/service.py +++ b/lemur/endpoints/service.py @@ -94,6 +94,8 @@ def update(endpoint_id, **kwargs): endpoint.policy = kwargs['policy'] endpoint.certificate = kwargs['certificate'] + endpoint.source = kwargs['source'] + metrics.send('endpoint_added', 'counter', 1) database.update(endpoint) return endpoint @@ -104,7 +106,7 @@ def rotate_certificate(endpoint, new_cert): endpoint.source.plugin.update_endpoint(endpoint, new_cert) endpoint.certificate = new_cert except Exception as e: - metrics.send('rotate_failure', 'counter', 1, tags={'endpoint': endpoint.name}) + metrics.send('rotate_failure', 'counter', 1, metric_tags={'endpoint': endpoint.name}) current_app.logger.exception(e) raise e diff --git a/lemur/manage.py b/lemur/manage.py index b8bed77f..794479b2 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -518,81 +518,32 @@ def print_certificate_details(details): ) -class RotateCertificate(Command): - """ - Rotates certificate on all endpoints managed by Lemur. - """ - option_list = ( - Option('-n', '--cert-name', dest='new_cert_name'), - Option('-o', '--old-cert-name', dest='old_cert_name', required=True), - Option('-c', '--commit', dest='commit', action='store_true', default=False) - ) +certificate_manager = Manager(usage="Handles all certificate related tasks.") - def run(self, new_cert_name, old_cert_name, commit): - from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives - from lemur.endpoints.service import rotate_certificate - old_cert = get_by_name(old_cert_name) +@certificate_manager.command +def rotate(new_certificate_name, old_certificate_name, commit=False): + from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives + from lemur.endpoints.service import rotate_certificate - if not old_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name)) + old_cert = get_by_name(old_certificate_name) + + if not old_cert: + sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) + sys.exit(1) + + if new_certificate_name: + new_cert = get_by_name(new_certificate_name) + + if not new_cert: + sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) sys.exit(1) - if new_cert_name: - new_cert = get_by_name(new_cert_name) + if commit: + sys.stdout.write("[!] Running in COMMIT mode.\n") - if not new_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name)) - sys.exit(1) - - if commit: - sys.stdout.write("[!] Running in COMMIT mode.\n") - - if not new_cert_name: - sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_cert_name)) - - details = get_certificate_primitives(old_cert) - print_certificate_details(details) - - if commit: - new_cert = reissue_certificate(old_cert, replace=True) - sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name)) - - sys.stdout.write("[+] Done! \n") - - if len(old_cert.endpoints) > 0: - for endpoint in old_cert.endpoints: - sys.stdout.write("[+] Certificate found deployed onto {0}\n ".format(endpoint.name)) - sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n ".format(old_cert_name, new_cert.name)) - - if commit: - rotate_certificate(endpoint, new_cert_name) - - sys.stdout.write("[+] Done! \n") - else: - sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n") - - -class ReissueCertificate(Command): - """ - Reissues a certificate based on a given certificate. - """ - option_list = ( - Option('-o', '--old-cert-name', dest='old_cert_name', required=True), - Option('-c', '--commit', dest='commit', action='store_true', default=False) - ) - - def run(self, old_cert_name, commit): - from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives - - old_cert = get_by_name(old_cert_name) - - if not old_cert: - sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_cert_name)) - sys.exit(1) - - if commit: - sys.stdout.write("[!] Running in COMMIT mode.\n") + if not new_certificate_name: + sys.stdout.write("[!] No new certificate provided. Attempting to re-issue old certificate: {0}.\n".format(old_certificate_name)) details = get_certificate_primitives(old_cert) print_certificate_details(details) @@ -603,6 +554,48 @@ class ReissueCertificate(Command): sys.stdout.write("[+] Done! \n") + if len(old_cert.endpoints) > 0: + for endpoint in old_cert.endpoints: + sys.stdout.write( + "[+] Certificate deployed on endpoint: name:{name} dnsname:{dnsname} port:{port} type:{type}\n".format( + name=endpoint.name, + dnsname=endpoint.dnsname, + port=endpoint.port, + type=endpoint.type + ) + ) + sys.stdout.write("[+] Rotating certificate from: {0} to: {1}\n".format(old_certificate_name, new_cert.name)) + + if commit: + rotate_certificate(endpoint, new_cert) + + sys.stdout.write("[+] Done! \n") + else: + sys.stdout.write("[!] Certificate not found on any existing endpoints. Nothing to rotate.\n") + + +@certificate_manager.command +def reissue(old_certificate_name, commit=False): + from lemur.certificates.service import get_by_name, reissue_certificate, get_certificate_primitives + + old_cert = get_by_name(old_certificate_name) + + if not old_cert: + sys.stdout.write("[-] No certificate found with name: {0}\n".format(old_certificate_name)) + sys.exit(1) + + if commit: + sys.stdout.write("[!] Running in COMMIT mode.\n") + + details = get_certificate_primitives(old_cert) + print_certificate_details(details) + + if commit: + new_cert = reissue_certificate(old_cert, replace=True) + sys.stdout.write("[+] Issued new certificate named: {0}\n".format(new_cert.name)) + + sys.stdout.write("[+] Done! \n") + @manager.command def publish_verisign_units(): @@ -743,46 +736,37 @@ class Report(Command): sys.stdout.write(tabulate(rows, headers=["Authority Name", "Description", "Daily Average", "Monthy Average", "Yearly Average"]) + "\n") -class Sources(Command): - """ - Defines a set of actions to take against Lemur's sources. - """ - option_list = ( - Option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.', required=True), - Option('-a', '--action', choices=['sync', 'clean'], dest='action', help='Action to take on source.', required=True) - ) +source_manager = Manager(usage="Handles all source related tasks.") - def run(self, source_strings, action): - sources = [] - if not source_strings: - table = [] - for source in source_service.get_all(): - table.append([source.label, source.active, source.description]) - sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description'])) - sys.exit(1) +def validate_sources(source_strings): + sources = [] + if not source_strings: + table = [] + for source in source_service.get_all(): + table.append([source.label, source.active, source.description]) - elif 'all' in source_strings: - sources = source_service.get_all() + sys.stdout.write("No source specified choose from below:\n") + sys.stdout.write(tabulate(table, headers=['Label', 'Active', 'Description'])) + sys.exit(1) - else: - for source_str in source_strings: - source = source_service.get_by_label(source_str) + if 'all' in source_strings: + sources = source_service.get_all() - if not source: - sys.stderr.write("Unable to find specified source with label: {0}".format(source_str)) + for source_str in source_strings: + source = source_service.get_by_label(source_str) - sources.append(source) + if not source: + sys.stderr.write("Unable to find specified source with label: {0}".format(source_str)) - for source in sources: - if action == 'sync': - self.sync(source) + sources.append(source) + return sources - if action == 'clean': - self.clean(source) - @staticmethod - def sync(source): +@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') +def sync(source_strings): + source_objs = validate_sources(source_strings) + for source in source_objs: start_time = time.time() sys.stdout.write("[+] Staring to sync source: {label}!\n".format(label=source.label)) @@ -805,8 +789,11 @@ class Sources(Command): metrics.send('sync_failed', 'counter', 1, metric_tags={'source': source.label}) - @staticmethod - def clean(source): + +@source_manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.') +def clean(source_strings): + source_objs = validate_sources(source_strings) + for source in source_objs: start_time = time.time() sys.stdout.write("[+] Staring to clean source: {label}!\n".format(label=source.label)) source_service.clean(source) @@ -828,10 +815,9 @@ def main(): manager.add_command("create_user", CreateUser()) manager.add_command("reset_password", ResetPassword()) manager.add_command("create_role", CreateRole()) - manager.add_command("sources", Sources()) + manager.add_command("source", source_manager) manager.add_command("report", Report()) - manager.add_command("rotate_certificate", RotateCertificate()) - manager.add_command("reissue_certificate", ReissueCertificate()) + manager.add_command("certificate", certificate_manager) manager.run() diff --git a/lemur/plugins/lemur_aws/elb.py b/lemur/plugins/lemur_aws/elb.py index 385bf0d4..c75a8370 100644 --- a/lemur/plugins/lemur_aws/elb.py +++ b/lemur/plugins/lemur_aws/elb.py @@ -11,7 +11,7 @@ from flask import current_app from retrying import retry from lemur.exceptions import InvalidListener -from lemur.plugins.lemur_aws.sts import sts_client, assume_service +from lemur.plugins.lemur_aws.sts import sts_client def retry_throttled(exception): @@ -104,15 +104,13 @@ def describe_load_balancer_types(policies, **kwargs): @sts_client('elb') -def attach_certificate(account_number, region, name, port, certificate_id): +def attach_certificate(name, port, certificate_id, **kwargs): """ Attaches a certificate to a listener, throws exception if certificate specified does not exist in a particular account. - :param account_number: - :param region: :param name: :param port: :param certificate_id: """ - return assume_service(account_number, 'elb', region).set_lb_listener_SSL_certificate(name, port, certificate_id) + return kwargs['client'].set_load_balancer_listener_ssl_certificate(LoadBalancerName=name, LoadBalancerPort=port, SSLCertificateId=certificate_id) diff --git a/lemur/plugins/lemur_aws/iam.py b/lemur/plugins/lemur_aws/iam.py index b332484f..2051c5b2 100644 --- a/lemur/plugins/lemur_aws/iam.py +++ b/lemur/plugins/lemur_aws/iam.py @@ -88,8 +88,7 @@ def create_arn_from_cert(account_number, region, certificate_name): :param certificate_name: :return: """ - return "arn:aws:iam:{region}:{account_number}:{certificate_name}".format( - region=region, + return "arn:aws:iam::{account_number}:server-certificate/{certificate_name}".format( account_number=account_number, certificate_name=certificate_name) diff --git a/lemur/plugins/lemur_aws/plugin.py b/lemur/plugins/lemur_aws/plugin.py index da100fb3..5e9605a8 100644 --- a/lemur/plugins/lemur_aws/plugin.py +++ b/lemur/plugins/lemur_aws/plugin.py @@ -40,6 +40,10 @@ from lemur.plugins.lemur_aws import iam, s3, elb, ec2 from lemur.plugins import lemur_aws as aws +def get_region_from_dns(dns): + return dns.split('.')[-4] + + class AWSDestinationPlugin(DestinationPlugin): title = 'AWS' slug = 'aws-destination' @@ -149,20 +153,21 @@ class AWSSourcePlugin(SourcePlugin): ) if listener['PolicyNames']: - policy = e.describe_load_balancer_policies(e['LoadBalancerName'], listener['PolicyNames'], account_number=account_number, region=region) + policy = elb.describe_load_balancer_policies(e['LoadBalancerName'], listener['PolicyNames'], account_number=account_number, region=region) endpoint['policy'] = format_elb_cipher_policy(policy) endpoints.append(endpoint) return endpoints - def update_endpoint(self, options, endpoint, certificate): + def update_endpoint(self, endpoint, certificate): + options = endpoint.source.options account_number = self.get_option('accountNumber', options) - regions = self.get_option('regions', options) - for region in regions: - arn = iam.create_arn_from_cert(account_number, region, certificate.name) - elb.attach_certificate(account_number, region, certificate.name, endpoint.port, arn) + # relies on the fact that region is included in DNS name + region = get_region_from_dns(endpoint.dnsname) + arn = iam.create_arn_from_cert(account_number, region, certificate.name) + elb.attach_certificate(endpoint.name, endpoint.port, arn, account_number=account_number, region=region) def clean(self, options, **kwargs): account_number = self.get_option('accountNumber', options) diff --git a/lemur/plugins/lemur_digicert/plugin.py b/lemur/plugins/lemur_digicert/plugin.py index aa3f1dcd..28ddb5d7 100644 --- a/lemur/plugins/lemur_digicert/plugin.py +++ b/lemur/plugins/lemur_digicert/plugin.py @@ -259,6 +259,11 @@ class DigiCertIssuerPlugin(IssuerPlugin): def __init__(self, *args, **kwargs): """Initialize the issuer with the appropriate details.""" required_vars = [ + 'DIGICERT_API_KEY', + 'DIGICERT_URL', + 'DIGICERT_ORG_ID', + 'DIGICERT_ROOT', + 'DIGICERT_INTERMEDIATE' ] validate_conf(current_app, required_vars) diff --git a/lemur/sources/models.py b/lemur/sources/models.py index 92f7eb08..fcf91e48 100644 --- a/lemur/sources/models.py +++ b/lemur/sources/models.py @@ -6,11 +6,12 @@ .. moduleauthor:: Kevin Glisson """ from sqlalchemy.orm import relationship -from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean +from sqlalchemy import Column, Integer, String, Text, Boolean from sqlalchemy_utils import JSONType from lemur.database import db from lemur.plugins.base import plugins +from sqlalchemy_utils import ArrowType class Source(db.Model): @@ -21,7 +22,7 @@ class Source(db.Model): description = Column(Text()) plugin_name = Column(String(32)) active = Column(Boolean, default=True) - last_run = Column(DateTime) + last_run = Column(ArrowType) endpoints = relationship("Endpoint", back_populates="source") @property diff --git a/lemur/sources/service.py b/lemur/sources/service.py index 42762489..878cdbc7 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -141,6 +141,7 @@ def sync_endpoints(source): policy['ciphers'] = policy_ciphers endpoint['policy'] = endpoint_service.get_or_create_policy(**policy) + endpoint['source'] = source if not exists: endpoint_service.create(**endpoint)