From e7e6a99ff4272125e88f9b7caecfe9f838500d6d Mon Sep 17 00:00:00 2001 From: kevgliss Date: Sat, 1 Aug 2015 18:31:38 -0700 Subject: [PATCH] Adding more source syncing logic --- docs/administration/index.rst | 12 ++++- docs/developer/plugins/index.rst | 6 +-- lemur/manage.py | 83 +++++++++++++++++--------------- lemur/notifications/service.py | 2 +- lemur/sources/models.py | 4 +- lemur/sources/service.py | 71 +++++++++++++++++++++++++++ lemur/sources/sync.py | 46 ------------------ lemur/sources/views.py | 8 +++ setup.py | 3 +- 9 files changed, 143 insertions(+), 92 deletions(-) delete mode 100644 lemur/sources/sync.py diff --git a/docs/administration/index.rst b/docs/administration/index.rst index 9a1846e2..b609cbfe 100644 --- a/docs/administration/index.rst +++ b/docs/administration/index.rst @@ -505,11 +505,19 @@ All commands default to `~/.lemur/lemur.conf.py` if a configuration is not speci .. data:: sync - Sync attempts to discover certificates in the environment that were not created by Lemur. There + Sync attempts to discover certificates in the environment that were not created by Lemur. If you wish to only sync + a few sources you can pass a comma delimited list of sources to sync :: - lemur sync --all + lemur sync source1,source2 + + + Additionally you can also list the available sources that Lemur can sync + + :: + + lemur sync -list Identity and Access Management diff --git a/docs/developer/plugins/index.rst b/docs/developer/plugins/index.rst index ae07f1c2..c813b4f7 100644 --- a/docs/developer/plugins/index.rst +++ b/docs/developer/plugins/index.rst @@ -215,9 +215,9 @@ certificate Lemur does not know about and adding the certificate to it's invento The `SourcePlugin` object has one default option of `pollRate`. This controls the number of seconds which to get new certificates. .. warning:: - Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. - This means special consideration needs to be taken such that running all `SourcePlugins` does not take >15min to run. It also means - that the minimum resolution of a source plugin poll rate is effectively 15min. + Lemur currently has a very basic polling system of running a cron job every 15min to see which source plugins need to be run. A lock file is generated to guarentee that ] + only one sync is running at a time. It also means that the minimum resolution of a source plugin poll rate is effectively 15min. You can always specify a faster cron + job if you need a higher resolution sync job. The `SourcePlugin` object requires implementation of one function:: diff --git a/lemur/manage.py b/lemur/manage.py index 877b34e1..5821f4e4 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -1,12 +1,15 @@ import os import sys import base64 +import time from gunicorn.config import make_settings from cryptography.fernet import Fernet +from lockfile import LockFile, LockTimeout + from flask import current_app -from flask.ext.script import Manager, Command, Option, Group, prompt_pass +from flask.ext.script import Manager, Command, Option, prompt_pass from flask.ext.migrate import Migrate, MigrateCommand, stamp from flask_script.commands import ShowUrls, Clean, Server @@ -15,11 +18,12 @@ from lemur.users import service as user_service from lemur.roles import service as role_service from lemur.destinations import service as destination_service from lemur.certificates import service as cert_service +from lemur.sources import service as source_service from lemur.plugins.base import plugins from lemur.certificates.verify import verify_string -from lemur.sources import sync +from lemur.sources.service import sync from lemur import create_app @@ -176,51 +180,55 @@ def generate_settings(): return output -class Sync(Command): +@manager.option('-s', '--sources', dest='labels', default='', required=False) +@manager.option('-l', '--list', dest='view', default=False, required=False) +def sync_sources(labels, view): """ Attempts to run several methods Certificate discovery. This is run on a periodic basis and updates the Lemur datastore with the information it discovers. """ + if view: + for source in source_service.get_all(): + sys.stdout.write( + "[{active}]\t{label}\t{description}!\n".format( + label=source.label, + description=source.description, + active=source.active + ) + ) + else: + start_time = time.time() + lock_file = "/tmp/.lemur_lock" + sync_lock = LockFile(lock_file) - # TODO create these commands dynamically - option_list = [ - Group( - Option('-a', '--all', action="store_true"), - exclusive=True, required=True - ) - ] - - def run(self, all, aws, cloudca, source): - sys.stdout.write("[!] Starting to sync with external sources!\n") - - if all or aws: - sys.stdout.write("[!] Starting to sync with AWS!\n") + while not sync_lock.i_am_locking(): try: - sync.aws() - # sync_all_elbs() - sys.stdout.write("[+] Finished syncing with AWS!\n") - except Exception as e: - sys.stdout.write("[-] Syncing with AWS failed!\n") + sync_lock.acquire(timeout=10) # wait up to 10 seconds - if all or cloudca: - sys.stdout.write("[!] Starting to sync with CloudCA!\n") - try: - sync.cloudca() - sys.stdout.write("[+] Finished syncing with CloudCA!\n") - except Exception as e: - sys.stdout.write("[-] Syncing with CloudCA failed!\n") + if labels: + sys.stdout.write("[+] Staring to sync sources: {labels}!\n".format(labels)) + labels = labels.split(",") + else: + sys.stdout.write("[+] Starting to sync ALL sources!\n".format(labels)) - sys.stdout.write("[!] Starting to sync with Source Code!\n") + sync(labels=labels) + sys.stdout.write( + "[+] Finished syncing sources. Run Time: {time}\n".format( + time=(time.time() - start_time) + ) + ) + except LockTimeout: + sys.stderr.write( + "[!] Unable to acquire file lock on {file}, is there another sync running?\n".format( + file=lock_file + ) + ) + sync_lock.break_lock() + sync_lock.acquire() + sync_lock.release() - if all or source: - try: - sync.source() - sys.stdout.write("[+] Finished syncing with Source Code!\n") - except Exception as e: - sys.stdout.write("[-] Syncing with Source Code failed!\n") - - sys.stdout.write("[+] Finished syncing with external sources!\n") + sync_lock.release() class InitializeApp(Command): @@ -473,7 +481,6 @@ def main(): manager.add_command("init", InitializeApp()) manager.add_command("create_user", CreateUser()) manager.add_command("create_role", CreateRole()) - manager.add_command("sync", Sync()) manager.run() diff --git a/lemur/notifications/service.py b/lemur/notifications/service.py index 517bc8ab..63890d8a 100644 --- a/lemur/notifications/service.py +++ b/lemur/notifications/service.py @@ -64,7 +64,7 @@ def send_expiration_notifications(): """ notifications = 0 - for plugin_name, notifications in database.get_all(Notification, 'active', field='status').group_by(Notification.plugin_name): + for plugin_name, notifications in database.get_all(Notification, True, field='active').group_by(Notification.plugin_name): notifications += 1 messages = _deduplicate(notifications) diff --git a/lemur/sources/models.py b/lemur/sources/models.py index 85beaeea..2d9870af 100644 --- a/lemur/sources/models.py +++ b/lemur/sources/models.py @@ -6,7 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import copy -from sqlalchemy import Column, Integer, String, Text +from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean from sqlalchemy_utils import JSONType from lemur.database import db @@ -20,6 +20,8 @@ class Source(db.Model): options = Column(JSONType) description = Column(Text()) plugin_name = Column(String(32)) + active = Column(Boolean, default=True) + last_run = Column(DateTime) @property def plugin(self): diff --git a/lemur/sources/service.py b/lemur/sources/service.py index dd7eaa1a..e3f53094 100644 --- a/lemur/sources/service.py +++ b/lemur/sources/service.py @@ -5,9 +5,80 @@ :license: Apache, see LICENSE for more details. .. moduleauthor:: Kevin Glisson """ +from flask import current_app + from lemur import database from lemur.sources.models import Source from lemur.certificates.models import Certificate +from lemur.certificates import service as cert_service + +from lemur.plugins.base import plugins + + +def _disassociate_certs_from_source(current_certificates, found_certificates, source_label): + missing = [] + for cc in current_certificates: + for fc in found_certificates: + if fc.body == cc.body: + break + else: + missing.append(cc) + + for c in missing: + for s in c.sources: + if s.label == source_label: + current_app.logger.info( + "Certificate {name} is no longer associated with {source}".format( + name=c.name, + source=source_label + ) + ) + c.sources.delete(s) + + +def sync(labels=None): + new, updated = 0, 0 + c_certificates = cert_service.get_all_certs() + + for source in database.get_all(Source, True, field='active'): + # we should be able to specify, individual sources to sync + if labels: + if source.label not in labels: + continue + + current_app.logger.error("Retrieving certificates from {0}".format(source.title)) + s = plugins.get(source.plugin_name) + certificates = s.get_certificates(source.options) + + for certificate in certificates: + exists = cert_service.find_duplicates(certificate) + + if not exists: + cert = cert_service.import_certificate(**certificate) + cert.sources.append(source) + database.update(cert) + + new += 1 + + # check to make sure that existing certificates have the current source associated with it + if len(exists) == 1: + for s in cert.sources: + if s.label == source.label: + break + else: + cert.sources.append(source) + + updated += 1 + + else: + current_app.logger.warning( + "Multiple certificates found, attempt to deduplicate the following certificates: {0}".format( + ",".join([x.name for x in exists]) + ) + ) + + # we need to try and find the absent of certificates so we can properly disassociate them when they are deleted + _disassociate_certs_from_source(c_certificates, certificates, source) def create(label, plugin_name, options, description=None): diff --git a/lemur/sources/sync.py b/lemur/sources/sync.py deleted file mode 100644 index 5b37897f..00000000 --- a/lemur/sources/sync.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -.. module: lemur.sources.sync - :platform: Unix - :synopsis: This module contains various certificate syncing operations. - Because of the nature of the SSL environment there are multiple ways - a certificate could be created without Lemur's knowledge. Lemur attempts - to 'sync' with as many different datasources as possible to try and track - any certificate that may be in use. - - These operations are typically run on a periodic basis from either the command - line or a cron job. - - :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more - :license: Apache, see LICENSE for more details. -.. moduleauthor:: Kevin Glisson -""" -from flask import current_app - -from lemur.certificates import service as cert_service - -from lemur.plugins.base import plugins -from lemur.plugins.bases.source import SourcePlugin - - -def sync(): - for plugin in plugins: - new = 0 - updated = 0 - if isinstance(plugin, SourcePlugin): - if plugin.is_enabled(): - current_app.logger.error("Retrieving certificates from {0}".format(plugin.title)) - certificates = plugin.get_certificates() - - for certificate in certificates: - exists = cert_service.find_duplicates(certificate) - - if not exists: - cert_service.import_certificate(**certificate) - new += 1 - - if len(exists) == 1: - updated += 1 - - # TODO associated cert with source - # TODO update cert if found from different source - # TODO disassociate source if missing diff --git a/lemur/sources/views.py b/lemur/sources/views.py index 807054cc..7e828f83 100644 --- a/lemur/sources/views.py +++ b/lemur/sources/views.py @@ -23,6 +23,7 @@ FIELDS = { 'description': fields.String, 'sourceOptions': fields.Raw(attribute='options'), 'pluginName': fields.String(attribute='plugin_name'), + 'lastRun': fields.DateTime(attribute='last_run', dt_format='iso8061'), 'label': fields.String, 'id': fields.Integer, } @@ -71,6 +72,7 @@ class SourcesList(AuthenticatedResource): } ], "pluginName": "aws-source", + "lastRun": "2015-08-01T15:40:58", "id": 3, "description": "test", "label": "test" @@ -120,6 +122,7 @@ class SourcesList(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } @@ -145,6 +148,7 @@ class SourcesList(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } @@ -203,6 +207,7 @@ class Sources(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } @@ -241,6 +246,7 @@ class Sources(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } @@ -266,6 +272,7 @@ class Sources(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } @@ -332,6 +339,7 @@ class CertificateSources(AuthenticatedResource): ], "pluginName": "aws-source", "id": 3, + "lastRun": "2015-08-01T15:40:58", "description": "test", "label": "test" } diff --git a/setup.py b/setup.py index 7371881c..5b315241 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ install_requires = [ 'cryptography>=1.0dev', 'pyopenssl==0.15.1', 'pyjwt==1.0.1', - 'xmltodict==0.9.2' + 'xmltodict==0.9.2', + 'lockfile=0.10.2' ] tests_require = [