Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding more source syncing logic

Browse Source
This commit is contained in:
kevgliss
2015-08-01 18:31:38 -07:00
parent 46652ba117
commit e7e6a99ff4
9 changed files with 143 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lemur/sources/models.py
View File

@ -6,7 +6,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import copy
from sqlalchemy import Column, Integer, String, Text
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean
from sqlalchemy_utils import JSONType
from lemur.database import db
@ -20,6 +20,8 @@ class Source(db.Model):
options = Column(JSONType)
description = Column(Text())
plugin_name = Column(String(32))
active = Column(Boolean, default=True)
last_run = Column(DateTime)
@property
def plugin(self):

71
lemur/sources/service.py
View File

@ -5,9 +5,80 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from lemur import database
from lemur.sources.models import Source
from lemur.certificates.models import Certificate
from lemur.certificates import service as cert_service
from lemur.plugins.base import plugins
def _disassociate_certs_from_source(current_certificates, found_certificates, source_label):
missing = []
for cc in current_certificates:
for fc in found_certificates:
if fc.body == cc.body:
break
else:
missing.append(cc)
for c in missing:
for s in c.sources:
if s.label == source_label:
current_app.logger.info(
"Certificate {name} is no longer associated with {source}".format(
name=c.name,
source=source_label
)
)
c.sources.delete(s)
def sync(labels=None):
new, updated = 0, 0
c_certificates = cert_service.get_all_certs()
for source in database.get_all(Source, True, field='active'):
# we should be able to specify, individual sources to sync
if labels:
if source.label not in labels:
continue
current_app.logger.error("Retrieving certificates from {0}".format(source.title))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = cert_service.find_duplicates(certificate)
if not exists:
cert = cert_service.import_certificate(**certificate)
cert.sources.append(source)
database.update(cert)
new += 1
# check to make sure that existing certificates have the current source associated with it
if len(exists) == 1:
for s in cert.sources:
if s.label == source.label:
break
else:
cert.sources.append(source)
updated += 1
else:
current_app.logger.warning(
"Multiple certificates found, attempt to deduplicate the following certificates: {0}".format(
",".join([x.name for x in exists])
)
)
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
_disassociate_certs_from_source(c_certificates, certificates, source)
def create(label, plugin_name, options, description=None):

46
lemur/sources/sync.py
View File

@ -1,46 +0,0 @@
"""
.. module: lemur.sources.sync
:platform: Unix
:synopsis: This module contains various certificate syncing operations.
Because of the nature of the SSL environment there are multiple ways
a certificate could be created without Lemur's knowledge. Lemur attempts
to 'sync' with as many different datasources as possible to try and track
any certificate that may be in use.
These operations are typically run on a periodic basis from either the command
line or a cron job.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask import current_app
from lemur.certificates import service as cert_service
from lemur.plugins.base import plugins
from lemur.plugins.bases.source import SourcePlugin
def sync():
for plugin in plugins:
new = 0
updated = 0
if isinstance(plugin, SourcePlugin):
if plugin.is_enabled():
current_app.logger.error("Retrieving certificates from {0}".format(plugin.title))
certificates = plugin.get_certificates()
for certificate in certificates:
exists = cert_service.find_duplicates(certificate)
if not exists:
cert_service.import_certificate(**certificate)
new += 1
if len(exists) == 1:
updated += 1
# TODO associated cert with source
# TODO update cert if found from different source
# TODO disassociate source if missing

8
lemur/sources/views.py
View File

@ -23,6 +23,7 @@ FIELDS = {
'description': fields.String,
'sourceOptions': fields.Raw(attribute='options'),
'pluginName': fields.String(attribute='plugin_name'),
'lastRun': fields.DateTime(attribute='last_run', dt_format='iso8061'),
'label': fields.String,
'id': fields.Integer,
}
@ -71,6 +72,7 @@ class SourcesList(AuthenticatedResource):
}
],
"pluginName": "aws-source",
"lastRun": "2015-08-01T15:40:58",
"id": 3,
"description": "test",
"label": "test"
@ -120,6 +122,7 @@ class SourcesList(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}
@ -145,6 +148,7 @@ class SourcesList(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}
@ -203,6 +207,7 @@ class Sources(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}
@ -241,6 +246,7 @@ class Sources(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}
@ -266,6 +272,7 @@ class Sources(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}
@ -332,6 +339,7 @@ class CertificateSources(AuthenticatedResource):
],
"pluginName": "aws-source",
"id": 3,
"lastRun": "2015-08-01T15:40:58",
"description": "test",
"label": "test"
}