Clean refactor (#635)
* Adding rotation to the UI. * Removing spinkit dependency. * refactoring source cleaning
This commit is contained in:
kevgliss
GitHub
@ -15,8 +15,12 @@ from flask_script import Manager
|
||||
from flask import current_app
|
||||
|
||||
from lemur.extensions import metrics
|
||||
from lemur.plugins.base import plugins
|
||||
|
||||
from lemur.sources import service as source_service
|
||||
from lemur.users import service as user_service
|
||||
from lemur.certificates import service as certificate_service
|
||||
|
||||
|
||||
manager = Manager(usage="Handles all source related tasks.")
|
||||
|
||||
@ -48,8 +52,8 @@ def validate_sources(source_strings):
|
||||
|
||||
@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
|
||||
def sync(source_strings):
|
||||
source_objs = validate_sources(source_strings)
|
||||
for source in source_objs:
|
||||
sources = validate_sources(source_strings)
|
||||
for source in sources:
|
||||
start_time = time.time()
|
||||
print("[+] Staring to sync source: {label}!\n".format(label=source.label))
|
||||
|
||||
@ -86,15 +90,45 @@ def sync(source_strings):
|
||||
|
||||
|
||||
@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
|
||||
def clean(source_strings):
|
||||
source_objs = validate_sources(source_strings)
|
||||
for source in source_objs:
|
||||
@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
|
||||
def clean(source_strings, commit):
|
||||
sources = validate_sources(source_strings)
|
||||
for source in sources:
|
||||
s = plugins.get(source.plugin_name)
|
||||
|
||||
if not hasattr(s, 'clean'):
|
||||
print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
|
||||
source.label
|
||||
))
|
||||
continue
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
print("[+] Staring to clean source: {label}!\n".format(label=source.label))
|
||||
source_service.clean(source)
|
||||
|
||||
cleaned = 0
|
||||
for certificate in certificate_service.get_all_pending_cleaning(source):
|
||||
if commit:
|
||||
try:
|
||||
s.clean(certificate, source.options)
|
||||
certificate.sources.remove(source)
|
||||
certificate_service.database.update(certificate)
|
||||
metrics.send('clean_success', 'counter', 1, metric_tags={'source': source.label})
|
||||
except Exception as e:
|
||||
current_app.logger.exception(e)
|
||||
metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label})
|
||||
|
||||
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
|
||||
certificate.name,
|
||||
source.label
|
||||
))
|
||||
|
||||
cleaned += 1
|
||||
|
||||
print(
|
||||
"[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
|
||||
"[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
|
||||
label=source.label,
|
||||
time=(time.time() - start_time)
|
||||
time=(time.time() - start_time),
|
||||
cleaned=cleaned
|
||||
)
|
||||
)
|
||||
|
||||
@ -12,7 +12,7 @@ from flask import current_app
|
||||
from lemur import database
|
||||
from lemur.sources.models import Source
|
||||
from lemur.certificates.models import Certificate
|
||||
from lemur.certificates import service as cert_service
|
||||
from lemur.certificates import service as certificate_service
|
||||
from lemur.endpoints import service as endpoint_service
|
||||
from lemur.destinations import service as destination_service
|
||||
|
||||
@ -21,29 +21,6 @@ from lemur.certificates.schemas import CertificateUploadInputSchema
|
||||
from lemur.plugins.base import plugins
|
||||
|
||||
|
||||
# TODO optimize via sql query
|
||||
def _disassociate_certs_from_source(certificates, source):
|
||||
current_certificates = cert_service.get_by_source(source_label=source.label)
|
||||
missing = []
|
||||
for cc in current_certificates:
|
||||
for fc in certificates:
|
||||
if fc['body'] == cc.body:
|
||||
break
|
||||
else:
|
||||
missing.append(cc)
|
||||
|
||||
for c in missing:
|
||||
for s in c.sources:
|
||||
if s.label == source:
|
||||
current_app.logger.info(
|
||||
"Certificate {name} is no longer associated with {source}.".format(
|
||||
name=c.name,
|
||||
source=source.label
|
||||
)
|
||||
)
|
||||
c.sources.delete(s)
|
||||
|
||||
|
||||
def certificate_create(certificate, source):
|
||||
data, errors = CertificateUploadInputSchema().load(certificate)
|
||||
|
||||
@ -52,7 +29,7 @@ def certificate_create(certificate, source):
|
||||
|
||||
data['creator'] = certificate['creator']
|
||||
|
||||
cert = cert_service.import_certificate(**data)
|
||||
cert = certificate_service.import_certificate(**data)
|
||||
cert.description = "This certificate was automatically discovered by Lemur"
|
||||
cert.sources.append(source)
|
||||
sync_update_destination(cert, source)
|
||||
@ -99,12 +76,12 @@ def sync_endpoints(source):
|
||||
certificate = endpoint.pop('certificate', None)
|
||||
|
||||
if certificate_name:
|
||||
cert = cert_service.get_by_name(certificate_name)
|
||||
cert = certificate_service.get_by_name(certificate_name)
|
||||
|
||||
elif certificate:
|
||||
cert = cert_service.find_duplicates(certificate)
|
||||
cert = certificate_service.find_duplicates(certificate)
|
||||
if not cert:
|
||||
cert = cert_service.import_certificate(**certificate)
|
||||
cert = certificate_service.import_certificate(**certificate)
|
||||
|
||||
if not cert:
|
||||
current_app.logger.error(
|
||||
@ -142,7 +119,7 @@ def sync_certificates(source, user):
|
||||
certificates = s.get_certificates(source.options)
|
||||
|
||||
for certificate in certificates:
|
||||
exists = cert_service.find_duplicates(certificate)
|
||||
exists = certificate_service.find_duplicates(certificate)
|
||||
|
||||
certificate['owner'] = user.email
|
||||
certificate['creator'] = user
|
||||
@ -162,9 +139,6 @@ def sync_certificates(source, user):
|
||||
)
|
||||
)
|
||||
|
||||
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
|
||||
_disassociate_certs_from_source(certificates, source)
|
||||
|
||||
return new, updated
|
||||
|
||||
|
||||
@ -178,33 +152,13 @@ def sync(source, user):
|
||||
return {'endpoints': (new_endpoints, updated_endpoints), 'certificates': (new_certs, updated_certs)}
|
||||
|
||||
|
||||
def clean(source):
|
||||
s = plugins.get(source.plugin_name)
|
||||
|
||||
try:
|
||||
certificates = s.clean(source.options)
|
||||
except NotImplementedError:
|
||||
current_app.logger.warning("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
|
||||
source.label
|
||||
))
|
||||
return
|
||||
|
||||
for certificate in certificates:
|
||||
cert = cert_service.get_by_name(certificate)
|
||||
|
||||
if cert:
|
||||
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
|
||||
cert.name,
|
||||
source.label
|
||||
))
|
||||
cert.sources.remove(source)
|
||||
|
||||
|
||||
def create(label, plugin_name, options, description=None):
|
||||
"""
|
||||
Creates a new source, that can then be used as a source for certificates.
|
||||
|
||||
:param label: Source common name
|
||||
:param plugin_name:
|
||||
:param options:
|
||||
:param description:
|
||||
:rtype : Source
|
||||
:return: New source
|
||||
@ -219,6 +173,8 @@ def update(source_id, label, options, description):
|
||||
|
||||
:param source_id: Lemur assigned ID
|
||||
:param label: Source common name
|
||||
:param options:
|
||||
:param description:
|
||||
:rtype : Source
|
||||
:return:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user