Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Clean refactor (#635) 

* Adding rotation to the UI.

* Removing spinkit dependency.

* refactoring source cleaning
This commit is contained in:
kevgliss 2016-12-27 10:31:33 -08:00 committed by GitHub
parent 700c57b807
commit de7cec35c6
8 changed files with 86 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lemur/certificates/service.py
View File

@ -68,14 +68,15 @@ def get_all_certs():
return Certificate.query.all()
def get_by_source(source_label):
def get_all_pending_cleaning(source):
"""
Retrieves all certificates from a given source.
Retrieves all certificates that are available for cleaning.
:param source_label:
:param source:
:return:
"""
return Certificate.query.filter(Certificate.sources.any(label=source_label))
return Certificate.query.filter(Certificate.sources.any(id=source.id))\
.filter(not_(Certificate.endpoints.any())).all()
def get_all_pending_reissue():

2
lemur/plugins/bases/source.py
View File

@ -28,7 +28,7 @@ class SourcePlugin(Plugin):
def get_endpoints(self, options, **kwargs):
raise NotImplementedError
def clean(self, options, **kwargs):
def clean(self, certificate, options, **kwargs):
raise NotImplementedError
@property

6
lemur/plugins/lemur_aws/elb.py
View File

@ -16,14 +16,14 @@ from lemur.plugins.lemur_aws.sts import sts_client
def retry_throttled(exception):
"""
Determiens if this exception is due to throttling
Determines if this exception is due to throttling
:param exception:
:return:
"""
if isinstance(exception, botocore.exceptions.ClientError):
if exception.response['Error']['Code'] == 'LoadBalancerNotFound':
return True
return False
return False
return True
def is_valid(listener_tuple):

25
lemur/plugins/lemur_aws/iam.py
View File

@ -6,7 +6,24 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import botocore
from retrying import retry
from lemur.plugins.lemur_aws.sts import assume_service
from lemur.plugins.lemur_aws.sts import sts_client
def retry_throttled(exception):
"""
Determines if this exception is due to throttling
:param exception:
:return:
"""
if isinstance(exception, botocore.exceptions.ClientError):
if exception.response['Error']['Code'] == 'NoSuchEntity':
return False
return True
def get_name_from_arn(arn):
@ -33,15 +50,17 @@ def upload_cert(account_number, name, body, private_key, cert_chain=None):
cert_chain=str(cert_chain))
def delete_cert(account_number, cert_name):
@sts_client('iam')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
def delete_cert(cert_name, **kwargs):
"""
Delete a certificate from AWS
:param account_number:
:param cert_name:
:return:
"""
return assume_service(account_number, 'iam').delete_server_cert(cert_name)
client = kwargs.pop('client')
client.delete_server_certificate(ServerCertificateName=cert_name)
def get_all_server_certs(account_number):

16
lemur/plugins/lemur_aws/plugin.py
View File

@ -261,21 +261,9 @@ class AWSSourcePlugin(SourcePlugin):
else:
elb.attach_certificate(endpoint.name, endpoint.port, arn, account_number=account_number, region=region)
def clean(self, options, **kwargs):
def clean(self, certificate, options, **kwargs):
account_number = self.get_option('accountNumber', options)
certificates = self.get_certificates(options)
endpoints = self.get_endpoints(options)
orphaned = []
for certificate in certificates:
for endpoint in endpoints:
if certificate['name'] == endpoint['certificate_name']:
break
else:
orphaned.append(certificate['name'])
iam.delete_cert(account_number, certificate)
return orphaned
iam.delete_cert(certificate.name, account_number=account_number)
class S3DestinationPlugin(DestinationPlugin):

1
lemur/plugins/lemur_aws/sts.py
View File

@ -56,6 +56,7 @@ def sts_client(service, service_type='client'):
kwargs.pop('account_number'),
current_app.config.get('LEMUR_INSTANCE_PROFILE', 'Lemur')
)
# TODO add user specific information to RoleSessionName
role = sts.assume_role(RoleArn=arn, RoleSessionName='lemur')

50
lemur/sources/cli.py
View File

@ -15,8 +15,12 @@ from flask_script import Manager
from flask import current_app
from lemur.extensions import metrics
from lemur.plugins.base import plugins
from lemur.sources import service as source_service
from lemur.users import service as user_service
from lemur.certificates import service as certificate_service
manager = Manager(usage="Handles all source related tasks.")
@ -48,8 +52,8 @@ def validate_sources(source_strings):
@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
def sync(source_strings):
source_objs = validate_sources(source_strings)
for source in source_objs:
sources = validate_sources(source_strings)
for source in sources:
start_time = time.time()
print("[+] Staring to sync source: {label}!\n".format(label=source.label))
@ -86,15 +90,45 @@ def sync(source_strings):
@manager.option('-s', '--sources', dest='source_strings', action='append', help='Sources to operate on.')
def clean(source_strings):
source_objs = validate_sources(source_strings)
for source in source_objs:
@manager.option('-c', '--commit', dest='commit', action='store_true', default=False, help='Persist changes.')
def clean(source_strings, commit):
sources = validate_sources(source_strings)
for source in sources:
s = plugins.get(source.plugin_name)
if not hasattr(s, 'clean'):
print("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
source.label
))
continue
start_time = time.time()
print("[+] Staring to clean source: {label}!\n".format(label=source.label))
source_service.clean(source)
cleaned = 0
for certificate in certificate_service.get_all_pending_cleaning(source):
if commit:
try:
s.clean(certificate, source.options)
certificate.sources.remove(source)
certificate_service.database.update(certificate)
metrics.send('clean_success', 'counter', 1, metric_tags={'source': source.label})
except Exception as e:
current_app.logger.exception(e)
metrics.send('clean_failed', 'counter', 1, metric_tags={'source': source.label})
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
certificate.name,
source.label
))
cleaned += 1
print(
"[+] Finished cleaning source: {label}. Run Time: {time}\n".format(
"[+] Finished cleaning source: {label}. Removed {cleaned} certificates from source. Run Time: {time}\n".format(
label=source.label,
time=(time.time() - start_time)
time=(time.time() - start_time),
cleaned=cleaned
)
)

64
lemur/sources/service.py
View File

@ -12,7 +12,7 @@ from flask import current_app
from lemur import database
from lemur.sources.models import Source
from lemur.certificates.models import Certificate
from lemur.certificates import service as cert_service
from lemur.certificates import service as certificate_service
from lemur.endpoints import service as endpoint_service
from lemur.destinations import service as destination_service
@ -21,29 +21,6 @@ from lemur.certificates.schemas import CertificateUploadInputSchema
from lemur.plugins.base import plugins
# TODO optimize via sql query
def _disassociate_certs_from_source(certificates, source):
current_certificates = cert_service.get_by_source(source_label=source.label)
missing = []
for cc in current_certificates:
for fc in certificates:
if fc['body'] == cc.body:
break
else:
missing.append(cc)
for c in missing:
for s in c.sources:
if s.label == source:
current_app.logger.info(
"Certificate {name} is no longer associated with {source}.".format(
name=c.name,
source=source.label
)
)
c.sources.delete(s)
def certificate_create(certificate, source):
data, errors = CertificateUploadInputSchema().load(certificate)
@ -52,7 +29,7 @@ def certificate_create(certificate, source):
data['creator'] = certificate['creator']
cert = cert_service.import_certificate(**data)
cert = certificate_service.import_certificate(**data)
cert.description = "This certificate was automatically discovered by Lemur"
cert.sources.append(source)
sync_update_destination(cert, source)
@ -99,12 +76,12 @@ def sync_endpoints(source):
certificate = endpoint.pop('certificate', None)
if certificate_name:
cert = cert_service.get_by_name(certificate_name)
cert = certificate_service.get_by_name(certificate_name)
elif certificate:
cert = cert_service.find_duplicates(certificate)
cert = certificate_service.find_duplicates(certificate)
if not cert:
cert = cert_service.import_certificate(**certificate)
cert = certificate_service.import_certificate(**certificate)
if not cert:
current_app.logger.error(
@ -142,7 +119,7 @@ def sync_certificates(source, user):
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = cert_service.find_duplicates(certificate)
exists = certificate_service.find_duplicates(certificate)
certificate['owner'] = user.email
certificate['creator'] = user
@ -162,9 +139,6 @@ def sync_certificates(source, user):
)
)
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
_disassociate_certs_from_source(certificates, source)
return new, updated
@ -178,33 +152,13 @@ def sync(source, user):
return {'endpoints': (new_endpoints, updated_endpoints), 'certificates': (new_certs, updated_certs)}
def clean(source):
s = plugins.get(source.plugin_name)
try:
certificates = s.clean(source.options)
except NotImplementedError:
current_app.logger.warning("Cannot clean source: {0}, source plugin does not implement 'clean()'".format(
source.label
))
return
for certificate in certificates:
cert = cert_service.get_by_name(certificate)
if cert:
current_app.logger.warning("Removed {0} from source {1} during cleaning".format(
cert.name,
source.label
))
cert.sources.remove(source)
def create(label, plugin_name, options, description=None):
"""
Creates a new source, that can then be used as a source for certificates.
:param label: Source common name
:param plugin_name:
:param options:
:param description:
:rtype : Source
:return: New source
@ -219,6 +173,8 @@ def update(source_id, label, options, description):
:param source_id: Lemur assigned ID
:param label: Source common name
:param options:
:param description:
:rtype : Source
:return:
"""