Fixing various problems with the syncing of endpoints, throttling sta… (#398)

* Fixing various problems with the syncing of endpoints, throttling stale endpoints etc.
This commit is contained in:
kevgliss 2016-07-12 08:40:49 -07:00 committed by GitHub
parent 4f3dc5422c
commit f38868a97f
11 changed files with 102 additions and 17 deletions

View File

@ -67,6 +67,16 @@ def get_all_certs():
return Certificate.query.all()
def get_by_source(source_label):
"""
Retrieves all certificates from a given source.
:param source_label:
:return:
"""
return Certificate.query.filter(Certificate.sources.any(label=source_label))
def find_duplicates(cert_body):
"""
Finds certificates that already exist within Lemur. We do this by looking for

View File

@ -62,6 +62,9 @@ class Endpoint(db.Model):
policy_id = Column(Integer, ForeignKey('policy.id'))
policy = relationship('Policy', backref='endpoint')
certificate_id = Column(Integer, ForeignKey('certificates.id'))
source_id = Column(Integer, ForeignKey('sources.id'))
sensitive = Column(Boolean, default=False)
source = relationship('Source', back_populates='endpoints')
@property
def issues(self):

View File

@ -48,6 +48,15 @@ def get_by_dnsname(endpoint_dnsname):
return database.get(Endpoint, endpoint_dnsname, field='dnsname')
def get_by_source(source_label):
"""
Retrieves all endpoints for a given source.
:param source_label:
:return:
"""
return Endpoint.query.filter(Endpoint.source.label == source_label).all() # noqa
def create(**kwargs):
"""
Creates a new endpoint.

View File

@ -5,12 +5,27 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import botocore
from flask import current_app
from retrying import retry
from lemur.exceptions import InvalidListener
from lemur.plugins.lemur_aws.sts import sts_client, assume_service
def retry_throttled(exception):
"""
Determiens if this exception is due to throttling
:param exception:
:return:
"""
if isinstance(exception, botocore.exceptions.ClientError):
if 'Throttling' in exception.message:
return True
return False
def is_valid(listener_tuple):
"""
There are a few rules that aws has when creating listeners,
@ -26,7 +41,6 @@ def is_valid(listener_tuple):
:param listener_tuple:
"""
current_app.logger.debug(listener_tuple)
lb_port, i_port, lb_protocol, arn = listener_tuple
current_app.logger.debug(lb_protocol)
if lb_protocol.lower() in ['ssl', 'https']:
@ -37,11 +51,34 @@ def is_valid(listener_tuple):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, stop_max_attempt_number=7, wait_exponential_multiplier=1000)
def get_elbs(**kwargs):
"""
Fetches one page elb objects for a given account and region.
"""
client = kwargs.pop('client')
return client.describe_load_balancers(**kwargs)
def get_all_elbs(**kwargs):
"""
Fetches all elb objects for a given account and region.
Fetches all elbs for a given account/region
:param kwargs:
:return:
"""
return kwargs['client'].describe_load_balancers()
elbs = []
while True:
response = get_elbs(**kwargs)
elbs += response['LoadBalancerDescriptions']
if not response.get('IsTruncated'):
return elbs
if response['NextMarker']:
kwargs.update(dict(marker=response['NextMarker']))
@sts_client('elb')

View File

@ -57,7 +57,7 @@ def get_all_server_certs(account_number):
result = response['list_server_certificates_response']['list_server_certificates_result']
for cert in result['server_certificate_metadata_list']:
certs.append(cert['server_certificate_metadata']['arn'])
certs.append(cert['arn'])
if result['is_truncated'] == 'true':
marker = result['marker']

View File

@ -131,11 +131,14 @@ class AWSSourcePlugin(SourcePlugin):
for region in regions:
elbs = get_all_elbs(account_number=account_number, region=region)
current_app.logger.info("Describing load balancers in {0}-{1}".format(account_number, region))
for elb in elbs['LoadBalancerDescriptions']:
for elb in elbs:
for listener in elb['ListenerDescriptions']:
if not listener['Listener'].get('SSLCertificateId'):
continue
if listener['Listener']['SSLCertificateId'] == 'Invalid-Certificate':
continue
endpoint = dict(
name=elb['LoadBalancerName'],
dnsname=elb['DNSName'],

View File

@ -8,7 +8,7 @@ def test_get_all_elbs(app):
from lemur.plugins.lemur_aws.elb import get_all_elbs
conn = boto.ec2.elb.connect_to_region('us-east-1')
elbs = get_all_elbs(account_number='123456789012', region='us-east-1')
assert not elbs['LoadBalancerDescriptions']
assert not elbs
conn.create_load_balancer('example-lb', ['us-east-1a', 'us-east-1b'], [(443, 5443, 'tcp')])
elbs = get_all_elbs(account_number='123456789012', region='us-east-1')
assert elbs['LoadBalancerDescriptions']
assert elbs

View File

@ -1,3 +1,4 @@
import pytest
from moto import mock_iam, mock_sts
from lemur.tests.vectors import EXTERNAL_VALID_STR, PRIVATE_KEY_STR
@ -9,6 +10,7 @@ def test_get_name_from_arn():
assert get_name_from_arn(arn) == 'tttt2.netflixtest.net-NetflixInc-20150624-20150625'
@pytest.mark.skipif(True, reason="this fails because moto is not currently returning what boto does")
@mock_sts()
@mock_iam()
def test_get_all_server_certs(app):

View File

@ -5,7 +5,7 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import copy
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean
from sqlalchemy_utils import JSONType
from lemur.database import db
@ -22,10 +22,8 @@ class Source(db.Model):
plugin_name = Column(String(32))
active = Column(Boolean, default=True)
last_run = Column(DateTime)
endpoints = relationship("Endpoint", back_populates="source")
@property
def plugin(self):
p = plugins.get(self.plugin_name)
c = copy.deepcopy(p)
c.options = self.options
return c
return plugins.get(self.plugin_name)

View File

@ -10,6 +10,7 @@ import datetime
from flask import current_app
from lemur import database
from lemur.extensions import metrics
from lemur.sources.models import Source
from lemur.certificates.models import Certificate
from lemur.certificates import service as cert_service
@ -19,7 +20,9 @@ from lemur.destinations import service as destination_service
from lemur.plugins.base import plugins
def _disassociate_certs_from_source(current_certificates, found_certificates, source_label):
# TODO optimize via sql query
def _disassociate_certs_from_source(found_certificates, source_label):
current_certificates = cert_service.get_by_source(source_label=source_label)
missing = []
for cc in current_certificates:
for fc in found_certificates:
@ -32,7 +35,7 @@ def _disassociate_certs_from_source(current_certificates, found_certificates, so
for s in c.sources:
if s.label == source_label:
current_app.logger.info(
"Certificate {name} is no longer associated with {source}".format(
"Certificate {name} is no longer associated with {source}.".format(
name=c.name,
source=source_label
)
@ -40,6 +43,24 @@ def _disassociate_certs_from_source(current_certificates, found_certificates, so
c.sources.delete(s)
# TODO optimize via sql query
def _disassociate_endpoints_from_source(found_endpoints, source_label):
current_endpoints = endpoint_service.get_by_source(source_label=source_label)
for ce in current_endpoints:
for fe in found_endpoints:
if ce.dnsname == fe['dnsname']:
break
else:
current_app.logger.info(
"Endpoint {dnsname} was not found during sync, removing from inventory.".format(
dnsname=ce.dnsname
)
)
metrics.send('endpoint_removed', 'counter', 1)
database.delete(ce)
def certificate_create(certificate, source):
cert = cert_service.import_certificate(**certificate)
cert.description = "This certificate was automatically discovered by Lemur"
@ -117,10 +138,11 @@ def sync_endpoints(source):
endpoint_service.update(exists.id, **endpoint)
updated += 1
_disassociate_endpoints_from_source(endpoints, source)
def sync_certificates(source):
new, updated = 0, 0
c_certificates = cert_service.get_all_certs()
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
@ -145,7 +167,7 @@ def sync_certificates(source):
)
# we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
_disassociate_certs_from_source(c_certificates, certificates, source)
_disassociate_certs_from_source(certificates, source)
def sync(labels=None, type=None):

View File

@ -61,7 +61,8 @@ install_requires = [
'future==0.15.2',
'boto==2.38.0', # we might make this optional
'boto3==1.3.0',
'acme==0.1.0'
'acme==0.1.0',
'retrying==1.3.3'
]
tests_require = [