Fixing various problems with the syncing of endpoints, throttling sta… (#398)

* Fixing various problems with the syncing of endpoints, throttling stale endpoints etc.

lemur/sources/models.py

@@ -5,7 +5,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import copy
+from sqlalchemy.orm import relationship
 from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean
 from sqlalchemy_utils import JSONType
 from lemur.database import db
@@ -22,10 +22,8 @@ class Source(db.Model):
     plugin_name = Column(String(32))
     active = Column(Boolean, default=True)
     last_run = Column(DateTime)
+    endpoints = relationship("Endpoint", back_populates="source")
 
     @property
     def plugin(self):
-        p = plugins.get(self.plugin_name)
-        c = copy.deepcopy(p)
-        c.options = self.options
-        return c
+        return plugins.get(self.plugin_name)

lemur/sources/service.py

@@ -10,6 +10,7 @@ import datetime
 from flask import current_app
 
 from lemur import database
+from lemur.extensions import metrics
 from lemur.sources.models import Source
 from lemur.certificates.models import Certificate
 from lemur.certificates import service as cert_service
@@ -19,7 +20,9 @@ from lemur.destinations import service as destination_service
 from lemur.plugins.base import plugins
 
 
-def _disassociate_certs_from_source(current_certificates, found_certificates, source_label):
+# TODO optimize via sql query
+def _disassociate_certs_from_source(found_certificates, source_label):
+    current_certificates = cert_service.get_by_source(source_label=source_label)
     missing = []
     for cc in current_certificates:
         for fc in found_certificates:
@@ -32,7 +35,7 @@ def _disassociate_certs_from_source(current_certificates, found_certificates, so
         for s in c.sources:
             if s.label == source_label:
                 current_app.logger.info(
-                    "Certificate {name} is no longer associated with {source}".format(
+                    "Certificate {name} is no longer associated with {source}.".format(
                         name=c.name,
                         source=source_label
                     )
@@ -40,6 +43,24 @@ def _disassociate_certs_from_source(current_certificates, found_certificates, so
                 c.sources.delete(s)
 
 
+# TODO optimize via sql query
+def _disassociate_endpoints_from_source(found_endpoints, source_label):
+    current_endpoints = endpoint_service.get_by_source(source_label=source_label)
+
+    for ce in current_endpoints:
+        for fe in found_endpoints:
+            if ce.dnsname == fe['dnsname']:
+                break
+        else:
+            current_app.logger.info(
+                "Endpoint {dnsname} was not found during sync, removing from inventory.".format(
+                    dnsname=ce.dnsname
+                )
+            )
+            metrics.send('endpoint_removed', 'counter', 1)
+            database.delete(ce)
+
+
 def certificate_create(certificate, source):
     cert = cert_service.import_certificate(**certificate)
     cert.description = "This certificate was automatically discovered by Lemur"
@@ -117,10 +138,11 @@ def sync_endpoints(source):
             endpoint_service.update(exists.id, **endpoint)
             updated += 1
 
+    _disassociate_endpoints_from_source(endpoints, source)
+
 
 def sync_certificates(source):
     new, updated = 0, 0
-    c_certificates = cert_service.get_all_certs()
 
     current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
     s = plugins.get(source.plugin_name)
@@ -145,7 +167,7 @@ def sync_certificates(source):
             )
 
     # we need to try and find the absent of certificates so we can properly disassociate them when they are deleted
-    _disassociate_certs_from_source(c_certificates, certificates, source)
+    _disassociate_certs_from_source(certificates, source)
 
 
 def sync(labels=None, type=None):