Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #2869 from hosseinsh/cert-sync-endpoint-find-by-hash

Browse Source 
Cert sync endpoint find by hash
This commit is contained in:
Hossein Shafagh 2019-10-18 11:29:57 -07:00 committed by GitHub
commit da1080e041
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 108 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lemur/plugins/lemur_aws/iam.py
View File

@ -10,7 +10,7 @@ import botocore
from retrying import retry
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.plugins.lemur_aws.sts import sts_client
@ -122,9 +122,11 @@ def get_certificate(name, **kwargs):
"""
client = kwargs.pop("client")
metrics.send("get_certificate", "counter", 1, metric_tags={"name": name})
return client.get_server_certificate(ServerCertificateName=name)[
"ServerCertificate"
]
try:
return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"]
except client.exceptions.NoSuchEntityException:
sentry.captureException()
return None
@sts_client("iam")

25
lemur/plugins/lemur_aws/plugin.py
View File

@ -32,7 +32,9 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Harm Weites <harm@weites.com>
"""
from acme.errors import ClientError
from flask import current_app
from lemur.extensions import sentry, metrics
from lemur.plugins import lemur_aws as aws
from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
@ -271,6 +273,29 @@ class AWSSourcePlugin(SourcePlugin):
account_number = self.get_option("accountNumber", options)
iam.delete_cert(certificate.name, account_number=account_number)
def get_certificate_by_name(self, certificate_name, options):
account_number = self.get_option("accountNumber", options)
# certificate name may contain path, in which case we remove it
if "/" in certificate_name:
certificate_name = certificate_name.split('/')[-1]
try:
cert = iam.get_certificate(certificate_name, account_number=account_number)
if cert:
return dict(
body=cert["CertificateBody"],
chain=cert.get("CertificateChain"),
name=cert["ServerCertificateMetadata"]["ServerCertificateName"],
)
except ClientError:
current_app.logger.warning(
"get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name))
sentry.captureException()
metrics.send(
"get_elb_certificate_failed", "counter", 1,
metric_tags={"certificate_name": certificate_name, "account_number": account_number}
)
return None
class AWSDestinationPlugin(DestinationPlugin):
title = "AWS"

104
lemur/sources/service.py
View File

@ -15,7 +15,7 @@ from lemur.sources.models import Source
from lemur.certificates.models import Certificate
from lemur.certificates import service as certificate_service
from lemur.endpoints import service as endpoint_service
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.destinations import service as destination_service
from lemur.certificates.schemas import CertificateUploadInputSchema
@ -66,7 +66,7 @@ def sync_update_destination(certificate, source):
def sync_endpoints(source):
new, updated = 0, 0
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
s = plugins.get(source.plugin_name)
@ -89,6 +89,39 @@ def sync_endpoints(source):
endpoint["certificate"] = certificate_service.get_by_name(certificate_name)
# if get cert by name failed, we attempt a search via serial number and hash comparison
# and link the endpoint certificate to Lemur certificate
if not endpoint["certificate"]:
certificate_attached_to_endpoint = None
try:
certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options)
except NotImplementedError:
current_app.logger.warning(
"Unable to describe server certificate for endpoints in source {0}:"
" plugin has not implemented 'get_certificate_by_name'".format(
source.label
)
)
sentry.captureException()
if certificate_attached_to_endpoint:
lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint)
updated_by_hash += updated_by_hash_tmp
if lemur_matching_cert:
endpoint["certificate"] = lemur_matching_cert[0]
if len(lemur_matching_cert) > 1:
current_app.logger.error(
"Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format(
len(lemur_matching_cert), certificate_name, endpoint["name"]
)
)
metrics.send("endpoint.certificate.conflict",
"gauge", len(lemur_matching_cert),
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
"acct": s.get_option("accountNumber", source.options)})
if not endpoint["certificate"]:
current_app.logger.error(
"Certificate Not Found. Name: {0} Endpoint: {1}".format(
@ -97,7 +130,8 @@ def sync_endpoints(source):
)
metrics.send("endpoint.certificate.not.found",
"counter", 1,
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)})
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
"acct": s.get_option("accountNumber", source.options)})
continue
policy = endpoint.pop("policy")
@ -122,42 +156,50 @@ def sync_endpoints(source):
endpoint_service.update(exists.id, **endpoint)
updated += 1
return new, updated
return new, updated, updated_by_hash
def find_cert(certificate):
updated_by_hash = 0
exists = False
if certificate.get("search", None):
conditions = certificate.pop("search")
exists = certificate_service.get_by_attributes(conditions)
if not exists and certificate.get("name"):
result = certificate_service.get_by_name(certificate["name"])
if result:
exists = [result]
if not exists and certificate.get("serial"):
exists = certificate_service.get_by_serial(certificate["serial"])
if not exists:
cert = parse_certificate(certificate["body"])
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
updated_by_hash += 1
exists = [x for x in exists if x]
return exists, updated_by_hash
# TODO this is very slow as we don't batch update certificates
def sync_certificates(source, user):
new, updated = 0, 0
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists = False
if certificate.get("search", None):
conditions = certificate.pop("search")
exists = certificate_service.get_by_attributes(conditions)
if not exists and certificate.get("name"):
result = certificate_service.get_by_name(certificate["name"])
if result:
exists = [result]
if not exists and certificate.get("serial"):
exists = certificate_service.get_by_serial(certificate["serial"])
if not exists:
cert = parse_certificate(certificate["body"])
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
exists, updated_by_hash = find_cert(certificate)
if not certificate.get("owner"):
certificate["owner"] = user.email
certificate["creator"] = user
exists = [x for x in exists if x]
if not exists:
certificate_create(certificate, source)
@ -172,12 +214,20 @@ def sync_certificates(source, user):
certificate_update(e, source)
updated += 1
return new, updated
return new, updated, updated_by_hash
def sync(source, user):
new_certs, updated_certs = sync_certificates(source, user)
new_endpoints, updated_endpoints = sync_endpoints(source)
new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user)
new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source)
metrics.send("sync.updated_certs_by_hash",
"gauge", updated_certs_by_hash,
metric_tags={"source": source.label})
metrics.send("sync.updated_endpoints_by_hash",
"gauge", updated_endpoints_by_hash,
metric_tags={"source": source.label})
source.last_run = arrow.utcnow()
database.update(source)