Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into ilabun/optimize-certificates-sql

Browse Source
This commit is contained in:
Hossein Shafagh
2020-06-09 14:20:31 -07:00
committed by GitHub
parent 50091cca1d 1b05971268
commit c40d297735
14 changed files with 10554 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
lemur/certificates/cli.py
View File

@ -23,7 +23,7 @@ from lemur.certificates.service import (
get_certificate_primitives,
get_all_pending_reissue,
get_by_name,
get_all_certs,
get_all_valid_certs,
get,
get_all_certs_attached_to_endpoint_without_autorotate,
)
@ -210,6 +210,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
status = FAILURE_METRIC_STATUS
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
}
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
@ -219,26 +223,43 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
log_data["message"] = "Rotating endpoint"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
elif old_cert and new_cert:
print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
log_data["message"] = "Rotating all endpoints"
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
for endpoint in old_cert.endpoints:
print(f"[+] Rotating {endpoint.name}")
log_data["endpoint"] = endpoint.dnsname
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
else:
print("[+] Rotating all endpoints that have new certificates available")
log_data["message"] = "Rotating all endpoints that have new certificates available"
for endpoint in endpoint_service.get_all_pending_rotation():
log_data["endpoint"] = endpoint.dnsname
if len(endpoint.certificate.replaced) == 1:
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
log_data["certificate"] = endpoint.certificate.replaced[0].name
request_rotation(
endpoint, endpoint.certificate.replaced[0], message, commit
)
current_app.logger.info(log_data)
else:
log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
metrics.send(
"endpoint_rotation",
"counter",
@ -636,7 +657,14 @@ def check_revoked():
encounters an issue with verification it marks the certificate status
as `unknown`.
"""
for cert in get_all_certs():
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Checking for revoked Certificates"
}
certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
for cert in certs:
try:
if cert.chain:
status = verify_string(cert.body, cert.chain)
@ -645,6 +673,20 @@ def check_revoked():
cert.status = "valid" if status else "revoked"
if cert.status == "revoked":
log_data["valid"] = cert.status
log_data["certificate_name"] = cert.name
log_data["certificate_id"] = cert.id
metrics.send(
"certificate_revoked",
"counter",
1,
metric_tags={"status": log_data["valid"],
"certificate_name": log_data["certificate_name"],
"certificate_id": log_data["certificate_id"]},
)
current_app.logger.info(log_data)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)

21
lemur/certificates/service.py
View File

@ -103,6 +103,27 @@ def get_all_certs():
return Certificate.query.all()
def get_all_valid_certs(authority_plugin_name):
"""
Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
ignored if no authority_plugin_name provided.
Note that depending on the DB size retrieving all certificates might an expensive operation
:return:
"""
if authority_plugin_name:
return (
Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
Authority.plugin_name.in_(authority_plugin_name)).all()
)
else:
return (
Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
)
def get_all_pending_cleaning_expired(source):
"""
Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not

13
lemur/certificates/verify.py
View File

@ -8,6 +8,7 @@
import requests
import subprocess
from flask import current_app
from lemur.extensions import sentry
from requests.exceptions import ConnectionError, InvalidSchema
from cryptography import x509
from cryptography.hazmat.backends import default_backend
@ -152,10 +153,18 @@ def verify(cert_path, issuer_chain_path):
# OCSP is our main source of truth, in a lot of cases CRLs
# have been deprecated and are no longer updated
verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
try:
verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
if verify_result is None:
verify_result = crl_verify(cert, cert_path)
try:
verify_result = crl_verify(cert, cert_path)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
if verify_result is None:
current_app.logger.debug("Failed to verify {}".format(cert.serial_number))

6
lemur/dns_providers/util.py
View File

@ -31,11 +31,11 @@ class DNSResolveError(DNSError):
def is_valid_domain(domain):
"""Checks if a domain is syntactically valid and returns a bool"""
if len(domain) > 253:
return False
if domain[-1] == ".":
domain = domain[:-1]
fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(?<!-)\.?)+(?:[a-zA-Z]{2,})$)", re.IGNORECASE)
if len(domain) > 253:
return False
fqdn_re = re.compile("(?=^.{1,63}$)(^(?:[a-z0-9_](?:-*[a-z0-9_])+)$|^[a-z0-9]$)", re.IGNORECASE)
return all(fqdn_re.match(d) for d in domain.split("."))

3
lemur/plugins/lemur_email/templates/expiration.html
View File

@ -75,7 +75,8 @@
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used.
<br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used,
and disable notifications via the Notify toggle in Lemur, if applicable.
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>

23
lemur/tests/test_dns_providers.py
View File

@ -4,9 +4,20 @@ from lemur.dns_providers import util as dnsutil
class TestDNSProvider(unittest.TestCase):
def test_is_valid_domain(self):
self.assertTrue(dnsutil.is_valid_domain("example.com"))
self.assertTrue(dnsutil.is_valid_domain("foo.bar.org"))
self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com"))
self.assertFalse(dnsutil.is_valid_domain("e/xample.com"))
self.assertFalse(dnsutil.is_valid_domain("exam\ple.com"))
self.assertFalse(dnsutil.is_valid_domain("*.example.com"))
self.assertTrue(dnsutil.is_valid_domain('example.com'))
self.assertTrue(dnsutil.is_valid_domain('foo.bar.org'))
self.assertTrue(dnsutil.is_valid_domain('exam--ple.io'))
self.assertTrue(dnsutil.is_valid_domain('a.example.com'))
self.assertTrue(dnsutil.is_valid_domain('example.io'))
self.assertTrue(dnsutil.is_valid_domain('example-of-under-63-character-domain-label-length-limit-1234567.com'))
self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com'))
self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com'))
self.assertFalse(dnsutil.is_valid_domain('e/xample.com'))
self.assertFalse(dnsutil.is_valid_domain('exam\ple.com'))
self.assertFalse(dnsutil.is_valid_domain('<example.com'))
self.assertFalse(dnsutil.is_valid_domain('*.example.com'))
self.assertFalse(dnsutil.is_valid_domain('-example.io'))
self.assertFalse(dnsutil.is_valid_domain('example-.io'))
self.assertFalse(dnsutil.is_valid_domain('example..io'))
self.assertFalse(dnsutil.is_valid_domain('exa mple.io'))
self.assertFalse(dnsutil.is_valid_domain('-'))