Merge branch 'master' into ilabun/optimize-certificates-sql

This commit is contained in:
Hossein Shafagh 2020-06-09 14:20:31 -07:00 committed by GitHub
commit c40d297735
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 10554 additions and 42 deletions

1
.gitignore vendored
View File

@ -14,7 +14,6 @@
MANIFEST
test.conf
pip-log.txt
package-lock.json
/htmlcov
/cover
/build

View File

@ -3,6 +3,8 @@ FROM alpine:3.8
ARG VERSION
ENV VERSION master
ARG URLCONTEXT
ENV uid 1337
ENV gid 1337
ENV user lemur
@ -22,6 +24,7 @@ RUN addgroup -S ${group} -g ${gid} && \
gcc \
autoconf \
automake \
libtool \
make \
nasm \
zlib-dev \
@ -42,7 +45,7 @@ WORKDIR /opt/lemur
RUN npm install --unsafe-perm && \
pip3 install -e . && \
node_modules/.bin/gulp build && \
node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
apk del build-dependencies
COPY entrypoint /

View File

@ -3,6 +3,8 @@ FROM alpine:3.8
ARG VERSION
ENV VERSION master
ARG URLCONTEXT
ENV uid 1337
ENV gid 1337
ENV user lemur
@ -22,6 +24,7 @@ RUN addgroup -S ${group} -g ${gid} && \
gcc \
autoconf \
automake \
libtool \
make \
nasm \
zlib-dev \
@ -42,7 +45,7 @@ RUN chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/ && \
npm install --unsafe-perm && \
pip3 install -e . && \
node_modules/.bin/gulp build && \
node_modules/.bin/gulp package --urlContextPath=$(urlContextPath) && \
node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \
apk del build-dependencies
COPY docker/entrypoint /

View File

@ -23,7 +23,7 @@ from lemur.certificates.service import (
get_certificate_primitives,
get_all_pending_reissue,
get_by_name,
get_all_certs,
get_all_valid_certs,
get,
get_all_certs_attached_to_endpoint_without_autorotate,
)
@ -210,6 +210,10 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
status = FAILURE_METRIC_STATUS
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
}
try:
old_cert = validate_certificate(old_certificate_name)
new_cert = validate_certificate(new_certificate_name)
@ -219,26 +223,43 @@ def rotate(endpoint_name, new_certificate_name, old_certificate_name, message, c
print(
f"[+] Rotating endpoint: {endpoint.name} to certificate {new_cert.name}"
)
log_data["message"] = "Rotating endpoint"
log_data["endpoint"] = endpoint.dnsname
log_data["certificate"] = new_cert.name
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
elif old_cert and new_cert:
print(f"[+] Rotating all endpoints from {old_cert.name} to {new_cert.name}")
log_data["message"] = "Rotating all endpoints"
log_data["certificate"] = new_cert.name
log_data["certificate_old"] = old_cert.name
log_data["message"] = "Rotating endpoint from old to new cert"
for endpoint in old_cert.endpoints:
print(f"[+] Rotating {endpoint.name}")
log_data["endpoint"] = endpoint.dnsname
request_rotation(endpoint, new_cert, message, commit)
current_app.logger.info(log_data)
else:
print("[+] Rotating all endpoints that have new certificates available")
log_data["message"] = "Rotating all endpoints that have new certificates available"
for endpoint in endpoint_service.get_all_pending_rotation():
log_data["endpoint"] = endpoint.dnsname
if len(endpoint.certificate.replaced) == 1:
print(
f"[+] Rotating {endpoint.name} to {endpoint.certificate.replaced[0].name}"
)
log_data["certificate"] = endpoint.certificate.replaced[0].name
request_rotation(
endpoint, endpoint.certificate.replaced[0], message, commit
)
current_app.logger.info(log_data)
else:
log_data["message"] = "Failed to rotate endpoint due to Multiple replacement certificates found"
print(log_data)
metrics.send(
"endpoint_rotation",
"counter",
@ -636,7 +657,14 @@ def check_revoked():
encounters an issue with verification it marks the certificate status
as `unknown`.
"""
for cert in get_all_certs():
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Checking for revoked Certificates"
}
certs = get_all_valid_certs(current_app.config.get("SUPPORTED_REVOCATION_AUTHORITY_PLUGINS", []))
for cert in certs:
try:
if cert.chain:
status = verify_string(cert.body, cert.chain)
@ -645,6 +673,20 @@ def check_revoked():
cert.status = "valid" if status else "revoked"
if cert.status == "revoked":
log_data["valid"] = cert.status
log_data["certificate_name"] = cert.name
log_data["certificate_id"] = cert.id
metrics.send(
"certificate_revoked",
"counter",
1,
metric_tags={"status": log_data["valid"],
"certificate_name": log_data["certificate_name"],
"certificate_id": log_data["certificate_id"]},
)
current_app.logger.info(log_data)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)

View File

@ -103,6 +103,27 @@ def get_all_certs():
return Certificate.query.all()
def get_all_valid_certs(authority_plugin_name):
"""
Retrieves all valid (not expired) certificates within Lemur, for the given authority plugin names
ignored if no authority_plugin_name provided.
Note that depending on the DB size retrieving all certificates might an expensive operation
:return:
"""
if authority_plugin_name:
return (
Certificate.query.outerjoin(Authority, Authority.id == Certificate.authority_id).filter(
Certificate.not_after > arrow.now().format("YYYY-MM-DD")).filter(
Authority.plugin_name.in_(authority_plugin_name)).all()
)
else:
return (
Certificate.query.filter(Certificate.not_after > arrow.now().format("YYYY-MM-DD")).all()
)
def get_all_pending_cleaning_expired(source):
"""
Retrieves all certificates that are available for cleaning. These are certificates which are expired and are not

View File

@ -8,6 +8,7 @@
import requests
import subprocess
from flask import current_app
from lemur.extensions import sentry
from requests.exceptions import ConnectionError, InvalidSchema
from cryptography import x509
from cryptography.hazmat.backends import default_backend
@ -152,10 +153,18 @@ def verify(cert_path, issuer_chain_path):
# OCSP is our main source of truth, in a lot of cases CRLs
# have been deprecated and are no longer updated
verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
try:
verify_result = ocsp_verify(cert, cert_path, issuer_chain_path)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
if verify_result is None:
verify_result = crl_verify(cert, cert_path)
try:
verify_result = crl_verify(cert, cert_path)
except Exception as e:
sentry.captureException()
current_app.logger.exception(e)
if verify_result is None:
current_app.logger.debug("Failed to verify {}".format(cert.serial_number))

View File

@ -31,11 +31,11 @@ class DNSResolveError(DNSError):
def is_valid_domain(domain):
"""Checks if a domain is syntactically valid and returns a bool"""
if len(domain) > 253:
return False
if domain[-1] == ".":
domain = domain[:-1]
fqdn_re = re.compile("(?=^.{1,254}$)(^(?:(?!\d+\.|-)[a-zA-Z0-9_\-]{1,63}(?<!-)\.?)+(?:[a-zA-Z]{2,})$)", re.IGNORECASE)
if len(domain) > 253:
return False
fqdn_re = re.compile("(?=^.{1,63}$)(^(?:[a-z0-9_](?:-*[a-z0-9_])+)$|^[a-z0-9]$)", re.IGNORECASE)
return all(fqdn_re.match(d) for d in domain.split("."))

View File

@ -75,7 +75,8 @@
</tr>
<tr>
<td style="font-family:Roboto-Regular,Helvetica,Arial,sans-serif;font-size:13px;color:#202020;line-height:1.5">
<br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used.
<br>This is a Lemur certificate expiration notice. Please verify that the following certificates are no longer used,
and disable notifications via the Notify toggle in Lemur, if applicable.
<table border="0" cellspacing="0" cellpadding="0"
style="margin-top:48px;margin-bottom:48px">
<tbody>

View File

@ -4,9 +4,20 @@ from lemur.dns_providers import util as dnsutil
class TestDNSProvider(unittest.TestCase):
def test_is_valid_domain(self):
self.assertTrue(dnsutil.is_valid_domain("example.com"))
self.assertTrue(dnsutil.is_valid_domain("foo.bar.org"))
self.assertTrue(dnsutil.is_valid_domain("_acme-chall.example.com"))
self.assertFalse(dnsutil.is_valid_domain("e/xample.com"))
self.assertFalse(dnsutil.is_valid_domain("exam\ple.com"))
self.assertFalse(dnsutil.is_valid_domain("*.example.com"))
self.assertTrue(dnsutil.is_valid_domain('example.com'))
self.assertTrue(dnsutil.is_valid_domain('foo.bar.org'))
self.assertTrue(dnsutil.is_valid_domain('exam--ple.io'))
self.assertTrue(dnsutil.is_valid_domain('a.example.com'))
self.assertTrue(dnsutil.is_valid_domain('example.io'))
self.assertTrue(dnsutil.is_valid_domain('example-of-under-63-character-domain-label-length-limit-1234567.com'))
self.assertFalse(dnsutil.is_valid_domain('example-of-over-63-character-domain-label-length-limit-123456789.com'))
self.assertTrue(dnsutil.is_valid_domain('_acme-chall.example.com'))
self.assertFalse(dnsutil.is_valid_domain('e/xample.com'))
self.assertFalse(dnsutil.is_valid_domain('exam\ple.com'))
self.assertFalse(dnsutil.is_valid_domain('<example.com'))
self.assertFalse(dnsutil.is_valid_domain('*.example.com'))
self.assertFalse(dnsutil.is_valid_domain('-example.io'))
self.assertFalse(dnsutil.is_valid_domain('example-.io'))
self.assertFalse(dnsutil.is_valid_domain('example..io'))
self.assertFalse(dnsutil.is_valid_domain('exa mple.io'))
self.assertFalse(dnsutil.is_valid_domain('-'))

10423
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -6,7 +6,7 @@
#
appdirs==1.4.3 # via virtualenv
bleach==3.1.4 # via readme-renderer
certifi==2020.4.5.1 # via requests
certifi==2020.4.5.2 # via requests
cffi==1.14.0 # via cryptography
cfgv==3.1.0 # via pre-commit
chardet==3.0.4 # via requests
@ -21,7 +21,7 @@ invoke==1.4.1 # via -r requirements-dev.in
jeepney==0.4.3 # via keyring, secretstorage
keyring==21.2.0 # via twine
mccabe==0.6.1 # via flake8
nodeenv==1.3.5 # via -r requirements-dev.in, pre-commit
nodeenv==1.4.0 # via -r requirements-dev.in, pre-commit
pkginfo==1.5.0.1 # via twine
pre-commit==2.4.0 # via -r requirements-dev.in
pycodestyle==2.3.1 # via flake8

View File

@ -4,7 +4,7 @@
#
# pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
#
acme==1.4.0 # via -r requirements.txt
acme==1.5.0 # via -r requirements.txt
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2 # via -r requirements.txt
alembic==1.4.2 # via -r requirements.txt, flask-migrate
@ -16,10 +16,10 @@ babel==2.8.0 # via sphinx
bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko
billiard==3.6.3.0 # via -r requirements.txt, celery
blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven
boto3==1.13.18 # via -r requirements.txt
botocore==1.16.18 # via -r requirements.txt, boto3, s3transfer
boto3==1.13.19 # via -r requirements.txt
botocore==1.16.25 # via -r requirements.txt, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.txt
certifi==2020.4.5.1 # via -r requirements.txt, requests
certifi==2020.4.5.2 # via -r requirements.txt, requests
certsrv==2.1.1 # via -r requirements.txt
cffi==1.14.0 # via -r requirements.txt, bcrypt, cryptography, pynacl
chardet==3.0.4 # via -r requirements.txt, requests
@ -38,14 +38,14 @@ flask-principal==0.4.0 # via -r requirements.txt
flask-replicated==1.3 # via -r requirements.txt
flask-restful==0.3.8 # via -r requirements.txt
flask-script==2.0.6 # via -r requirements.txt
flask-sqlalchemy==2.4.1 # via -r requirements.txt, flask-migrate
flask-sqlalchemy==2.4.3 # via -r requirements.txt, flask-migrate
flask==1.1.2 # via -r requirements.txt, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
future==0.18.2 # via -r requirements.txt, cloudflare
gunicorn==20.0.4 # via -r requirements.txt
hvac==0.10.3 # via -r requirements.txt
idna==2.9 # via -r requirements.txt, requests
imagesize==1.2.0 # via sphinx
inflection==0.4.0 # via -r requirements.txt
inflection==0.5.0 # via -r requirements.txt
itsdangerous==1.1.0 # via -r requirements.txt, flask
javaobj-py3==0.4.0.1 # via -r requirements.txt, pyjks
jinja2==2.11.2 # via -r requirements.txt, flask, sphinx
@ -57,7 +57,7 @@ lockfile==0.12.2 # via -r requirements.txt
logmatic-python==0.1.7 # via -r requirements.txt
mako==1.1.2 # via -r requirements.txt, alembic
markupsafe==1.1.1 # via -r requirements.txt, jinja2, mako
marshmallow-sqlalchemy==0.23.0 # via -r requirements.txt
marshmallow-sqlalchemy==0.23.1 # via -r requirements.txt
marshmallow==2.20.4 # via -r requirements.txt, marshmallow-sqlalchemy
ndg-httpsclient==0.5.1 # via -r requirements.txt
packaging==20.3 # via sphinx
@ -82,7 +82,7 @@ python-ldap==3.2.0 # via -r requirements.txt
pytz==2019.3 # via -r requirements.txt, acme, babel, celery, flask-restful, pyrfc3339
pyyaml==5.3.1 # via -r requirements.txt, cloudflare
raven[flask]==6.10.0 # via -r requirements.txt
redis==3.5.2 # via -r requirements.txt, celery
redis==3.5.3 # via -r requirements.txt, celery
requests-toolbelt==0.9.1 # via -r requirements.txt, acme
requests[security]==2.23.0 # via -r requirements.txt, acme, certsrv, cloudflare, hvac, requests-toolbelt, sphinx
retrying==1.3.3 # via -r requirements.txt
@ -90,7 +90,7 @@ s3transfer==0.3.3 # via -r requirements.txt, boto3
six==1.15.0 # via -r requirements.txt, acme, bcrypt, cryptography, flask-cors, flask-restful, hvac, josepy, jsonlines, packaging, pynacl, pyopenssl, python-dateutil, retrying, sphinxcontrib-httpdomain, sqlalchemy-utils
snowballstemmer==2.0.0 # via sphinx
sphinx-rtd-theme==0.4.3 # via -r requirements-docs.in
sphinx==3.0.3 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
sphinx==3.1.0 # via -r requirements-docs.in, sphinx-rtd-theme, sphinxcontrib-httpdomain
sphinxcontrib-applehelp==1.0.2 # via sphinx
sphinxcontrib-devhelp==1.0.2 # via sphinx
sphinxcontrib-htmlhelp==1.0.3 # via sphinx

View File

@ -10,10 +10,10 @@ aws-sam-translator==1.22.0 # via cfn-lint
aws-xray-sdk==2.5.0 # via moto
bandit==1.6.2 # via -r requirements-tests.in
black==19.10b0 # via -r requirements-tests.in
boto3==1.13.18 # via aws-sam-translator, moto
boto3==1.13.19 # via aws-sam-translator, moto
boto==2.49.0 # via moto
botocore==1.16.18 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2020.4.5.1 # via requests
botocore==1.16.25 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2020.4.5.2 # via requests
cffi==1.14.0 # via cryptography
cfn-lint==0.29.5 # via moto
chardet==3.0.4 # via requests
@ -59,13 +59,13 @@ pyflakes==2.2.0 # via -r requirements-tests.in
pyparsing==2.4.7 # via packaging
pyrsistent==0.16.0 # via jsonschema
pytest-flask==1.0.0 # via -r requirements-tests.in
pytest-mock==3.1.0 # via -r requirements-tests.in
pytest==5.4.2 # via -r requirements-tests.in, pytest-flask, pytest-mock
pytest-mock==3.1.1 # via -r requirements-tests.in
pytest==5.4.3 # via -r requirements-tests.in, pytest-flask, pytest-mock
python-dateutil==2.8.1 # via botocore, faker, freezegun, moto
python-jose==3.1.0 # via moto
pytz==2019.3 # via moto
pyyaml==5.3.1 # via -r requirements-tests.in, bandit, cfn-lint, moto
redis==3.5.2 # via fakeredis
redis==3.5.3 # via fakeredis
regex==2020.4.4 # via black
requests-mock==1.8.0 # via -r requirements-tests.in
requests==2.23.0 # via docker, moto, requests-mock, responses

View File

@ -4,7 +4,7 @@
#
# pip-compile --no-index --output-file=requirements.txt requirements.in
#
acme==1.4.0 # via -r requirements.in
acme==1.5.0 # via -r requirements.in
alembic-autogenerate-enums==0.0.2 # via -r requirements.in
alembic==1.4.2 # via flask-migrate
amqp==2.5.2 # via kombu
@ -14,10 +14,10 @@ asyncpool==1.0 # via -r requirements.in
bcrypt==3.1.7 # via flask-bcrypt, paramiko
billiard==3.6.3.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.13.18 # via -r requirements.in
botocore==1.16.18 # via -r requirements.in, boto3, s3transfer
boto3==1.13.19 # via -r requirements.in
botocore==1.16.25 # via -r requirements.in, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.in
certifi==2020.4.5.1 # via -r requirements.in, requests
certifi==2020.4.5.2 # via -r requirements.in, requests
certsrv==2.1.1 # via -r requirements.in
cffi==1.14.0 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
@ -36,13 +36,13 @@ flask-principal==0.4.0 # via -r requirements.in
flask-replicated==1.3 # via -r requirements.in
flask-restful==0.3.8 # via -r requirements.in
flask-script==2.0.6 # via -r requirements.in
flask-sqlalchemy==2.4.1 # via -r requirements.in, flask-migrate
flask-sqlalchemy==2.4.3 # via -r requirements.in, flask-migrate
flask==1.1.2 # via -r requirements.in, flask-bcrypt, flask-cors, flask-mail, flask-migrate, flask-principal, flask-restful, flask-script, flask-sqlalchemy, raven
future==0.18.2 # via -r requirements.in, cloudflare
gunicorn==20.0.4 # via -r requirements.in
hvac==0.10.3 # via -r requirements.in
idna==2.9 # via requests
inflection==0.4.0 # via -r requirements.in
inflection==0.5.0 # via -r requirements.in
itsdangerous==1.1.0 # via flask
javaobj-py3==0.4.0.1 # via pyjks
jinja2==2.11.2 # via -r requirements.in, flask
@ -54,7 +54,7 @@ lockfile==0.12.2 # via -r requirements.in
logmatic-python==0.1.7 # via -r requirements.in
mako==1.1.2 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.23.0 # via -r requirements.in
marshmallow-sqlalchemy==0.23.1 # via -r requirements.in
marshmallow==2.20.4 # via -r requirements.in, marshmallow-sqlalchemy
ndg-httpsclient==0.5.1 # via -r requirements.in
paramiko==2.7.1 # via -r requirements.in
@ -76,7 +76,7 @@ python-ldap==3.2.0 # via -r requirements.in
pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339
pyyaml==5.3.1 # via -r requirements.in, cloudflare
raven[flask]==6.10.0 # via -r requirements.in
redis==3.5.2 # via -r requirements.in, celery
redis==3.5.3 # via -r requirements.in, celery
requests-toolbelt==0.9.1 # via acme
requests[security]==2.23.0 # via -r requirements.in, acme, certsrv, cloudflare, hvac, requests-toolbelt
retrying==1.3.3 # via -r requirements.in