Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into master

This commit is contained in:
Hossein Shafagh 2019-10-18 11:32:21 -07:00 committed by GitHub
parent f0652ca6a9 da1080e041
commit f077b19126
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 170 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lemur/plugins/lemur_acme/plugin.py
View File

@ -32,6 +32,7 @@ from lemur.extensions import metrics, sentry
from lemur.plugins import lemur_acme as acme
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns
from retrying import retry
class AuthorizationRecord(object):
@ -197,6 +198,7 @@ class AcmeHandler(object):
)
return pem_certificate, pem_certificate_chain
@retry(stop_max_attempt_number=5, wait_fixed=5000)
def setup_acme_client(self, authority):
if not authority.options:
raise InvalidAuthority("Invalid authority. Options not set")

10
lemur/plugins/lemur_aws/iam.py
View File

@ -10,7 +10,7 @@ import botocore
from retrying import retry
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.plugins.lemur_aws.sts import sts_client
@ -122,9 +122,11 @@ def get_certificate(name, **kwargs):
"""
client = kwargs.pop("client")
metrics.send("get_certificate", "counter", 1, metric_tags={"name": name})
return client.get_server_certificate(ServerCertificateName=name)[
"ServerCertificate"
]
try:
return client.get_server_certificate(ServerCertificateName=name)["ServerCertificate"]
except client.exceptions.NoSuchEntityException:
sentry.captureException()
return None
@sts_client("iam")

25
lemur/plugins/lemur_aws/plugin.py
View File

@ -32,7 +32,9 @@
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Harm Weites <harm@weites.com>
"""
from acme.errors import ClientError
from flask import current_app
from lemur.extensions import sentry, metrics
from lemur.plugins import lemur_aws as aws
from lemur.plugins.bases import DestinationPlugin, ExportDestinationPlugin, SourcePlugin
@ -271,6 +273,29 @@ class AWSSourcePlugin(SourcePlugin):
account_number = self.get_option("accountNumber", options)
iam.delete_cert(certificate.name, account_number=account_number)
def get_certificate_by_name(self, certificate_name, options):
account_number = self.get_option("accountNumber", options)
# certificate name may contain path, in which case we remove it
if "/" in certificate_name:
certificate_name = certificate_name.split('/')[-1]
try:
cert = iam.get_certificate(certificate_name, account_number=account_number)
if cert:
return dict(
body=cert["CertificateBody"],
chain=cert.get("CertificateChain"),
name=cert["ServerCertificateMetadata"]["ServerCertificateName"],
)
except ClientError:
current_app.logger.warning(
"get_elb_certificate_failed: Unable to get certificate for {0}".format(certificate_name))
sentry.captureException()
metrics.send(
"get_elb_certificate_failed", "counter", 1,
metric_tags={"certificate_name": certificate_name, "account_number": account_number}
)
return None
class AWSDestinationPlugin(DestinationPlugin):
title = "AWS"

84
lemur/sources/service.py
View File

@ -15,7 +15,7 @@ from lemur.sources.models import Source
from lemur.certificates.models import Certificate
from lemur.certificates import service as certificate_service
from lemur.endpoints import service as endpoint_service
from lemur.extensions import metrics
from lemur.extensions import metrics, sentry
from lemur.destinations import service as destination_service
from lemur.certificates.schemas import CertificateUploadInputSchema
@ -66,7 +66,7 @@ def sync_update_destination(certificate, source):
def sync_endpoints(source):
new, updated = 0, 0
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving endpoints from {0}".format(source.label))
s = plugins.get(source.plugin_name)
@ -89,6 +89,39 @@ def sync_endpoints(source):
endpoint["certificate"] = certificate_service.get_by_name(certificate_name)
# if get cert by name failed, we attempt a search via serial number and hash comparison
# and link the endpoint certificate to Lemur certificate
if not endpoint["certificate"]:
certificate_attached_to_endpoint = None
try:
certificate_attached_to_endpoint = s.get_certificate_by_name(certificate_name, source.options)
except NotImplementedError:
current_app.logger.warning(
"Unable to describe server certificate for endpoints in source {0}:"
" plugin has not implemented 'get_certificate_by_name'".format(
source.label
)
)
sentry.captureException()
if certificate_attached_to_endpoint:
lemur_matching_cert, updated_by_hash_tmp = find_cert(certificate_attached_to_endpoint)
updated_by_hash += updated_by_hash_tmp
if lemur_matching_cert:
endpoint["certificate"] = lemur_matching_cert[0]
if len(lemur_matching_cert) > 1:
current_app.logger.error(
"Too Many Certificates Found{0}. Name: {1} Endpoint: {2}".format(
len(lemur_matching_cert), certificate_name, endpoint["name"]
)
)
metrics.send("endpoint.certificate.conflict",
"gauge", len(lemur_matching_cert),
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
"acct": s.get_option("accountNumber", source.options)})
if not endpoint["certificate"]:
current_app.logger.error(
"Certificate Not Found. Name: {0} Endpoint: {1}".format(
@ -97,7 +130,8 @@ def sync_endpoints(source):
)
metrics.send("endpoint.certificate.not.found",
"counter", 1,
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"], "acct": s.get_option("accountNumber", source.options)})
metric_tags={"cert": certificate_name, "endpoint": endpoint["name"],
"acct": s.get_option("accountNumber", source.options)})
continue
policy = endpoint.pop("policy")
@ -122,18 +156,11 @@ def sync_endpoints(source):
endpoint_service.update(exists.id, **endpoint)
updated += 1
return new, updated
return new, updated, updated_by_hash
# TODO this is very slow as we don't batch update certificates
def sync_certificates(source, user):
new, updated = 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
def find_cert(certificate):
updated_by_hash = 0
exists = False
if certificate.get("search", None):
@ -152,12 +179,27 @@ def sync_certificates(source, user):
cert = parse_certificate(certificate["body"])
matching_serials = certificate_service.get_by_serial(serial(cert))
exists = find_matching_certificates_by_hash(cert, matching_serials)
updated_by_hash += 1
exists = [x for x in exists if x]
return exists, updated_by_hash
# TODO this is very slow as we don't batch update certificates
def sync_certificates(source, user):
new, updated, updated_by_hash = 0, 0, 0
current_app.logger.debug("Retrieving certificates from {0}".format(source.label))
s = plugins.get(source.plugin_name)
certificates = s.get_certificates(source.options)
for certificate in certificates:
exists, updated_by_hash = find_cert(certificate)
if not certificate.get("owner"):
certificate["owner"] = user.email
certificate["creator"] = user
exists = [x for x in exists if x]
if not exists:
certificate_create(certificate, source)
@ -172,12 +214,20 @@ def sync_certificates(source, user):
certificate_update(e, source)
updated += 1
return new, updated
return new, updated, updated_by_hash
def sync(source, user):
new_certs, updated_certs = sync_certificates(source, user)
new_endpoints, updated_endpoints = sync_endpoints(source)
new_certs, updated_certs, updated_certs_by_hash = sync_certificates(source, user)
new_endpoints, updated_endpoints, updated_endpoints_by_hash = sync_endpoints(source)
metrics.send("sync.updated_certs_by_hash",
"gauge", updated_certs_by_hash,
metric_tags={"source": source.label})
metrics.send("sync.updated_endpoints_by_hash",
"gauge", updated_endpoints_by_hash,
metric_tags={"source": source.label})
source.last_run = arrow.utcnow()
database.update(source)

6
requirements-dev.txt
View File

@ -30,11 +30,11 @@ requests==2.22.0 # via requests-toolbelt, twine
six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.36.1 # via twine
twine==1.15.0
urllib3==1.25.5 # via requests
twine==2.0.0
urllib3==1.25.6 # via requests
virtualenv==16.7.5 # via pre-commit
webencodings==0.5.1 # via bleach
zipp==0.6.0 # via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.2.0 # via twine
# setuptools==41.4.0 # via twine

36
requirements-docs.txt
View File

@ -4,25 +4,25 @@
#
# pip-compile --no-index --output-file=requirements-docs.txt requirements-docs.in
#
acme==0.38.0
acme==0.39.0
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.2.0
amqp==2.5.1
alembic==1.2.1
amqp==2.5.2
aniso8601==8.0.0
arrow==0.15.2
asn1crypto==0.24.0
asn1crypto==1.1.0
asyncpool==1.0
babel==2.7.0 # via sphinx
bcrypt==3.1.7
billiard==3.6.1.0
blinker==1.4
boto3==1.9.232
botocore==1.12.232
boto3==1.9.250
botocore==1.12.250
celery[redis]==4.3.0
certifi==2019.9.11
certsrv==2.1.1
cffi==1.12.3
cffi==1.13.0
chardet==3.0.4
click==7.0
cloudflare==2.3.0
@ -39,9 +39,9 @@ flask-principal==0.4.0
flask-replicated==1.3
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.4.0
flask-sqlalchemy==2.4.1
flask==1.1.1
future==0.17.1
future==0.18.0
gunicorn==19.9.0
hvac==0.9.5
idna==2.8
@ -49,7 +49,7 @@ imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.1.0
javaobj-py3==0.3.0
jinja2==2.10.1
jinja2==2.10.3
jmespath==0.9.4
josepy==1.2.0
jsonlines==1.2.0
@ -66,7 +66,7 @@ packaging==19.2 # via sphinx
paramiko==2.6.0
pem==19.2.0
psycopg2==2.8.3
pyasn1-modules==0.2.6
pyasn1-modules==0.2.7
pyasn1==0.4.7
pycparser==2.19
pycryptodomex==3.9.0
@ -80,16 +80,16 @@ pyrfc3339==1.1
python-dateutil==2.8.0
python-editor==1.0.4
python-json-logger==0.1.11
pytz==2019.2
pytz==2019.3
pyyaml==5.1.2
raven[flask]==6.10.0
redis==3.3.8
redis==3.3.11
requests-toolbelt==0.9.1
requests[security]==2.22.0
retrying==1.3.3
s3transfer==0.2.1
six==1.12.0
snowballstemmer==1.9.1 # via sphinx
snowballstemmer==2.0.0 # via sphinx
sphinx-rtd-theme==0.4.3
sphinx==2.2.0
sphinxcontrib-applehelp==1.0.1 # via sphinx
@ -100,13 +100,13 @@ sphinxcontrib-jsmath==1.0.1 # via sphinx
sphinxcontrib-qthelp==1.0.2 # via sphinx
sphinxcontrib-serializinghtml==1.1.3 # via sphinx
sqlalchemy-utils==0.34.2
sqlalchemy==1.3.8
tabulate==0.8.3
sqlalchemy==1.3.10
tabulate==0.8.5
twofish==0.3.0
urllib3==1.25.5
urllib3==1.25.6
vine==1.3.0
werkzeug==0.16.0
xmltodict==0.12.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.2.0 # via acme, josepy, sphinx
# setuptools==41.4.0 # via acme, josepy, sphinx

44
requirements-tests.txt
View File

@ -5,45 +5,45 @@
# pip-compile --no-index --output-file=requirements-tests.txt requirements-tests.in
#
appdirs==1.4.3 # via black
asn1crypto==0.24.0 # via cryptography
asn1crypto==1.1.0 # via cryptography
atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via black, jsonschema, pytest
aws-sam-translator==1.14.0 # via cfn-lint
attrs==19.3.0 # via black, jsonschema, pytest
aws-sam-translator==1.15.1 # via cfn-lint
aws-xray-sdk==2.4.2 # via moto
bandit==1.6.2
black==19.3b0
boto3==1.9.232 # via aws-sam-translator, moto
boto3==1.9.250 # via aws-sam-translator, moto
boto==2.49.0 # via moto
botocore==1.12.232 # via aws-xray-sdk, boto3, moto, s3transfer
botocore==1.12.250 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2019.9.11 # via requests
cffi==1.12.3 # via cryptography
cfn-lint==0.24.1 # via moto
cffi==1.13.0 # via cryptography
cfn-lint==0.24.4 # via moto
chardet==3.0.4 # via requests
click==7.0 # via black, flask
coverage==4.5.4
cryptography==2.7 # via moto, sshpubkeys
datetime==4.3 # via moto
docker==4.0.2 # via moto
docker==4.1.0 # via moto
docutils==0.15.2 # via botocore
ecdsa==0.13.2 # via python-jose, sshpubkeys
ecdsa==0.13.3 # via python-jose, sshpubkeys
factory-boy==2.12.0
faker==2.0.2
faker==2.0.3
fakeredis==1.0.5
flask==1.1.1 # via pytest-flask
freezegun==0.3.12
future==0.17.1 # via aws-xray-sdk, python-jose
gitdb2==2.0.5 # via gitpython
gitpython==3.0.2 # via bandit
future==0.18.0 # via aws-xray-sdk, python-jose
gitdb2==2.0.6 # via gitpython
gitpython==3.0.3 # via bandit
idna==2.8 # via moto, requests
importlib-metadata==0.23 # via pluggy, pytest
importlib-metadata==0.23 # via jsonschema, pluggy, pytest
itsdangerous==1.1.0 # via flask
jinja2==2.10.1 # via flask, moto
jinja2==2.10.3 # via flask, moto
jmespath==0.9.4 # via boto3, botocore
jsondiff==1.1.2 # via moto
jsonpatch==1.24 # via cfn-lint
jsonpickle==1.2 # via aws-xray-sdk
jsonpointer==2.0 # via jsonpatch
jsonschema==3.0.2 # via aws-sam-translator, cfn-lint
jsonschema==3.1.1 # via aws-sam-translator, cfn-lint
markupsafe==1.1.1 # via jinja2
mock==3.0.5 # via moto
more-itertools==7.2.0 # via pytest, zipp
@ -59,13 +59,13 @@ pyflakes==2.1.1
pyparsing==2.4.2 # via packaging
pyrsistent==0.15.4 # via jsonschema
pytest-flask==0.15.0
pytest-mock==1.10.4
pytest==5.1.2
pytest-mock==1.11.1
pytest==5.2.1
python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==3.0.1 # via moto
pytz==2019.2 # via datetime, moto
pytz==2019.3 # via datetime, moto
pyyaml==5.1.2
redis==3.3.8 # via fakeredis
redis==3.3.11 # via fakeredis
requests-mock==1.7.0
requests==2.22.0 # via docker, moto, requests-mock, responses
responses==0.10.6 # via moto
@ -78,7 +78,7 @@ sshpubkeys==3.1.0 # via moto
stevedore==1.31.0 # via bandit
text-unidecode==1.3 # via faker
toml==0.10.0 # via black
urllib3==1.25.5 # via botocore, requests
urllib3==1.25.6 # via botocore, requests
wcwidth==0.1.7 # via pytest
websocket-client==0.56.0 # via docker
werkzeug==0.16.0 # via flask, moto, pytest-flask
@ -88,4 +88,4 @@ zipp==0.6.0 # via importlib-metadata
zope.interface==4.6.0 # via datetime
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.2.0 # via cfn-lint, jsonschema, zope.interface
# setuptools==41.4.0 # via cfn-lint, jsonschema, zope.interface

34
requirements.txt
View File

@ -4,23 +4,23 @@
#
# pip-compile --no-index --output-file=requirements.txt requirements.in
#
acme==0.38.0
acme==0.39.0
alembic-autogenerate-enums==0.0.2
alembic==1.2.0 # via flask-migrate
amqp==2.5.1 # via kombu
alembic==1.2.1 # via flask-migrate
amqp==2.5.2 # via kombu
aniso8601==8.0.0 # via flask-restful
arrow==0.15.2
asn1crypto==0.24.0 # via cryptography
asn1crypto==1.1.0 # via cryptography
asyncpool==1.0
bcrypt==3.1.7 # via flask-bcrypt, paramiko
billiard==3.6.1.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.232
botocore==1.12.232
boto3==1.9.250
botocore==1.12.250
celery[redis]==4.3.0
certifi==2019.9.11
certsrv==2.1.1
cffi==1.12.3 # via bcrypt, cryptography, pynacl
cffi==1.13.0 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
click==7.0 # via flask
cloudflare==2.3.0
@ -37,16 +37,16 @@ flask-principal==0.4.0
flask-replicated==1.3
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.4.0
flask-sqlalchemy==2.4.1
flask==1.1.1
future==0.17.1
future==0.18.0
gunicorn==19.9.0
hvac==0.9.5
idna==2.8 # via requests
inflection==0.3.1
itsdangerous==1.1.0 # via flask
javaobj-py3==0.3.0 # via pyjks
jinja2==2.10.1
jinja2==2.10.3
jmespath==0.9.4 # via boto3, botocore
josepy==1.2.0 # via acme
jsonlines==1.2.0 # via cloudflare
@ -62,7 +62,7 @@ ndg-httpsclient==0.5.1
paramiko==2.6.0
pem==19.2.0
psycopg2==2.8.3
pyasn1-modules==0.2.6 # via pyjks, python-ldap
pyasn1-modules==0.2.7 # via pyjks, python-ldap
pyasn1==0.4.7 # via ndg-httpsclient, pyasn1-modules, pyjks, python-ldap
pycparser==2.19 # via cffi
pycryptodomex==3.9.0 # via pyjks
@ -75,23 +75,23 @@ python-dateutil==2.8.0 # via alembic, arrow, botocore
python-editor==1.0.4 # via alembic
python-json-logger==0.1.11 # via logmatic-python
python-ldap==3.2.0
pytz==2019.2 # via acme, celery, flask-restful, pyrfc3339
pytz==2019.3 # via acme, celery, flask-restful, pyrfc3339
pyyaml==5.1.2
raven[flask]==6.10.0
redis==3.3.8
redis==3.3.11
requests-toolbelt==0.9.1 # via acme
requests[security]==2.22.0
retrying==1.3.3
s3transfer==0.2.1 # via boto3
six==1.12.0
sqlalchemy-utils==0.34.2
sqlalchemy==1.3.8 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
sqlalchemy==1.3.10 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.5
twofish==0.3.0 # via pyjks
urllib3==1.25.5 # via botocore, requests
urllib3==1.25.6 # via botocore, requests
vine==1.3.0 # via amqp, celery
werkzeug==0.16.0 # via flask
xmltodict==0.12.0
# The following packages are considered to be unsafe in a requirements file:
# setuptools==41.2.0 # via acme, josepy
# setuptools==41.4.0 # via acme, josepy