Merge branch 'lemur_vault_source' of github.com:/alwaysjolley/lemur into lemur_vault_source

This commit is contained in:
alwaysjolley 2019-05-07 10:06:09 -04:00
commit ef7a8587fe
8 changed files with 143 additions and 96 deletions

View File

@ -1,6 +1,6 @@
FROM python:3.5
RUN apt-get update
RUN apt-get install -y make python-software-properties curl
RUN apt-get install -y make software-properties-common curl
RUN curl -sL https://deb.nodesource.com/setup_7.x | bash -
RUN apt-get update
RUN apt-get install -y nodejs libldap2-dev libsasl2-dev libldap2-dev libssl-dev

View File

@ -64,11 +64,10 @@ def wait_for_dns_change(change_id, account_number=None):
metrics.send('wait_for_dns_change_fail', 'counter', 1)
sentry.captureException(
extra={
"fqdn": fqdn, "txt_record": token}
"fqdn": str(fqdn), "txt_record": str(token)}
)
metrics.send('wait_for_dns_change_error', 'counter', 1,
metric_tags={'fqdn': fqdn, 'txt_record': token})
raise Exception("Unable to query DNS token for fqdn {}.".format(fqdn))
return
@ -155,8 +154,8 @@ def delete_txt_record(change_id, account_number, domain, token):
except DynectDeleteError:
sentry.captureException(
extra={
"fqdn": fqdn, "zone_name": zone_name, "node_name": node_name,
"txt_record": txt_record.txtdata}
"fqdn": str(fqdn), "zone_name": str(zone_name), "node_name": str(node_name),
"txt_record": str(txt_record.txtdata)}
)
metrics.send('delete_txt_record_deleteerror', 'counter', 1,
metric_tags={'fqdn': fqdn, 'txt_record': txt_record.txtdata})
@ -166,11 +165,11 @@ def delete_txt_record(change_id, account_number, domain, token):
except DynectUpdateError:
sentry.captureException(
extra={
"fqdn": fqdn, "zone_name": zone_name, "node_name": node_name,
"txt_record": txt_record.txtdata}
"fqdn": str(fqdn), "zone_name": str(zone_name), "node_name": str(node_name),
"txt_record": str(txt_record.txtdata)}
)
metrics.send('delete_txt_record_publish_error', 'counter', 1,
metric_tags={'fqdn': fqdn, 'txt_record': txt_record.txtdata})
metric_tags={'fqdn': str(fqdn), 'txt_record': str(txt_record.txtdata)})
def delete_acme_txt_records(domain):
@ -201,8 +200,8 @@ def delete_acme_txt_records(domain):
except DynectDeleteError:
sentry.captureException(
extra={
"fqdn": fqdn, "zone_name": zone_name, "node_name": node_name,
"txt_record": txt_record.txtdata}
"fqdn": str(fqdn), "zone_name": str(zone_name), "node_name": str(node_name),
"txt_record": str(txt_record.txtdata)}
)
metrics.send('delete_txt_record_deleteerror', 'counter', 1,
metric_tags={'fqdn': fqdn, 'txt_record': txt_record.txtdata})

View File

@ -19,7 +19,7 @@ import OpenSSL.crypto
import josepy as jose
from acme import challenges, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.errors import PollError, WildcardUnsupportedError
from acme.errors import PollError, TimeoutError, WildcardUnsupportedError
from acme.messages import Error as AcmeError
from botocore.exceptions import ClientError
from flask import current_app
@ -56,7 +56,7 @@ class AcmeHandler(object):
def find_dns_challenge(self, host, authorizations):
dns_challenges = []
for authz in authorizations:
if not authz.body.identifier.value == host:
if not authz.body.identifier.value.lower() == host.lower():
continue
for combo in authz.body.challenges:
if isinstance(combo.chall, challenges.DNS01):
@ -77,8 +77,13 @@ class AcmeHandler(object):
change_ids = []
host_to_validate = self.maybe_remove_wildcard(host)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
dns_challenges = self.find_dns_challenge(host_to_validate, order.authorizations)
host_to_validate = self.maybe_add_extension(host_to_validate, dns_provider_options)
if not dns_challenges:
sentry.captureException()
metrics.send('start_dns_challenge_error_no_dns_challenges', 'counter', 1)
raise Exception("Unable to determine DNS challenges from authorizations")
for dns_challenge in dns_challenges:
change_id = dns_provider.create_txt_record(
@ -127,24 +132,26 @@ class AcmeHandler(object):
acme_client.client.net.key.public_key()
)
if not verified:
metrics.send('complete_dns_challenge_verification_error', 'counter', 1)
raise ValueError("Failed verification")
if not verified:
metrics.send('complete_dns_challenge_verification_error', 'counter', 1)
raise ValueError("Failed verification")
time.sleep(5)
acme_client.answer_challenge(dns_challenge, response)
time.sleep(5)
res = acme_client.answer_challenge(dns_challenge, response)
current_app.logger.debug(f"answer_challenge response: {res}")
def request_certificate(self, acme_client, authorizations, order):
for authorization in authorizations:
for authz in authorization.authz:
authorization_resource, _ = acme_client.poll(authz)
deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
deadline = datetime.datetime.now() + datetime.timedelta(seconds=360)
try:
orderr = acme_client.poll_and_finalize(order, deadline)
except AcmeError:
sentry.captureException(extra={"order_url": order.uri})
except (AcmeError, TimeoutError):
sentry.captureException(extra={"order_url": str(order.uri)})
metrics.send('request_certificate_error', 'counter', 1)
current_app.logger.error(f"Unable to resolve Acme order: {order.uri}", exc_info=True)
raise

View File

@ -21,14 +21,22 @@ def retry_throttled(exception):
:param exception:
:return:
"""
# Log details about the exception
try:
raise exception
except Exception as e:
current_app.logger.error("ELB retry_throttled triggered", exc_info=True)
metrics.send('elb_retry', 'counter', 1,
metric_tags={"exception": e})
sentry.captureException()
if isinstance(exception, botocore.exceptions.ClientError):
if exception.response['Error']['Code'] == 'LoadBalancerNotFound':
return False
if exception.response['Error']['Code'] == 'CertificateNotFound':
return False
metrics.send('elb_retry', 'counter', 1)
return True
@ -63,16 +71,20 @@ def get_all_elbs(**kwargs):
:return:
"""
elbs = []
try:
while True:
response = get_elbs(**kwargs)
while True:
response = get_elbs(**kwargs)
elbs += response['LoadBalancerDescriptions']
elbs += response['LoadBalancerDescriptions']
if not response.get('NextMarker'):
return elbs
else:
kwargs.update(dict(Marker=response['NextMarker']))
if not response.get('NextMarker'):
return elbs
else:
kwargs.update(dict(Marker=response['NextMarker']))
except Exception as e: # noqa
metrics.send('get_all_elbs_error', 'counter', 1)
sentry.captureException()
raise
def get_all_elbs_v2(**kwargs):
@ -84,18 +96,23 @@ def get_all_elbs_v2(**kwargs):
"""
elbs = []
while True:
response = get_elbs_v2(**kwargs)
elbs += response['LoadBalancers']
try:
while True:
response = get_elbs_v2(**kwargs)
elbs += response['LoadBalancers']
if not response.get('NextMarker'):
return elbs
else:
kwargs.update(dict(Marker=response['NextMarker']))
if not response.get('NextMarker'):
return elbs
else:
kwargs.update(dict(Marker=response['NextMarker']))
except Exception as e: # noqa
metrics.send('get_all_elbs_v2_error', 'counter', 1)
sentry.captureException()
raise
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
"""
Get a listener ARN from an endpoint.
@ -103,27 +120,40 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
:param endpoint_port:
:return:
"""
client = kwargs.pop('client')
elbs = client.describe_load_balancers(Names=[endpoint_name])
for elb in elbs['LoadBalancers']:
listeners = client.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])
for listener in listeners['Listeners']:
if listener['Port'] == endpoint_port:
return listener['ListenerArn']
try:
client = kwargs.pop('client')
elbs = client.describe_load_balancers(Names=[endpoint_name])
for elb in elbs['LoadBalancers']:
listeners = client.describe_listeners(LoadBalancerArn=elb['LoadBalancerArn'])
for listener in listeners['Listeners']:
if listener['Port'] == endpoint_port:
return listener['ListenerArn']
except Exception as e: # noqa
metrics.send('get_listener_arn_from_endpoint_error', 'counter', 1,
metric_tags={"error": e, "endpoint_name": endpoint_name, "endpoint_port": endpoint_port})
sentry.captureException(extra={"endpoint_name": str(endpoint_name),
"endpoint_port": str(endpoint_port)})
raise
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_elbs(**kwargs):
"""
Fetches one page elb objects for a given account and region.
"""
client = kwargs.pop('client')
return client.describe_load_balancers(**kwargs)
try:
client = kwargs.pop('client')
return client.describe_load_balancers(**kwargs)
except Exception as e: # noqa
metrics.send('get_elbs_error', 'counter', 1,
metric_tags={"error": e})
sentry.captureException()
raise
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_elbs_v2(**kwargs):
"""
Fetches one page of elb objects for a given account and region.
@ -131,12 +161,18 @@ def get_elbs_v2(**kwargs):
:param kwargs:
:return:
"""
client = kwargs.pop('client')
return client.describe_load_balancers(**kwargs)
try:
client = kwargs.pop('client')
return client.describe_load_balancers(**kwargs)
except Exception as e: # noqa
metrics.send('get_elbs_v2_error', 'counter', 1,
metric_tags={"error": e})
sentry.captureException()
raise
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def describe_listeners_v2(**kwargs):
"""
Fetches one page of listener objects for a given elb arn.
@ -144,8 +180,14 @@ def describe_listeners_v2(**kwargs):
:param kwargs:
:return:
"""
client = kwargs.pop('client')
return client.describe_listeners(**kwargs)
try:
client = kwargs.pop('client')
return client.describe_listeners(**kwargs)
except Exception as e: # noqa
metrics.send('describe_listeners_v2_error', 'counter', 1,
metric_tags={"error": e})
sentry.captureException()
raise
@sts_client('elb')
@ -157,13 +199,15 @@ def describe_load_balancer_policies(load_balancer_name, policy_names, **kwargs):
:param load_balancer_name:
:return:
"""
try:
return kwargs['client'].describe_load_balancer_policies(LoadBalancerName=load_balancer_name,
PolicyNames=policy_names)
except Exception as e: # noqa
metrics.send('describe_load_balancer_policies_fail', 'counter', 1,
metrics.send('describe_load_balancer_policies_error', 'counter', 1,
metric_tags={"load_balancer_name": load_balancer_name, "policy_names": policy_names, "error": e})
sentry.captureException(extra={"load_balancer_name": load_balancer_name, "policy_names": policy_names})
sentry.captureException(extra={"load_balancer_name": str(load_balancer_name),
"policy_names": str(policy_names)})
raise
@ -179,14 +223,14 @@ def describe_ssl_policies_v2(policy_names, **kwargs):
try:
return kwargs['client'].describe_ssl_policies(Names=policy_names)
except Exception as e: # noqa
metrics.send('describe_ssl_policies_v2_fail', 'counter', 1,
metrics.send('describe_ssl_policies_v2_error', 'counter', 1,
metric_tags={"policy_names": policy_names, "error": e})
sentry.captureException(extra={"policy_names": policy_names})
sentry.captureException(extra={"policy_names": str(policy_names)})
raise
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def describe_load_balancer_types(policies, **kwargs):
"""
Describe the policies with policy details.
@ -198,7 +242,7 @@ def describe_load_balancer_types(policies, **kwargs):
@sts_client('elb')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def attach_certificate(name, port, certificate_id, **kwargs):
"""
Attaches a certificate to a listener, throws exception
@ -218,7 +262,7 @@ def attach_certificate(name, port, certificate_id, **kwargs):
@sts_client('elbv2')
@retry(retry_on_exception=retry_throttled, wait_fixed=2000)
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def attach_certificate_v2(listener_arn, port, certificates, **kwargs):
"""
Attaches a certificate to a listener, throws exception

View File

@ -11,14 +11,14 @@ cfgv==1.6.0 # via pre-commit
chardet==3.0.4 # via requests
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.4.1 # via pre-commit
identify==1.4.2 # via pre-commit
idna==2.8 # via requests
importlib-metadata==0.9 # via pre-commit
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.3
pkginfo==1.5.0.1 # via twine
pre-commit==1.15.2
pre-commit==1.16.0
pycodestyle==2.3.1 # via flake8
pyflakes==1.6.0 # via flake8
pygments==2.3.1 # via readme-renderer
@ -30,7 +30,7 @@ six==1.12.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.31.1 # via twine
twine==1.13.0
urllib3==1.24.2 # via requests
virtualenv==16.4.3 # via pre-commit
urllib3==1.24.3 # via requests
virtualenv==16.5.0 # via pre-commit
webencodings==0.5.1 # via bleach
zipp==0.3.3 # via importlib-metadata
zipp==0.4.0 # via importlib-metadata

View File

@ -4,10 +4,10 @@
#
# pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
#
acme==0.33.1
acme==0.34.1
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.9
alembic==1.0.10
amqp==2.4.2
aniso8601==6.0.0
arrow==0.13.1
@ -17,8 +17,8 @@ babel==2.6.0 # via sphinx
bcrypt==3.1.6
billiard==3.6.0.0
blinker==1.4
boto3==1.9.134
botocore==1.12.134
boto3==1.9.143
botocore==1.12.143
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
@ -38,7 +38,7 @@ flask-migrate==2.4.0
flask-principal==0.4.0
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask-sqlalchemy==2.4.0
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
@ -47,7 +47,7 @@ idna==2.8
imagesize==1.1.0 # via sphinx
inflection==0.3.1
itsdangerous==1.1.0
javaobj-py3==0.2.4
javaobj-py3==0.3.0
jinja2==2.10.1
jmespath==0.9.4
josepy==1.1.0
@ -56,16 +56,15 @@ kombu==4.5.0
lockfile==0.12.2
mako==1.0.9
markupsafe==1.1.1
marshmallow-sqlalchemy==0.16.2
marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2
mock==2.0.0
mock==3.0.4
ndg-httpsclient==0.5.1
packaging==19.0 # via sphinx
paramiko==2.4.2
pbr==5.1.3
pem==19.1.0
psycopg2==2.8.2
pyasn1-modules==0.2.4
pyasn1-modules==0.2.5
pyasn1==0.4.5
pycparser==2.19
pycryptodomex==3.8.1
@ -101,7 +100,7 @@ sqlalchemy-utils==0.33.11
sqlalchemy==1.3.3
tabulate==0.8.3
twofish==0.3.0
urllib3==1.24.2
urllib3==1.24.3
vine==1.3.0
werkzeug==0.15.2
xmltodict==0.12.0

View File

@ -7,11 +7,11 @@
asn1crypto==0.24.0 # via cryptography
atomicwrites==1.3.0 # via pytest
attrs==19.1.0 # via pytest
aws-sam-translator==1.10.0 # via cfn-lint
aws-sam-translator==1.11.0 # via cfn-lint
aws-xray-sdk==2.4.2 # via moto
boto3==1.9.134 # via aws-sam-translator, moto
boto3==1.9.143 # via aws-sam-translator, moto
boto==2.49.0 # via moto
botocore==1.12.134 # via aws-xray-sdk, boto3, moto, s3transfer
botocore==1.12.143 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2019.3.9 # via requests
cffi==1.12.3 # via cryptography
cfn-lint==0.19.1 # via moto
@ -38,11 +38,10 @@ jsonpickle==1.1 # via aws-xray-sdk
jsonpointer==2.0 # via jsonpatch
jsonschema==2.6.0 # via aws-sam-translator, cfn-lint
markupsafe==1.1.1 # via jinja2
mock==2.0.0 # via moto
mock==3.0.4 # via moto
more-itertools==7.0.0 # via pytest
moto==1.3.8
nose==1.3.7
pbr==5.1.3 # via mock
pluggy==0.9.0 # via pytest
py==1.8.0 # via pytest
pyasn1==0.4.5 # via rsa
@ -55,14 +54,14 @@ python-dateutil==2.8.0 # via botocore, faker, freezegun, moto
python-jose==3.0.1 # via moto
pytz==2019.1 # via moto
pyyaml==5.1
requests-mock==1.5.2
requests-mock==1.6.0
requests==2.21.0 # via cfn-lint, docker, moto, requests-mock, responses
responses==0.10.6 # via moto
rsa==4.0 # via python-jose
s3transfer==0.2.0 # via boto3
six==1.12.0 # via aws-sam-translator, cfn-lint, cryptography, docker, docker-pycreds, faker, freezegun, mock, moto, pytest, python-dateutil, python-jose, requests-mock, responses, websocket-client
text-unidecode==1.2 # via faker
urllib3==1.24.2 # via botocore, requests
urllib3==1.24.3 # via botocore, requests
websocket-client==0.56.0 # via docker
werkzeug==0.15.2 # via flask, moto, pytest-flask
wrapt==1.11.1 # via aws-xray-sdk

View File

@ -4,9 +4,9 @@
#
# pip-compile --output-file requirements.txt requirements.in -U --no-index
#
acme==0.33.1
acme==0.34.1
alembic-autogenerate-enums==0.0.2
alembic==1.0.9 # via flask-migrate
alembic==1.0.10 # via flask-migrate
amqp==2.4.2 # via kombu
aniso8601==6.0.0 # via flask-restful
arrow==0.13.1
@ -15,8 +15,8 @@ asyncpool==1.0
bcrypt==3.1.6 # via flask-bcrypt, paramiko
billiard==3.6.0.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.9.134
botocore==1.12.134
boto3==1.9.143
botocore==1.12.143
celery[redis]==4.3.0
certifi==2019.3.9
certsrv==2.1.1
@ -36,7 +36,7 @@ flask-migrate==2.4.0
flask-principal==0.4.0
flask-restful==0.3.7
flask-script==2.0.6
flask-sqlalchemy==2.3.2
flask-sqlalchemy==2.4.0
flask==1.0.2
future==0.17.1
gunicorn==19.9.0
@ -44,7 +44,7 @@ hvac==0.8.2
idna==2.8 # via requests
inflection==0.3.1
itsdangerous==1.1.0 # via flask
javaobj-py3==0.2.4 # via pyjks
javaobj-py3==0.3.0 # via pyjks
jinja2==2.10.1
jmespath==0.9.4 # via boto3, botocore
josepy==1.1.0 # via acme
@ -53,15 +53,14 @@ kombu==4.5.0
lockfile==0.12.2
mako==1.0.9 # via alembic
markupsafe==1.1.1 # via jinja2, mako
marshmallow-sqlalchemy==0.16.2
marshmallow-sqlalchemy==0.16.3
marshmallow==2.19.2
mock==2.0.0 # via acme
mock==3.0.4 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.2
pbr==5.1.3 # via mock
pem==19.1.0
psycopg2==2.8.2
pyasn1-modules==0.2.4 # via pyjks, python-ldap
pyasn1-modules==0.2.5 # via pyjks, python-ldap
pyasn1==0.4.5 # via ndg-httpsclient, paramiko, pyasn1-modules, pyjks, python-ldap
pycparser==2.19 # via cffi
pycryptodomex==3.8.1 # via pyjks
@ -86,7 +85,7 @@ sqlalchemy-utils==0.33.11
sqlalchemy==1.3.3 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.3
twofish==0.3.0 # via pyjks
urllib3==1.24.2 # via botocore, requests
urllib3==1.24.3 # via botocore, requests
vine==1.3.0 # via amqp, celery
werkzeug==0.15.2 # via flask
xmltodict==0.12.0