Merge branch 'entrust_source' of github.com:sirferl/lemur into entrust_source

This commit is contained in:
sirferl 2020-12-03 08:35:02 +01:00
commit 6585bcaa55
12 changed files with 283 additions and 25 deletions

View File

@ -115,10 +115,10 @@ endif
@echo "--> Updating Python requirements" @echo "--> Updating Python requirements"
pip install --upgrade pip pip install --upgrade pip
pip install --upgrade pip-tools pip install --upgrade pip-tools
pip-compile --output-file requirements.txt requirements.in -U --no-index pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
@echo "--> Done updating Python requirements" @echo "--> Done updating Python requirements"
@echo "--> Removing python-ldap from requirements-docs.txt" @echo "--> Removing python-ldap from requirements-docs.txt"
grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

View File

@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput
from lemur.common.utils import generate_private_key, truthiness from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination from lemur.destinations.models import Destination
from lemur.domains.models import Domain from lemur.domains.models import Domain
from lemur.endpoints import service as endpoint_service
from lemur.extensions import metrics, sentry, signals from lemur.extensions import metrics, sentry, signals
from lemur.models import certificate_associations from lemur.models import certificate_associations
from lemur.notifications.models import Notification from lemur.notifications.models import Notification
@ -797,3 +798,61 @@ def reissue_certificate(certificate, replace=None, user=None):
new_cert = create(**primitives) new_cert = create(**primitives)
return new_cert return new_cert
def is_attached_to_endpoint(certificate_name, endpoint_name):
"""
Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
This method talks to elb and finds the real time information.
:param certificate_name:
:param endpoint_name:
:return: True if certificate is attached to the given endpoint, False otherwise
"""
endpoint = endpoint_service.get_by_name(endpoint_name)
attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
return certificate_name in attached_certificates
def remove_from_destination(certificate, destination):
"""
Remove the certificate from given destination if clean() is implemented
:param certificate:
:param destination:
:return:
"""
plugin = plugins.get(destination.plugin_name)
if not hasattr(plugin, "clean"):
info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
current_app.logger.warning(info_text)
else:
plugin.clean(certificate=certificate, options=destination.options)
def cleanup_after_revoke(certificate):
"""
Perform the needed cleanup for a revoked certificate. This includes -
1. Disabling notification
2. Disabling auto-rotation
3. Update certificate status to 'revoked'
4. Remove from AWS
:param certificate: Certificate object to modify and update in DB
:return: None
"""
certificate.notify = False
certificate.rotation = False
certificate.status = 'revoked'
error_message = ""
for destination in list(certificate.destinations):
try:
remove_from_destination(certificate, destination)
certificate.destinations.remove(destination)
except Exception as e:
# This cleanup is the best-effort since certificate is already revoked at this point.
# We will capture the exception and move on to the next destination
sentry.captureException()
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
database.update(certificate)
return error_message

View File

@ -19,6 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
from lemur.certificates import service from lemur.certificates import service
from lemur.certificates.models import Certificate from lemur.certificates.models import Certificate
from lemur.extensions import sentry
from lemur.plugins.base import plugins from lemur.plugins.base import plugins
from lemur.certificates.schemas import ( from lemur.certificates.schemas import (
certificate_input_schema, certificate_input_schema,
@ -888,8 +889,24 @@ class Certificates(AuthenticatedResource):
if cert.owner != data["owner"]: if cert.owner != data["owner"]:
service.cleanup_owner_roles_notification(cert.owner, data) service.cleanup_owner_roles_notification(cert.owner, data)
error_message = ""
# if destination is removed, cleanup the certificate from AWS
for destination in cert.destinations:
if destination not in data["destinations"]:
try:
service.remove_from_destination(cert, destination)
except Exception as e:
sentry.captureException()
# Add the removed destination back
data["destinations"].append(destination)
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
# go ahead with DB update
cert = service.update(certificate_id, **data) cert = service.update(certificate_id, **data)
log_service.create(g.current_user, "update_cert", certificate=cert) log_service.create(g.current_user, "update_cert", certificate=cert)
if error_message:
return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
return cert return cert
@validate_schema(certificate_edit_input_schema, certificate_output_schema) @validate_schema(certificate_edit_input_schema, certificate_output_schema)
@ -1433,16 +1450,24 @@ class CertificateRevoke(AuthenticatedResource):
return dict(message="Cannot revoke certificate. No external id found."), 400 return dict(message="Cannot revoke certificate. No external id found."), 400
if cert.endpoints: if cert.endpoints:
return ( for endpoint in cert.endpoints:
dict( if service.is_attached_to_endpoint(cert.name, endpoint.name):
message="Cannot revoke certificate. Endpoints are deployed with the given certificate." return (
), dict(
403, message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
) ),
403,
)
plugin = plugins.get(cert.authority.plugin_name) plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data) plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, "revoke_cert", certificate=cert) log_service.create(g.current_user, "revoke_cert", certificate=cert)
# Perform cleanup after revoke
error_message = service.cleanup_after_revoke(cert)
if error_message:
return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
return dict(id=cert.id) return dict(id=cert.id)

View File

@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
raise raise
@sts_client("elbv2")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5)
def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs):
"""
Get a load balancer ARN from an endpoint.
:param endpoint_name:
:return:
"""
try:
client = kwargs.pop("client")
elbs = client.describe_load_balancers(Names=[endpoint_name])
if "LoadBalancers" in elbs and elbs["LoadBalancers"]:
return elbs["LoadBalancers"][0]["LoadBalancerArn"]
except Exception as e: # noqa
metrics.send(
"get_load_balancer_arn_from_endpoint",
"counter",
1,
metric_tags={
"error": str(e),
"endpoint_name": endpoint_name,
},
)
sentry.captureException(
extra={
"endpoint_name": str(endpoint_name),
}
)
raise
@sts_client("elb") @sts_client("elb")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20) @retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_elbs(**kwargs): def get_elbs(**kwargs):

View File

@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin):
) )
return None return None
def get_endpoint_certificate_names(self, endpoint):
options = endpoint.source.options
account_number = self.get_option("accountNumber", options)
region = get_region_from_dns(endpoint.dnsname)
certificate_names = []
if endpoint.type == "elb":
elb_details = elb.get_elbs(account_number=account_number,
region=region,
LoadBalancerNames=[endpoint.name],)
for lb_description in elb_details["LoadBalancerDescriptions"]:
for listener_description in lb_description["ListenerDescriptions"]:
listener = listener_description.get("Listener")
if not listener.get("SSLCertificateId"):
continue
certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId")))
elif endpoint.type == "elbv2":
listeners = elb.describe_listeners_v2(
account_number=account_number,
region=region,
LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name,
account_number=account_number,
region=region),
)
for listener in listeners["Listeners"]:
if not listener.get("Certificates"):
continue
for certificate in listener["Certificates"]:
certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"]))
return certificate_names
class AWSDestinationPlugin(DestinationPlugin): class AWSDestinationPlugin(DestinationPlugin):
title = "AWS" title = "AWS"
@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin):
def deploy(self, elb_name, account, region, certificate): def deploy(self, elb_name, account, region, certificate):
pass pass
def clean(self, certificate, options, **kwargs):
account_number = self.get_option("accountNumber", options)
iam.delete_cert(certificate.name, account_number=account_number)
class S3DestinationPlugin(ExportDestinationPlugin): class S3DestinationPlugin(ExportDestinationPlugin):
title = "AWS-S3" title = "AWS-S3"

View File

@ -1,5 +1,5 @@
import boto3 import boto3
from moto import mock_sts, mock_elb from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam
@mock_sts() @mock_sts()
@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials):
elbs = get_all_elbs(account_number="123456789012", region="us-east-1") elbs = get_all_elbs(account_number="123456789012", region="us-east-1")
assert elbs assert elbs
@mock_sts()
@mock_ec2
@mock_elbv2()
@mock_iam
def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials):
from lemur.plugins.lemur_aws import iam, elb
endpoint_name = "example-lbv2"
account_number = "123456789012"
region_ue1 = "us-east-1"
client = boto3.client("elbv2", region_name="us-east-1")
ec2 = boto3.resource("ec2", region_name="us-east-1")
# Create VPC
vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24")
# Create LB (elbv2) in above VPC
assert create_load_balancer(client, ec2, vpc.id, endpoint_name)
# Create target group
target_group_arn = create_target_group(client, vpc.id)
assert target_group_arn
# Test get_load_balancer_arn_from_endpoint
lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name,
account_number=account_number,
region=region_ue1)
assert lb_arn
# Test describe_listeners_v2
listeners = elb.describe_listeners_v2(account_number=account_number,
region=region_ue1,
LoadBalancerArn=lb_arn)
assert listeners
assert not listeners["Listeners"]
# Upload cert
response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2",
account_number=account_number)
assert response
cert_arn = response["ServerCertificateMetadata"]["Arn"]
assert cert_arn
# Create https listener using above cert
listeners = client.create_listener(
LoadBalancerArn=lb_arn,
Protocol="HTTPS",
Port=443,
Certificates=[{"CertificateArn": cert_arn}],
DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
)
assert listeners
listener_arn = listeners["Listeners"][0]["ListenerArn"]
assert listener_arn
assert listeners["Listeners"]
for listener in listeners["Listeners"]:
if listener["Port"] == 443:
assert listener["Certificates"]
assert cert_arn == listener["Certificates"][0]["CertificateArn"]
# Test get_listener_arn_from_endpoint
assert listener_arn == elb.get_listener_arn_from_endpoint(
endpoint_name,
443,
account_number=account_number,
region=region_ue1,
)
@mock_sts()
@mock_elb()
def test_get_all_elbs_v2():
from lemur.plugins.lemur_aws.elb import get_all_elbs_v2
elbs = get_all_elbs_v2(account_number="123456789012",
region="us-east-1")
assert elbs
def create_load_balancer(client, ec2, vpc_id, endpoint_name):
subnet1 = ec2.create_subnet(
VpcId=vpc_id,
CidrBlock="172.28.7.192/26",
AvailabilityZone="us-east-1a"
)
return client.create_load_balancer(
Name=endpoint_name,
Subnets=[
subnet1.id,
],
)
def create_target_group(client, vpc_id):
response = client.create_target_group(
Name="a-target",
Protocol="HTTPS",
Port=443,
VpcId=vpc_id,
)
return response.get("TargetGroups")[0]["TargetGroupArn"]

View File

@ -33,7 +33,7 @@
<div class="col-sm-10"> <div class="col-sm-10">
<select class="form-control" ng-model="certificate.keyType" <select class="form-control" ng-model="certificate.keyType"
ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']" ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']"
ng-init="certificate.keyType = 'RSA2048'"></select> ng-init="certificate.keyType = 'ECCPRIME256V1'"></select>
</div> </div>
</div> </div>
<div class="form-group"> <div class="form-group">

View File

@ -26,9 +26,8 @@
</div> </div>
</form> </form>
<div ng-if="certificate.endpoints.length"> <div ng-if="certificate.endpoints.length">
<p><strong>Certificate cannot be revoked, it is associated with the following endpoints. Disassociate this <p><strong>Certificate might be associated with the following endpoints. Disassociate this
certificate certificate before revoking or continue if you've already done so.</strong></p>
before revoking.</strong></p>
<ul class="list-group"> <ul class="list-group">
<li class="list-group-item" ng-repeat="endpoint in certificate.endpoints"> <li class="list-group-item" ng-repeat="endpoint in certificate.endpoints">
<span class="pull-right"><label class="label label-default">{{ endpoint.type }}</label></span> <span class="pull-right"><label class="label label-default">{{ endpoint.type }}</label></span>

View File

@ -24,7 +24,7 @@ keyring==21.2.0 # via twine
mccabe==0.6.1 # via flake8 mccabe==0.6.1 # via flake8
nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit
pkginfo==1.5.0.1 # via twine pkginfo==1.5.0.1 # via twine
pre-commit==2.9.0 # via -r requirements-dev.in pre-commit==2.9.2 # via -r requirements-dev.in
pycodestyle==2.6.0 # via flake8 pycodestyle==2.6.0 # via flake8
pycparser==2.20 # via cffi pycparser==2.20 # via cffi
pyflakes==2.2.0 # via flake8 pyflakes==2.2.0 # via flake8

View File

@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare
billiard==3.6.3.0 # via -r requirements.txt, celery billiard==3.6.3.0 # via -r requirements.txt, celery
blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven
boto3==1.16.24 # via -r requirements.txt boto3==1.16.25 # via -r requirements.txt
botocore==1.19.24 # via -r requirements.txt, boto3, s3transfer botocore==1.19.25 # via -r requirements.txt, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.txt celery[redis]==4.4.2 # via -r requirements.txt
certifi==2020.11.8 # via -r requirements.txt, requests certifi==2020.11.8 # via -r requirements.txt, requests
certsrv==2.1.1 # via -r requirements.txt certsrv==2.1.1 # via -r requirements.txt
@ -73,7 +73,7 @@ pygments==2.6.1 # via sphinx
pyjks==20.0.0 # via -r requirements.txt pyjks==20.0.0 # via -r requirements.txt
pyjwt==1.7.1 # via -r requirements.txt pyjwt==1.7.1 # via -r requirements.txt
pynacl==1.3.0 # via -r requirements.txt, paramiko pynacl==1.3.0 # via -r requirements.txt, paramiko
pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests pyopenssl==20.0.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
pyparsing==2.4.7 # via packaging pyparsing==2.4.7 # via packaging
pyrfc3339==1.1 # via -r requirements.txt, acme pyrfc3339==1.1 # via -r requirements.txt, acme
python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore

View File

@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint
aws-xray-sdk==2.5.0 # via moto aws-xray-sdk==2.5.0 # via moto
bandit==1.6.2 # via -r requirements-tests.in bandit==1.6.2 # via -r requirements-tests.in
black==20.8b1 # via -r requirements-tests.in black==20.8b1 # via -r requirements-tests.in
boto3==1.16.24 # via aws-sam-translator, moto boto3==1.16.25 # via aws-sam-translator, moto
boto==2.49.0 # via moto boto==2.49.0 # via moto
botocore==1.19.24 # via aws-xray-sdk, boto3, moto, s3transfer botocore==1.19.25 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2020.11.8 # via requests certifi==2020.11.8 # via requests
cffi==1.14.0 # via cryptography cffi==1.14.0 # via cryptography
cfn-lint==0.29.5 # via moto cfn-lint==0.29.5 # via moto
@ -25,7 +25,7 @@ docker==4.2.0 # via moto
ecdsa==0.14.1 # via moto, python-jose, sshpubkeys ecdsa==0.14.1 # via moto, python-jose, sshpubkeys
factory-boy==3.1.0 # via -r requirements-tests.in factory-boy==3.1.0 # via -r requirements-tests.in
faker==4.17.1 # via -r requirements-tests.in, factory-boy faker==4.17.1 # via -r requirements-tests.in, factory-boy
fakeredis==1.4.4 # via -r requirements-tests.in fakeredis==1.4.5 # via -r requirements-tests.in
flask==1.1.2 # via pytest-flask flask==1.1.2 # via pytest-flask
freezegun==1.0.0 # via -r requirements-tests.in freezegun==1.0.0 # via -r requirements-tests.in
future==0.18.2 # via aws-xray-sdk future==0.18.2 # via aws-xray-sdk

View File

@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via cloudflare beautifulsoup4==4.9.1 # via cloudflare
billiard==3.6.3.0 # via celery billiard==3.6.3.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.16.24 # via -r requirements.in boto3==1.16.25 # via -r requirements.in
botocore==1.19.24 # via -r requirements.in, boto3, s3transfer botocore==1.19.25 # via -r requirements.in, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.in celery[redis]==4.4.2 # via -r requirements.in
certifi==2020.11.8 # via -r requirements.in, requests certifi==2020.11.8 # via -r requirements.in, requests
certsrv==2.1.1 # via -r requirements.in certsrv==2.1.1 # via -r requirements.in
@ -67,7 +67,7 @@ pycryptodomex==3.9.7 # via pyjks
pyjks==20.0.0 # via -r requirements.in pyjks==20.0.0 # via -r requirements.in
pyjwt==1.7.1 # via -r requirements.in pyjwt==1.7.1 # via -r requirements.in
pynacl==1.3.0 # via paramiko pynacl==1.3.0 # via paramiko
pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests pyopenssl==20.0.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
pyrfc3339==1.1 # via acme pyrfc3339==1.1 # via acme
python-dateutil==2.8.1 # via alembic, arrow, botocore python-dateutil==2.8.1 # via alembic, arrow, botocore
python-editor==1.0.4 # via alembic python-editor==1.0.4 # via alembic