Merge branch 'master' into entrust_source

This commit is contained in:
Hossein Shafagh 2020-12-02 16:42:19 -08:00 committed by GitHub
commit a951e7623c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 283 additions and 25 deletions

View File

@ -115,10 +115,10 @@ endif
@echo "--> Updating Python requirements"
pip install --upgrade pip
pip install --upgrade pip-tools
pip-compile --output-file requirements.txt requirements.in -U --no-index
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-index
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-index
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-index
pip-compile --output-file requirements.txt requirements.in -U --no-emit-index-url
pip-compile --output-file requirements-docs.txt requirements-docs.in -U --no-emit-index-url
pip-compile --output-file requirements-dev.txt requirements-dev.in -U --no-emit-index-url
pip-compile --output-file requirements-tests.txt requirements-tests.in -U --no-emit-index-url
@echo "--> Done updating Python requirements"
@echo "--> Removing python-ldap from requirements-docs.txt"
grep -v "python-ldap" requirements-docs.txt > tempreqs && mv tempreqs requirements-docs.txt

View File

@ -21,6 +21,7 @@ from lemur.certificates.schemas import CertificateOutputSchema, CertificateInput
from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.endpoints import service as endpoint_service
from lemur.extensions import metrics, sentry, signals
from lemur.models import certificate_associations
from lemur.notifications.models import Notification
@ -797,3 +798,61 @@ def reissue_certificate(certificate, replace=None, user=None):
new_cert = create(**primitives)
return new_cert
def is_attached_to_endpoint(certificate_name, endpoint_name):
"""
Find if given certificate is attached to the endpoint. Both, certificate and endpoint, are identified by name.
This method talks to elb and finds the real time information.
:param certificate_name:
:param endpoint_name:
:return: True if certificate is attached to the given endpoint, False otherwise
"""
endpoint = endpoint_service.get_by_name(endpoint_name)
attached_certificates = endpoint.source.plugin.get_endpoint_certificate_names(endpoint)
return certificate_name in attached_certificates
def remove_from_destination(certificate, destination):
"""
Remove the certificate from given destination if clean() is implemented
:param certificate:
:param destination:
:return:
"""
plugin = plugins.get(destination.plugin_name)
if not hasattr(plugin, "clean"):
info_text = f"Cannot clean certificate {certificate.name}, {destination.plugin_name} plugin does not implement 'clean()'"
current_app.logger.warning(info_text)
else:
plugin.clean(certificate=certificate, options=destination.options)
def cleanup_after_revoke(certificate):
"""
Perform the needed cleanup for a revoked certificate. This includes -
1. Disabling notification
2. Disabling auto-rotation
3. Update certificate status to 'revoked'
4. Remove from AWS
:param certificate: Certificate object to modify and update in DB
:return: None
"""
certificate.notify = False
certificate.rotation = False
certificate.status = 'revoked'
error_message = ""
for destination in list(certificate.destinations):
try:
remove_from_destination(certificate, destination)
certificate.destinations.remove(destination)
except Exception as e:
# This cleanup is the best-effort since certificate is already revoked at this point.
# We will capture the exception and move on to the next destination
sentry.captureException()
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
database.update(certificate)
return error_message

View File

@ -19,6 +19,7 @@ from lemur.auth.permissions import AuthorityPermission, CertificatePermission
from lemur.certificates import service
from lemur.certificates.models import Certificate
from lemur.extensions import sentry
from lemur.plugins.base import plugins
from lemur.certificates.schemas import (
certificate_input_schema,
@ -888,8 +889,24 @@ class Certificates(AuthenticatedResource):
if cert.owner != data["owner"]:
service.cleanup_owner_roles_notification(cert.owner, data)
error_message = ""
# if destination is removed, cleanup the certificate from AWS
for destination in cert.destinations:
if destination not in data["destinations"]:
try:
service.remove_from_destination(cert, destination)
except Exception as e:
sentry.captureException()
# Add the removed destination back
data["destinations"].append(destination)
error_message = error_message + f"Failed to remove destination: {destination.label}. {str(e)}. "
# go ahead with DB update
cert = service.update(certificate_id, **data)
log_service.create(g.current_user, "update_cert", certificate=cert)
if error_message:
return dict(message=f"Edit Successful except -\n\n {error_message}"), 400
return cert
@validate_schema(certificate_edit_input_schema, certificate_output_schema)
@ -1433,6 +1450,8 @@ class CertificateRevoke(AuthenticatedResource):
return dict(message="Cannot revoke certificate. No external id found."), 400
if cert.endpoints:
for endpoint in cert.endpoints:
if service.is_attached_to_endpoint(cert.name, endpoint.name):
return (
dict(
message="Cannot revoke certificate. Endpoints are deployed with the given certificate."
@ -1442,7 +1461,13 @@ class CertificateRevoke(AuthenticatedResource):
plugin = plugins.get(cert.authority.plugin_name)
plugin.revoke_certificate(cert, data)
log_service.create(g.current_user, "revoke_cert", certificate=cert)
# Perform cleanup after revoke
error_message = service.cleanup_after_revoke(cert)
if error_message:
return dict(message=f"Certificate (id:{cert.id}) is revoked - {error_message}"), 400
return dict(id=cert.id)

View File

@ -149,6 +149,38 @@ def get_listener_arn_from_endpoint(endpoint_name, endpoint_port, **kwargs):
raise
@sts_client("elbv2")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=5)
def get_load_balancer_arn_from_endpoint(endpoint_name, **kwargs):
"""
Get a load balancer ARN from an endpoint.
:param endpoint_name:
:return:
"""
try:
client = kwargs.pop("client")
elbs = client.describe_load_balancers(Names=[endpoint_name])
if "LoadBalancers" in elbs and elbs["LoadBalancers"]:
return elbs["LoadBalancers"][0]["LoadBalancerArn"]
except Exception as e: # noqa
metrics.send(
"get_load_balancer_arn_from_endpoint",
"counter",
1,
metric_tags={
"error": str(e),
"endpoint_name": endpoint_name,
},
)
sentry.captureException(
extra={
"endpoint_name": str(endpoint_name),
}
)
raise
@sts_client("elb")
@retry(retry_on_exception=retry_throttled, wait_fixed=2000, stop_max_attempt_number=20)
def get_elbs(**kwargs):

View File

@ -300,6 +300,41 @@ class AWSSourcePlugin(SourcePlugin):
)
return None
def get_endpoint_certificate_names(self, endpoint):
options = endpoint.source.options
account_number = self.get_option("accountNumber", options)
region = get_region_from_dns(endpoint.dnsname)
certificate_names = []
if endpoint.type == "elb":
elb_details = elb.get_elbs(account_number=account_number,
region=region,
LoadBalancerNames=[endpoint.name],)
for lb_description in elb_details["LoadBalancerDescriptions"]:
for listener_description in lb_description["ListenerDescriptions"]:
listener = listener_description.get("Listener")
if not listener.get("SSLCertificateId"):
continue
certificate_names.append(iam.get_name_from_arn(listener.get("SSLCertificateId")))
elif endpoint.type == "elbv2":
listeners = elb.describe_listeners_v2(
account_number=account_number,
region=region,
LoadBalancerArn=elb.get_load_balancer_arn_from_endpoint(endpoint.name,
account_number=account_number,
region=region),
)
for listener in listeners["Listeners"]:
if not listener.get("Certificates"):
continue
for certificate in listener["Certificates"]:
certificate_names.append(iam.get_name_from_arn(certificate["CertificateArn"]))
return certificate_names
class AWSDestinationPlugin(DestinationPlugin):
title = "AWS"
@ -344,6 +379,10 @@ class AWSDestinationPlugin(DestinationPlugin):
def deploy(self, elb_name, account, region, certificate):
pass
def clean(self, certificate, options, **kwargs):
account_number = self.get_option("accountNumber", options)
iam.delete_cert(certificate.name, account_number=account_number)
class S3DestinationPlugin(ExportDestinationPlugin):
title = "AWS-S3"

View File

@ -1,5 +1,5 @@
import boto3
from moto import mock_sts, mock_elb
from moto import mock_sts, mock_ec2, mock_elb, mock_elbv2, mock_iam
@mock_sts()
@ -27,3 +27,107 @@ def test_get_all_elbs(app, aws_credentials):
elbs = get_all_elbs(account_number="123456789012", region="us-east-1")
assert elbs
@mock_sts()
@mock_ec2
@mock_elbv2()
@mock_iam
def test_create_elb_with_https_listener_miscellaneous(app, aws_credentials):
from lemur.plugins.lemur_aws import iam, elb
endpoint_name = "example-lbv2"
account_number = "123456789012"
region_ue1 = "us-east-1"
client = boto3.client("elbv2", region_name="us-east-1")
ec2 = boto3.resource("ec2", region_name="us-east-1")
# Create VPC
vpc = ec2.create_vpc(CidrBlock="172.28.7.0/24")
# Create LB (elbv2) in above VPC
assert create_load_balancer(client, ec2, vpc.id, endpoint_name)
# Create target group
target_group_arn = create_target_group(client, vpc.id)
assert target_group_arn
# Test get_load_balancer_arn_from_endpoint
lb_arn = elb.get_load_balancer_arn_from_endpoint(endpoint_name,
account_number=account_number,
region=region_ue1)
assert lb_arn
# Test describe_listeners_v2
listeners = elb.describe_listeners_v2(account_number=account_number,
region=region_ue1,
LoadBalancerArn=lb_arn)
assert listeners
assert not listeners["Listeners"]
# Upload cert
response = iam.upload_cert("LemurTestCert", "testCert", "cert1", "cert2",
account_number=account_number)
assert response
cert_arn = response["ServerCertificateMetadata"]["Arn"]
assert cert_arn
# Create https listener using above cert
listeners = client.create_listener(
LoadBalancerArn=lb_arn,
Protocol="HTTPS",
Port=443,
Certificates=[{"CertificateArn": cert_arn}],
DefaultActions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
)
assert listeners
listener_arn = listeners["Listeners"][0]["ListenerArn"]
assert listener_arn
assert listeners["Listeners"]
for listener in listeners["Listeners"]:
if listener["Port"] == 443:
assert listener["Certificates"]
assert cert_arn == listener["Certificates"][0]["CertificateArn"]
# Test get_listener_arn_from_endpoint
assert listener_arn == elb.get_listener_arn_from_endpoint(
endpoint_name,
443,
account_number=account_number,
region=region_ue1,
)
@mock_sts()
@mock_elb()
def test_get_all_elbs_v2():
from lemur.plugins.lemur_aws.elb import get_all_elbs_v2
elbs = get_all_elbs_v2(account_number="123456789012",
region="us-east-1")
assert elbs
def create_load_balancer(client, ec2, vpc_id, endpoint_name):
subnet1 = ec2.create_subnet(
VpcId=vpc_id,
CidrBlock="172.28.7.192/26",
AvailabilityZone="us-east-1a"
)
return client.create_load_balancer(
Name=endpoint_name,
Subnets=[
subnet1.id,
],
)
def create_target_group(client, vpc_id):
response = client.create_target_group(
Name="a-target",
Protocol="HTTPS",
Port=443,
VpcId=vpc_id,
)
return response.get("TargetGroups")[0]["TargetGroupArn"]

View File

@ -33,7 +33,7 @@
<div class="col-sm-10">
<select class="form-control" ng-model="certificate.keyType"
ng-options="option for option in ['RSA2048', 'RSA4096', 'ECCPRIME256V1', 'ECCSECP384R1']"
ng-init="certificate.keyType = 'RSA2048'"></select>
ng-init="certificate.keyType = 'ECCPRIME256V1'"></select>
</div>
</div>
<div class="form-group">

View File

@ -26,9 +26,8 @@
</div>
</form>
<div ng-if="certificate.endpoints.length">
<p><strong>Certificate cannot be revoked, it is associated with the following endpoints. Disassociate this
certificate
before revoking.</strong></p>
<p><strong>Certificate might be associated with the following endpoints. Disassociate this
certificate before revoking or continue if you've already done so.</strong></p>
<ul class="list-group">
<li class="list-group-item" ng-repeat="endpoint in certificate.endpoints">
<span class="pull-right"><label class="label label-default">{{ endpoint.type }}</label></span>

View File

@ -24,7 +24,7 @@ keyring==21.2.0 # via twine
mccabe==0.6.1 # via flake8
nodeenv==1.5.0 # via -r requirements-dev.in, pre-commit
pkginfo==1.5.0.1 # via twine
pre-commit==2.9.0 # via -r requirements-dev.in
pre-commit==2.9.2 # via -r requirements-dev.in
pycodestyle==2.6.0 # via flake8
pycparser==2.20 # via cffi
pyflakes==2.2.0 # via flake8

View File

@ -17,8 +17,8 @@ bcrypt==3.1.7 # via -r requirements.txt, flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via -r requirements.txt, cloudflare
billiard==3.6.3.0 # via -r requirements.txt, celery
blinker==1.4 # via -r requirements.txt, flask-mail, flask-principal, raven
boto3==1.16.24 # via -r requirements.txt
botocore==1.19.24 # via -r requirements.txt, boto3, s3transfer
boto3==1.16.25 # via -r requirements.txt
botocore==1.19.25 # via -r requirements.txt, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.txt
certifi==2020.11.8 # via -r requirements.txt, requests
certsrv==2.1.1 # via -r requirements.txt
@ -73,7 +73,7 @@ pygments==2.6.1 # via sphinx
pyjks==20.0.0 # via -r requirements.txt
pyjwt==1.7.1 # via -r requirements.txt
pynacl==1.3.0 # via -r requirements.txt, paramiko
pyopenssl==19.1.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
pyopenssl==20.0.0 # via -r requirements.txt, acme, josepy, ndg-httpsclient, requests
pyparsing==2.4.7 # via packaging
pyrfc3339==1.1 # via -r requirements.txt, acme
python-dateutil==2.8.1 # via -r requirements.txt, alembic, arrow, botocore

View File

@ -10,9 +10,9 @@ aws-sam-translator==1.22.0 # via cfn-lint
aws-xray-sdk==2.5.0 # via moto
bandit==1.6.2 # via -r requirements-tests.in
black==20.8b1 # via -r requirements-tests.in
boto3==1.16.24 # via aws-sam-translator, moto
boto3==1.16.25 # via aws-sam-translator, moto
boto==2.49.0 # via moto
botocore==1.19.24 # via aws-xray-sdk, boto3, moto, s3transfer
botocore==1.19.25 # via aws-xray-sdk, boto3, moto, s3transfer
certifi==2020.11.8 # via requests
cffi==1.14.0 # via cryptography
cfn-lint==0.29.5 # via moto
@ -25,7 +25,7 @@ docker==4.2.0 # via moto
ecdsa==0.14.1 # via moto, python-jose, sshpubkeys
factory-boy==3.1.0 # via -r requirements-tests.in
faker==4.17.1 # via -r requirements-tests.in, factory-boy
fakeredis==1.4.4 # via -r requirements-tests.in
fakeredis==1.4.5 # via -r requirements-tests.in
flask==1.1.2 # via pytest-flask
freezegun==1.0.0 # via -r requirements-tests.in
future==0.18.2 # via aws-xray-sdk

View File

@ -15,8 +15,8 @@ bcrypt==3.1.7 # via flask-bcrypt, paramiko
beautifulsoup4==4.9.1 # via cloudflare
billiard==3.6.3.0 # via celery
blinker==1.4 # via flask-mail, flask-principal, raven
boto3==1.16.24 # via -r requirements.in
botocore==1.19.24 # via -r requirements.in, boto3, s3transfer
boto3==1.16.25 # via -r requirements.in
botocore==1.19.25 # via -r requirements.in, boto3, s3transfer
celery[redis]==4.4.2 # via -r requirements.in
certifi==2020.11.8 # via -r requirements.in, requests
certsrv==2.1.1 # via -r requirements.in
@ -67,7 +67,7 @@ pycryptodomex==3.9.7 # via pyjks
pyjks==20.0.0 # via -r requirements.in
pyjwt==1.7.1 # via -r requirements.in
pynacl==1.3.0 # via paramiko
pyopenssl==19.1.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
pyopenssl==20.0.0 # via -r requirements.in, acme, josepy, ndg-httpsclient, requests
pyrfc3339==1.1 # via acme
python-dateutil==2.8.1 # via alembic, arrow, botocore
python-editor==1.0.4 # via alembic