Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into improved_verify

Browse Source
This commit is contained in:
Non Sequitur
2018-09-27 11:19:06 -04:00
parent 590fac4aa8 c7b57e21a5
commit 50919d85a8
17 changed files with 261 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lemur/certificates/schemas.py
View File

@ -235,8 +235,9 @@ class CertificateOutputSchema(LemurOutputSchema):
class CertificateUploadInputSchema(CertificateCreationSchema):
name = fields.String()
authority = fields.Nested(AssociatedAuthoritySchema, required=False)
notify = fields.Boolean(missing=True)
external_id = fields.String(missing=None, allow_none=True)
private_key = fields.String(validate=validators.private_key)
body = fields.String(required=True, validate=validators.public_certificate)
chain = fields.String(validate=validators.public_certificate, missing=None,

43
lemur/certificates/service.py
View File

@ -6,31 +6,25 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
from flask import current_app
from sqlalchemy import func, or_, not_, cast, Integer
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from flask import current_app
from sqlalchemy import func, or_, not_, cast, Integer
from lemur import database
from lemur.extensions import metrics, sentry, signals
from lemur.plugins.base import plugins
from lemur.common.utils import generate_private_key, truthiness
from lemur.roles.models import Role
from lemur.domains.models import Domain
from lemur.authorities.models import Authority
from lemur.destinations.models import Destination
from lemur.certificates.models import Certificate
from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
from lemur.common.utils import generate_private_key, truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.extensions import metrics, sentry, signals
from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate
from lemur.certificates.schemas import CertificateOutputSchema, CertificateInputSchema
from lemur.plugins.base import plugins
from lemur.roles import service as role_service
from lemur.roles.models import Role
csr_created = signals.signal('csr_created', "CSR generated")
csr_imported = signals.signal('csr_imported', "CSR imported from external source")
@ -95,8 +89,8 @@ def get_all_pending_cleaning(source):
:param source:
:return:
"""
return Certificate.query.filter(Certificate.sources.any(id=source.id))\
.filter(not_(Certificate.endpoints.any())).all()
return Certificate.query.filter(Certificate.sources.any(id=source.id)) \
.filter(not_(Certificate.endpoints.any())).all()
def get_all_pending_reissue():
@ -109,8 +103,8 @@ def get_all_pending_reissue():
:return:
"""
return Certificate.query.filter(Certificate.rotation == True)\
.filter(not_(Certificate.replaced.any()))\
return Certificate.query.filter(Certificate.rotation == True) \
.filter(not_(Certificate.replaced.any())) \
.filter(Certificate.in_rotation_window == True).all() # noqa
@ -233,7 +227,7 @@ def upload(**kwargs):
kwargs['private_key'] = private_key.encode('utf-8')
cert = Certificate(**kwargs)
cert.authority = kwargs.get('authority')
cert = database.create(cert)
kwargs['creator'].certificates.append(cert)
@ -280,6 +274,7 @@ def create(**kwargs):
if isinstance(cert, Certificate):
certificate_issued.send(certificate=cert, authority=cert.authority)
metrics.send('certificate_issued', 'counter', 1, metric_tags=dict(owner=cert.owner, issuer=cert.issuer))
return cert
@ -310,8 +305,8 @@ def render(args):
if 'issuer' in terms:
# we can't rely on issuer being correct in the cert directly so we combine queries
sub_query = database.session_query(Authority.id)\
.filter(Authority.name.ilike(term))\
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike(term)) \
.subquery()
query = query.filter(
@ -450,8 +445,8 @@ def stats(**kwargs):
if kwargs.get('metric') == 'not_after':
start = arrow.utcnow()
end = start.replace(weeks=+32)
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id))\
.group_by(Certificate.issuer)\
items = database.db.session.query(Certificate.issuer, func.count(Certificate.id)) \
.group_by(Certificate.issuer) \
.filter(Certificate.not_after <= end.format('YYYY-MM-DD')) \
.filter(Certificate.not_after >= start.format('YYYY-MM-DD')).all()

135
lemur/common/celery.py Normal file
View File

@ -0,0 +1,135 @@
"""
This module controls defines celery tasks and their applicable schedules. The celery beat server and workers will start
when invoked.
When ran in development mode (LEMUR_CONFIG=<location of development configuration file. To run both the celery
beat scheduler and a worker simultaneously, and to have jobs kick off starting at the next minute, run the following
command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
"""
import copy
import datetime
import sys
from datetime import timezone
from celery import Celery
from flask import current_app
from lemur.authorities.service import get as get_authority
from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
flask_app = create_app()
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
celery = make_celery(flask_app)
@celery.task()
def fetch_acme_cert(id):
"""
Attempt to get the full certificate for the pending certificate listed.
Args:
id: an id of a PendingCertificate
"""
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
}
pending_certs = pending_certificate_service.get_pending_certs([id])
user = user_service.get_by_username('lemur')
new = 0
failed = 0
wrong_issuer = 0
acme_certs = []
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
acme_certs.append(cert)
else:
wrong_issuer += 1
authority = plugins.get("acme-issuer")
resolved_certs = authority.get_ordered_certificates(acme_certs)
for cert in resolved_certs:
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# add metrics to metrics extension
new += 1
else:
failed += 1
error_log = copy.deepcopy(log_data)
error_log["message"] = "Pending certificate creation failure"
error_log["pending_cert_id"] = pending_cert.id
error_log["last_error"] = cert.get("last_error")
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(
cert.get("pending_cert").id,
status=str(cert.get("last_error"))
)
# Add failed pending cert task back to queue
fetch_acme_cert.delay(id)
current_app.logger.error(error_log)
log_data["message"] = "Complete"
log_data["new"] = new
log_data["failed"] = failed
log_data["wrong_issuer"] = wrong_issuer
current_app.logger.debug(log_data)
print(
"[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
new=new,
failed=failed,
wrong_issuer=wrong_issuer
)
)
@celery.task()
def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_pending_certs('all')
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert.last_updated == cert.date_created or datetime.datetime.now(
timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
fetch_acme_cert.delay(cert.id)

4
lemur/common/defaults.py
View File

@ -232,8 +232,8 @@ def issuer(cert):
delchars = ''.join(c for c in map(chr, range(256)) if not c.isalnum())
try:
# Try organization name or fall back to CN
issuer = (cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME)
or cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME))
issuer = (cert.issuer.get_attributes_for_oid(x509.OID_COMMON_NAME) or
cert.issuer.get_attributes_for_oid(x509.OID_ORGANIZATION_NAME))
issuer = str(issuer[0].value)
for c in delchars:
issuer = issuer.replace(c, "")

24
lemur/migrations/versions/9392b9f9a805_.py Normal file
View File

@ -0,0 +1,24 @@
"""Add last_updated field to Pending Certs
Revision ID: 9392b9f9a805
Revises: 5ae0ecefb01f
Create Date: 2018-09-17 08:33:37.087488
"""
# revision identifiers, used by Alembic.
revision = '9392b9f9a805'
down_revision = '5ae0ecefb01f'
from alembic import op
from sqlalchemy_utils import ArrowType
import sqlalchemy as sa
def upgrade():
op.add_column('pending_certs', sa.Column('last_updated', ArrowType, server_default=sa.text('now()'), onupdate=sa.text('now()'),
nullable=False))
def downgrade():
op.drop_column('pending_certs', 'last_updated')

6
lemur/notifications/messaging.py
View File

@ -165,6 +165,7 @@ def send_rotation_notification(certificate, notification_plugin=None):
notification_plugin.send('rotation', data, [data['owner']])
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.error('Unable to send notification to {}.'.format(data['owner']), exc_info=True)
sentry.captureException()
metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': 'rotation'})
@ -196,6 +197,8 @@ def send_pending_failure_notification(pending_cert, notify_owner=True, notify_se
notification_plugin.send('failed', data, [data['owner']], pending_cert)
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.error('Unable to send pending failure notification to {}.'.format(data['owner']),
exc_info=True)
sentry.captureException()
if notify_security:
@ -203,6 +206,9 @@ def send_pending_failure_notification(pending_cert, notify_owner=True, notify_se
notification_plugin.send('failed', data, data["security_email"], pending_cert)
status = SUCCESS_METRIC_STATUS
except Exception as e:
current_app.logger.error('Unable to send pending failure notification to '
'{}.'.format(data['security_email']),
exc_info=True)
sentry.captureException()
metrics.send('notification', 'counter', 1, metric_tags={'status': status, 'event_type': 'rotation'})

19
lemur/pending_certificates/models.py
View File

@ -5,19 +5,18 @@
"""
from datetime import datetime as dt
from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
from sqlalchemy_utils.types.arrow import ArrowType
from sqlalchemy.orm import relationship
from sqlalchemy_utils import JSONType
from sqlalchemy_utils.types.arrow import ArrowType
from lemur.certificates.models import get_or_increase_name
from lemur.common import defaults, utils
from lemur.database import db
from lemur.utils import Vault
from lemur.models import pending_cert_source_associations, \
pending_cert_destination_associations, pending_cert_notification_associations, \
pending_cert_replacement_associations, pending_cert_role_associations
from lemur.utils import Vault
class PendingCertificate(db.Model):
@ -40,6 +39,7 @@ class PendingCertificate(db.Model):
dns_provider_id = Column(Integer, ForeignKey('dns_providers.id', ondelete="CASCADE"))
status = Column(Text(), nullable=True)
last_updated = Column(ArrowType, PassiveDefault(func.now()), onupdate=func.now(), nullable=False)
rotation = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('users.id'))
@ -47,9 +47,12 @@ class PendingCertificate(db.Model):
root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
notifications = relationship('Notification', secondary=pending_cert_notification_associations, backref='pending_cert', passive_deletes=True)
destinations = relationship('Destination', secondary=pending_cert_destination_associations, backref='pending_cert', passive_deletes=True)
sources = relationship('Source', secondary=pending_cert_source_associations, backref='pending_cert', passive_deletes=True)
notifications = relationship('Notification', secondary=pending_cert_notification_associations,
backref='pending_cert', passive_deletes=True)
destinations = relationship('Destination', secondary=pending_cert_destination_associations, backref='pending_cert',
passive_deletes=True)
sources = relationship('Source', secondary=pending_cert_source_associations, backref='pending_cert',
passive_deletes=True)
roles = relationship('Role', secondary=pending_cert_role_associations, backref='pending_cert', passive_deletes=True)
replaces = relationship('Certificate',
secondary=pending_cert_replacement_associations,
@ -77,7 +80,7 @@ class PendingCertificate(db.Model):
# TODO: Fix auto-generated name, it should be renamed on creation
self.name = get_or_increase_name(
defaults.certificate_name(kwargs['common_name'], kwargs['authority'].name,
dt.now(), dt.now(), False), self.external_id)
dt.now(), dt.now(), False), self.external_id)
self.rename = True
self.cn = defaults.common_name(utils.parse_csr(self.csr))

24
lemur/pending_certificates/schemas.py
View File

@ -1,5 +1,14 @@
from marshmallow import fields, post_load
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates.schemas import CertificateNestedOutputSchema
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.notifications import service as notification_service
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.schemas import (
AssociatedCertificateSchema,
AssociatedDestinationSchema,
@ -8,18 +17,7 @@ from lemur.schemas import (
EndpointNestedOutputSchema,
ExtensionSchema
)
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.users.schemas import UserNestedOutputSchema
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.certificates.schemas import CertificateNestedOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.notifications import service as notification_service
class PendingCertificateSchema(LemurInputSchema):
@ -38,6 +36,7 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
name = fields.String()
number_attempts = fields.Integer()
date_created = fields.Date()
last_updated = fields.Date()
rotation = fields.Boolean()
@ -88,7 +87,8 @@ class PendingCertificateEditInputSchema(PendingCertificateSchema):
"""
if data['owner']:
notification_name = "DEFAULT_{0}".format(data['owner'].split('@')[0].upper())
data['notifications'] += notification_service.create_default_expiration_notifications(notification_name, [data['owner']])
data['notifications'] += notification_service.create_default_expiration_notifications(notification_name,
[data['owner']])
return data

10
lemur/tests/test_certificates.py
View File

@ -61,7 +61,7 @@ def test_certificate_output_schema(session, certificate, issuer_plugin):
# Make sure serialization parses the cert only once (uses cached 'parsed_cert' attribute)
with patch('lemur.common.utils.parse_certificate', side_effect=utils.parse_certificate) as wrapper:
data, errors = CertificateOutputSchema().dump(certificate)
assert data['issuer'] == 'LemurTrustEnterprisesLtd'
assert data['issuer'] == 'LemurTrustUnittestsClass1CA2018'
assert wrapper.call_count == 1
@ -455,8 +455,8 @@ def test_create_certificate(issuer_plugin, authority, user):
cert = create(authority=authority, csr=CSR_STR, owner='joe@example.com', creator=user['user'])
assert str(cert.not_after) == '2047-12-31T22:00:00+00:00'
assert str(cert.not_before) == '2017-12-31T22:00:00+00:00'
assert cert.issuer == 'LemurTrustEnterprisesLtd'
assert cert.name == 'SAN-san.example.org-LemurTrustEnterprisesLtd-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333'
assert cert.issuer == 'LemurTrustUnittestsClass1CA2018'
assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333'
cert = create(authority=authority, csr=CSR_STR, owner='joe@example.com', name='ACustomName1', creator=user['user'])
assert cert.name == 'ACustomName1'
@ -486,8 +486,8 @@ def test_import(user):
cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, creator=user['user'])
assert str(cert.not_after) == '2047-12-31T22:00:00+00:00'
assert str(cert.not_before) == '2017-12-31T22:00:00+00:00'
assert cert.issuer == 'LemurTrustEnterprisesLtd'
assert cert.name == 'SAN-san.example.org-LemurTrustEnterprisesLtd-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2'
assert cert.issuer == 'LemurTrustUnittestsClass1CA2018'
assert cert.name == 'SAN-san.example.org-LemurTrustUnittestsClass1CA2018-20171231-20471231-AFF2DB4F8D2D4D8E80FA382AE27C2333-2'
cert = import_certificate(body=SAN_CERT_STR, chain=INTERMEDIATE_CERT_STR, private_key=SAN_CERT_KEY, owner='joe@example.com', name='ACustomName2', creator=user['user'])
assert cert.name == 'ACustomName2'

2
lemur/tests/test_defaults.py
View File

@ -35,7 +35,7 @@ def test_cert_bitstrength(client):
def test_cert_issuer(client):
from lemur.common.defaults import issuer
assert issuer(INTERMEDIATE_CERT) == 'LemurTrustEnterprisesLtd'
assert issuer(INTERMEDIATE_CERT) == 'LemurTrustUnittestsRootCA2018'
def test_text_to_slug(client):

2
lemur/tests/test_pending_certificates.py
View File

@ -26,7 +26,7 @@ def test_create_pending(pending_certificate, user, session):
from lemur.pending_certificates.service import create_certificate, get
cert = {'body': WILDCARD_CERT_STR,
'chain': INTERMEDIATE_CERT_STR,
'external_id': 54321}
'external_id': '54321'}
# Weird copy because the session behavior. pending_certificate is a valid object but the
# return of vars(pending_certificate) is a sessionobject, and so nothing from the pending_cert