Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding the ability to specify a per-certificate rotation policy. (#851)

This commit is contained in:
kevgliss 2017-07-12 16:46:11 -07:00 committed by GitHub
parent 560bd5a872
commit 443eb43d1f
18 changed files with 291 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.pre-commit-config.yaml
View File

@ -1,10 +1,10 @@
- repo: git://github.com/pre-commit/pre-commit-hooks
sha: 18d7035de5388cc7775be57f529c154bf541aab9
sha: v0.9.1
hooks:
- id: trailing-whitespace
- id: flake8
- id: check-merge-conflict
- repo: git://github.com/pre-commit/mirrors-jshint
sha: e72140112bdd29b18b0c8257956c896c4c3cebcb
sha: v2.9.5
hooks:
- id: jshint

4
CHANGELOG.rst
View File

@ -4,6 +4,10 @@ Changelog
0.6 - `master`
~~~~~~~~~~~~~~
Adds per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates
is 30 days. Every certificate will gain a policy regardless is auto-rotation is used.
.. note:: This version is not yet released and is under active development

51
lemur/certificates/models.py
View File

@ -6,6 +6,7 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
from datetime import timedelta
from flask import current_app
@ -15,7 +16,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from idna.core import InvalidCodepoint
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import case
from sqlalchemy.sql.expression import case, extract
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
@ -37,6 +38,7 @@ from lemur.models import certificate_associations, certificate_source_associatio
certificate_replacement_associations, roles_certificates
from lemur.domains.models import Domain
from lemur.policies.models import RotationPolicy
def get_sequence(name):
@ -102,10 +104,10 @@ class Certificate(db.Model):
san = Column(String(1024)) # TODO this should be migrated to boolean
rotation = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('users.id'))
authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
@ -120,6 +122,7 @@ class Certificate(db.Model):
logs = relationship('Log', backref='certificate')
endpoints = relationship('Endpoint', backref='certificate')
rotation_policy = relationship("RotationPolicy")
def __init__(self, **kwargs):
cert = lemur.common.utils.parse_certificate(kwargs['body'])
@ -134,7 +137,8 @@ class Certificate(db.Model):
if kwargs.get('name'):
self.name = get_or_increase_name(kwargs['name'])
else:
self.name = get_or_increase_name(defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
self.name = get_or_increase_name(
defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
self.owner = kwargs['owner']
self.body = kwargs['body'].strip()
@ -152,6 +156,7 @@ class Certificate(db.Model):
self.roles = list(set(kwargs.get('roles', [])))
self.replaces = kwargs.get('replaces', [])
self.rotation = kwargs.get('rotation')
self.rotation_policy = kwargs.get('rotation_policy')
self.signing_algorithm = defaults.signing_algorithm(cert)
self.bits = defaults.bitstrength(cert)
self.serial = defaults.serial(cert)
@ -240,6 +245,33 @@ class Certificate(db.Model):
else_=False
)
@hybrid_property
def in_rotation_window(self):
"""
Determines if a certificate is available for rotation based
on the rotation policy associated.
:return:
"""
now = arrow.utcnow()
end = now + timedelta(days=self.rotation_policy.days)
if self.not_after <= end:
return True
@in_rotation_window.expression
def in_rotation_window(cls):
"""
Determines if a certificate is available for rotation based
on the rotation policy associated.
:return:
"""
return case(
[
(extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
],
else_=False
)
@property
def extensions(self):
# setup default values
@ -298,16 +330,6 @@ class Certificate(db.Model):
return return_extensions
def get_arn(self, account_number):
"""
Generate a valid AWS IAM arn
:rtype : str
:param account_number:
:return:
"""
return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
def __repr__(self):
return "Certificate(name={name})".format(name=self.name)
@ -329,7 +351,8 @@ def update_destinations(target, value, initiator):
destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
except Exception as e:
current_app.logger.exception(e)
metrics.send('destination_upload_failure', 'counter', 1, metric_tags={'certificate': target.name, 'destination': value.label})
metrics.send('destination_upload_failure', 'counter', 1,
metric_tags={'certificate': target.name, 'destination': value.label})
@event.listens_for(Certificate.replaces, 'append')

17
lemur/certificates/schemas.py
View File

@ -9,8 +9,17 @@ from flask import current_app
from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.schemas import AssociatedAuthoritySchema, AssociatedDestinationSchema, AssociatedCertificateSchema, \
AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema, EndpointNestedOutputSchema
from lemur.schemas import (
AssociatedAuthoritySchema,
AssociatedDestinationSchema,
AssociatedCertificateSchema,
AssociatedNotificationSchema,
PluginInputSchema,
ExtensionSchema,
AssociatedRoleSchema,
EndpointNestedOutputSchema,
AssociatedRotationPolicySchema
)
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
@ -18,6 +27,7 @@ from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.users.schemas import UserNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.common import validators, missing
@ -63,6 +73,7 @@ class CertificateInputSchema(CertificateCreationSchema):
notify = fields.Boolean(default=True)
rotation = fields.Boolean()
rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, default={'name': 'default'})
# certificate body fields
organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
@ -133,6 +144,7 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
rotation = fields.Boolean()
notify = fields.Boolean()
rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
# Note aliasing is the first step in deprecating these fields.
cn = fields.String() # deprecated
@ -198,6 +210,7 @@ class CertificateOutputSchema(LemurOutputSchema):
roles = fields.Nested(RoleNestedOutputSchema, many=True)
endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
class CertificateUploadInputSchema(CertificateCreationSchema):

12
lemur/certificates/service.py
View File

@ -6,7 +6,6 @@
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
from datetime import timedelta
from flask import current_app
from sqlalchemy import func, or_, not_, cast, Boolean, Integer
@ -85,20 +84,15 @@ def get_all_pending_reissue():
"""
Retrieves all certificates that need to be rotated.
Must be X days from expiration, uses `LEMUR_DEFAULT_ROTATION_INTERVAL`
to determine how many days from expiration the certificate must be
Must be X days from expiration, uses the certificates rotation
policy to determine how many days from expiration the certificate must be
for rotation to be pending.
:return:
"""
now = arrow.utcnow()
interval = current_app.config.get('LEMUR_DEFAULT_ROTATION_INTERVAL', 30)
end = now + timedelta(days=interval)
return Certificate.query.filter(Certificate.rotation == True)\
.filter(Certificate.endpoints.any())\
.filter(not_(Certificate.replaced.any()))\
.filter(Certificate.not_after <= end.format('YYYY-MM-DD')).all() # noqa
.filter(Certificate.in_rotation_window == True).all() # noqa
def find_duplicates(cert):

29
lemur/certificates/views.py
View File

@ -232,6 +232,8 @@ class CertificatesList(AuthenticatedResource):
"replaces": [{
"id": 1
}],
"rotation": True,
"rotationPolicy": {"name": "default"},
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,
@ -241,18 +243,6 @@ class CertificatesList(AuthenticatedResource):
"san": null
}
:arg extensions: extensions to be used in the certificate
:arg description: description for new certificate
:arg owner: owner email
:arg validityStart: when the certificate should start being valid
:arg validityEnd: when the certificate should expire
:arg authority: authority that should issue the certificate
:arg country: country for the CSR
:arg state: state for the CSR
:arg location: location for the CSR
:arg organization: organization for CSR
:arg commonName: certificate common name
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
@ -356,6 +346,8 @@ class CertificatesUpload(AuthenticatedResource):
"name": "*.test.example.net"
}],
"replaces": [],
"rotation": True,
"rotationPolicy": {"name": "default"},
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,
@ -365,11 +357,6 @@ class CertificatesUpload(AuthenticatedResource):
"san": null
}
:arg owner: owner email for certificate
:arg publicCert: valid PEM public key for certificate
:arg intermediateCert valid PEM intermediate key for certificate
:arg privateKey: valid PEM private key for certificate
:arg destinations: list of aws destinations to upload the certificate to
:reqheader Authorization: OAuth token to authenticate
:statuscode 403: unauthenticated
:statuscode 200: no error
@ -522,6 +509,8 @@ class Certificates(AuthenticatedResource):
"id": 1090,
"name": "*.test.example.net"
}],
"rotation": True,
"rotationPolicy": {"name": "default"},
"replaces": [],
"replaced": [],
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
@ -616,6 +605,8 @@ class Certificates(AuthenticatedResource):
"description": "This is a google group based role created by Lemur",
"name": "joe@example.com"
}],
"rotation": True,
"rotationPolicy": {"name": "default"},
"san": null
}
@ -722,6 +713,8 @@ class NotificationCertificatesList(AuthenticatedResource):
}],
"replaces": [],
"replaced": [],
"rotation": True,
"rotationPolicy": {"name": "default"},
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,
@ -827,6 +820,8 @@ class CertificatesReplacementsList(AuthenticatedResource):
}],
"replaces": [],
"replaced": [],
"rotation": True,
"rotationPolicy": {"name": "default"},
"name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
"roles": [{
"id": 464,

1
lemur/deployment/service.py
View File

@ -10,7 +10,6 @@ def rotate_certificate(endpoint, new_cert):
:return:
"""
# ensure that certificate is available for rotation
endpoint.source.plugin.update_endpoint(endpoint, new_cert)
endpoint.certificate = new_cert
database.update(endpoint)

17
lemur/manage.py
View File

@ -16,16 +16,18 @@ from flask_migrate import Migrate, MigrateCommand, stamp
from flask_script.commands import ShowUrls, Clean, Server
from lemur.sources.cli import manager as source_manager
from lemur.policies.cli import manager as policy_manager
from lemur.reporting.cli import manager as report_manager
from lemur.endpoints.cli import manager as endpoint_manager
from lemur.certificates.cli import manager as certificate_manager
from lemur.notifications.cli import manager as notification_manager
from lemur.endpoints.cli import manager as endpoint_manager
from lemur.reporting.cli import manager as report_manager
from lemur import database
from lemur.users import service as user_service
from lemur.roles import service as role_service
from lemur.policies import service as policy_service
from lemur.notifications import service as notification_service
from lemur.common.utils import validate_conf
from lemur import create_app
@ -40,6 +42,8 @@ from lemur.domains.models import Domain # noqa
from lemur.notifications.models import Notification # noqa
from lemur.sources.models import Source # noqa
from lemur.logs.models import Log # noqa
from lemur.endpoints.models import Endpoint # noqa
from lemur.policies.models import RotationPolicy # noqa
manager = Manager(create_app)
@ -242,6 +246,12 @@ class InitializeApp(Command):
recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients)
days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
days=days
))
policy_service.create(days=days)
sys.stdout.write("[/] Done!\n")
@ -531,6 +541,7 @@ def main():
manager.add_command("notify", notification_manager)
manager.add_command("endpoint", endpoint_manager)
manager.add_command("report", report_manager)
manager.add_command("policy", policy_manager)
manager.run()

54
lemur/migrations/versions/a02a678ddc25_.py Normal file
View File

@ -0,0 +1,54 @@
"""Adds support for rotation policies.
Creates a default rotation policy (30 days) with the name
'default' ensures that all existing certificates use the default
policy.
Revision ID: a02a678ddc25
Revises: 8ae67285ff14
Create Date: 2017-07-12 11:45:49.257927
"""
# revision identifiers, used by Alembic.
revision = 'a02a678ddc25'
down_revision = '8ae67285ff14'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.sql import text
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('rotation_policies',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=True),
sa.Column('days', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.add_column('certificates', sa.Column('rotation_policy_id', sa.Integer(), nullable=True))
op.create_foreign_key(None, 'certificates', 'rotation_policies', ['rotation_policy_id'], ['id'])
conn = op.get_bind()
stmt = text('insert into rotation_policies (days, name) values (:days, :name)')
stmt = stmt.bindparams(days=30, name='default')
conn.execute(stmt)
stmt = text('select id from rotation_policies where name=:name')
stmt = stmt.bindparams(name='default')
rotation_policy_id = conn.execute(stmt).fetchone()[0]
stmt = text('update certificates set rotation_policy_id=:rotation_policy_id')
stmt = stmt.bindparams(rotation_policy_id=rotation_policy_id)
conn.execute(stmt)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, 'certificates', type_='foreignkey')
op.drop_column('certificates', 'rotation_policy_id')
op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations')
op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['replaced_certificate_id', 'certificate_id'], unique=True)
op.drop_table('rotation_policies')
# ### end Alembic commands ###

0
lemur/policies/__init__.py Normal file
View File

24
lemur/policies/cli.py Normal file
View File

@ -0,0 +1,24 @@
"""
.. module: lemur.policies.cli
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from flask_script import Manager
from lemur.policies import service as policy_service
manager = Manager(usage="Handles all policy related tasks.")
@manager.option('-d', '--days', dest='days', help='Number of days before expiration.')
@manager.option('-n', '--name', dest='name', help='Policy name.')
def create(days, name):
"""
Create a new certificate rotation policy
:return:
"""
print("[+] Creating a new certificate rotation policy.")
policy_service.create(days=days, name=name)
print("[+] Successfully created a new certificate rotation policy")

21
lemur/policies/models.py Normal file
View File

@ -0,0 +1,21 @@
"""
.. module: lemur.policies.models
:platform: unix
:synopsis: This module contains all of the models need to create a certificate policy within Lemur.
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from sqlalchemy import Column, Integer, String
from lemur.database import db
class RotationPolicy(db.Model):
__tablename__ = 'rotation_policies'
id = Column(Integer, primary_key=True)
name = Column(String)
days = Column(Integer)
def __repr__(self):
return "RotationPolicy(days={days}, name={name})".format(days=self.days, name=self.name)

19
lemur/policies/schemas.py Normal file
View File

@ -0,0 +1,19 @@
"""
.. module: lemur.policies.schemas
:platform: unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from marshmallow import fields
from lemur.common.schema import LemurOutputSchema
class RotationPolicyOutputSchema(LemurOutputSchema):
id = fields.Integer()
days = fields.Integer()
class RotationPolicyNestedOutputSchema(RotationPolicyOutputSchema):
pass

62
lemur/policies/service.py Normal file
View File

@ -0,0 +1,62 @@
"""
.. module: lemur.policies.service
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
from lemur import database
from lemur.policies.models import RotationPolicy
def get(policy_id):
"""
Retrieves policy by its ID.
:param policy_id:
:return:
"""
return database.get(RotationPolicy, policy_id)
def delete(policy_id):
"""
Delete a rotation policy.
:param policy_id:
:return:
"""
database.delete(get(policy_id))
def get_all_policies():
"""
Retrieves all rotation policies.
:return:
"""
return RotationPolicy.query.all()
def create(**kwargs):
"""
Creates a new rotation policy.
:param kwargs:
:return:
"""
policy = RotationPolicy(**kwargs)
database.create(policy)
return policy
def update(policy_id, **kwargs):
"""
Updates a policy.
:param policy_id:
:param kwargs:
:return:
"""
policy = get(policy_id)
for key, value in kwargs.items():
setattr(policy, key, value)
return database.update(policy)

10
lemur/schemas.py
View File

@ -21,6 +21,7 @@ from lemur.plugins.utils import get_plugin_option
from lemur.roles.models import Role
from lemur.users.models import User
from lemur.authorities.models import Authority
from lemur.policies.models import RotationPolicy
from lemur.certificates.models import Certificate
from lemur.destinations.models import Destination
from lemur.notifications.models import Notification
@ -149,6 +150,15 @@ class AssociatedUserSchema(LemurInputSchema):
return fetch_objects(User, data, many=many)
class AssociatedRotationPolicySchema(LemurInputSchema):
id = fields.Int()
name = fields.String()
@post_load
def get_object(self, data, many=False):
return fetch_objects(RotationPolicy, data, many=many)
class PluginInputSchema(LemurInputSchema):
plugin_options = fields.List(fields.Dict(), validate=validate_options)
slug = fields.String(required=True)

3
lemur/tests/conftest.py
View File

@ -6,7 +6,7 @@ from lemur.database import db as _db
from lemur.auth.service import create_token
from .factories import AuthorityFactory, NotificationFactory, DestinationFactory, \
CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory
CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, RotationPolicyFactory
def pytest_runtest_setup(item):
@ -51,6 +51,7 @@ def db(app, request):
UserFactory()
r = RoleFactory(name='admin')
UserFactory(roles=[r])
rp = RotationPolicyFactory(name='default')
_db.session.commit()
yield _db

11
lemur/tests/factories.py
View File

@ -15,6 +15,7 @@ from lemur.notifications.models import Notification
from lemur.users.models import User
from lemur.roles.models import Role
from lemur.endpoints.models import Policy, Endpoint
from lemur.policies.models import RotationPolicy
from .vectors import INTERNAL_VALID_SAN_STR, PRIVATE_KEY_STR
@ -137,6 +138,16 @@ class AuthorityFactory(BaseFactory):
self.roles.append(role)
class RotationPolicyFactory(BaseFactory):
"""Rotation Factory."""
name = Sequence(lambda n: 'policy{0}'.format(n))
days = 30
class Meta:
"""Factory configuration."""
model = RotationPolicy
class DestinationFactory(BaseFactory):
"""Destination factory."""
plugin_name = 'test-destination'

2
lemur/tests/test_certificates.py
View File

@ -155,7 +155,7 @@ def test_certificate_input_schema(client, authority):
assert data['country'] == 'US'
assert data['location'] == 'Los Gatos'
assert len(data.keys()) == 17
assert len(data.keys()) == 18
def test_certificate_input_with_extensions(client, authority):