diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0855b3de..f3d19151 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ - repo: git://github.com/pre-commit/pre-commit-hooks - sha: 18d7035de5388cc7775be57f529c154bf541aab9 + sha: v0.9.1 hooks: - id: trailing-whitespace - id: flake8 - id: check-merge-conflict - repo: git://github.com/pre-commit/mirrors-jshint - sha: e72140112bdd29b18b0c8257956c896c4c3cebcb + sha: v2.9.5 hooks: - id: jshint diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e423611..1073d2d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,10 @@ Changelog 0.6 - `master` ~~~~~~~~~~~~~~ + +Adds per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates +is 30 days. Every certificate will gain a policy regardless is auto-rotation is used. + .. note:: This version is not yet released and is under active development diff --git a/lemur/certificates/models.py b/lemur/certificates/models.py index 20ed1262..59c9e90a 100644 --- a/lemur/certificates/models.py +++ b/lemur/certificates/models.py @@ -6,6 +6,7 @@ .. moduleauthor:: Kevin Glisson """ import arrow +from datetime import timedelta from flask import current_app @@ -15,7 +16,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from idna.core import InvalidCodepoint from sqlalchemy.orm import relationship -from sqlalchemy.sql.expression import case +from sqlalchemy.sql.expression import case, extract from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean @@ -37,6 +38,7 @@ from lemur.models import certificate_associations, certificate_source_associatio certificate_replacement_associations, roles_certificates from lemur.domains.models import Domain +from lemur.policies.models import RotationPolicy def get_sequence(name): @@ -102,10 +104,10 @@ class Certificate(db.Model): san = Column(String(1024)) # TODO this should be migrated to boolean rotation = Column(Boolean, default=False) - user_id = Column(Integer, ForeignKey('users.id')) authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE")) root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE")) + rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id')) notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate') destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate') @@ -120,6 +122,7 @@ class Certificate(db.Model): logs = relationship('Log', backref='certificate') endpoints = relationship('Endpoint', backref='certificate') + rotation_policy = relationship("RotationPolicy") def __init__(self, **kwargs): cert = lemur.common.utils.parse_certificate(kwargs['body']) @@ -134,7 +137,8 @@ class Certificate(db.Model): if kwargs.get('name'): self.name = get_or_increase_name(kwargs['name']) else: - self.name = get_or_increase_name(defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san)) + self.name = get_or_increase_name( + defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san)) self.owner = kwargs['owner'] self.body = kwargs['body'].strip() @@ -152,6 +156,7 @@ class Certificate(db.Model): self.roles = list(set(kwargs.get('roles', []))) self.replaces = kwargs.get('replaces', []) self.rotation = kwargs.get('rotation') + self.rotation_policy = kwargs.get('rotation_policy') self.signing_algorithm = defaults.signing_algorithm(cert) self.bits = defaults.bitstrength(cert) self.serial = defaults.serial(cert) @@ -240,6 +245,33 @@ class Certificate(db.Model): else_=False ) + @hybrid_property + def in_rotation_window(self): + """ + Determines if a certificate is available for rotation based + on the rotation policy associated. + :return: + """ + now = arrow.utcnow() + end = now + timedelta(days=self.rotation_policy.days) + + if self.not_after <= end: + return True + + @in_rotation_window.expression + def in_rotation_window(cls): + """ + Determines if a certificate is available for rotation based + on the rotation policy associated. + :return: + """ + return case( + [ + (extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True) + ], + else_=False + ) + @property def extensions(self): # setup default values @@ -298,16 +330,6 @@ class Certificate(db.Model): return return_extensions - def get_arn(self, account_number): - """ - Generate a valid AWS IAM arn - - :rtype : str - :param account_number: - :return: - """ - return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name) - def __repr__(self): return "Certificate(name={name})".format(name=self.name) @@ -329,7 +351,8 @@ def update_destinations(target, value, initiator): destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options) except Exception as e: current_app.logger.exception(e) - metrics.send('destination_upload_failure', 'counter', 1, metric_tags={'certificate': target.name, 'destination': value.label}) + metrics.send('destination_upload_failure', 'counter', 1, + metric_tags={'certificate': target.name, 'destination': value.label}) @event.listens_for(Certificate.replaces, 'append') diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index fd1a5b27..7f3aaea6 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -9,8 +9,17 @@ from flask import current_app from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError -from lemur.schemas import AssociatedAuthoritySchema, AssociatedDestinationSchema, AssociatedCertificateSchema, \ - AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema, EndpointNestedOutputSchema +from lemur.schemas import ( + AssociatedAuthoritySchema, + AssociatedDestinationSchema, + AssociatedCertificateSchema, + AssociatedNotificationSchema, + PluginInputSchema, + ExtensionSchema, + AssociatedRoleSchema, + EndpointNestedOutputSchema, + AssociatedRotationPolicySchema +) from lemur.authorities.schemas import AuthorityNestedOutputSchema from lemur.destinations.schemas import DestinationNestedOutputSchema @@ -18,6 +27,7 @@ from lemur.notifications.schemas import NotificationNestedOutputSchema from lemur.roles.schemas import RoleNestedOutputSchema from lemur.domains.schemas import DomainNestedOutputSchema from lemur.users.schemas import UserNestedOutputSchema +from lemur.policies.schemas import RotationPolicyNestedOutputSchema from lemur.common.schema import LemurInputSchema, LemurOutputSchema from lemur.common import validators, missing @@ -63,6 +73,7 @@ class CertificateInputSchema(CertificateCreationSchema): notify = fields.Boolean(default=True) rotation = fields.Boolean() + rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, default={'name': 'default'}) # certificate body fields organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT')) @@ -133,6 +144,7 @@ class CertificateNestedOutputSchema(LemurOutputSchema): rotation = fields.Boolean() notify = fields.Boolean() + rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema) # Note aliasing is the first step in deprecating these fields. cn = fields.String() # deprecated @@ -198,6 +210,7 @@ class CertificateOutputSchema(LemurOutputSchema): roles = fields.Nested(RoleNestedOutputSchema, many=True) endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[]) replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced') + rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema) class CertificateUploadInputSchema(CertificateCreationSchema): diff --git a/lemur/certificates/service.py b/lemur/certificates/service.py index b6d3bf92..cb14d1fd 100644 --- a/lemur/certificates/service.py +++ b/lemur/certificates/service.py @@ -6,7 +6,6 @@ .. moduleauthor:: Kevin Glisson """ import arrow -from datetime import timedelta from flask import current_app from sqlalchemy import func, or_, not_, cast, Boolean, Integer @@ -85,20 +84,15 @@ def get_all_pending_reissue(): """ Retrieves all certificates that need to be rotated. - Must be X days from expiration, uses `LEMUR_DEFAULT_ROTATION_INTERVAL` - to determine how many days from expiration the certificate must be + Must be X days from expiration, uses the certificates rotation + policy to determine how many days from expiration the certificate must be for rotation to be pending. :return: """ - now = arrow.utcnow() - interval = current_app.config.get('LEMUR_DEFAULT_ROTATION_INTERVAL', 30) - end = now + timedelta(days=interval) - return Certificate.query.filter(Certificate.rotation == True)\ - .filter(Certificate.endpoints.any())\ .filter(not_(Certificate.replaced.any()))\ - .filter(Certificate.not_after <= end.format('YYYY-MM-DD')).all() # noqa + .filter(Certificate.in_rotation_window == True).all() # noqa def find_duplicates(cert): diff --git a/lemur/certificates/views.py b/lemur/certificates/views.py index 63b8cff1..147e0273 100644 --- a/lemur/certificates/views.py +++ b/lemur/certificates/views.py @@ -232,6 +232,8 @@ class CertificatesList(AuthenticatedResource): "replaces": [{ "id": 1 }], + "rotation": True, + "rotationPolicy": {"name": "default"}, "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", "roles": [{ "id": 464, @@ -241,18 +243,6 @@ class CertificatesList(AuthenticatedResource): "san": null } - - :arg extensions: extensions to be used in the certificate - :arg description: description for new certificate - :arg owner: owner email - :arg validityStart: when the certificate should start being valid - :arg validityEnd: when the certificate should expire - :arg authority: authority that should issue the certificate - :arg country: country for the CSR - :arg state: state for the CSR - :arg location: location for the CSR - :arg organization: organization for CSR - :arg commonName: certificate common name :reqheader Authorization: OAuth token to authenticate :statuscode 200: no error :statuscode 403: unauthenticated @@ -356,6 +346,8 @@ class CertificatesUpload(AuthenticatedResource): "name": "*.test.example.net" }], "replaces": [], + "rotation": True, + "rotationPolicy": {"name": "default"}, "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", "roles": [{ "id": 464, @@ -365,11 +357,6 @@ class CertificatesUpload(AuthenticatedResource): "san": null } - :arg owner: owner email for certificate - :arg publicCert: valid PEM public key for certificate - :arg intermediateCert valid PEM intermediate key for certificate - :arg privateKey: valid PEM private key for certificate - :arg destinations: list of aws destinations to upload the certificate to :reqheader Authorization: OAuth token to authenticate :statuscode 403: unauthenticated :statuscode 200: no error @@ -522,6 +509,8 @@ class Certificates(AuthenticatedResource): "id": 1090, "name": "*.test.example.net" }], + "rotation": True, + "rotationPolicy": {"name": "default"}, "replaces": [], "replaced": [], "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", @@ -616,6 +605,8 @@ class Certificates(AuthenticatedResource): "description": "This is a google group based role created by Lemur", "name": "joe@example.com" }], + "rotation": True, + "rotationPolicy": {"name": "default"}, "san": null } @@ -722,6 +713,8 @@ class NotificationCertificatesList(AuthenticatedResource): }], "replaces": [], "replaced": [], + "rotation": True, + "rotationPolicy": {"name": "default"}, "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", "roles": [{ "id": 464, @@ -827,6 +820,8 @@ class CertificatesReplacementsList(AuthenticatedResource): }], "replaces": [], "replaced": [], + "rotation": True, + "rotationPolicy": {"name": "default"}, "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112", "roles": [{ "id": 464, diff --git a/lemur/deployment/service.py b/lemur/deployment/service.py index 9c2a0a12..0a81693f 100644 --- a/lemur/deployment/service.py +++ b/lemur/deployment/service.py @@ -10,7 +10,6 @@ def rotate_certificate(endpoint, new_cert): :return: """ # ensure that certificate is available for rotation - endpoint.source.plugin.update_endpoint(endpoint, new_cert) endpoint.certificate = new_cert database.update(endpoint) diff --git a/lemur/manage.py b/lemur/manage.py index 3617ce5d..9516ae03 100755 --- a/lemur/manage.py +++ b/lemur/manage.py @@ -16,16 +16,18 @@ from flask_migrate import Migrate, MigrateCommand, stamp from flask_script.commands import ShowUrls, Clean, Server from lemur.sources.cli import manager as source_manager +from lemur.policies.cli import manager as policy_manager +from lemur.reporting.cli import manager as report_manager +from lemur.endpoints.cli import manager as endpoint_manager from lemur.certificates.cli import manager as certificate_manager from lemur.notifications.cli import manager as notification_manager -from lemur.endpoints.cli import manager as endpoint_manager -from lemur.reporting.cli import manager as report_manager + from lemur import database from lemur.users import service as user_service from lemur.roles import service as role_service +from lemur.policies import service as policy_service from lemur.notifications import service as notification_service - from lemur.common.utils import validate_conf from lemur import create_app @@ -40,6 +42,8 @@ from lemur.domains.models import Domain # noqa from lemur.notifications.models import Notification # noqa from lemur.sources.models import Source # noqa from lemur.logs.models import Log # noqa +from lemur.endpoints.models import Endpoint # noqa +from lemur.policies.models import RotationPolicy # noqa manager = Manager(create_app) @@ -242,6 +246,12 @@ class InitializeApp(Command): recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL') notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients) + days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30) + sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format( + days=days + )) + + policy_service.create(days=days) sys.stdout.write("[/] Done!\n") @@ -531,6 +541,7 @@ def main(): manager.add_command("notify", notification_manager) manager.add_command("endpoint", endpoint_manager) manager.add_command("report", report_manager) + manager.add_command("policy", policy_manager) manager.run() diff --git a/lemur/migrations/versions/a02a678ddc25_.py b/lemur/migrations/versions/a02a678ddc25_.py new file mode 100644 index 00000000..603bc06a --- /dev/null +++ b/lemur/migrations/versions/a02a678ddc25_.py @@ -0,0 +1,54 @@ +"""Adds support for rotation policies. + +Creates a default rotation policy (30 days) with the name +'default' ensures that all existing certificates use the default +policy. + +Revision ID: a02a678ddc25 +Revises: 8ae67285ff14 +Create Date: 2017-07-12 11:45:49.257927 +""" + +# revision identifiers, used by Alembic. +revision = 'a02a678ddc25' +down_revision = '8ae67285ff14' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import text + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('rotation_policies', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('days', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.add_column('certificates', sa.Column('rotation_policy_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'certificates', 'rotation_policies', ['rotation_policy_id'], ['id']) + + conn = op.get_bind() + stmt = text('insert into rotation_policies (days, name) values (:days, :name)') + stmt = stmt.bindparams(days=30, name='default') + conn.execute(stmt) + + stmt = text('select id from rotation_policies where name=:name') + stmt = stmt.bindparams(name='default') + rotation_policy_id = conn.execute(stmt).fetchone()[0] + + stmt = text('update certificates set rotation_policy_id=:rotation_policy_id') + stmt = stmt.bindparams(rotation_policy_id=rotation_policy_id) + conn.execute(stmt) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'certificates', type_='foreignkey') + op.drop_column('certificates', 'rotation_policy_id') + op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations') + op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['replaced_certificate_id', 'certificate_id'], unique=True) + op.drop_table('rotation_policies') + # ### end Alembic commands ### diff --git a/lemur/policies/__init__.py b/lemur/policies/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lemur/policies/cli.py b/lemur/policies/cli.py new file mode 100644 index 00000000..15accd98 --- /dev/null +++ b/lemur/policies/cli.py @@ -0,0 +1,24 @@ +""" +.. module: lemur.policies.cli + :platform: Unix + :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from flask_script import Manager +from lemur.policies import service as policy_service + + +manager = Manager(usage="Handles all policy related tasks.") + + +@manager.option('-d', '--days', dest='days', help='Number of days before expiration.') +@manager.option('-n', '--name', dest='name', help='Policy name.') +def create(days, name): + """ + Create a new certificate rotation policy + :return: + """ + print("[+] Creating a new certificate rotation policy.") + policy_service.create(days=days, name=name) + print("[+] Successfully created a new certificate rotation policy") diff --git a/lemur/policies/models.py b/lemur/policies/models.py new file mode 100644 index 00000000..99e33cc0 --- /dev/null +++ b/lemur/policies/models.py @@ -0,0 +1,21 @@ +""" +.. module: lemur.policies.models + :platform: unix + :synopsis: This module contains all of the models need to create a certificate policy within Lemur. + :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from sqlalchemy import Column, Integer, String + +from lemur.database import db + + +class RotationPolicy(db.Model): + __tablename__ = 'rotation_policies' + id = Column(Integer, primary_key=True) + name = Column(String) + days = Column(Integer) + + def __repr__(self): + return "RotationPolicy(days={days}, name={name})".format(days=self.days, name=self.name) diff --git a/lemur/policies/schemas.py b/lemur/policies/schemas.py new file mode 100644 index 00000000..190600fe --- /dev/null +++ b/lemur/policies/schemas.py @@ -0,0 +1,19 @@ +""" +.. module: lemur.policies.schemas + :platform: unix + :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from marshmallow import fields + +from lemur.common.schema import LemurOutputSchema + + +class RotationPolicyOutputSchema(LemurOutputSchema): + id = fields.Integer() + days = fields.Integer() + + +class RotationPolicyNestedOutputSchema(RotationPolicyOutputSchema): + pass diff --git a/lemur/policies/service.py b/lemur/policies/service.py new file mode 100644 index 00000000..603fb3af --- /dev/null +++ b/lemur/policies/service.py @@ -0,0 +1,62 @@ +""" +.. module: lemur.policies.service + :platform: Unix + :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more + :license: Apache, see LICENSE for more details. +.. moduleauthor:: Kevin Glisson +""" +from lemur import database +from lemur.policies.models import RotationPolicy + + +def get(policy_id): + """ + Retrieves policy by its ID. + :param policy_id: + :return: + """ + return database.get(RotationPolicy, policy_id) + + +def delete(policy_id): + """ + Delete a rotation policy. + :param policy_id: + :return: + """ + database.delete(get(policy_id)) + + +def get_all_policies(): + """ + Retrieves all rotation policies. + :return: + """ + return RotationPolicy.query.all() + + +def create(**kwargs): + """ + Creates a new rotation policy. + + :param kwargs: + :return: + """ + policy = RotationPolicy(**kwargs) + database.create(policy) + return policy + + +def update(policy_id, **kwargs): + """ + Updates a policy. + :param policy_id: + :param kwargs: + :return: + """ + policy = get(policy_id) + + for key, value in kwargs.items(): + setattr(policy, key, value) + + return database.update(policy) diff --git a/lemur/schemas.py b/lemur/schemas.py index e50ac0a0..b897f378 100644 --- a/lemur/schemas.py +++ b/lemur/schemas.py @@ -21,6 +21,7 @@ from lemur.plugins.utils import get_plugin_option from lemur.roles.models import Role from lemur.users.models import User from lemur.authorities.models import Authority +from lemur.policies.models import RotationPolicy from lemur.certificates.models import Certificate from lemur.destinations.models import Destination from lemur.notifications.models import Notification @@ -149,6 +150,15 @@ class AssociatedUserSchema(LemurInputSchema): return fetch_objects(User, data, many=many) +class AssociatedRotationPolicySchema(LemurInputSchema): + id = fields.Int() + name = fields.String() + + @post_load + def get_object(self, data, many=False): + return fetch_objects(RotationPolicy, data, many=many) + + class PluginInputSchema(LemurInputSchema): plugin_options = fields.List(fields.Dict(), validate=validate_options) slug = fields.String(required=True) diff --git a/lemur/tests/conftest.py b/lemur/tests/conftest.py index cbb7224c..46ac7a27 100644 --- a/lemur/tests/conftest.py +++ b/lemur/tests/conftest.py @@ -6,7 +6,7 @@ from lemur.database import db as _db from lemur.auth.service import create_token from .factories import AuthorityFactory, NotificationFactory, DestinationFactory, \ - CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory + CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, RotationPolicyFactory def pytest_runtest_setup(item): @@ -51,6 +51,7 @@ def db(app, request): UserFactory() r = RoleFactory(name='admin') UserFactory(roles=[r]) + rp = RotationPolicyFactory(name='default') _db.session.commit() yield _db diff --git a/lemur/tests/factories.py b/lemur/tests/factories.py index 064337dd..5a47982d 100644 --- a/lemur/tests/factories.py +++ b/lemur/tests/factories.py @@ -15,6 +15,7 @@ from lemur.notifications.models import Notification from lemur.users.models import User from lemur.roles.models import Role from lemur.endpoints.models import Policy, Endpoint +from lemur.policies.models import RotationPolicy from .vectors import INTERNAL_VALID_SAN_STR, PRIVATE_KEY_STR @@ -137,6 +138,16 @@ class AuthorityFactory(BaseFactory): self.roles.append(role) +class RotationPolicyFactory(BaseFactory): + """Rotation Factory.""" + name = Sequence(lambda n: 'policy{0}'.format(n)) + days = 30 + + class Meta: + """Factory configuration.""" + model = RotationPolicy + + class DestinationFactory(BaseFactory): """Destination factory.""" plugin_name = 'test-destination' diff --git a/lemur/tests/test_certificates.py b/lemur/tests/test_certificates.py index 302327b5..4484d2cf 100644 --- a/lemur/tests/test_certificates.py +++ b/lemur/tests/test_certificates.py @@ -155,7 +155,7 @@ def test_certificate_input_schema(client, authority): assert data['country'] == 'US' assert data['location'] == 'Los Gatos' - assert len(data.keys()) == 17 + assert len(data.keys()) == 18 def test_certificate_input_with_extensions(client, authority):