Adding the ability to specify a per-certificate rotation policy. (#851)

.pre-commit-config.yaml

@@ -1,10 +1,10 @@
 -   repo: git://github.com/pre-commit/pre-commit-hooks
-    sha: 18d7035de5388cc7775be57f529c154bf541aab9
+    sha: v0.9.1
     hooks:
     -   id: trailing-whitespace
     -   id: flake8
     -   id: check-merge-conflict
 -   repo: git://github.com/pre-commit/mirrors-jshint
-    sha: e72140112bdd29b18b0c8257956c896c4c3cebcb
+    sha: v2.9.5
     hooks:
     -   id: jshint

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 0.6 - `master`
 ~~~~~~~~~~~~~~
 
+
+Adds per-certificate rotation policies, requires a database migration. The default rotation policy for all certificates
+is 30 days. Every certificate will gain a policy regardless is auto-rotation is used.
+
 .. note:: This version is not yet released and is under active development
 
 

lemur/certificates/models.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
+from datetime import timedelta
 
 from flask import current_app
 
@@ -15,7 +16,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
 from idna.core import InvalidCodepoint
 
 from sqlalchemy.orm import relationship
-from sqlalchemy.sql.expression import case
+from sqlalchemy.sql.expression import case, extract
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
 
@@ -37,6 +38,7 @@ from lemur.models import certificate_associations, certificate_source_associatio
     certificate_replacement_associations, roles_certificates
 
 from lemur.domains.models import Domain
+from lemur.policies.models import RotationPolicy
 
 
 def get_sequence(name):
@@ -102,10 +104,10 @@ class Certificate(db.Model):
     san = Column(String(1024))  # TODO this should be migrated to boolean
 
     rotation = Column(Boolean, default=False)
-
     user_id = Column(Integer, ForeignKey('users.id'))
     authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
     root_authority_id = Column(Integer, ForeignKey('authorities.id', ondelete="CASCADE"))
+    rotation_policy_id = Column(Integer, ForeignKey('rotation_policies.id'))
 
     notifications = relationship('Notification', secondary=certificate_notification_associations, backref='certificate')
     destinations = relationship('Destination', secondary=certificate_destination_associations, backref='certificate')
@@ -120,6 +122,7 @@ class Certificate(db.Model):
 
     logs = relationship('Log', backref='certificate')
     endpoints = relationship('Endpoint', backref='certificate')
+    rotation_policy = relationship("RotationPolicy")
 
     def __init__(self, **kwargs):
         cert = lemur.common.utils.parse_certificate(kwargs['body'])
@@ -134,7 +137,8 @@ class Certificate(db.Model):
         if kwargs.get('name'):
             self.name = get_or_increase_name(kwargs['name'])
         else:
-            self.name = get_or_increase_name(defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
+            self.name = get_or_increase_name(
+                defaults.certificate_name(self.cn, self.issuer, self.not_before, self.not_after, self.san))
 
         self.owner = kwargs['owner']
         self.body = kwargs['body'].strip()
@@ -152,6 +156,7 @@ class Certificate(db.Model):
         self.roles = list(set(kwargs.get('roles', [])))
         self.replaces = kwargs.get('replaces', [])
         self.rotation = kwargs.get('rotation')
+        self.rotation_policy = kwargs.get('rotation_policy')
         self.signing_algorithm = defaults.signing_algorithm(cert)
         self.bits = defaults.bitstrength(cert)
         self.serial = defaults.serial(cert)
@@ -240,6 +245,33 @@ class Certificate(db.Model):
             else_=False
         )
 
+    @hybrid_property
+    def in_rotation_window(self):
+        """
+        Determines if a certificate is available for rotation based
+        on the rotation policy associated.
+        :return:
+        """
+        now = arrow.utcnow()
+        end = now + timedelta(days=self.rotation_policy.days)
+
+        if self.not_after <= end:
+            return True
+
+    @in_rotation_window.expression
+    def in_rotation_window(cls):
+        """
+        Determines if a certificate is available for rotation based
+        on the rotation policy associated.
+        :return:
+        """
+        return case(
+            [
+                (extract('day', cls.not_after - func.now()) <= RotationPolicy.days, True)
+            ],
+            else_=False
+        )
+
     @property
     def extensions(self):
         # setup default values
@@ -298,16 +330,6 @@ class Certificate(db.Model):
 
         return return_extensions
 
-    def get_arn(self, account_number):
-        """
-        Generate a valid AWS IAM arn
-
-        :rtype : str
-        :param account_number:
-        :return:
-        """
-        return "arn:aws:iam::{}:server-certificate/{}".format(account_number, self.name)
-
     def __repr__(self):
         return "Certificate(name={name})".format(name=self.name)
 
@@ -329,7 +351,8 @@ def update_destinations(target, value, initiator):
             destination_plugin.upload(target.name, target.body, target.private_key, target.chain, value.options)
     except Exception as e:
         current_app.logger.exception(e)
-        metrics.send('destination_upload_failure', 'counter', 1, metric_tags={'certificate': target.name, 'destination': value.label})
+        metrics.send('destination_upload_failure', 'counter', 1,
+                     metric_tags={'certificate': target.name, 'destination': value.label})
 
 
 @event.listens_for(Certificate.replaces, 'append')

lemur/certificates/schemas.py

@@ -9,8 +9,17 @@ from flask import current_app
 from marshmallow import fields, validate, validates_schema, post_load, pre_load
 from marshmallow.exceptions import ValidationError
 
-from lemur.schemas import AssociatedAuthoritySchema, AssociatedDestinationSchema, AssociatedCertificateSchema, \
+from lemur.schemas import (
-    AssociatedNotificationSchema, PluginInputSchema, ExtensionSchema, AssociatedRoleSchema, EndpointNestedOutputSchema
+    AssociatedAuthoritySchema,
+    AssociatedDestinationSchema,
+    AssociatedCertificateSchema,
+    AssociatedNotificationSchema,
+    PluginInputSchema,
+    ExtensionSchema,
+    AssociatedRoleSchema,
+    EndpointNestedOutputSchema,
+    AssociatedRotationPolicySchema
+)
 
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
 from lemur.destinations.schemas import DestinationNestedOutputSchema
@@ -18,6 +27,7 @@ from lemur.notifications.schemas import NotificationNestedOutputSchema
 from lemur.roles.schemas import RoleNestedOutputSchema
 from lemur.domains.schemas import DomainNestedOutputSchema
 from lemur.users.schemas import UserNestedOutputSchema
+from lemur.policies.schemas import RotationPolicyNestedOutputSchema
 
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.common import validators, missing
@@ -63,6 +73,7 @@ class CertificateInputSchema(CertificateCreationSchema):
 
     notify = fields.Boolean(default=True)
     rotation = fields.Boolean()
+    rotation_policy = fields.Nested(AssociatedRotationPolicySchema, missing={'name': 'default'}, default={'name': 'default'})
 
     # certificate body fields
     organizational_unit = fields.String(missing=lambda: current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'))
@@ -133,6 +144,7 @@ class CertificateNestedOutputSchema(LemurOutputSchema):
 
     rotation = fields.Boolean()
     notify = fields.Boolean()
+    rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
 
     # Note aliasing  is the first step in deprecating these fields.
     cn = fields.String()  # deprecated
@@ -198,6 +210,7 @@ class CertificateOutputSchema(LemurOutputSchema):
     roles = fields.Nested(RoleNestedOutputSchema, many=True)
     endpoints = fields.Nested(EndpointNestedOutputSchema, many=True, missing=[])
     replaced_by = fields.Nested(CertificateNestedOutputSchema, many=True, attribute='replaced')
+    rotation_policy = fields.Nested(RotationPolicyNestedOutputSchema)
 
 
 class CertificateUploadInputSchema(CertificateCreationSchema):

lemur/certificates/service.py

@@ -6,7 +6,6 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 import arrow
-from datetime import timedelta
 
 from flask import current_app
 from sqlalchemy import func, or_, not_, cast, Boolean, Integer
@@ -85,20 +84,15 @@ def get_all_pending_reissue():
     """
     Retrieves all certificates that need to be rotated.
 
-    Must be X days from expiration, uses `LEMUR_DEFAULT_ROTATION_INTERVAL`
+    Must be X days from expiration, uses the certificates rotation
-    to determine how many days from expiration the certificate must be
+    policy to determine how many days from expiration the certificate must be
     for rotation to be pending.
 
     :return:
     """
-    now = arrow.utcnow()
-    interval = current_app.config.get('LEMUR_DEFAULT_ROTATION_INTERVAL', 30)
-    end = now + timedelta(days=interval)
-
     return Certificate.query.filter(Certificate.rotation == True)\
-        .filter(Certificate.endpoints.any())\
         .filter(not_(Certificate.replaced.any()))\
-        .filter(Certificate.not_after <= end.format('YYYY-MM-DD')).all()  # noqa
+        .filter(Certificate.in_rotation_window == True).all()  # noqa
 
 
 def find_duplicates(cert):

lemur/certificates/views.py

@@ -232,6 +232,8 @@ class CertificatesList(AuthenticatedResource):
                 "replaces": [{
                     "id": 1
                 }],
+                "rotation": True,
+                "rotationPolicy": {"name": "default"},
                 "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
                 "roles": [{
                     "id": 464,
@@ -241,18 +243,6 @@ class CertificatesList(AuthenticatedResource):
                 "san": null
               }
 
-
-           :arg extensions: extensions to be used in the certificate
-           :arg description: description for new certificate
-           :arg owner: owner email
-           :arg validityStart: when the certificate should start being valid
-           :arg validityEnd: when the certificate should expire
-           :arg authority: authority that should issue the certificate
-           :arg country: country for the CSR
-           :arg state: state for the CSR
-           :arg location: location for the CSR
-           :arg organization: organization for CSR
-           :arg commonName: certificate common name
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 200: no error
            :statuscode 403: unauthenticated
@@ -356,6 +346,8 @@ class CertificatesUpload(AuthenticatedResource):
                     "name": "*.test.example.net"
                 }],
                 "replaces": [],
+                "rotation": True,
+                "rotationPolicy": {"name": "default"},
                 "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
                 "roles": [{
                     "id": 464,
@@ -365,11 +357,6 @@ class CertificatesUpload(AuthenticatedResource):
                 "san": null
               }
 
-           :arg owner: owner email for certificate
-           :arg publicCert: valid PEM public key for certificate
-           :arg intermediateCert valid PEM intermediate key for certificate
-           :arg privateKey: valid PEM private key for certificate
-           :arg destinations: list of aws destinations to upload the certificate to
            :reqheader Authorization: OAuth token to authenticate
            :statuscode 403: unauthenticated
            :statuscode 200: no error
@@ -522,6 +509,8 @@ class Certificates(AuthenticatedResource):
                     "id": 1090,
                     "name": "*.test.example.net"
                 }],
+                "rotation": True,
+                "rotationPolicy": {"name": "default"},
                 "replaces": [],
                 "replaced": [],
                 "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
@@ -616,6 +605,8 @@ class Certificates(AuthenticatedResource):
                     "description": "This is a google group based role created by Lemur",
                     "name": "joe@example.com"
                 }],
+                "rotation": True,
+                "rotationPolicy": {"name": "default"},
                 "san": null
               }
 
@@ -722,6 +713,8 @@ class NotificationCertificatesList(AuthenticatedResource):
                     }],
                     "replaces": [],
                     "replaced": [],
+                    "rotation": True,
+                    "rotationPolicy": {"name": "default"},
                     "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
                     "roles": [{
                         "id": 464,
@@ -827,6 +820,8 @@ class CertificatesReplacementsList(AuthenticatedResource):
                     }],
                     "replaces": [],
                     "replaced": [],
+                    "rotation": True,
+                    "rotationPolicy": {"name": "default"},
                     "name": "WILDCARD.test.example.net-SymantecCorporation-20160603-20180112",
                     "roles": [{
                         "id": 464,

lemur/deployment/service.py

@@ -10,7 +10,6 @@ def rotate_certificate(endpoint, new_cert):
     :return:
     """
     # ensure that certificate is available for rotation
-
     endpoint.source.plugin.update_endpoint(endpoint, new_cert)
     endpoint.certificate = new_cert
     database.update(endpoint)

lemur/manage.py

@@ -16,16 +16,18 @@ from flask_migrate import Migrate, MigrateCommand, stamp
 from flask_script.commands import ShowUrls, Clean, Server
 
 from lemur.sources.cli import manager as source_manager
+from lemur.policies.cli import manager as policy_manager
+from lemur.reporting.cli import manager as report_manager
+from lemur.endpoints.cli import manager as endpoint_manager
 from lemur.certificates.cli import manager as certificate_manager
 from lemur.notifications.cli import manager as notification_manager
-from lemur.endpoints.cli import manager as endpoint_manager
+
-from lemur.reporting.cli import manager as report_manager
 from lemur import database
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
+from lemur.policies import service as policy_service
 from lemur.notifications import service as notification_service
 
-
 from lemur.common.utils import validate_conf
 
 from lemur import create_app
@@ -40,6 +42,8 @@ from lemur.domains.models import Domain  # noqa
 from lemur.notifications.models import Notification  # noqa
 from lemur.sources.models import Source  # noqa
 from lemur.logs.models import Log  # noqa
+from lemur.endpoints.models import Endpoint  # noqa
+from lemur.policies.models import RotationPolicy  # noqa
 
 
 manager = Manager(create_app)
@@ -242,6 +246,12 @@ class InitializeApp(Command):
         recipients = current_app.config.get('LEMUR_SECURITY_TEAM_EMAIL')
         notification_service.create_default_expiration_notifications("DEFAULT_SECURITY", recipients=recipients)
 
+        days = current_app.config.get("LEMUR_DEFAULT_ROTATION_INTERVAL", 30)
+        sys.stdout.write("[+] Creating default certificate rotation policy of {days} days before issuance.\n".format(
+            days=days
+        ))
+
+        policy_service.create(days=days)
         sys.stdout.write("[/] Done!\n")
 
 
@@ -531,6 +541,7 @@ def main():
     manager.add_command("notify", notification_manager)
     manager.add_command("endpoint", endpoint_manager)
     manager.add_command("report", report_manager)
+    manager.add_command("policy", policy_manager)
     manager.run()
 
 

lemur/migrations/versions/a02a678ddc25_.py

@@ -0,0 +1,54 @@
+"""Adds support for rotation policies.
+
+Creates a default rotation policy (30 days) with the name
+'default' ensures that all existing certificates use the default
+policy.
+
+Revision ID: a02a678ddc25
+Revises: 8ae67285ff14
+Create Date: 2017-07-12 11:45:49.257927
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'a02a678ddc25'
+down_revision = '8ae67285ff14'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('rotation_policies',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(), nullable=True),
+        sa.Column('days', sa.Integer(), nullable=True),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('certificates', sa.Column('rotation_policy_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'certificates', 'rotation_policies', ['rotation_policy_id'], ['id'])
+
+    conn = op.get_bind()
+    stmt = text('insert into rotation_policies (days, name) values (:days, :name)')
+    stmt = stmt.bindparams(days=30, name='default')
+    conn.execute(stmt)
+
+    stmt = text('select id from rotation_policies where name=:name')
+    stmt = stmt.bindparams(name='default')
+    rotation_policy_id = conn.execute(stmt).fetchone()[0]
+
+    stmt = text('update certificates set rotation_policy_id=:rotation_policy_id')
+    stmt = stmt.bindparams(rotation_policy_id=rotation_policy_id)
+    conn.execute(stmt)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'certificates', type_='foreignkey')
+    op.drop_column('certificates', 'rotation_policy_id')
+    op.drop_index('certificate_replacement_associations_ix', table_name='certificate_replacement_associations')
+    op.create_index('certificate_replacement_associations_ix', 'certificate_replacement_associations', ['replaced_certificate_id', 'certificate_id'], unique=True)
+    op.drop_table('rotation_policies')
+    # ### end Alembic commands ###

lemur/policies/cli.py

@@ -0,0 +1,24 @@
+"""
+.. module: lemur.policies.cli
+    :platform: Unix
+    :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from flask_script import Manager
+from lemur.policies import service as policy_service
+
+
+manager = Manager(usage="Handles all policy related tasks.")
+
+
+@manager.option('-d', '--days', dest='days', help='Number of days before expiration.')
+@manager.option('-n', '--name', dest='name', help='Policy name.')
+def create(days, name):
+    """
+    Create a new certificate rotation policy
+    :return:
+    """
+    print("[+] Creating a new certificate rotation policy.")
+    policy_service.create(days=days, name=name)
+    print("[+] Successfully created a new certificate rotation policy")

lemur/policies/models.py

@@ -0,0 +1,21 @@
+"""
+.. module: lemur.policies.models
+    :platform: unix
+    :synopsis: This module contains all of the models need to create a certificate policy within Lemur.
+    :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from sqlalchemy import Column, Integer, String
+
+from lemur.database import db
+
+
+class RotationPolicy(db.Model):
+    __tablename__ = 'rotation_policies'
+    id = Column(Integer, primary_key=True)
+    name = Column(String)
+    days = Column(Integer)
+
+    def __repr__(self):
+        return "RotationPolicy(days={days}, name={name})".format(days=self.days, name=self.name)

lemur/policies/schemas.py

@@ -0,0 +1,19 @@
+"""
+.. module: lemur.policies.schemas
+    :platform: unix
+    :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from marshmallow import fields
+
+from lemur.common.schema import LemurOutputSchema
+
+
+class RotationPolicyOutputSchema(LemurOutputSchema):
+    id = fields.Integer()
+    days = fields.Integer()
+
+
+class RotationPolicyNestedOutputSchema(RotationPolicyOutputSchema):
+    pass

lemur/policies/service.py

@@ -0,0 +1,62 @@
+"""
+.. module: lemur.policies.service
+    :platform: Unix
+    :copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
+"""
+from lemur import database
+from lemur.policies.models import RotationPolicy
+
+
+def get(policy_id):
+    """
+    Retrieves policy by its ID.
+    :param policy_id:
+    :return:
+    """
+    return database.get(RotationPolicy, policy_id)
+
+
+def delete(policy_id):
+    """
+    Delete a rotation policy.
+    :param policy_id:
+    :return:
+    """
+    database.delete(get(policy_id))
+
+
+def get_all_policies():
+    """
+    Retrieves all rotation policies.
+    :return:
+    """
+    return RotationPolicy.query.all()
+
+
+def create(**kwargs):
+    """
+    Creates a new rotation policy.
+
+    :param kwargs:
+    :return:
+    """
+    policy = RotationPolicy(**kwargs)
+    database.create(policy)
+    return policy
+
+
+def update(policy_id, **kwargs):
+    """
+    Updates a policy.
+    :param policy_id:
+    :param kwargs:
+    :return:
+    """
+    policy = get(policy_id)
+
+    for key, value in kwargs.items():
+        setattr(policy, key, value)
+
+    return database.update(policy)

lemur/schemas.py

@@ -21,6 +21,7 @@ from lemur.plugins.utils import get_plugin_option
 from lemur.roles.models import Role
 from lemur.users.models import User
 from lemur.authorities.models import Authority
+from lemur.policies.models import RotationPolicy
 from lemur.certificates.models import Certificate
 from lemur.destinations.models import Destination
 from lemur.notifications.models import Notification
@@ -149,6 +150,15 @@ class AssociatedUserSchema(LemurInputSchema):
         return fetch_objects(User, data, many=many)
 
 
+class AssociatedRotationPolicySchema(LemurInputSchema):
+    id = fields.Int()
+    name = fields.String()
+
+    @post_load
+    def get_object(self, data, many=False):
+        return fetch_objects(RotationPolicy, data, many=many)
+
+
 class PluginInputSchema(LemurInputSchema):
     plugin_options = fields.List(fields.Dict(), validate=validate_options)
     slug = fields.String(required=True)

lemur/tests/conftest.py

@@ -6,7 +6,7 @@ from lemur.database import db as _db
 from lemur.auth.service import create_token
 
 from .factories import AuthorityFactory, NotificationFactory, DestinationFactory, \
-    CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory
+    CertificateFactory, UserFactory, RoleFactory, SourceFactory, EndpointFactory, RotationPolicyFactory
 
 
 def pytest_runtest_setup(item):
@@ -51,6 +51,7 @@ def db(app, request):
     UserFactory()
     r = RoleFactory(name='admin')
     UserFactory(roles=[r])
+    rp = RotationPolicyFactory(name='default')
 
     _db.session.commit()
     yield _db

lemur/tests/factories.py

@@ -15,6 +15,7 @@ from lemur.notifications.models import Notification
 from lemur.users.models import User
 from lemur.roles.models import Role
 from lemur.endpoints.models import Policy, Endpoint
+from lemur.policies.models import RotationPolicy
 
 from .vectors import INTERNAL_VALID_SAN_STR, PRIVATE_KEY_STR
 
@@ -137,6 +138,16 @@ class AuthorityFactory(BaseFactory):
                 self.roles.append(role)
 
 
+class RotationPolicyFactory(BaseFactory):
+    """Rotation Factory."""
+    name = Sequence(lambda n: 'policy{0}'.format(n))
+    days = 30
+
+    class Meta:
+        """Factory configuration."""
+        model = RotationPolicy
+
+
 class DestinationFactory(BaseFactory):
     """Destination factory."""
     plugin_name = 'test-destination'

lemur/tests/test_certificates.py

@@ -155,7 +155,7 @@ def test_certificate_input_schema(client, authority):
     assert data['country'] == 'US'
     assert data['location'] == 'Los Gatos'
 
-    assert len(data.keys()) == 17
+    assert len(data.keys()) == 18
 
 
 def test_certificate_input_with_extensions(client, authority):