From e68b3d2cbdd61a8c3fa6fe46bef8ac4f268cab0d Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Mon, 7 May 2018 09:58:24 -0700 Subject: [PATCH] 0.7 release --- CHANGELOG.rst | 26 ++++++++++++++++-- lemur/__about__.py | 2 +- lemur/certificates/schemas.py | 30 +++++++++------------ lemur/common/utils.py | 6 ++--- lemur/dns_providers/models.py | 3 ++- lemur/models.py | 2 +- lemur/plugins/lemur_acme/cloudflare.py | 2 +- lemur/plugins/lemur_acme/dyn.py | 4 +-- lemur/plugins/lemur_acme/plugin.py | 15 +++++------ lemur/plugins/lemur_acme/route53.py | 1 + lemur/plugins/lemur_acme/tests/test_acme.py | 3 ++- lemur/utils.py | 10 +++---- requirements-dev.txt | 2 +- requirements-docs.txt | 12 ++++----- requirements-tests.txt | 8 +++--- requirements.in | 2 -- requirements.txt | 7 ++--- 17 files changed, 72 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2262a6c8..2f9a2b96 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,12 +2,34 @@ Changelog ========= -0.7 - `master` +0.7 - `2018-05-07` ~~~~~~~~~~~~~~ -.. note:: This version is not yet released and is under active development +This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality. + +The pending_dns_authorizations and dns_providers tables were created. New columns +were added to the certificates and pending_certificates tables, (For the DNS provider ID), and authorities (For options). +Please run a database migration when upgrading. + +The Let's Encrypt flow will run asynchronously. When a certificate is requested through the acme-issuer, a pending certificate +will be created. A cron needs to be defined to run `lemur pending_certs fetch_all_acme`. This command will iterate through all of the pending +certificates, request a DNS challenge token from Let's Encrypt, and set the appropriate _acme-challenge TXT entry. It will +then iterate through and resolve the challenges before requesting a certificate for each pending certificate. If a certificate +is successfully obtained, the pending_certificate will be moved to the certificates table with the appropriate properties. + +Special thanks to all who helped with this release, notably: + +- The folks at Cloudflare +- dmitryzykov +- jchuong +- seils +- titouanc +Upgrading +--------- + +.. note:: This release will need a migration change. Please follow the `documentation `_ to upgrade Lemur. 0.6 - `2018-01-02` ~~~~~~~~~~~~~~~~~~ diff --git a/lemur/__about__.py b/lemur/__about__.py index ee4e6a6d..c40d4829 100644 --- a/lemur/__about__.py +++ b/lemur/__about__.py @@ -9,7 +9,7 @@ __title__ = "lemur" __summary__ = ("Certificate management and orchestration service") __uri__ = "https://github.com/Netflix/lemur" -__version__ = "0.7.0dev" +__version__ = "0.7.0" __author__ = "The Lemur developers" __email__ = "security@netflix.com" diff --git a/lemur/certificates/schemas.py b/lemur/certificates/schemas.py index ac02e7e0..727c67f8 100644 --- a/lemur/certificates/schemas.py +++ b/lemur/certificates/schemas.py @@ -9,6 +9,17 @@ from flask import current_app from marshmallow import fields, validate, validates_schema, post_load, pre_load from marshmallow.exceptions import ValidationError +from lemur.authorities.schemas import AuthorityNestedOutputSchema +from lemur.common import validators, missing +from lemur.common.fields import ArrowDateTime, Hex +from lemur.common.schema import LemurInputSchema, LemurOutputSchema +from lemur.constants import CERTIFICATE_KEY_TYPES +from lemur.destinations.schemas import DestinationNestedOutputSchema +from lemur.domains.schemas import DomainNestedOutputSchema +from lemur.notifications import service as notification_service +from lemur.notifications.schemas import NotificationNestedOutputSchema +from lemur.policies.schemas import RotationPolicyNestedOutputSchema +from lemur.roles.schemas import RoleNestedOutputSchema from lemur.schemas import ( AssociatedAuthoritySchema, AssociatedDestinationSchema, @@ -21,20 +32,7 @@ from lemur.schemas import ( AssociatedRotationPolicySchema, DnsProviderSchema ) - -from lemur.authorities.schemas import AuthorityNestedOutputSchema -from lemur.destinations.schemas import DestinationNestedOutputSchema -from lemur.notifications.schemas import NotificationNestedOutputSchema -from lemur.roles.schemas import RoleNestedOutputSchema -from lemur.domains.schemas import DomainNestedOutputSchema from lemur.users.schemas import UserNestedOutputSchema -from lemur.policies.schemas import RotationPolicyNestedOutputSchema - -from lemur.common.schema import LemurInputSchema, LemurOutputSchema -from lemur.common import validators, missing -from lemur.notifications import service as notification_service - -from lemur.common.fields import ArrowDateTime, Hex class CertificateSchema(LemurInputSchema): @@ -76,11 +74,7 @@ class CertificateInputSchema(CertificateCreationSchema): csr = fields.String(validate=validators.csr) key_type = fields.String( - validate=validate.OneOf( - ['RSA2048', 'RSA4096', 'ECCPRIME192V1', 'ECCPRIME256V1', 'ECCSECP192R1', 'ECCSECP224R1', - 'ECCSECP256R1', 'ECCSECP384R1', 'ECCSECP521R1', 'ECCSECP256K1', 'ECCSECT163K1', 'ECCSECT233K1', - 'ECCSECT283K1', 'ECCSECT409K1', 'ECCSECT571K1', 'ECCSECT163R2', 'ECCSECT233R1', 'ECCSECT283R1', - 'ECCSECT409R1', 'ECCSECT571R2']), + validate=validate.OneOf(CERTIFICATE_KEY_TYPES), missing='RSA2048') notify = fields.Boolean(default=True) diff --git a/lemur/common/utils.py b/lemur/common/utils.py index 02f55340..fd621076 100644 --- a/lemur/common/utils.py +++ b/lemur/common/utils.py @@ -6,17 +6,15 @@ .. moduleauthor:: Kevin Glisson """ -import string import random +import string import sqlalchemy -from sqlalchemy import and_, func - from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa, ec - from flask_restful.reqparse import RequestParser +from sqlalchemy import and_, func from lemur.constants import CERTIFICATE_KEY_TYPES from lemur.exceptions import InvalidConfiguration diff --git a/lemur/dns_providers/models.py b/lemur/dns_providers/models.py index 42cc992a..751e1fbe 100644 --- a/lemur/dns_providers/models.py +++ b/lemur/dns_providers/models.py @@ -4,6 +4,7 @@ from sqlalchemy_utils import ArrowType from lemur.database import db from lemur.plugins.base import plugins +from lemur.utils import Vault class DnsProviders(db.Model): @@ -15,7 +16,7 @@ class DnsProviders(db.Model): name = Column(String(length=256), unique=True, nullable=True) description = Column(String(length=1024), nullable=True) provider_type = Column(String(length=256), nullable=True) - credentials = Column(String(length=256), nullable=True) + credentials = Column(Vault, nullable=True) api_endpoint = Column(String(length=256), nullable=True) date_created = Column(ArrowType(), server_default=text('now()'), nullable=False) status = Column(String(length=128), nullable=True) diff --git a/lemur/models.py b/lemur/models.py index 02c64dbe..900bbb0f 100644 --- a/lemur/models.py +++ b/lemur/models.py @@ -55,7 +55,7 @@ certificate_replacement_associations = db.Table('certificate_replacement_associa ForeignKey('certificates.id', ondelete='cascade')) ) -Index('certificate_replacement_associations_ix', certificate_replacement_associations.c.replaced_certificate_id, certificate_replacement_associations.c.certificate_id) +Index('certificate_replacement_associations_ix', certificate_replacement_associations.c.replaced_certificate_id, certificate_replacement_associations.c.certificate_id, unique=True) roles_authorities = db.Table('roles_authorities', Column('authority_id', Integer, ForeignKey('authorities.id')), diff --git a/lemur/plugins/lemur_acme/cloudflare.py b/lemur/plugins/lemur_acme/cloudflare.py index 2a665b38..77052242 100644 --- a/lemur/plugins/lemur_acme/cloudflare.py +++ b/lemur/plugins/lemur_acme/cloudflare.py @@ -1,6 +1,6 @@ import time -import CloudFlare +import CloudFlare from flask import current_app diff --git a/lemur/plugins/lemur_acme/dyn.py b/lemur/plugins/lemur_acme/dyn.py index 5d7abf66..b413180f 100644 --- a/lemur/plugins/lemur_acme/dyn.py +++ b/lemur/plugins/lemur_acme/dyn.py @@ -1,7 +1,7 @@ -import dns.exception -import dns.resolver import time +import dns.exception +import dns.resolver from dyn.tm.session import DynectSession from dyn.tm.zones import Node, Zone from flask import current_app diff --git a/lemur/plugins/lemur_acme/plugin.py b/lemur/plugins/lemur_acme/plugin.py index e95fe3b7..61f22eb4 100644 --- a/lemur/plugins/lemur_acme/plugin.py +++ b/lemur/plugins/lemur_acme/plugin.py @@ -11,25 +11,22 @@ .. moduleauthor:: Mikhail Khodorovskiy .. moduleauthor:: Curtis Castrapel """ -import josepy as jose import json -from flask import current_app - -from acme.client import Client +import OpenSSL.crypto +import josepy as jose from acme import challenges, messages +from acme.client import Client from acme.errors import PollError from botocore.exceptions import ClientError - -from lemur.common.utils import generate_private_key - -import OpenSSL.crypto +from flask import current_app from lemur.authorizations import service as authorization_service +from lemur.common.utils import generate_private_key from lemur.dns_providers import service as dns_provider_service from lemur.exceptions import InvalidAuthority, InvalidConfiguration -from lemur.plugins.bases import IssuerPlugin from lemur.plugins import lemur_acme as acme +from lemur.plugins.bases import IssuerPlugin def find_dns_challenge(authz): diff --git a/lemur/plugins/lemur_acme/route53.py b/lemur/plugins/lemur_acme/route53.py index a95cfcd1..a6d3ca92 100644 --- a/lemur/plugins/lemur_acme/route53.py +++ b/lemur/plugins/lemur_acme/route53.py @@ -1,4 +1,5 @@ import time + from lemur.plugins.lemur_aws.sts import sts_client diff --git a/lemur/plugins/lemur_acme/tests/test_acme.py b/lemur/plugins/lemur_acme/tests/test_acme.py index b148f1c7..88af1b11 100644 --- a/lemur/plugins/lemur_acme/tests/test_acme.py +++ b/lemur/plugins/lemur_acme/tests/test_acme.py @@ -1,8 +1,9 @@ import unittest -from lemur.plugins.lemur_acme import plugin from mock import MagicMock, Mock, patch +from lemur.plugins.lemur_acme import plugin + class TestAcme(unittest.TestCase): diff --git a/lemur/utils.py b/lemur/utils.py index f8c6154c..090ba992 100644 --- a/lemur/utils.py +++ b/lemur/utils.py @@ -6,12 +6,12 @@ .. moduleauthor:: Kevin Glisson """ import os -from flask import current_app -from cryptography.fernet import Fernet, MultiFernet -import sqlalchemy.types as types - -from contextlib import contextmanager import tempfile +from contextlib import contextmanager + +import sqlalchemy.types as types +from cryptography.fernet import Fernet, MultiFernet +from flask import current_app @contextmanager diff --git a/requirements-dev.txt b/requirements-dev.txt index 7047b699..36d90b53 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ certifi==2018.4.16 # via requests cfgv==1.0.0 # via pre-commit chardet==3.0.4 # via requests flake8==3.5.0 -identify==1.0.13 # via pre-commit +identify==1.0.15 # via pre-commit idna==2.6 # via requests invoke==0.23.0 mccabe==0.6.1 # via flake8 diff --git a/requirements-docs.txt b/requirements-docs.txt index 17b86edb..95bd84ca 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,7 +4,7 @@ # # pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in # -acme==0.23.0 +acme==0.24.0 alabaster==0.7.10 # via sphinx alembic-autogenerate-enums==0.0.2 alembic==0.9.9 @@ -15,13 +15,15 @@ asyncpool==1.0 babel==2.5.3 # via sphinx bcrypt==3.1.4 blinker==1.4 -boto3==1.7.11 -botocore==1.10.11 +boto3==1.7.14 +botocore==1.10.14 certifi==2018.4.16 cffi==1.11.5 click==6.7 cloudflare==2.1.0 cryptography==2.2.2 +dnspython3==1.15.0 +dnspython==1.15.0 docutils==0.14 dyn==1.8.1 flask-bcrypt==0.7.1 @@ -34,14 +36,11 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gevent==1.2.2 -greenlet==0.4.13 gunicorn==19.8.1 idna==2.6 imagesize==1.0.0 # via sphinx inflection==0.3.1 itsdangerous==0.24 -janus==0.3.1 jinja2==2.10 jmespath==0.9.3 josepy==1.1.0 @@ -84,5 +83,6 @@ sphinxcontrib-websupport==1.0.1 # via sphinx sqlalchemy-utils==0.33.3 sqlalchemy==1.2.7 tabulate==0.8.2 +tld==0.7.10 werkzeug==0.14.1 xmltodict==0.11.0 diff --git a/requirements-tests.txt b/requirements-tests.txt index 54ae9226..f1736305 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -5,11 +5,11 @@ # pip-compile --no-index --output-file requirements-tests.txt requirements-tests.in # asn1crypto==0.24.0 # via cryptography -attrs==17.4.0 # via pytest +attrs==18.1.0 # via pytest aws-xray-sdk==0.95 # via moto -boto3==1.7.11 # via moto +boto3==1.7.14 # via moto boto==2.48.0 # via moto -botocore==1.10.11 # via boto3, moto, s3transfer +botocore==1.10.14 # via boto3, moto, s3transfer certifi==2018.4.16 # via requests cffi==1.11.5 # via cryptography chardet==3.0.4 # via requests @@ -20,7 +20,7 @@ cryptography==2.2.2 # via moto docker-pycreds==0.2.3 # via docker docker==3.3.0 # via moto docutils==0.14 # via botocore -factory-boy==2.10.0 +factory-boy==2.11.1 faker==0.8.13 flask==1.0.2 # via pytest-flask freezegun==0.3.10 diff --git a/requirements.in b/requirements.in index 5acccc62..4d287c0d 100644 --- a/requirements.in +++ b/requirements.in @@ -20,10 +20,8 @@ Flask-SQLAlchemy Flask==0.12 Flask-Cors future -gevent gunicorn inflection -janus jinja2 lockfile marshmallow-sqlalchemy diff --git a/requirements.txt b/requirements.txt index e880bb02..73bda8b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,8 @@ asn1crypto==0.24.0 # via cryptography asyncpool==1.0 bcrypt==3.1.4 # via flask-bcrypt, paramiko blinker==1.4 # via flask-mail, flask-principal, raven -boto3==1.7.11 -botocore==1.10.11 # via boto3, s3transfer +boto3==1.7.14 +botocore==1.10.14 # via boto3, s3transfer certifi==2018.4.16 cffi==1.11.5 # via bcrypt, cryptography, pynacl click==6.7 # via flask @@ -34,13 +34,10 @@ flask-script==2.0.6 flask-sqlalchemy==2.3.2 flask==0.12 future==0.16.0 -gevent==1.2.2 -greenlet==0.4.13 # via gevent gunicorn==19.8.1 idna==2.6 # via cryptography inflection==0.3.1 itsdangerous==0.24 # via flask -janus==0.3.1 jinja2==2.10 jmespath==0.9.3 # via boto3, botocore josepy==1.1.0 # via acme