Infra
/
lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'master' into linuxdst

This commit is contained in:
Curtis 2018-06-12 07:40:40 -07:00 committed by GitHub
parent 2a3af5214e 3800d67d71
commit 038f5dc554
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
147 changed files with 2008 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
CHANGELOG.rst
View File

@ -2,12 +2,35 @@ Changelog
=========
0.7 - `master`
0.7 - `2018-05-07`
~~~~~~~~~~~~~~
.. note:: This version is not yet released and is under active development
This release adds LetsEncrypt support with DNS providers Dyn, Route53, and Cloudflare, and expands on the pending certificate functionality.
The linux_dst plugin will also be deprecated and removed.
The pending_dns_authorizations and dns_providers tables were created. New columns
were added to the certificates and pending_certificates tables, (For the DNS provider ID), and authorities (For options).
Please run a database migration when upgrading.
The Let's Encrypt flow will run asynchronously. When a certificate is requested through the acme-issuer, a pending certificate
will be created. A cron needs to be defined to run `lemur pending_certs fetch_all_acme`. This command will iterate through all of the pending
certificates, request a DNS challenge token from Let's Encrypt, and set the appropriate _acme-challenge TXT entry. It will
then iterate through and resolve the challenges before requesting a certificate for each pending certificate. If a certificate
is successfully obtained, the pending_certificate will be moved to the certificates table with the appropriate properties.
Special thanks to all who helped with this release, notably:
- The folks at Cloudflare
- dmitryzykov
- jchuong
- seils
- titouanc
Upgrading
---------
.. note:: This release will need a migration change. Please follow the `documentation <https://lemur.readthedocs.io/en/latest/administration.html#upgrading-lemur>`_ to upgrade Lemur.
0.6 - `2018-01-02`
~~~~~~~~~~~~~~~~~~

2
LICENSE
View File

@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2014 Netflix, Inc.
Copyright 2018 Netflix, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

2
docs/conf.py
View File

@ -59,7 +59,7 @@ master_doc = 'index'
# General information about the project.
project = u'lemur'
copyright = u'2015, Netflix Inc.'
copyright = u'2018, Netflix Inc.'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the

4
lemur/__about__.py
View File

@ -9,10 +9,10 @@ __title__ = "lemur"
__summary__ = ("Certificate management and orchestration service")
__uri__ = "https://github.com/Netflix/lemur"
__version__ = "0.7.0dev"
__version__ = "0.7.0"
__author__ = "The Lemur developers"
__email__ = "security@netflix.com"
__license__ = "Apache License, Version 2.0"
__copyright__ = "Copyright 2017 {0}".format(__author__)
__copyright__ = "Copyright 2018 {0}".format(__author__)

4
lemur/__init__.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
@ -29,6 +29,7 @@ from lemur.endpoints.views import mod as endpoints_bp
from lemur.logs.views import mod as logs_bp
from lemur.api_keys.views import mod as api_key_bp
from lemur.pending_certificates.views import mod as pending_certificates_bp
from lemur.dns_providers.views import mod as dns_providers_bp
from lemur.__about__ import (
__author__, __copyright__, __email__, __license__, __summary__, __title__,
@ -57,6 +58,7 @@ LEMUR_BLUEPRINTS = (
logs_bp,
api_key_bp,
pending_certificates_bp,
dns_providers_bp,
)

2
lemur/api_keys/cli.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.api_keys.cli
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Eric Coan <kungfury@instructure.com>
"""

2
lemur/api_keys/models.py
View File

@ -2,7 +2,7 @@
.. module: lemur.api_keys.models
:platform: Unix
:synopsis: This module contains all of the models need to create an api key within Lemur.
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Eric Coan <kungfury@instructure.com>
"""

2
lemur/api_keys/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.api_keys.schemas
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Eric Coan <kungfury@instructure.com>
"""

2
lemur/api_keys/service.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.api_keys.service
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Eric Coan <kungfury@instructure.com>
"""

2
lemur/api_keys/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.api_keys.views
:platform: Unix
:copyright: (c) 2017 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Eric Coan <kungfury@instructure.com>

2
lemur/auth/ldap.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.auth.ldap
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Ian Stahnke <ian.stahnke@myob.com>
"""

2
lemur/auth/permissions.py
View File

@ -2,7 +2,7 @@
.. module: lemur.auth.permissions
:platform: Unix
:synopsis: This module defines all the permission used within Lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/auth/service.py
View File

@ -3,7 +3,7 @@
:platform: Unix
:synopsis: This module contains all of the authentication duties for
lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/auth/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.auth.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

3
lemur/authorities/models.py
View File

@ -2,7 +2,7 @@
.. module: lemur.authorities.models
:platform: unix
:synopsis: This module contains all of the models need to create an authority within Lemur.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
@ -42,6 +42,7 @@ class Authority(db.Model):
self.description = kwargs.get('description')
self.authority_certificate = kwargs['authority_certificate']
self.plugin_name = kwargs['plugin']['slug']
self.options = kwargs.get('options')
@property
def plugin(self):

2
lemur/authorities/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.authorities.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

7
lemur/authorities/service.py
View File

@ -3,11 +3,14 @@
:platform: Unix
:synopsis: This module contains all of the services level functions used to
administer authorities in Lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import json
from lemur import database
from lemur.common.utils import truthiness
from lemur.extensions import metrics
@ -107,6 +110,8 @@ def create(**kwargs):
cert = upload(**kwargs)
kwargs['authority_certificate'] = cert
if kwargs.get('plugin', {}).get('plugin_options', []):
kwargs['options'] = json.dumps(kwargs['plugin']['plugin_options'])
authority = Authority(**kwargs)
authority = database.create(authority)

2
lemur/authorities/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.authorities.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

0
lemur/authorizations/__init__.py Normal file
View File

34
lemur/authorizations/models.py Normal file
View File

@ -0,0 +1,34 @@
"""
.. module: lemur.authorizations.models
:platform: unix
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Netflix Secops <secops@netflix.com>
"""
from sqlalchemy import Column, Integer, String
from sqlalchemy_utils import JSONType
from lemur.database import db
from lemur.plugins.base import plugins
class Authorization(db.Model):
__tablename__ = 'pending_dns_authorizations'
id = Column(Integer, primary_key=True, autoincrement=True)
account_number = Column(String(128))
domains = Column(JSONType)
dns_provider_type = Column(String(128))
options = Column(JSONType)
@property
def plugin(self):
return plugins.get(self.plugin_name)
def __repr__(self):
return "Authorization(id={id})".format(label=self.id)
def __init__(self, account_number, domains, dns_provider_type, options=None):
self.account_number = account_number
self.domains = domains
self.dns_provider_type = dns_provider_type
self.options = options

24
lemur/authorizations/service.py Normal file
View File

@ -0,0 +1,24 @@
"""
.. module: lemur.pending_certificates.service
Copyright (c) 2018 and onwards Netflix, Inc. All rights reserved.
.. moduleauthor:: Secops <secops@netflix.com>
"""
from lemur import database
from lemur.authorizations.models import Authorization
def get(authorization_id):
"""
Retrieve dns authorization by ID
"""
return database.get(Authorization, authorization_id)
def create(account_number, domains, dns_provider_type, options=None):
"""
Creates a new dns authorization.
"""
authorization = Authorization(account_number, domains, dns_provider_type, options)
return database.create(authorization)

4
lemur/certificates/cli.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificate.cli
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
@ -154,6 +154,7 @@ def request_reissue(certificate, commit):
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error reissuing certificate.", exc_info=True)
print(
"[!] Failed to reissue certificates. Reason: {}".format(
e
@ -245,6 +246,7 @@ def reissue(old_certificate_name, commit):
print("[+] Done!")
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error reissuing certificate.", exc_info=True)
print(
"[!] Failed to reissue certificates. Reason: {}".format(
e

2
lemur/certificates/hooks.py
View File

@ -3,7 +3,7 @@ Debugging hooks for dumping imported or generated CSR and certificate details to
.. module: lemur.certificates.hooks
:platform: Unix
:copyright: (c) 2016-2017 by Marti Raudsepp, see AUTHORS for more
:copyright: (c) 2018 by Marti Raudsepp, see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Marti Raudsepp <marti@juffo.org>

5
lemur/certificates/models.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificates.models
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
@ -102,6 +102,7 @@ class Certificate(db.Model):
serial = Column(String(128))
cn = Column(String(128))
deleted = Column(Boolean, index=True)
dns_provider_id = Column(Integer(), ForeignKey('dns_providers.id', ondelete='cascade'), nullable=True)
not_before = Column(ArrowType)
not_after = Column(ArrowType)
@ -177,6 +178,8 @@ class Certificate(db.Model):
self.signing_algorithm = defaults.signing_algorithm(cert)
self.bits = defaults.bitstrength(cert)
self.external_id = kwargs.get('external_id')
self.authority_id = kwargs.get('authority_id')
self.dns_provider_id = kwargs.get('dns_provider_id')
for domain in defaults.domains(cert):
self.domains.append(Domain(name=domain))

36
lemur/certificates/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificates.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
@ -9,31 +9,30 @@ from flask import current_app
from marshmallow import fields, validate, validates_schema, post_load, pre_load
from marshmallow.exceptions import ValidationError
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.common import validators, missing
from lemur.common.fields import ArrowDateTime, Hex
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.constants import CERTIFICATE_KEY_TYPES
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.notifications import service as notification_service
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.schemas import (
AssociatedAuthoritySchema,
AssociatedDestinationSchema,
AssociatedCertificateSchema,
AssociatedNotificationSchema,
AssociatedDnsProviderSchema,
PluginInputSchema,
ExtensionSchema,
AssociatedRoleSchema,
EndpointNestedOutputSchema,
AssociatedRotationPolicySchema
AssociatedRotationPolicySchema,
)
from lemur.authorities.schemas import AuthorityNestedOutputSchema
from lemur.destinations.schemas import DestinationNestedOutputSchema
from lemur.notifications.schemas import NotificationNestedOutputSchema
from lemur.roles.schemas import RoleNestedOutputSchema
from lemur.domains.schemas import DomainNestedOutputSchema
from lemur.users.schemas import UserNestedOutputSchema
from lemur.policies.schemas import RotationPolicyNestedOutputSchema
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from lemur.common import validators, missing
from lemur.notifications import service as notification_service
from lemur.common.fields import ArrowDateTime, Hex
class CertificateSchema(LemurInputSchema):
@ -70,9 +69,13 @@ class CertificateInputSchema(CertificateCreationSchema):
replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True) # deprecated
roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
dns_provider = fields.Nested(AssociatedDnsProviderSchema, missing=None, allow_none=True, required=False)
csr = fields.String(validate=validators.csr)
key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')
key_type = fields.String(
validate=validate.OneOf(CERTIFICATE_KEY_TYPES),
missing='RSA2048')
notify = fields.Boolean(default=True)
rotation = fields.Boolean()
@ -183,6 +186,7 @@ class CertificateOutputSchema(LemurOutputSchema):
description = fields.String()
issuer = fields.String()
name = fields.String()
dns_provider_id = fields.Integer(required=False, allow_none=True)
rotation = fields.Boolean()

2
lemur/certificates/service.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificate.service
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/certificates/verify.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificates.verify
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/certificates/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.certificates.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/common/fields.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.common.fields
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/common/health.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.common.health
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/common/managers.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.common.managers
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

3
lemur/common/schema.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.common.schema
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
@ -135,7 +135,6 @@ def unwrap_pagination(data, output_schema):
marshaled_data = {'total': len(data)}
marshaled_data['items'] = output_schema.dump(data, many=True).data
return marshaled_data
return output_schema.dump(data).data

8
lemur/common/utils.py
View File

@ -1,22 +1,20 @@
"""
.. module: lemur.common.utils
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import string
import random
import string
import sqlalchemy
from sqlalchemy import and_, func
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.asymmetric import rsa, ec
from flask_restful.reqparse import RequestParser
from sqlalchemy import and_, func
from lemur.constants import CERTIFICATE_KEY_TYPES
from lemur.exceptions import InvalidConfiguration

2
lemur/constants.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.constants
:copyright: (c) 2015 by Netflix Inc.
:copyright: (c) 2018 by Netflix Inc.
:license: Apache, see LICENSE for more details.
"""
SAN_NAMING_TEMPLATE = "SAN-{subject}-{issuer}-{not_before}-{not_after}"

2
lemur/database.py
View File

@ -4,7 +4,7 @@
:synopsis: This module contains all of the database related methods
needed for lemur to interact with a datastore
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/defaults/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.defaults.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

7
lemur/defaults/views.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.defaults.views
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
"""
from flask import current_app, Blueprint
@ -50,7 +50,8 @@ class LemurDefaults(AuthenticatedResource):
"state": "CA",
"location": "Los Gatos",
"organization": "Netflix",
"organizationalUnit": "Operations"
"organizationalUnit": "Operations",
"dnsProviders": [{"name": "test", ...}, {...}],
}
:reqheader Authorization: OAuth token to authenticate
@ -67,7 +68,7 @@ class LemurDefaults(AuthenticatedResource):
organization=current_app.config.get('LEMUR_DEFAULT_ORGANIZATION'),
organizational_unit=current_app.config.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT'),
issuer_plugin=current_app.config.get('LEMUR_DEFAULT_ISSUER_PLUGIN'),
authority=default_authority
authority=default_authority,
)

2
lemur/destinations/models.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.destinations.models
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/destinations/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.destinations.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/destinations/service.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.destinations.service
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/destinations/views.py
View File

@ -2,7 +2,7 @@
.. module: lemur.destinations.views
:platform: Unix
:synopsis: This module contains all of the accounts view code.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

37
lemur/dns_providers/models.py Normal file
View File

@ -0,0 +1,37 @@
from sqlalchemy import Column, Integer, String, text, Text
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy_utils import ArrowType
from lemur.database import db
from lemur.plugins.base import plugins
from lemur.utils import Vault
class DnsProvider(db.Model):
__tablename__ = 'dns_providers'
id = Column(
Integer(),
primary_key=True,
)
name = Column(String(length=256), unique=True, nullable=True)
description = Column(Text(), nullable=True)
provider_type = Column(String(length=256), nullable=True)
credentials = Column(Vault, nullable=True)
api_endpoint = Column(String(length=256), nullable=True)
date_created = Column(ArrowType(), server_default=text('now()'), nullable=False)
status = Column(String(length=128), nullable=True)
options = Column(JSON, nullable=True)
domains = Column(JSON, nullable=True)
def __init__(self, name, description, provider_type, credentials):
self.name = name
self.description = description
self.provider_type = provider_type
self.credentials = credentials
@property
def plugin(self):
return plugins.get(self.plugin_name)
def __repr__(self):
return "DnsProvider(name={name})".format(name=self.name)

27
lemur/dns_providers/schemas.py Normal file
View File

@ -0,0 +1,27 @@
from lemur.common.fields import ArrowDateTime
from lemur.common.schema import LemurInputSchema, LemurOutputSchema
from marshmallow import fields
class DnsProvidersNestedOutputSchema(LemurOutputSchema):
__envelope__ = False
id = fields.Integer()
name = fields.String()
providerType = fields.String()
description = fields.String()
credentials = fields.String()
api_endpoint = fields.String()
date_created = ArrowDateTime()
class DnsProvidersNestedInputSchema(LemurInputSchema):
__envelope__ = False
name = fields.String()
description = fields.String()
provider_type = fields.Dict()
dns_provider_output_schema = DnsProvidersNestedOutputSchema()
dns_provider_input_schema = DnsProvidersNestedInputSchema()

111
lemur/dns_providers/service.py Normal file
View File

@ -0,0 +1,111 @@
import json
from flask import current_app
from lemur import database
from lemur.dns_providers.models import DnsProvider
def render(args):
"""
Helper that helps us render the REST Api responses.
:param args:
:return:
"""
query = database.session_query(DnsProvider)
return database.sort_and_page(query, DnsProvider, args)
def get(dns_provider_id):
provider = database.get(DnsProvider, dns_provider_id)
return provider
def get_friendly(dns_provider_id):
"""
Retrieves a dns provider by its lemur assigned ID.
:param dns_provider_id: Lemur assigned ID
:rtype : DnsProvider
:return:
"""
dns_provider = get(dns_provider_id)
dns_provider_friendly = {
"name": dns_provider.name,
"description": dns_provider.description,
"providerType": dns_provider.provider_type,
"options": dns_provider.options,
"credentials": dns_provider.credentials,
}
if dns_provider.provider_type == "route53":
dns_provider_friendly["account_id"] = json.loads(dns_provider.credentials).get("account_id")
return dns_provider_friendly
def delete(dns_provider_id):
"""
Deletes a DNS provider.
:param dns_provider_id: Lemur assigned ID
"""
database.delete(get(dns_provider_id))
def get_types():
provider_config = current_app.config.get(
'ACME_DNS_PROVIDER_TYPES',
{"items": [
{
'name': 'route53',
'requirements': [
{
'name': 'account_id',
'type': 'int',
'required': True,
'helpMessage': 'AWS Account number'
},
]
},
{
'name': 'cloudflare',
'requirements': [
{
'name': 'email',
'type': 'str',
'required': True,
'helpMessage': 'Cloudflare Email'
},
{
'name': 'key',
'type': 'str',
'required': True,
'helpMessage': 'Cloudflare Key'
},
]
},
{
'name': 'dyn',
},
]}
)
if not provider_config:
raise Exception("No DNS Provider configuration specified.")
provider_config["total"] = len(provider_config.get("items"))
return provider_config
def create(data):
provider_name = data.get("name")
credentials = {}
for item in data.get("provider_type", {}).get("requirements", []):
credentials[item["name"]] = item["value"]
dns_provider = DnsProvider(
name=provider_name,
description=data.get("description"),
provider_type=data.get("provider_type").get("name"),
credentials=json.dumps(credentials),
)
created = database.create(dns_provider)
return created.id

170
lemur/dns_providers/views.py Normal file
View File

@ -0,0 +1,170 @@
"""
.. module: lemur.dns)providers.views
:platform: Unix
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
"""
from flask import Blueprint, g
from flask_restful import reqparse, Api
from lemur.auth.permissions import admin_permission
from lemur.auth.service import AuthenticatedResource
from lemur.common.schema import validate_schema
from lemur.common.utils import paginated_parser
from lemur.dns_providers import service
from lemur.dns_providers.schemas import dns_provider_output_schema, dns_provider_input_schema
mod = Blueprint('dns_providers', __name__)
api = Api(mod)
class DnsProvidersList(AuthenticatedResource):
""" Defines the 'dns_providers' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(DnsProvidersList, self).__init__()
@validate_schema(None, dns_provider_output_schema)
def get(self):
"""
.. http:get:: /dns_providers
The current list of DNS Providers
**Example request**:
.. sourcecode:: http
GET /dns_providers HTTP/1.1
Host: example.com
Accept: application/json, text/javascript
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Vary: Accept
Content-Type: text/javascript
{
"items": [{
"id": 1,
"name": "test",
"description": "test",
"provider_type": "dyn",
"status": "active",
}],
"total": 1
}
:query sortBy: field to sort on
:query sortDir: asc or desc
:query page: int. default is 1
:query filter: key value pair format is k;v
:query count: count number. default is 10
:reqheader Authorization: OAuth token to authenticate
:statuscode 200: no error
:statuscode 403: unauthenticated
"""
parser = paginated_parser.copy()
parser.add_argument('dns_provider_id', type=int, location='args')
parser.add_argument('name', type=str, location='args')
parser.add_argument('type', type=str, location='args')
args = parser.parse_args()
args['user'] = g.user
return service.render(args)
@validate_schema(dns_provider_input_schema, None)
@admin_permission.require(http_exception=403)
def post(self, data=None):
"""
Creates a DNS Provider
**Example request**:
{
"providerType": {
"name": "route53",
"requirements": [
{
"name": "account_id",
"type": "int",
"required": true,
"helpMessage": "AWS Account number",
"value": 12345
}
],
"route": "dns_provider_options",
"reqParams": null,
"restangularized": true,
"fromServer": true,
"parentResource": null,
"restangularCollection": false
},
"name": "provider_name",
"description": "provider_description"
}
**Example request 2**
{
"providerType": {
"name": "cloudflare",
"requirements": [
{
"name": "email",
"type": "str",
"required": true,
"helpMessage": "Cloudflare Email",
"value": "test@example.com"
},
{
"name": "key",
"type": "str",
"required": true,
"helpMessage": "Cloudflare Key",
"value": "secretkey"
}
],
"route": "dns_provider_options",
"reqParams": null,
"restangularized": true,
"fromServer": true,
"parentResource": null,
"restangularCollection": false
},
"name": "provider_name",
"description": "provider_description"
}
:return:
"""
return service.create(data)
class DnsProviders(AuthenticatedResource):
@validate_schema(None, dns_provider_output_schema)
def get(self, dns_provider_id):
return service.get_friendly(dns_provider_id)
@admin_permission.require(http_exception=403)
def delete(self, dns_provider_id):
service.delete(dns_provider_id)
return {'result': True}
class DnsProviderOptions(AuthenticatedResource):
""" Defines the 'dns_provider_types' endpoint """
def __init__(self):
self.reqparse = reqparse.RequestParser()
super(DnsProviderOptions, self).__init__()
def get(self):
return service.get_types()
api.add_resource(DnsProvidersList, '/dns_providers', endpoint='dns_providers')
api.add_resource(DnsProviders, '/dns_providers/<int:dns_provider_id>', endpoint='dns_provider')
api.add_resource(DnsProviderOptions, '/dns_provider_options', endpoint='dns_provider_options')

2
lemur/domains/models.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.domains.models
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/domains/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.domains.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/domains/service.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.domains.service
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/domains/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.domains.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/endpoints/cli.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.endpoints.cli
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/endpoints/models.py
View File

@ -2,7 +2,7 @@
.. module: lemur.endpoints.models
:platform: unix
:synopsis: This module contains all of the models need to create an authority within Lemur.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/endpoints/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.endpoints.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/endpoints/service.py
View File

@ -3,7 +3,7 @@
:platform: Unix
:synopsis: This module contains all of the services level functions used to
administer endpoints in Lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/endpoints/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.endpoints.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

10
lemur/exceptions.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.exceptions
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
"""
from flask import current_app
@ -34,3 +34,11 @@ class AttrNotFound(LemurException):
class InvalidConfiguration(Exception):
pass
class InvalidAuthority(Exception):
pass
class UnknownProvider(Exception):
pass

2
lemur/extensions.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.extensions
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
"""
from flask_sqlalchemy import SQLAlchemy

2
lemur/factory.py
View File

@ -4,7 +4,7 @@
:synopsis: This module contains all the needed functions to allow
the factory app creation.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/logs/models.py
View File

@ -2,7 +2,7 @@
.. module: lemur.logs.models
:platform: unix
:synopsis: This module contains all of the models related private key audit log.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/logs/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.logs.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/logs/service.py
View File

@ -3,7 +3,7 @@
:platform: Unix
:synopsis: This module contains all of the services level functions used to
administer logs in Lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/logs/views.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.logs.views
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/metrics.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.metrics
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
"""
from flask import current_app

105
lemur/migrations/versions/3adfdd6598df_.py Normal file
View File

@ -0,0 +1,105 @@
"""Create tables and columns for the acme issuer.
Revision ID: 3adfdd6598df
Revises: 556ceb3e3c3e
Create Date: 2018-04-10 13:25:47.007556
"""
# revision identifiers, used by Alembic.
revision = '3adfdd6598df'
down_revision = '556ceb3e3c3e'
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import JSON
from sqlalchemy_utils import ArrowType
from lemur.utils import Vault
def upgrade():
# create provider table
print("Creating dns_providers table")
op.create_table(
'dns_providers',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=256), nullable=True),
sa.Column('description', sa.String(length=1024), nullable=True),
sa.Column('provider_type', sa.String(length=256), nullable=True),
sa.Column('credentials', Vault(), nullable=True),
sa.Column('api_endpoint', sa.String(length=256), nullable=True),
sa.Column('date_created', ArrowType(), server_default=sa.text('now()'), nullable=False),
sa.Column('status', sa.String(length=128), nullable=True),
sa.Column('options', JSON),
sa.Column('domains', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
print("Adding dns_provider_id column to certificates")
op.add_column('certificates', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
print("Adding dns_provider_id column to pending_certs")
op.add_column('pending_certs', sa.Column('dns_provider_id', sa.Integer(), nullable=True))
print("Adding options column to pending_certs")
op.add_column('pending_certs', sa.Column('options', JSON))
print("Creating pending_dns_authorizations table")
op.create_table(
'pending_dns_authorizations',
sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
sa.Column('account_number', sa.String(length=128), nullable=True),
sa.Column('domains', JSON, nullable=True),
sa.Column('dns_provider_type', sa.String(length=128), nullable=True),
sa.Column('options', JSON, nullable=True),
)
print("Creating certificates_dns_providers_fk foreign key")
op.create_foreign_key('certificates_dns_providers_fk', 'certificates', 'dns_providers', ['dns_provider_id'], ['id'],
ondelete='cascade')
print("Altering column types in the api_keys table")
op.alter_column('api_keys', 'issued_at',
existing_type=sa.BIGINT(),
nullable=True)
op.alter_column('api_keys', 'revoked',
existing_type=sa.BOOLEAN(),
nullable=True)
op.alter_column('api_keys', 'ttl',
existing_type=sa.BIGINT(),
nullable=True)
op.alter_column('api_keys', 'user_id',
existing_type=sa.INTEGER(),
nullable=True)
print("Creating dns_providers_id foreign key on pending_certs table")
op.create_foreign_key(None, 'pending_certs', 'dns_providers', ['dns_provider_id'], ['id'], ondelete='CASCADE')
def downgrade():
print("Removing dns_providers_id foreign key on pending_certs table")
op.drop_constraint(None, 'pending_certs', type_='foreignkey')
print("Reverting column types in the api_keys table")
op.alter_column('api_keys', 'user_id',
existing_type=sa.INTEGER(),
nullable=False)
op.alter_column('api_keys', 'ttl',
existing_type=sa.BIGINT(),
nullable=False)
op.alter_column('api_keys', 'revoked',
existing_type=sa.BOOLEAN(),
nullable=False)
op.alter_column('api_keys', 'issued_at',
existing_type=sa.BIGINT(),
nullable=False)
print("Reverting certificates_dns_providers_fk foreign key")
op.drop_constraint('certificates_dns_providers_fk', 'certificates', type_='foreignkey')
print("Dropping pending_dns_authorizations table")
op.drop_table('pending_dns_authorizations')
print("Undoing modifications to pending_certs table")
op.drop_column('pending_certs', 'options')
op.drop_column('pending_certs', 'dns_provider_id')
print("Undoing modifications to certificates table")
op.drop_column('certificates', 'dns_provider_id')
print("Deleting dns_providers table")
op.drop_table('dns_providers')

4
lemur/models.py
View File

@ -4,7 +4,7 @@
:synopsis: This module contains all of the associative tables
that help define the many to many relationships established in Lemur
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
@ -55,7 +55,7 @@ certificate_replacement_associations = db.Table('certificate_replacement_associa
ForeignKey('certificates.id', ondelete='cascade'))
)
Index('certificate_replacement_associations_ix', certificate_replacement_associations.c.replaced_certificate_id, certificate_replacement_associations.c.certificate_id)
Index('certificate_replacement_associations_ix', certificate_replacement_associations.c.replaced_certificate_id, certificate_replacement_associations.c.certificate_id, unique=True)
roles_authorities = db.Table('roles_authorities',
Column('authority_id', Integer, ForeignKey('authorities.id')),

2
lemur/notifications/cli.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.notifications.cli
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/notifications/messaging.py
View File

@ -2,7 +2,7 @@
.. module: lemur.notifications.messaging
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/notifications/models.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.notifications.models
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/notifications/schemas.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.notifications.schemas
:platform: unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/notifications/service.py
View File

@ -2,7 +2,7 @@
.. module: lemur.notifications.service
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/notifications/views.py
View File

@ -2,7 +2,7 @@
.. module: lemur.notifications.views
:platform: Unix
:synopsis: This module contains all of the accounts view code.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

60
lemur/pending_certificates/cli.py
View File

@ -2,10 +2,11 @@
.. module: lemur.pending_certificates.cli
.. moduleauthor:: James Chuong <jchuong@instartlogic.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
"""
from flask_script import Manager
from lemur.authorities.service import get as get_authority
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
@ -22,14 +23,14 @@ def fetch(ids):
ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
`python manager.py pending_certs fetch -i 123 321 all`
"""
new = 0
failed = 0
pending_certs = pending_certificate_service.get_pending_certs(ids)
user = user_service.get_by_username('lemur')
new = 0
failed = 0
for cert in pending_certs:
authority = plugins.get(cert.authority.plugin_name)
real_cert = authority.get_ordered_certificate(cert.external_id)
real_cert = authority.get_ordered_certificate(cert)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
@ -43,6 +44,55 @@ def fetch(ids):
print(
"[+] Certificates: New: {new} Failed: {failed}".format(
new=new,
failed=failed
failed=failed,
)
)
@manager.command
def fetch_all_acme():
"""
Attempt to get full certificates for each pending certificate listed with the acme-issuer. This is more efficient
for acme-issued certificates because it will configure all of the DNS challenges prior to resolving any
certificates.
"""
pending_certs = pending_certificate_service.get_pending_certs('all')
user = user_service.get_by_username('lemur')
new = 0
failed = 0
wrong_issuer = 0
acme_certs = []
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
acme_certs.append(cert)
else:
wrong_issuer += 1
authority = plugins.get("acme-issuer")
resolved_certs = authority.get_ordered_certificates(acme_certs)
for cert in resolved_certs:
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# add metrics to metrics extension
new += 1
else:
pending_certificate_service.increment_attempt(pending_cert)
failed += 1
print(
"[+] Certificates: New: {new} Failed: {failed} Not using ACME: {wrong_issuer}".format(
new=new,
failed=failed,
wrong_issuer=wrong_issuer
)
)

9
lemur/pending_certificates/models.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.pending_certificates.models
Copyright (c) 2017 and onwards Instart Logic, Inc. All rights reserved.
Copyright (c) 2018 and onwards Netflix, Inc. All rights reserved.
.. moduleauthor:: James Chuong <jchuong@instartlogic.com>
"""
from datetime import datetime as dt
@ -8,6 +8,7 @@ from datetime import datetime as dt
from sqlalchemy.orm import relationship
from sqlalchemy import Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
from sqlalchemy_utils.types.arrow import ArrowType
from sqlalchemy_utils import JSONType
import lemur.common.utils
from lemur.certificates.models import get_or_increase_name
@ -37,6 +38,7 @@ class PendingCertificate(db.Model):
private_key = Column(Vault, nullable=True)
date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
dns_provider_id = Column(Integer, ForeignKey('dns_providers.id', ondelete="CASCADE"))
status = Column(String(128))
@ -54,6 +56,7 @@ class PendingCertificate(db.Model):
secondary=pending_cert_replacement_associations,
backref='pending_cert',
passive_deletes=True)
options = Column(JSONType)
rotation_policy = relationship("RotationPolicy")
@ -93,3 +96,7 @@ class PendingCertificate(db.Model):
self.replaces = kwargs.get('replaces', [])
self.rotation = kwargs.get('rotation')
self.rotation_policy = kwargs.get('rotation_policy')
try:
self.dns_provider_id = kwargs.get('dns_provider').id
except (AttributeError, KeyError, TypeError, Exception):
pass

6
lemur/pending_certificates/service.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.pending_certificates.service
Copyright (c) 2017 and onwards Instart Logic, Inc. All rights reserved.
Copyright (c) 2018 and onwards Netflix, Inc. All rights reserved.
.. moduleauthor:: James Chuong <jchuong@instartlogic.com>
"""
import arrow
@ -59,6 +59,10 @@ def delete(pending_certificate):
database.delete(pending_certificate)
def delete_by_id(id):
database.delete(get(id))
def get_pending_certs(pending_ids):
"""
Retrieve a list of pending certs given a list of ids

2
lemur/plugins/base/__init__.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.base
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/base/manager.py
View File

@ -1,6 +1,6 @@
"""
.. module: lemur.plugins.base.manager
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson (kglisson@netflix.com)

2
lemur/plugins/base/v1.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.base.v1
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/bases/destination.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.destination
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/bases/export.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.export
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

4
lemur/plugins/bases/issuer.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.issuer
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
@ -25,7 +25,7 @@ class IssuerPlugin(Plugin):
def revoke_certificate(self, certificate, comments):
raise NotImplementedError
def get_ordered_certificate(self, order_id):
def get_ordered_certificate(self, certificate):
raise NotImplementedError
def cancel_ordered_certificate(self, pending_cert, **kwargs):

2
lemur/plugins/bases/metric.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.metric
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/bases/notification.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.notification
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/bases/source.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.bases.source
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/lemur_acme/cloudflare.py
View File

@ -1,6 +1,6 @@
import time
import CloudFlare
import CloudFlare
from flask import current_app

131
lemur/plugins/lemur_acme/dyn.py Normal file
View File

@ -0,0 +1,131 @@
import time
import dns
import dns.exception
import dns.name
import dns.query
import dns.resolver
from dyn.tm.errors import DynectCreateError
from dyn.tm.session import DynectSession
from dyn.tm.zones import Node, Zone
from flask import current_app
from tld import get_tld
def get_dynect_session():
dynect_session = DynectSession(
current_app.config.get('ACME_DYN_CUSTOMER_NAME', ''),
current_app.config.get('ACME_DYN_USERNAME', ''),
current_app.config.get('ACME_DYN_PASSWORD', ''),
)
return dynect_session
def _has_dns_propagated(name, token):
txt_records = []
try:
dns_resolver = dns.resolver.Resolver()
dns_resolver.nameservers = [get_authoritative_nameserver(name)]
dns_response = dns_resolver.query(name, 'TXT')
for rdata in dns_response:
for txt_record in rdata.strings:
txt_records.append(txt_record.decode("utf-8"))
except dns.exception.DNSException:
return False
for txt_record in txt_records:
if txt_record == token:
return True
return False
def wait_for_dns_change(change_id, account_number=None):
fqdn, token = change_id
while True:
status = _has_dns_propagated(fqdn, token)
current_app.logger.debug("Record status for fqdn: {}: {}".format(fqdn, status))
if status:
break
time.sleep(20)
return
def create_txt_record(domain, token, account_number):
get_dynect_session()
zone_name = get_tld('http://' + domain)
zone_parts = len(zone_name.split('.'))
node_name = '.'.join(domain.split('.')[:-zone_parts])
fqdn = "{0}.{1}".format(node_name, zone_name)
zone = Zone(zone_name)
try:
zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
except DynectCreateError:
delete_txt_record(None, None, domain, token)
zone.add_record(node_name, record_type='TXT', txtdata="\"{}\"".format(token), ttl=5)
node = zone.get_node(node_name)
zone.publish()
current_app.logger.debug("TXT record created: {0}".format(fqdn))
change_id = (fqdn, token)
return change_id
def delete_txt_record(change_id, account_number, domain, token):
get_dynect_session()
if not domain:
current_app.logger.debug("delete_txt_record: No domain passed")
return
zone_name = get_tld('http://' + domain)
zone_parts = len(zone_name.split('.'))
node_name = '.'.join(domain.split('.')[:-zone_parts])
fqdn = "{0}.{1}".format(node_name, zone_name)
zone = Zone(zone_name)
node = Node(zone_name, fqdn)
all_txt_records = node.get_all_records_by_type('TXT')
for txt_record in all_txt_records:
if txt_record.txtdata == ("{}".format(token)):
current_app.logger.debug("Deleting TXT record name: {0}".format(fqdn))
txt_record.delete()
zone.publish()
def get_authoritative_nameserver(domain):
n = dns.name.from_text(domain)
depth = 2
default = dns.resolver.get_default_resolver()
nameserver = default.nameservers[0]
last = False
while not last:
s = n.split(depth)
last = s[0].to_unicode() == u'@'
sub = s[1]
query = dns.message.make_query(sub, dns.rdatatype.NS)
response = dns.query.udp(query, nameserver)
rcode = response.rcode()
if rcode != dns.rcode.NOERROR:
if rcode == dns.rcode.NXDOMAIN:
raise Exception('%s does not exist.' % sub)
else:
raise Exception('Error %s' % dns.rcode.to_text(rcode))
if len(response.authority) > 0:
rrset = response.authority[0]
else:
rrset = response.answer[0]
rr = rrset[0]
if rr.rdtype != dns.rdatatype.SOA:
authority = rr.target
nameserver = default.query(authority).rrset[0].to_text()
depth += 1
return nameserver

365
lemur/plugins/lemur_acme/plugin.py
View File

@ -2,38 +2,44 @@
.. module: lemur.plugins.lemur_acme.plugin
:platform: Unix
:synopsis: This module is responsible for communicating with an ACME CA.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
Snippets from https://raw.githubusercontent.com/alex/letsencrypt-aws/master/letsencrypt-aws.py
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
.. moduleauthor:: Mikhail Khodorovskiy <mikhail.khodorovskiy@jivesoftware.com>
.. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
"""
import josepy as jose
from flask import current_app
from acme.client import Client
from acme import messages
from acme import challenges
from lemur.common.utils import generate_private_key
import datetime
import json
import time
import OpenSSL.crypto
import josepy as jose
from acme import challenges, messages
from acme.client import BackwardsCompatibleClientV2, ClientNetwork
from acme.messages import Error as AcmeError
from acme.errors import PollError, WildcardUnsupportedError
from botocore.exceptions import ClientError
from flask import current_app
from lemur.common.utils import validate_conf
from lemur.plugins.bases import IssuerPlugin
from lemur.authorizations import service as authorization_service
from lemur.common.utils import generate_private_key
from lemur.dns_providers import service as dns_provider_service
from lemur.exceptions import InvalidAuthority, InvalidConfiguration, UnknownProvider
from lemur.plugins import lemur_acme as acme
from lemur.plugins.bases import IssuerPlugin
from lemur.plugins.lemur_acme import cloudflare, dyn, route53
def find_dns_challenge(authz):
for combo in authz.body.resolved_combinations:
if (
len(combo) == 1 and
isinstance(combo[0].chall, challenges.DNS01)
):
yield combo[0]
def find_dns_challenge(authorizations):
dns_challenges = []
for authz in authorizations:
for combo in authz.body.challenges:
if isinstance(combo.chall, challenges.DNS01):
dns_challenges.append(combo)
return dns_challenges
class AuthorizationRecord(object):
@ -44,85 +50,90 @@ class AuthorizationRecord(object):
self.change_id = change_id
def start_dns_challenge(acme_client, account_number, host, dns_provider):
def maybe_remove_wildcard(host):
return host.replace("*.", "")
def start_dns_challenge(acme_client, account_number, host, dns_provider, order):
current_app.logger.debug("Starting DNS challenge for {0}".format(host))
authz = acme_client.request_domain_challenges(host)
[dns_challenge] = find_dns_challenge(authz)
dns_challenges = find_dns_challenge(order.authorizations)
change_ids = []
change_id = dns_provider.create_txt_record(
dns_challenge.validation_domain_name(host),
dns_challenge.validation(acme_client.key),
account_number
)
for dns_challenge in find_dns_challenge(order.authorizations):
change_id = dns_provider.create_txt_record(
dns_challenge.validation_domain_name(maybe_remove_wildcard(host)),
dns_challenge.validation(acme_client.client.net.key),
account_number
)
change_ids.append(change_id)
return AuthorizationRecord(
host,
authz,
dns_challenge,
change_id,
order.authorizations,
dns_challenges,
change_ids
)
def complete_dns_challenge(acme_client, account_number, authz_record, dns_provider):
dns_provider.wait_for_dns_change(authz_record.change_id, account_number=account_number)
current_app.logger.debug("Finalizing DNS challenge for {0}".format(authz_record.authz[0].body.identifier.value))
for change_id in authz_record.change_id:
dns_provider.wait_for_dns_change(change_id, account_number=account_number)
response = authz_record.dns_challenge.response(acme_client.key)
for dns_challenge in authz_record.dns_challenge:
verified = response.simple_verify(
authz_record.dns_challenge.chall,
authz_record.host,
acme_client.key.public_key()
)
response = dns_challenge.response(acme_client.client.net.key)
if not verified:
raise ValueError("Failed verification")
verified = response.simple_verify(
dns_challenge.chall,
authz_record.host,
acme_client.client.net.key.public_key()
)
acme_client.answer_challenge(authz_record.dns_challenge, response)
if not verified:
raise ValueError("Failed verification")
time.sleep(5)
acme_client.answer_challenge(dns_challenge, response)
def request_certificate(acme_client, authorizations, csr):
cert_response, _ = acme_client.poll_and_request_issuance(
jose.util.ComparableX509(
OpenSSL.crypto.load_certificate_request(
OpenSSL.crypto.FILETYPE_PEM,
csr
)
),
authzrs=[authz_record.authz for authz_record in authorizations],
)
def request_certificate(acme_client, authorizations, csr, order):
for authorization in authorizations:
for authz in authorization.authz:
authorization_resource, _ = acme_client.poll(authz)
pem_certificate = OpenSSL.crypto.dump_certificate(
OpenSSL.crypto.FILETYPE_PEM, cert_response.body
).decode('utf-8')
deadline = datetime.datetime.now() + datetime.timedelta(seconds=90)
orderr = acme_client.finalize_order(order, deadline)
pem_certificate = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM,
OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM,
orderr.fullchain_pem)).decode()
pem_certificate_chain = orderr.fullchain_pem[len(pem_certificate):].lstrip()
pem_certificate_chain = "\n".join(
OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert.decode("utf-8"))
for cert in acme_client.fetch_chain(cert_response)
).decode('utf-8')
current_app.logger.debug("{0} {1}".format(type(pem_certificate). type(pem_certificate_chain)))
current_app.logger.debug("{0} {1}".format(type(pem_certificate), type(pem_certificate_chain)))
return pem_certificate, pem_certificate_chain
def setup_acme_client():
email = current_app.config.get('ACME_EMAIL')
tel = current_app.config.get('ACME_TEL')
directory_url = current_app.config.get('ACME_DIRECTORY_URL')
contact = ('mailto:{}'.format(email), 'tel:{}'.format(tel))
def setup_acme_client(authority):
if not authority.options:
raise InvalidAuthority("Invalid authority. Options not set")
options = {}
for option in json.loads(authority.options):
options[option["name"]] = option.get("value")
email = options.get('email', current_app.config.get('ACME_EMAIL'))
tel = options.get('telephone', current_app.config.get('ACME_TEL'))
directory_url = options.get('acme_url', current_app.config.get('ACME_DIRECTORY_URL'))
key = jose.JWKRSA(key=generate_private_key('RSA2048'))
current_app.logger.debug("Connecting with directory at {0}".format(directory_url))
client = Client(directory_url, key)
registration = client.register(
messages.NewRegistration.from_data(email=email)
)
net = ClientNetwork(key, account=None)
client = BackwardsCompatibleClientV2(net, key, directory_url)
registration = client.new_account_and_tos(messages.NewRegistration.from_data(email=email))
current_app.logger.debug("Connected: {0}".format(registration.uri))
client.agree_to_tos(registration)
return client, registration
@ -143,23 +154,25 @@ def get_domains(options):
return domains
def get_authorizations(acme_client, account_number, domains, dns_provider):
def get_authorizations(acme_client, order, order_info, dns_provider):
authorizations = []
try:
for domain in domains:
authz_record = start_dns_challenge(acme_client, account_number, domain, dns_provider)
authorizations.append(authz_record)
for domain in order_info.domains:
authz_record = start_dns_challenge(acme_client, order_info.account_number, domain, dns_provider, order)
authorizations.append(authz_record)
return authorizations
for authz_record in authorizations:
complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
finally:
for authz_record in authorizations:
dns_challenge = authz_record.dns_challenge
def finalize_authorizations(acme_client, account_number, dns_provider, authorizations):
for authz_record in authorizations:
complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
for authz_record in authorizations:
dns_challenges = authz_record.dns_challenge
for dns_challenge in dns_challenges:
dns_provider.delete_txt_record(
authz_record.change_id,
account_number,
dns_challenge.validation_domain_name(authz_record.host),
dns_challenge.validation(acme_client.key)
dns_challenge.validation_domain_name(maybe_remove_wildcard(authz_record.host)),
dns_challenge.validation(acme_client.client.net.key)
)
return authorizations
@ -171,24 +184,139 @@ class ACMEIssuerPlugin(IssuerPlugin):
description = 'Enables the creation of certificates via ACME CAs (including Let\'s Encrypt)'
version = acme.VERSION
author = 'Kevin Glisson'
author = 'Netflix'
author_url = 'https://github.com/netflix/lemur.git'
def __init__(self, *args, **kwargs):
required_vars = [
'ACME_DIRECTORY_URL',
'ACME_TEL',
'ACME_EMAIL',
'ACME_AWS_ACCOUNT_NUMBER',
'ACME_ROOT'
]
options = [
{
'name': 'acme_url',
'type': 'str',
'required': True,
'validation': '/^http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$/',
'helpMessage': 'Must be a valid web url starting with http[s]://',
},
{
'name': 'telephone',
'type': 'str',
'default': '',
'helpMessage': 'Telephone to use'
},
{
'name': 'email',
'type': 'str',
'default': '',
'validation': '/^?([-a-zA-Z0-9.`?{}]+@\w+\.\w+)$/',
'helpMessage': 'Email to use'
},
{
'name': 'certificate',
'type': 'textarea',
'default': '',
'validation': '/^-----BEGIN CERTIFICATE-----/',
'helpMessage': 'Certificate to use'
},
]
validate_conf(current_app, required_vars)
self.dns_provider_name = current_app.config.get('ACME_DNS_PROVIDER', 'route53')
current_app.logger.debug("Using DNS provider: {0}".format(self.dns_provider_name))
self.dns_provider = __import__(self.dns_provider_name, globals(), locals(), [], 1)
def __init__(self, *args, **kwargs):
super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
def get_dns_provider(self, type):
provider_types = {
'cloudflare': cloudflare,
'dyn': dyn,
'route53': route53,
}
provider = provider_types.get(type)
if not provider:
raise UnknownProvider("No such DNS provider: {}".format(type))
return provider
def get_ordered_certificate(self, pending_cert):
acme_client, registration = setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
try:
authorizations = get_authorizations(
acme_client, order_info.account_number, order_info.domains, dns_provider_type)
except ClientError:
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert.name), exc_info=True)
return False
authorizations = finalize_authorizations(
acme_client, order_info.account_number, dns_provider_type, authorizations)
pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr)
cert = {
'body': "\n".join(str(pem_certificate).splitlines()),
'chain': "\n".join(str(pem_certificate_chain).splitlines()),
'external_id': str(pending_cert.external_id)
}
return cert
def get_ordered_certificates(self, pending_certs):
pending = []
certs = []
for pending_cert in pending_certs:
try:
acme_client, registration = setup_acme_client(pending_cert.authority)
order_info = authorization_service.get(pending_cert.external_id)
dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
dns_provider_type = self.get_dns_provider(dns_provider.provider_type)
try:
order = acme_client.new_order(pending_cert.csr)
except WildcardUnsupportedError:
raise Exception("The currently selected ACME CA endpoint does"
" not support issuing wildcard certificates.")
authorizations = get_authorizations(acme_client, order, order_info, dns_provider_type)
pending.append({
"acme_client": acme_client,
"account_number": order_info.account_number,
"dns_provider_type": dns_provider_type,
"authorizations": authorizations,
"pending_cert": pending_cert,
"order": order,
})
except (ClientError, ValueError, Exception):
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
certs.append({
"cert": False,
"pending_cert": pending_cert,
})
for entry in pending:
try:
entry["authorizations"] = finalize_authorizations(
entry["acme_client"],
entry["account_number"],
entry["dns_provider_type"],
entry["authorizations"],
)
pem_certificate, pem_certificate_chain = request_certificate(
entry["acme_client"],
entry["authorizations"],
entry["pending_cert"].csr,
entry["order"]
)
cert = {
'body': "\n".join(str(pem_certificate).splitlines()),
'chain': "\n".join(str(pem_certificate_chain).splitlines()),
'external_id': str(entry["pending_cert"].external_id)
}
certs.append({
"cert": cert,
"pending_cert": entry["pending_cert"],
})
except (PollError, AcmeError, Exception):
current_app.logger.error("Unable to resolve pending cert: {}".format(pending_cert), exc_info=True)
certs.append({
"cert": False,
"pending_cert": entry["pending_cert"],
})
return certs
def create_certificate(self, csr, issuer_options):
"""
Creates an ACME certificate.
@ -197,11 +325,37 @@ class ACMEIssuerPlugin(IssuerPlugin):
:param issuer_options:
:return: :raise Exception:
"""
current_app.logger.debug("Requesting a new acme certificate: {0}".format(issuer_options))
acme_client, registration = setup_acme_client()
account_number = current_app.config.get('ACME_AWS_ACCOUNT_NUMBER')
authority = issuer_options.get('authority')
create_immediately = issuer_options.get('create_immediately', False)
acme_client, registration = setup_acme_client(authority)
dns_provider = issuer_options.get('dns_provider')
if not dns_provider:
raise InvalidConfiguration("DNS Provider setting is required for ACME certificates.")
credentials = json.loads(dns_provider.credentials)
current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type))
dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
account_number = credentials.get("account_id")
if dns_provider.provider_type == 'route53' and not account_number:
error = "Route53 DNS Provider {} does not have an account number configured.".format(dns_provider.name)
current_app.logger.error(error)
raise InvalidConfiguration(error)
domains = get_domains(issuer_options)
authorizations = get_authorizations(acme_client, account_number, domains, self.dns_provider)
if not create_immediately:
# Create pending authorizations that we'll need to do the creation
authz_domains = []
for d in domains:
if type(d) == str:
authz_domains.append(d)
else:
authz_domains.append(d.value)
dns_authorization = authorization_service.create(account_number, authz_domains, dns_provider.provider_type)
# Return id of the DNS Authorization
return None, None, dns_authorization.id
authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type)
finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations)
pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr)
# TODO add external ID (if possible)
return pem_certificate, pem_certificate_chain, None
@ -216,4 +370,15 @@ class ACMEIssuerPlugin(IssuerPlugin):
:return:
"""
role = {'username': '', 'password': '', 'name': 'acme'}
return current_app.config.get('ACME_ROOT'), "", [role]
plugin_options = options.get('plugin', {}).get('plugin_options')
if not plugin_options:
error = "Invalid options for lemur_acme plugin: {}".format(options)
current_app.logger.error(error)
raise InvalidConfiguration(error)
# Define static acme_root based off configuration variable by default. However, if user has passed a
# certificate, use this certificate as the root.
acme_root = current_app.config.get('ACME_ROOT')
for option in plugin_options:
if option.get('name') == 'certificate':
acme_root = option.get('value')
return acme_root, "", [role]

23
lemur/plugins/lemur_acme/route53.py
View File

@ -1,4 +1,5 @@
import time
from lemur.plugins.lemur_aws.sts import sts_client
@ -58,21 +59,23 @@ def change_txt_record(action, zone_id, domain, value, client=None):
def create_txt_record(host, value, account_number):
zone_id = find_zone_id(host, account_number=account_number)
change_id = change_txt_record(
"CREATE",
"UPSERT",
zone_id,
host,
value,
account_number=account_number
)
return zone_id, change_id
def delete_txt_record(change_id, account_number, host, value):
zone_id, _ = change_id
change_txt_record(
"DELETE",
zone_id,
host,
value,
account_number=account_number
)
def delete_txt_record(change_ids, account_number, host, value):
for change_id in change_ids:
zone_id, _ = change_id
change_txt_record(
"DELETE",
zone_id,
host,
value,
account_number=account_number
)

307
lemur/plugins/lemur_acme/tests/test_acme.py
View File

@ -1,4 +1,305 @@
import unittest
def test_get_certificates(app):
from lemur.plugins.base import plugins
p = plugins.get('acme-issuer')
from mock import MagicMock, Mock, patch
from lemur.plugins.lemur_acme import plugin
class TestAcme(unittest.TestCase):
def setUp(self):
self.ACMEIssuerPlugin = plugin.ACMEIssuerPlugin()
@patch('lemur.plugins.lemur_acme.plugin.len', return_value=1)
def test_find_dns_challenge(self, mock_len):
assert mock_len
from acme import challenges
c = challenges.DNS01()
mock_authz = Mock()
mock_authz.body.resolved_combinations = []
mock_entry = Mock()
mock_entry.chall = c
mock_authz.body.resolved_combinations.append(mock_entry)
result = yield plugin.find_dns_challenge(mock_authz)
self.assertEqual(result, mock_entry)
def test_authz_record(self):
a = plugin.AuthorizationRecord("host", "authz", "challenge", "id")
self.assertEqual(type(a), plugin.AuthorizationRecord)
@patch('acme.client.Client')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
@patch('lemur.plugins.lemur_acme.plugin.len', return_value=1)
@patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge')
def test_start_dns_challenge(self, mock_find_dns_challenge, mock_len, mock_app, mock_acme):
assert mock_len
mock_order = Mock()
mock_app.logger.debug = Mock()
mock_authz = Mock()
mock_authz.body.resolved_combinations = []
mock_entry = MagicMock()
from acme import challenges
c = challenges.DNS01()
mock_entry.chall = c
mock_authz.body.resolved_combinations.append(mock_entry)
mock_acme.request_domain_challenges = Mock(return_value=mock_authz)
mock_dns_provider = Mock()
mock_dns_provider.create_txt_record = Mock(return_value=1)
values = [mock_entry]
iterable = mock_find_dns_challenge.return_value
iterator = iter(values)
iterable.__iter__.return_value = iterator
result = plugin.start_dns_challenge(mock_acme, "accountid", "host", mock_dns_provider, mock_order)
self.assertEqual(type(result), plugin.AuthorizationRecord)
@patch('acme.client.Client')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_complete_dns_challenge_success(self, mock_current_app, mock_acme):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
mock_authz = Mock()
mock_authz.dns_challenge.response = Mock()
mock_authz.dns_challenge.response.simple_verify = Mock(return_value=True)
mock_authz.authz = []
mock_authz_record = Mock()
mock_authz_record.body.identifier.value = "test"
mock_authz.authz.append(mock_authz_record)
mock_authz.change_id = []
mock_authz.change_id.append("123")
mock_authz.dns_challenge = []
dns_challenge = Mock()
mock_authz.dns_challenge.append(dns_challenge)
plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
@patch('acme.client.Client')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_complete_dns_challenge_fail(self, mock_current_app, mock_acme):
mock_dns_provider = Mock()
mock_dns_provider.wait_for_dns_change = Mock(return_value=True)
mock_authz = Mock()
mock_authz.dns_challenge.response = Mock()
mock_authz.dns_challenge.response.simple_verify = Mock(return_value=False)
mock_authz.authz = []
mock_authz_record = Mock()
mock_authz_record.body.identifier.value = "test"
mock_authz.authz.append(mock_authz_record)
mock_authz.change_id = []
mock_authz.change_id.append("123")
mock_authz.dns_challenge = []
dns_challenge = Mock()
mock_authz.dns_challenge.append(dns_challenge)
self.assertRaises(
ValueError,
plugin.complete_dns_challenge(mock_acme, "accountid", mock_authz, mock_dns_provider)
)
@patch('acme.client.Client')
@patch('OpenSSL.crypto', return_value="mock_cert")
@patch('josepy.util.ComparableX509')
@patch('lemur.plugins.lemur_acme.plugin.find_dns_challenge')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_request_certificate(self, mock_current_app, mock_find_dns_challenge, mock_jose, mock_crypto, mock_acme):
mock_cert_response = Mock()
mock_cert_response.body = "123"
mock_cert_response_full = [mock_cert_response, True]
mock_acme.poll_and_request_issuance = Mock(return_value=mock_cert_response_full)
mock_authz = []
mock_authz_record = MagicMock()
mock_authz_record.authz = Mock()
mock_authz.append(mock_authz_record)
mock_acme.fetch_chain = Mock(return_value="mock_chain")
mock_crypto.dump_certificate = Mock(return_value=b'chain')
mock_order = Mock()
plugin.request_certificate(mock_acme, [], "mock_csr", mock_order)
def test_setup_acme_client_fail(self):
mock_authority = Mock()
mock_authority.options = []
with self.assertRaises(Exception):
plugin.setup_acme_client(mock_authority)
@patch('lemur.plugins.lemur_acme.plugin.BackwardsCompatibleClientV2')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_setup_acme_client_success(self, mock_current_app, mock_acme):
mock_authority = Mock()
mock_authority.options = '[{"name": "mock_name", "value": "mock_value"}]'
mock_client = Mock()
mock_registration = Mock()
mock_registration.uri = "http://test.com"
mock_client.register = mock_registration
mock_client.agree_to_tos = Mock(return_value=True)
mock_acme.return_value = mock_client
result_client, result_registration = plugin.setup_acme_client(mock_authority)
assert result_client
assert result_registration
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_get_domains_single(self, mock_current_app):
options = {
"common_name": "test.netflix.net"
}
result = plugin.get_domains(options)
self.assertEqual(result, [options["common_name"]])
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_get_domains_multiple(self, mock_current_app):
options = {
"common_name": "test.netflix.net",
"extensions": {
"sub_alt_names": {
"names": [
"test2.netflix.net",
"test3.netflix.net"
]
}
}
}
result = plugin.get_domains(options)
self.assertEqual(result, [options["common_name"], "test2.netflix.net", "test3.netflix.net"])
@patch('lemur.plugins.lemur_acme.plugin.start_dns_challenge', return_value="test")
def test_get_authorizations(self, mock_start_dns_challenge):
mock_order = Mock()
mock_order.body.identifiers = []
mock_domain = Mock()
mock_order.body.identifiers.append(mock_domain)
mock_order_info = Mock()
mock_order_info.account_number = 1
mock_order_info.domains = ["test.fakedomain.net"]
result = plugin.get_authorizations("acme_client", mock_order, mock_order_info, "dns_provider")
self.assertEqual(result, ["test"])
@patch('lemur.plugins.lemur_acme.plugin.complete_dns_challenge', return_value="test")
def test_finalize_authorizations(self, mock_complete_dns_challenge):
mock_authz = []
mock_authz_record = MagicMock()
mock_authz_record.authz = Mock()
mock_authz_record.change_id = 1
mock_authz_record.dns_challenge.validation_domain_name = Mock()
mock_authz_record.dns_challenge.validation = Mock()
mock_authz.append(mock_authz_record)
mock_dns_provider = Mock()
mock_dns_provider.delete_txt_record = Mock()
mock_acme_client = Mock()
result = plugin.finalize_authorizations(mock_acme_client, "account_number", mock_dns_provider, mock_authz)
self.assertEqual(result, mock_authz)
@patch('lemur.plugins.lemur_acme.plugin.current_app')
def test_create_authority(self, mock_current_app):
mock_current_app.config = Mock()
options = {
"plugin": {
"plugin_options": [{
"name": "certificate",
"value": "123"
}]
}
}
acme_root, b, role = self.ACMEIssuerPlugin.create_authority(options)
self.assertEqual(acme_root, "123")
self.assertEqual(b, "")
self.assertEqual(role, [{'username': '', 'password': '', 'name': 'acme'}])
@patch('lemur.plugins.lemur_acme.plugin.current_app')
@patch('lemur.plugins.lemur_acme.dyn.current_app')
@patch('lemur.plugins.lemur_acme.cloudflare.current_app')
def test_get_dns_provider(self, mock_current_app_cloudflare, mock_current_app_dyn, mock_current_app):
provider = plugin.ACMEIssuerPlugin()
route53 = provider.get_dns_provider("route53")
assert route53
cloudflare = provider.get_dns_provider("cloudflare")
assert cloudflare
dyn = provider.get_dns_provider("dyn")
assert dyn
@patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
@patch('lemur.plugins.lemur_acme.plugin.authorization_service')
@patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
@patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.request_certificate')
def test_get_ordered_certificate(
self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations,
mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme):
mock_client = Mock()
mock_acme.return_value = (mock_client, "")
mock_request_certificate.return_value = ("pem_certificate", "chain")
mock_cert = Mock()
mock_cert.external_id = 1
provider = plugin.ACMEIssuerPlugin()
provider.get_dns_provider = Mock()
result = provider.get_ordered_certificate(mock_cert)
self.assertEqual(
result,
{
'body': "pem_certificate",
'chain': "chain",
'external_id': "1"
}
)
@patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
@patch('lemur.plugins.lemur_acme.plugin.authorization_service')
@patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
@patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.request_certificate')
def test_get_ordered_certificates(
self, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations,
mock_dns_provider_service, mock_authorization_service, mock_current_app, mock_acme):
mock_client = Mock()
mock_acme.return_value = (mock_client, "")
mock_request_certificate.return_value = ("pem_certificate", "chain")
mock_cert = Mock()
mock_cert.external_id = 1
mock_cert2 = Mock()
mock_cert2.external_id = 2
provider = plugin.ACMEIssuerPlugin()
provider.get_dns_provider = Mock()
result = provider.get_ordered_certificates([mock_cert, mock_cert2])
self.assertEqual(len(result), 2)
self.assertEqual(result[0]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '1'})
self.assertEqual(result[1]['cert'], {'body': 'pem_certificate', 'chain': 'chain', 'external_id': '2'})
@patch('lemur.plugins.lemur_acme.plugin.setup_acme_client')
@patch('lemur.plugins.lemur_acme.plugin.dns_provider_service')
@patch('lemur.plugins.lemur_acme.plugin.current_app')
@patch('lemur.plugins.lemur_acme.plugin.get_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.finalize_authorizations')
@patch('lemur.plugins.lemur_acme.plugin.request_certificate')
@patch('lemur.plugins.lemur_acme.plugin.authorization_service')
def test_create_certificate(self, mock_authorization_service, mock_request_certificate, mock_finalize_authorizations, mock_get_authorizations,
mock_current_app, mock_dns_provider_service, mock_acme):
provider = plugin.ACMEIssuerPlugin()
mock_authority = Mock()
mock_client = Mock()
mock_acme.return_value = (mock_client, "")
mock_dns_provider = Mock()
mock_dns_provider.credentials = '{"account_id": 1}'
mock_dns_provider.provider_type = "route53"
mock_dns_provider_service.get.return_value = mock_dns_provider
issuer_options = {
'authority': mock_authority,
'dns_provider': mock_dns_provider,
"common_name": "test.netflix.net"
}
csr = "123"
mock_request_certificate.return_value = ("pem_certificate", "chain")
result = provider.create_certificate(csr, issuer_options)
assert result

2
lemur/plugins/lemur_atlas/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_atlas.plugin
:platform: Unix
:copyright: (c) 2016 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/lemur_aws/iam.py
View File

@ -2,7 +2,7 @@
.. module: lemur.plugins.lemur_aws.iam
:platform: Unix
:synopsis: Contains helper functions for interactive with AWS IAM Apis.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/plugins/lemur_aws/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_aws.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
Terraform example to setup the destination bucket:

2
lemur/plugins/lemur_aws/s3.py
View File

@ -2,7 +2,7 @@
.. module: lemur.plugins.lemur_aws.s3
:platform: Unix
:synopsis: Contains helper functions for interactive with AWS S3 Apis.
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/plugins/lemur_aws/sts.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_aws.sts
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""

2
lemur/plugins/lemur_cfssl/plugin.py
View File

@ -2,7 +2,7 @@
.. module: lemur.plugins.lemur_cfssl.plugin
:platform: Unix
:synopsis: This module is responsible for communicating with the CFSSL private CA.
:copyright: (c) 2016 by Thomson Reuters
:copyright: (c) 2018 by Thomson Reuters
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Charles Hendrie <chad.hendrie@tr.com>

2
lemur/plugins/lemur_cryptography/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_cryptography.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

3
lemur/plugins/lemur_digicert/plugin.py
View File

@ -325,8 +325,9 @@ class DigiCertIssuerPlugin(IssuerPlugin):
response = self.session.put(create_url, data=json.dumps({'comments': comments}))
return handle_response(response)
def get_ordered_certificate(self, order_id):
def get_ordered_certificate(self, pending_cert):
""" Retrieve a certificate via order id """
order_id = pending_cert.external_id
base_url = current_app.config.get('DIGICERT_URL')
try:
certificate_id = get_certificate_id(self.session, base_url, order_id)

12
lemur/plugins/lemur_digicert/tests/test_digicert.py
View File

@ -150,11 +150,7 @@ def test_signature_hash(app):
signature_hash('sdfdsf')
def test_issuer_plugin_create_certificate():
import requests_mock
from lemur.plugins.lemur_digicert.plugin import DigiCertIssuerPlugin
pem_fixture = """\
def test_issuer_plugin_create_certificate(certificate_="""\
-----BEGIN CERTIFICATE-----
abc
-----END CERTIFICATE-----
@ -164,7 +160,11 @@ def
-----BEGIN CERTIFICATE-----
ghi
-----END CERTIFICATE-----
"""
"""):
import requests_mock
from lemur.plugins.lemur_digicert.plugin import DigiCertIssuerPlugin
pem_fixture = certificate_
subject = DigiCertIssuerPlugin()
adapter = requests_mock.Adapter()

2
lemur/plugins/lemur_email/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_email.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/lemur_java/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_java.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

2
lemur/plugins/lemur_kubernetes/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_kubernetes.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.

2
lemur/plugins/lemur_openssl/plugin.py
View File

@ -1,7 +1,7 @@
"""
.. module: lemur.plugins.lemur_openssl.plugin
:platform: Unix
:copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
:copyright: (c) 2018 by Netflix Inc., see AUTHORS for more
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>

Some files were not shown because too many files have changed in this diff Show More