Working acme flow. Pending DNS providers UI

lemur/authorizations/models.py

@@ -0,0 +1,34 @@
+"""
+.. module: lemur.authorizations.models
+    :platform: unix
+    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
+    :license: Apache, see LICENSE for more details.
+.. moduleauthor:: Netflix Secops <secops@netflix.com>
+"""
+from sqlalchemy import Column, Integer, String
+from sqlalchemy_utils import JSONType
+from lemur.database import db
+
+from lemur.plugins.base import plugins
+
+
+class Authorizations(db.Model):
+    __tablename__ = 'pending_dns_authorizations'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    account_number = Column(String(128))
+    domains = Column(JSONType)
+    dns_provider_type = Column(String(128))
+    options = Column(JSONType)
+
+    @property
+    def plugin(self):
+        return plugins.get(self.plugin_name)
+
+    def __repr__(self):
+        return "Authorizations(id={id})".format(label=self.id)
+
+    def __init__(self, account_number, domains, dns_provider_type, options=None):
+        self.account_number = account_number
+        self.domains = domains
+        self.dns_provider_type = dns_provider_type
+        self.options = options

lemur/authorizations/service.py

@@ -0,0 +1,24 @@
+"""
+.. module: lemur.pending_certificates.service
+    Copyright (c) 2017 and onwards Instart Logic, Inc.  All rights reserved.
+.. moduleauthor:: Secops <secops@netflix.com>
+"""
+from lemur import database
+
+from lemur.authorizations.models import Authorizations
+
+
+def get(authorization_id):
+    """
+    Retrieve dns authorization by ID
+    """
+    return database.get(Authorizations, authorization_id)
+
+
+def create(account_number, domains, dns_provider_type, options=None):
+    """
+    Creates a new dns authorization.
+    """
+
+    authorization = Authorizations(account_number, domains, dns_provider_type, options)
+    return database.create(authorization)

lemur/certificates/schemas.py

@@ -71,7 +71,7 @@ class CertificateInputSchema(CertificateCreationSchema):
     replaces = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)
     replacements = fields.Nested(AssociatedCertificateSchema, missing=[], many=True)  # deprecated
     roles = fields.Nested(AssociatedRoleSchema, missing=[], many=True)
-    dns_provider = fields.Nested(DnsProviderSchema, missing={}, required=False)
+    dns_provider = fields.Nested(DnsProviderSchema, missing={}, required=False, allow_none=True)
 
     csr = fields.String(validate=validators.csr)
     key_type = fields.String(validate=validate.OneOf(['RSA2048', 'RSA4096']), missing='RSA2048')

lemur/dns_providers/models.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, PrimaryKeyConstraint, String, text, UniqueConstraint
+from sqlalchemy import Column, Integer, String, text
 from sqlalchemy.dialects.postgresql import JSON
 from sqlalchemy_utils import ArrowType
 

lemur/models.py

@@ -8,9 +8,7 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-from sqlalchemy import Column, Integer, ForeignKey, Index, PrimaryKeyConstraint, String, text, UniqueConstraint
+from sqlalchemy import Column, Integer, ForeignKey, Index, UniqueConstraint
-from sqlalchemy.dialects.postgresql import JSON
-from sqlalchemy_utils import ArrowType
 
 from lemur.database import db
 

lemur/pending_certificates/cli.py

@@ -29,7 +29,7 @@ def fetch(ids):
 
     for cert in pending_certs:
         authority = plugins.get(cert.authority.plugin_name)
-        real_cert = authority.get_ordered_certificate(cert.external_id)
+        real_cert = authority.get_ordered_certificate(cert)
         if real_cert:
             # If a real certificate was returned from issuer, then create it in Lemur and delete
             # the pending certificate

lemur/pending_certificates/models.py

@@ -8,6 +8,7 @@ from datetime import datetime as dt
 from sqlalchemy.orm import relationship
 from sqlalchemy import Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
 from sqlalchemy_utils.types.arrow import ArrowType
+from sqlalchemy_utils import JSONType
 
 import lemur.common.utils
 from lemur.certificates.models import get_or_increase_name
@@ -37,6 +38,7 @@ class PendingCertificate(db.Model):
     private_key = Column(Vault, nullable=True)
 
     date_created = Column(ArrowType, PassiveDefault(func.now()), nullable=False)
+    dns_provider_id = Column(Integer(), nullable=True)
 
     status = Column(String(128))
 
@@ -54,6 +56,7 @@ class PendingCertificate(db.Model):
                             secondary=pending_cert_replacement_associations,
                             backref='pending_cert',
                             passive_deletes=True)
+    options = Column(JSONType)
 
     rotation_policy = relationship("RotationPolicy")
 
@@ -93,3 +96,7 @@ class PendingCertificate(db.Model):
         self.replaces = kwargs.get('replaces', [])
         self.rotation = kwargs.get('rotation')
         self.rotation_policy = kwargs.get('rotation_policy')
+        try:
+            self.dns_provider_id = kwargs.get('dns_provider', {}).get("id")
+        except AttributeError:
+            self.dns_provider_id = None

lemur/plugins/bases/issuer.py

@@ -25,7 +25,7 @@ class IssuerPlugin(Plugin):
     def revoke_certificate(self, certificate, comments):
         raise NotImplementedError
 
-    def get_ordered_certificate(self, order_id):
+    def get_ordered_certificate(self, certificate):
         raise NotImplementedError
 
     def cancel_ordered_certificate(self, pending_cert, **kwargs):

lemur/plugins/lemur_acme/plugin.py

@@ -24,6 +24,8 @@ from lemur.common.utils import generate_private_key
 
 import OpenSSL.crypto
 
+from lemur.authorizations import service as authorization_service
+from lemur.dns_providers import service as dns_provider_service
 from lemur.plugins.bases import IssuerPlugin
 from lemur.plugins import lemur_acme as acme
 
@@ -153,11 +155,14 @@ def get_domains(options):
 
 def get_authorizations(acme_client, account_number, domains, dns_provider):
     authorizations = []
-    try:
     for domain in domains:
         authz_record = start_dns_challenge(acme_client, account_number, domain, dns_provider)
         authorizations.append(authz_record)
+    return authorizations
 
+
+def finalize_authorizations(acme_client, account_number, dns_provider, authorizations):
+    try:
         for authz_record in authorizations:
             complete_dns_challenge(acme_client, account_number, authz_record, dns_provider)
     finally:
@@ -215,6 +220,23 @@ class ACMEIssuerPlugin(IssuerPlugin):
     def __init__(self, *args, **kwargs):
         super(ACMEIssuerPlugin, self).__init__(*args, **kwargs)
 
+    def get_ordered_certificate(self, pending_cert):
+        acme_client, registration = setup_acme_client(pending_cert.authority)
+        order_info = authorization_service.get(pending_cert.external_id)
+        dns_provider = dns_provider_service.get(pending_cert.dns_provider_id)
+        dns_provider_type = __import__(dns_provider.provider_type, globals(), locals(), [], 1)
+        authorizations = get_authorizations(
+            acme_client, order_info.account_number, order_info.domains, dns_provider_type)
+
+        finalize_authorizations(acme_client, order_info.account_number, dns_provider_type, authorizations)
+        pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, pending_cert.csr)
+        cert = {
+            'body': "\n".join(str(pem_certificate).splitlines()),
+            'chain': "\n".join(str(pem_certificate_chain).splitlines()),
+            'external_id': str(pending_cert.external_id)
+        }
+        return cert
+
     def create_certificate(self, csr, issuer_options):
         """
         Creates an ACME certificate.
@@ -224,10 +246,12 @@ class ACMEIssuerPlugin(IssuerPlugin):
         :return: :raise Exception:
         """
         authority = issuer_options.get('authority')
+        create_immediately = issuer_options.get('create_immediately', False)
         acme_client, registration = setup_acme_client(authority)
-        dns_provider = issuer_options.get('dns_provider')
+        dns_provider_d = issuer_options.get('dns_provider')
-        if not dns_provider:
+        if not dns_provider_d:
             raise Exception("DNS Provider setting is required for ACME certificates.")
+        dns_provider = dns_provider_service.get(dns_provider_d.get("id"))
         credentials = json.loads(dns_provider.credentials)
 
         current_app.logger.debug("Using DNS provider: {0}".format(dns_provider.provider_type))
@@ -238,7 +262,21 @@ class ACMEIssuerPlugin(IssuerPlugin):
             current_app.logger.error(error)
             raise Exception(error)
         domains = get_domains(issuer_options)
+        if not create_immediately:
+            # Create pending authorizations that we'll need to do the creation
+            authz_domains = []
+            for d in domains:
+                if type(d) == str:
+                    authz_domains.append(d)
+                else:
+                    authz_domains.append(d.value)
+
+            dns_authorization = authorization_service.create(account_number, authz_domains, dns_provider.provider_type)
+            # Return id of the DNS Authorization
+            return None, None, dns_authorization.id
+
         authorizations = get_authorizations(acme_client, account_number, domains, dns_provider_type)
+        finalize_authorizations(acme_client, account_number, dns_provider_type, authorizations)
         pem_certificate, pem_certificate_chain = request_certificate(acme_client, authorizations, csr)
         # TODO add external ID (if possible)
         return pem_certificate, pem_certificate_chain, None

lemur/plugins/lemur_acme/route53.py

@@ -58,7 +58,7 @@ def change_txt_record(action, zone_id, domain, value, client=None):
 def create_txt_record(host, value, account_number):
     zone_id = find_zone_id(host, account_number=account_number)
     change_id = change_txt_record(
-        "CREATE",
+        "UPSERT",
         zone_id,
         host,
         value,

lemur/plugins/lemur_digicert/plugin.py

@@ -325,8 +325,9 @@ class DigiCertIssuerPlugin(IssuerPlugin):
         response = self.session.put(create_url, data=json.dumps({'comments': comments}))
         return handle_response(response)
 
-    def get_ordered_certificate(self, order_id):
+    def get_ordered_certificate(self, pending_cert):
         """ Retrieve a certificate via order id """
+        order_id = pending_cert.external_id
         base_url = current_app.config.get('DIGICERT_URL')
         try:
             certificate_id = get_certificate_id(self.session, base_url, order_id)

lemur/schemas.py

@@ -164,10 +164,6 @@ class DnsProviderSchema(LemurInputSchema):
     id = fields.Integer()
     name = fields.String()
 
-    @post_load
-    def get_object(self, data, many=False):
-        return fetch_objects(DnsProviders, data, many=many)
-
 
 class PluginInputSchema(LemurInputSchema):
     plugin_options = fields.List(fields.Dict(), validate=validate_options)

lemur/static/app/angular/certificates/services.js

@@ -248,7 +248,7 @@ angular.module('lemur')
 
     CertificateService.getDnsProviders = function () {
       return DnsProviders.get();
-    }
+    };
 
     CertificateService.loadPrivateKey = function (certificate) {
       return certificate.customGET('key');

lemur/static/app/angular/dns_providers/dns_provider/dns_provider.js

@@ -14,7 +14,7 @@ angular.module('lemur')
     });
 
     $scope.save = function (dns_provider) {
-      DnsProvider.create(dns_provider.then(
+      DnsProviderService.create(dns_provider.then(
         function () {
           toaster.pop({
             type: 'success',
@@ -31,7 +31,7 @@ angular.module('lemur')
             directiveData: response.data,
             timeout: 100000
           });
-        });
+        }));
     };
 
     $scope.cancel = function () {

lemur/static/app/angular/dns_providers/services.js

@@ -14,7 +14,7 @@ angular.module('lemur')
 
     DnsProviderService.getDnsProviders = function () {
       return DnsProviders.get();
-    }
+    };
 
     DnsProviderService.create = function (dns_provider) {
       return DnsProviderApi.post(dns_provider);

lemur/tests/plugins/issuer_plugin.py

@@ -37,7 +37,8 @@ class TestAsyncIssuerPlugin(IssuerPlugin):
     def create_certificate(self, csr, issuer_options):
         return "", "", 12345
 
-    def get_ordered_certificate(self, order_id):
+    def get_ordered_certificate(self, pending_cert):
+        order_id = pending_cert.external_id
         return INTERNAL_VALID_LONG_STR, INTERNAL_VALID_SAN_STR, 54321
 
     @staticmethod

lemur/tests/test_certificates.py

@@ -63,7 +63,7 @@ def test_get_certificate_primitives(certificate):
 
     with freeze_time(datetime.date(year=2016, month=10, day=30)):
         primitives = get_certificate_primitives(certificate)
-        assert len(primitives) == 23
+        assert len(primitives) == 24
 
 
 def test_certificate_edit_schema(session):
@@ -152,7 +152,8 @@ def test_certificate_input_schema(client, authority):
         'authority': {'id': authority.id},
         'description': 'testtestest',
         'validityEnd': arrow.get(2016, 11, 9).isoformat(),
-        'validityStart': arrow.get(2015, 11, 9).isoformat()
+        'validityStart': arrow.get(2015, 11, 9).isoformat(),
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -165,7 +166,7 @@ def test_certificate_input_schema(client, authority):
     assert data['country'] == 'US'
     assert data['location'] == 'Los Gatos'
 
-    assert len(data.keys()) == 18
+    assert len(data.keys()) == 19
 
 
 def test_certificate_input_with_extensions(client, authority):
@@ -192,7 +193,8 @@ def test_certificate_input_with_extensions(client, authority):
                     {'nameType': 'DNSName', 'value': 'test.example.com'}
                 ]
             }
-        }
+        },
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -206,7 +208,8 @@ def test_certificate_out_of_range_date(client, authority):
         'owner': 'jim@example.com',
         'authority': {'id': authority.id},
         'description': 'testtestest',
-        'validityYears': 100
+        'validityYears': 100,
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -230,7 +233,8 @@ def test_certificate_valid_years(client, authority):
         'owner': 'jim@example.com',
         'authority': {'id': authority.id},
         'description': 'testtestest',
-        'validityYears': 1
+        'validityYears': 1,
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -245,7 +249,8 @@ def test_certificate_valid_dates(client, authority):
         'authority': {'id': authority.id},
         'description': 'testtestest',
         'validityStart': '2020-01-01T00:00:00',
-        'validityEnd': '2020-01-01T00:00:01'
+        'validityEnd': '2020-01-01T00:00:01',
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -262,6 +267,7 @@ def test_certificate_cn_admin(client, authority, logged_in_admin):
         'description': 'testtestest',
         'validityStart': '2020-01-01T00:00:00',
         'validityEnd': '2020-01-01T00:00:01',
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -285,7 +291,8 @@ def test_certificate_allowed_names(client, authority, session, logged_in_user):
                     {'nameType': 'IPAddress', 'value': '127.0.0.1'},
                 ]
             }
-        }
+        },
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -306,6 +313,7 @@ def test_certificate_incative_authority(client, authority, session, logged_in_us
         'description': 'testtestest',
         'validityStart': '2020-01-01T00:00:00',
         'validityEnd': '2020-01-01T00:00:01',
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -329,7 +337,8 @@ def test_certificate_disallowed_names(client, authority, session, logged_in_user
                     {'nameType': 'DNSName', 'value': 'evilhacker.org'},
                 ]
             }
-        }
+        },
+        'dns_provider': None,
     }
 
     data, errors = CertificateInputSchema().load(input_data)
@@ -348,6 +357,7 @@ def test_certificate_sensitive_name(client, authority, session, logged_in_user):
         'description': 'testtestest',
         'validityStart': '2020-01-01T00:00:00',
         'validityEnd': '2020-01-01T00:00:01',
+        'dns_provider': None,
     }
     session.add(Domain(name='sensitive.example.com', sensitive=True))