Enabling CSR generation and reducing complexity of encryption/decrypting the 'key' dir.

lemur/__init__.py

@@ -8,8 +8,6 @@
 
 
 """
-from flask import jsonify
-
 from lemur import factory
 
 from lemur.users.views import mod as users_bp

lemur/auth/views.py

@@ -14,8 +14,6 @@ from flask import g, Blueprint, current_app, abort
 from flask.ext.restful import reqparse, Resource, Api
 from flask.ext.principal import Identity, identity_changed
 
-from lemur.common.crypto import unlock
-
 from lemur.auth.permissions import admin_permission
 from lemur.users import service as user_service
 from lemur.roles import service as role_service
@@ -234,24 +232,7 @@ class Ping(Resource):
         return dict(token=create_token(user))
 
 
-class Unlock(AuthenticatedResource):
-    def __init__(self):
-        self.reqparse = reqparse.RequestParser()
-        super(Unlock, self).__init__()
-
-    @admin_permission.require(http_exception=403)
-    def post(self):
-        self.reqparse.add_argument('password', type=str, required=True, location='json')
-        args = self.reqparse.parse_args()
-        unlock(args['password'])
-        return {
-            "message": "You have successfully unlocked this Lemur instance",
-            "type": "success"
-        }
-
-
 api.add_resource(Login, '/auth/login', endpoint='login')
 api.add_resource(Ping, '/auth/ping', endpoint='ping')
-api.add_resource(Unlock, '/auth/unlock', endpoint='unlock')
 
 

lemur/certificates/service.py

@@ -26,10 +26,11 @@ from lemur.roles.models import Role
 
 from cryptography import x509
 from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
 
 
+
 def get(cert_id):
     """
     Retrieves certificate by it's ID.
@@ -145,7 +146,7 @@ def import_certificate(**kwargs):
     """
     Uploads already minted certificates and pulls the required information into Lemur.
 
-    This is to be used for certificates that are reated outside of Lemur but
+    This is to be used for certificates that are created outside of Lemur but
     should still be tracked.
 
     Internally this is used to bootstrap Lemur with external
@@ -315,64 +316,71 @@ def create_csr(csr_config):
         x509.BasicConstraints(ca=False, path_length=None), critical=True,
     )
 
-    for name in csr_config['extensions']['subAltNames']['names']:
+    for k, v in csr_config.get('extensions', {}).items():
-        builder.add_extension(
+        if k == 'subAltNames':
-            x509.SubjectAlternativeName(x509.DNSName, name['value'])
+            builder = builder.add_extension(
+                x509.SubjectAlternativeName([x509.DNSName(n) for n in v]), critical=True,
             )
 
-# TODO support more CSR options
+    # TODO support more CSR options, none of the authorities support these atm
-#    csr_config['extensions']['keyUsage']
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.KeyUsage(
-#        x509.KeyUsage(
+    #            digital_signature=digital_signature,
-#            digital_signature=digital_signature,
+    #            content_commitment=content_commitment,
-#            content_commitment=content_commitment,
+    #            key_encipherment=key_enipherment,
-#            key_encipherment=key_enipherment,
+    #            data_encipherment=data_encipherment,
-#            data_encipherment=data_encipherment,
+    #            key_agreement=key_agreement,
-#            key_agreement=key_agreement,
+    #            key_cert_sign=key_cert_sign,
-#            key_cert_sign=key_cert_sign,
+    #            crl_sign=crl_sign,
-#            crl_sign=crl_sign,
+    #            encipher_only=enchipher_only,
-#            encipher_only=enchipher_only,
+    #            decipher_only=decipher_only
-#            decipher_only=decipher_only
+    #        ), critical=True
-#        ), critical=True
+    #    )
-#    )
+    #
-#
+    #    # we must maintain our own list of OIDs here
-#    # we must maintain our own list of OIDs here
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.ExtendedKeyUsage(
-#        x509.ExtendedKeyUsage(
+    #            server_authentication=server_authentication,
-#            server_authentication=server_authentication,
+    #            email=
-#            email=
+    #        )
-#        )
+    #    )
-#    )
+    #
-#
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.AuthorityInformationAccess()
-#        x509.AuthorityInformationAccess()
+    #    )
-#    )
+    #
-#
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.AuthorityKeyIdentifier()
-#        x509.AuthorityKeyIdentifier()
+    #    )
-#    )
+    #
-#
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.SubjectKeyIdentifier()
-#        x509.SubjectKeyIdentifier()
+    #    )
-#    )
+    #
-#
+    #    builder.add_extension(
-#    builder.add_extension(
+    #        x509.CRLDistributionPoints()
-#        x509.CRLDistributionPoints()
+    #    )
-#    )
+    #
+    #    builder.add_extension(
+    #        x509.ObjectIdentifier(oid)
+    #    )
 
     request = builder.sign(
         private_key, hashes.SHA256(), default_backend()
     )
 
-    # here we try and support arbitrary oids
+    # serialize our private key and CSR
-    for oid in csr_config['extensions']['custom']:
+    pem = private_key.private_bytes(
-        builder.add_extension(
+        encoding=serialization.Encoding.PEM,
-            x509.ObjectIdentifier(oid)
+        format=serialization.PrivateFormat.PKCS8,
+        encryption_algorithm=serialization.NoEncryption()
     )
 
+    csr = request.public_bytes(
+        encoding=serialization.Encoding.PEM
+    )
 
-    return request.public_bytes("PEM"), private_key.public_bytes("PEM")
+    return csr, pem
-
 
 def create_challenge():
     """

lemur/common/crypto.py

@@ -1,185 +0,0 @@
-"""
-.. module: lemur.common.crypto
-    :platform: Unix
-    :synopsis: This module contains all cryptographic function's in Lemur
-    :copyright: (c) 2015 by Netflix Inc., see AUTHORS for more
-    :license: Apache, see LICENSE for more details.
-.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
-
-"""
-import os
-import ssl
-import StringIO
-import functools
-from Crypto import Random
-from Crypto.Cipher import AES
-from hashlib import sha512
-
-from flask import current_app
-
-from lemur.factory import create_app
-
-
-old_init = ssl.SSLSocket.__init__
-
-@functools.wraps(old_init)
-def ssl_bug(self, *args, **kwargs):
-  kwargs['ssl_version'] = ssl.PROTOCOL_TLSv1
-  old_init(self, *args, **kwargs)
-
-ssl.SSLSocket.__init__ = ssl_bug
-
-
-def derive_key_and_iv(password, salt, key_length, iv_length):
-    """
-    Derives the key and iv from the password and salt.
-
-    :param password:
-    :param salt:
-    :param key_length:
-    :param iv_length:
-    :return: key, iv
-    """
-    d = d_i = ''
-
-    while len(d) < key_length + iv_length:
-        d_i = sha512(d_i + password + salt).digest()
-        d += d_i
-
-    return d[:key_length], d[key_length:key_length+iv_length]
-
-
-def encrypt(in_file, out_file, password, key_length=32):
-    """
-    Encrypts a file.
-
-    :param in_file:
-    :param out_file:
-    :param password:
-    :param key_length:
-    """
-    bs = AES.block_size
-    salt = Random.new().read(bs - len('Salted__'))
-    key, iv = derive_key_and_iv(password, salt, key_length, bs)
-    cipher = AES.new(key, AES.MODE_CBC, iv)
-    out_file.write('Salted__' + salt)
-    finished = False
-    while not finished:
-        chunk = in_file.read(1024 * bs)
-        if len(chunk) == 0 or len(chunk) % bs != 0:
-            padding_length = bs - (len(chunk) % bs)
-            chunk += padding_length * chr(padding_length)
-            finished = True
-        out_file.write(cipher.encrypt(chunk))
-
-
-def decrypt(in_file, out_file, password, key_length=32):
-    """
-    Decrypts a file.
-
-    :param in_file:
-    :param out_file:
-    :param password:
-    :param key_length:
-    :raise ValueError:
-    """
-    bs = AES.block_size
-    salt = in_file.read(bs)[len('Salted__'):]
-    key, iv = derive_key_and_iv(password, salt, key_length, bs)
-    cipher = AES.new(key, AES.MODE_CBC, iv)
-    next_chunk = ''
-    finished = False
-    while not finished:
-        chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * bs))
-        if len(next_chunk) == 0:
-            padding_length = ord(chunk[-1])
-            if padding_length < 1 or padding_length > bs:
-                raise ValueError("bad decrypt pad (%d)" % padding_length)
-            # all the pad-bytes must be the same
-            if chunk[-padding_length:] != (padding_length * chr(padding_length)):
-                # this is similar to the bad decrypt:evp_enc.c from openssl program
-                raise ValueError("bad decrypt")
-            chunk = chunk[:-padding_length]
-            finished = True
-        out_file.write(chunk)
-
-
-def encrypt_string(string, password):
-    """
-    Encrypts a string.
-
-    :param string:
-    :param password:
-    :return:
-    """
-    in_file = StringIO.StringIO(string)
-    enc_file = StringIO.StringIO()
-    encrypt(in_file, enc_file, password)
-    enc_file.seek(0)
-    return enc_file.read()
-
-
-def decrypt_string(string, password):
-    """
-    Decrypts a string.
-
-    :param string:
-    :param password:
-    :return:
-    """
-    in_file = StringIO.StringIO(string)
-    out_file = StringIO.StringIO()
-    decrypt(in_file, out_file, password)
-    out_file.seek(0)
-    return out_file.read()
-
-
-def lock(password):
-    """
-    Encrypts Lemur's KEY_PATH. This directory can be used to store secrets needed for normal
-    Lemur operation. This is especially useful for storing secrets needed for communication
-    with third parties (e.g. external certificate authorities).
-
-    Lemur does not assume anything about the contents of the directory and will attempt to
-    encrypt all files contained within. Currently this has only been tested against plain
-    text files.
-
-    :param password:
-    """
-    dest_dir = os.path.join(current_app.config.get("KEY_PATH"), "encrypted")
-
-    if not os.path.exists(dest_dir):
-        current_app.logger.debug("Creating encryption directory: {0}".format(dest_dir))
-        os.makedirs(dest_dir)
-
-    for root, dirs, files in os.walk(os.path.join(current_app.config.get("KEY_PATH"), 'decrypted')):
-        for f in files:
-            source = os.path.join(root, f)
-            dest = os.path.join(dest_dir, f + ".enc")
-            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
-                encrypt(in_file, out_file, password)
-
-
-def unlock(password):
-    """
-    Decrypts Lemur's KEY_PATH, allowing lemur to use the secrets within.
-
-    This reverses the :func:`lock` function.
-
-    :param password:
-    """
-    dest_dir = os.path.join(current_app.config.get("KEY_PATH"), "decrypted")
-    source_dir = os.path.join(current_app.config.get("KEY_PATH"), "encrypted")
-
-    if not os.path.exists(dest_dir):
-        current_app.logger.debug("Creating decryption directory: {0}".format(dest_dir))
-        os.makedirs(dest_dir)
-
-    for root, dirs, files in os.walk(source_dir):
-        for f in files:
-            source = os.path.join(source_dir, f)
-            dest = os.path.join(dest_dir, ".".join(f.split(".")[:-1]))
-            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
-                current_app.logger.debug("Writing file: {0} Source: {1}".format(dest, source))
-                decrypt(in_file, out_file, password)
-

lemur/manage.py

@@ -3,6 +3,8 @@ import sys
 import base64
 from gunicorn.config import make_settings
 
+from cryptography.fernet import Fernet
+
 from flask import current_app
 from flask.ext.script import Manager, Command, Option, Group, prompt_pass
 from flask.ext.migrate import Migrate, MigrateCommand, stamp
@@ -19,7 +21,6 @@ from lemur.certificates import sync
 from lemur.elbs.sync import sync_all_elbs
 
 from lemur import create_app
-from lemur.common.crypto import encrypt, decrypt, lock, unlock
 
 # Needed to be imported so that SQLAlchemy create_all can find our models
 from lemur.users.models import User
@@ -132,78 +133,6 @@ def create():
     stamp(revision='head')
 
 
-@manager.command
-def lock():
-    """
-    Encrypts all of the files in the `keys` directory with the password
-    given. This is a useful function to ensure that you do no check in
-    your key files into source code in clear text.
-
-    :return:
-    """
-    password = prompt_pass("Please enter the encryption password")
-    lock(password)
-    sys.stdout.write("[+] Lemur keys have been encrypted!\n")
-
-
-@manager.command
-def unlock():
-    """
-    Decrypts all of the files in the `keys` directory with the password
-    given. This is most commonly used during the startup sequence of Lemur
-    allowing it to go from source code to something that can communicate
-    with external services.
-
-    :return:
-    """
-    password = prompt_pass("Please enter the encryption password")
-    unlock(password)
-    sys.stdout.write("[+] Lemur keys have been unencrypted!\n")
-
-
-@manager.command
-def encrypt_file(source):
-    """
-    Utility to encrypt sensitive files, Lemur will decrypt these
-    files when admin enters the correct password.
-
-    Uses AES-256-CBC encryption
-    """
-    dest = source + ".encrypted"
-    password = prompt_pass("Please enter the encryption password")
-    password1 = prompt_pass("Please confirm the encryption password")
-    if password != password1:
-        sys.stdout.write("[!] Encryption passwords do not match!\n")
-        return
-
-    with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
-        encrypt(in_file, out_file, password)
-
-    sys.stdout.write("[+] Writing encryption files... {0}!\n".format(dest))
-
-
-@manager.command
-def decrypt_file(source):
-    """
-    Utility to decrypt, Lemur will decrypt these
-    files when admin enters the correct password.
-
-    Assumes AES-256-CBC encryption
-    """
-    # cleanup extensions a bit
-    if ".encrypted" in source:
-        dest = ".".join(source.split(".")[:-1]) + ".decrypted"
-    else:
-        dest = source + ".decrypted"
-
-    password = prompt_pass("Please enter the encryption password")
-
-    with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
-        decrypt(in_file, out_file, password)
-
-    sys.stdout.write("[+] Writing decrypted files... {0}!\n".format(dest))
-
-
 @manager.command
 def check_revoked():
     """
@@ -491,7 +420,84 @@ def create_config(config_path=None):
     with open(config_path, 'w') as f:
         f.write(config)
 
-    sys.stdout.write("Created a new configuration file {0}\n".format(config_path))
+    sys.stdout.write("[+] Created a new configuration file {0}\n".format(config_path))
+
+
+@manager.command
+def lock(path=None):
+    """
+    Encrypts a given path. This directory can be used to store secrets needed for normal
+    Lemur operation. This is especially useful for storing secrets needed for communication
+    with third parties (e.g. external certificate authorities).
+
+    Lemur does not assume anything about the contents of the directory and will attempt to
+    encrypt all files contained within. Currently this has only been tested against plain
+    text files.
+
+    Path defaults ~/.lemur/keys
+
+    :param: path
+    """
+    if not path:
+        path = os.path.expanduser('~/.lemur/keys')
+
+    dest_dir = os.path.join(path, "encrypted")
+    sys.stdout.write("[!] Generating a new key...\n")
+
+    key = Fernet.generate_key()
+
+    if not os.path.exists(dest_dir):
+        sys.stdout.write("[+] Creating encryption directory: {0}\n".format(dest_dir))
+        os.makedirs(dest_dir)
+
+    for root, dirs, files in os.walk(os.path.join(path, 'decrypted')):
+        for f in files:
+            source = os.path.join(root, f)
+            dest = os.path.join(dest_dir, f + ".enc")
+            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
+                f = Fernet(key)
+                data = f.encrypt(in_file.read())
+                out_file.write(data)
+                sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
+
+    sys.stdout.write("[+] Keys have been encrypted with key {0}\n".format(key))
+
+
+@manager.command
+def unlock(path=None):
+    """
+    Decrypts all of the files in a given directory with provided password.
+    This is most commonly used during the startup sequence of Lemur
+    allowing it to go from source code to something that can communicate
+    with external services.
+
+    Path defaults ~/.lemur/keys
+
+    :param: path
+    """
+    key = prompt_pass("[!] Please enter the encryption password")
+
+    if not path:
+        path = os.path.expanduser('~/.lemur/keys')
+
+    dest_dir = os.path.join(path, "decrypted")
+    source_dir = os.path.join(path, "encrypted")
+
+    if not os.path.exists(dest_dir):
+        sys.stdout.write("[+] Creating decryption directory: {0}\n".format(dest_dir))
+        os.makedirs(dest_dir)
+
+    for root, dirs, files in os.walk(source_dir):
+        for f in files:
+            source = os.path.join(source_dir, f)
+            dest = os.path.join(dest_dir, ".".join(f.split(".")[:-1]))
+            with open(source, 'rb') as in_file, open(dest, 'wb') as out_file:
+                f = Fernet(key)
+                data = f.decrypt(in_file.read())
+                out_file.write(data)
+                sys.stdout.write("[+] Writing file: {0} Source: {1}\n".format(dest, source))
+
+    sys.stdout.write("[+] Keys have been unencrypted!\n")
 
 
 def main():

lemur/tests/conftest.py

@@ -1,3 +1,4 @@
+import os
 import pytest
 
 from lemur import create_app

lemur/tests/test_certificates.py

@@ -1,6 +1,4 @@
-import os
 import pytest
-from mock import mock_open, patch
 from lemur.certificates.views import *
 
 #def test_crud(session):
@@ -33,25 +31,23 @@ def test_private_key_str():
         private_key_str('dfsdfsdf', 'test')
 
 
-def test_create_csr():
+def test_create_basic_csr():
-    from lemur.tests.certs import CSR_CONFIG
     from lemur.certificates.service import create_csr
-    m = mock_open()
+    csr_config = dict(
-    with patch('lemur.certificates.service.open', m, create=True):
+        commonName=u'example.com',
-        path = create_csr(CSR_CONFIG)
+        organization=u'Example, Inc.',
-        assert path == ''
+        organizationalUnit=u'Operations',
+        country=u'US',
+        state=u'CA',
+        location=u'A place',
+        extensions=dict(subAltNames=[u'test.example.com', u'test2.example.com'])
+    )
+    csr, pem = create_csr(csr_config)
 
-
+    private_key = serialization.load_pem_private_key(pem, password=None, backend=default_backend())
-def test_create_path():
+    csr = x509.load_pem_x509_csr(csr, default_backend())
-    assert 1 == 2
+    for name in csr.subject:
-
+        assert name.value in csr_config.values()
-
-def test_load_ssl_pack():
-    assert 1 == 2
-
-
-def test_delete_ssl_path():
-    assert 1 == 2
 
 
 def test_import_certificate(session):