Merge pull request #236 from kevgliss/migration_script_fixups

Removing per 2.0 migration scripts

docs/quickstart/index.rst

@@ -158,6 +158,7 @@ Additional notifications can be created through the UI or API.  See :ref:`Creati
 
 .. code-block:: bash
 
+    $ cd /www/lemur/lemur
     $ lemur init
 
 .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access.  There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO.  This can be done through the CLI or UI.  See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.

lemur/migrations/versions/1ff763f5b80b_.py

@@ -1,42 +0,0 @@
-"""Adding in models for certificate sources
-
-Revision ID: 1ff763f5b80b
-Revises: 4dc5ddd111b8
-Create Date: 2015-08-01 15:24:20.412725
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '1ff763f5b80b'
-down_revision = '4dc5ddd111b8'
-
-from alembic import op
-import sqlalchemy as sa
-
-import sqlalchemy_utils
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('sources',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('label', sa.String(length=32), nullable=True),
-    sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
-    sa.Column('description', sa.Text(), nullable=True),
-    sa.Column('plugin_name', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('certificate_source_associations',
-    sa.Column('source_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['source_id'], ['destinations.id'], ondelete='cascade')
-    )
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_source_associations')
-    op.drop_table('sources')
-    ### end Alembic commands ###

lemur/migrations/versions/33de094da890_.py

@@ -8,7 +8,7 @@ Create Date: 2015-11-30 15:40:19.827272
 
 # revision identifiers, used by Alembic.
 revision = '33de094da890'
-down_revision = 'ed422fc58ba'
+down_revision = None
 
 from alembic import op
 import sqlalchemy as sa

lemur/migrations/versions/3b718f59b8ce_.py

@@ -1,49 +0,0 @@
-"""Refactors Accounts to Destinations
-
-Revision ID: 3b718f59b8ce
-Revises: None
-Create Date: 2015-07-09 17:44:55.626221
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '3b718f59b8ce'
-down_revision = None
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_account_associations')
-    op.drop_table('accounts')
-    op.add_column('destinations', sa.Column('plugin_name', sa.String(length=32), nullable=True))
-    op.drop_index('ix_elbs_account_id', table_name='elbs')
-    op.drop_constraint(u'elbs_account_id_fkey', 'elbs', type_='foreignkey')
-    op.drop_column('elbs', 'account_id')
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('elbs', sa.Column('account_id', sa.BIGINT(), autoincrement=False, nullable=True))
-    op.create_foreign_key(u'elbs_account_id_fkey', 'elbs', 'accounts', ['account_id'], ['id'])
-    op.create_index('ix_elbs_account_id', 'elbs', ['account_id'], unique=False)
-    op.drop_column('destinations', 'plugin_name')
-    op.create_table('accounts',
-    sa.Column('id', sa.INTEGER(), server_default=sa.text(u"nextval('accounts_id_seq'::regclass)"), nullable=False),
-    sa.Column('account_number', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
-    sa.Column('label', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
-    sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.PrimaryKeyConstraint('id', name=u'accounts_pkey'),
-    sa.UniqueConstraint('account_number', name=u'accounts_account_number_key'),
-        postgresql_ignore_search_path=False
-    )
-    op.create_table('certificate_account_associations',
-    sa.Column('account_id', sa.INTEGER(), autoincrement=False, nullable=True),
-    sa.Column('certificate_id', sa.INTEGER(), autoincrement=False, nullable=True),
-    sa.ForeignKeyConstraint(['account_id'], [u'accounts.id'], name=u'certificate_account_associations_account_id_fkey', ondelete=u'CASCADE'),
-    sa.ForeignKeyConstraint(['certificate_id'], [u'certificates.id'], name=u'certificate_account_associations_certificate_id_fkey', ondelete=u'CASCADE')
-    )
-    ### end Alembic commands ###

lemur/migrations/versions/4bcfa2c36623_.py

@@ -1,26 +0,0 @@
-"""Adding certificate signing algorithm
-
-Revision ID: 4bcfa2c36623
-Revises: 1ff763f5b80b
-Create Date: 2015-10-06 10:03:47.993204
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4bcfa2c36623'
-down_revision = '1ff763f5b80b'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('certificates', sa.Column('signing_algorithm', sa.String(length=128), nullable=True))
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column('certificates', 'signing_algorithm')
-    ### end Alembic commands ###

lemur/migrations/versions/4c8915e461b3_.py

@@ -1,41 +0,0 @@
-"""Adding notifications
-
-Revision ID: 4c8915e461b3
-Revises: 3b718f59b8ce
-Create Date: 2015-07-24 14:34:57.316273
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4c8915e461b3'
-down_revision = '3b718f59b8ce'
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-import sqlalchemy_utils
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('notifications',
-    sa.Column('id', sa.Integer(), nullable=False),
-    sa.Column('label', sa.String(length=128), nullable=True),
-    sa.Column('description', sa.Text(), nullable=True),
-    sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
-    sa.Column('active', sa.Boolean(), nullable=True),
-    sa.Column('plugin_name', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.drop_column(u'certificates', 'challenge')
-    op.drop_column(u'certificates', 'csr_config')
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.add_column(u'certificates', sa.Column('csr_config', sa.TEXT(), autoincrement=False, nullable=True))
-    op.add_column(u'certificates', sa.Column('challenge', postgresql.BYTEA(), autoincrement=False, nullable=True))
-    op.drop_table('notifications')
-    ### end Alembic commands ###

lemur/migrations/versions/4dc5ddd111b8_.py

@@ -1,31 +0,0 @@
-"""Creating a one-to-many relationship for notifications
-
-Revision ID: 4dc5ddd111b8
-Revises: 4c8915e461b3
-Create Date: 2015-07-24 15:02:04.398262
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '4dc5ddd111b8'
-down_revision = '4c8915e461b3'
-
-from alembic import op
-import sqlalchemy as sa
-
-
-def upgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('certificate_notification_associations',
-    sa.Column('notification_id', sa.Integer(), nullable=True),
-    sa.Column('certificate_id', sa.Integer(), nullable=True),
-    sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
-    sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade')
-    )
-    ### end Alembic commands ###
-
-
-def downgrade():
-    ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('certificate_notification_associations')
-    ### end Alembic commands ###

lemur/migrations/versions/ed422fc58ba_.py

@@ -1,255 +0,0 @@
-"""Migrates the private key encrypted column from AES to fernet encryption scheme.
-
-Revision ID: ed422fc58ba
-Revises: 4bcfa2c36623
-Create Date: 2015-10-23 09:19:28.654126
-
-"""
-import base64
-
-# revision identifiers, used by Alembic.
-revision = 'ed422fc58ba'
-down_revision = '4bcfa2c36623'
-import six
-
-from StringIO import StringIO
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.sql import text
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-from cryptography.fernet import Fernet, MultiFernet
-
-from flask import current_app
-from lemur.common.utils import get_psuedo_random_string
-
-conn = op.get_bind()
-
-op.drop_table('encrypted_keys')
-op.drop_table('encrypted_passwords')
-
-# helper tables to migrate data
-temp_key_table = op.create_table('encrypted_keys',
-                                 sa.Column('id', sa.Integer(), nullable=False),
-                                 sa.Column('aes', sa.Binary()),
-                                 sa.Column('fernet', sa.Binary()),
-                                 sa.PrimaryKeyConstraint('id')
-                                 )
-
-# helper table to migrate data
-temp_password_table = op.create_table('encrypted_passwords',
-                                      sa.Column('id', sa.Integer(), nullable=False),
-                                      sa.Column('aes', sa.Binary()),
-                                      sa.Column('fernet', sa.Binary()),
-                                      sa.PrimaryKeyConstraint('id')
-                                      )
-
-
-# From http://sqlalchemy-utils.readthedocs.org/en/latest/_modules/sqlalchemy_utils/types/encrypted.html#EncryptedType
-# for migration purposes only
-class EncryptionDecryptionBaseEngine(object):
-    """A base encryption and decryption engine.
-
-    This class must be sub-classed in order to create
-    new engines.
-    """
-
-    def _update_key(self, key):
-        if isinstance(key, six.string_types):
-            key = key.encode()
-        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
-        digest.update(key)
-        engine_key = digest.finalize()
-
-        self._initialize_engine(engine_key)
-
-    def encrypt(self, value):
-        raise NotImplementedError('Subclasses must implement this!')
-
-    def decrypt(self, value):
-        raise NotImplementedError('Subclasses must implement this!')
-
-
-class AesEngine(EncryptionDecryptionBaseEngine):
-    """Provide AES encryption and decryption methods."""
-
-    BLOCK_SIZE = 16
-    PADDING = six.b('*')
-
-    def _initialize_engine(self, parent_class_key):
-        self.secret_key = parent_class_key
-        self.iv = self.secret_key[:16]
-        self.cipher = Cipher(
-            algorithms.AES(self.secret_key),
-            modes.CBC(self.iv),
-            backend=default_backend()
-        )
-
-    def _pad(self, value):
-        """Pad the message to be encrypted, if needed."""
-        BS = self.BLOCK_SIZE
-        P = self.PADDING
-        padded = (value + (BS - len(value) % BS) * P)
-        return padded
-
-    def encrypt(self, value):
-        if not isinstance(value, six.string_types):
-            value = repr(value)
-        if isinstance(value, six.text_type):
-            value = str(value)
-        value = value.encode()
-        value = self._pad(value)
-        encryptor = self.cipher.encryptor()
-        encrypted = encryptor.update(value) + encryptor.finalize()
-        encrypted = base64.b64encode(encrypted)
-        return encrypted
-
-    def decrypt(self, value):
-        if isinstance(value, six.text_type):
-            value = str(value)
-        decryptor = self.cipher.decryptor()
-        decrypted = base64.b64decode(value)
-        decrypted = decryptor.update(decrypted) + decryptor.finalize()
-        decrypted = decrypted.rstrip(self.PADDING)
-        if not isinstance(decrypted, six.string_types):
-            decrypted = decrypted.decode('utf-8')
-        return decrypted
-
-
-def migrate_to_fernet(aes_encrypted, old_key, new_key):
-    """
-    Will attempt to migrate an aes encrypted to fernet encryption
-    :param aes_encrypted:
-    :return: fernet encrypted value
-    """
-    engine = AesEngine()
-    engine._update_key(old_key)
-
-    if not isinstance(aes_encrypted, six.string_types):
-        return
-
-    aes_decrypted = engine.decrypt(aes_encrypted)
-    fernet_encrypted = MultiFernet([Fernet(k) for k in new_key]).encrypt(bytes(aes_decrypted))
-
-    # sanity check
-    fernet_decrypted = MultiFernet([Fernet(k) for k in new_key]).decrypt(fernet_encrypted)
-    if fernet_decrypted != aes_decrypted:
-        raise Exception("WARNING: Decrypted values do not match!")
-
-    return fernet_encrypted
-
-
-def migrate_from_fernet(fernet_encrypted, old_key, new_key):
-    """
-    Will attempt to migrate from a fernet encryption to aes
-    :param fernet_encrypted:
-    :return:
-    """
-    engine = AesEngine()
-    engine._update_key(new_key)
-
-    fernet_decrypted = MultiFernet([Fernet(k) for k in old_key]).decrypt(fernet_encrypted)
-    aes_encrypted = engine.encrypt(fernet_decrypted)
-
-    # sanity check
-    aes_decrypted = engine.decrypt(aes_encrypted)
-    if fernet_decrypted != aes_decrypted:
-        raise Exception("WARNING: Decrypted values do not match!")
-
-    return aes_encrypted
-
-
-def upgrade():
-    old_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
-    print "Using: {0} as decryption key".format(old_key)
-    # generate a new fernet token
-
-    if current_app.config.get('LEMUR_ENCRYPTION_KEYS'):
-        new_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
-    else:
-        new_key = [Fernet.generate_key()]
-
-    print "Using: {0} as new encryption key, save this and place it in your configuration!".format(new_key)
-
-    # migrate private_keys
-    temp_keys = []
-    for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
-        aes_encrypted = StringIO(private_key).read()
-        fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
-        temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_key_table, temp_keys)
-
-    for id, fernet in conn.execute(text('select id, fernet from encrypted_keys')):
-        stmt = text("update certificates set private_key=:key where id=:id")
-        stmt = stmt.bindparams(key=fernet, id=id)
-        op.execute(stmt)
-        print "Certificate {0} has been migrated".format(id)
-
-    # migrate role_passwords
-    temp_passwords = []
-    for id, password in conn.execute(text('select id, password from roles where password is not null')):
-        aes_encrypted = StringIO(password).read()
-        fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
-        temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_password_table, temp_passwords)
-
-    for id, fernet in conn.execute(text('select id, fernet from encrypted_passwords')):
-        stmt = text("update roles set password=:password where id=:id")
-        stmt = stmt.bindparams(password=fernet, id=id)
-        print stmt
-        op.execute(stmt)
-        print "Password {0} has been migrated".format(id)
-
-    op.drop_table('encrypted_keys')
-    op.drop_table('encrypted_passwords')
-
-
-def downgrade():
-    old_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
-    print "Using: {0} as decryption key(s)".format(old_key)
-
-    # generate aes valid key
-    if current_app.config.get('LEMUR_ENCRYPTION_KEY'):
-        new_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
-    else:
-        new_key = get_psuedo_random_string()
-    print "Using: {0} as the encryption key, save this and place it in your configuration!".format(new_key)
-
-    # migrate keys
-    temp_keys = []
-    for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
-        fernet_encrypted = StringIO(private_key).read()
-        aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
-        temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_key_table, temp_keys)
-
-    for id, aes in conn.execute(text('select id, aes from encrypted_keys')):
-        stmt = text("update certificates set private_key=:key where id=:id")
-        stmt = stmt.bindparams(key=aes, id=id)
-        print stmt
-        op.execute(stmt)
-        print "Certificate {0} has been migrated".format(id)
-
-    # migrate role_passwords
-    temp_passwords = []
-    for id, password in conn.execute(text('select id, password from roles where password is not null')):
-        fernet_encrypted = StringIO(password).read()
-        aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
-        temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
-
-    op.bulk_insert(temp_password_table, temp_passwords)
-
-    for id, aes in conn.execute(text('select id, aes from encrypted_passwords')):
-        stmt = text("update roles set password=:password where id=:id")
-        stmt = stmt.bindparams(password=aes, id=id)
-        op.execute(stmt)
-        print "Password {0} has been migrated".format(id)
-
-    op.drop_table('encrypted_keys')
-    op.drop_table('encrypted_passwords')