Merge pull request #236 from kevgliss/migration_script_fixups

Removing per 2.0 migration scripts
This commit is contained in:
kevgliss 2016-01-29 10:30:41 -08:00
commit 3ad317fb6d
8 changed files with 2 additions and 445 deletions

View File

@ -158,6 +158,7 @@ Additional notifications can be created through the UI or API. See :ref:`Creati
.. code-block:: bash .. code-block:: bash
$ cd /www/lemur/lemur
$ lemur init $ lemur init
.. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details. .. note:: It is recommended that once the ``lemur`` user is created that you create individual users for every day access. There is currently no way for a user to self enroll for Lemur access, they must have an administrator create an account for them or be enrolled automatically through SSO. This can be done through the CLI or UI. See :ref:`Creating Users <CreatingUsers>` and :ref:`Command Line Interface <CommandLineInterface>` for details.

View File

@ -1,42 +0,0 @@
"""Adding in models for certificate sources
Revision ID: 1ff763f5b80b
Revises: 4dc5ddd111b8
Create Date: 2015-08-01 15:24:20.412725
"""
# revision identifiers, used by Alembic.
revision = '1ff763f5b80b'
down_revision = '4dc5ddd111b8'
from alembic import op
import sqlalchemy as sa
import sqlalchemy_utils
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('sources',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('label', sa.String(length=32), nullable=True),
sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('plugin_name', sa.String(length=32), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('certificate_source_associations',
sa.Column('source_id', sa.Integer(), nullable=True),
sa.Column('certificate_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['source_id'], ['destinations.id'], ondelete='cascade')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('certificate_source_associations')
op.drop_table('sources')
### end Alembic commands ###

View File

@ -8,7 +8,7 @@ Create Date: 2015-11-30 15:40:19.827272
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
revision = '33de094da890' revision = '33de094da890'
down_revision = 'ed422fc58ba' down_revision = None
from alembic import op from alembic import op
import sqlalchemy as sa import sqlalchemy as sa

View File

@ -1,49 +0,0 @@
"""Refactors Accounts to Destinations
Revision ID: 3b718f59b8ce
Revises: None
Create Date: 2015-07-09 17:44:55.626221
"""
# revision identifiers, used by Alembic.
revision = '3b718f59b8ce'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('certificate_account_associations')
op.drop_table('accounts')
op.add_column('destinations', sa.Column('plugin_name', sa.String(length=32), nullable=True))
op.drop_index('ix_elbs_account_id', table_name='elbs')
op.drop_constraint(u'elbs_account_id_fkey', 'elbs', type_='foreignkey')
op.drop_column('elbs', 'account_id')
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('elbs', sa.Column('account_id', sa.BIGINT(), autoincrement=False, nullable=True))
op.create_foreign_key(u'elbs_account_id_fkey', 'elbs', 'accounts', ['account_id'], ['id'])
op.create_index('ix_elbs_account_id', 'elbs', ['account_id'], unique=False)
op.drop_column('destinations', 'plugin_name')
op.create_table('accounts',
sa.Column('id', sa.INTEGER(), server_default=sa.text(u"nextval('accounts_id_seq'::regclass)"), nullable=False),
sa.Column('account_number', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
sa.Column('label', sa.VARCHAR(length=32), autoincrement=False, nullable=True),
sa.Column('notes', sa.TEXT(), autoincrement=False, nullable=True),
sa.PrimaryKeyConstraint('id', name=u'accounts_pkey'),
sa.UniqueConstraint('account_number', name=u'accounts_account_number_key'),
postgresql_ignore_search_path=False
)
op.create_table('certificate_account_associations',
sa.Column('account_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.Column('certificate_id', sa.INTEGER(), autoincrement=False, nullable=True),
sa.ForeignKeyConstraint(['account_id'], [u'accounts.id'], name=u'certificate_account_associations_account_id_fkey', ondelete=u'CASCADE'),
sa.ForeignKeyConstraint(['certificate_id'], [u'certificates.id'], name=u'certificate_account_associations_certificate_id_fkey', ondelete=u'CASCADE')
)
### end Alembic commands ###

View File

@ -1,26 +0,0 @@
"""Adding certificate signing algorithm
Revision ID: 4bcfa2c36623
Revises: 1ff763f5b80b
Create Date: 2015-10-06 10:03:47.993204
"""
# revision identifiers, used by Alembic.
revision = '4bcfa2c36623'
down_revision = '1ff763f5b80b'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column('certificates', sa.Column('signing_algorithm', sa.String(length=128), nullable=True))
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_column('certificates', 'signing_algorithm')
### end Alembic commands ###

View File

@ -1,41 +0,0 @@
"""Adding notifications
Revision ID: 4c8915e461b3
Revises: 3b718f59b8ce
Create Date: 2015-07-24 14:34:57.316273
"""
# revision identifiers, used by Alembic.
revision = '4c8915e461b3'
down_revision = '3b718f59b8ce'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
import sqlalchemy_utils
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('notifications',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('label', sa.String(length=128), nullable=True),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('options', sqlalchemy_utils.types.json.JSONType(), nullable=True),
sa.Column('active', sa.Boolean(), nullable=True),
sa.Column('plugin_name', sa.String(length=32), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.drop_column(u'certificates', 'challenge')
op.drop_column(u'certificates', 'csr_config')
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.add_column(u'certificates', sa.Column('csr_config', sa.TEXT(), autoincrement=False, nullable=True))
op.add_column(u'certificates', sa.Column('challenge', postgresql.BYTEA(), autoincrement=False, nullable=True))
op.drop_table('notifications')
### end Alembic commands ###

View File

@ -1,31 +0,0 @@
"""Creating a one-to-many relationship for notifications
Revision ID: 4dc5ddd111b8
Revises: 4c8915e461b3
Create Date: 2015-07-24 15:02:04.398262
"""
# revision identifiers, used by Alembic.
revision = '4dc5ddd111b8'
down_revision = '4c8915e461b3'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('certificate_notification_associations',
sa.Column('notification_id', sa.Integer(), nullable=True),
sa.Column('certificate_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['certificate_id'], ['certificates.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['notification_id'], ['notifications.id'], ondelete='cascade')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('certificate_notification_associations')
### end Alembic commands ###

View File

@ -1,255 +0,0 @@
"""Migrates the private key encrypted column from AES to fernet encryption scheme.
Revision ID: ed422fc58ba
Revises: 4bcfa2c36623
Create Date: 2015-10-23 09:19:28.654126
"""
import base64
# revision identifiers, used by Alembic.
revision = 'ed422fc58ba'
down_revision = '4bcfa2c36623'
import six
from StringIO import StringIO
from alembic import op
import sqlalchemy as sa
from sqlalchemy.sql import text
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.fernet import Fernet, MultiFernet
from flask import current_app
from lemur.common.utils import get_psuedo_random_string
conn = op.get_bind()
op.drop_table('encrypted_keys')
op.drop_table('encrypted_passwords')
# helper tables to migrate data
temp_key_table = op.create_table('encrypted_keys',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('aes', sa.Binary()),
sa.Column('fernet', sa.Binary()),
sa.PrimaryKeyConstraint('id')
)
# helper table to migrate data
temp_password_table = op.create_table('encrypted_passwords',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('aes', sa.Binary()),
sa.Column('fernet', sa.Binary()),
sa.PrimaryKeyConstraint('id')
)
# From http://sqlalchemy-utils.readthedocs.org/en/latest/_modules/sqlalchemy_utils/types/encrypted.html#EncryptedType
# for migration purposes only
class EncryptionDecryptionBaseEngine(object):
"""A base encryption and decryption engine.
This class must be sub-classed in order to create
new engines.
"""
def _update_key(self, key):
if isinstance(key, six.string_types):
key = key.encode()
digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
digest.update(key)
engine_key = digest.finalize()
self._initialize_engine(engine_key)
def encrypt(self, value):
raise NotImplementedError('Subclasses must implement this!')
def decrypt(self, value):
raise NotImplementedError('Subclasses must implement this!')
class AesEngine(EncryptionDecryptionBaseEngine):
"""Provide AES encryption and decryption methods."""
BLOCK_SIZE = 16
PADDING = six.b('*')
def _initialize_engine(self, parent_class_key):
self.secret_key = parent_class_key
self.iv = self.secret_key[:16]
self.cipher = Cipher(
algorithms.AES(self.secret_key),
modes.CBC(self.iv),
backend=default_backend()
)
def _pad(self, value):
"""Pad the message to be encrypted, if needed."""
BS = self.BLOCK_SIZE
P = self.PADDING
padded = (value + (BS - len(value) % BS) * P)
return padded
def encrypt(self, value):
if not isinstance(value, six.string_types):
value = repr(value)
if isinstance(value, six.text_type):
value = str(value)
value = value.encode()
value = self._pad(value)
encryptor = self.cipher.encryptor()
encrypted = encryptor.update(value) + encryptor.finalize()
encrypted = base64.b64encode(encrypted)
return encrypted
def decrypt(self, value):
if isinstance(value, six.text_type):
value = str(value)
decryptor = self.cipher.decryptor()
decrypted = base64.b64decode(value)
decrypted = decryptor.update(decrypted) + decryptor.finalize()
decrypted = decrypted.rstrip(self.PADDING)
if not isinstance(decrypted, six.string_types):
decrypted = decrypted.decode('utf-8')
return decrypted
def migrate_to_fernet(aes_encrypted, old_key, new_key):
"""
Will attempt to migrate an aes encrypted to fernet encryption
:param aes_encrypted:
:return: fernet encrypted value
"""
engine = AesEngine()
engine._update_key(old_key)
if not isinstance(aes_encrypted, six.string_types):
return
aes_decrypted = engine.decrypt(aes_encrypted)
fernet_encrypted = MultiFernet([Fernet(k) for k in new_key]).encrypt(bytes(aes_decrypted))
# sanity check
fernet_decrypted = MultiFernet([Fernet(k) for k in new_key]).decrypt(fernet_encrypted)
if fernet_decrypted != aes_decrypted:
raise Exception("WARNING: Decrypted values do not match!")
return fernet_encrypted
def migrate_from_fernet(fernet_encrypted, old_key, new_key):
"""
Will attempt to migrate from a fernet encryption to aes
:param fernet_encrypted:
:return:
"""
engine = AesEngine()
engine._update_key(new_key)
fernet_decrypted = MultiFernet([Fernet(k) for k in old_key]).decrypt(fernet_encrypted)
aes_encrypted = engine.encrypt(fernet_decrypted)
# sanity check
aes_decrypted = engine.decrypt(aes_encrypted)
if fernet_decrypted != aes_decrypted:
raise Exception("WARNING: Decrypted values do not match!")
return aes_encrypted
def upgrade():
old_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
print "Using: {0} as decryption key".format(old_key)
# generate a new fernet token
if current_app.config.get('LEMUR_ENCRYPTION_KEYS'):
new_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
else:
new_key = [Fernet.generate_key()]
print "Using: {0} as new encryption key, save this and place it in your configuration!".format(new_key)
# migrate private_keys
temp_keys = []
for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
aes_encrypted = StringIO(private_key).read()
fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
op.bulk_insert(temp_key_table, temp_keys)
for id, fernet in conn.execute(text('select id, fernet from encrypted_keys')):
stmt = text("update certificates set private_key=:key where id=:id")
stmt = stmt.bindparams(key=fernet, id=id)
op.execute(stmt)
print "Certificate {0} has been migrated".format(id)
# migrate role_passwords
temp_passwords = []
for id, password in conn.execute(text('select id, password from roles where password is not null')):
aes_encrypted = StringIO(password).read()
fernet_encrypted = migrate_to_fernet(aes_encrypted, old_key, new_key)
temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
op.bulk_insert(temp_password_table, temp_passwords)
for id, fernet in conn.execute(text('select id, fernet from encrypted_passwords')):
stmt = text("update roles set password=:password where id=:id")
stmt = stmt.bindparams(password=fernet, id=id)
print stmt
op.execute(stmt)
print "Password {0} has been migrated".format(id)
op.drop_table('encrypted_keys')
op.drop_table('encrypted_passwords')
def downgrade():
old_key = current_app.config.get('LEMUR_ENCRYPTION_KEYS')
print "Using: {0} as decryption key(s)".format(old_key)
# generate aes valid key
if current_app.config.get('LEMUR_ENCRYPTION_KEY'):
new_key = current_app.config.get('LEMUR_ENCRYPTION_KEY')
else:
new_key = get_psuedo_random_string()
print "Using: {0} as the encryption key, save this and place it in your configuration!".format(new_key)
# migrate keys
temp_keys = []
for id, private_key in conn.execute(text('select id, private_key from certificates where private_key is not null')):
fernet_encrypted = StringIO(private_key).read()
aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
temp_keys.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
op.bulk_insert(temp_key_table, temp_keys)
for id, aes in conn.execute(text('select id, aes from encrypted_keys')):
stmt = text("update certificates set private_key=:key where id=:id")
stmt = stmt.bindparams(key=aes, id=id)
print stmt
op.execute(stmt)
print "Certificate {0} has been migrated".format(id)
# migrate role_passwords
temp_passwords = []
for id, password in conn.execute(text('select id, password from roles where password is not null')):
fernet_encrypted = StringIO(password).read()
aes_encrypted = migrate_from_fernet(fernet_encrypted, old_key, new_key)
temp_passwords.append({'id': id, 'aes': aes_encrypted, 'fernet': fernet_encrypted})
op.bulk_insert(temp_password_table, temp_passwords)
for id, aes in conn.execute(text('select id, aes from encrypted_passwords')):
stmt = text("update roles set password=:password where id=:id")
stmt = stmt.bindparams(password=aes, id=id)
op.execute(stmt)
print "Password {0} has been migrated".format(id)
op.drop_table('encrypted_keys')
op.drop_table('encrypted_passwords')