Infra
/
tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remote sqlalchemy storage (unworking) and add setting to storage

This commit is contained in:
Emmanuel Garette 2019-02-22 07:25:57 +01:00
parent fd95c6dd4a
commit 4a737c5b9d
11 changed files with 59 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tiramisu/__init__.py
View File

@ -12,12 +12,15 @@
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from .function import Params, ParamOption, ParamValue, ParamContext, tiramisu_copy
from .function import Params, ParamOption, ParamValue, ParamContext, \
tiramisu_copy
from .option import *
from .error import APIError
from .api import Config, MetaConfig, GroupConfig, MixConfig
from .option import __all__ as all_options
from .setting import owners, undefined
from .storage import default_storage_type, StorageType, list_sessions, \
delete_session
allfuncs = ['Params',
@ -30,6 +33,10 @@ allfuncs = ['Params',
'Config',
'APIError',
'undefined',
'default_storage_type',
'StorageType',
'list_sessions',
'delete_session',
'tiramisu_copy']
allfuncs.extend(all_options)
del(all_options)

37
tiramisu/storage/__init__.py
View File

@ -51,22 +51,19 @@ class StorageType(object):
if self.storage_type is not None: # pragma: no cover
if self.storage_type == name:
return
raise ConfigError(_('storage_type is already set, cannot rebind it'))
raise ConfigError(_('storage_type is already set, '
'cannot rebind it'))
self.storage_type = name
def get(self):
if self.storage_type is None:
self.storage_type = self.default_storage
set_to_default = True
else:
set_to_default = False
if self.mod is None:
modulepath = '{0}.storage.{1}'.format(MODULE_PATH, self.storage_type)
modulepath = '{0}.storage.{1}'.format(MODULE_PATH,
self.storage_type)
try:
mod = __import__(modulepath)
except ImportError: # pragma: no cover
import traceback
traceback.print_exc()
raise SystemError(_('cannot import the storage {0}').format(
self.default_storage))
for token in modulepath.split(".")[1:]:
@ -74,10 +71,13 @@ class StorageType(object):
self.mod = mod
return self.mod
def setting(self, **kwargs):
mod = self.get()
for key, value in kwargs.items():
setattr(mod.SETTING, key, value)
storage_type = StorageType()
#storage_option_type = StorageType()
#storage_option_type.set(DEFAULT_STORAGE)
default_storage_type = StorageType()
memory_storage = StorageType()
memory_storage.set(MEMORY_STORAGE)
@ -89,12 +89,17 @@ def gen_storage_id(session_id,
return 'c' + str(id(config)) + str(int(time())) + str(randint(0, 500))
def get_storages(context, session_id, persistent, storage):
session_id = gen_storage_id(session_id, context)
def get_storages(context,
session_id,
persistent,
storage):
session_id = gen_storage_id(session_id,
context)
if storage is None:
storage = storage_type
storage = default_storage_type
imp = storage.get()
imp_storage = imp.Storage(session_id, persistent)
imp_storage = imp.Storage(session_id,
persistent)
properties = imp.Properties(imp_storage)
permissives = imp.Permissives(imp_storage)
values = imp.Values(imp_storage)
@ -118,7 +123,7 @@ def get_default_settings_storages():
def list_sessions():
"""List all available session (persistent or not persistent)
"""
return storage_type.get().list_sessions()
return default_storage_type.get().list_sessions()
def delete_session(session_id):
@ -126,7 +131,7 @@ def delete_session(session_id):
use by an other instance
:params session_id: id of session to delete
"""
storage_module = storage_type.get()
storage_module = default_storage_type.get()
session = storage_module.storage.getsession()
storage_module.value.delete_session(session_id)
storage_module.storage.delete_session(session_id)

10
tiramisu/storage/dictionary/__init__.py
View File

@ -24,6 +24,12 @@ use it. But if something goes wrong, you will lost your modifications.
"""
from .value import Values
from .setting import Properties, Permissives
from .storage import setting, Storage, list_sessions
from .storage import SETTING, Storage, list_sessions
__all__ = ('setting', 'Values', 'Properties', 'Permissives', 'Storage', 'list_sessions')
__all__ = ('SETTING',
'Values',
'Properties',
'Permissives',
'Storage',
'list_sessions')

8
tiramisu/storage/dictionary/storage.py
View File

@ -18,13 +18,13 @@ from ...i18n import _
from ...error import ConflictError
class Setting(object):
class Setting:
"""Dictionary storage has no particular setting.
"""
pass
__slots__ = tuple()
setting = Setting()
SETTING = Setting()
_list_sessions = []
@ -35,7 +35,7 @@ def list_sessions():
class Storage(object):
__slots__ = ('session_id', 'persistent')
storage = 'dictionary'
#if object could be serializable
# if object could be serializable
serializable = True
def __init__(self, session_id, persistent, test=False):

36
tiramisu/storage/sqlalchemy/__init__.py
View File

@ -1,36 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
"""Default plugin for storage. All informations are store in a simple
dictionary in memory.
You cannot have persistente informations with this kind of storage.
The advantage of this solution is that you can easily create a Config and
use it. But if something goes wrong, you will lost your modifications.
"""
from .value import Values
from .setting import Settings
from .storage import Storage, list_sessions, delete_session, storage_setting
from .util import load
load()
__all__ = (storage_setting, Values, Settings, Storage, list_sessions, delete_session,
StorageBase)
# Base, OptionDescription)

138
tiramisu/storage/sqlalchemy/setting.py
View File

@ -1,138 +0,0 @@
# -*- coding: utf-8 -*-
"default plugin for setting: set it in a simple dictionary"
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from ..util import Cache
from .util import SqlAlchemyBase
import util
from sqlalchemy import Column, Integer, String, PickleType, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm.collections import attribute_mapped_collection
#____________________________________________________________
#
# properties|permissives
class _Property(SqlAlchemyBase):
__tablename__ = 'property'
id = Column(Integer, primary_key=True)
setting = Column(Integer, ForeignKey('settings.id'), nullable=False)
path = Column(String)
properties = Column(PickleType)
def __init__(self, path, properties):
self.path = path
self.properties = properties
class _Permissive (SqlAlchemyBase):
__tablename__ = 'permissive'
id = Column(Integer, primary_key=True)
setting = Column(Integer, ForeignKey('settings.id'), nullable=False)
path = Column(String)
permissives = Column(PickleType)
def __init__(self, path, permissives):
self.path = path
self.permissives = permissives
#____________________________________________________________
#FIXME marche pas le cache ... de toute facon je vais faire un storage separe !
class Settings(Cache, SqlAlchemyBase):
__tablename__ = 'settings'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
_props = relationship("_Property",
collection_class=attribute_mapped_collection('path'),
cascade="all, delete-orphan")
_properties = association_proxy("_props", "properties")
_perms = relationship("_Permissive",
collection_class=attribute_mapped_collection('path'),
cascade="all, delete-orphan")
_permissives = association_proxy("_perms", "permissives")
def __init__(self, session_id, storage):
session = self.getsession()
self.session_id = session_id
super(Settings, self).__init__(storage)
session.commit()
def getsession(self):
return util.Session()
# properties
def setproperties(self, path, properties):
session = self.getsession()
self._properties[path] = properties
session.commit()
del(session)
def getproperties(self, path, default_properties):
ret = self._properties.get(path)
if ret is None:
return set(default_properties)
return ret
def hasproperties(self, path):
return path in self._properties
def reset_all_properties(self):
session = self.getsession()
self._properties.clear()
session.commit()
del(session)
def delproperties(self, path):
try:
session = self.getsession()
del(self._properties[path])
session.commit()
del(session)
except KeyError:
pass
# permissive
def setpermissive(self, path, permissive):
session = self.getsession()
self._permissives[path] = frozenset(permissive)
session.commit()
def getpermissive(self, path=None):
ret = self._permissives.get(path, frozenset())
#replace None by a frozenset()
return {None: frozenset()}.get(ret, ret)
def exportation(self):
"""return all modified settings in a dictionary
example: {'path1': set(['prop1', 'prop2'])}
"""
return self._properties
def importation(self):
"""return all modified permissives in a dictionary
example: {'path1': set(['perm1', 'perm2'])}
"""
return self._permissives
def delete_session(session_id, session):
settings_id = session.query(Settings).filter_by(session_id=session_id).first().id
for val in session.query(_Property).filter_by(settings=settings_id).all():
session.delete(val)
for val in session.query(_Permissive).filter_by(settings=settings_id).all():
session.delete(val)

81
tiramisu/storage/sqlalchemy/storage.py
View File

@ -1,81 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from tiramisu.i18n import _
from .util import SqlAlchemyBase
import util
from sqlalchemy import Column, Integer, String
class Setting(object):
""":param extension: database file extension (by default: db)
:param dir_database: root database directory (by default: /tmp)
"""
#FIXME
extension = 'db'
dir_database = '/tmp'
storage_setting = Setting()
class Session(SqlAlchemyBase):
__tablename__ = 'session'
id = Column(Integer, primary_key=True)
session = Column(String, index=True)
def __init__(self, session_id):
self.session = session_id
def list_sessions():
session = util.Session()
ret = []
for val in session.query(Session).all():
ret.append(val.session)
del(session)
return ret
def delete_session(session_id, session):
session.delete(session.query(Session).filter_by(session=session_id).first())
session.commit()
def getsession():
return util.Session()
class Storage(object):
__slots__ = ('session_id', 'persistent')
storage = 'sqlalchemy'
#if object could be serializable
serializable = True
def __init__(self, session_id, persistent, test=False):
session = getsession()
self.session_id = session_id
self.persistent = persistent
if not session.query(Session).filter_by(session=session_id).first():
session.add(Session(session_id))
session.commit()
del(session)
def __del__(self):
if not self.persistent:
session = getsession()
delete_session(self.session_id, session)
del(session)

38
tiramisu/storage/sqlalchemy/util.py
View File

@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
""
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# ____________________________________________________________
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:')
SqlAlchemyBase = declarative_base()
global session, Session
Session = None
def load():
global session, Session
if Session is None:
#engine.echo = True
#print SqlAlchemyBase.metadata.tables.keys()
SqlAlchemyBase.metadata.create_all(engine)
Session = sessionmaker(bind=engine, expire_on_commit=False)

228
tiramisu/storage/sqlalchemy/value.py
View File

@ -1,228 +0,0 @@
# -*- coding: utf-8 -*-
"plugin for value: set it in sqlalchemy"
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from ..util import Cache
from .util import SqlAlchemyBase
import util
from ...setting import undefined
from sqlalchemy import Column, Integer, String, PickleType
from sqlalchemy import func
from tiramisu.setting import owners
#____________________________________________________________
#
# information
class _Vinformation(SqlAlchemyBase):
__tablename__ = 'vinformation'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
path = Column(String, index=True)
key = Column(String)
value = Column(PickleType)
def __init__(self, session_id, key, value):
self.session_id = session_id
self.key = key
self.value = value
class Value(SqlAlchemyBase):
__tablename__ = 'value'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
path = Column(String, index=True)
key = Column(String)
value = Column(PickleType)
owner = Column(String)
indx = Column(Integer, index=True)
def __init__(self, session_id, path, value, owner, index):
self.session_id = session_id
self.path = path
self.value = value
self.owner = owner
self.indx = index
class Values(Cache):
def getsession(self):
return util.Session()
# value
def setvalue(self, path, value, owner, index, session):
"""set value for a path
a specified value must be associated to an owner
"""
#if it's a multi
if isinstance(value, list):
value = list(value)
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if val is None:
session.add(Value(self._storage.session_id, path, value,
owner, index))
else:
val.value = value
val.owner = owner
session.commit()
def getvalue(self, path, session, index=None):
"""get value for a path
return: only value, not the owner
"""
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if not val:
raise KeyError('no value found')
return val.value
def hasvalue(self, path, session):
"""if path has a value
return: boolean
"""
return session.query(Value).filter_by(
path=path, session_id=self._storage.session_id).first() is not None
def resetvalue(self, path, session):
"""remove value means delete value in storage
"""
vals = session.query(Value).filter_by(
path=path, session_id=self._storage.session_id).all()
if vals != []:
for val in vals:
session.delete(val)
session.commit()
# owner
def setowner(self, path, owner, session, index=None):
"""change owner for a path
"""
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if val is None:
raise KeyError('no value found')
else:
val.owner = owner
session.commit()
def get_max_length(self, path, session):
val = session.query(Value, func.max(Value.indx)).filter_by(
path=path, session_id=self._storage.session_id).first()
if val[1] is None:
maxval = 0
else:
maxval = val[1] + 1
return maxval
def getowner(self, path, default, session, index=None, only_default=False):
#FIXME support de only_default
"""get owner for a path
return: owner object
"""
session.commit()
val = session.query(Value).filter_by(
path=path, session_id=self._storage.session_id,
indx=index).first()
if val is None:
return default
else:
owner = val.owner
# autocreate owners
try:
return getattr(owners, owner)
except AttributeError:
owners.addowner(owner)
return getattr(owners, owner)
def set_information(self, key, value):
"""updates the information's attribute
(which is a dictionary)
:param key: information's key (ex: "help", "doc"
:param value: information's value (ex: "the help string")
"""
session = self.getsession()
val = session.query(_Vinformation).filter_by(
key=key, session_id=self._storage.session_id).first()
if val is None:
session.add(_Vinformation(self._storage.session_id, key,
value))
else:
val.value = value
session.commit()
del(session)
def get_information(self, key, default):
"""retrieves one information's item
:param key: the item string (ex: "help")
"""
session = self.getsession()
val = session.query(_Vinformation).filter_by(
key=key, session_id=self._storage.session_id).first()
del(session)
if not val:
if default is not undefined:
return default
raise ValueError("not found")
return val.value
def exportation(self, session, fake=False):
if fake:
#(('path1',), (index1,), (value1,), ('owner1'))
paths = []
indexes = []
values = []
owners_ = []
slaves = {}
for val in session.query(Value).filter_by(
session_id=self._storage.session_id).all():
if val.indx is not None:
slaves.setdefault(val.path, []).append((val.indx, val.value, getattr(owners, val.owner)))
else:
paths.append(val.path)
indexes.append(val.indx)
values.append(val.value)
owners_.append(getattr(owners, val.owner))
for path, vals in slaves.items():
paths.append(path)
t_idxes = []
t_vals = []
t_owners = []
for val in vals:
t_idxes.append(val[0])
t_vals.append(val[1])
t_owners.append(val[2])
indexes.append(tuple(t_idxes))
values.append(t_vals)
owners_.append(t_owners)
return (paths, indexes, values, owners_)
pass
def importation(self, value):
pass
def delete_session(session_id, session):
for val in session.query(_Vinformation).filter_by(session_id=session_id).all():
session.delete(val)
for val in session.query(Value).filter_by(session_id=session_id).all():
session.delete(val)

10
tiramisu/storage/sqlite3/__init__.py
View File

@ -22,6 +22,12 @@ You should not configure differents Configs with same session_id.
"""
from .value import Values
from .setting import Properties, Permissives
from .storage import Storage, list_sessions, delete_session
from .storage import SETTING, Storage, list_sessions
__all__ = ('Values', 'Properties', 'Permissives', 'Storage', 'list_sessions', 'delete_session')
__all__ = ('SETTING',
'Values',
'Properties',
'Permissives',
'Storage',
'list_sessions')

17
tiramisu/storage/sqlite3/storage.py
View File

@ -17,20 +17,23 @@
# ____________________________________________________________
from ...i18n import _
from os import unlink
from os.path import basename, splitext, join, isfile
from os.path import join
import sqlite3
from glob import glob
from ...error import ConflictError
class Setting(object):
class Setting:
""":param extension: database file extension (by default: db)
:param dir_database: root database directory (by default: /tmp)
"""
extension = 'db'
dir_database = '/tmp'
name = 'tiramisu'
__slots__ = ('extension',
'dir_database',
'name')
def __init__(self):
self.extension = 'db'
self.dir_database = '/tmp'
self.name = 'tiramisu'
SETTING = Setting()