Infra
/
tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remote sqlalchemy storage (unworking) and add setting to storage

This commit is contained in:
Emmanuel Garette 2019-02-22 07:25:57 +01:00
parent fd95c6dd4a
commit 4a737c5b9d
11 changed files with 59 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tiramisu/__init__.py
View File

@ -12,12 +12,15 @@
# #
# You should have received a copy of the GNU Lesser General Public License # You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
from .function import Params, ParamOption, ParamValue, ParamContext, tiramisu_copy from .function import Params, ParamOption, ParamValue, ParamContext, \
tiramisu_copy
from .option import * from .option import *
from .error import APIError from .error import APIError
from .api import Config, MetaConfig, GroupConfig, MixConfig from .api import Config, MetaConfig, GroupConfig, MixConfig
from .option import __all__ as all_options from .option import __all__ as all_options
from .setting import owners, undefined from .setting import owners, undefined
from .storage import default_storage_type, StorageType, list_sessions, \
delete_session
allfuncs = ['Params', allfuncs = ['Params',
@ -30,6 +33,10 @@ allfuncs = ['Params',
'Config', 'Config',
'APIError', 'APIError',
'undefined', 'undefined',
'default_storage_type',
'StorageType',
'list_sessions',
'delete_session',
'tiramisu_copy'] 'tiramisu_copy']
allfuncs.extend(all_options) allfuncs.extend(all_options)
del(all_options) del(all_options)

37
tiramisu/storage/__init__.py
View File

@ -51,22 +51,19 @@ class StorageType(object):
if self.storage_type is not None: # pragma: no cover if self.storage_type is not None: # pragma: no cover
if self.storage_type == name: if self.storage_type == name:
return return
raise ConfigError(_('storage_type is already set, cannot rebind it')) raise ConfigError(_('storage_type is already set, '
'cannot rebind it'))
self.storage_type = name self.storage_type = name
def get(self): def get(self):
if self.storage_type is None: if self.storage_type is None:
self.storage_type = self.default_storage self.storage_type = self.default_storage
set_to_default = True
else:
set_to_default = False
if self.mod is None: if self.mod is None:
modulepath = '{0}.storage.{1}'.format(MODULE_PATH, self.storage_type) modulepath = '{0}.storage.{1}'.format(MODULE_PATH,
self.storage_type)
try: try:
mod = __import__(modulepath) mod = __import__(modulepath)
except ImportError: # pragma: no cover except ImportError: # pragma: no cover
import traceback
traceback.print_exc()
raise SystemError(_('cannot import the storage {0}').format( raise SystemError(_('cannot import the storage {0}').format(
self.default_storage)) self.default_storage))
for token in modulepath.split(".")[1:]: for token in modulepath.split(".")[1:]:
@ -74,10 +71,13 @@ class StorageType(object):
self.mod = mod self.mod = mod
return self.mod return self.mod
def setting(self, **kwargs):
mod = self.get()
for key, value in kwargs.items():
setattr(mod.SETTING, key, value)
storage_type = StorageType()
#storage_option_type = StorageType() default_storage_type = StorageType()
#storage_option_type.set(DEFAULT_STORAGE)
memory_storage = StorageType() memory_storage = StorageType()
memory_storage.set(MEMORY_STORAGE) memory_storage.set(MEMORY_STORAGE)
@ -89,12 +89,17 @@ def gen_storage_id(session_id,
return 'c' + str(id(config)) + str(int(time())) + str(randint(0, 500)) return 'c' + str(id(config)) + str(int(time())) + str(randint(0, 500))
def get_storages(context, session_id, persistent, storage): def get_storages(context,
session_id = gen_storage_id(session_id, context) session_id,
persistent,
storage):
session_id = gen_storage_id(session_id,
context)
if storage is None: if storage is None:
storage = storage_type storage = default_storage_type
imp = storage.get() imp = storage.get()
imp_storage = imp.Storage(session_id, persistent) imp_storage = imp.Storage(session_id,
persistent)
properties = imp.Properties(imp_storage) properties = imp.Properties(imp_storage)
permissives = imp.Permissives(imp_storage) permissives = imp.Permissives(imp_storage)
values = imp.Values(imp_storage) values = imp.Values(imp_storage)
@ -118,7 +123,7 @@ def get_default_settings_storages():
def list_sessions(): def list_sessions():
"""List all available session (persistent or not persistent) """List all available session (persistent or not persistent)
""" """
return storage_type.get().list_sessions() return default_storage_type.get().list_sessions()
def delete_session(session_id): def delete_session(session_id):
@ -126,7 +131,7 @@ def delete_session(session_id):
use by an other instance use by an other instance
:params session_id: id of session to delete :params session_id: id of session to delete
""" """
storage_module = storage_type.get() storage_module = default_storage_type.get()
session = storage_module.storage.getsession() session = storage_module.storage.getsession()
storage_module.value.delete_session(session_id) storage_module.value.delete_session(session_id)
storage_module.storage.delete_session(session_id) storage_module.storage.delete_session(session_id)

10
tiramisu/storage/dictionary/__init__.py
View File

@ -24,6 +24,12 @@ use it. But if something goes wrong, you will lost your modifications.
""" """
from .value import Values from .value import Values
from .setting import Properties, Permissives from .setting import Properties, Permissives
from .storage import setting, Storage, list_sessions from .storage import SETTING, Storage, list_sessions
__all__ = ('setting', 'Values', 'Properties', 'Permissives', 'Storage', 'list_sessions')
__all__ = ('SETTING',
'Values',
'Properties',
'Permissives',
'Storage',
'list_sessions')

8
tiramisu/storage/dictionary/storage.py
View File

@ -18,13 +18,13 @@ from ...i18n import _
from ...error import ConflictError from ...error import ConflictError
class Setting(object): class Setting:
"""Dictionary storage has no particular setting. """Dictionary storage has no particular setting.
""" """
pass __slots__ = tuple()
setting = Setting() SETTING = Setting()
_list_sessions = [] _list_sessions = []
@ -35,7 +35,7 @@ def list_sessions():
class Storage(object): class Storage(object):
__slots__ = ('session_id', 'persistent') __slots__ = ('session_id', 'persistent')
storage = 'dictionary' storage = 'dictionary'
#if object could be serializable # if object could be serializable
serializable = True serializable = True
def __init__(self, session_id, persistent, test=False): def __init__(self, session_id, persistent, test=False):

36
tiramisu/storage/sqlalchemy/__init__.py
View File

@ -1,36 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
"""Default plugin for storage. All informations are store in a simple
dictionary in memory.
You cannot have persistente informations with this kind of storage.
The advantage of this solution is that you can easily create a Config and
use it. But if something goes wrong, you will lost your modifications.
"""
from .value import Values
from .setting import Settings
from .storage import Storage, list_sessions, delete_session, storage_setting
from .util import load
load()
__all__ = (storage_setting, Values, Settings, Storage, list_sessions, delete_session,
StorageBase)
# Base, OptionDescription)

138
tiramisu/storage/sqlalchemy/setting.py
View File

@ -1,138 +0,0 @@
# -*- coding: utf-8 -*-
"default plugin for setting: set it in a simple dictionary"
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from ..util import Cache
from .util import SqlAlchemyBase
import util
from sqlalchemy import Column, Integer, String, PickleType, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm.collections import attribute_mapped_collection
#____________________________________________________________
#
# properties|permissives
class _Property(SqlAlchemyBase):
__tablename__ = 'property'
id = Column(Integer, primary_key=True)
setting = Column(Integer, ForeignKey('settings.id'), nullable=False)
path = Column(String)
properties = Column(PickleType)
def __init__(self, path, properties):
self.path = path
self.properties = properties
class _Permissive (SqlAlchemyBase):
__tablename__ = 'permissive'
id = Column(Integer, primary_key=True)
setting = Column(Integer, ForeignKey('settings.id'), nullable=False)
path = Column(String)
permissives = Column(PickleType)
def __init__(self, path, permissives):
self.path = path
self.permissives = permissives
#____________________________________________________________
#FIXME marche pas le cache ... de toute facon je vais faire un storage separe !
class Settings(Cache, SqlAlchemyBase):
__tablename__ = 'settings'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
_props = relationship("_Property",
collection_class=attribute_mapped_collection('path'),
cascade="all, delete-orphan")
_properties = association_proxy("_props", "properties")
_perms = relationship("_Permissive",
collection_class=attribute_mapped_collection('path'),
cascade="all, delete-orphan")
_permissives = association_proxy("_perms", "permissives")
def __init__(self, session_id, storage):
session = self.getsession()
self.session_id = session_id
super(Settings, self).__init__(storage)
session.commit()
def getsession(self):
return util.Session()
# properties
def setproperties(self, path, properties):
session = self.getsession()
self._properties[path] = properties
session.commit()
del(session)
def getproperties(self, path, default_properties):
ret = self._properties.get(path)
if ret is None:
return set(default_properties)
return ret
def hasproperties(self, path):
return path in self._properties
def reset_all_properties(self):
session = self.getsession()
self._properties.clear()
session.commit()
del(session)
def delproperties(self, path):
try:
session = self.getsession()
del(self._properties[path])
session.commit()
del(session)
except KeyError:
pass
# permissive
def setpermissive(self, path, permissive):
session = self.getsession()
self._permissives[path] = frozenset(permissive)
session.commit()
def getpermissive(self, path=None):
ret = self._permissives.get(path, frozenset())
#replace None by a frozenset()
return {None: frozenset()}.get(ret, ret)
def exportation(self):
"""return all modified settings in a dictionary
example: {'path1': set(['prop1', 'prop2'])}
"""
return self._properties
def importation(self):
"""return all modified permissives in a dictionary
example: {'path1': set(['perm1', 'perm2'])}
"""
return self._permissives
def delete_session(session_id, session):
settings_id = session.query(Settings).filter_by(session_id=session_id).first().id
for val in session.query(_Property).filter_by(settings=settings_id).all():
session.delete(val)
for val in session.query(_Permissive).filter_by(settings=settings_id).all():
session.delete(val)

81
tiramisu/storage/sqlalchemy/storage.py
View File

@ -1,81 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from tiramisu.i18n import _
from .util import SqlAlchemyBase
import util
from sqlalchemy import Column, Integer, String
class Setting(object):
""":param extension: database file extension (by default: db)
:param dir_database: root database directory (by default: /tmp)
"""
#FIXME
extension = 'db'
dir_database = '/tmp'
storage_setting = Setting()
class Session(SqlAlchemyBase):
__tablename__ = 'session'
id = Column(Integer, primary_key=True)
session = Column(String, index=True)
def __init__(self, session_id):
self.session = session_id
def list_sessions():
session = util.Session()
ret = []
for val in session.query(Session).all():
ret.append(val.session)
del(session)
return ret
def delete_session(session_id, session):
session.delete(session.query(Session).filter_by(session=session_id).first())
session.commit()
def getsession():
return util.Session()
class Storage(object):
__slots__ = ('session_id', 'persistent')
storage = 'sqlalchemy'
#if object could be serializable
serializable = True
def __init__(self, session_id, persistent, test=False):
session = getsession()
self.session_id = session_id
self.persistent = persistent
if not session.query(Session).filter_by(session=session_id).first():
session.add(Session(session_id))
session.commit()
del(session)
def __del__(self):
if not self.persistent:
session = getsession()
delete_session(self.session_id, session)
del(session)

38
tiramisu/storage/sqlalchemy/util.py
View File

@ -1,38 +0,0 @@
# -*- coding: utf-8 -*-
""
# Copyright (C) 2014-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# ____________________________________________________________
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:')
SqlAlchemyBase = declarative_base()
global session, Session
Session = None
def load():
global session, Session
if Session is None:
#engine.echo = True
#print SqlAlchemyBase.metadata.tables.keys()
SqlAlchemyBase.metadata.create_all(engine)
Session = sessionmaker(bind=engine, expire_on_commit=False)

228
tiramisu/storage/sqlalchemy/value.py
View File

@ -1,228 +0,0 @@
# -*- coding: utf-8 -*-
"plugin for value: set it in sqlalchemy"
# Copyright (C) 2013-2019 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
from ..util import Cache
from .util import SqlAlchemyBase
import util
from ...setting import undefined
from sqlalchemy import Column, Integer, String, PickleType
from sqlalchemy import func
from tiramisu.setting import owners
#____________________________________________________________
#
# information
class _Vinformation(SqlAlchemyBase):
__tablename__ = 'vinformation'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
path = Column(String, index=True)
key = Column(String)
value = Column(PickleType)
def __init__(self, session_id, key, value):
self.session_id = session_id
self.key = key
self.value = value
class Value(SqlAlchemyBase):
__tablename__ = 'value'
id = Column(Integer, primary_key=True)
session_id = Column(String, index=True)
path = Column(String, index=True)
key = Column(String)
value = Column(PickleType)
owner = Column(String)
indx = Column(Integer, index=True)
def __init__(self, session_id, path, value, owner, index):
self.session_id = session_id
self.path = path
self.value = value
self.owner = owner
self.indx = index
class Values(Cache):
def getsession(self):
return util.Session()
# value
def setvalue(self, path, value, owner, index, session):
"""set value for a path
a specified value must be associated to an owner
"""
#if it's a multi
if isinstance(value, list):
value = list(value)
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if val is None:
session.add(Value(self._storage.session_id, path, value,
owner, index))
else:
val.value = value
val.owner = owner
session.commit()
def getvalue(self, path, session, index=None):
"""get value for a path
return: only value, not the owner
"""
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if not val:
raise KeyError('no value found')
return val.value
def hasvalue(self, path, session):
"""if path has a value
return: boolean
"""
return session.query(Value).filter_by(
path=path, session_id=self._storage.session_id).first() is not None
def resetvalue(self, path, session):
"""remove value means delete value in storage
"""
vals = session.query(Value).filter_by(
path=path, session_id=self._storage.session_id).all()
if vals != []:
for val in vals:
session.delete(val)
session.commit()
# owner
def setowner(self, path, owner, session, index=None):
"""change owner for a path
"""
val = session.query(Value).filter_by(
path=path, indx=index, session_id=self._storage.session_id).first()
if val is None:
raise KeyError('no value found')
else:
val.owner = owner
session.commit()
def get_max_length(self, path, session):
val = session.query(Value, func.max(Value.indx)).filter_by(
path=path, session_id=self._storage.session_id).first()
if val[1] is None:
maxval = 0
else:
maxval = val[1] + 1
return maxval
def getowner(self, path, default, session, index=None, only_default=False):
#FIXME support de only_default
"""get owner for a path
return: owner object
"""
session.commit()
val = session.query(Value).filter_by(
path=path, session_id=self._storage.session_id,
indx=index).first()
if val is None:
return default
else:
owner = val.owner
# autocreate owners
try:
return getattr(owners, owner)
except AttributeError:
owners.addowner(owner)
return getattr(owners, owner)
def set_information(self, key, value):
"""updates the information's attribute
(which is a dictionary)
:param key: information's key (ex: "help", "doc"
:param value: information's value (ex: "the help string")
"""
session = self.getsession()
val = session.query(_Vinformation).filter_by(
key=key, session_id=self._storage.session_id).first()
if val is None:
session.add(_Vinformation(self._storage.session_id, key,
value))
else:
val.value = value
session.commit()
del(session)
def get_information(self, key, default):
"""retrieves one information's item
:param key: the item string (ex: "help")
"""
session = self.getsession()
val = session.query(_Vinformation).filter_by(
key=key, session_id=self._storage.session_id).first()
del(session)
if not val:
if default is not undefined:
return default
raise ValueError("not found")
return val.value
def exportation(self, session, fake=False):
if fake:
#(('path1',), (index1,), (value1,), ('owner1'))
paths = []
indexes = []
values = []
owners_ = []
slaves = {}
for val in session.query(Value).filter_by(
session_id=self._storage.session_id).all():
if val.indx is not None:
slaves.setdefault(val.path, []).append((val.indx, val.value, getattr(owners, val.owner)))
else:
paths.append(val.path)
indexes.append(val.indx)
values.append(val.value)
owners_.append(getattr(owners, val.owner))
for path, vals in slaves.items():
paths.append(path)
t_idxes = []
t_vals = []
t_owners = []
for val in vals:
t_idxes.append(val[0])
t_vals.append(val[1])
t_owners.append(val[2])
indexes.append(tuple(t_idxes))
values.append(t_vals)
owners_.append(t_owners)
return (paths, indexes, values, owners_)
pass
def importation(self, value):
pass
def delete_session(session_id, session):
for val in session.query(_Vinformation).filter_by(session_id=session_id).all():
session.delete(val)
for val in session.query(Value).filter_by(session_id=session_id).all():
session.delete(val)

10
tiramisu/storage/sqlite3/__init__.py
View File

@ -22,6 +22,12 @@ You should not configure differents Configs with same session_id.
""" """
from .value import Values from .value import Values
from .setting import Properties, Permissives from .setting import Properties, Permissives
from .storage import Storage, list_sessions, delete_session from .storage import SETTING, Storage, list_sessions
__all__ = ('Values', 'Properties', 'Permissives', 'Storage', 'list_sessions', 'delete_session')
__all__ = ('SETTING',
'Values',
'Properties',
'Permissives',
'Storage',
'list_sessions')

17
tiramisu/storage/sqlite3/storage.py
View File

@ -17,20 +17,23 @@
# ____________________________________________________________ # ____________________________________________________________
from ...i18n import _ from ...i18n import _
from os import unlink from os.path import join
from os.path import basename, splitext, join, isfile
import sqlite3 import sqlite3
from glob import glob
from ...error import ConflictError from ...error import ConflictError
class Setting(object): class Setting:
""":param extension: database file extension (by default: db) """:param extension: database file extension (by default: db)
:param dir_database: root database directory (by default: /tmp) :param dir_database: root database directory (by default: /tmp)
""" """
extension = 'db' __slots__ = ('extension',
dir_database = '/tmp' 'dir_database',
name = 'tiramisu' 'name')
def __init__(self):
self.extension = 'db'
self.dir_database = '/tmp'
self.name = 'tiramisu'
SETTING = Setting() SETTING = Setting()