Infra/tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

better support for sqlalchemy storage

Browse Source
This commit is contained in:
Emmanuel Garette
2014-07-06 15:31:57 +02:00
parent 3cc2d9ca3d
commit 71f8926fca
16 changed files with 369 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tiramisu/storage/__init__.py
View File

@ -117,15 +117,20 @@ def get_storages(context, session_id, persistent):
session_id = gen_id(context)
imp = storage_type.get()
storage = imp.Storage(session_id, persistent)
return imp.Settings(storage), imp.Values(storage)
try:
return imp.Settings(storage), imp.Values(storage)
except:
import traceback
traceback.print_exc()
raise Exception('rah')
def get_storages_option(type_):
imp = storage_option_type.get()
if type_ == 'base':
return imp.Base
return imp.StorageBase
else:
return imp.OptionDescription
return imp.StorageOptionDescription
def list_sessions(type_): # pragma: optional cover

6
tiramisu/storage/dictionary/__init__.py
View File

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2013 Team tiramisu (see AUTHORS for all contributors)
# Copyright (C) 2013-2014 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
@ -25,7 +25,7 @@ use it. But if something goes wrong, you will lost your modifications.
from .value import Values
from .setting import Settings
from .storage import setting, Storage, list_sessions, delete_session
from .option import Base, OptionDescription
from .option import StorageBase, StorageOptionDescription
__all__ = (setting, Values, Settings, Storage, list_sessions, delete_session,
Base, OptionDescription)
StorageBase, StorageOptionDescription)

65
tiramisu/storage/dictionary/option.py
View File

@ -18,14 +18,14 @@
#
# ____________________________________________________________
from tiramisu.i18n import _
from tiramisu.setting import groups, undefined
from tiramisu.setting import undefined
from tiramisu.error import ConfigError
#____________________________________________________________
#
# Base
class Base(object):
class StorageBase(object):
__slots__ = ('_name', '_requires', '_properties', '_readonly',
'_calc_properties', '_informations',
'_state_readonly', '_state_requires', '_stated',
@ -34,13 +34,14 @@ class Base(object):
'_state_callback_params', '_callback_params', '_multitype',
'_consistencies', '_warnings_only', '_master_slaves',
'_state_consistencies', '_extra', '_subdyn', '__weakref__',
'_state_master_slaves')
'_state_master_slaves', '_choice_values',
'_choice_values_params')
def __init__(self):
try:
self._subdyn
except AttributeError:
self._subdyn = False
self._subdyn = None
try:
self._consistencies
except AttributeError:
@ -52,7 +53,7 @@ class Base(object):
try:
self._callback_params
except AttributeError:
self._callback_params = None
self._callback_params = {}
try:
self._validator
except AttributeError:
@ -71,19 +72,22 @@ class Base(object):
def _get_id(self):
return id(self)
def _is_subdyn(self):
try:
return self._subdyn is not False
except AttributeError:
return False
def _impl_getsubdyn(self):
return self._subdyn
def _impl_setsubdyn(self, subdyn):
self._subdyn = subdyn
def commit(self):
pass
class OptionDescription(Base):
class StorageOptionDescription(StorageBase):
__slots__ = ('_children', '_cache_paths', '_cache_consistencies',
'_group_type', '_is_build_cache', '_state_group_type')
def __init__(self):
pass
self._cache_paths = None
def _add_children(self, child_names, children):
self._children = (tuple(child_names), tuple(children))
@ -104,10 +108,14 @@ class OptionDescription(Base):
raise AttributeError(_('no option {0} found').format(opt))
def impl_get_group_type(self): # pragma: optional cover
return getattr(groups, self._group_type)
return self._group_type
def impl_build_cache_option(self, _currpath=None, cache_path=None,
cache_option=None):
try:
self._cache_paths
except AttributeError:
self._cache_paths = None
if _currpath is None and self._cache_paths is not None: # pragma: optional cover
# cache already set
return
@ -145,8 +153,7 @@ class OptionDescription(Base):
found = True
break
if not found:
#FIXME
raise ConfigError(_('hu?'))
raise ConfigError(_('cannot find dynpath'))
subpath = subpath + suffix
for slength in xrange(length, len(spath)):
subpath = subpath + '.' + spath[slength] + suffix
@ -226,21 +233,17 @@ class OptionDescription(Base):
return find_results[0]
return find_results
def _impl_st_getchildren(self):
return self._children[1]
def _impl_st_getchildren(self, context, only_dyn=False):
for child in self._children[1]:
if only_dyn is False or child.impl_is_dynoptiondescription():
yield(child)
def __getattr__(self, name, context=undefined):
if name == '_name':
return object.__getattribute__(self, name)
return self._getattr(name, context=context)
def _getattr(self, name, dyn_od=undefined, suffix=undefined,
context=undefined, dyn=True):
def _getattr(self, name, suffix=undefined, context=undefined, dyn=True):
error = False
if suffix is not undefined:
try:
if undefined in [dyn_od, suffix, context]: # pragma: optional cover
raise ConfigError(_("dyn_od, suffix and context needed if "
if undefined in [suffix, context]: # pragma: optional cover
raise ConfigError(_("suffix and context needed if "
"it's a dyn option"))
if name.endswith(suffix):
oname = name[:-len(suffix)]
@ -270,3 +273,13 @@ class OptionDescription(Base):
raise AttributeError(_('unknown Option {0} '
'in OptionDescription {1}'
'').format(name, self._name))
def _get_force_store_value(self):
#FIXME faire des tests (notamment pas ajouter à un config)
#FIXME devrait faire un cache !
opts, paths = self._cache_paths
for index in range(0, len(paths)):
opt = opts[index]
path = paths[index]
if 'force_store_value' in opt._properties:
yield (opt, path)

3
tiramisu/storage/dictionary/setting.py
View File

@ -42,13 +42,12 @@ class Settings(Cache):
def reset_all_properties(self):
self._properties.clear()
def reset_properties(self, path):
def delproperties(self, path):
try:
del(self._properties[path])
except KeyError:
pass
# permissive
def setpermissive(self, path, permissive):
self._permissives[path] = frozenset(permissive)

2
tiramisu/storage/dictionary/value.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
"default plugin for value: set it in a simple dictionary"
# Copyright (C) 2013 Team tiramisu (see AUTHORS for all contributors)
# Copyright (C) 2013-2014 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the

37
tiramisu/storage/sqlalchemy/__init__.py
View File

@ -0,0 +1,37 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Team tiramisu (see AUTHORS for all contributors)
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# ____________________________________________________________
"""Default plugin for storage. All informations are store in a simple
dictionary in memory.
You cannot have persistente informations with this kind of storage.
The advantage of this solution is that you can easily create a Config and
use it. But if something goes wrong, you will lost your modifications.
"""
from .value import Values
from .setting import Settings
from .storage import Storage, list_sessions, delete_session, setting
from .option import StorageBase, StorageOptionDescription
from .util import load
load()
__all__ = (setting, Values, Settings, Storage, list_sessions, delete_session,
StorageBase, StorageOptionDescription)
# Base, OptionDescription)

272
tiramisu/storage/sqlalchemy/option.py
View File

@ -18,28 +18,20 @@
#
# ____________________________________________________________
from tiramisu.i18n import _
from tiramisu.setting import groups
from tiramisu.setting import groups, undefined
from tiramisu.error import ConfigError
from .util import SqlAlchemyBase
import util
from sqlalchemy import not_, or_
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy import create_engine, Column, Integer, String, Boolean, \
PickleType, ForeignKey, Table
from sqlalchemy import Column, Integer, String, Boolean, PickleType, \
ForeignKey, Table
from sqlalchemy.orm import relationship, backref
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.collections import attribute_mapped_collection
#FIXME
engine = create_engine('sqlite:///:memory:')
SqlAlchemyBase = declarative_base()
#FIXME a voir:
# # Organization.members will be a Query object - no loading
# # of the entire collection occurs unless requested
# lazy="dynamic",
#____________________________________________________________
#
# require
from itertools import chain
def load_requires(collection_type, proxy):
@ -49,7 +41,7 @@ def load_requires(collection_type, proxy):
ret = []
requires = getattr(obj, proxy.value_attr)
for require in requires:
option = session.query(_Base).filter_by(id=require.option).first()
option = util.session.query(_Base).filter_by(id=require.option).first()
ret.append(tuple([option, require.expected, require.action, require.inverse, require.transitive, require.same_action]))
return tuple(ret)
@ -158,7 +150,7 @@ def load_callback_parm(collection_type, proxy):
if require.value is not None:
ret.append(require.value)
else:
option = session.query(_Base).filter_by(id=require.option).first()
option = util.session.query(_Base).filter_by(id=require.option).first()
ret.append((option, require.force_permissive))
return tuple(ret)
@ -175,10 +167,10 @@ class _CallbackParamOption(SqlAlchemyBase):
force_permissive = Column(Boolean)
value = Column(PickleType)
def __init__(self, option=None, force_permissive=None, value=None):
if value is not None:
def __init__(self, option=undefined, force_permissive=undefined, value=undefined):
if value is not undefined:
self.value = value
else:
elif option is not undefined:
self.option = option.id
self.force_permissive = force_permissive
@ -194,8 +186,11 @@ class _CallbackParam(SqlAlchemyBase):
self.key = key
for param in params:
if isinstance(param, tuple):
self.params.append(_CallbackParamOption(option=param[0],
force_permissive=param[1]))
if param == (None,):
self.params.append(_CallbackParamOption())
else:
self.params.append(_CallbackParamOption(option=param[0],
force_permissive=param[1]))
else:
self.params.append(_CallbackParamOption(value=param))
@ -215,7 +210,7 @@ class _Consistency(SqlAlchemyBase):
func = Column(PickleType)
params = Column(PickleType)
def __init__(self, func, all_cons_opts):
def __init__(self, func, all_cons_opts, params):
self.func = func
for option in all_cons_opts:
option._consistencies.append(self)
@ -249,9 +244,16 @@ class _Base(SqlAlchemyBase):
_informations = association_proxy("_infos", "value")
_default = Column(PickleType)
_default_multi = Column(PickleType)
_subdyn = Column(Integer)
_choice_values = Column(PickleType)
_cho_params = relationship('_CallbackParam',
collection_class=
attribute_mapped_collection('key'))
_choice_params = association_proxy("_cho_params", "params",
getset_factory=load_callback_parm)
_reqs = relationship("_Require", collection_class=list)
_requires = association_proxy("_reqs", "requires", getset_factory=load_requires)
_multi = Column(Boolean)
_multi = Column(Integer)
_multitype = Column(String)
######
_callback = Column(PickleType)
@ -264,7 +266,7 @@ class _Base(SqlAlchemyBase):
_val_params = relationship('_CallbackParam',
collection_class=
attribute_mapped_collection('key'))
_validator_params = association_proxy("_call_params", "params",
_validator_params = association_proxy("_val_params", "params",
getset_factory=load_callback_parm)
######
#FIXME pas 2 fois la meme properties dans la base ...
@ -290,11 +292,11 @@ class _Base(SqlAlchemyBase):
_is_build_cache = Column(Boolean, default=False)
def __init__(self):
util.session.add(self)
self.commit()
def commit(self):
session.add(self)
session.commit()
util.session.commit()
def _add_consistency(self, func, all_cons_opts, params):
_Consistency(func, all_cons_opts, params)
@ -306,52 +308,35 @@ class _Base(SqlAlchemyBase):
def _get_id(self):
return self.id
# ____________________________________________________________
# information
#def impl_set_information(self, key, value):
# """updates the information's attribute
# (which is a dictionary)
def _impl_getsubdyn(self):
return self._subdyn
# :param key: information's key (ex: "help", "doc"
# :param value: information's value (ex: "the help string")
# """
# info = session.query(_Information).filter_by(option=self.id, key=key).first()
# #FIXME pas append ! remplacer !
# if info is None:
# self._informations.append(_Information(key, value))
# else:
# info.value = value
#def impl_get_information(self, key, default=None):
# """retrieves one information's item
# :param key: the item string (ex: "help")
# """
# info = session.query(_Information).filter_by(option=self.id, key=key).first()
# if info is not None:
# return info.value
# return self._informations[key]
# elif default is not None:
# return default
# else:
# raise ValueError(_("information's item not found: {0}").format(
# key))
def _impl_setsubdyn(self, subdyn):
self._subdyn = subdyn.id
class Cache(SqlAlchemyBase):
__tablename__ = 'cache'
id = Column(Integer, primary_key=True)
#FIXME indexer ... les 3
path = Column(String, nullable=False)
descr = Column(Integer, nullable=False)
option = Column(Integer, nullable=False)
opt_type = Column(String, nullable=False)
path = Column(String, nullable=False, index=True)
descr = Column(Integer, nullable=False, index=True)
parent = Column(Integer, nullable=False, index=True)
option = Column(Integer, nullable=False, index=True)
opt_type = Column(String, nullable=False, index=True)
is_subdyn = Column(Boolean, nullable=False, index=True)
subdyn_path = Column(String)
def __init__(self, descr, option, path):
def __init__(self, descr, parent, option, path, subdyn_path):
self.descr = descr.id
self.parent = parent.id
self.option = option.id
self.path = path
self.opt_type = option.__class__.__name__
#is_subdyn = option._is_subdyn()
is_subdyn = option.impl_is_dynoptiondescription()
self.is_subdyn = is_subdyn
if is_subdyn:
self.subdyn_path = subdyn_path
class StorageOptionDescription(object):
@ -359,14 +344,14 @@ class StorageOptionDescription(object):
return self._is_build_cache
def impl_get_opt_by_path(self, path):
ret = session.query(Cache).filter_by(descr=self.id, path=path).first()
ret = util.session.query(Cache).filter_by(descr=self.id, path=path).first()
if ret is None:
raise AttributeError(_('no option for path {0}').format(path))
return session.query(_Base).filter_by(id=ret.option).first()
return util.session.query(_Base).filter_by(id=ret.option).first()
def impl_get_path_by_opt(self, opt):
ret = session.query(Cache).filter_by(descr=self.id,
option=opt.id).first()
ret = util.session.query(Cache).filter_by(descr=self.id,
option=opt.id).first()
if ret is None:
raise AttributeError(_('no option {0} found').format(opt))
return ret.path
@ -374,30 +359,36 @@ class StorageOptionDescription(object):
def impl_get_group_type(self):
return getattr(groups, self._group_type)
def impl_build_cache_option(self, descr=None, _currpath=None):
def impl_build_cache_option(self, descr=None, _currpath=None,
subdyn_path=None):
if descr is None:
save = True
descr = self
_currpath = []
else:
save = False
for option in self.impl_getchildren():
for option in self._impl_getchildren(dyn=False):
attr = option.impl_getname()
session.add(Cache(descr, option,
str('.'.join(_currpath + [attr]))))
util.session.add(Cache(descr, self, option,
str('.'.join(_currpath + [attr])), subdyn_path))
if isinstance(option, StorageOptionDescription):
if option.impl_is_dynoptiondescription():
subdyn_path = '.'.join(_currpath)
_currpath.append(attr)
option.impl_build_cache_option(descr,
_currpath)
_currpath,
subdyn_path)
_currpath.pop()
if save:
self._is_build_cache = True
session.commit()
util.session.commit()
def impl_get_options_paths(self, bytype, byname, _subpath, only_first):
sqlquery = session.query(Cache).filter_by(descr=self.id)
def impl_get_options_paths(self, bytype, byname, _subpath, only_first,
context):
sqlquery = util.session.query(Cache).filter_by(descr=self.id)
if bytype is None:
sqlquery = sqlquery.filter(not_(Cache.opt_type == 'OptionDescription'))
sqlquery = sqlquery.filter(not_(
Cache.opt_type == 'OptionDescription'))
else:
sqlquery = sqlquery.filter_by(opt_type=bytype.__name__)
@ -413,46 +404,111 @@ class StorageOptionDescription(object):
if or_query != '':
filter_query = or_(Cache.path == or_query, filter_query)
sqlquery = sqlquery.filter(filter_query)
if only_first:
opt = sqlquery.first()
if opt is None:
return tuple()
option = session.query(_Base).filter_by(id=opt.option).first()
return ((opt.path, option),)
else:
ret = []
for opt in sqlquery.all():
option = session.query(_Base).filter_by(id=opt.option).first()
ret.append((opt.path, option))
return ret
#if only_first:
# opt = sqlquery.first()
# if opt is None:
# return tuple()
# option = util.session.query(_Base).filter_by(id=opt.option).first()
# return ((opt.path, option),)
#else:
ret = []
for opt in sqlquery.all():
option = util.session.query(_Base).filter_by(id=opt.option).first()
if opt.is_subdyn:
name = option.impl_getname()
if byname is not None and byname.startswith(name):
found = False
for suffix in option._subdyn._impl_get_suffixes(
context):
if byname == name + suffix:
found = True
subdyn_path = opt.subdyn_path
dynpaths = opt.path[len(subdyn_path):].split('.')
path = subdyn_path
for dynpath in dynpaths:
path += '.' + dynpath + suffix
option = option._impl_to_dyn(
name + suffix, path)
break
if not found:
break
else:
ret_opt = (opt.path, option)
if only_first:
return ret_opt
ret.append(ret_opt)
return ret
def _add_children(self, child_names, children):
for child in children:
session.add(_Parent(self, child))
util.session.add(_Parent(self, child))
def impl_getchildren(self):
for child in session.query(_Parent).filter_by(parent_id=self.id).all():
yield(session.query(_Base).filter_by(id=child.child_id).first())
#return
def _impl_st_getchildren(self, context, only_dyn=False):
if only_dyn is False or context is undefined:
for child in util.session.query(_Parent).filter_by(
parent_id=self.id).all():
yield(util.session.query(_Base).filter_by(id=child.child_id
).first())
else:
descr = context.cfgimpl_get_description().id
for child in util.session.query(Cache).filter_by(descr=descr,
parent=self.id
).filter_by(
is_subdyn=True).all():
yield(util.session.query(_Base).filter_by(id=child.option).first())
def __getattr__(self, name):
if name.startswith('_') or name.startswith('impl_'):
return object.__getattribute__(self, name)
child = session.query(_Parent).filter_by(parent_id=self.id, child_name=name).first()
if child is None:
raise AttributeError(_('unknown Option {0} '
'in OptionDescription {1}'
'').format(name, self.impl_getname()))
return session.query(_Base).filter_by(id=child.child_id).first()
def _getattr(self, name, suffix=undefined, context=undefined, dyn=True):
error = False
if suffix is not undefined:
try:
if undefined in [suffix, context]: # pragma: optional cover
raise ConfigError(_("suffix and context needed if "
"it's a dyn option"))
if name.endswith(suffix):
oname = name[:-len(suffix)]
#child = self._children[1][self._children[0].index(oname)]
child = util.session.query(_Parent).filter_by(
parent_id=self.id, child_name=oname).first()
if child is None:
error = True
else:
opt = util.session.query(_Base).filter_by(
id=child.child_id).first()
return self._impl_get_dynchild(opt, suffix)
else:
error = True
except ValueError: # pragma: optional cover
error = True
else:
child = util.session.query(_Parent).filter_by(parent_id=self.id,
child_name=name
).first()
if child is None:
child = self._impl_search_dynchild(name, context=context)
if child != []:
return child
error = True
if error is False:
return util.session.query(_Base).filter_by(id=child.child_id
).first()
if error:
raise AttributeError(_('unknown Option {0} in OptionDescription {1}'
'').format(name, self.impl_getname()))
def _get_force_store_value(self):
#only option in current tree
current_ids = tuple(chain(*util.session.query(Cache.option).filter_by(
descr=self.id).all()))
for prop in util.session.query(_PropertyOption).filter(
_PropertyOption.option.in_(current_ids),
_PropertyOption.name == 'force_store_value').all():
opt = util.session.query(_Base).filter_by(id=prop.option).first()
path = self.impl_get_path_by_opt(opt)
yield (opt, path)
class StorageBase(_Base):
@declared_attr
def __mapper_args__(self):
return {'polymorphic_identity': self.__name__.lower()}
#engine.echo = True
SqlAlchemyBase.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()