Infra
/
tiramisu
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

reduce memory usage

This commit is contained in:
Emmanuel Garette 2014-10-25 22:11:31 +02:00
parent 58c22aa70f
commit 9f3d676280
9 changed files with 427 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ChangeLog
View File

@ -13,7 +13,7 @@ Sun Apr 27 10:32:40 2014 +0200 Emmanuel Garette <egarette@cadoles.com>
* behavior change in ChoiceOption: * behavior change in ChoiceOption:
remove open_values, that no sens (no type validation is possible) if remove open_values, that no sens (no type validation is possible) if
you wan't something like open_values, please use a typed option and you want something like open_values, please use a typed option and
add impl_(s|g)et_information to add proposed values and use it in your add impl_(s|g)et_information to add proposed values and use it in your
code code
* add dynamic ChoiceOption: * add dynamic ChoiceOption:

2
test/test_config.py
View File

@ -343,4 +343,4 @@ def test_config_od_function():
print cfg.impl_get_opt_by_path() print cfg.impl_get_opt_by_path()
except AttributeError, err: except AttributeError, err:
assert str(err) == _('unknown Option {0} in OptionDescription {1}' assert str(err) == _('unknown Option {0} in OptionDescription {1}'
'').format('impl_get_opt_by_path', descr._name) '').format('impl_get_opt_by_path', descr.impl_getname())

40
test/test_state.py
View File

@ -12,7 +12,7 @@ from py.test import raises
def return_value(value=None): def return_value(value=None):
return value return value
def _get_slots(opt): def _get_slots(opt):
@ -52,14 +52,20 @@ def _diff_opt(opt1, opt2):
val2 = None val2 = None
try: try:
val1 = getattr(opt1, attr) val1 = getattr(opt1, attr)
except: msg1 = "exists"
except Exception, err:
err1 = True err1 = True
msg1 = "not exists"
try: try:
val2 = getattr(opt2, attr) val2 = getattr(opt2, attr)
msg2 = "exists"
except: except:
err2 = True err2 = True
assert err1 == err2 msg2 = "not exists"
if not err1 == err2:
raise ValueError("{0} {1} before but {2} after for {3}".format(attr, msg1, msg2, opt1.impl_getname()))
if val1 is None: if val1 is None:
assert val1 == val2 assert val1 == val2
elif attr == '_children': elif attr == '_children':
@ -81,19 +87,23 @@ def _diff_opt(opt1, opt2):
assert consistency[0] == val2[index][0] assert consistency[0] == val2[index][0]
for idx, opt in enumerate(consistency[1]): for idx, opt in enumerate(consistency[1]):
assert opt._name == val2[index][1][idx]._name assert opt._name == val2[index][1][idx]._name
elif attr == '_callback': elif attr == '_val_call':
assert val1 == val2 for idx, v in enumerate(val1):
elif attr == '_callback_params': if v is None:
if val1 is not None: assert val2[idx] is None
for key, values in val1.items(): else:
for idx, value in enumerate(values): assert v[0] == val2[idx][0]
if isinstance(value, tuple): if len(v) == 2:
assert val1[key][idx][0]._name == val2[key][idx][0]._name if v[1] is not None:
assert val1[key][idx][1] == val2[key][idx][1] for key, values in v[1].items():
for i, value in enumerate(values):
if isinstance(value, tuple):
assert v[1][key][i][0].impl_getname() == val2[idx][1][key][i][0].impl_getname()
assert v[1][key][i][1] == val2[idx][1][key][i][1]
else:
assert v[1][key][i] == val2[idx][1][key][i]
else: else:
assert val1[key][idx] == val2[key][idx] assert v[1] == val2[idx][1]
else:
assert val1 == val2
elif attr == '_master_slaves': elif attr == '_master_slaves':
assert val1.master.impl_getname() == val2.master.impl_getname() assert val1.master.impl_getname() == val2.master.impl_getname()
sval1 = [opt.impl_getname() for opt in val1.slaves] sval1 = [opt.impl_getname() for opt in val1.slaves]

292
tiramisu/option/baseoption.py
View File

@ -19,7 +19,6 @@
# the whole pypy projet is under MIT licence # the whole pypy projet is under MIT licence
# ____________________________________________________________ # ____________________________________________________________
import re import re
from copy import copy
from types import FunctionType from types import FunctionType
import warnings import warnings
@ -31,12 +30,8 @@ from tiramisu.storage import get_storages_option
StorageBase = get_storages_option('base') StorageBase = get_storages_option('base')
submulti = 2 submulti = 2
allowed_character = '[a-zA-Z\d\-_]'
allowed_character = '[a-z\d\-_]'
name_regexp = re.compile(r'^[a-z]{0}*$'.format(allowed_character)) name_regexp = re.compile(r'^[a-z]{0}*$'.format(allowed_character))
forbidden_names = ('iter_all', 'iter_group', 'find', 'find_first', forbidden_names = ('iter_all', 'iter_group', 'find', 'find_first',
'make_dict', 'unwrap_from_path', 'read_only', 'make_dict', 'unwrap_from_path', 'read_only',
@ -101,86 +96,62 @@ def validate_callback(callback, callback_params, type_):
class Base(StorageBase): class Base(StorageBase):
__slots__ = tuple() __slots__ = tuple()
def impl_set_callback(self, callback, callback_params=None):
if callback is None and callback_params is not None: # pragma: optional cover
raise ValueError(_("params defined for a callback function but "
"no callback defined"
" yet for option {0}").format(
self.impl_getname()))
self._validate_callback(callback, callback_params)
if callback is not None:
validate_callback(callback, callback_params, 'callback')
self._callback = callback
if callback_params is None:
self._callback_params = {}
else:
self._callback_params = callback_params
def __init__(self, name, doc, default=None, default_multi=None, def __init__(self, name, doc, default=None, default_multi=None,
requires=None, multi=False, callback=None, requires=None, multi=False, callback=None,
callback_params=None, validator=None, validator_params=None, callback_params=None, validator=None, validator_params=None,
properties=None, warnings_only=False): properties=None, warnings_only=False, extra=None):
if not valid_name(name): # pragma: optional cover if not valid_name(name): # pragma: optional cover
raise ValueError(_("invalid name: {0} for option").format(name)) raise ValueError(_("invalid name: {0} for option").format(name))
self._name = name
self._readonly = False
self._informations = {}
self.impl_set_information('doc', doc)
if requires is not None: if requires is not None:
self._calc_properties, self._requires = validate_requires_arg( self._calc_properties, self._requires = validate_requires_arg(
requires, self._name) requires, name)
else: #else:
self._calc_properties = frozenset() # self._calc_properties = frozenset()
self._requires = [] # self._requires = []
if not multi and default_multi is not None: # pragma: optional cover if not multi and default_multi is not None: # pragma: optional cover
raise ValueError(_("a default_multi is set whereas multi is False" raise ValueError(_("a default_multi is set whereas multi is False"
" in option: {0}").format(name)) " in option: {0}").format(name))
if default_multi is not None:
try:
self._validate(default_multi)
except ValueError as err: # pragma: optional cover
raise ValueError(_("invalid default_multi value {0} "
"for option {1}: {2}").format(
str(default_multi), name, err))
if multi is True: if multi is True:
self._multi = 0 _multi = 0
elif multi is False: elif multi is False:
self._multi = 1 _multi = 1
elif multi is submulti: elif multi is submulti:
self._multi = submulti _multi = submulti
if self._multi != 1:
if default is None:
default = []
self._default_multi = default_multi
if properties is None: if properties is None:
properties = tuple() properties = tuple()
if not isinstance(properties, tuple): # pragma: optional cover if not isinstance(properties, tuple): # pragma: optional cover
raise TypeError(_('invalid properties type {0} for {1},' raise TypeError(_('invalid properties type {0} for {1},'
' must be a tuple').format( ' must be a tuple').format(
type(properties), type(properties),
self._name)) name))
if validator is not None: if validator is not None:
validate_callback(validator, validator_params, 'validator') validate_callback(validator, validator_params, 'validator')
self._validator = validator self._set_validator(validator, validator_params)
if validator_params is not None: if self.impl_get_calc_properties() != frozenset([]) and properties is not tuple(): # pragma: optional cover
self._validator_params = validator_params set_forbidden_properties = self.impl_get_calc_properties() & set(properties)
if self._calc_properties != frozenset([]) and properties is not tuple(): # pragma: optional cover
set_forbidden_properties = self._calc_properties & set(properties)
if set_forbidden_properties != frozenset(): if set_forbidden_properties != frozenset():
raise ValueError('conflict: properties already set in ' raise ValueError('conflict: properties already set in '
'requirement {0}'.format( 'requirement {0}'.format(
list(set_forbidden_properties))) list(set_forbidden_properties)))
if multi and default is None: super(Base, self).__init__(name, _multi, warnings_only, doc, extra)
self._default = [] self._set_default_values(default, default_multi)
else:
self._default = default
if callback is not False: if callback is not False:
self.impl_set_callback(callback, callback_params) self.impl_set_callback(callback, callback_params)
self._properties = properties self._properties = properties
self._warnings_only = warnings_only
ret = super(Base, self).__init__() def impl_set_callback(self, callback, callback_params=None):
self.impl_validate(self._default) if callback is None and callback_params is not None: # pragma: optional cover
return ret raise ValueError(_("params defined for a callback function but "
"no callback defined"
" yet for option {0}").format(
self.impl_getname()))
if self.impl_get_callback()[0] is not None:
raise ConfigError(_("a callback is already set for option {0}, "
"cannot set another one's".format(self.impl_getname())))
self._validate_callback(callback, callback_params)
if callback is not None:
validate_callback(callback, callback_params, 'callback')
self._set_callback(callback, callback_params)
def impl_is_optiondescription(self): def impl_is_optiondescription(self):
return self.__class__.__name__ in ['OptionDescription', return self.__class__.__name__ in ['OptionDescription',
@ -199,29 +170,6 @@ class BaseOption(Base):
""" """
__slots__ = tuple() __slots__ = tuple()
# information
def impl_set_information(self, key, value):
"""updates the information's attribute
(which is a dictionary)
:param key: information's key (ex: "help", "doc"
:param value: information's value (ex: "the help string")
"""
self._informations[key] = value
def impl_get_information(self, key, default=undefined):
"""retrieves one information's item
:param key: the item string (ex: "help")
"""
if key in self._informations:
return self._informations[key]
elif default is not undefined: # pragma: optional cover
return default
else: # pragma: optional cover
raise ValueError(_("information's item not found: {0}").format(
key))
# ____________________________________________________________ # ____________________________________________________________
# serialize object # serialize object
def _impl_convert_requires(self, descr, load=False): def _impl_convert_requires(self, descr, load=False):
@ -231,16 +179,15 @@ class BaseOption(Base):
:param load: `True` if we are at the init of the option description :param load: `True` if we are at the init of the option description
:type load: bool :type load: bool
""" """
if not load and self._requires is None: if not load and self.impl_getrequires() is None:
self._state_requires = None self._state_requires = None
elif load and self._state_requires is None: elif load and self._state_requires is None:
self._requires = None
del(self._state_requires) del(self._state_requires)
else: else:
if load: if load:
_requires = self._state_requires _requires = self._state_requires
else: else:
_requires = self._requires _requires = self.impl_getrequires()
new_value = [] new_value = []
for requires in _requires: for requires in _requires:
new_requires = [] new_requires = []
@ -254,7 +201,8 @@ class BaseOption(Base):
new_value.append(tuple(new_requires)) new_value.append(tuple(new_requires))
if load: if load:
del(self._state_requires) del(self._state_requires)
self._requires = new_value if new_value != []:
self._requires = new_value
else: else:
self._state_requires = new_value self._state_requires = new_value
@ -262,12 +210,10 @@ class BaseOption(Base):
if self.__class__.__name__ == 'OptionDescription' or \ if self.__class__.__name__ == 'OptionDescription' or \
isinstance(self, SymLinkOption): isinstance(self, SymLinkOption):
return return
if not load and self._callback is None: if not load and self.impl_get_callback() is None:
self._state_callback = None self._state_callback = None
self._state_callback_params = {} self._state_callback_params = {}
elif load and self._state_callback is None: elif load and self._state_callback is None:
self._callback = None
self._callback_params = {}
del(self._state_callback) del(self._state_callback)
del(self._state_callback_params) del(self._state_callback_params)
else: else:
@ -275,8 +221,7 @@ class BaseOption(Base):
callback = self._state_callback callback = self._state_callback
callback_params = self._state_callback_params callback_params = self._state_callback_params
else: else:
callback = self._callback callback, callback_params = self.impl_get_callback()
callback_params = self._callback_params
self._state_callback_params = {} self._state_callback_params = {}
if callback_params is not None: if callback_params is not None:
cllbck_prms = {} cllbck_prms = {}
@ -298,8 +243,7 @@ class BaseOption(Base):
if load: if load:
del(self._state_callback) del(self._state_callback)
del(self._state_callback_params) del(self._state_callback_params)
self._callback = callback self._set_callback(callback, cllbck_prms)
self._callback_params = cllbck_prms
else: else:
self._state_callback = callback self._state_callback = callback
self._state_callback_params = cllbck_prms self._state_callback_params = cllbck_prms
@ -311,11 +255,11 @@ class BaseOption(Base):
:param descr: the parent :class:`tiramisu.option.OptionDescription` :param descr: the parent :class:`tiramisu.option.OptionDescription`
""" """
#super(BaseOption, self)._impl_getstate()
self._stated = True self._stated = True
for func in dir(self): for func in dir(self):
if func.startswith('_impl_convert_'): if func.startswith('_impl_convert_'):
getattr(self, func)(descr) getattr(self, func)(descr)
self._state_readonly = self._readonly
def __getstate__(self, stated=True): def __getstate__(self, stated=True):
"""special method to enable the serialization with pickle """special method to enable the serialization with pickle
@ -366,8 +310,6 @@ class BaseOption(Base):
if func.startswith('_impl_convert_'): if func.startswith('_impl_convert_'):
getattr(self, func)(descr, load=True) getattr(self, func)(descr, load=True)
try: try:
self._readonly = self._state_readonly
del(self._state_readonly)
del(self._stated) del(self._stated)
except AttributeError: # pragma: optional cover except AttributeError: # pragma: optional cover
pass pass
@ -401,7 +343,7 @@ class BaseOption(Base):
# never change _name # never change _name
if name == '_name': if name == '_name':
try: try:
if self._name is not None: if self.impl_getname() is not None:
#so _name is already set #so _name is already set
is_readonly = True is_readonly = True
except (KeyError, AttributeError): except (KeyError, AttributeError):
@ -412,30 +354,16 @@ class BaseOption(Base):
raise AttributeError(_("'{0}' ({1}) object attribute '{2}' is" raise AttributeError(_("'{0}' ({1}) object attribute '{2}' is"
" read-only").format( " read-only").format(
self.__class__.__name__, self.__class__.__name__,
self._name, self.impl_getname(),
name)) name))
super(BaseOption, self).__setattr__(name, value) super(BaseOption, self).__setattr__(name, value)
def impl_is_readonly(self):
try:
if self._readonly is True:
return True
except AttributeError:
pass
return False
def impl_getname(self):
return self._name
def impl_getpath(self, context): def impl_getpath(self, context):
return context.cfgimpl_get_description().impl_get_path_by_opt(self) return context.cfgimpl_get_description().impl_get_path_by_opt(self)
def impl_get_callback(self):
return self._callback, self._callback_params
def impl_has_callback(self): def impl_has_callback(self):
"to know if a callback has been defined or not" "to know if a callback has been defined or not"
return self._callback is not None return self.impl_get_callback()[0] is not None
def _is_subdyn(self): def _is_subdyn(self):
try: try:
@ -464,40 +392,6 @@ class Option(OnlyOption):
__slots__ = tuple() __slots__ = tuple()
_empty = '' _empty = ''
def __init__(self, name, doc, default=None, default_multi=None,
requires=None, multi=False, callback=None,
callback_params=None, validator=None, validator_params=None,
properties=None, warnings_only=False):
"""
:param name: the option's name
:param doc: the option's description
:param default: specifies the default value of the option,
for a multi : ['bla', 'bla', 'bla']
:param default_multi: 'bla' (used in case of a reset to default only at
a given index)
:param requires: is a list of names of options located anywhere
in the configuration.
:param multi: if true, the option's value is a list
:param callback: the name of a function. If set, the function's output
is responsible of the option's value
:param callback_params: the callback's parameter
:param validator: the name of a function which stands for a custom
validation of the value
:param validator_params: the validator's parameters
:param properties: tuple of default properties
:param warnings_only: _validator and _consistencies don't raise if True
Values()._warning contain message
"""
super(Option, self).__init__(name, doc, default, default_multi,
requires, multi, callback,
callback_params, validator,
validator_params, properties,
warnings_only)
def impl_getrequires(self):
return self._requires
def _launch_consistency(self, func, option, value, context, index, def _launch_consistency(self, func, option, value, context, index,
submulti_index, all_cons_opts, warnings_only): submulti_index, all_cons_opts, warnings_only):
"""Launch consistency now """Launch consistency now
@ -581,24 +475,25 @@ class Option(OnlyOption):
current_opt = self current_opt = self
def val_validator(val): def val_validator(val):
if self._validator is not None: validator, validator_params = self.impl_get_validator()
if self._validator_params is not None: if validator is not None:
validator_params = {} if validator_params != {}:
for val_param, values in self._validator_params.items(): validator_params_ = {}
validator_params[val_param] = values for val_param, values in validator_params.items():
validator_params_[val_param] = values
#inject value in calculation #inject value in calculation
if '' in validator_params: if '' in validator_params_:
lst = list(validator_params['']) lst = list(validator_params_[''])
lst.insert(0, val) lst.insert(0, val)
validator_params[''] = tuple(lst) validator_params_[''] = tuple(lst)
else: else:
validator_params[''] = (val,) validator_params_[''] = (val,)
else: else:
validator_params = {'': (val,)} validator_params_ = {'': (val,)}
# Raise ValueError if not valid # Raise ValueError if not valid
carry_out_calculation(self, config=context, carry_out_calculation(self, config=context,
callback=self._validator, callback=validator,
callback_params=validator_params) callback_params=validator_params_)
def do_validation(_value, _index, submulti_index): def do_validation(_value, _index, submulti_index):
if _value is None: if _value is None:
@ -622,11 +517,11 @@ class Option(OnlyOption):
if context is not undefined: if context is not undefined:
descr._valid_consistency(current_opt, _value, context, descr._valid_consistency(current_opt, _value, context,
_index, submulti_index) _index, submulti_index)
self._second_level_validation(_value, self._warnings_only) self._second_level_validation(_value, self._is_warnings_only())
except ValueError as error: except ValueError as error:
log.debug(_('do_validation for {0}: error in value').format( log.debug(_('do_validation for {0}: error in value').format(
self.impl_getname()), exc_info=True) self.impl_getname()), exc_info=True)
if self._warnings_only: if self._is_warnings_only():
warning = error warning = error
error = None error = None
except ValueWarning as warning: except ValueWarning as warning:
@ -655,7 +550,7 @@ class Option(OnlyOption):
self.__class__.__name__, 0) self.__class__.__name__, 0)
elif error: elif error:
raise ValueError(_("invalid value for option {0}: {1}").format( raise ValueError(_("invalid value for option {0}: {1}").format(
self._name, error)) self.impl_getname(), error))
# generic calculation # generic calculation
if context is not undefined: if context is not undefined:
@ -690,16 +585,6 @@ class Option(OnlyOption):
else: else:
do_validation(val, idx, force_submulti_index) do_validation(val, idx, force_submulti_index)
def impl_getdefault(self):
"accessing the default value"
if isinstance(self._default, list):
return copy(self._default)
return self._default
def impl_getdefault_multi(self):
"accessing the default value for a multi"
return self._default_multi
def impl_is_master_slaves(self, type_='both'): def impl_is_master_slaves(self, type_='both'):
"""FIXME """FIXME
""" """
@ -720,9 +605,9 @@ class Option(OnlyOption):
def impl_is_empty_by_default(self): def impl_is_empty_by_default(self):
"no default value has been set yet" "no default value has been set yet"
if ((not self.impl_is_multi() and self._default is None) or if ((not self.impl_is_multi() and self.impl_getdefault() is None) or
(self.impl_is_multi() and (self._default == [] (self.impl_is_multi() and (self.impl_getdefault() == []
or None in self._default))): or None in self.impl_getdefault()))):
return True return True
return False return False
@ -733,12 +618,6 @@ class Option(OnlyOption):
#def impl_getkey(self, value): #def impl_getkey(self, value):
# return value # return value
def impl_is_multi(self):
return self._multi == 0 or self._multi is submulti
def impl_is_submulti(self):
return self._multi is submulti
def impl_add_consistency(self, func, *other_opts, **params): def impl_add_consistency(self, func, *other_opts, **params):
"""Add consistency means that value will be validate with other_opts """Add consistency means that value will be validate with other_opts
option's values. option's values.
@ -753,7 +632,7 @@ class Option(OnlyOption):
raise AttributeError(_("'{0}' ({1}) cannont add consistency, option is" raise AttributeError(_("'{0}' ({1}) cannont add consistency, option is"
" read-only").format( " read-only").format(
self.__class__.__name__, self.__class__.__name__,
self._name)) self.impl_getname()))
warnings_only = params.get('warnings_only', False) warnings_only = params.get('warnings_only', False)
if self._is_subdyn(): if self._is_subdyn():
dynod = self._impl_getsubdyn() dynod = self._impl_getsubdyn()
@ -821,16 +700,15 @@ class Option(OnlyOption):
:param load: `True` if we are at the init of the option description :param load: `True` if we are at the init of the option description
:type load: bool :type load: bool
""" """
if not load and self._consistencies is None: if not load and self._get_consistencies() is None:
self._state_consistencies = None self._state_consistencies = None
elif load and self._state_consistencies is None: elif load and self._state_consistencies is None:
self._consistencies = None
del(self._state_consistencies) del(self._state_consistencies)
else: else:
if load: if load:
consistencies = self._state_consistencies consistencies = self._state_consistencies
else: else:
consistencies = self._consistencies consistencies = self._get_consistencies()
new_value = [] new_value = []
for consistency in consistencies: for consistency in consistencies:
values = [] values = []
@ -842,7 +720,8 @@ class Option(OnlyOption):
new_value.append((consistency[0], tuple(values), consistency[2])) new_value.append((consistency[0], tuple(values), consistency[2]))
if load: if load:
del(self._state_consistencies) del(self._state_consistencies)
self._consistencies = new_value for new_val in new_value:
self._add_consistency(new_val[0], new_val[1], new_val[2])
else: else:
self._state_consistencies = new_value self._state_consistencies = new_value
@ -854,14 +733,14 @@ class Option(OnlyOption):
def _validate_callback(self, callback, callback_params): def _validate_callback(self, callback, callback_params):
try: try:
default_multi = self._default_multi default_multi = self.impl_getdefault_multi()
except AttributeError: except AttributeError:
default_multi = None default_multi = None
if callback is not None and ((self._multi == 1 and if callback is not None and ((not self.impl_is_multi() and
(self._default is not None or (self.impl_getdefault() is not None or
default_multi is not None)) default_multi is not None))
or (self._multi != 1 and or (self.impl_is_multi() and
(self._default != [] or (self.impl_getdefault() != [] or
default_multi is not None)) default_multi is not None))
): # pragma: optional cover ): # pragma: optional cover
raise ValueError(_("default value not allowed if option: {0} " raise ValueError(_("default value not allowed if option: {0} "
@ -885,7 +764,7 @@ def validate_requires_arg(requires, name):
# start parsing all requires given by user (has dict) # start parsing all requires given by user (has dict)
# transforme it to a tuple # transforme it to a tuple
for require in requires: for require in requires:
if not type(require) == dict: # pragma: optional cover if not isinstance(require, dict): # pragma: optional cover
raise ValueError(_("malformed requirements type for option:" raise ValueError(_("malformed requirements type for option:"
" {0}, must be a dict").format(name)) " {0}, must be a dict").format(name))
valid_keys = ('option', 'expected', 'action', 'inverse', 'transitive', valid_keys = ('option', 'expected', 'action', 'inverse', 'transitive',
@ -963,46 +842,57 @@ def validate_requires_arg(requires, name):
class SymLinkOption(OnlyOption): class SymLinkOption(OnlyOption):
__slots__ = ('_opt', '_state_opt') __slots__ = ('_opt', '_state_opt', '_readonly')
def __init__(self, name, opt): def __init__(self, name, opt):
self._name = name
if not isinstance(opt, Option): # pragma: optional cover if not isinstance(opt, Option): # pragma: optional cover
raise ValueError(_('malformed symlinkoption ' raise ValueError(_('malformed symlinkoption '
'must be an option ' 'must be an option '
'for symlink {0}').format(name)) 'for symlink {0}').format(name))
self._opt = opt self._opt = opt
self._readonly = True self._set_readonly()
super(Base, self).__init__() super(Base, self).__init__(name, undefined, undefined, undefined, undefined)
def __getattr__(self, name, context=undefined): def __getattr__(self, name, context=undefined):
if name in ('_opt', '_opt_type', '_readonly', 'impl_getpath'): if name in ('_opt', '_opt_type', '_readonly', 'impl_getpath', '_name', '_state_opt'):
return object.__getattr__(self, name) return object.__getattr__(self, name)
else: else:
return getattr(self._opt, name) return getattr(self._opt, name)
def _impl_getstate(self, descr): def _impl_getstate(self, descr):
super(SymLinkOption, self)._impl_getstate(descr) self._stated = True
self._state_opt = descr.impl_get_path_by_opt(self._opt) self._state_opt = descr.impl_get_path_by_opt(self._opt)
def _impl_setstate(self, descr): def _impl_setstate(self, descr):
self._opt = descr.impl_get_opt_by_path(self._state_opt) self._opt = descr.impl_get_opt_by_path(self._state_opt)
del(self._state_opt) del(self._state_opt)
super(SymLinkOption, self)._impl_setstate(descr) try:
del(self._stated)
except AttributeError: # pragma: optional cover
pass
self._set_readonly()
def impl_get_information(self, key, default=undefined): def impl_get_information(self, key, default=undefined):
return self._opt.impl_get_information(key, default) return self._opt.impl_get_information(key, default)
#FIXME utile tout ca ? c'est un peu de la duplication ... def _set_readonly(self):
self._readonly = True
def impl_is_readonly(self):
try:
return self._readonly
except AttributeError:
return False
def impl_getproperties(self): def impl_getproperties(self):
return self._opt._properties return self._opt._properties
def impl_get_callback(self): def impl_get_callback(self):
return self._opt._callback, self._opt._callback_params return self._opt.impl_get_callback()
def impl_has_callback(self): def impl_has_callback(self):
"to know if a callback has been defined or not" "to know if a callback has been defined or not"
return self._opt._callback is not None return self._opt.impl_has_callback()
def _is_subdyn(self): def _is_subdyn(self):
try: try:

53
tiramisu/option/option.py
View File

@ -155,8 +155,8 @@ class IPOption(Option):
callback_params=None, validator=None, validator_params=None, callback_params=None, validator=None, validator_params=None,
properties=None, private_only=False, allow_reserved=False, properties=None, private_only=False, allow_reserved=False,
warnings_only=False): warnings_only=False):
self._extra = {'_private_only': private_only, extra = {'_private_only': private_only,
'_allow_reserved': allow_reserved} '_allow_reserved': allow_reserved}
super(IPOption, self).__init__(name, doc, default=default, super(IPOption, self).__init__(name, doc, default=default,
default_multi=default_multi, default_multi=default_multi,
callback=callback, callback=callback,
@ -166,7 +166,8 @@ class IPOption(Option):
validator=validator, validator=validator,
validator_params=validator_params, validator_params=validator_params,
properties=properties, properties=properties,
warnings_only=warnings_only) warnings_only=warnings_only,
extra=extra)
def _validate(self, value, context=undefined): def _validate(self, value, context=undefined):
# sometimes an ip term starts with a zero # sometimes an ip term starts with a zero
@ -186,13 +187,13 @@ class IPOption(Option):
def _second_level_validation(self, value, warnings_only): def _second_level_validation(self, value, warnings_only):
ip = IP('{0}/32'.format(value)) ip = IP('{0}/32'.format(value))
if not self._extra['_allow_reserved'] and ip.iptype() == 'RESERVED': # pragma: optional cover if not self._get_extra('_allow_reserved') and ip.iptype() == 'RESERVED': # pragma: optional cover
if warnings_only: if warnings_only:
msg = _("IP is in reserved class") msg = _("IP is in reserved class")
else: else:
msg = _("invalid IP, mustn't be in reserved class") msg = _("invalid IP, mustn't be in reserved class")
raise ValueError(msg) raise ValueError(msg)
if self._extra['_private_only'] and not ip.iptype() == 'PRIVATE': # pragma: optional cover if self._get_extra('_private_only') and not ip.iptype() == 'PRIVATE': # pragma: optional cover
if warnings_only: if warnings_only:
msg = _("IP is not in private class") msg = _("IP is not in private class")
else: else:
@ -257,7 +258,6 @@ class PortOption(Option):
if extra['_max_value'] is None: if extra['_max_value'] is None:
raise ValueError(_('max value is empty')) # pragma: optional cover raise ValueError(_('max value is empty')) # pragma: optional cover
self._extra = extra
super(PortOption, self).__init__(name, doc, default=default, super(PortOption, self).__init__(name, doc, default=default,
default_multi=default_multi, default_multi=default_multi,
callback=callback, callback=callback,
@ -267,10 +267,11 @@ class PortOption(Option):
validator=validator, validator=validator,
validator_params=validator_params, validator_params=validator_params,
properties=properties, properties=properties,
warnings_only=warnings_only) warnings_only=warnings_only,
extra=extra)
def _validate(self, value, context=undefined): def _validate(self, value, context=undefined):
if self._extra['_allow_range'] and ":" in str(value): # pragma: optional cover if self._get_extra('_allow_range') and ":" in str(value): # pragma: optional cover
value = str(value).split(':') value = str(value).split(':')
if len(value) != 2: if len(value) != 2:
raise ValueError(_('invalid port, range must have two values ' raise ValueError(_('invalid port, range must have two values '
@ -286,10 +287,10 @@ class PortOption(Option):
val = int(val) val = int(val)
except ValueError: # pragma: optional cover except ValueError: # pragma: optional cover
raise ValueError(_('invalid port')) raise ValueError(_('invalid port'))
if not self._extra['_min_value'] <= val <= self._extra['_max_value']: # pragma: optional cover if not self._get_extra('_min_value') <= val <= self._get_extra('_max_value'): # pragma: optional cover
raise ValueError(_('invalid port, must be an between {0} ' raise ValueError(_('invalid port, must be an between {0} '
'and {1}').format(self._extra['_min_value'], 'and {1}').format(self._get_extra('_min_value'),
self._extra['_max_value'])) self._get_extra('_max_value')))
class NetworkOption(Option): class NetworkOption(Option):
@ -400,34 +401,34 @@ class DomainnameOption(Option):
warnings_only=False, allow_without_dot=False): warnings_only=False, allow_without_dot=False):
if type_ not in ['netbios', 'hostname', 'domainname']: if type_ not in ['netbios', 'hostname', 'domainname']:
raise ValueError(_('unknown type_ {0} for hostname').format(type_)) # pragma: optional cover raise ValueError(_('unknown type_ {0} for hostname').format(type_)) # pragma: optional cover
self._extra = {'_dom_type': type_} extra = {'_dom_type': type_}
if allow_ip not in [True, False]: if allow_ip not in [True, False]:
raise ValueError(_('allow_ip must be a boolean')) # pragma: optional cover raise ValueError(_('allow_ip must be a boolean')) # pragma: optional cover
if allow_without_dot not in [True, False]: if allow_without_dot not in [True, False]:
raise ValueError(_('allow_without_dot must be a boolean')) # pragma: optional cover raise ValueError(_('allow_without_dot must be a boolean')) # pragma: optional cover
self._extra['_allow_ip'] = allow_ip extra['_allow_ip'] = allow_ip
self._extra['_allow_without_dot'] = allow_without_dot extra['_allow_without_dot'] = allow_without_dot
end = '' end = ''
extrachar = '' extrachar = ''
extrachar_mandatory = '' extrachar_mandatory = ''
if self._extra['_dom_type'] != 'netbios': if extra['_dom_type'] != 'netbios':
allow_number = '\d' allow_number = '\d'
else: else:
allow_number = '' # pragma: optional cover allow_number = '' # pragma: optional cover
if self._extra['_dom_type'] == 'netbios': if extra['_dom_type'] == 'netbios':
length = 14 # pragma: optional cover length = 14 # pragma: optional cover
elif self._extra['_dom_type'] == 'hostname': elif extra['_dom_type'] == 'hostname':
length = 62 # pragma: optional cover length = 62 # pragma: optional cover
elif self._extra['_dom_type'] == 'domainname': elif extra['_dom_type'] == 'domainname':
length = 62 length = 62
if allow_without_dot is False: if allow_without_dot is False:
extrachar_mandatory = '\.' extrachar_mandatory = '\.'
else: else:
extrachar = '\.' # pragma: optional cover extrachar = '\.' # pragma: optional cover
end = '+[a-z]*' end = '+[a-z]*'
self._extra['_domain_re'] = re.compile(r'^(?:[a-z{0}][a-z\d\-{1}]{{,{2}}}{3}){4}$' extra['_domain_re'] = re.compile(r'^(?:[a-z{0}][a-z\d\-{1}]{{,{2}}}{3}){4}$'
''.format(allow_number, extrachar, length, ''.format(allow_number, extrachar, length,
extrachar_mandatory, end)) extrachar_mandatory, end))
super(DomainnameOption, self).__init__(name, doc, default=default, super(DomainnameOption, self).__init__(name, doc, default=default,
default_multi=default_multi, default_multi=default_multi,
callback=callback, callback=callback,
@ -437,23 +438,25 @@ class DomainnameOption(Option):
validator=validator, validator=validator,
validator_params=validator_params, validator_params=validator_params,
properties=properties, properties=properties,
warnings_only=warnings_only) warnings_only=warnings_only,
extra=extra)
def _validate(self, value, context=undefined): def _validate(self, value, context=undefined):
if self._extra['_allow_ip'] is True: # pragma: optional cover if self._get_extra('_allow_ip') is True: # pragma: optional cover
try: try:
IP('{0}/32'.format(value)) IP('{0}/32'.format(value))
return return
except ValueError: except ValueError:
pass pass
if self._extra['_dom_type'] == 'domainname' and not self._extra['_allow_without_dot'] and \ if self._get_extra('_dom_type') == 'domainname' and \
not self._get_extra('_allow_without_dot') and \
'.' not in value: # pragma: optional cover '.' not in value: # pragma: optional cover
raise ValueError(_("invalid domainname, must have dot")) raise ValueError(_("invalid domainname, must have dot"))
if len(value) > 255: if len(value) > 255:
raise ValueError(_("invalid domainname's length (max 255)")) # pragma: optional cover raise ValueError(_("invalid domainname's length (max 255)")) # pragma: optional cover
if len(value) < 2: if len(value) < 2:
raise ValueError(_("invalid domainname's length (min 2)")) # pragma: optional cover raise ValueError(_("invalid domainname's length (min 2)")) # pragma: optional cover
if not self._extra['_domain_re'].search(value): if not self._get_extra('_domain_re').search(value):
raise ValueError(_('invalid domainname')) # pragma: optional cover raise ValueError(_('invalid domainname')) # pragma: optional cover

11
tiramisu/option/optiondescription.py
View File

@ -79,9 +79,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
self._group_type = groups.default self._group_type = groups.default
self._is_build_cache = False self._is_build_cache = False
def impl_getrequires(self):
return self._requires
def impl_getdoc(self): def impl_getdoc(self):
return self.impl_get_information('doc') return self.impl_get_information('doc')
@ -141,11 +138,11 @@ class OptionDescription(BaseOption, StorageOptionDescription):
if option._get_id() in cache_option: # pragma: optional cover if option._get_id() in cache_option: # pragma: optional cover
raise ConflictError(_('duplicate option: {0}').format(option)) raise ConflictError(_('duplicate option: {0}').format(option))
cache_option.append(option._get_id()) cache_option.append(option._get_id())
option._readonly = True option._set_readonly()
if isinstance(option, OptionDescription): if isinstance(option, OptionDescription):
option.impl_validate_options(cache_option) option.impl_validate_options(cache_option)
if init: if init:
self._readonly = True self._set_readonly()
# ____________________________________________________________ # ____________________________________________________________
def impl_set_group_type(self, group_type): def impl_set_group_type(self, group_type):
@ -281,7 +278,7 @@ class OptionDescription(BaseOption, StorageOptionDescription):
def _impl_get_dynchild(self, child, suffix): def _impl_get_dynchild(self, child, suffix):
name = child.impl_getname() + suffix name = child.impl_getname() + suffix
path = self._name + suffix + '.' + name path = self.impl_getname() + suffix + '.' + name
if isinstance(child, OptionDescription): if isinstance(child, OptionDescription):
return SynDynOptionDescription(child, name, path, suffix) return SynDynOptionDescription(child, name, path, suffix)
else: else:
@ -289,7 +286,7 @@ class OptionDescription(BaseOption, StorageOptionDescription):
def _impl_getchildren(self, dyn=True, context=undefined): def _impl_getchildren(self, dyn=True, context=undefined):
for child in self._impl_st_getchildren(context): for child in self._impl_st_getchildren(context):
cname = child._name cname = child.impl_getname()
if dyn and child.impl_is_dynoptiondescription(): if dyn and child.impl_is_dynoptiondescription():
path = cname path = cname
for value in child._impl_get_suffixes(context): for value in child._impl_get_suffixes(context):

6
tiramisu/setting.py
View File

@ -251,7 +251,7 @@ class Property(object):
:type propname: string :type propname: string
""" """
if self._opt is not None and self._opt.impl_getrequires() is not None \ if self._opt is not None and self._opt.impl_getrequires() is not None \
and propname in self._opt._calc_properties: # pragma: optional cover and propname in self._opt.impl_get_calc_properties(): # pragma: optional cover
raise ValueError(_('cannot append {0} property for option {1}: ' raise ValueError(_('cannot append {0} property for option {1}: '
'this property is calculated').format( 'this property is calculated').format(
propname, self._opt.impl_getname())) propname, self._opt.impl_getname()))
@ -573,13 +573,13 @@ class Settings(object):
:param path: the option's path in the config :param path: the option's path in the config
:type path: str :type path: str
""" """
if opt._requires is None: if opt.impl_getrequires() == []:
return frozenset() return frozenset()
# filters the callbacks # filters the callbacks
calc_properties = set() calc_properties = set()
context = self._getcontext() context = self._getcontext()
for requires in opt._requires: for requires in opt.impl_getrequires():
for require in requires: for require in requires:
option, expected, action, inverse, \ option, expected, action, inverse, \
transitive, same_action = require transitive, same_action = require

314
tiramisu/storage/dictionary/option.py
View File

@ -25,49 +25,188 @@ from tiramisu.error import ConfigError
#____________________________________________________________ #____________________________________________________________
# #
# Base # Base
#('_name', '_informations', '_multi', '_multitype', '_warnings_only', '_extra', '_readonly', '_subdyn)
class StorageBase(object): class StorageBase(object):
__slots__ = ('_name', '_requires', '_properties', '_readonly', __slots__ = ('_name',
'_calc_properties', '_informations', '_informations',
'_state_readonly', '_state_requires', '_stated', '_multi',
'_multi', '_validator', '_validator_params', '_extra',
'_default', '_default_multi', '_state_callback', '_callback', '_warnings_only',
'_state_callback_params', '_callback_params', '_multitype', #valeur
'_consistencies', '_warnings_only', '_master_slaves', '_default',
'_state_consistencies', '_extra', '_subdyn', '__weakref__', '_default_multi',
'_state_master_slaves', '_choice_values', #calcul
'_choice_values_params') '_subdyn',
'_requires',
'_properties',
'_calc_properties',
'_val_call',
#'_validator',
#'_validator_params',
#'_callback',
#'_callback_params',
'_consistencies',
'_master_slaves',
'_choice_values',
'_choice_values_params',
#autre
'_state_master_slaves',
'_state_callback',
'_state_callback_params',
'_state_requires',
'_stated',
'_state_consistencies',
'_state_informations',
'_state_readonly',
'__weakref__'
)
def __init__(self): def __init__(self, name, multi, warnings_only, doc, extra):
try: self._name = name
self._subdyn if doc is not undefined:
except AttributeError: self._informations = {'doc': doc}
self._subdyn = None if multi != 1:
try: self._multi = multi
self._consistencies if extra is not None:
except AttributeError: self._extra = extra
self._consistencies = [] if warnings_only is True:
try: self._warnings_only = warnings_only
self._callback
except AttributeError: def _set_default_values(self, default, default_multi):
self._callback = None if self.impl_is_multi() and default is None:
try: default = []
self._callback_params self.impl_validate(default)
except AttributeError: if ((self.impl_is_multi() and default != tuple()) or
self._callback_params = {} (not self.impl_is_multi() and default is not None)):
try: if self.impl_is_multi():
self._validator default = tuple(default)
except AttributeError: self._default = default
self._validator = None
try: if self.impl_is_multi() and default_multi is not None:
self._validator_params try:
except AttributeError: self._validate(default_multi)
self._validator_params = None except ValueError as err: # pragma: optional cover
raise ValueError(_("invalid default_multi value {0} "
"for option {1}: {2}").format(
str(default_multi),
self.impl_getname(), err))
self._default_multi = default_multi
# information
def impl_set_information(self, key, value):
"""updates the information's attribute
(which is a dictionary)
:param key: information's key (ex: "help", "doc"
:param value: information's value (ex: "the help string")
"""
self._informations[key] = value
def impl_get_information(self, key, default=undefined):
"""retrieves one information's item
:param key: the item string (ex: "help")
"""
error = False
dico = self._informations
if dico is None or isinstance(dico, str) or isinstance(dico, unicode):
if key == 'doc':
return dico
error = True
elif isinstance(dico, tuple):
try:
return dico[1][dico[0].index(key)]
except AttributeError:
if default is not undefined:
return default
error = True
else:
if default is not undefined:
return self._informations.get(key, default)
try:
return self._informations[key]
except KeyError: # pragma: optional cover
error = True
if error:
raise ValueError(_("information's item not found: {0}").format(
key))
def _add_consistency(self, func, all_cons_opts, params): def _add_consistency(self, func, all_cons_opts, params):
self._consistencies.append((func, all_cons_opts, params)) cons = (func, all_cons_opts, params)
try:
self._consistencies.append(cons)
except AttributeError:
self._consistencies = [cons]
def _get_consistencies(self): def _get_consistencies(self):
return self._consistencies try:
return self._consistencies
except AttributeError:
return tuple()
def _set_callback(self, callback, callback_params):
if callback_params is None or callback_params == {}:
val_call = (callback,)
else:
val_call = tuple([callback, callback_params])
try:
self._val_call = (self._val_call[0], val_call)
except AttributeError:
self._val_call = (None, val_call)
def impl_get_callback(self):
default = (None, {})
try:
call = self._val_call[1]
except (AttributeError, IndexError):
ret_call = default
else:
if call is None:
ret_call = default
else:
if len(call) == 1:
ret_call = (call[0], default[1])
else:
ret_call = call
return ret_call
def impl_get_calc_properties(self):
try:
return self._calc_properties
except AttributeError:
return frozenset()
def impl_getrequires(self):
try:
return self._requires
except AttributeError:
return []
def _set_validator(self, validator, validator_params):
if validator_params is None:
val_call = (validator,)
else:
val_call = (validator, validator_params)
try:
self._val_call = (val_call, self._val_call[1])
except (AttributeError, IndexError):
self._val_call = (val_call, None)
def impl_get_validator(self):
default = (None, {})
try:
val = self._val_call[0]
except (AttributeError, IndexError):
ret_val = default
else:
if val is None:
ret_val = default
else:
if len(val) == 1:
ret_val = (val[0], default[1])
else:
ret_val = val
return ret_val
def _get_id(self): def _get_id(self):
return id(self) return id(self)
@ -75,9 +214,103 @@ class StorageBase(object):
def _impl_getsubdyn(self): def _impl_getsubdyn(self):
return self._subdyn return self._subdyn
def _set_readonly(self):
if not self.impl_is_readonly():
dico = self._informations
if not (dico is None or isinstance(dico, str) or isinstance(dico, unicode)):
keys = tuple(dico.keys())
if keys == ('doc',):
dico = dico['doc']
else:
dico = tuple([tuple(dico.keys()), tuple(dico.values())])
self._informations = dico
try:
extra = self._extra
self._extra = tuple([tuple(extra.keys()), tuple(extra.values())])
except AttributeError:
pass
def _impl_setsubdyn(self, subdyn): def _impl_setsubdyn(self, subdyn):
self._subdyn = subdyn self._subdyn = subdyn
def _impl_convert_informations(self, descr, load=False):
if not load:
infos = self._informations
if isinstance(infos, tuple):
self._state_informations = {}
for key, value in infos:
self._state_informations[key] = value
elif isinstance(infos, str) or isinstance(infos, unicode):
self._state_informations = {'doc': infos}
else:
self._state_informations = infos
self._state_readonly = self.impl_is_readonly()
else:
try:
self._informations = self._state_informations
del(self._state_informations)
except AttributeError:
pass
if self._state_readonly:
self._set_readonly()
del(self._state_readonly)
def impl_is_readonly(self):
try:
return not isinstance(self._informations, dict)
except AttributeError:
return False
def impl_getname(self):
return self._name
def impl_is_multi(self):
try:
_multi = self._multi
except AttributeError:
_multi = 1
return _multi == 0 or _multi == 2
def impl_is_submulti(self):
try:
_multi = self._multi
except IndexError:
_multi = 1
return _multi == 2
def _get_extra(self, key):
extra = self._extra
if isinstance(extra, tuple):
return extra[1][extra[0].index(key)]
else:
return extra[key]
def _is_warnings_only(self):
try:
return self._warnings_only
except:
return False
def impl_getdefault(self):
"accessing the default value"
try:
ret = self._default
if self.impl_is_multi():
return list(ret)
return ret
except AttributeError:
if self.impl_is_multi():
return []
return None
def impl_getdefault_multi(self):
"accessing the default value for a multi"
if self.impl_is_multi():
try:
return self._default_multi
except (AttributeError, IndexError):
pass
def commit(self): def commit(self):
pass pass
@ -86,7 +319,8 @@ class StorageOptionDescription(StorageBase):
__slots__ = ('_children', '_cache_paths', '_cache_consistencies', __slots__ = ('_children', '_cache_paths', '_cache_consistencies',
'_group_type', '_is_build_cache', '_state_group_type') '_group_type', '_is_build_cache', '_state_group_type')
def __init__(self): def __init__(self, name, multi, warnings_only, doc, extra):
super(StorageOptionDescription, self).__init__(name, multi, warnings_only, doc, None)
self._cache_paths = None self._cache_paths = None
def _add_children(self, child_names, children): def _add_children(self, child_names, children):
@ -128,7 +362,7 @@ class StorageOptionDescription(StorageBase):
cache_path = [] cache_path = []
cache_option = [] cache_option = []
for option in self._impl_getchildren(dyn=False): for option in self._impl_getchildren(dyn=False):
attr = option._name attr = option.impl_getname()
path = str('.'.join(_currpath + [attr])) path = str('.'.join(_currpath + [attr]))
cache_option.append(option) cache_option.append(option)
cache_path.append(path) cache_path.append(path)
@ -272,7 +506,7 @@ class StorageOptionDescription(StorageBase):
if error: if error:
raise AttributeError(_('unknown Option {0} ' raise AttributeError(_('unknown Option {0} '
'in OptionDescription {1}' 'in OptionDescription {1}'
'').format(name, self._name)) '').format(name, self.impl_getname()))
def _get_force_store_value(self): def _get_force_store_value(self):
#FIXME faire des tests (notamment pas ajouter à un config) #FIXME faire des tests (notamment pas ajouter à un config)

1
tiramisu/storage/sqlalchemy/option.py
View File

@ -254,7 +254,6 @@ class _Base(SqlAlchemyBase):
_reqs = relationship("_Require", collection_class=list) _reqs = relationship("_Require", collection_class=list)
_requires = association_proxy("_reqs", "requires", getset_factory=load_requires) _requires = association_proxy("_reqs", "requires", getset_factory=load_requires)
_multi = Column(Integer) _multi = Column(Integer)
_multitype = Column(String)
###### ######
_callback = Column(PickleType) _callback = Column(PickleType)
_call_params = relationship('_CallbackParam', _call_params = relationship('_CallbackParam',