From 9f3d676280abba62fd3021c3ef642a75c1f184fc Mon Sep 17 00:00:00 2001 From: Emmanuel Garette Date: Sat, 25 Oct 2014 22:11:31 +0200 Subject: [PATCH] reduce memory usage --- ChangeLog | 2 +- test/test_config.py | 2 +- test/test_state.py | 40 ++-- tiramisu/option/baseoption.py | 292 ++++++++---------------- tiramisu/option/option.py | 53 +++-- tiramisu/option/optiondescription.py | 11 +- tiramisu/setting.py | 6 +- tiramisu/storage/dictionary/option.py | 314 ++++++++++++++++++++++---- tiramisu/storage/sqlalchemy/option.py | 1 - 9 files changed, 427 insertions(+), 294 deletions(-) diff --git a/ChangeLog b/ChangeLog index cf73539..02bb016 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,7 +13,7 @@ Sun Apr 27 10:32:40 2014 +0200 Emmanuel Garette * behavior change in ChoiceOption: remove open_values, that no sens (no type validation is possible) if - you wan't something like open_values, please use a typed option and + you want something like open_values, please use a typed option and add impl_(s|g)et_information to add proposed values and use it in your code * add dynamic ChoiceOption: diff --git a/test/test_config.py b/test/test_config.py index 0cefc13..ba16545 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -343,4 +343,4 @@ def test_config_od_function(): print cfg.impl_get_opt_by_path() except AttributeError, err: assert str(err) == _('unknown Option {0} in OptionDescription {1}' - '').format('impl_get_opt_by_path', descr._name) + '').format('impl_get_opt_by_path', descr.impl_getname()) diff --git a/test/test_state.py b/test/test_state.py index e66eb34..1a5d5bf 100644 --- a/test/test_state.py +++ b/test/test_state.py @@ -12,7 +12,7 @@ from py.test import raises def return_value(value=None): - return value + return value def _get_slots(opt): @@ -52,14 +52,20 @@ def _diff_opt(opt1, opt2): val2 = None try: val1 = getattr(opt1, attr) - except: + msg1 = "exists" + except Exception, err: err1 = True + msg1 = "not exists" try: val2 = getattr(opt2, attr) + msg2 = "exists" except: err2 = True - assert err1 == err2 + msg2 = "not exists" + + if not err1 == err2: + raise ValueError("{0} {1} before but {2} after for {3}".format(attr, msg1, msg2, opt1.impl_getname())) if val1 is None: assert val1 == val2 elif attr == '_children': @@ -81,19 +87,23 @@ def _diff_opt(opt1, opt2): assert consistency[0] == val2[index][0] for idx, opt in enumerate(consistency[1]): assert opt._name == val2[index][1][idx]._name - elif attr == '_callback': - assert val1 == val2 - elif attr == '_callback_params': - if val1 is not None: - for key, values in val1.items(): - for idx, value in enumerate(values): - if isinstance(value, tuple): - assert val1[key][idx][0]._name == val2[key][idx][0]._name - assert val1[key][idx][1] == val2[key][idx][1] + elif attr == '_val_call': + for idx, v in enumerate(val1): + if v is None: + assert val2[idx] is None + else: + assert v[0] == val2[idx][0] + if len(v) == 2: + if v[1] is not None: + for key, values in v[1].items(): + for i, value in enumerate(values): + if isinstance(value, tuple): + assert v[1][key][i][0].impl_getname() == val2[idx][1][key][i][0].impl_getname() + assert v[1][key][i][1] == val2[idx][1][key][i][1] + else: + assert v[1][key][i] == val2[idx][1][key][i] else: - assert val1[key][idx] == val2[key][idx] - else: - assert val1 == val2 + assert v[1] == val2[idx][1] elif attr == '_master_slaves': assert val1.master.impl_getname() == val2.master.impl_getname() sval1 = [opt.impl_getname() for opt in val1.slaves] diff --git a/tiramisu/option/baseoption.py b/tiramisu/option/baseoption.py index 8eec7fa..3c3f3af 100644 --- a/tiramisu/option/baseoption.py +++ b/tiramisu/option/baseoption.py @@ -19,7 +19,6 @@ # the whole pypy projet is under MIT licence # ____________________________________________________________ import re -from copy import copy from types import FunctionType import warnings @@ -31,12 +30,8 @@ from tiramisu.storage import get_storages_option StorageBase = get_storages_option('base') - - submulti = 2 - - -allowed_character = '[a-z\d\-_]' +allowed_character = '[a-zA-Z\d\-_]' name_regexp = re.compile(r'^[a-z]{0}*$'.format(allowed_character)) forbidden_names = ('iter_all', 'iter_group', 'find', 'find_first', 'make_dict', 'unwrap_from_path', 'read_only', @@ -101,86 +96,62 @@ def validate_callback(callback, callback_params, type_): class Base(StorageBase): __slots__ = tuple() - def impl_set_callback(self, callback, callback_params=None): - if callback is None and callback_params is not None: # pragma: optional cover - raise ValueError(_("params defined for a callback function but " - "no callback defined" - " yet for option {0}").format( - self.impl_getname())) - self._validate_callback(callback, callback_params) - if callback is not None: - validate_callback(callback, callback_params, 'callback') - self._callback = callback - if callback_params is None: - self._callback_params = {} - else: - self._callback_params = callback_params - def __init__(self, name, doc, default=None, default_multi=None, requires=None, multi=False, callback=None, callback_params=None, validator=None, validator_params=None, - properties=None, warnings_only=False): + properties=None, warnings_only=False, extra=None): if not valid_name(name): # pragma: optional cover raise ValueError(_("invalid name: {0} for option").format(name)) - self._name = name - self._readonly = False - self._informations = {} - self.impl_set_information('doc', doc) if requires is not None: self._calc_properties, self._requires = validate_requires_arg( - requires, self._name) - else: - self._calc_properties = frozenset() - self._requires = [] + requires, name) + #else: + # self._calc_properties = frozenset() + # self._requires = [] if not multi and default_multi is not None: # pragma: optional cover raise ValueError(_("a default_multi is set whereas multi is False" " in option: {0}").format(name)) - if default_multi is not None: - try: - self._validate(default_multi) - except ValueError as err: # pragma: optional cover - raise ValueError(_("invalid default_multi value {0} " - "for option {1}: {2}").format( - str(default_multi), name, err)) if multi is True: - self._multi = 0 + _multi = 0 elif multi is False: - self._multi = 1 + _multi = 1 elif multi is submulti: - self._multi = submulti - if self._multi != 1: - if default is None: - default = [] - self._default_multi = default_multi + _multi = submulti if properties is None: properties = tuple() if not isinstance(properties, tuple): # pragma: optional cover raise TypeError(_('invalid properties type {0} for {1},' ' must be a tuple').format( type(properties), - self._name)) + name)) if validator is not None: validate_callback(validator, validator_params, 'validator') - self._validator = validator - if validator_params is not None: - self._validator_params = validator_params - if self._calc_properties != frozenset([]) and properties is not tuple(): # pragma: optional cover - set_forbidden_properties = self._calc_properties & set(properties) + self._set_validator(validator, validator_params) + if self.impl_get_calc_properties() != frozenset([]) and properties is not tuple(): # pragma: optional cover + set_forbidden_properties = self.impl_get_calc_properties() & set(properties) if set_forbidden_properties != frozenset(): raise ValueError('conflict: properties already set in ' 'requirement {0}'.format( list(set_forbidden_properties))) - if multi and default is None: - self._default = [] - else: - self._default = default + super(Base, self).__init__(name, _multi, warnings_only, doc, extra) + self._set_default_values(default, default_multi) if callback is not False: self.impl_set_callback(callback, callback_params) self._properties = properties - self._warnings_only = warnings_only - ret = super(Base, self).__init__() - self.impl_validate(self._default) - return ret + + def impl_set_callback(self, callback, callback_params=None): + if callback is None and callback_params is not None: # pragma: optional cover + raise ValueError(_("params defined for a callback function but " + "no callback defined" + " yet for option {0}").format( + self.impl_getname())) + if self.impl_get_callback()[0] is not None: + raise ConfigError(_("a callback is already set for option {0}, " + "cannot set another one's".format(self.impl_getname()))) + self._validate_callback(callback, callback_params) + if callback is not None: + validate_callback(callback, callback_params, 'callback') + self._set_callback(callback, callback_params) def impl_is_optiondescription(self): return self.__class__.__name__ in ['OptionDescription', @@ -199,29 +170,6 @@ class BaseOption(Base): """ __slots__ = tuple() - # information - def impl_set_information(self, key, value): - """updates the information's attribute - (which is a dictionary) - - :param key: information's key (ex: "help", "doc" - :param value: information's value (ex: "the help string") - """ - self._informations[key] = value - - def impl_get_information(self, key, default=undefined): - """retrieves one information's item - - :param key: the item string (ex: "help") - """ - if key in self._informations: - return self._informations[key] - elif default is not undefined: # pragma: optional cover - return default - else: # pragma: optional cover - raise ValueError(_("information's item not found: {0}").format( - key)) - # ____________________________________________________________ # serialize object def _impl_convert_requires(self, descr, load=False): @@ -231,16 +179,15 @@ class BaseOption(Base): :param load: `True` if we are at the init of the option description :type load: bool """ - if not load and self._requires is None: + if not load and self.impl_getrequires() is None: self._state_requires = None elif load and self._state_requires is None: - self._requires = None del(self._state_requires) else: if load: _requires = self._state_requires else: - _requires = self._requires + _requires = self.impl_getrequires() new_value = [] for requires in _requires: new_requires = [] @@ -254,7 +201,8 @@ class BaseOption(Base): new_value.append(tuple(new_requires)) if load: del(self._state_requires) - self._requires = new_value + if new_value != []: + self._requires = new_value else: self._state_requires = new_value @@ -262,12 +210,10 @@ class BaseOption(Base): if self.__class__.__name__ == 'OptionDescription' or \ isinstance(self, SymLinkOption): return - if not load and self._callback is None: + if not load and self.impl_get_callback() is None: self._state_callback = None self._state_callback_params = {} elif load and self._state_callback is None: - self._callback = None - self._callback_params = {} del(self._state_callback) del(self._state_callback_params) else: @@ -275,8 +221,7 @@ class BaseOption(Base): callback = self._state_callback callback_params = self._state_callback_params else: - callback = self._callback - callback_params = self._callback_params + callback, callback_params = self.impl_get_callback() self._state_callback_params = {} if callback_params is not None: cllbck_prms = {} @@ -298,8 +243,7 @@ class BaseOption(Base): if load: del(self._state_callback) del(self._state_callback_params) - self._callback = callback - self._callback_params = cllbck_prms + self._set_callback(callback, cllbck_prms) else: self._state_callback = callback self._state_callback_params = cllbck_prms @@ -311,11 +255,11 @@ class BaseOption(Base): :param descr: the parent :class:`tiramisu.option.OptionDescription` """ + #super(BaseOption, self)._impl_getstate() self._stated = True for func in dir(self): if func.startswith('_impl_convert_'): getattr(self, func)(descr) - self._state_readonly = self._readonly def __getstate__(self, stated=True): """special method to enable the serialization with pickle @@ -366,8 +310,6 @@ class BaseOption(Base): if func.startswith('_impl_convert_'): getattr(self, func)(descr, load=True) try: - self._readonly = self._state_readonly - del(self._state_readonly) del(self._stated) except AttributeError: # pragma: optional cover pass @@ -401,7 +343,7 @@ class BaseOption(Base): # never change _name if name == '_name': try: - if self._name is not None: + if self.impl_getname() is not None: #so _name is already set is_readonly = True except (KeyError, AttributeError): @@ -412,30 +354,16 @@ class BaseOption(Base): raise AttributeError(_("'{0}' ({1}) object attribute '{2}' is" " read-only").format( self.__class__.__name__, - self._name, + self.impl_getname(), name)) super(BaseOption, self).__setattr__(name, value) - def impl_is_readonly(self): - try: - if self._readonly is True: - return True - except AttributeError: - pass - return False - - def impl_getname(self): - return self._name - def impl_getpath(self, context): return context.cfgimpl_get_description().impl_get_path_by_opt(self) - def impl_get_callback(self): - return self._callback, self._callback_params - def impl_has_callback(self): "to know if a callback has been defined or not" - return self._callback is not None + return self.impl_get_callback()[0] is not None def _is_subdyn(self): try: @@ -464,40 +392,6 @@ class Option(OnlyOption): __slots__ = tuple() _empty = '' - def __init__(self, name, doc, default=None, default_multi=None, - requires=None, multi=False, callback=None, - callback_params=None, validator=None, validator_params=None, - properties=None, warnings_only=False): - """ - :param name: the option's name - :param doc: the option's description - :param default: specifies the default value of the option, - for a multi : ['bla', 'bla', 'bla'] - :param default_multi: 'bla' (used in case of a reset to default only at - a given index) - :param requires: is a list of names of options located anywhere - in the configuration. - :param multi: if true, the option's value is a list - :param callback: the name of a function. If set, the function's output - is responsible of the option's value - :param callback_params: the callback's parameter - :param validator: the name of a function which stands for a custom - validation of the value - :param validator_params: the validator's parameters - :param properties: tuple of default properties - :param warnings_only: _validator and _consistencies don't raise if True - Values()._warning contain message - - """ - super(Option, self).__init__(name, doc, default, default_multi, - requires, multi, callback, - callback_params, validator, - validator_params, properties, - warnings_only) - - def impl_getrequires(self): - return self._requires - def _launch_consistency(self, func, option, value, context, index, submulti_index, all_cons_opts, warnings_only): """Launch consistency now @@ -581,24 +475,25 @@ class Option(OnlyOption): current_opt = self def val_validator(val): - if self._validator is not None: - if self._validator_params is not None: - validator_params = {} - for val_param, values in self._validator_params.items(): - validator_params[val_param] = values + validator, validator_params = self.impl_get_validator() + if validator is not None: + if validator_params != {}: + validator_params_ = {} + for val_param, values in validator_params.items(): + validator_params_[val_param] = values #inject value in calculation - if '' in validator_params: - lst = list(validator_params['']) + if '' in validator_params_: + lst = list(validator_params_['']) lst.insert(0, val) - validator_params[''] = tuple(lst) + validator_params_[''] = tuple(lst) else: - validator_params[''] = (val,) + validator_params_[''] = (val,) else: - validator_params = {'': (val,)} + validator_params_ = {'': (val,)} # Raise ValueError if not valid carry_out_calculation(self, config=context, - callback=self._validator, - callback_params=validator_params) + callback=validator, + callback_params=validator_params_) def do_validation(_value, _index, submulti_index): if _value is None: @@ -622,11 +517,11 @@ class Option(OnlyOption): if context is not undefined: descr._valid_consistency(current_opt, _value, context, _index, submulti_index) - self._second_level_validation(_value, self._warnings_only) + self._second_level_validation(_value, self._is_warnings_only()) except ValueError as error: log.debug(_('do_validation for {0}: error in value').format( self.impl_getname()), exc_info=True) - if self._warnings_only: + if self._is_warnings_only(): warning = error error = None except ValueWarning as warning: @@ -655,7 +550,7 @@ class Option(OnlyOption): self.__class__.__name__, 0) elif error: raise ValueError(_("invalid value for option {0}: {1}").format( - self._name, error)) + self.impl_getname(), error)) # generic calculation if context is not undefined: @@ -690,16 +585,6 @@ class Option(OnlyOption): else: do_validation(val, idx, force_submulti_index) - def impl_getdefault(self): - "accessing the default value" - if isinstance(self._default, list): - return copy(self._default) - return self._default - - def impl_getdefault_multi(self): - "accessing the default value for a multi" - return self._default_multi - def impl_is_master_slaves(self, type_='both'): """FIXME """ @@ -720,9 +605,9 @@ class Option(OnlyOption): def impl_is_empty_by_default(self): "no default value has been set yet" - if ((not self.impl_is_multi() and self._default is None) or - (self.impl_is_multi() and (self._default == [] - or None in self._default))): + if ((not self.impl_is_multi() and self.impl_getdefault() is None) or + (self.impl_is_multi() and (self.impl_getdefault() == [] + or None in self.impl_getdefault()))): return True return False @@ -733,12 +618,6 @@ class Option(OnlyOption): #def impl_getkey(self, value): # return value - def impl_is_multi(self): - return self._multi == 0 or self._multi is submulti - - def impl_is_submulti(self): - return self._multi is submulti - def impl_add_consistency(self, func, *other_opts, **params): """Add consistency means that value will be validate with other_opts option's values. @@ -753,7 +632,7 @@ class Option(OnlyOption): raise AttributeError(_("'{0}' ({1}) cannont add consistency, option is" " read-only").format( self.__class__.__name__, - self._name)) + self.impl_getname())) warnings_only = params.get('warnings_only', False) if self._is_subdyn(): dynod = self._impl_getsubdyn() @@ -821,16 +700,15 @@ class Option(OnlyOption): :param load: `True` if we are at the init of the option description :type load: bool """ - if not load and self._consistencies is None: + if not load and self._get_consistencies() is None: self._state_consistencies = None elif load and self._state_consistencies is None: - self._consistencies = None del(self._state_consistencies) else: if load: consistencies = self._state_consistencies else: - consistencies = self._consistencies + consistencies = self._get_consistencies() new_value = [] for consistency in consistencies: values = [] @@ -842,7 +720,8 @@ class Option(OnlyOption): new_value.append((consistency[0], tuple(values), consistency[2])) if load: del(self._state_consistencies) - self._consistencies = new_value + for new_val in new_value: + self._add_consistency(new_val[0], new_val[1], new_val[2]) else: self._state_consistencies = new_value @@ -854,14 +733,14 @@ class Option(OnlyOption): def _validate_callback(self, callback, callback_params): try: - default_multi = self._default_multi + default_multi = self.impl_getdefault_multi() except AttributeError: default_multi = None - if callback is not None and ((self._multi == 1 and - (self._default is not None or + if callback is not None and ((not self.impl_is_multi() and + (self.impl_getdefault() is not None or default_multi is not None)) - or (self._multi != 1 and - (self._default != [] or + or (self.impl_is_multi() and + (self.impl_getdefault() != [] or default_multi is not None)) ): # pragma: optional cover raise ValueError(_("default value not allowed if option: {0} " @@ -885,7 +764,7 @@ def validate_requires_arg(requires, name): # start parsing all requires given by user (has dict) # transforme it to a tuple for require in requires: - if not type(require) == dict: # pragma: optional cover + if not isinstance(require, dict): # pragma: optional cover raise ValueError(_("malformed requirements type for option:" " {0}, must be a dict").format(name)) valid_keys = ('option', 'expected', 'action', 'inverse', 'transitive', @@ -963,46 +842,57 @@ def validate_requires_arg(requires, name): class SymLinkOption(OnlyOption): - __slots__ = ('_opt', '_state_opt') + __slots__ = ('_opt', '_state_opt', '_readonly') def __init__(self, name, opt): - self._name = name if not isinstance(opt, Option): # pragma: optional cover raise ValueError(_('malformed symlinkoption ' 'must be an option ' 'for symlink {0}').format(name)) self._opt = opt - self._readonly = True - super(Base, self).__init__() + self._set_readonly() + super(Base, self).__init__(name, undefined, undefined, undefined, undefined) def __getattr__(self, name, context=undefined): - if name in ('_opt', '_opt_type', '_readonly', 'impl_getpath'): + if name in ('_opt', '_opt_type', '_readonly', 'impl_getpath', '_name', '_state_opt'): return object.__getattr__(self, name) else: return getattr(self._opt, name) def _impl_getstate(self, descr): - super(SymLinkOption, self)._impl_getstate(descr) + self._stated = True self._state_opt = descr.impl_get_path_by_opt(self._opt) def _impl_setstate(self, descr): self._opt = descr.impl_get_opt_by_path(self._state_opt) del(self._state_opt) - super(SymLinkOption, self)._impl_setstate(descr) + try: + del(self._stated) + except AttributeError: # pragma: optional cover + pass + self._set_readonly() def impl_get_information(self, key, default=undefined): return self._opt.impl_get_information(key, default) -#FIXME utile tout ca ? c'est un peu de la duplication ... + def _set_readonly(self): + self._readonly = True + + def impl_is_readonly(self): + try: + return self._readonly + except AttributeError: + return False + def impl_getproperties(self): return self._opt._properties def impl_get_callback(self): - return self._opt._callback, self._opt._callback_params + return self._opt.impl_get_callback() def impl_has_callback(self): "to know if a callback has been defined or not" - return self._opt._callback is not None + return self._opt.impl_has_callback() def _is_subdyn(self): try: diff --git a/tiramisu/option/option.py b/tiramisu/option/option.py index 1f5df9d..687a5a8 100644 --- a/tiramisu/option/option.py +++ b/tiramisu/option/option.py @@ -155,8 +155,8 @@ class IPOption(Option): callback_params=None, validator=None, validator_params=None, properties=None, private_only=False, allow_reserved=False, warnings_only=False): - self._extra = {'_private_only': private_only, - '_allow_reserved': allow_reserved} + extra = {'_private_only': private_only, + '_allow_reserved': allow_reserved} super(IPOption, self).__init__(name, doc, default=default, default_multi=default_multi, callback=callback, @@ -166,7 +166,8 @@ class IPOption(Option): validator=validator, validator_params=validator_params, properties=properties, - warnings_only=warnings_only) + warnings_only=warnings_only, + extra=extra) def _validate(self, value, context=undefined): # sometimes an ip term starts with a zero @@ -186,13 +187,13 @@ class IPOption(Option): def _second_level_validation(self, value, warnings_only): ip = IP('{0}/32'.format(value)) - if not self._extra['_allow_reserved'] and ip.iptype() == 'RESERVED': # pragma: optional cover + if not self._get_extra('_allow_reserved') and ip.iptype() == 'RESERVED': # pragma: optional cover if warnings_only: msg = _("IP is in reserved class") else: msg = _("invalid IP, mustn't be in reserved class") raise ValueError(msg) - if self._extra['_private_only'] and not ip.iptype() == 'PRIVATE': # pragma: optional cover + if self._get_extra('_private_only') and not ip.iptype() == 'PRIVATE': # pragma: optional cover if warnings_only: msg = _("IP is not in private class") else: @@ -257,7 +258,6 @@ class PortOption(Option): if extra['_max_value'] is None: raise ValueError(_('max value is empty')) # pragma: optional cover - self._extra = extra super(PortOption, self).__init__(name, doc, default=default, default_multi=default_multi, callback=callback, @@ -267,10 +267,11 @@ class PortOption(Option): validator=validator, validator_params=validator_params, properties=properties, - warnings_only=warnings_only) + warnings_only=warnings_only, + extra=extra) def _validate(self, value, context=undefined): - if self._extra['_allow_range'] and ":" in str(value): # pragma: optional cover + if self._get_extra('_allow_range') and ":" in str(value): # pragma: optional cover value = str(value).split(':') if len(value) != 2: raise ValueError(_('invalid port, range must have two values ' @@ -286,10 +287,10 @@ class PortOption(Option): val = int(val) except ValueError: # pragma: optional cover raise ValueError(_('invalid port')) - if not self._extra['_min_value'] <= val <= self._extra['_max_value']: # pragma: optional cover + if not self._get_extra('_min_value') <= val <= self._get_extra('_max_value'): # pragma: optional cover raise ValueError(_('invalid port, must be an between {0} ' - 'and {1}').format(self._extra['_min_value'], - self._extra['_max_value'])) + 'and {1}').format(self._get_extra('_min_value'), + self._get_extra('_max_value'))) class NetworkOption(Option): @@ -400,34 +401,34 @@ class DomainnameOption(Option): warnings_only=False, allow_without_dot=False): if type_ not in ['netbios', 'hostname', 'domainname']: raise ValueError(_('unknown type_ {0} for hostname').format(type_)) # pragma: optional cover - self._extra = {'_dom_type': type_} + extra = {'_dom_type': type_} if allow_ip not in [True, False]: raise ValueError(_('allow_ip must be a boolean')) # pragma: optional cover if allow_without_dot not in [True, False]: raise ValueError(_('allow_without_dot must be a boolean')) # pragma: optional cover - self._extra['_allow_ip'] = allow_ip - self._extra['_allow_without_dot'] = allow_without_dot + extra['_allow_ip'] = allow_ip + extra['_allow_without_dot'] = allow_without_dot end = '' extrachar = '' extrachar_mandatory = '' - if self._extra['_dom_type'] != 'netbios': + if extra['_dom_type'] != 'netbios': allow_number = '\d' else: allow_number = '' # pragma: optional cover - if self._extra['_dom_type'] == 'netbios': + if extra['_dom_type'] == 'netbios': length = 14 # pragma: optional cover - elif self._extra['_dom_type'] == 'hostname': + elif extra['_dom_type'] == 'hostname': length = 62 # pragma: optional cover - elif self._extra['_dom_type'] == 'domainname': + elif extra['_dom_type'] == 'domainname': length = 62 if allow_without_dot is False: extrachar_mandatory = '\.' else: extrachar = '\.' # pragma: optional cover end = '+[a-z]*' - self._extra['_domain_re'] = re.compile(r'^(?:[a-z{0}][a-z\d\-{1}]{{,{2}}}{3}){4}$' - ''.format(allow_number, extrachar, length, - extrachar_mandatory, end)) + extra['_domain_re'] = re.compile(r'^(?:[a-z{0}][a-z\d\-{1}]{{,{2}}}{3}){4}$' + ''.format(allow_number, extrachar, length, + extrachar_mandatory, end)) super(DomainnameOption, self).__init__(name, doc, default=default, default_multi=default_multi, callback=callback, @@ -437,23 +438,25 @@ class DomainnameOption(Option): validator=validator, validator_params=validator_params, properties=properties, - warnings_only=warnings_only) + warnings_only=warnings_only, + extra=extra) def _validate(self, value, context=undefined): - if self._extra['_allow_ip'] is True: # pragma: optional cover + if self._get_extra('_allow_ip') is True: # pragma: optional cover try: IP('{0}/32'.format(value)) return except ValueError: pass - if self._extra['_dom_type'] == 'domainname' and not self._extra['_allow_without_dot'] and \ + if self._get_extra('_dom_type') == 'domainname' and \ + not self._get_extra('_allow_without_dot') and \ '.' not in value: # pragma: optional cover raise ValueError(_("invalid domainname, must have dot")) if len(value) > 255: raise ValueError(_("invalid domainname's length (max 255)")) # pragma: optional cover if len(value) < 2: raise ValueError(_("invalid domainname's length (min 2)")) # pragma: optional cover - if not self._extra['_domain_re'].search(value): + if not self._get_extra('_domain_re').search(value): raise ValueError(_('invalid domainname')) # pragma: optional cover diff --git a/tiramisu/option/optiondescription.py b/tiramisu/option/optiondescription.py index f9e9401..fa8bdcc 100644 --- a/tiramisu/option/optiondescription.py +++ b/tiramisu/option/optiondescription.py @@ -79,9 +79,6 @@ class OptionDescription(BaseOption, StorageOptionDescription): self._group_type = groups.default self._is_build_cache = False - def impl_getrequires(self): - return self._requires - def impl_getdoc(self): return self.impl_get_information('doc') @@ -141,11 +138,11 @@ class OptionDescription(BaseOption, StorageOptionDescription): if option._get_id() in cache_option: # pragma: optional cover raise ConflictError(_('duplicate option: {0}').format(option)) cache_option.append(option._get_id()) - option._readonly = True + option._set_readonly() if isinstance(option, OptionDescription): option.impl_validate_options(cache_option) if init: - self._readonly = True + self._set_readonly() # ____________________________________________________________ def impl_set_group_type(self, group_type): @@ -281,7 +278,7 @@ class OptionDescription(BaseOption, StorageOptionDescription): def _impl_get_dynchild(self, child, suffix): name = child.impl_getname() + suffix - path = self._name + suffix + '.' + name + path = self.impl_getname() + suffix + '.' + name if isinstance(child, OptionDescription): return SynDynOptionDescription(child, name, path, suffix) else: @@ -289,7 +286,7 @@ class OptionDescription(BaseOption, StorageOptionDescription): def _impl_getchildren(self, dyn=True, context=undefined): for child in self._impl_st_getchildren(context): - cname = child._name + cname = child.impl_getname() if dyn and child.impl_is_dynoptiondescription(): path = cname for value in child._impl_get_suffixes(context): diff --git a/tiramisu/setting.py b/tiramisu/setting.py index d2435f2..eae628e 100644 --- a/tiramisu/setting.py +++ b/tiramisu/setting.py @@ -251,7 +251,7 @@ class Property(object): :type propname: string """ if self._opt is not None and self._opt.impl_getrequires() is not None \ - and propname in self._opt._calc_properties: # pragma: optional cover + and propname in self._opt.impl_get_calc_properties(): # pragma: optional cover raise ValueError(_('cannot append {0} property for option {1}: ' 'this property is calculated').format( propname, self._opt.impl_getname())) @@ -573,13 +573,13 @@ class Settings(object): :param path: the option's path in the config :type path: str """ - if opt._requires is None: + if opt.impl_getrequires() == []: return frozenset() # filters the callbacks calc_properties = set() context = self._getcontext() - for requires in opt._requires: + for requires in opt.impl_getrequires(): for require in requires: option, expected, action, inverse, \ transitive, same_action = require diff --git a/tiramisu/storage/dictionary/option.py b/tiramisu/storage/dictionary/option.py index 3cb61e7..f58826e 100644 --- a/tiramisu/storage/dictionary/option.py +++ b/tiramisu/storage/dictionary/option.py @@ -25,49 +25,188 @@ from tiramisu.error import ConfigError #____________________________________________________________ # # Base +#('_name', '_informations', '_multi', '_multitype', '_warnings_only', '_extra', '_readonly', '_subdyn) class StorageBase(object): - __slots__ = ('_name', '_requires', '_properties', '_readonly', - '_calc_properties', '_informations', - '_state_readonly', '_state_requires', '_stated', - '_multi', '_validator', '_validator_params', - '_default', '_default_multi', '_state_callback', '_callback', - '_state_callback_params', '_callback_params', '_multitype', - '_consistencies', '_warnings_only', '_master_slaves', - '_state_consistencies', '_extra', '_subdyn', '__weakref__', - '_state_master_slaves', '_choice_values', - '_choice_values_params') + __slots__ = ('_name', + '_informations', + '_multi', + '_extra', + '_warnings_only', + #valeur + '_default', + '_default_multi', + #calcul + '_subdyn', + '_requires', + '_properties', + '_calc_properties', + '_val_call', + #'_validator', + #'_validator_params', + #'_callback', + #'_callback_params', + '_consistencies', + '_master_slaves', + '_choice_values', + '_choice_values_params', + #autre + '_state_master_slaves', + '_state_callback', + '_state_callback_params', + '_state_requires', + '_stated', + '_state_consistencies', + '_state_informations', + '_state_readonly', + '__weakref__' + ) - def __init__(self): - try: - self._subdyn - except AttributeError: - self._subdyn = None - try: - self._consistencies - except AttributeError: - self._consistencies = [] - try: - self._callback - except AttributeError: - self._callback = None - try: - self._callback_params - except AttributeError: - self._callback_params = {} - try: - self._validator - except AttributeError: - self._validator = None - try: - self._validator_params - except AttributeError: - self._validator_params = None + def __init__(self, name, multi, warnings_only, doc, extra): + self._name = name + if doc is not undefined: + self._informations = {'doc': doc} + if multi != 1: + self._multi = multi + if extra is not None: + self._extra = extra + if warnings_only is True: + self._warnings_only = warnings_only + + def _set_default_values(self, default, default_multi): + if self.impl_is_multi() and default is None: + default = [] + self.impl_validate(default) + if ((self.impl_is_multi() and default != tuple()) or + (not self.impl_is_multi() and default is not None)): + if self.impl_is_multi(): + default = tuple(default) + self._default = default + + if self.impl_is_multi() and default_multi is not None: + try: + self._validate(default_multi) + except ValueError as err: # pragma: optional cover + raise ValueError(_("invalid default_multi value {0} " + "for option {1}: {2}").format( + str(default_multi), + self.impl_getname(), err)) + self._default_multi = default_multi + + # information + def impl_set_information(self, key, value): + """updates the information's attribute + (which is a dictionary) + + :param key: information's key (ex: "help", "doc" + :param value: information's value (ex: "the help string") + """ + self._informations[key] = value + + def impl_get_information(self, key, default=undefined): + """retrieves one information's item + + :param key: the item string (ex: "help") + """ + error = False + dico = self._informations + if dico is None or isinstance(dico, str) or isinstance(dico, unicode): + if key == 'doc': + return dico + error = True + elif isinstance(dico, tuple): + try: + return dico[1][dico[0].index(key)] + except AttributeError: + if default is not undefined: + return default + error = True + else: + if default is not undefined: + return self._informations.get(key, default) + try: + return self._informations[key] + except KeyError: # pragma: optional cover + error = True + if error: + raise ValueError(_("information's item not found: {0}").format( + key)) def _add_consistency(self, func, all_cons_opts, params): - self._consistencies.append((func, all_cons_opts, params)) + cons = (func, all_cons_opts, params) + try: + self._consistencies.append(cons) + except AttributeError: + self._consistencies = [cons] def _get_consistencies(self): - return self._consistencies + try: + return self._consistencies + except AttributeError: + return tuple() + + def _set_callback(self, callback, callback_params): + if callback_params is None or callback_params == {}: + val_call = (callback,) + else: + val_call = tuple([callback, callback_params]) + try: + self._val_call = (self._val_call[0], val_call) + except AttributeError: + self._val_call = (None, val_call) + + def impl_get_callback(self): + default = (None, {}) + try: + call = self._val_call[1] + except (AttributeError, IndexError): + ret_call = default + else: + if call is None: + ret_call = default + else: + if len(call) == 1: + ret_call = (call[0], default[1]) + else: + ret_call = call + return ret_call + + def impl_get_calc_properties(self): + try: + return self._calc_properties + except AttributeError: + return frozenset() + + def impl_getrequires(self): + try: + return self._requires + except AttributeError: + return [] + + def _set_validator(self, validator, validator_params): + if validator_params is None: + val_call = (validator,) + else: + val_call = (validator, validator_params) + try: + self._val_call = (val_call, self._val_call[1]) + except (AttributeError, IndexError): + self._val_call = (val_call, None) + + def impl_get_validator(self): + default = (None, {}) + try: + val = self._val_call[0] + except (AttributeError, IndexError): + ret_val = default + else: + if val is None: + ret_val = default + else: + if len(val) == 1: + ret_val = (val[0], default[1]) + else: + ret_val = val + return ret_val def _get_id(self): return id(self) @@ -75,9 +214,103 @@ class StorageBase(object): def _impl_getsubdyn(self): return self._subdyn + def _set_readonly(self): + if not self.impl_is_readonly(): + dico = self._informations + if not (dico is None or isinstance(dico, str) or isinstance(dico, unicode)): + keys = tuple(dico.keys()) + if keys == ('doc',): + dico = dico['doc'] + else: + dico = tuple([tuple(dico.keys()), tuple(dico.values())]) + self._informations = dico + try: + extra = self._extra + self._extra = tuple([tuple(extra.keys()), tuple(extra.values())]) + except AttributeError: + pass + def _impl_setsubdyn(self, subdyn): self._subdyn = subdyn + def _impl_convert_informations(self, descr, load=False): + if not load: + infos = self._informations + if isinstance(infos, tuple): + self._state_informations = {} + for key, value in infos: + self._state_informations[key] = value + elif isinstance(infos, str) or isinstance(infos, unicode): + self._state_informations = {'doc': infos} + else: + self._state_informations = infos + self._state_readonly = self.impl_is_readonly() + else: + try: + self._informations = self._state_informations + del(self._state_informations) + except AttributeError: + pass + if self._state_readonly: + self._set_readonly() + del(self._state_readonly) + + def impl_is_readonly(self): + try: + return not isinstance(self._informations, dict) + except AttributeError: + return False + + def impl_getname(self): + return self._name + + def impl_is_multi(self): + try: + _multi = self._multi + except AttributeError: + _multi = 1 + return _multi == 0 or _multi == 2 + + def impl_is_submulti(self): + try: + _multi = self._multi + except IndexError: + _multi = 1 + return _multi == 2 + + def _get_extra(self, key): + extra = self._extra + if isinstance(extra, tuple): + return extra[1][extra[0].index(key)] + else: + return extra[key] + + def _is_warnings_only(self): + try: + return self._warnings_only + except: + return False + + def impl_getdefault(self): + "accessing the default value" + try: + ret = self._default + if self.impl_is_multi(): + return list(ret) + return ret + except AttributeError: + if self.impl_is_multi(): + return [] + return None + + def impl_getdefault_multi(self): + "accessing the default value for a multi" + if self.impl_is_multi(): + try: + return self._default_multi + except (AttributeError, IndexError): + pass + def commit(self): pass @@ -86,7 +319,8 @@ class StorageOptionDescription(StorageBase): __slots__ = ('_children', '_cache_paths', '_cache_consistencies', '_group_type', '_is_build_cache', '_state_group_type') - def __init__(self): + def __init__(self, name, multi, warnings_only, doc, extra): + super(StorageOptionDescription, self).__init__(name, multi, warnings_only, doc, None) self._cache_paths = None def _add_children(self, child_names, children): @@ -128,7 +362,7 @@ class StorageOptionDescription(StorageBase): cache_path = [] cache_option = [] for option in self._impl_getchildren(dyn=False): - attr = option._name + attr = option.impl_getname() path = str('.'.join(_currpath + [attr])) cache_option.append(option) cache_path.append(path) @@ -272,7 +506,7 @@ class StorageOptionDescription(StorageBase): if error: raise AttributeError(_('unknown Option {0} ' 'in OptionDescription {1}' - '').format(name, self._name)) + '').format(name, self.impl_getname())) def _get_force_store_value(self): #FIXME faire des tests (notamment pas ajouter à un config) diff --git a/tiramisu/storage/sqlalchemy/option.py b/tiramisu/storage/sqlalchemy/option.py index a2c103f..6578189 100644 --- a/tiramisu/storage/sqlalchemy/option.py +++ b/tiramisu/storage/sqlalchemy/option.py @@ -254,7 +254,6 @@ class _Base(SqlAlchemyBase): _reqs = relationship("_Require", collection_class=list) _requires = association_proxy("_reqs", "requires", getset_factory=load_requires) _multi = Column(Integer) - _multitype = Column(String) ###### _callback = Column(PickleType) _call_params = relationship('_CallbackParam',