Some optimisations

This commit is contained in:
Emmanuel Garette 2015-05-03 09:56:03 +02:00
parent 487b99b32c
commit 6cc74506fb
7 changed files with 156 additions and 128 deletions

View File

@ -257,27 +257,27 @@ class SubConfig(object):
return homeconfig.getattr(name, force_permissive=force_permissive,
validate=validate)
context = self._cfgimpl_get_context()
opt_or_descr = self.cfgimpl_get_description().__getattr__(
name, context=context)
option = self.cfgimpl_get_description().__getattr__(name,
context=context)
subpath = self._get_subpath(name)
if isinstance(opt_or_descr, DynSymLinkOption):
if isinstance(option, DynSymLinkOption):
return self.cfgimpl_get_values()._get_cached_item(
opt_or_descr, path=subpath,
option, path=subpath,
validate=validate,
force_permissive=force_permissive)
elif isinstance(opt_or_descr, SymLinkOption): # pragma: no dynoptiondescription cover
elif isinstance(option, SymLinkOption): # pragma: no dynoptiondescription cover
path = context.cfgimpl_get_description().impl_get_path_by_opt(
opt_or_descr._impl_getopt())
option._impl_getopt())
return context.getattr(path, validate=validate,
force_permissive=force_permissive)
elif opt_or_descr.impl_is_optiondescription():
elif option.impl_is_optiondescription():
self.cfgimpl_get_settings().validate_properties(
opt_or_descr, True, False, path=subpath,
option, True, False, path=subpath,
force_permissive=force_permissive)
return SubConfig(opt_or_descr, self._impl_context, subpath)
return SubConfig(option, self._impl_context, subpath)
else:
return self.cfgimpl_get_values()._get_cached_item(
opt_or_descr, path=subpath,
option, path=subpath,
validate=validate,
force_permissive=force_permissive)
@ -499,10 +499,11 @@ class _CommonConfig(SubConfig):
__slots__ = ('_impl_values', '_impl_settings', '_impl_meta', '_impl_test')
def _impl_build_all_caches(self):
if not self.cfgimpl_get_description().impl_already_build_caches():
self.cfgimpl_get_description().impl_build_cache_consistency()
self.cfgimpl_get_description().impl_build_cache_option()
self.cfgimpl_get_description().impl_validate_options()
descr = self.cfgimpl_get_description()
if not descr.impl_already_build_caches():
descr.impl_build_cache_consistency()
descr.impl_build_cache_option()
descr.impl_validate_options()
def read_only(self):
"read only is a global config's setting, see `settings.py`"
@ -600,10 +601,9 @@ class _CommonConfig(SubConfig):
def _gen_fake_values(self):
fake_config = Config(self._impl_descr, persistent=False,
force_storages=get_storages_validation())
force_storages=get_storages_validation(),
force_settings=self.cfgimpl_get_settings())
fake_config.cfgimpl_get_values()._p_._values = copy(self.cfgimpl_get_values()._p_.get_modified_values())
fake_config.cfgimpl_get_settings()._p_._properties = copy(self.cfgimpl_get_settings()._p_.get_modified_properties())
fake_config.cfgimpl_get_settings()._p_._permissives = copy(self.cfgimpl_get_settings()._p_.get_modified_permissives())
return fake_config
@ -613,7 +613,7 @@ class Config(_CommonConfig):
__slots__ = ('__weakref__', '_impl_test', '_impl_name')
def __init__(self, descr, session_id=None, persistent=False,
name=undefined, force_storages=None):
name=undefined, force_storages=None, force_settings=None):
""" Configuration option management master class
:param descr: describes the configuration schema
@ -627,22 +627,27 @@ class Config(_CommonConfig):
:type persistent: `boolean`
"""
#if force_storages is None:
settings, values = get_storages(self, session_id, persistent)
settings, values = get_storages(self, session_id, persistent,
only_value=not force_settings is None)
#else:
# settings, values = force_storages
self._impl_settings = Settings(self, settings)
self._impl_values = Values(self, values)
super(Config, self).__init__(descr, weakref.ref(self))
self._impl_build_all_caches()
self._impl_meta = None
#undocumented option used only in test script
self._impl_test = False
if name is undefined:
name = 'config'
if session_id is not None:
name += session_id
if name is not None and not valid_name(name): # pragma: optional cover
raise ValueError(_("invalid name: {0} for config").format(name))
if force_settings is None:
self._impl_settings = Settings(self, settings)
else:
self._impl_settings = force_settings
self._impl_values = Values(self, values)
super(Config, self).__init__(descr, weakref.ref(self))
if force_settings is None:
self._impl_build_all_caches()
self._impl_meta = None
#undocumented option used only in test script
self._impl_test = False
self._impl_name = name
def cfgimpl_reset_cache(self,

View File

@ -553,10 +553,6 @@ class Option(OnlyOption):
try:
# valid with self._validator
calculation_validator(_value)
# if not context launch consistency validation
if context is not undefined:
descr._valid_consistency(current_opt, _value, context,
_index, submulti_index)
self._second_level_validation(_value, self._is_warnings_only())
except ValueError as error:
log.debug(_('do_validation for {0}: error in value').format(
@ -786,17 +782,16 @@ class Option(OnlyOption):
return DynSymLinkOption(name, self, dyn=path)
def _validate_callback(self, callback, callback_params):
if callback is None:
return
try:
default_multi = self.impl_getdefault_multi()
except AttributeError:
default_multi = None
if callback is not None and ((not self.impl_is_multi() and
(self.impl_getdefault() is not None or
default_multi is not None))
or (self.impl_is_multi() and
(self.impl_getdefault() != [] or
default_multi is not None))
): # pragma: optional cover
if (not self.impl_is_multi() and (self.impl_getdefault() is not None or
default_multi is not None)) or \
(self.impl_is_multi() and (self.impl_getdefault() != [] or
default_multi is not None)): # pragma: optional cover
raise ValueError(_("default value not allowed if option: {0} "
"is calculated").format(self.impl_getname()))

View File

@ -115,25 +115,25 @@ class MasterSlaves(object):
def getitem(self, values, opt, path, validate, force_permissive,
force_properties, validate_properties, slave_path=undefined,
slave_value=undefined, setting_properties=undefined):
slave_value=undefined, setting_properties=undefined, settings=undefined):
if self.is_master(opt):
return self._getmaster(values, opt, path, validate,
force_permissive, force_properties,
validate_properties, slave_path,
slave_value, setting_properties)
slave_value, settings)
else:
return self._getslave(values, opt, path, validate,
force_permissive, force_properties,
validate_properties, setting_properties)
validate_properties, setting_properties, settings)
def _getmaster(self, values, opt, path, validate, force_permissive,
force_properties, validate_properties, c_slave_path,
c_slave_value, setting_properties):
c_slave_value, settings):
value = values._get_validated_value(opt, path, validate,
force_permissive,
force_properties,
validate_properties,
setting_properties=setting_properties)
settings=settings)
if validate is True:
masterlen = len(value)
for slave in self.getslaves(opt):
@ -148,7 +148,7 @@ class MasterSlaves(object):
False,
None, False,
None,
setting_properties=setting_properties)
settings=settings)
slavelen = len(slave_value)
self.validate_slave_length(masterlen, slavelen, slave.impl_getname(), opt)
except ConfigError: # pragma: optional cover
@ -156,7 +156,8 @@ class MasterSlaves(object):
return value
def _getslave(self, values, opt, path, validate, force_permissive,
force_properties, validate_properties, setting_properties):
force_properties, validate_properties, setting_properties,
settings):
"""
if master has length 0:
return []
@ -191,7 +192,7 @@ class MasterSlaves(object):
False,
None, # not undefined
with_meta=master_is_meta,
setting_properties=setting_properties)
settings=settings)
#if slave, had values until master's one
path = opt.impl_getpath(context)
valuelen = len(value)
@ -206,7 +207,7 @@ class MasterSlaves(object):
validate_properties=False,
with_meta=master_is_meta,
index=index,
setting_properties=setting_properties),
settings=settings),
setitem=False,
force=True,
validate=validate)

View File

@ -93,7 +93,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
return _impl_getpaths(self, include_groups, _currpath)
def impl_build_cache_consistency(self, _consistencies=None, cache_option=None):
#FIXME cache_option !
if _consistencies is None:
init = True
_consistencies = {}
@ -114,7 +113,6 @@ class OptionDescription(BaseOption, StorageOptionDescription):
if init and _consistencies != {}:
self._cache_consistencies = {}
for opt, cons in _consistencies.items():
#FIXME dans le cache ...
if opt._get_id() not in cache_option: # pragma: optional cover
raise ConfigError(_('consistency with option {0} '
'which is not in Config').format(

View File

@ -357,36 +357,35 @@ class Settings(object):
self._p_.delproperties(_path)
self._getcontext().cfgimpl_reset_cache()
def _getproperties(self, opt=None, path=None, _is_apply_req=True,
self_properties=undefined):
def _getproperties(self, opt=None, path=None,
self_properties=undefined, read_write=True):
"""
be careful, _is_apply_req doesn't copy properties
"""
if opt is None:
props = copy(self._p_.getproperties(path, default_properties))
props = self._p_.getproperties(path, default_properties)
else:
if self_properties is undefined:
self_properties = self._getproperties()
if path is None: # pragma: optional cover
raise ValueError(_('if opt is not None, path should not be'
' None in _getproperties'))
is_cached = False
if 'cache' in self_properties and 'expire' in self_properties:
ntime = int(time())
else:
ntime = None
if 'cache' in self_properties and self._p_.hascache(path):
if 'expire' in self_properties:
ntime = int(time())
is_cached, props = self._p_.getcache(path, ntime)
if is_cached:
return copy(props)
props = self._p_.getproperties(path, opt.impl_getproperties())
if _is_apply_req:
props = copy(props)
if not is_cached:
props = copy(self._p_.getproperties(path, opt.impl_getproperties()))
props |= self.apply_requires(opt, path)
if 'cache' in self_properties:
if 'expire' in self_properties:
if ntime is None:
ntime = int(time())
ntime = ntime + expires_time
self._p_.setcache(path, copy(props), ntime)
self._p_.setcache(path, props, ntime)
if read_write:
return copy(props)
else:
return props
def append(self, propname):
@ -442,7 +441,7 @@ class Settings(object):
"""
# opt properties
if self_properties is undefined:
self_properties = self._getproperties()
self_properties = self._getproperties(read_write=False)
properties = self._getproperties(opt_or_descr, path,
self_properties=self_properties)
# remove opt permissive
@ -455,12 +454,14 @@ class Settings(object):
if force_permissives is not None:
properties -= force_permissives
# global properties
if force_properties is not None:
self_properties.update(force_properties)
forced_properties = copy(self_properties)
forced_properties.update(force_properties)
else:
forced_properties = self_properties
# calc properties
properties &= self_properties
properties &= forced_properties
# mandatory and frozen are special properties
if is_descr:
properties -= frozenset(('mandatory', 'frozen'))
@ -469,7 +470,7 @@ class Settings(object):
not self._getcontext().cfgimpl_get_values()._isempty(
opt_or_descr, value, opt_or_descr.impl_allow_empty_list()):
properties.remove('mandatory')
if is_write and 'everything_frozen' in self_properties:
if is_write and 'everything_frozen' in forced_properties:
properties.add('frozen')
elif 'frozen' in properties and not is_write:
properties.remove('frozen')

View File

@ -114,7 +114,7 @@ def get_storage(type_, session_id, persistent, test): # pragma: optional cover
return storage_validation.get().Storage(session_id, persistent, test)
def get_storages(context, session_id, persistent):
def get_storages(context, session_id, persistent, only_value=False):
def gen_id(config):
return str(id(config)) + str(time()) + str(randint(0, 500))
@ -122,8 +122,13 @@ def get_storages(context, session_id, persistent):
session_id = gen_id(context)
imp = storage_type.get()
storage = imp.Storage(session_id, persistent)
if only_value:
settings = None
else:
settings = imp.Settings(storage)
values = imp.Values(storage)
try:
return imp.Settings(storage), imp.Values(storage)
return settings, values
except Exception, err:
raise Exception(_('unable to get storages:') + str(err))

View File

@ -55,7 +55,7 @@ class Values(object):
return context
def _getvalue(self, opt, path, is_default, index=undefined,
with_meta=True, setting_properties=undefined):
with_meta=True, settings=undefined):
"""actually retrieves the value
:param opt: the `option.Option()` object
@ -63,11 +63,12 @@ class Values(object):
"""
if opt.impl_is_optiondescription(): # pragma: optional cover
raise ValueError(_('optiondescription has no value'))
setting = self._getcontext().cfgimpl_get_settings()
force_default = 'frozen' in setting._getitem(opt, path,
self_properties=setting_properties) and \
'force_default_on_freeze' in setting._getitem(opt, path,
self_properties=setting_properties)
if settings is undefined:
settings = self._getcontext().cfgimpl_get_settings()._getproperties(
opt, path, read_write=False)
force_default = 'frozen' in settings and \
'force_default_on_freeze' in settings
if not is_default and not force_default:
value = self._p_.getvalue(path)
if index is not undefined:
@ -102,7 +103,6 @@ class Values(object):
if with_meta:
meta = self._getcontext().cfgimpl_get_meta()
if meta is not None:
#FIXME : possible problème de longueur si slave en SymLinkOption
try:
value = meta.cfgimpl_get_values(
)._get_cached_item(opt, path)
@ -161,10 +161,10 @@ class Values(object):
hasvalue = self._contains(path)
if hasvalue and validate:
setting = context.cfgimpl_get_settings()
setting_properties = setting._getproperties(read_write=False)
if 'validator' in setting_properties and validate and hasvalue:
fake_context = context._gen_fake_values()
setting_properties = setting._getproperties()
fake_value = fake_context.cfgimpl_get_values()
fake_value.reset(opt, path, validate=False)
opt.impl_validate(getattr(fake_context, path),
@ -207,7 +207,9 @@ class Values(object):
ntime = None
if setting_properties is undefined:
setting_properties = self._getcontext().cfgimpl_get_settings(
)._getproperties()
)._getproperties(read_write=False)
settings = self._getcontext().cfgimpl_get_settings()._getproperties(
opt, path, read_write=False, self_properties=setting_properties)
if 'cache' in setting_properties and self._p_.hascache(path):
if 'expire' in setting_properties:
ntime = int(time())
@ -219,7 +221,7 @@ class Values(object):
return value
val = self._getitem(opt, path, validate, force_permissive,
force_properties, validate_properties,
setting_properties)
setting_properties, settings=settings)
if 'cache' in setting_properties and validate and validate_properties \
and force_permissive is False and force_properties is None:
if 'expire' in setting_properties:
@ -230,34 +232,42 @@ class Values(object):
return val
def _getitem(self, opt, path, validate, force_permissive, force_properties,
validate_properties, setting_properties=undefined):
validate_properties, setting_properties=undefined,
settings=undefined):
if opt.impl_is_master_slaves():
return opt.impl_get_master_slaves().getitem(self, opt, path,
validate,
force_permissive,
force_properties,
validate_properties,
setting_properties=setting_properties)
setting_properties=setting_properties,
settings=settings)
else:
return self._get_validated_value(opt, path, validate,
force_permissive,
force_properties,
validate_properties,
setting_properties=setting_properties)
setting_properties=setting_properties,
settings=settings)
def _get_validated_value(self, opt, path, validate, force_permissive,
force_properties, validate_properties,
index=undefined, submulti_index=undefined,
with_meta=True, setting_properties=undefined):
with_meta=True, setting_properties=undefined,
settings=undefined):
"""same has getitem but don't touch the cache
index is None for slave value, if value returned is not a list, just return []
"""
context = self._getcontext()
setting = context.cfgimpl_get_settings()
if setting_properties is undefined:
setting_properties = setting._getproperties(read_write=False)
if settings is undefined:
settings = setting._getproperties(opt, path, read_write=False)
is_default = self._is_default_owner(opt, path,
validate_properties=False,
validate_meta=False,
setting_properties=setting_properties)
settings=settings)
try:
if index is None:
gv_index = undefined
@ -265,7 +275,7 @@ class Values(object):
gv_index = index
value = self._getvalue(opt, path, is_default, index=gv_index,
with_meta=with_meta,
setting_properties=setting_properties)
settings=settings)
config_error = None
except ConfigError as err:
# For calculating properties, we need value (ie for mandatory
@ -301,8 +311,6 @@ class Values(object):
force_submulti_index = None
else:
force_submulti_index = submulti_index
if setting_properties is undefined:
setting_properties = setting._getproperties()
try:
opt.impl_validate(value, context,
'validator' in setting_properties,
@ -312,11 +320,7 @@ class Values(object):
config_error = err
value = None
#FIXME pas de test avec les metas ...
#FIXME et les symlinkoption ...
if is_default and 'force_store_value' in setting._getitem(opt,
path,
self_properties=setting_properties):
if is_default and 'force_store_value' in settings:
if isinstance(value, Multi):
item = list(value)
else:
@ -342,22 +346,25 @@ class Values(object):
# user didn't change value, so not write
# valid opt
context = self._getcontext()
setting_properties = context.cfgimpl_get_settings()._getproperties()
setting_properties = context.cfgimpl_get_settings()._getproperties(read_write=False)
if 'validator' in setting_properties:
fake_context = context._gen_fake_values()
fake_context.cfgimpl_get_values()._setitem(opt, value, path,
force_permissive, is_write,
force_permissive,
is_write,
setting_properties)
opt.impl_validate(value, fake_context, 'validator' in setting_properties)
opt.impl_validate(value, fake_context)
self._setitem(opt, value, path, force_permissive, is_write,
setting_properties)
setting_properties, validate_properties=False)
def _setitem(self, opt, value, path, force_permissive, is_write,
setting_properties):
setting_properties, validate_properties=True):
if opt.impl_is_master_slaves():
opt.impl_get_master_slaves().setitem(self, opt, value, path)
self._setvalue(opt, path, value, force_permissive=force_permissive,
is_write=is_write,
setting_properties=setting_properties)
setting_properties=setting_properties,
validate_properties=validate_properties)
def _setvalue(self, opt, path, value, force_permissive=False,
is_write=True, validate_properties=True,
@ -370,19 +377,19 @@ class Values(object):
value=value, path=path,
force_permissive=force_permissive,
self_properties=setting_properties)
owner = context.cfgimpl_get_settings().getowner()
if isinstance(value, Multi):
value = list(value)
if opt.impl_is_submulti():
for idx, val in enumerate(value):
if isinstance(val, SubMulti):
value[idx] = list(val)
owner = context.cfgimpl_get_settings().getowner()
self._p_.setvalue(path, value, owner)
def _is_meta(self, opt, path):
context = self._getcontext()
setting = context.cfgimpl_get_settings()
settings = setting._getitem(opt, path)
settings = setting._getproperties(opt, path, read_write=False)
if 'frozen' in settings and 'force_default_on_freeze' in settings:
return False
if self._p_.getowner(path, owners.default) is not owners.default:
@ -408,18 +415,21 @@ class Values(object):
def _getowner(self, opt, path, validate_properties=True,
force_permissive=False, validate_meta=undefined,
setting_properties=undefined):
settings=undefined):
"""get owner of an option
"""
if not isinstance(opt, Option) and not isinstance(opt,
DynSymLinkOption):
raise ConfigError(_('owner only avalaible for an option'))
context = self._getcontext()
setting = context.cfgimpl_get_settings()
settings = setting._getitem(opt, path,
self_properties=setting_properties)
if settings is undefined:
settings = context.cfgimpl_get_settings()._getproperties(
opt, path, read_write=False)
if 'frozen' in settings and 'force_default_on_freeze' in settings:
return owners.default
if validate_properties:
self._getitem(opt, path, True, force_permissive, None, True)
self._getitem(opt, path, True, force_permissive, None, True,
settings=settings)
owner = self._p_.getowner(path, owners.default)
if validate_meta is undefined:
if opt.impl_is_master_slaves('slave'):
@ -471,10 +481,10 @@ class Values(object):
validate_meta=validate_meta)
def _is_default_owner(self, opt, path, validate_properties=True,
validate_meta=True, setting_properties=undefined):
validate_meta=True, settings=undefined):
return self._getowner(opt, path, validate_properties,
validate_meta=validate_meta,
setting_properties=setting_properties) == \
settings=settings) == \
owners.default
def reset_cache(self, only_expired):
@ -519,8 +529,10 @@ class Values(object):
def _mandatory_warnings(description):
#if value in cache, properties are not calculated
_ret = []
for opt in description._impl_getchildren(
context=self._getcontext()):
context = self._getcontext()
setting_properties = context.cfgimpl_get_settings()._getproperties(
read_write=False)
for opt in description._impl_getchildren(context=context):
if opt.impl_is_optiondescription():
_ret.extend(_mandatory_warnings(opt))
elif isinstance(opt, SymLinkOption) and \
@ -531,7 +543,8 @@ class Values(object):
try:
self._get_cached_item(opt, path=path,
force_properties=frozenset(('mandatory',)),
force_permissive=force_permissive)
force_permissive=force_permissive,
setting_properties=setting_properties)
except PropertiesOptionError as err:
if err.proptype == ['mandatory']:
_ret.append(path)
@ -610,7 +623,6 @@ class Multi(list):
context,
opt, path,
idx))
#FIXME weakref ??
self[idx].submulti = weakref.ref(self)
def _getcontext(self):
@ -628,16 +640,18 @@ class Multi(list):
self._setitem(index, value)
def _setitem(self, index, value, validate=True):
if validate:
fake_context = self._getcontext()._gen_fake_values()
context = self._getcontext()
setting = context.cfgimpl_get_settings()
setting_properties = setting._getproperties(read_write=False)
if 'validator' in setting_properties and validate:
fake_context = context._gen_fake_values()
fake_multi = fake_context.cfgimpl_get_values()._get_cached_item(
self.opt, path=self.path, validate=False)
fake_multi._setitem(index, value, validate=False)
self._validate(value, fake_context, index, True)
#assume not checking mandatory property
super(Multi, self).__setitem__(index, value)
self._getcontext().cfgimpl_get_values()._setvalue(self.opt, self.path,
self)
context.cfgimpl_get_values()._setvalue(self.opt, self.path, self)
#def __repr__(self, *args, **kwargs):
# return super(Multi, self).__repr__(*args, **kwargs)
@ -664,8 +678,11 @@ class Multi(list):
value = self._get_validated_value(index)
except IndexError:
value = None
if validate and value not in [None, undefined]:
fake_context = self._getcontext()._gen_fake_values()
context = self._getcontext()
setting = context.cfgimpl_get_settings()
setting_properties = setting._getproperties(read_write=False)
if 'validator' in setting_properties and validate and value not in [None, undefined]:
fake_context = context._gen_fake_values()
fake_multi = fake_context.cfgimpl_get_values()._get_cached_item(
self.opt, path=self.path, validate=False)
fake_multi.append(value, validate=False, force=True)
@ -703,8 +720,11 @@ class Multi(list):
if self.opt.impl_is_master_slaves():
raise SlaveError(_("cannot insert multi option {0} if master or "
"slave").format(self.opt.impl_getname()))
if value is not None and validate:
fake_context = self._getcontext()._gen_fake_values()
context = self._getcontext()
setting = setting = context.cfgimpl_get_settings()
setting_properties = setting._getproperties(read_write=False)
if 'validator' in setting_properties and validate and value is not None:
fake_context = context._gen_fake_values()
fake_multi = fake_context.cfgimpl_get_values()._get_cached_item(
self.opt, path=self.path, validate=False)
fake_multi.insert(index, value, validate=False)
@ -720,8 +740,11 @@ class Multi(list):
index = self._index
except:
index = None
if validate:
fake_context = self._getcontext()._gen_fake_values()
context = self._getcontext()
setting = context.cfgimpl_get_settings()
setting_properties = setting._getproperties(read_write=False)
if 'validator' in setting_properties and validate:
fake_context = context._gen_fake_values()
fake_multi = fake_context.cfgimpl_get_values()._get_cached_item(
self.opt, path=self.path, validate=False)
fake_multi.extend(iterable, validate=False)