from lxml import etree from os.path import isfile, join, isdir from pytest import fixture, raises from os import listdir from tiramisu.option import SymLinkOption, OptionDescription, ChoiceOption, Option from tiramisu.setting import groups from creole.objspace import CreoleObjSpace from creole.error import CreoleDictConsistencyError from creole.xml_compare import xml_compare #from creole.loader import creole_loader from creole import loader1 as loader, loader as loader2, config #from creole.loader2 import creole_loader as creole_loader2 from creole.eosfunc import gen_random, random_int from pyeole.encode import normalize dtdfile = 'data/creole.dtd' if isfile(dtdfile): relative = True else: relative = False dtdfile = '/usr/share/creole/creole.dtd' if not isfile(dtdfile): raise Exception('unable to find dtdfile') destfile = '/tmp/test.xml' if relative: dico_dirs = 'tests/flattener_dicos' else: dico_dirs = '/usr/share/creole/tests/flattener_dicos' TEST = set() for testfile in listdir(dico_dirs): if isdir(join(dico_dirs, testfile)): if isdir(join(dico_dirs, testfile, 'result')): TEST.add(testfile) TEST = list(TEST) TEST.remove('51redefine_help') TEST.remove('52exists_redefine') TEST.remove('60extra_load') TEST.remove('60extra_redefine') TEST.remove('60extra_mandatory') TEST.remove('60action_external') TEST.remove('60extra_externalspacecondition') TEST.remove('60familyaction') TEST.remove('60familyaction_accent') TEST.remove('60familyaction_disable') TEST.remove('60familyaction_twoactions') TEST.remove('60familyaction_mandatory') TEST.remove('60familyaction_empty') TEST.remove('60familyactionexternal') TEST.remove('10check_option') TEST.remove('10valid_enum_numberdefault') TEST.remove('60familyaction_save') TEST.remove('60extra_help') EOLE_DIR = '/usr/share/eole/creole/dicos' if isdir(EOLE_DIR): TEST.append(EOLE_DIR) TEST.sort() @fixture(scope="module", params=TEST) def test_dir(request): return request.param def fake_config_save_values(*args, **kwargs): return True def setup_module(module): module.config_save_values_ori = loader2.config_save_values loader2.config_save_values = fake_config_save_values def teardown_module(module): loader2.config_save_values = module.config_save_values_ori loader2.eoleextradico = config.eoleextradico loader2.eoledirs = config.eoledirs def launch_flattener(test_dir): is_eole = test_dir == EOLE_DIR dirs = [test_dir] if not is_eole: subfolder = join(test_dir, 'subfolder') if isdir(subfolder): dirs.append(subfolder) extra = list() subfolder = join(test_dir, 'extra_dirs') load_extra = False if isdir(subfolder): load_extra = True loader.eoleextradico = subfolder loader2.eoleextradico = subfolder #FIXME cannot redefine from now ... :/ #return None, None force_dirs = dirs else: load_extra = False force_dirs = None loader.eoledirs = dirs loader2.eoledirs = dirs try: config1 = loader.creole_loader(load_values=False, load_extra=load_extra) except OSError: # for disknod return None, None config2 = loader2.creole_loader(load_values=False, load_extra=load_extra, force_flattened=destfile) #eolobj = CreoleObjSpace(dtdfile) #eolobj.create_or_populate_from_xml('creole', dirs) #if not is_eole: # subfolder = join(test_dir, 'extra') # if isdir(subfolder): # eolobj.create_or_populate_from_xml('extra', [subfolder]) # subfolder = join(test_dir, 'extra1') # if isdir(subfolder): # eolobj.create_or_populate_from_xml('extra1', [subfolder]) #eolobj.space_visitor() #eolobj.save(destfile) #config2 = creole_loader2(destfile) return config1, config2 def filter_path(path): if path.startswith('containers.'): return not (path.endswith('.level') or path.endswith('_type') or path.endswith('.redefine') or path.startswith('containers.interfaces.')) else: return True def get_descr1_paths(descr1, config1): set1 = set() options1 = {} dict1 = {} containers = {} redefinable = {} for path in descr1._cache_paths[1]: if path.startswith('containers.') and not path.startswith('containers.network.') and not path.startswith('containers.containers'): spath = path.split('.') opt = getattr(config1.cfgimpl_get_description(), path) if len(spath) == 3: _, type_, idx = spath name = None elif len(spath) == 4: _, type_, idx, name = spath else: if not filter_path(path): continue set1.add(path) options1[path] = opt if not isinstance(opt, OptionDescription): try: value = getattr(config1, path) dict1[path] = value except: pass continue if not filter_path(path): continue num = int(idx[len(type_) - 1:]) d = {'name': name, 'opt': opt} try: value = getattr(config1, path) d['value'] = value except: pass containers.setdefault(type_, {}).setdefault(num, []).append(d) if name == None: copt = getattr(config1.cfgimpl_get_description(), 'containers.{}.{}.container'.format(type_, idx)) container = copt.impl_getdefault() copt = getattr(config1.cfgimpl_get_description(), 'containers.{}.{}.name'.format(type_, idx)) name = copt.impl_getdefault() if not isinstance(name, unicode): continue try: copt = getattr(config1.cfgimpl_get_description(), 'containers.{}.{}.redefine'.format(type_, idx)) redefine = copt.impl_getdefault() == 'True' except AttributeError: redefine = False if redefine: del(containers[type_][redefinable[name][container]]) redefinable.setdefault(name, {})[container] = num elif filter_path(path): opt = getattr(config1.cfgimpl_get_description(), path) options1[path] = opt set1.add(path) if not isinstance(opt, OptionDescription): try: value = getattr(config1, path) dict1[path] = value except: pass for container, cont_values in containers.items(): idx = 0 for lst in cont_values.values(): name = container[:-1] + str(idx) subpath = 'containers.{}.{}'.format(container, name) for name in lst: if name['name'] is None: set1.add(subpath) options1[subpath] = name['opt'] else: path = subpath + '.' + name['name'] set1.add(path) options1[path] = name['opt'] if 'value' in name: dict1[path] = name['value'] idx += 1 return set1, options1, dict1 def test_variable_name(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1 = get_descr1_paths(descr1, config1)[0] set2 = set() for path in descr2._cache_paths[1]: if filter_path(path): set2.add(path) for path in set1: try: assert path in set2 except AssertionError as err: print msg = ('path in set1 not present in set2 in {}'.format(test_dir)) print(msg) print('-' * len(msg)) print(set1 - set2) print msg = ('path in set2 not present in set1 in {}'.format(test_dir)) print(msg) print('-' * len(msg)) print(set2 - set1) raise err for path in set2: try: assert path in set1 except AssertionError as err: print msg = ('path in set1 not present in set2 in {}'.format(test_dir)) print(msg) print('-' * len(msg)) print(set1 - set2) print msg = ('path in set2 not present in set1 in {}'.format(test_dir)) print(msg) print('-' * len(msg)) print(set2 - set1) raise err def test_variable_multi(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() #paths1 = descr1._cache_paths[1] set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: try: #opt1 = getattr(descr1, path) opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): assert opt1.impl_is_multi() == opt2.impl_is_multi() assert opt1.impl_is_submulti() == opt2.impl_is_submulti() assert opt1.impl_allow_empty_list() == opt2.impl_allow_empty_list() except AssertionError as err: print msg = 'multi differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) print(opt1.impl_is_multi(), opt1.impl_is_submulti()) print(opt2.impl_is_multi(), opt2.impl_is_submulti()) raise err def test_variable_information(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): information1 = opt1._informations if isinstance(information1, str): information1 = normalize(information1) if isinstance(information1, tuple): tuple1 = [] for info1 in information1[1]: if isinstance(info1, str): info1 = normalize(info1) tuple1.append(info1) information1 = list(information1) information1[1] = tuple(tuple1) information1 = tuple(information1) check_info = True if isinstance(opt1, OptionDescription) and opt1.impl_get_group_type() == groups.master: check_info = False if check_info: assert information1 == opt2._informations except AssertionError as err: import traceback traceback.print_exc() print msg = 'informations differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) print information1 print opt2._informations raise err def test_variable_consistencies(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): assert opt1._is_warnings_only() == opt2._is_warnings_only() cons1 = opt1._get_consistencies() cons2 = opt2._get_consistencies() if cons1 != tuple(): assert len(cons1) == len(cons2) for idx, val1 in enumerate(cons1): val2 = cons2[idx] for subidx, c1 in enumerate(val1): c2 = val2[subidx] if isinstance(c1, tuple): for validx, val in enumerate(c1): assert descr1.impl_get_path_by_opt(val) == descr2.impl_get_path_by_opt(c2[validx]) elif isinstance(c1, dict): assert set(c1.keys()) == set(c2.keys()) for key, val in c1.items(): assert val == c2[key] else: assert c1 == c2 else: assert cons1 == cons2 except AssertionError as err: print msg = 'consistencies differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) print print cons1 print cons2 raise err def test_variable_callback(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: if not (path.startswith('containers.disknods.disknod') and (path.endswith('.major') or path.endswith('.minor') or path.endswith('.type'))): try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): callback1 = opt1.impl_get_callback() callback2 = opt2.impl_get_callback() if not callback1 == (None, {}): assert callback1[0] == callback2[0] assert set(callback1[1].keys()) == set(callback2[1].keys()) for key, val1 in callback1[1].items(): val2 = callback2[1][key] assert len(val1) == len(val2) for idx, idx1 in enumerate(val1): idx2 = val2[idx] if not isinstance(idx1, tuple): assert idx1 == idx2 else: if idx1[0] == idx1[0] == None: continue assert descr1.impl_get_path_by_opt(idx1[0]) == descr2.impl_get_path_by_opt(idx2[0]) assert idx1[1] == idx2[1] else: assert callback1 == callback2 except AssertionError as err: import traceback traceback.print_exc() print msg = 'callback differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) raise err def test_variable_validator(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): validator1 = opt1.impl_get_validator() validator2 = opt2.impl_get_validator() if validator1 == (None, {}): assert validator1 == validator2 else: assert validator1[0] == validator2[0] assert validator1[1].keys() == validator2[1].keys() for key, vals1 in validator1[1].items(): vals2 = validator2[1][key] assert len(vals2) == len(vals1) for idx, val1 in enumerate(vals1): val2 = vals2[idx] if isinstance(val1[0], Option) and isinstance(val1, tuple): assert len(val1) == len(val2) == 2 path1 = config1.cfgimpl_get_description().impl_get_path_by_opt(val1[0]) path2 = config2.cfgimpl_get_description().impl_get_path_by_opt(val2[0]) assert path1 == path2 assert val1[1] == val2[1] else: assert val1 == val2 except AssertionError as err: print msg = 'consistencies differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) raise err def test_variable_choice(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: try: opt1 = options1[path] opt2 = getattr(descr2, path) ischoice1 = isinstance(opt1, ChoiceOption) ischoice2 = isinstance(opt2, ChoiceOption) assert ischoice1 == ischoice2 if ischoice1: assert opt1.impl_get_values(None) == opt2.impl_get_values(None) assert opt1.impl_get_choice_values_params() == opt2.impl_get_choice_values_params() except AssertionError as err: print msg = 'multi differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) raise err def test_variable_default(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: if not (path.startswith('containers.disknods.disknod') and (path.endswith('.major') or path.endswith('.minor') or path.endswith('.type'))): try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): assert opt1.impl_getdefault() == opt2.impl_getdefault() assert opt1.impl_getdefault_multi() == opt2.impl_getdefault_multi() assert opt1.impl_is_unique() == opt2.impl_is_unique() #if hasattr(opt1, '_extra'): # assert opt1._extra == opt2._extra except AssertionError as err: print msg = 'default value differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) print(opt1.impl_getdefault(), opt1.impl_getdefault_multi()) print(opt2.impl_getdefault(), opt2.impl_getdefault_multi()) raise err def test_variable_symlink(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: opt1 = options1[path] opt2 = getattr(descr2, path) try: symlink1 = isinstance(opt1, SymLinkOption) symlink2 = isinstance(opt2, SymLinkOption) assert symlink1 == symlink2 if symlink1: path1 = descr1.impl_get_path_by_opt(opt1._opt) path2 = descr2.impl_get_path_by_opt(opt2._opt) assert path1 == path2 except AssertionError as err: print msg = 'symlink differents for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) raise err def test_variable_properties(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return descr1 = config1.cfgimpl_get_description() descr2 = config2.cfgimpl_get_description() set1, options1, _ = get_descr1_paths(descr1, config1) for path in set1: if not (path.startswith('containers.disknods.disknod') and (path.endswith('.major') or path.endswith('.minor') or path.endswith('.type'))): try: opt1 = options1[path] opt2 = getattr(descr2, path) if not isinstance(opt1, SymLinkOption): assert set(opt1._properties) == set(opt2._properties) requires1 = opt1.impl_getrequires() requires2 = opt2.impl_getrequires() assert opt1.impl_get_calc_properties() == opt2.impl_get_calc_properties() requires1 = opt1.impl_getrequires() requires2 = opt2.impl_getrequires() assert len(requires1) == len(requires2) dictreq1 = dict() dictreq2 = dict() for idx, require1 in enumerate(requires1): require2 = requires2[idx] assert len(require1) == len(require2) for req_idx, req1 in enumerate(require1): req2 = require2[req_idx] dictreq1.setdefault(descr1.impl_get_path_by_opt(req1[0]), []).append(req1[1:]) dictreq2.setdefault(descr2.impl_get_path_by_opt(req2[0]), []).append(req2[1:]) for key, val1 in dictreq1.items(): val2 = dictreq2[key] try: assert val1 == val2 except AssertionError as err: # sur horus on a : [('non', 'non'), 'disabled', False, True, True] au lieu de [('non'), 'disabled', False, True, True] # sur l'ancien loader, on fait un "set" pour supprimer les doublons # pour containers.service_accesss.service_access3 for idx, req1 in enumerate(val1): lval1 = list(req1) lval2 = list(val2[idx]) if lval1[1:] != lval2[1:] or set(lval1[0]) != set(lval2[0]): raise err except AssertionError as err: import traceback traceback.print_exc() print msg = 'properties different for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) raise err def test_variable_value(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return config1.cfgimpl_get_settings().remove('mandatory') config2.cfgimpl_get_settings().remove('mandatory') descr1 = config1.cfgimpl_get_description() dict1 = get_descr1_paths(descr1, config1)[2] dict2 = config2.make_dict() for path in dict1.keys(): try: assert dict1[path] == dict2[path] except AssertionError as err: opt1 = config1.cfgimpl_get_description().impl_get_opt_by_path(path) opt2 = config2.cfgimpl_get_description().impl_get_opt_by_path(path) callback1 = opt1.impl_get_callback() callback2 = opt2.impl_get_callback() if callback1[0] in [gen_random, random_int]: continue print msg = 'value different for {} in {}'.format(path, test_dir) print(msg) print('-' * len(msg)) print(opt1) print(opt2) raise err def test_config(test_dir): test_dir = join(dico_dirs, test_dir) config1, config2 = launch_flattener(test_dir) if config1 is None: return # information information1 = config1.cfgimpl_get_values()._p_._informations information2 = config2.cfgimpl_get_values()._p_._informations try: if isinstance(information1, str): information1 = normalize(information1) if isinstance(information1, tuple): tuple1 = [] for info1 in information1[1]: if isinstance(info1, str): info1 = normalize(info1) tuple1.append(info1) information1 = list(information1) information1[1] = tuple(tuple1) information1 = tuple(information1) assert information1 == information2 except AssertionError as err: print msg = 'informations differents for config in {}'.format(test_dir) print(msg) print('-' * len(msg)) print information1 print information2 raise err # properties props1 = config1.cfgimpl_get_settings()._p_._properties props2 = config2.cfgimpl_get_settings()._p_._properties try: lst1 = set(props1.keys()) lst2 = set(props2.keys()) assert lst1 == lst2 for lst in lst1: assert set(props1[lst]) == set(props2[lst]) except AssertionError as err: print msg = 'properties differents for config in {}'.format(test_dir) print(msg) print('-' * len(msg)) print props1 print props2 raise err # permissives props1 = config1.cfgimpl_get_settings()._p_._permissives props2 = config2.cfgimpl_get_settings()._p_._permissives try: lst1 = set(props1.keys()) lst2 = set(props2.keys()) assert lst1 == lst2 for lst in lst1: assert set(props1[lst]) == set(props2[lst]) except AssertionError as err: print msg = 'permissives differents for config in {}'.format(test_dir) print(msg) print('-' * len(msg)) print props1 print props2 raise err