# -*- coding: utf-8 -*- #import cjson import json import fcntl import stat import logging from os.path import isdir, isfile, join, basename, dirname, splitext from os import listdir, makedirs, major, minor from os import stat as os_stat from distutils.version import StrictVersion try: from collections import OrderedDict except: from pyeole.odict import OrderedDict from tiramisu.option import UnicodeOption, OptionDescription, \ IntOption, ChoiceOption, BoolOption, SymLinkOption, IPOption, \ NetworkOption, NetmaskOption from tiramisu.error import PropertiesOptionError, LeadershipError from tiramisu.setting import owners from .config import configeol, eoledirs, dtdfilename, eoleextradico, \ eoleextraconfig, forbiddenextra, VIRTROOT, \ VIRTBASE, VIRTMASTER, templatedir from .error import ConfigError from .var_loader import modes_level, CreoleFamily, CreoleConstraint, \ CreoleVarLoader try: from .client import CreoleClient, CreoleClientError client = CreoleClient() except: client = None from pyeole.encode import normalize try: from .eosfunc import is_instanciate, get_version except: pass from .i18n import _ log = logging.getLogger(__name__) class CreoleContainer(): """ Charge les conteneurs, les fichiers, les packages, services, interfaces et disknods """ def gen_containers(self, paths): """ Generate Containers information in tiramisu tree :paths: paths variables (for added new option in paths's dictionnary) """ containers = [] for name, container in self._get_containers().items(): container['path'] = 'container_path_{0}'.format(name) container['ip'] = 'container_ip_{0}'.format(name) containers.append(container) key_type = {'id': IntOption, 'group': UnicodeOption, 'ip': SymLinkOption, 'path': SymLinkOption, 'level': UnicodeOption} return self._gen_tiramisu_config(paths, "container", containers, key_type) def gen_networks(self, paths): var = [] descr = None namespace = paths['adresse_ip_br0'].split('.')[0] for descr_ in self.space: if descr_._name == namespace: descr = descr_ break if descr == None: raise Exception(_(u'Unable to find namespace: {0}').format( namespace)) for name in ['adresse_ip_br0', 'adresse_netmask_br0', 'adresse_network_br0', 'adresse_broadcast_br0']: path = paths[name] subpath = path.split('.')[1:] opt = descr for p in subpath: opt = getattr(opt, p) var.append(SymLinkOption(name, opt)) return OptionDescription('network', '', var) def gen_interfaces(self, paths): """Add per container interface linked to inter-containers bridge Theses interfaces must come before other containers ones as default gateway. """ lxc_net = OrderedDict() if self.containers_enabled: interfaces = OrderedDict() containers = self._get_containers() for name, container in containers.items(): if name in ['all', 'root']: continue lxc_net[name] = {'name': 'containers', 'container': name, 'linkto': 'br0', 'method': 'bridge', 'ip': 'container_ip_{0}'.format(name), 'mask': 'adresse_netmask_br0', 'bcast': 'adresse_broadcast_br0', 'gateway': 'adresse_ip_br0'} # Insert default interfaces before self.generic['interfaces'] = lxc_net.values() \ + self.generic['interfaces'] return self.gen_generic('interfaces', paths, copy_requires='ip') def gen_service_accesss(self, paths): return self.__gen_service_access_restriction('service_access', paths) def gen_service_restrictions(self, paths): return self.__gen_service_access_restriction('service_restriction', paths) def __gen_service_access_restriction(self, service_type, paths): """Add services requires to service_access/service_restriction If a service is disabled, we remove, also, access to this service """ generic_name = service_type + 's' list_name = service_type + 'list' if 'service' in self.requires: for gen in self.generic[generic_name]: service_name = gen['service'] requires_name = gen.get(list_name) if requires_name is None: requires_name = '___auto_{0}'.format(service_name) gen[list_name] = requires_name self.requires[service_type][requires_name] = {'optional': True, 'list': []} if service_name in self.requires['service']: service_requires = self.requires['service'][service_name]['list'] if self.requires['service'][service_name]['optional'] is False: self.requires['service'][service_name]['optional'] = False self.requires[service_type][requires_name]['list'].extend(service_requires) return self.gen_generic(generic_name, paths, verify_exists_redefine=False) def _gen_file(self, fdata, container, containers): """Generate one file structure for one container :param fdata: file informations :type fdata: `dict` :param container: container of the file :type container: `dict` :return: file information for a container :rtype: `dict` """ file_infos = fdata.copy() # take care of os.path.join and absolute part after first # argument. _file = fdata['name'] if _file[0] == '/': _file = _file[1:] file_infos['container'] = container['name'] file_infos['full_name'] = fdata['name'] if self.containers_enabled and container['name'] != VIRTMASTER: # Prefix the full path with container rootfs if fdata['container'] == 'all': cont_grp = container['group'] else: cont_grp = fdata['container'] cont_name = self.get_real_container_name(containers, cont_grp) _file = join(VIRTROOT, cont_name, VIRTBASE, _file) file_infos['full_name'] = _file source = file_infos.get('source', basename(_file)) source = join(templatedir, source) file_infos['source'] = source return file_infos def gen_files(self, paths): containers = self._get_containers() files = [] for fdata in self.generic.get('files', []): if fdata['container'] == 'all': # Generate a file per container for container in containers.values(): if container['name'] in ['all', VIRTMASTER]: continue files.append(self._gen_file(fdata, container, containers)) else: container = containers[fdata['container']] files.append(self._gen_file(fdata, container, containers)) key_type = {'source': UnicodeOption, 'mode': UnicodeOption, 'full_name': UnicodeOption, 'owner': UnicodeOption, 'group': UnicodeOption, 'mkdir': BoolOption, 'rm': BoolOption, 'del_comment': UnicodeOption, 'level': UnicodeOption} return self._gen_tiramisu_config(paths, "file", files, key_type, requires_key='activate') def gen_disknods(self, paths): containers = self._get_containers() disknods = [] for fdata in self.generic.get('disknods', []): stats = os_stat(fdata['name']) if stat.S_ISBLK(stats.st_mode): dev_type = u'b' device = stats.st_rdev elif stat.S_ISCHR(stats.st_mode): dev_type = u'c' device = stats.st_rdev elif stat.S_ISDIR(stats.st_mode): dev_type = u'b' device = stats.st_dev else: dev_type = None device = None fdata['type'] = dev_type if device is not None: fdata['major'] = major(device) fdata['minor'] = minor(device) else: fdata['major'] = None fdata['minor'] = None fdata['mode'] = u'rwm' fdata['permission'] = 'allow' disknods.append(fdata) key_type = {'major': IntOption, 'minor': IntOption, 'name': UnicodeOption, 'permission': UnicodeOption, 'mode': UnicodeOption, 'type': UnicodeOption, 'level': UnicodeOption} return self._gen_tiramisu_config(paths, "disknod", disknods, key_type) def gen_packages(self, paths): # c'est le dernier 'package' qui a raison # (si présence de deux balises package avec le même nom dans le # même conteneur) return self.gen_generic('packages', paths, verify_exists_redefine=False) class CreoleLoader(CreoleVarLoader, CreoleContainer): """ charge les variables + les conteneurs """ pass def _gen_eol_file(namespace, root_path=None): if namespace == 'creole': return unicode(configeol) else: if root_path is None: root_path = eoleextraconfig return unicode(join(root_path, namespace, 'config.eol')) def _list_extras(extradico=eoleextradico): extranames = [] if isdir(extradico): for directory in listdir(extradico): content = listdir(join(extradico, directory)) if not len(content) == 0: extensions = [splitext(filename)[1] for filename in content] if ".xml" in extensions: extranames.append(directory) return extranames def set_mandatory_permissive(config, action): descr = config.cfgimpl_get_description() parent = getattr(descr, action, None) if parent is not None: for family in parent.impl_getchildren(): for option in family.impl_getchildren(): if 'mandatory' in option.impl_getproperties(): config.cfgimpl_get_settings().setpermissive(('mandatory',), option) def load_extras(config, load_values=True, mandatory_permissive=False, extradico=eoleextradico, force_eoleextraconfig=None): actions = set() if mandatory_permissive and hasattr(config, 'actions'): for name, family in config.actions.iter_groups(): for aname, action in family.iter_groups(): actions.add(action.name) for extraname in _list_extras(extradico=extradico): if extraname in ['creole', 'containers', 'actions']: raise Exception(_('extra name {} not allowed').format(extraname)) eol_file = _gen_eol_file(extraname, root_path=force_eoleextraconfig) config.impl_set_information(extraname, eol_file) if extraname in actions: set_mandatory_permissive(config, extraname) if not load_values: continue #if file not exists, create it (for auto_freeze value) if not isfile(eol_file): try: config_save_values(config, extraname, reload_config=False, check_mandatory=False) except PropertiesOptionError: pass if isfile(eol_file): config_load_values(config, extraname) def load_config_eol(config, configfile=None, try_upgrade=True, force_load_owner=None, current_eol_version=None, force_instanciate=None): if not configfile: configfile = _gen_eol_file('creole') config.impl_set_information('creole', configfile) config_load_values(config, 'creole', force_load_owner=force_load_owner, force_instanciate=force_instanciate) load_values(config, configfile=configfile, try_upgrade=try_upgrade, force_load_owner=force_load_owner, current_eol_version=current_eol_version) def load_config_store(config, store, unset_default=False, force_load_owner=None, current_eol_version=None, force_instanciate=None, remove_unknown_vars=False, try_upgrade=False): """used on Zéphir to upgrade values (2.4.X -> 2.4.X+1) on a configuration that has already been migrated (2.2/2.3 −> 2.4) """ config_load_store(config, 'creole', store, force_load_owner=force_load_owner, unset_default=unset_default, force_instanciate=force_instanciate) load_values(config, try_upgrade=try_upgrade, force_load_owner=force_load_owner, current_eol_version=current_eol_version, remove_unknown_vars=remove_unknown_vars) def load_values(config, configfile=None, try_upgrade=True, force_load_owner=None, current_eol_version=None, remove_unknown_vars=False): load_error = config.impl_get_information('load_error', False) if load_error and try_upgrade: #Try to upgrade from .upgrade import upgrade try: store_dico, version = upgrade(config, configfile) config_load_store(config, 'creole', store_dico, unset_default=True, eol_version='1.0') config.impl_set_information('upgrade', version) remove_unknown_vars = True load_error = False except Exception as e: log.error(_('Error when trying to upgrade config file: {}').format(e)) config.impl_set_information('load_error', True) #print "fichier de configuration invalide 2.2 ou 2.3: {0} : {1}".format(configfile, e) if current_eol_version == None: current_eol_version = get_version('EOLE_RELEASE') eol_version = str(config.impl_get_information('eol_version')) if try_upgrade and not load_error: if StrictVersion(eol_version) > StrictVersion(current_eol_version): raise Exception(_('eol_version ({0}) is greater than current version ({1})').format(eol_version, current_eol_version)) if StrictVersion(eol_version) < StrictVersion(current_eol_version): #can be used to edit lower versions on Zéphir from .upgrade24 import upgrade2 try: # 2.4.x (greater than 2.4.0) if StrictVersion(current_eol_version) >= StrictVersion('2.4.0') and StrictVersion(eol_version) < StrictVersion('2.5.0'): upgrade2('2.4', eol_version, current_eol_version, config) # 2.5.x (greater than 2.5.0) if StrictVersion(current_eol_version) >= StrictVersion('2.5.0') and StrictVersion(eol_version) < StrictVersion('2.6.0'): upgrade2('2.5', eol_version, current_eol_version, config) # 2.6.x (greater than 2.6.0) if StrictVersion(current_eol_version) >= StrictVersion('2.6.0') and StrictVersion(eol_version) < StrictVersion('2.7.0'): upgrade2('2.6', eol_version, current_eol_version, config) if config.impl_get_information('upgrade', '') == '': #set the version only if it is the first upgrade config.impl_set_information('upgrade', eol_version) except Exception as e: log.error(_('Error when trying to upgrade config file: {}').format(normalize(str(e)))) config.impl_set_information('upgrade', False) config.impl_set_information('load_error', True) if remove_unknown_vars: # nettoyage des variables inconnues en dernier (#9858) config.impl_set_information('unknown_options', {}) def creole_loader(load_values=True, rw=False, namespace='creole', load_extra=False, reload_config=True, owner=None, disable_mandatory=False, force_configeol=None, try_upgrade=True, force_load_creole_owner=None, force_dirs=None, warnings=None, force_instanciate=None): """ charge les dictionnaires Creole et retourne une config Tiramisu :load_values: boolean. Charge ou non le fichier config.eol (default True) :rw: boolean. Mode de travail (lecture seule ou lecture/écriture) :namespace: string. Espace de travail (ex: "creole", "bacula", ...) :load_extra: boolean. Charge ou non les dictionnaire extra (si namespace='creole') :reload_config: boolean. Cette option est conservée pour raison de compatibilité ascendante mais n'a plus de justification, a ne pas utiliser :owner: string. Owner forcé sur les variables modifiées :disable_mandatory: boolean. :force_configeol: string. Force le nom du fichier de configuration utilisé :try_upgrade: boolean. :force_dirs: string. Force le nom du réprtoire contenant les dictionnaires :force_load_creole_owner: Owner forcé pour les variables chargées :warnings: affiche les warnings de validation """ if force_configeol is not None: if not isfile(force_configeol): raise ConfigError(_(u"Configuration file unexistent : {0}").format( force_configeol)) if load_extra: #if force_configeol, cannot calculated extra configfile name raise Exception(_(u'Unable to force_configeol with load_extra.')) if force_dirs is not None and (load_extra is True or namespace != 'creole'): raise Exception(_(u'If force_dirs is defined, namespace must be set to creole and load_extra must be set to False.')) if namespace != 'creole' and load_extra: raise ValueError(_(u'namespace is not creole, so load_extra is forbidden.')) #should not load value now because create a Config loader = CreoleLoader() if force_dirs is not None: dirs = force_dirs elif namespace == 'creole': dirs = eoledirs else: dirs = join(eoleextradico, namespace) #load config loader.read_dir(dirs, namespace) if load_extra: extranames = _list_extras() if isdir(eoleextradico): for directory in extranames: if directory in forbiddenextra: raise ValueError( _(u'Namespace {} for extra dictionary not allowed').format(directory)) loader.read_dir(join(eoleextradico, directory), directory) config = loader.get_config() if warnings is None: # warnings is disabled in read-only mode and enabled in read-write mode by default warnings = rw if warnings is False: config.cfgimpl_get_settings().remove('warnings') if owner is not None: if owner not in dir(owners): owners.addowner(owner) config.cfgimpl_get_settings().setowner(getattr(owners, owner)) #load values if force_configeol is not None: configfile = force_configeol else: configfile = _gen_eol_file(namespace) if load_values and isfile(configfile): disable_mandatory = False load_config_eol(config, configfile=configfile, try_upgrade=try_upgrade, force_load_owner=force_load_creole_owner, force_instanciate=force_instanciate) else: config.impl_set_information(namespace, configfile) if load_extra: load_extras(config, load_values=load_values) if rw: config.read_write() elif rw is False: config.read_only() if disable_mandatory: config.cfgimpl_get_settings().remove('mandatory') config.cfgimpl_get_settings().remove('empty') return config def valid_store(store): if not isinstance(store, dict): raise Exception('store is not a dict: {0}'.format(store)) for key, value in store.items(): if not isinstance(key, unicode): raise Exception('store key is not an unicode for {0}'.format(key)) if key != '___version___' and (not isinstance(value, dict) or value.keys() != ['owner', 'val']): raise Exception('store value is not a dict for {0}'.format(key)) def load_store(config, eol_file=configeol): if not isfile(eol_file): store = {} else: fh = open(eol_file, 'r') fcntl.lockf(fh, fcntl.LOCK_SH) try: store = cjson.decode(fh.read(), all_unicode=True) except cjson.DecodeError: config.impl_set_information('load_error', True) store = {} fh.close() try: valid_store(store) except Exception as err: config.impl_set_information('load_error', True) store = {} return store def config_load_store(config, namespace, store, force_instanciate=None, unset_default=False, force_load_owner=None, eol_version='2.4.0'): subconfig = getattr(config, namespace) cache_paths = config.cfgimpl_get_description()._cache_paths[1] unknown_options = {} def reorder_store(path1, path2): """ sorter function. sort description : if varname1 is a master and varname 2 is a slave, returns [varname1, varname2] """ idx_1 = cache_paths.index(path1) idx_2 = cache_paths.index(path2) return cmp(idx_1, idx_2) def store_path_and_reorder(eol_version): """Convenience function to replace varnames with full paths and to sort an unordered ConfigObj's :returns: a sorted ordereddict. """ store_path = {} if namespace == 'creole': paths = {} for path in subconfig.cfgimpl_get_description().impl_getpaths(): vname = path.split('.')[-1] paths[vname] = namespace + '.' + path #variable pas dans Tiramisu for vname, value in store.items(): if vname == '___version___': eol_version = value elif vname not in paths: unknown_options[vname] = value if vname not in paths or value == {}: continue store_path[paths[vname]] = value else: paths = [] subpaths = subconfig.cfgimpl_get_description().impl_getpaths() for path in subpaths: paths.append(namespace + '.' + path) for vname, value in store.items(): if vname == '___version___': eol_version = value continue elif vname not in paths: continue store_path[vname] = value store_order = OrderedDict() store_key = store_path.keys() store_key.sort(reorder_store) for path in store_key: store_order[path] = store_path[path] return eol_version, store_order #don't frozen auto_freeze before instance (or enregistrement_zephir for Zephir) if force_instanciate is not None: is_inst = force_instanciate else: is_inst = is_instanciate() eol_version, store = store_path_and_reorder(eol_version) orig_values = {} for path, values in store.items(): value = values['val'] option = config.unwrap_from_path(path) settings = config.cfgimpl_get_settings() tiramisu_values = config.cfgimpl_get_values() if force_load_owner is not None: owner = force_load_owner else: owner = values['owner'] if isinstance(owner, dict): for towner in owner.values(): if towner not in dir(owners): owners.addowner(towner) else: if owner not in dir(owners): owners.addowner(owner) try: #si unset_default, remet à la valeur par défaut si == à la valeur if unset_default and value == getattr(config, path): continue if isinstance(value, tuple): value = list(value) values['val'] = value orig_values[path.split('.')[-1]] = values if option.impl_is_master_slaves('slave'): if not isinstance(owner, dict): new_owner = getattr(owners, owner) multi = config.getattr(path, force_permissive=True) if isinstance(value, list): tval = {} for idx, val in enumerate(value): tval[idx] = val value = tval for idx, val in value.items(): index = int(idx) if len(multi) > index: multi[index] = val if isinstance(owner, dict): new_owner = getattr(owners, owner[idx]) tiramisu_values.setowner(option, new_owner, index=index) else: log.error(_("master's len is lower than the slave variable ({})").format(path)) else: if isinstance(owner, str): owner = unicode(owner) if not isinstance(owner, unicode): raise Exception(_('owner must be a string for {}').format(path)) new_owner = getattr(owners, owner) try: config.setattr(path, value, force_permissive=True) except ValueError as e: if path == 'schedule.schedule.weekday' and 'schedule.schedule.monthday' in store: settings.remove('validator') config.setattr(path, value, force_permissive=True) config.setattr('schedule.schedule.monthday', store['schedule.schedule.monthday'], force_permissive=True) settings.append('validator') else: raise e tiramisu_values.setowner(option, new_owner) except ValueError as e: msg = str(e).decode('utf8') #msg = unicode(e) log.error(_('unable to load variable {} with value {}: {}').format(path, value, msg)) settings[option].append('load_error') config.impl_set_information('error_msg_{}'.format(path), msg) config.impl_set_information('orig_value_{}'.format(path), value) except LeadershipError: # ne pas faire d'erreur #8380 pass try: config.impl_get_information('force_store_vars').remove(path) except (KeyError, ValueError) as err: pass path_split = path.split('.') family_option = config.unwrap_from_path(namespace + '.' + path_split[1]) settings.setpermissive(tuple(modes_level), opt=family_option) if len(path_split) == 4: parent_option = config.unwrap_from_path(namespace + '.' + path_split[1] + '.' + path_split[2]) settings.setpermissive(tuple(modes_level), opt=parent_option) settings.setpermissive(tuple(modes_level), opt=option) setting = config.cfgimpl_get_settings() if 'auto_freeze' in setting[option] and is_inst == 'oui' and \ not tiramisu_values.is_default_owner(option): setting[option].append('frozen') if namespace == 'creole': config.impl_set_information('unknown_options', unknown_options) config.impl_set_information('eol_version', eol_version) config.impl_set_information('orig_values', orig_values) def config_load_values(config, namespace, eol_file=None, force_instanciate=None, force_load_owner=None): subconfig = getattr(config, namespace, None) if subconfig is None: return if eol_file is None: try: eol_file = config.impl_get_information(namespace) except AttributeError: raise Exception(_(u'config must have eol_file attribute')) else: config.impl_set_information(namespace, eol_file) if not isfile(eol_file): raise IOError(_(u'Can not find file {0}').format( eol_file)) store = load_store(config, eol_file) config_load_store(config, namespace, store, force_instanciate=force_instanciate, force_load_owner=force_load_owner) def config_get_values(config, namespace, check_mandatory=True, ignore_autofreeze=False): """check_mandatory: allows to disable mandatory checking (i.e : when returning values for partial configuration in Zéphir) """ def _get_varname(path): if namespace == 'creole': value_name = path.split('.')[-1] else: value_name = path return value_name subconfig = getattr(config, namespace) if check_mandatory: mandatory_errors = list(config.cfgimpl_get_values( ).mandatory_warnings(force_permissive=True)) if mandatory_errors != []: text = [] for error in mandatory_errors: if not error.startswith(namespace + '.'): continue error = error.split('.') text.append(_(u"Mandatory variable '{0}' from family '{1}'" u" is not set !").format(unicode(error[-1]), unicode(error[1].capitalize())).encode('utf-8')) if text != []: raise PropertiesOptionError("\n".join(text), ('mandatory',)) store = {} opt_values = subconfig.cfgimpl_get_values().get_modified_values() force_store_values = config.impl_get_information('force_store_values', None) for path, own_val in opt_values.items(): #for variable not related to current namespace if not path.startswith(namespace+'.'): continue if force_store_values and path in force_store_values: force_store_values.remove(path) store[_get_varname(path)] = {'val': own_val[1], 'owner': own_val[0]} if force_store_values: for path in force_store_values: varname = _get_varname(path) if varname not in store: try: store[varname] = {'val': config.getattr(path, force_permissive=True), 'owner': u'forced'} except PropertiesOptionError: pass if namespace == 'creole': #update with values in store with no known options store.update(config.impl_get_information('unknown_options', {})) return store def add_eol_version(store, eol_version=None): # on stocke la version passée en paramètre (si >= 2.4.1) ou celle du système le cas échéant if eol_version: if StrictVersion(eol_version) >= StrictVersion('2.4.1'): store['___version___'] = eol_version else: store['___version___'] = get_version('EOLE_RELEASE') def config_save_values(config, namespace, reload_config=True, eol_file=None, check_mandatory=True, eol_version=None): subconfig = getattr(config, namespace) if eol_file is not None: config.impl_set_information(namespace, eol_file) try: eol_file = config.impl_get_information(namespace) except AttributeError: raise Exception(_(u'config must have eol_file attribute')) store = config_get_values(config, namespace, check_mandatory) add_eol_version(store, eol_version) try: dirn = dirname(eol_file) if not isdir(dirn): makedirs(dirn) if not isfile(eol_file): fh = file(eol_file, 'w') fcntl.lockf(fh, fcntl.LOCK_EX) else: fh = file(eol_file, 'r+') fcntl.lockf(fh, fcntl.LOCK_EX) fh.truncate() # Here's where the magic happens #7073 fh.write(cjson.encode(store)) fh.close() except Exception as err: raise Exception(_(u"Error saving file: {0}").format(err)) if client is not None and reload_config: try: client.reload_eol() #client.reload_config() except CreoleClientError: pass return True