rougail/creole/template.py

# -*- coding: utf-8 -*-
"""
Gestion du mini-langage de template
On travaille sur les fichiers cibles
"""

import sys
import shutil
import logging

import traceback
import os
from os import listdir, unlink
from os.path import basename, join

from tempfile import mktemp

from Cheetah import Parser
# l'encoding du template est déterminé par une regexp (encodingDirectiveRE dans Parser.py)
# il cherche un ligne qui ressemble à '#encoding: utf-8
# cette classe simule le module 're' et retourne toujours l'encoding utf-8
# 6224
class FakeEncoding():
    def groups(self):
        return ('utf-8',)

    def search(self, *args):
        return self
Parser.encodingDirectiveRE = FakeEncoding()

from Cheetah.Template import Template as ChtTemplate
from Cheetah.NameMapper import NotFound as CheetahNotFound

from config import patch_dir, templatedir, distrib_dir

from .client import CreoleClient, CreoleClientError
from .error import FileNotFound, TemplateError, TemplateDisabled
import eosfunc

from .i18n import _

import pyeole

log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())

class IsDefined(object):
    """
    filtre permettant de ne pas lever d'exception au cas où
    la variable Creole n'est pas définie
     """
    def __init__(self, context):
        self.context = context

    def __call__(self, varname):
        if '.' in varname:
            splitted_var = varname.split('.')
            if len(splitted_var) != 2:
                msg = _(u"Group variables must be of type master.slave")
                raise KeyError(msg)
            master, slave = splitted_var
            if master in self.context:
                return slave in self.context[master].slave.keys()
            return False
        else:
            return varname in self.context


class CreoleGet(object):
    def __init__(self, context):
        self.context = context

    def __call__(self, varname):
        return self.context[varname]

    def __getitem__(self, varname):
        """For bracket and dotted notation
        """
        return self.context[varname]

    def __contains__(self, varname):
        """Check variable existence in context
        """
        return varname in self.context


@classmethod
def cl_compile(kls, *args, **kwargs):
    kwargs['compilerSettings'] = {'directiveStartToken' : u'%',
                   'cheetahVarStartToken' : u'%%',
                   'EOLSlurpToken' : u'%',
                   'PSPStartToken' : u'µ' * 10,
                   'PSPEndToken' : u'µ' * 10,
                   'commentStartToken' : u'µ' * 10,
                   'commentEndToken' : u'µ' * 10,
                   'multiLineCommentStartToken' : u'µ' * 10,
                   'multiLineCommentEndToken' : u'µ' * 10}
    return kls.old_compile(*args, **kwargs)
ChtTemplate.old_compile = ChtTemplate.compile
ChtTemplate.compile = cl_compile


class CheetahTemplate(ChtTemplate):
    """classe pour personnaliser et faciliter la construction
    du template Cheetah
    """
    def __init__(self, filename, context, current_container):
        """Initialize Creole CheetahTemplate

        @param filename: name of the file to process
        @type filename: C{str}
        @param context: flat dictionary of creole variables as 'name':'value',
        @type context: C{dict}
        @param current_container: flat dictionary describing the current container
        @type current_container: C{dict}
        """
        eos = {}
        for func in dir(eosfunc):
            if not func.startswith('_'):
                eos[func] = getattr(eosfunc, func)
        # ajout des variables decrivant les conteneurs
        #FIXME chercher les infos dans le client !
        ChtTemplate.__init__(self, file=filename,
                             searchList=[context, eos, {u'is_defined' : IsDefined(context),
                                                        u'creole_client' : CreoleClient(),
                                                        u'current_container':CreoleGet(current_container),
                                                    }])


class CreoleMaster(object):
    def __init__(self, value, slave=None, index=None):
        """
        On rend la variable itérable pour pouvoir faire:
        for ip in iplist:
        print ip.network
        print ip.netmask
        print ip
        index is used for CreoleLint
        """
        self._value = value
        if slave is not None:
            self.slave = slave
        else:
            self.slave = {}
        self._index = index

    def __getattr__(self, name):
        """Get slave variable or attribute of master value.

        If the attribute is a name of a slave variable, return its value.
        Otherwise, returns the requested attribute of master value.
        """
        if name in self.slave:
            value = self.slave[name]
            if isinstance(value, Exception):
                raise value
            return value
        else:
            return getattr(self._value, name)

    def __getitem__(self, index):
        """Get a master.slave at requested index.
        """
        ret = {}
        for key, values in self.slave.items():
            ret[key] = values[index]
        return CreoleMaster(self._value[index], ret, index)

    def __iter__(self):
        """Iterate over master.slave.

        Return synchronised value of master.slave.
        """
        for i in range(len(self._value)):
            ret = {}
            for key, values in self.slave.items():
                ret[key] = values[i]
            yield CreoleMaster(self._value[i], ret, i)

    def __len__(self):
        """Delegate to master value
        """
        return len(self._value)

    def __repr__(self):
        """Show CreoleMaster as dictionary.

        The master value is stored under 'value' key.
        The slaves are stored under 'slave' key.
        """
        return repr({u'value': self._value, u'slave': self.slave})

    def __eq__(self, value):
        return value == self._value

    def __ne__(self, value):
        return value != self._value

    def __lt__(self, value):
        return self._value < value

    def __le__(self, value):
        return self._value <= value

    def __gt__(self, value):
        return self._value > value

    def __ge__(self, value):
        return self._value >= value

    def __str__(self):
        """Delegate to master value
        """
        return str(self._value)

    def __add__(self, val):
        return self._value.__add__(val)

    def __radd__(self, val):
        return val + self._value

    def __contains__(self, item):
        return item in self._value

    def add_slave(self, name, value):
        """Add a slave variable

        Minimal check on type and value of the slave in regards to the
        master one.

        @param name: name of the slave variable
        @type name: C{str}
        @param value: value of the slave variable
        """
        if isinstance(self._value, list):
            if not isinstance(value, list):
                raise TypeError
            elif len(value) != len(self._value):
                raise ValueError(_(u'length mismatch'))
            new_value = []
            for val in value:
                if isinstance(val, dict):
                    new_value.append(ValueError(val['err']))
                else:
                    new_value.append(val)
            value = new_value
        elif isinstance(value, list):
            raise TypeError
        self.slave[name] = value

class CreoleTemplateEngine(object):
    """Engine to process Creole cheetah template
    """
    def __init__(self, force_values=None):
        #force_values permit inject value and not used CreoleClient (used by CreoleLint)
        self.client = CreoleClient()
        self.creole_variables_dict = {}
        self.force_values = force_values
        self.load_eole_variables()

    def load_eole_variables(self):
        # remplacement des variables EOLE
        self.creole_variables_dict = {}
        if self.force_values is not None:
            values = self.force_values
        else:
            values = self.client.get_creole()
        for varname, value in values.items():
            if varname in self.creole_variables_dict:
                # Creation of a slave create the master
                continue
            if varname.find('.') != -1:
                #support des groupes
                mastername, slavename = varname.split('.')
                if not mastername in self.creole_variables_dict or not \
                        isinstance(self.creole_variables_dict [mastername],
                                CreoleMaster):
                    # Create the master variable
                    if mastername in values:
                        self.creole_variables_dict[mastername] = CreoleMaster(values[mastername])
                    else:
                        #only for CreoleLint
                        self.creole_variables_dict[mastername] = CreoleMaster(value)
                #test only for CreoleLint
                if mastername != slavename:
                    self.creole_variables_dict[mastername].add_slave(slavename, value)
            else:
                self.creole_variables_dict[varname] = value

    def patch_template(self, filevar, force_no_active=False):
        """Apply patch to a template
        """
        var_dir = os.path.join(patch_dir,'variante')
        patch_cmd = ['patch', '-d', templatedir, '-N', '-p1']
        patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']

        tmpl_filename = os.path.split(filevar[u'source'])[1]
        # patches variante + locaux
        for directory in [var_dir, patch_dir]:
            patch_file = os.path.join(directory, '{0}.patch'.format(tmpl_filename))
            if os.access(patch_file, os.F_OK):
                msg = _(u"Patching template '{0}' with '{1}'")
                log.info(msg.format(filevar[u'source'], patch_file))
                ret, out, err = pyeole.process.system_out(patch_cmd + patch_no_debug + ['-i', patch_file])
                if ret != 0:
                    msg = _(u"Error applying patch: '{0}'\nTo reproduce and fix this error {1}")
                    log.error(msg.format(patch_file, ' '.join(patch_cmd + ['-i', patch_file])))
                    #8307 : recopie le template original et n'arrête pas le processus
                    self._copy_to_template_dir(filevar, force_no_active)
                    #raise TemplateError(msg.format(patch_file, err))

    def strip_template_comment(self, filevar):
        """Strip comment from template

        This apply if filevar has a del_comment attribut
        """
        # suppression des commentaires si demandé (attribut del_comment)
        strip_cmd = ['sed', '-i']
        if u'del_comment' in filevar and filevar[u'del_comment'] != '':
            log.info(_(u"Cleaning file '{0}'").format( filevar[u'source'] ))
            ret, out, err = pyeole.process.system_out(strip_cmd
                                                      + ['/^\s*{0}/d ; /^$/d'.format(filevar[u'del_comment']),
                                                         filevar[u'source'] ])
            if ret != 0:
                msg = _(u"Error removing comments '{0}': {1}")
                raise TemplateError(msg.format(filevar[u'del_comment'], err))

    def _check_filevar(self, filevar, force_no_active=False):
        """Verify that filevar is processable

        :param filevar: template file informations
        :type filevar: `dict`

        :raise CreoleClientError: if :data:`filevar` is disabled
                                  inexistant or unknown.

        """
        if not force_no_active and (u'activate' not in filevar or not filevar[u'activate']):

            raise CreoleClientError(_(u"Template file not enabled:"
                                    u" {0}").format(basename(filevar[u'source'])))
        if u'source' not in filevar or filevar[u'source'] is None:
            raise CreoleClientError(_(u"Template file not set:"
                                    u" {0}").format(basename(filevar['source'])))

        if u'name' not in filevar or filevar[u'name'] is None:
            raise CreoleClientError(_(u"Template target not set:"
                                    u" {0}").format(basename(filevar[u'source'])))

    def _copy_to_template_dir(self, filevar, force_no_active=False):
        """Copy template to processing temporary directory.

        :param filevar: template file informations
        :type filevar: `dict`
        :param force_no_active: copy disabled template if `True`
        :type filevar: `bool`
        :raise FileNotFound: if source template does not exist

        """
        self._check_filevar(filevar, force_no_active)
        tmpl_source_name = os.path.split(filevar[u'source'])[1]
        tmpl_source_file = os.path.join(distrib_dir, tmpl_source_name)
        if not os.path.isfile(tmpl_source_file):
            msg = _(u"Template {0} unexistent").format(tmpl_source_file)
            raise FileNotFound(msg)
        else:
            log.info(_(u"Copy template: '{0}' -> '{1}'").format(tmpl_source_file, templatedir))
            shutil.copy(tmpl_source_file, templatedir)

    def prepare_template(self, filevar, force_no_active=False):
        """Prepare template source file
        """
        self._copy_to_template_dir(filevar, force_no_active)
        self.patch_template(filevar, force_no_active)
        self.strip_template_comment(filevar)

    def verify(self, filevar):
        """
        verifie que les fichiers existent
        @param mkdir : création du répertoire si nécessaire
        """
        if not os.path.isfile(filevar[u'source']):
            raise FileNotFound(_(u"File {0} does not exist.").format(filevar[u'source']))
        destfilename = filevar[u'full_name']
        dir_target = os.path.dirname(destfilename)
        if dir_target != '' and not os.path.isdir(dir_target):
            if not filevar[u'mkdir']:
                raise FileNotFound(_(u"Folder {0} does not exist but is required by {1}").format(dir_target, destfilename))
            os.makedirs(dir_target)
        # FIXME: pose plus de problème qu'autre chose (cf. #3048)
        #if not isfile(target):
        #    system('cp %s %s' % (source, target))

    def process(self, filevar, container):
        """Process a cheetah template

        Process a cheetah template and copy the file to destination.
        @param filevar: dictionary describing the file to process
        @type filevar: C{dict}
        @param container: dictionary describing the container
        @type container: C{dict}
        """
        UTF = "#encoding: utf-8"

        self._check_filevar(filevar)

        # full path of the destination file
        destfilename = filevar[u'full_name']

        log.info(_(u"Cheetah processing: '{0}' -> '{1}'").format(filevar[u'source'],
                                                              destfilename))

        # utilisation d'un fichier temporaire
        # afin de ne pas modifier l'original
        tmpfile = mktemp()
        shutil.copy(filevar[u'source'], tmpfile)

        # ajout de l'en-tête pour le support de l'UTF-8
        # FIXME: autres encodages ?
        #os.system("sed -i '1i{0}' {1}".format(UTF, tmpfile)) (supprimé depuis #6224)

        try:
            cheetah_template = CheetahTemplate(tmpfile, self.creole_variables_dict, container)
            os.unlink(tmpfile)
            # suppression de l'en-tête UTF-8 ajouté !!! (supprimé depuis #6224)
            data = str(cheetah_template)  # .replace("{0}\n".format(UTF), '', 1)
        except CheetahNotFound, err:
            varname = err.args[0][13:-1]
            msg = _(u"Error: unknown variable used in template {0} : {1}").format(filevar[u'name'], varname)
            raise TemplateError, msg
        except UnicodeDecodeError, err:
            msg = _(u"Encoding issue detected in template {0}").format(filevar[u'name'])
            raise TemplateError, msg
        except Exception, err:
            msg = _(u"Error while instantiating template {0}: {1}").format(filevar[u'name'], err)
            raise TemplateError, msg

        # écriture du fichier cible
        if destfilename == '':
            # CreoleCat may need to write on stdout (#10065)
            sys.stdout.write(data)
        else:
            try:
                file_h = file(destfilename, 'w')
                file_h.write(data)
                file_h.close()
            except IOError, e:
                msg = _(u"Unable to write in file '{0}': '{1}'").format(destfilename, e)
                raise FileNotFound, msg

    def change_properties(self, filevar, container=None, force_full_name=False):
        chowncmd = [u'chown']
        chownarg = ''
        chmodcmd = [u'chmod']
        chmodarg = ''

        if not force_full_name:
            destfilename = filevar[u'name']
        else:
            destfilename = filevar[u'full_name']

        if u'owner' in filevar and filevar[u'owner']:
            chownarg = filevar[u'owner']
        else:
            chownarg = u'root'

        if u'group' in filevar and filevar[u'group']:
            chownarg += ":" + filevar[u'group']
        else:
            chownarg += u':root'

        if u'mode' in filevar and filevar[u'mode']:
            chmodarg = filevar[u'mode']
        else:
            chmodarg = u'0644'

        chowncmd.extend( [chownarg, destfilename] )
        chmodcmd.extend( [chmodarg, destfilename] )

        log.info(_(u'Changing properties: {0}').format(' '.join(chowncmd)) )
        ret, out, err = pyeole.process.creole_system_out( chowncmd, container=container, context=False )
        if ret != 0:
            log.error(_(u'Error changing properties {0}: {1}').format(ret, err) )

        log.info(_(u'Changing properties: {0}').format(' '.join(chmodcmd)) )
        ret, out, err = pyeole.process.creole_system_out( chmodcmd, container=container, context=False )
        if ret != 0:
            log.error(_(u'Error changing properties {0}: {1}').format(ret, err) )

    def remove_destfile(self, filevar):
        """
        suppression du fichier de destination
        """
        destfilename = filevar[u'full_name']
        if os.path.isfile(destfilename):
            os.unlink(destfilename)
        else:
            log.debug(_(u"File '{0}' unexistent.").format(destfilename))


    def _instance_file(self, filevar, container=None):
        """Run templatisation on one file of one container

        @param filevar: Dictionary describing the file
        @type filevar: C{dict}
        @param container: Dictionary describing a container
        @type container: C{dict}
        """
        if not filevar.get(u'activate', False):
            try:
                # copy and patch disabled templates too (#11029)
                self.prepare_template(filevar, force_no_active=True)
            except FileNotFound:
                pass

            if u'rm' in filevar and filevar[u'rm']:
                log.info(_(u"Removing file '{0}'"
                     u" from container '{1}'").format(filevar[u'name'],
                                                   container[u'name']))
                self.remove_destfile(filevar)

            # The caller handles if it's an error
            raise TemplateDisabled(_(u"Instantiation of file '{0}' disabled").format(filevar[u'name']))

        log.info(_(u"Instantiating file '{0}'"
                 u" from '{1}'").format(filevar[u'name'], filevar[u'source']))
        self.prepare_template(filevar)
        self.verify(filevar)
        self.process(filevar, container)
        if filevar['name'].startswith('..') and container not in [None, 'root']:
            self.change_properties(filevar, None, True)
        else:
            self.change_properties(filevar, container)


    def instance_file(self, filename=None, container='root', ctx=None):
        """Run templatisation on one file

        @param filename: name of a file
        @type filename: C{str}
        @param container: name of a container
        @type container: C{str}
        """
        if container == 'all':
            if ctx is None:
                groups = self.client.get_groups()
            else:
                groups = ctx.keys()
            for group in groups:
                if group in ['all', 'root']:
                    continue
                if ctx is None:
                    lctx = None
                else:
                    lctx = ctx[group]
                self.instance_file(filename=filename, container=group, ctx=lctx)
        else:
            if ctx is None:
                ctx = self.client.get_container_infos(container)

            filevars = [f for f in ctx[u'files'] if f[u'name'] == filename]
            for f in filevars:
                self._instance_file(f, ctx)

    def instance_files(self, filenames=None, container=None, containers_ctx=None):
        """Run templatisation on all files of all containers

        @param filenames: names of files
        @type filename: C{list}
        @param container: name of a container
        @type container: C{str}
        """
        if containers_ctx is None:
            containers_ctx = []
            if container is not None:
                containers_ctx = [self.client.get_container_infos(container)]
            else:
                for group_name in self.client.get_groups():
                    containers_ctx.append(self.client.get_group_infos(group_name))
        if filenames is None:
            all_files = set(listdir(distrib_dir))
            prev_files = set(listdir(templatedir))
            all_declared_files = set()
            for ctx in containers_ctx:
                for fdict in ctx[u'files']:
                    all_declared_files.add(basename(fdict['source']))
            undeclared_files = all_files - all_declared_files
            toremove_files = prev_files - all_files
            # delete old templates (#6600)
            for fname in toremove_files:
                rm_file = join(templatedir, fname)
                log.debug(_(u"Removing file '{0}'").format(rm_file))
                unlink(rm_file)
            # copy template not referenced in a dictionary (#6303)
            for fname in undeclared_files:
                fobj = {'source': join(templatedir, fname), 'name': ''}
                self.prepare_template(fobj, True)

        for ctx in containers_ctx:
            for fdict in ctx[u'files']:
                if not filenames or fdict[u'name'] in filenames:
                    try:
                        self._instance_file(fdict, container=ctx)
                    except TemplateDisabled, err:
                        # Information on disabled template only useful
                        # in debug
                        log.debug(err, exc_info=True)