Infra/rougail
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

python3 and simplify import

Browse Source
This commit is contained in:
Emmanuel Garette 2019-12-02 10:31:55 +01:00
parent 5e3ff68325
commit 498e950610
31 changed files with 465 additions and 3433 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
creole/__init__.py
View File

14
creole/lint/TODO
View File

@ -1,14 +0,0 @@
XXX: Currently in progress, NOT IN WORKING STATE.
MAJOR REASON IS :
**revamping the implementation entirely for scalability**
AND :
NOT INTEGRATED YET
for pretty print in the console
- ansiprint.py
- terminalreport.py
- terminalwriter.py

0
creole/lint/__init__.py
View File

78
creole/lint/ansiwriter.py
View File

@ -1,78 +0,0 @@
# -*- coding: utf-8 -*-
"""Simple API fro creolelint reports"""
import sys
from creole.lint.warning import Warn
from creole.lint import warnsymb
def ansi_print(text=None, fhandle=None, newline=True, flush=False):
"""normalized (ansi) print >> file handle function"""
#sys.stdout.write(self.getvalue())
if fhandle is None:
fhandle = sys.stderr
if text != None:
#text = text.strip()
if newline:
text += '\n'
fhandle.write(text)
if flush:
fhandle.flush()
# if fhandle:
# fhandle.close()
class AnsiWriter(object):
"""Définit une interface d'écriture de warnings
"""
def __init__(self, write_level, output=sys.stdout):
self.write_level = write_level
self.output = output
def process(self, linter):
"""
parse a result from an item.check() dictionnary
which is made of {name: TmplVar}
"""
ident=1
itemname = linter.name
warnno = linter.warnno
warncomment = linter.warncomment
display = linter.display
name, level = warnsymb.errorcode[warnno]
if level > getattr(warnsymb, self.write_level):
print "\nLint {0} désactivé (niveau {1})".format(itemname, warnsymb.errorlevel[level])
return ''
level = warnsymb.errorlevel[level]
if not display:
ansi_print('')
ansi_print('%s (%s:%s:%s)'%(warncomment, itemname, name, level), self.output)
checks = linter.check()
warn = Warn(self.write_level, itemname, warnno, warncomment, checks)
dico_loc = warn.to_dict()
if dico_loc != '' and dico_loc != {}:
ansi_print('')
ansi_print('%s (%s:%s:%s)'%(warncomment, itemname, name, level), self.output)
def compare(x,y):
return cmp(x[0],y[0])
for vfile in dico_loc.keys():
if vfile != 'dictionnaire':
ansi_print('%s\-- fichier %s' % (' '*ident, vfile), self.output, newline=False)
vlines = dico_loc[vfile]
vlines.sort(compare)
oldline=0
for vline, var in vlines:
if hasattr(var, 'name'):
vname = '%%%%%s'%str(var.name)
else:
vname = str(var)
if vline != None:
if vline != oldline:
ansi_print('', self.output)
ansi_print('%s|-- ligne %s' % (' '*(ident+1), vline), self.output, newline=False)
pass
oldline=vline
if vfile != 'dictionnaire':
ansi_print(" %s" %vname, self.output, newline=False)
else:
ansi_print("%s\-- %s" %(' '*ident, vname), self.output)
if vfile != 'dictionnaire':
ansi_print('', self.output)

39
creole/lint/cmdexec.py
View File

@ -1,39 +0,0 @@
"a system command launcher"
import os, sys
import subprocess
from subprocess import Popen, PIPE
def cmdexec(cmd):
""" return output of executing 'cmd' in a separate process.
raise ExecutionFailed exception if the command failed.
the exception will provide an 'err' attribute containing
the error-output from the command.
"""
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
status = process.poll()
if status:
raise ExecutionFailed(status, status, cmd, out, err)
return out
class ExecutionFailed(Exception):
def __init__(self, status, systemstatus, cmd, out, err):
Exception.__init__(self)
self.status = status
self.systemstatus = systemstatus
self.cmd = cmd
self.err = err
self.out = out
def __str__(self):
return "ExecutionFailed: %d %s\n%s" %(self.status, self.cmd, self.err)
# export the exception under the name 'Error'
Error = ExecutionFailed
try:
ExecutionFailed.__module__ = 'cmdexec'
ExecutionFailed.__name__ = 'Error'
except (AttributeError, TypeError):
pass

1195
creole/lint/creolelint.py
View File

@ -1,1195 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
creole lint main module
"""
import types
import sys
import re
from glob import glob
from os.path import join, basename, isfile, abspath, normpath, isabs
from lxml.etree import parse
from creole import config, eosfunc
from pyeole.process import system_code
from creole.lint.parsetemplate import parse_templates
from creole.lint.normalize import is_correct
from creole.var_loader import parse_dtd
from creole.lxml_parser import parse_xml_file
from creole.config import FLATTENED_CREOLE_DIR, dtdfilename
from creole.loader import PopulateTiramisuObjects
# variables internes aux dictionnaires
DICO_TEST_VARS = ['test_', 'tmp_']
# variables de conteneur calculées dynamiquement
CONTAINER_VARS = ['container_path_', 'container_ip_', 'container_name_',
'adresse_ip_ftp', 'adresse_ip_mysql', 'adresse_ip_dhcp',
'adresse_ip_internet', 'adresse_ip_interbase',
'adresse_ip_postgresql']
# faux-positifs sur les variables d'activation
EXCLUDE_ACTIVATION_VARS = ['activer_cntlm_eth0', 'activer_cntlm_eth1',
'activer_cntlm_eth2', 'activer_cntlm_eth3',
'activer_cntlm_eth4', 'activer_supp_proxy_eth0',
'activer_bash_completion', 'activer_log_martian',
'activer_dns_eth0', 'activer_ctrl_alt_suppr',
'activer_ipv6', 'activer_courier_commun',
'activer_web_valider_ca', 'activer_admin_passfile',
'activer_regles_filtrage_port_source',
'activer_ftp_anonymous_access', 'activer_ftp_access',
'activer_pydio_local', 'activer_courier_imap_sso',
'activer_pydio_ftp', 'activer_client_ldap',
]
# templates à ne pas tester par défaut
EXCLUDE_TMPL = ['/usr/share/eole/creole/distrib/named.conf',
'/usr/share/eole/creole/distrib/common-squid1.conf',
'/usr/share/eole/creole/distrib/zstats.cfg',
'/usr/share/eole/creole/distrib/hosts',
'/usr/share/eole/creole/distrib/active_tags',
]
# dictionnaires conservés pour compatibilité 2.3
OLD_DICOS = ['/usr/share/eole/creole/dicos/51_gepi.xml',
'/usr/share/eole/creole/dicos/51_taskfreak.xml',
'/usr/share/eole/creole/dicos/51_wordpress.xml',
'/usr/share/eole/creole/dicos/60_roundcube.xml',
'/usr/share/eole/creole/dicos/61_ajaxplorer.xml',
'/usr/share/eole/creole/dicos/61_dokuwiki.xml',
'/usr/share/eole/creole/dicos/61_fluxbb.xml',
'/usr/share/eole/creole/dicos/61_piwigo.xml',
'/usr/share/eole/creole/dicos/51_grr.xml',
'/usr/share/eole/creole/dicos/51_cdt.xml',
'/usr/share/eole/creole/dicos/51_piwik.xml',
'/usr/share/eole/creole/dicos/51_spip.xml',
]
starttoken = '%'
varstarttoken = '%%'
builts = [u'ArithmeticError', u'AssertionError', u'AttributeError',
u'BaseException', u'DeprecationWarning', u'EOFError', u'Ellipsis',
u'EnvironmentError', u'Exception', u'False', u'FloatingPointError',
u'FutureWarning', u'GeneratorExit', u'IOError', u'ImportError',
u'ImportWarning', u'IndentationError', u'IndexError', u'KeyError',
u'KeyboardInterrupt', u'LookupError', u'MemoryError', u'NameError',
u'None', u'NotImplemented', u'NotImplementedError', u'OSError',
u'OverflowError', u'PendingDeprecationWarning', u'ReferenceError',
u'RuntimeError', u'RuntimeWarning', u'StandardError',
u'StopIteration', u'SyntaxError', u'SyntaxWarning', u'SystemError',
u'SystemExit', u'TabError', u'True', u'TypeError',
u'UnboundLocalError', u'UnicodeDecodeError', u'UnicodeEncodeError',
u'UnicodeError', u'UnicodeTranslateError', u'UnicodeWarning',
u'UserWarning', u'ValueError', u'Warning', u'ZeroDivisionError',
u'_', u'__debug__', u'__doc__', u'__import__', u'__name__', u'abs',
u'all', u'any', u'apply', u'basestring', u'bool', u'buffer',
u'callable', u'chr', u'classmethod', u'cmp', u'coerce', u'compile',
u'complex', u'copyright', u'credits', u'delattr', u'dict', u'dir',
u'divmod', u'enumerate', u'eval', u'execfile', u'exit', u'file',
u'filter', u'float', u'frozenset', u'getattr', u'globals',
u'hasattr', u'hash', u'help', u'hex', u'id', u'input', u'int',
u'intern', u'isinstance', u'issubclass', u'iter', u'len',
u'license', u'list', u'locals', u'long', u'map', u'max', u'min',
u'object', u'oct', u'open', u'ord', u'pow', u'property', u'quit',
u'range', u'raw_input', u'reduce', u'reload', u'repr', u'reversed',
u'round', u'set', u'setattr', u'slice', u'sorted', u'staticmethod',
u'str', u'sum', u'super', u'tuple', u'type', u'unichr', u'unicode',
u'vars', u'xrange', u'zip']
builts.append(u'is_defined')
builts.append(u'split')
builts.append(u'lower')
cmd_client = (u'creole_client', ('get', 'get_containers'))
for func in dir(eosfunc):
if not func.startswith('_'):
builts.append(unicode(func, 'utf-8'))
# FIXME: je sais pas où la mettre
def is_container_var(varname):
"""
variables de conteneur calculées dynamiquement
"""
for var in CONTAINER_VARS:
if varname.startswith(var):
return True
return False
class TmplVar():
def __init__(self, name, fd, line):
self.name = name
self.location = []
self.add_location(fd, line)
def add_location(self, fd, line):
fd = basename(fd)
self.location.append((fd, line+1))
def set_location(self, location):
self.location = location
def get_location(self):
return self.location
class Var():
def __init__(self, name, description, separator, help, defaultvalue, is_slave):
self.name = name
self.description = description
self.separator = separator
self.help = help
self.defaultvalue = defaultvalue
self.is_slave = is_slave
class CreoleLinter:
"""Base class for linters, collects creole vars and templates
**has to be launched once and only once**
"""
display = True
class __impl:
warnno = 1
warncomment = "Undefined comment"
""" Implementation of the singleton interface """
def set_config(self, tmpl_dir_or_file=None):
if tmpl_dir_or_file != None and type(tmpl_dir_or_file) != str:
raise TypeError('tmpl_dir_or_file doit être une string')
if self.tmpl_dir_or_file != None:
sys.stderr('Tentative de redefinition de tmpl_dir_or_file')
if tmpl_dir_or_file == None:
self.tmpl_dir_or_file = config.distrib_dir
else:
if not isabs(tmpl_dir_or_file):
tmpl_dir_or_file = normpath(join(config.distrib_dir, tmpl_dir_or_file))
if not isfile(tmpl_dir_or_file):
raise Exception("template doit etre le nom d'un template valide")
self.tmpl_dir_or_file = tmpl_dir_or_file
self.eoledirs = config.eoledirs
self.dtddir = config.dtddir
self.exclude_var = []
self.pkgname = None
def load_dics(self):
if self.config is None:
self._collect_vars_in_dicos()
# def load_tmpls(self):
# if self.tmplvars == None:
# self._collect_set_vars()
# self._collect_def_vars()
# self._collect_for_vars()
# self._collect_define_vars()
# self._collect_vars_in_tmplfiles()
#
def get_dicos_name(self):
if self.config is None:
raise Exception('Dictionnaire non chargé')
dic = self.variables.keys()
dic.sort()
return dic
# def get_dicos_files(self):
# if self.creoledic == None:
# raise Exception('Dictionnaire non chargé')
# dic = []
# for name in self.creoledic.generic['files']:
# dic.append(basename(name['source']))
# dic.sort()
# return dic
#
# def get_tmplvars_name(self):
# if self.tmplvars == None:
# raise Exception('Template non chargé')
# tmpl = self.tmplvars.keys()
# tmpl.sort()
# return tmpl
#
# def get_defvars_name(self):
# if self.defvars == None:
# raise Exception('Template non chargé')
# defv = self.defvars.keys()
# defv.sort()
# return defv
#
# def get_setvars_name(self):
# if self.setvars == None:
# raise Exception('Fonction non chargé')
# var = self.setvars.keys()
# var.sort()
# return var
#
# def get_forvars_name(self):
# if self.forvars == None:
# raise Exception('Fonction non chargé')
# var = self.forvars.keys()
# var.sort()
# return var
#
# def get_tmplvar(self, name):
# return self.tmplvars[name]
#
# def get_separators(self):
# return self.creoledic.get_separators()
#
def get_dico_file_names(self):
if self.eoledirs == None or self.tmpl_dir_or_file == None:
raise Exception('Utiliser la methode set_config avant')
ret = []
for eoledir in self.eoledirs:
eoledir = abspath(eoledir)
if isfile(eoledir):
ret.append(eoledir)
else:
ret.extend(glob(join(eoledir, '*.xml')))
return ret
def get_dtd(self):
if self.dtddir == None:
raise Exception('Utiliser la methode set_config avant')
return join(self.dtddir, 'creole.dtd')
def _collect_vars_in_dicos(self):
if self.eoledirs == None or self.tmpl_dir_or_file == None:
raise Exception('Utiliser la methode set_config avant')
flattened = join(FLATTENED_CREOLE_DIR, 'flattened_creole.xml')
with file(flattened, 'r') as fhd:
xmlroot = parse(fhd).getroot()
tiramisu_objects = PopulateTiramisuObjects()
tiramisu_objects.parse_dtd(dtdfilename)
tiramisu_objects.make_tiramisu_objects(xmlroot)
self.config = tiramisu_objects.build()
self.config.read_write()
self.config.cfgimpl_get_settings().remove('hidden')
self.config.cfgimpl_get_settings().remove('validator')
self.config.cfgimpl_get_settings().remove('disabled')
for path in self.config.creole.make_dict():
spath = path.split('.')
vname = spath[-1]
fname = spath[0]
is_slave = False
if len(spath) == 3:
master = spath[1]
if master != vname:
is_slave = True
option = self.config.unwrap_from_path('creole.' + path)
self.variables[vname] = Var(vname, option.impl_get_information('doc', None),
option.impl_get_information('separator', ''),
option.impl_get_information('help', None),
option.impl_getdefault(),
is_slave)
if fname not in self.families:
self.families.append(fname)
def _parse_tabs_in_dicos(self):
if self.eoledirs == None or self.tmpl_dir_or_file == None:
raise Exception('Utiliser la methode set_config avant')
tabs_in_dics = []
fnames = []
for directory in self.eoledirs:
if isfile(directory):
fnames.append(directory)
else:
fnames.extend(glob(join(directory, "*.xml")))
for fname in fnames:
fh = file(fname, 'r')
content = fh.read()
if '\t' in content:
tabs_in_dics.append(fname)
fh.close()
return tabs_in_dics
def _list_tmpl_files(self):
if isfile(self.tmpl_dir_or_file):
return [self.tmpl_dir_or_file]
ret = []
for filename in glob(join(self.tmpl_dir_or_file, '*')):
if filename.startswith('.') or filename.endswith('~'):
continue
if filename in EXCLUDE_TMPL:
print " \\-- template desactivé : {0}".format(filename)
continue
ret.append(filename)
return ret
# def _add_collected_var(self, dvar, var, fd, linenb):
# if dvar.has_key(var):
# dvar[var].add_location(fd=fd, line=linenb)
# else:
# dvar[var] = TmplVar(name=var, fd=fd, line=linenb)
#
# def _collect_var_in(self, dvar, var, fd, linenb, exists=False):
# not_added=True
# if exists == True:
# if self.forvars.has_key(var):
# #si deja en memoire
# if (basename(fd), linenb+1) in self.forvars[var].location:
# return
# self._add_collected_var(self.forvars, var, fd, linenb)
# not_added=False
# if self.setvars.has_key(var):
# self._add_collected_var(self.setvars, var, fd, linenb)
# not_added=False
# if self.defvars.has_key(var):
# self._add_collected_var(self.defvars, var, fd, linenb)
# not_added=False
# #test les builtsin seulement si variable
# if not_added == True and unicode(var, 'utf-8') in builts:
# #self.builtsvar.append(var)
# return
# if not_added == True:
# self._add_collected_var(dvar, var, fd, linenb)
#
# def _collect_vars_in(self, expr, dvar, fd, linenb, tvarstarttoken,
# exists=False):
# if self.unknown_client == None:
# self.unknown_client = []
# varcreole = re.compile(tvarstarttoken+'([a-zA-Z0-9_\.{}]+)')
# varcreole2 = re.compile(tvarstarttoken+'(\w+)')
# varcreolebr = re.compile(tvarstarttoken+'{(\w+)}')
# varmulti = re.compile('(\w+)\.(\w+)')
# for var in varcreole.findall(expr):
# ret = varmulti.match(var)
# if ret != None:
# if ret.group(1) == cmd_client[0]:
# if ret.group(2) not in cmd_client[1]:
# self.unknown_client.append(TmplVar(name=ret.group(2), fd=fd, line=linenb))
# else:
# #%%var.sousvar
# self._collect_var_in(dvar, ret.group(1), fd, linenb, exists)
# self._collect_var_in(dvar, ret.group(2), fd, linenb, exists)
# else:
# #%%var
# for var2 in varcreole2.findall(tvarstarttoken+var):
# self._collect_var_in(dvar, var2, fd, linenb, exists)
# #%%{var}
# for var2 in varcreolebr.findall(tvarstarttoken+var):
# self._collect_var_in(dvar, var2, fd, linenb, exists)
#
# def _collect_vars(self, tvars, tpattern, all_char=False, with_var=False, with_vars=True, broken=None):
# """
# collect vars in template for a specified pattern
#
# :tvars: all collected var are store in this variable
# :tpattern: re pattern
# :broken: if set, store broken variable
# """
# if tvars == None:
# tvars = {}
# tstarttoken = ''
# tvarstarttoken = ''
# for tmplfd in self._list_tmpl_files():
# fh = open(tmplfd, 'r')
# lines = fh.readlines()
# length = len(lines)
# settings = False
# if tstarttoken != starttoken or \
# tvarstarttoken != varstarttoken:
# if all_char:
# char = '(.*)'
# else:
# char = '( *)'
# pattern = re.compile(char+starttoken+tpattern)
# tstarttoken = starttoken
# tvarstarttoken = varstarttoken
# for linenb in range(length):
# line = lines[linenb]
# if line.strip() == '%compiler-settings':
# settings = True
# if settings and line.strip() == \
# '%end compiler-settings'.strip():
# settings = False
# if not settings:
# ret = pattern.match(line.strip())
# if ret != None:
# expr = ret.group(2).strip()
# if with_var:
# self._collect_var_in(tvars, expr, tmplfd, linenb)
# if with_vars:
# self._collect_vars_in(expr,
# tvars, tmplfd, linenb,
# tvarstarttoken)
# len_token = len(varstarttoken)
# if broken is not None and expr.strip()[:len_token] != tvarstarttoken:
# broken.append(TmplVar(
# name=line.strip(),
# fd=tmplfd, line=linenb))
# else:
# tline = line.split('=')
# tkey = tline[0].strip()
# if tkey == 'cheetahVarStartToken':
# tvarstarttoken = tline[1].strip()
# elif tkey == 'directiveStartToken':
# tstarttoken = tline[1].strip()
# pattern = re.compile(tstarttoken+tpattern)
# fh.close()
# return tvars
#
# def _collect_for_vars(self):
# """
# collect all vars generate in 'for'
# """
# self.brokenfor = []
# self.forvars = self._collect_vars(self.forvars, 'for (.+) in (.*)', broken=self.brokenfor)
#
# def _collect_def_vars(self):
# """
# collect all vars generate in 'def'
# """
# self.defvars = self._collect_vars(self.defvars, 'def (.*)\((.*)\)', with_var=True)
#
# def _collect_set_vars(self):
# """
# collect all vars generate in 'set'
# """
# self.setvars = self._collect_vars(self.setvars, 'set (.*)=.*')
#
# def _collect_define_vars(self):
# """
# collect all vars generate in 'def'
# """
# self.var_with_is_defined = self._collect_vars(
# self.var_with_is_defined,
# "is_defined\(\'(\w+)\'\)",
# all_char=True,
# with_var=True, with_vars=False)
# ##FIXME pas de support de cheetahVarStartToken, ...
# #if self.var_with_is_defined == None:
# # self.var_with_is_defined = {}
# # pattern = re.compile('(.*) %sis_defined\(\'(\w+)\'\)'%varstarttoken)
# # for tmplfd in self._list_tmpl_files():
# # fh = open(tmplfd, 'r')
# # lines = fh.readlines()
# # length = len(lines)
# # for linenb in range(length):
# # line = lines[linenb]
# # ret = pattern.match(line)
# # if ret != None:
# # self._collect_var_in(self.var_with_is_defined, ret.group(2), tmplfd, linenb)
# # fh.close()
#
# def _collect_vars_in_tmplfiles(self):
# if self.eoledirs == None or self.tmpl_dir_or_file == None:
# raise Exception('Utiliser la methode set_config avant')
# # XXX ".eolvars" is a good placeholder for var names to be kept in touch
# if self.tmplvars == None:
# self.tmplvars = {}
# for tmplfd in self._list_tmpl_files():
# fh = open(tmplfd, 'r')
# lines = fh.readlines()
# length = len(lines)
# settings = False
# tvarstarttoken = varstarttoken
# for linenb in range(length):
# line = lines[linenb]
# if line.strip() == '%compiler-settings':
# settings = True
# if settings and line.strip() == \
# '%end compiler-settings'.strip():
# settings = False
# if not settings:
# self._collect_vars_in(line, self.tmplvars, tmplfd,
# linenb, tvarstarttoken, True)
# else:
# tline = line.split('=')
# tkey = tline[0].strip()
# if tkey == 'cheetahVarStartToken':
# tvarstarttoken = tline[1].strip()
#
# fh.close()
#
# storage for the instance reference
__instance = None
def __init__(self):
""" Create singleton instance """
# Check whether we already have an instance
if CreoleLinter.__instance is None:
# Create and remember instance
CreoleLinter.__instance = CreoleLinter.__impl()
self.tmpl_dir_or_file = None
self.eoledirs = None
self.config = None
self.variables = {}
self.families = []
self.tmplvars = None
self.forvars = None
self.defvars = None
self.setvars = None
self.unknown_client = None
self.brokenfor = None
self.var_with_is_defined = None
self.exclude_var = None
self.skip_var = {}
self.conflevel = 'eole'
# Store instance reference as the only member in the handle
self.__dict__['_CreoleLinter__instance'] = CreoleLinter.__instance
def __getattr__(self, attr):
""" Delegate access to implementation """
return getattr(self.__instance, attr)
def __setattr__(self, attr, value):
""" Delegate access to implementation """
return setattr(self.__instance, attr, value)
#class SaveItem(CreoleLinter):
# """
# eolvars not present in the dicos
# """
# name = 'save'
# warnno = 1
# warncomment = "Ne pas utiliser la fonction SaveItem comme un test"
#
# def process(self):
# self.load_dics()
# if self.pkgname == None:
# raise Exception('fichier creolelint.conf incomplet (name)')
# filename = join(expanduser('~/.creolelint'), self.pkgname+'.conf')
# fh = open(filename, 'w')
# fh.write('vardico = '+str(self.get_dicos_name())+'\n')
# fh.close()
# print('fichier sauvegardé')
#
# def check(self):
# return []
#
#class OrphansInDicosItem(CreoleLinter):
# """
# eolvars not present in the dicos
# """
# name = 'orphans_in_dicos'
# warnno = 8
# warncomment = "Variable dictionnaire non utilisée dans un template"
#
# def check(self):
# self.load_dics()
# self.load_tmpls()
#
# vars_name = set(self.get_dicos_name())
# tmplvars_name = set(self.get_tmplvars_name())
# only_in_dicos = vars_name - tmplvars_name
# ret = []
# skip_var = self.skip_var.get(self.name, {})
# for var in only_in_dicos:
# if skip_var.has_key(var):
# continue
# test_start = False
# for start in DICO_TEST_VARS:
# if var.startswith(start):
# test_start = True
# break
# if not test_start:
# ret.append(self.variables[var])
# return ret
#
#class OrphansInTmplItem(CreoleLinter):
# """
# eolvars not present in the templates
# """
# name = 'orphans_in_tmpl'
# warnno = 8
# warncomment = "Variable template non présente dans le dictionnaire"
#
# def check(self):
# self.load_dics()
# self.load_tmpls()
#
# vars_name = self.get_dicos_name()
# if self.exclude_var != None:
# vars_name.extend(self.exclude_var)
# vars_name=set(vars_name)
# tmplvars_name = set(self.get_tmplvars_name())
# only_in_tmpl = tmplvars_name - vars_name
# #remove is_defined
# is_defined_name = set(self.var_with_is_defined.keys())
# only_in_tmpl = only_in_tmpl - is_defined_name
# set_var = set(self.get_setvars_name())
# only_in_tmpl = only_in_tmpl - set_var
# ret = []
# for var in only_in_tmpl:
# skipped_location = []
# for location in self.tmplvars[var].location:
# if self.skip_var.has_key(self.name) and self.skip_var[self.name].has_key(var):
# if not location in self.skip_var[self.name][var]:
# skipped_location.append(location)
# else:
# skipped_location.append(location)
# if skipped_location != []:
# tmplvar = TmplVar(var, '', 0)
# tmplvar.set_location(skipped_location)
# ret.append(tmplvar)
# #results.append(self.tmplvars[var])
# return ret
#
#class OrphansDefItem(CreoleLinter):
# name = 'orphans_def'
# warnno = 7
# warncomment = "Fonction définie mais non utilisée"
#
# def check(self):
# self.load_tmpls()
# results = []
# for defvar in self.get_defvars_name():
# defname = {}
# for filedesc, linenb in self.defvars[defvar].location:
# if not defname.has_key((defvar, filedesc)):
# defname[(defvar, filedesc)]=linenb
# else:
# defname[(defvar, filedesc)]="exists"
# for defvar, filedesc in defname.keys():
# if defname[(defvar, filedesc)] != "exists":
# results.append(TmplVar(name=defvar, fd=filedesc, line=defname[(defvar, filedesc)]))
#
# return results
#
#class OrphansSetItem(CreoleLinter):
# name = 'orphans_set'
# warnno = 7
# warncomment = "Variable définie dans le template mais non utilisée"
#
# def check(self):
# self.load_tmpls()
# results = []
# tmpl_vars = set(self.get_tmplvars_name())
# for setvar in self.get_setvars_name():
# setname = {}
# for filedesc, linenb in self.setvars[setvar].location:
# if setname.has_key((setvar, filedesc)) == False:
# setname[(setvar, filedesc)]=linenb
# else:
# setname[(setvar, filedesc)]="exists"
#
# for setvar, filedesc in setname.keys():
# if setname[(setvar, filedesc)] != "exists":
# results.append(TmplVar(name=setvar, fd=filedesc, line=setname[(setvar, filedesc)]))
#
# return results
#
#class OrphansForItem(CreoleLinter):
# name = 'orphans_for'
# warnno = 7
# warncomment = "Variable définie dans une boucle mais non utilisée"
#
# def check(self):
# self.load_tmpls()
# results = []
# for forvar in self.get_forvars_name():
# forname = {}
# for filedesc, linenb in self.forvars[forvar].location:
# if forname.has_key((forvar, filedesc)) == False:
# forname[(forvar, filedesc)]=linenb
# else:
# forname[(forvar, filedesc)]="exists"
#
# for forvar, filedesc in forname.keys():
# if forname[(forvar, filedesc)] != "exists":
# results.append(TmplVar(name=forvar, fd=filedesc, line=forname[(forvar, filedesc)]))
# return results
#
#class OrphansDicosFilesItem(CreoleLinter):
# """
# """
# name = 'orphans_dicos_files'
# warnno = 1
# warncomment = "Template déclaré dans le dicos inexistant"
#
# def check(self):
# self.load_dics()
#
# dicos_files = []
# for filen in self.get_dicos_files():
# dicos_files.append(basename(filen))
# dicos_files = set(dicos_files)
# tmpl_files = []
# for filen in self._list_tmpl_files():
# tmpl_files.append(unicode(basename(filen), 'utf-8'))
# tmpl_files=set(tmpl_files)
# orphans = dicos_files - tmpl_files
# ret = []
# for var in orphans:
# if self.skip_var.has_key(self.name) and not self.skip_var[self.name].has_key(var):
# ret.append(var)
# else:
# ret.append(var)
#
# return ret
#
#class OrphansTmplFilesItem(CreoleLinter):
# """
# """
# name = 'orphans_tmpl_files'
# warnno = 1
# warncomment = "Template non déclaré dans le dicos"
#
# def check(self):
# self.load_dics()
#
# dicos_files = []
# for filen in self.get_dicos_files():
# dicos_files.append(basename(filen))
# dicos_files = set(dicos_files)
# tmpl_files = []
# for filen in self._list_tmpl_files():
# tmpl_files.append(unicode(basename(filen), 'utf-8'))
# tmpl_files=set(tmpl_files)
# return tmpl_files - dicos_files
#
class WrongDicosNameItem(CreoleLinter):
name = 'wrong_dicos_name'
warnno = 6
warncomment = "Dictionnaire avec un nom invalide"
def check(self):
if self.conflevel == 'common':
pattern = '(0'
elif self.conflevel == 'conf':
pattern = '(1'
elif self.conflevel == 'eole':
pattern = '(2'
else:
pattern = '([3-9]'
pattern += '[0-9]_[a-z0-9_]+)'
cpattern = re.compile(pattern)
ret = []
for filename in self.get_dico_file_names():
name = basename(filename)
if cpattern.match(name) == None:
ret.append(name)
return ret
class HiddenIfInDicosItem(CreoleLinter):
name = 'hidden_if_in_dicos'
warnno = 5
warncomment = "Dictionnaire contenant un hidden_if_*"
def check(self):
ret = []
dtd = parse_dtd(self.get_dtd())
for filename in self.get_dico_file_names():
if filename in OLD_DICOS:
continue
parse = parse_xml_file(filename, dtd, parse_all=False)
for cond in parse['conditions'].values():
if cond[0]['name'].startswith('hidden_if_'):
ret.append(filename)
break
return ret
class ConditionWithoutTarget(CreoleLinter):
name = 'condition_without_target'
warnno = 5
warncomment = "Dictionnaire contenant une condition sans target"
def check(self):
ret = []
dtd = parse_dtd(self.get_dtd())
for filename in self.get_dico_file_names():
if filename in OLD_DICOS:
continue
parse = parse_xml_file(filename, dtd, parse_all=False)
for cond in parse['conditions'].values():
for con in cond:
if con['family'] == con['list'] == con['variable'] == []:
ret.append(filename)
break
return ret
class ObligatoireInDicosItem(CreoleLinter):
name = 'obligatoire_in_dicos'
warnno = 5
warncomment = "Dictionnaire contenant un check \"obligatoire\""
def check(self):
ret = []
dtd = parse_dtd(self.get_dtd())
for filename in self.get_dico_file_names():
if filename in OLD_DICOS:
continue
parse = parse_xml_file(filename, dtd, parse_all=False)
for cond in parse['checks'].values():
if cond[0][0] == 'obligatoire':
ret.append(filename)
break
return ret
class FamilyWithoutHelp(CreoleLinter):
name = 'family_without_help'
warnno = 5
warncomment = "Famille sans balise d'aide"
def check(self):
self.load_dics()
ret = []
for grp in self.config.creole.iter_groups():
doc = grp[1].cfgimpl_get_description().impl_get_information('help', None)
if doc is None:
ret.append(grp[0])
return ret
class FamilyWithoutIcon(CreoleLinter):
name = 'family_without_icon'
warnno = 5
warncomment = "Famille sans icône spécifique"
def check(self):
self.load_dics()
ret = []
for grp in self.config.creole.iter_groups():
if grp[1].cfgimpl_get_description().impl_get_information('icon') is None:
ret.append(grp[0])
return ret
#class DefineItem(CreoleLinter):
# """
# check for syntaxes
# """
# name = 'define'
# warnno = 4
# warncomment = "Redéfinition d'un variable d'un dictionnaire"
#
# def check(self):
# """
# verifie si une variable définie est une variable du dictionnaire
# """
# self.load_dics()
# self.load_tmpls()
# dicos = set(self.get_dicos_name())
# defv = set(self.get_defvars_name())
# ret=[]
# for var in defv & dicos:
# ret.append(self.defvars[var])
# return ret
#
class BuiltinsItem(CreoleLinter):
"""
verifier si une variable de dico n'est pas dans l'espace de nommage
"""
name = 'builtins'
warnno = 4
warncomment = "Variable identitique à une fonction python"
def check(self):
self.load_dics()
# self.load_tmpls()
ret = []
#dans le dictionnaire
for var in set(builts) & set(self.get_dicos_name()):
ret.append(self.variables[var])
# #dans les variables de template
# for var in set(builts) & set(self.get_tmplvars_name()):
# ret.append(self.tmplvars[var])
# #dans les boucles for
# for var in set(builts) & set(self.get_forvars_name()):
# ret.append(self.forvars[var])
# #dans la definition de variable dans un template
# for var in set(builts) & set(self.get_setvars_name()):
# ret.append(self.setvars[var])
# #dans les noms de fonction
# for var in set(builts) & set(self.get_defvars_name()):
# ret.append(self.defvars[var])
return ret
#class SyntaxForItem(CreoleLinter):
# """
# verifie la syntaxe de la ligne for
# """
# name = 'syntax_for'
# warnno = 1
# warncomment = "Syntaxe de la ligne for incorrect"
# def check(self):
# self.load_tmpls()
# return self.brokenfor
#
#class SyntaxVarItem(CreoleLinter):
# """
# verifie les variables suivant la syntaxe de pattern
# """
# name = 'syntax_var'
# pattern = '([a-z0-9][a-z0-9]+_[a-z0-9_]+)'
# warnno = 6
# warncomment = "La variable ne respecte pas la regexp %s" % pattern
#
# def check(self):
# cpattern = re.compile(self.pattern)
# self.load_dics()
# self.load_tmpls()
# ret=[]
# #dans le dictionnaire
# for var in self.get_dicos_name():
# if cpattern.match(var) == None:
# ret.append(self.variables[var])
# #dans les variables de template
# for var in self.get_tmplvars_name():
# if cpattern.match(var) == None:
# skipped_location = []
# for location in self.tmplvars[var].location:
# if self.skip_var.has_key(self.name) and self.skip_var[self.name].has_key(var):
# if not location in self.skip_var[self.name][var]:
# skipped_location.append(location)
# else:
# skipped_location.append(location)
# if skipped_location != []:
# tmplvar = TmplVar(var, '', 0)
# tmplvar.set_location(skipped_location)
# ret.append(tmplvar)
# #ret.append(self.tmplvars[var])
# #dans les boucles for
# for var in self.get_forvars_name():
# if cpattern.match(var) == None:
# ret.append(self.forvars[var])
# #dans la definition de variable dans un template
# for var in self.get_setvars_name():
# if cpattern.match(var) == None:
# ret.append(self.setvars[var])
# return ret
#
#class ForbiddenTemplateVarItem(CreoleLinter):
# """
# vérifie la présence des noms de variable interdits dans les templates
# """
# name = 'syntax_var2'
# warnno = 6
# warncomment = "Nom de variable interdit dans un template"
#
# def check(self):
# #self.load_dics()
# self.load_tmpls()
# ret=[]
# #dans les variables de template
# for var in self.get_tmplvars_name():
# for start in DICO_TEST_VARS:
# if var.startswith(start):
# ret.append(var)
# break
# return ret
#
#class SyntaxFunctionItem(CreoleLinter):
# """
# verifie les fonctions suivant la syntaxe de pattern
# """
# name = 'syntax_function'
# pattern = '([a-z0-9][a-z0-9]+_[a-z0-9_]+)'
# warnno = 6
# warncomment = "La fonction ne respecte pas la regexp %s" % pattern
#
# def check(self):
# cpattern = re.compile(self.pattern)
# self.load_tmpls()
# ret=[]
# #dans les noms de fonction
# for var in self.get_defvars_name():
# if cpattern.match(var) == None:
# ret.append(self.defvars[var])
# return ret
#
#class OrphansVarHelpItem(CreoleLinter):
# name = 'orphans_var_help'
# warnno = 3
# warncomment = "Aide définie dans le dictionnaire pour une variable inexistante"
#
# def check(self):
# self.load_dics()
# vars_name = set(self.get_dicos_name())
# vars_help = set(self.creoledic.get_helps()['variables'].keys())
# #print vars_help
# only_in_help = vars_help - vars_name
# results = []
# for tmpl in only_in_help:
# results.append(tmpl)
# return results
#class OrphansFamHelpItem(CreoleLinter):
# name = 'orphans_fam_help'
# warnno = 3
# warncomment = "Aide définie dans le dictionnaire pour une famille inexistante"
#
# def check(self):
# self.load_dics()
# #FIXME
# vars_name = set(self.families)
# vars_help = set(self.creoledic.get_helps()['families'].keys())
# only_in_help = vars_help - vars_name
# results = []
# for tmpl in only_in_help:
# results.append(tmpl)
# return results
#
class ValidVarLabelItem(CreoleLinter):
name = 'valid_var_label'
warnno = 5
warncomment = "Libellé de variable non valide dans un dictionnaire"
def check(self):
self.load_dics()
ret = []
for var in self.variables.values():
if not is_container_var(var.name):
ret.extend(is_correct(var.description, var.name))
return ret
class ActivationVarWithoutHelp(CreoleLinter):
name = 'activation_var_without_help'
warnno = 5
warncomment = "Variable d'activation sans balise d'aide"
def check(self):
self.load_dics()
ret = []
for var, var_obj in self.variables.items():
if var.startswith('activer_') and var not in EXCLUDE_ACTIVATION_VARS:
if var_obj.help is None:
ret.append(var)
return ret
class ValidSeparatorLabelItem(CreoleLinter):
name = 'valid_separator_label'
warnno = 5
warncomment = "Libellé de séparateur non valide dans un dictionnaire"
def check(self):
self.load_dics()
ret = []
for var, var_obj in self.variables.items():
if var_obj.separator == '':
#FIXME: variables de conteneur dynamiques
continue
ret.extend(is_correct(var_obj.separator[0], var))
return ret
class ValidHelpLabelItem(CreoleLinter):
name = 'valid_help_label'
warnno = 5
warncomment = "Libellé d'aide non valide dans un dictionnaire"
def check(self):
self.load_dics()
ret = []
for var, var_obj in self.variables.items():
# help/variable
ret.extend(is_correct(var_obj.help, var))
for grp in self.config.creole.iter_groups():
# help/family
ret.extend(is_correct(grp[1].cfgimpl_get_description().impl_get_information('help', ''),
grp[0], family=True))
return ret
class ValidSlaveValue(CreoleLinter):
name = 'valid_slave_value'
warnno = 5
warncomment = "Variable esclave avec une liste en valeur défaut"
def check(self):
self.load_dics()
ret = []
for var, var_obj in self.variables.items():
if var_obj.is_slave:
if len(var_obj.defaultvalue) > 1:
ret.append(var)
return ret
##class ValidCheckEnumOuiNon(CreoleLinter):
## name = 'valid_check_enum_ouinon'
## warnno = 6
## warncomment = "Variable avec un valid_enum à oui/non au lieu du type oui/non"
##
## def check(self):
## ret = []
## for var, content in self.creoledic.variables.items():
## print content
## if str(type(content)) != "<class 'creole.typeole.OuiNon'>":
## for check in content.choices:
## if check[0] == u'valid_enum':
## for valid_enum in check[1]:
## if valid_enum['value'].startswith('['):
## eval_valid_enum = eval(valid_enum['value'])
## if set(eval_valid_enum) == set(['non', 'oui']):
## ret.append(var)
## return ret
#
##class ValidCheckEnumOnOff(CreoleLinter):
## name = 'valid_check_enum_onoff'
## warnno = 6
## warncomment = "Variable avec un valid_enum à on/off au lieu du type on/off"
##
## def check(self):
## ret = []
## for var, content in self.creoledic.variables.items():
## if str(type(content)) != "<class 'creole.typeole.OnOff'>":
## for check in content.checks:
## if check[0] == u'valid_enum':
## for valid_enum in check[1]:
## if valid_enum['value'].startswith('['):
## eval_valid_enum = eval(valid_enum['value'])
## if set(eval_valid_enum) == set(['on', 'off']):
## ret.append(var)
## return ret
#
class TabsInDicosItem(CreoleLinter):
name = 'tabs_in_dicos'
warnno = 5
warncomment = "Tabulation dans le dictionnaire au lieu de 4 espaces"
def check(self):
return self._parse_tabs_in_dicos()
#class ValidClientOption(CreoleLinter):
# name = 'valid_client_option'
# warnno = 6
# warncomment = "Option inconnu pour %s" % cmd_client[0]
#
# def check(self):
# self.load_tmpls()
# return self.unknown_client
class ValidParseTmpl(CreoleLinter):
name = 'valid_parse_tmpl'
warnno = 1
warncomment = "Template Non valide"
display = False
def check(self):
parse_templates(self._list_tmpl_files())
return []
class ValidDTDItem(CreoleLinter):
name = 'valid_dtd'
warnno = 1
warncomment = "DTD Non valide"
display = False
def check(self):
dtd = self.get_dtd()
for filename in self.get_dico_file_names():
system_code(['xmllint', '--noout', '--dtdvalid', dtd, filename])
return []
class OldFwFile(CreoleLinter):
name = 'old_fw_file'
warnno = 5
warncomment = "Ancien fichier eole-firewall présent sur le serveur"
def check(self):
fw_templates = '/usr/share/eole/creole/distrib/*.fw'
fw_files = '/usr/share/eole/firewall/*.fw'
return glob(fw_files) + glob(fw_templates)
def validate(keyword, ansi, tmpl):
globs = globals()
classitem = None
for cls in globs:
if cls.startswith('_') and type(globs[cls])!=types.ClassType and cls == 'CreoleLinter' and cls == 'TmplVar':
continue
if hasattr(globs[cls], 'name'):
if globs[cls].name == keyword:
classitem = globs[cls]
break
if classitem == None:
raise Exception('test %s inconnu'%keyword)
cl = classitem()
if cl.eoledirs == None:
cl.set_config(tmpl_dir_or_file=tmpl)
ansi.process(cl)

132
creole/lint/entities.py
View File

@ -1,132 +0,0 @@
# -*- coding: utf-8 -*-
modules = [
'Zéphir',
'Eolebase',
'Amon',
'Sentinelle',
'Sphynx',
'Scribe',
'Eclair',
'Horus',
'ZéphirLog',
'PreludeManager',
'AmonEcole',
'EoleVZ',
'Seshat',
'ClientScribe',
'ClientHorus']
projets = [
' EOLE', # confusion avec eole-annuaire
'EAD',
'ead-web',
'ead-server',
'frontend',
'backend',
'Era',
'ESU',
'AGRIATES',
'RACINE-AGRIATES',
'Samba',
'Creole',
'GenConfig',
'EoleDB',
'EoleSSO',
'Zéphir',
"application Zéphir",
"Zéphir-web",
]
os_logiciels_protocoles = [
'Linux',
'GNU/Linux',
'Ubuntu',
'Unix',
'Windows',
'Microsoft',
# 'ALIAS',
'BlockInput',
'Epreuve@SSR',
'SSH',
'OpenSSH',
'DHCP',
'DHCPd',
'ClamAV',
'NuFW',
'NuWinC',
'Nuauth',
'DansGuardian',
'Bacula',
'Bareos',
'TCP',
'UDP',
'ICMP',
'IP',
' IPsec', # confusion avec la commande ipsec
'strongSwan',
'DMZ',
'FTP',
'SMB',
'XML',
'XML-RPC',
' SSO',
# 'CAS',
'SAML',
'Sympa',
'MySQL',
'SpamAssassin',
'web',
'phpMyAdmin',
'Grr',
'Gibii',
'Gepi',
'SPIP-EVA',
'Envole',
'Envole 2',
'WebShare',
' CSS', # confusion avec .css
'CUPS',
'OpenOffice.org',
'GDM',
'client léger',
'client lourd',
'OpenLDAP',
'ProFTPD',
'Vim',
'Controle-vnc',
'BE1D',
'RVP',
'PostgreSQL',
'Squid',
'NUT',
'PPPoE',
'VLAN',
'SSL',
'Nginx',
'reverse proxy',
'SquirrelMail',
'LDAP',
'FreeRADIUS',
'LightSquid',
'SARG',
'iptables',
'Netfilter',
'POSH',
'InterBase',
'OCS',
]
divers = [
'Éducation nationale',
'Internet',
'intranet',
'pare-feu',
'anti-virus',
'anti-spam',
'USB',
'relai',
]
entities = modules + projets + os_logiciels_protocoles + divers

82
creole/lint/error.py
View File

@ -1,82 +0,0 @@
"""errno-specific classes"""
import sys, os, errornb
class Error(EnvironmentError):
def __repr__(self):
return "%s.%s %r: %s " %(self.__class__.__module__,
self.__class__.__name__,
self.__class__.__doc__,
" ".join(map(str, self.args)),
#repr(self.args)
)
def __str__(self):
s = "[%s]: %s" %(self.__class__.__doc__,
" ".join(map(str, self.args)),
)
return s
#FIXME set the different error better suited than errno
_winerrnomap = {
2: errsymb.ENOENT,
3: errsymb.ENOENT,
17: errsymb.EEXIST,
22: errsymb.ENOTDIR,
267: errsymb.ENOTDIR,
5: errsymb.EACCES, # anything better?
}
class ErrorMaker(object):
""" lazily provides Exception classes for each possible POSIX errno
(as defined per the 'errno' module). All such instances
subclass EnvironmentError.
"""
Error = Error
_errno2class = {}
def __getattr__(self, name):
eno = getattr(errno, name)
cls = self._geterrnoclass(eno)
setattr(self, name, cls)
return cls
def _geterrnoclass(self, eno):
try:
return self._errno2class[eno]
except KeyError:
clsname = errno.errorcode.get(eno, "UnknownErrno%d" %(eno,))
errorcls = type(Error)(clsname, (Error,),
{'__module__':'py.error',
'__doc__': os.strerror(eno)})
self._errno2class[eno] = errorcls
return errorcls
def checked_call(self, func, *args):
""" call a function and raise an errno-exception if applicable. """
__tracebackhide__ = True
try:
return func(*args)
except self.Error:
raise
except EnvironmentError:
cls, value, tb = sys.exc_info()
if not hasattr(value, 'errno'):
raise
__tracebackhide__ = False
errno = value.errno
try:
if not isinstance(value, WindowsError):
raise NameError
except NameError:
# we are not on Windows, or we got a proper OSError
cls = self._geterrnoclass(errno)
else:
try:
cls = self._geterrnoclass(_winerrnomap[errno])
except KeyError:
raise value
raise cls("%s%r" % (func.__name__, args))
__tracebackhide__ = True
error = ErrorMaker()

76
creole/lint/normalize.py
View File

@ -1,76 +0,0 @@
# coding: utf-8
import re
import unicodedata
from entities import entities
# ______________________________________________________________________________
ENCODING = 'utf-8'
def strip_accents(string):
return unicodedata.normalize('NFKD', unicode(string, ENCODING)
).encode('ASCII', 'ignore')
def normalize_entities():
"""
enleve les accents de la liste des entites + minuscules
:return: entities normalisé
"""
norm_entities = []
for entitie in entities:
norm_entitie = strip_accents(entitie).lower()
norm_entities.append(norm_entitie)
return norm_entities
NORM_ENTITIES = normalize_entities()
# ______________________________________________________________________________
def parse_string(text):
"""
enlève les accents d'un texte
"""
# libelle = strip_accents(text)
words = re.findall('([a-zA-Zéèàùêôëö_]+)', text)
return words
def is_in_entities(text):
"""
donne l'index dans entities du texte
"""
norm_text = text.lower()
index = None
if norm_text in NORM_ENTITIES:
index = NORM_ENTITIES.index(norm_text)
return index
def is_correct(libelle, name, family=False):
if libelle is not None and type(libelle) != str:
libelle = unicode.encode(libelle, ENCODING)
ret = []
if libelle == '' or libelle is None:
return ret
if libelle[0].islower():
#FIXME: faux positifs connus
if not libelle.startswith('ejabberd') and \
not libelle.startswith('phpMyAdmin'):
ret.append('%%%%%s : phrase sans majuscule'%name)
for text in parse_string(libelle):
text_index = is_in_entities(text)
if not text_index == None:
if str(text) != str(entities[text_index]):
#FIXME: faux positifs connus
if 'ipsec.conf' in libelle or 'test-rvp' in libelle \
or 'bareos-' in libelle \
or 'bacula-' in libelle \
or '/var/log/zephir' in libelle \
or 'exemple : eolebase' in libelle:
continue
ent = str(unicode.encode((unicode(entities[text_index], ENCODING)), ENCODING))
if family:
ret.append('famille [%s] : %s => %s' % (str(name), text, ent))
else:
ret.append('%%%%%s : %s => %s' % (str(name), text, ent))
return ret
# ______________________________________________________________________________

660
creole/lint/parsetemplate.py
View File

@ -1,660 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
from os.path import basename
from creole.loader import creole_loader
from creole.client import CreoleClient
from creole.template import CreoleGet, IsDefined, CreoleTemplateEngine, CreoleMaster
from creole import eosfunc
from tiramisu.option import *
from tiramisu import Config
from tiramisu.error import ConfigError, PropertiesOptionError, \
RequirementError, ValueWarning
from Cheetah import Parser, Compiler
from Cheetah.Template import Template
from Cheetah.NameMapper import NotFound
from pyeole.ansiprint import print_red
from creole.eosfunc import valid_regexp
from Cheetah.Unspecified import Unspecified
import warnings
DEBUG = False
#DEBUG = True
client = CreoleClient()
compilerSettings = {'directiveStartToken' : u'%',
'cheetahVarStartToken' : u'%%', 'EOLSlurpToken' : u'%',
'PSPStartToken' : u'µ' * 10, 'PSPEndToken' : u'µ' * 10,
'commentStartToken' : u'µ' * 10, 'commentEndToken' : u'µ' * 10,
'multiLineCommentStartToken' : u'µ' * 10,
'multiLineCommentEndToken' : u'µ' * 10}
#======================= CHEETAH =======================
# This class is used to retrieve all template vars
#true_HighLevelParser = Parser._HighLevelParser
global cl_chunks, cl_vars
cl_chunks = set()
cl_vars = set()
class cl_Parser(Parser.Parser):
def getCheetahVarNameChunks(self, *args, **kwargs):
global cl_chunks
chunks = super(cl_Parser, self).getCheetahVarNameChunks(*args, **kwargs)
for chunk in chunks:
#if false, it's internal variable
if chunk[1]:
name = chunk[0]
#remove master if master/slave and add force adding master
if '.' in name:
cl_chunks.add(name.split('.')[-1])
cl_chunks.add(name.split('.')[0])
else:
cl_chunks.add(name)
return chunks
def getCheetahVar(self, *args, **kwargs):
global cl_vars
var = super(cl_Parser, self).getCheetahVar(*args, **kwargs)
if not var.startswith(u'VFFSL('):
cl_vars.add(var)
return var
def getVars():
global cl_chunks, cl_vars
#retrieve all calculated vars
ret = list(cl_chunks - cl_vars)
cl_chunks = set()
cl_vars = set()
return ret
class CompilerGetVars(Compiler.ModuleCompiler):
parserClass = cl_Parser
true_compile = Template.compile
@classmethod
def cl_compile(kls, *args, **kwargs):
kwargs['compilerClass'] = CompilerGetVars
kwargs['useCache'] = False
return true_compile(*args, **kwargs)
Template.compile = cl_compile
def CompilerGetVar(varName, default=Unspecified):
#remplace Cheetah's getVar function
#this function permite to known variable if getVar is used
if varName.startswith('%%'):
raise Exception('varname should not start with %% {0}'.format(varName))
global extra_vars, config
config.read_only()
try:
option = config.creole.find_first(byname=varName)
path = config.cfgimpl_get_description().impl_get_path_by_opt(option)
value = getattr(config, path)
except (AttributeError, ConfigError):
try:
option = config.creole.find_first(byname=varName, check_properties=False)
path = config.cfgimpl_get_description().impl_get_path_by_opt(option)
#populate_mandatory(config, option, path, raise_propertyerror=True)
config.read_write()
populate_mandatories()
config.read_only()
value = getattr(config, path)
except (AttributeError, RequirementError), err:
config.read_only()
#support default value
if default != Unspecified:
return default
else:
raise AttributeError('option:', varName, ':', err)
except PropertiesOptionError as err:
if default != Unspecified:
return default
else:
raise err
except Exception as err:
config.read_only()
raise err
except Exception as err:
config.read_only()
raise err
lpath = '.'.join(path.split('.')[2:])
dico = {lpath: value}
engine = CreoleTemplateEngine(force_values=dico)
name = path.split('.')[-1]
extra_vars[option] = name
if "." in lpath:
spath = lpath.split('.')
if spath[0] == spath[1]:
ret = engine.creole_variables_dict[name]
else:
ret = engine.creole_variables_dict[spath[0]].slave[spath[1]]
else:
ret = engine.creole_variables_dict[name]
return ret
def CompilerGetattr(creolemaster, name, default=None):
if not isinstance(creolemaster, CreoleMaster):
raise Exception('creolemaster must be CreoleMaster, not {0}'.format(type(creolemaster)))
if name not in creolemaster.slave:
#FIXME assume name is slave?
value = CompilerGetVar(name, default)
if creolemaster._index is not None:
value = value[creolemaster._index]
creolemaster.add_slave(name, value)
return getattr(creolemaster, name, default)
#======================= EOSFUNC =======================
eos = {}
for func in dir(eosfunc):
if not func.startswith('_'):
eos[func] = getattr(eosfunc, func)
#======================= CONFIG =======================
def populate_mandatory(config, option, path, raise_propertyerror=False):
def _build_network(path):
for num in range(0, 4):
if path.startswith('creole.interface_{0}'.format(num)):
return num
#si il y a un test de consistence de type _cons_in_network (l'IP doit être dans un network défini)
#on utilise le réseau de ce network #10714
if getattr(option, '_consistencies', None) is not None:
for const in option._consistencies:
if const[0] == '_cons_in_network':
try:
opt = const[1][1]
path = config.cfgimpl_get_description().impl_get_path_by_opt(opt)
val = config.getattr(path, force_permissive=True)
if isinstance(val, list):
val = val[0]
return val.split('.')[2]
except IndexError:
pass
return 5
def _build_ip(path):
if path.endswith('_fichier_link'):
return 3
elif path.endswith('_proxy_link'):
return 2
else:
#ne pas retourner la même valeur si elle est censé être différente
if getattr(option, '_consistencies', None) is not None:
for const in option._consistencies:
if const[0] == '_cons_not_equal':
return 4
return 1
if option.impl_getname().startswith('nom_carte_eth'):
value = unicode(option.impl_getname())
elif isinstance(option, UnicodeOption):
value = u'value'
elif isinstance(option, IPOption):
value = u'192.168.{0}.{1}'.format(_build_network(path), _build_ip(path))
elif isinstance(option, NetworkOption):
value = u'192.168.{0}.0'.format(_build_network(path))
elif isinstance(option, NetmaskOption):
value = u'255.255.255.0'
elif isinstance(option, BroadcastOption):
value = u'192.168.{0}.255'.format(_build_network(path))
elif isinstance(option, EmailOption):
value = u'foo@bar.com'
elif isinstance(option, URLOption):
value = u'http://foo.com/bar'
elif isinstance(option, DomainnameOption):
allow_without_dot = option._get_extra('_allow_without_dot')
o_type = option._get_extra('_dom_type')
if option._name == 'smb_workgroup':
value = u'othervalue'
elif o_type in ['netbios', 'hostname']:
value = u'value'
else:
value = u'value.lan'
elif isinstance(option, FilenameOption):
value = u'/tmp/foo'
elif isinstance(option, ChoiceOption):
#FIXME devrait le faire tout seul non ?
value = option.impl_get_values(config)[0]
elif isinstance(option, IntOption):
value = 1
elif isinstance(option, PortOption):
value = 80
elif isinstance(option, DomainnameOption):
value = 'foo.com'
elif isinstance(option, UsernameOption):
value = 'toto'
elif isinstance(option, PasswordOption):
value = 'P@ssWord'
else:
raise Exception('the Tiramisu type {0} is not supported by CreoleLint (variable : {1})'.format(type(option), path))
validator = option.impl_get_validator()
if validator is not None and validator[0] == valid_regexp:
regexp = validator[1][''][0]
# génération d'une "value" valide
# en cas de valid_regexp sans valeur par défaut
if regexp == u'^[A-Z][0-9]$':
value = u'A1'
elif option._name == 'additional_repository_source':
# variable avec expression (très) spécifique #20291
value = u"deb http://test dist"
elif not regexp.startswith(u'^[a-z0-9]') and regexp.startswith('^'):
value = regexp[1:]
if option.impl_is_multi():
if option.impl_is_master_slaves('slave'):
#slave should have same length as master
masterpath = '.'.join(path.split('.')[:-1]+[path.split('.')[-2]])
try:
len_master = len(getattr(config, masterpath))
val = []
for i in range(0, len_master):
val.append(value)
value = val
except:
value = [value]
else:
value = [value]
try:
config.setattr(path, value, force_permissive=True)
except ValueError, err:
msg = str('error for {0} type {1}: {2}'.format(path, type(option), err))
raise Exception(msg)
except PropertiesOptionError, err:
if 'frozen' not in err.proptype:
if raise_propertyerror:
raise err
msg = str('error for {0} type {1}: {2}'.format(path, type(option), err))
raise Exception(msg)
class Reload(Exception):
pass
class Check_Template:
def __init__(self, template_name):
self.all_requires = {}
self.current_opt = {}
self.od_list = {}
global extra_vars
#reinit extra_vars
extra_vars = {}
self.old_dico = []
self.current_var = []
self.ori_options = []
self.file_path = None
self.template_name = template_name
self.current_container = client.get_container_infos('mail')
self.tmpl = None
self.is_tmpl = False
self.filename_ok = False
def populate_requires(self, option, path, force=False):
def _parse_requires(_option):
o_requires = _option.impl_getrequires()
if o_requires is not None:
for requires in o_requires:
for require in requires:
opt_ = require[0]
path_ = config.cfgimpl_get_description().impl_get_path_by_opt(opt_)
self.populate_requires(opt_, path_, force=True)
if not force and not path.startswith('creole.'):
return
if option in self.current_opt:
return
o_requires = option.impl_getrequires()
if o_requires is not None:
for requires in o_requires:
for require in requires:
if require[0].impl_is_master_slaves('slave'):
path_ = config.cfgimpl_get_description().impl_get_path_by_opt(require[0])
s_path = path_.split('.')
master_path = 'creole.' + s_path[1] + '.' + s_path[2] + '.' + s_path[2]
try:
opt_master = config.unwrap_from_path(master_path)
config.cfgimpl_get_settings().remove('everything_frozen')
populate_mandatory(config, opt_master, master_path)
except:
pass
self.all_requires.setdefault(option, []).append(require[0])
if isinstance(option, OptionDescription):
self.od_list[path] = option
if force and not option._name in self.current_var:
self.current_var.append(option._name)
if option._name in self.current_var or not path.startswith('creole.'):
if not isinstance(option, OptionDescription):
if path.startswith('creole.'):
self.current_opt[option] = '.'.join(path.split('.')[1:])
else:
self.current_opt[option] = None
_parse_requires(option)
#requires could be in parent's too
opath = ''
for parent in path.split('.')[:-1]:
opath += parent
if opath in self.od_list:
desc = self.od_list[opath]
self.current_opt[desc] = None
_parse_requires(desc)
opath += '.'
try:
if option._callback is not None:
for params in option._callback[1].values():
for param in params:
if isinstance(param, tuple):
opt = param[0]
path = config.cfgimpl_get_description().impl_get_path_by_opt(opt)
self.populate_requires(opt, path, force=True)
except (AttributeError, KeyError):
pass
def read_write(self):
config.read_write()
config.cfgimpl_get_settings().remove('disabled')
config.cfgimpl_get_settings().remove('hidden')
config.cfgimpl_get_settings().remove('frozen')
def change_value(self, path, value, multi, parse_message, option):
self.read_write()
config.cfgimpl_get_settings()[option].remove('force_default_on_freeze')
if multi:
if option.impl_is_master_slaves('slave'):
s_path = path.split('.')
master_path = s_path[0] + '.' + s_path[1] + '.' + s_path[2] + '.' + s_path[2]
master_option = config.cfgimpl_get_description().impl_get_opt_by_path(master_path)
if getattr(config, master_path) == []:
populate_mandatory(config, master_option, master_path)
value = [value]
if parse_message:
print parse_message, value
setattr(config, path, value)
config.read_only()
def template(self):
self.last_notfound = []
def get_value(opt_, path_):
try:
return getattr(config.creole, path_)
except PropertiesOptionError, err:
if err.proptype == ['mandatory']:
self.read_write()
config.cfgimpl_get_settings().remove('mandatory')
s_path = path_.split('.')
#set value to master
if len(s_path) == 3 and s_path[1] != s_path[2]:
master_path = 'creole.' + s_path[0] + '.' + s_path[1] + '.' + s_path[1]
opt_master = config.unwrap_from_path(master_path)
populate_mandatory(config, opt_master, master_path)
populate_mandatory(config, opt_, 'creole.' + path_)
config.read_only()
config.cfgimpl_get_settings().remove('mandatory')
try:
ret = getattr(config.creole, path_)
config.cfgimpl_get_settings().append('mandatory')
return ret
except PropertiesOptionError:
pass
raise NotFound('no value')
except ConfigError:
self.read_write()
populate_mandatory(config, opt_, 'creole.' + path_)
config.read_only()
try:
return getattr(config.creole, path_)
except ConfigError, err:
raise err
except PropertiesOptionError, err:
raise NotFound('no value')
try:
is_gen_file = getattr(config, self.file_path)
except PropertiesOptionError, err:
is_gen_file = False
if not is_gen_file:
return
try:
config.read_write()
populate_mandatories()
config.read_only()
dico = {}
for opt_, path_ in self.current_opt.items():
#path_ is None if it's an OptionDescription
if path_ is None:
continue
try:
dico[path_] = get_value(opt_, path_)
except NotFound:
pass
#FIXME revoir le strip_full_path
ndico = {}
for path_, value in dico.items():
sdico = path_.split('.')
if len(sdico) == 2:
ndico[sdico[1]] = value
elif len(sdico) == 3:
if sdico[1] == sdico[2]:
ndico[sdico[1]] = value
else:
ndico['.'.join(sdico[1:])] = value
else:
raise Exception('chemin de longueur inconnu {}'.format(path_))
engine = CreoleTemplateEngine(force_values=ndico)
dico = engine.creole_variables_dict
self.read_write()
except ConfigError, err:
msg = 'erreur de templating', err
raise ValueError(msg)
diff = True
for old in self.old_dico:
if dico.keys() == old.keys():
for key in old.keys():
if old[key] != dico[key]:
diff = False
break
if not diff:
break
if not diff:
return
try:
self.old_dico.append(dico)
searchlist = [dico, eos, {'is_defined' : IsDefined(dico),
'creole_client' : CreoleClient(),
'current_container': CreoleGet(self.current_container),
}]
rtmpl = self.tmpl(searchList=searchlist)
rtmpl.getVar = CompilerGetVar
rtmpl.getattr = CompilerGetattr
rtmpl = str(rtmpl)
#print rtmpl
self.is_tmpl = True
except NotFound, err:
lst = getVars()
if lst == []:
raise Exception("Il manque une option", err, 'avec le dictionnaire', dico)
for ls in lst:
try:
CompilerGetVar(ls)
except AttributeError:
self.last_notfound.append(ls)
raise Reload('')
except Exception, err:
raise Exception("Il y a une erreur", err, 'avec le dictionnaire', dico)
def check_reload_with_extra(self):
#if extra_vars has value, check if not already in current_opt
global extra_vars
if extra_vars != {}:
oret = set(extra_vars.keys())
opt_requires = oret & set(self.all_requires.keys())
for opt_ in opt_requires:
oret.update(self.all_requires[opt_])
dont_exists = set(oret) - set(self.current_opt.keys())
ret = []
for opt_ in dont_exists:
try:
ret.append(extra_vars[opt_])
except KeyError:
ret.append(opt_._name)
extra_vars = {}
if ret == []:
return None
return ret
def test_all_values_for(self, options, cpt):
option = options[0]
parse_message = None
if DEBUG:
parse_message = '*' * cpt + '>' + option._name
if not isinstance(option, ChoiceOption):
msg = str('pas simple la... ' + option._name)
raise NotImplementedError(msg)
multi = option.impl_is_multi()
path = config.cfgimpl_get_description().impl_get_path_by_opt(option)
for value in option.impl_get_values(config):
self.change_value(path, value, multi, parse_message, option)
if options[1:] != []:
#if already value to test, restart test_all_values_for
ret = self.test_all_values_for(options[1:], cpt + 1)
if ret != None:
return ret
else:
need_reload = False
try:
self.template()
except Reload:
need_reload = True
ret = self.check_reload_with_extra()
if need_reload and ret is None:
notfound = []
paths = config.cfgimpl_get_description()._cache_paths[1]
for ls in self.last_notfound:
#if variable is locale (means template) variable, not config's one
for path in paths:
if path.endswith('.' + ls):
notfound.append(ls)
break
if notfound != []:
raise Exception('variable not found after reload {0}'.format(notfound))
if ret is not None:
return ret
def open_file(self, force_var):
# Open template and compile it
# retrieve template vars (add force_var if needed)
filecontent = open(self.template_name).read()
#try to convert content in unicode
self.tmpl = Template.compile(filecontent, compilerSettings=compilerSettings) # ,
#compilerClass=CompilerGetVars)
self.current_var = getVars()
if force_var:
self.current_var.extend(force_var)
def populate_file(self, path, option):
if path.startswith('containers.files.file'):
if path.endswith('.source') and option.impl_getdefault().endswith('/{0}'.format(self.template_name.split('/')[-1])):
self.filename_ok = True
if self.filename_ok and path.endswith('.activate'):
self.file_path = path
self.filename_ok = False
self.populate_requires(option, path, force=True)
def test_all_values(self):
try:
options = list(set(self.all_requires.keys())&set(self.current_opt.keys()))
need_tmpl = False
if options != []:
requires_options = set()
for opt in options:
for op in self.all_requires[opt]:
if 'frozen' not in config.cfgimpl_get_settings()[op]:
requires_options.add(op)
if requires_options == set([]):
need_tmpl = True
else:
self.ori_options = requires_options
ret = self.test_all_values_for(list(requires_options), 0)
if ret is not None:
if DEBUG:
print "reload with", ret
self.check_template(ret, already_load=True)
else:
need_tmpl = True
if need_tmpl is True:
try:
self.template()
except:
self.test_all_values()
except Exception, err:
if DEBUG:
import traceback
traceback.print_exc()
msg = self.template_name, ':', err
raise Exception(msg)
def check_template(self, force_var=None, already_load=False):
#remove all modification (value, properties, ...)
open_error = None
try:
self.open_file(force_var)
except Exception, err:
open_error = "problème à l'ouverture du fichier {}".format(self.template_name)
config.read_only()
for index, option in enumerate(config.cfgimpl_get_description()._cache_paths[0]):
path = config.cfgimpl_get_description()._cache_paths[1][index]
self.populate_file(path, option)
self.populate_requires(option, path)
if self.file_path is None:
if open_error is not None:
print "le fichier {0} non présent dans un dictionnaire a un problème : {1}".format(basename(self.template_name),
open_error)
else:
print " \\-- fichier non présent dans un dictionnaire {0}".format(self.template_name)
return
if open_error is not None:
raise Exception(open_error)
if not already_load:
print " \\--", self.template_name
self.test_all_values()
if not self.is_tmpl:
print "pas de templating !"
def populate_mandatories():
for path in config.cfgimpl_get_values().mandatory_warnings(config):
if path.startswith('creole.'):
option = config.cfgimpl_get_description().impl_get_opt_by_path(path)
try:
populate_mandatory(config, option, path)
except PropertiesOptionError:
pass
def parse_templates(templates_name):
global config, cl_chunks, cl_vars, extra_vars
config = creole_loader(load_values=False, load_extra=True)
config.read_write()
populate_mandatories()
cfg = config
for template_name in templates_name:
cl_chunks = set()
cl_vars = set()
extra_vars = {}
config = cfg.duplicate()
config.read_write()
populate_mandatories()
ctmpl = Check_Template(template_name)
try:
ctmpl.check_template()
except Exception, err:
if DEBUG:
import traceback
traceback.print_exc()
print_red(str(err))
sys.exit(1)

125
creole/lint/terminalreport.py
View File

@ -1,125 +0,0 @@
"""
Implements terminal reporting of the full validation process.
Implements the various reporting hooks.
XXX: Currently in progress, NOT IN WORKING STATE.
"""
import sys
def pytest_addoption(parser):
group = parser.getgroup("terminal reporting", after="general")
group._addoption('-v', '--verbose', action="count",
dest="verbose", default=0, help="increase verbosity."),
group.addoption('--report',
action="store", dest="report", default=None, metavar="opts",
help="comma separated options, valid: skipped,xfailed")
group._addoption('--fulltrace',
action="store_true", dest="fulltrace", default=False,
help="don't cut any tracebacks (default is to cut).")
group.addoption('--traceconfig',
action="store_true", dest="traceconfig", default=False,
help="trace considerations of conftest.py files."),
class TerminalReporter:
def __init__(self, config, file=None):
self.config = config
self.stats = {}
self.curdir = py.path.local()
if file is None:
file = sys.stdout
self._tw = TerminalWriter(file)
self.currentfspath = None
self._reportopt = getreportopt(config.getvalue('report'))
def hasopt(self, name):
return self._reportopt.get(name, False)
def write_fspath_result(self, fspath, res):
fspath = self.curdir.bestrelpath(fspath)
if fspath != self.currentfspath:
self._tw.line()
relpath = self.curdir.bestrelpath(fspath)
self._tw.write(relpath + " ")
self.currentfspath = fspath
self._tw.write(res)
def write_ensure_prefix(self, prefix, extra="", **kwargs):
if self.currentfspath != prefix:
self._tw.line()
self.currentfspath = prefix
self._tw.write(prefix)
if extra:
self._tw.write(extra, **kwargs)
self.currentfspath = -2
def ensure_newline(self):
if self.currentfspath:
self._tw.line()
self.currentfspath = None
def write_line(self, line, **markup):
line = str(line)
self.ensure_newline()
self._tw.line(line, **markup)
def write_sep(self, sep, title=None, **markup):
self.ensure_newline()
self._tw.sep(sep, title, **markup)
def getoutcomeword(self, rep):
if rep.passed:
return "PASS", dict(green=True)
elif rep.failed:
return "FAIL", dict(red=True)
elif rep.skipped:
return "SKIP"
else:
return "???", dict(red=True)
#
# summaries for sessionfinish
#
def summary_failures(self):
if 'failed' in self.stats and self.config.option.tbstyle != "no":
self.write_sep("=", "FAILURES")
for rep in self.stats['failed']:
msg = self._getfailureheadline(rep)
self.write_sep("_", msg)
self.write_platinfo(rep)
rep.toterminal(self._tw)
def summary_errors(self):
if 'error' in self.stats and self.config.option.tbstyle != "no":
self.write_sep("=", "ERRORS")
for rep in self.stats['error']:
msg = self._getfailureheadline(rep)
if not hasattr(rep, 'when'):
# collect
msg = "ERROR during collection " + msg
elif rep.when == "setup":
msg = "ERROR at setup of " + msg
elif rep.when == "teardown":
msg = "ERROR at teardown of " + msg
self.write_sep("_", msg)
self.write_platinfo(rep)
rep.toterminal(self._tw)
def summary_stats(self):
session_duration = py.std.time.time() - self._sessionstarttime
keys = "failed passed skipped deselected".split()
for key in self.stats.keys():
if key not in keys:
keys.append(key)
parts = []
for key in keys:
val = self.stats.get(key, None)
if val:
parts.append("%d %s" %(len(val), key))
line = ", ".join(parts)
# XXX coloring
self.write_sep("=", "%s in %.2f seconds" %(line, session_duration))

139
creole/lint/terminalwriter.py
View File

@ -1,139 +0,0 @@
# -*- coding: utf-8 -*-
"""
Helper functions for writing to terminals and files.
XXX: Currently in progress, NOT IN WORKING STATE.
"""
import sys, os
def _getdimensions():
import termios,fcntl,struct
call = fcntl.ioctl(0,termios.TIOCGWINSZ,"\000"*8)
height,width = struct.unpack( "hhhh", call ) [:2]
return height, width
def get_terminal_width():
try:
height, width = _getdimensions()
except (SystemExit, KeyboardInterrupt):
raise
except:
# FALLBACK
width = int(os.environ.get('COLUMNS', 80))-1
# XXX the windows getdimensions may be bogus, let's sanify a bit
width = max(width, 40) # we alaways need 40 chars
return width
terminal_width = get_terminal_width()
# XXX unify with _escaped func below
def ansi_print(text, file=None, newline=True, flush=False):
if file is None:
file = sys.stderr
text = text.strip()
if newline:
text += '\n'
file.write(text)
if flush:
file.flush()
if file:
file.close()
def should_do_markup(file):
return hasattr(file, 'isatty') and file.isatty() \
and os.environ.get('TERM') != 'dumb'
class TerminalWriter(object):
_esctable = dict(black=30, red=31, green=32, yellow=33,
blue=34, purple=35, cyan=36, white=37,
Black=40, Red=41, Green=42, Yellow=43,
Blue=44, Purple=45, Cyan=46, White=47,
bold=1, light=2, blink=5, invert=7)
def __init__(self, file=None, encoding=None):
self.encoding = encoding
if file is None:
file = sys.stdout
elif hasattr(file, '__call__'):
file = WriteFile(file, encoding=encoding)
self._file = file
self.fullwidth = get_terminal_width()
self.hasmarkup = should_do_markup(file)
def _escaped(self, text, esc):
if esc and self.hasmarkup:
text = (''.join(['\x1b[%sm' % cod for cod in esc]) +
text +'\x1b[0m')
return text
def markup(self, text, **kw):
esc = []
for name in kw:
if name not in self._esctable:
raise ValueError("unknown markup: %r" %(name,))
if kw[name]:
esc.append(self._esctable[name])
return self._escaped(text, tuple(esc))
def sep(self, sepchar, title=None, fullwidth=None, **kw):
if fullwidth is None:
fullwidth = self.fullwidth
# the goal is to have the line be as long as possible
# under the condition that len(line) <= fullwidth
if title is not None:
# we want 2 + 2*len(fill) + len(title) <= fullwidth
# i.e. 2 + 2*len(sepchar)*N + len(title) <= fullwidth
# 2*len(sepchar)*N <= fullwidth - len(title) - 2
# N <= (fullwidth - len(title) - 2) // (2*len(sepchar))
N = (fullwidth - len(title) - 2) // (2*len(sepchar))
fill = sepchar * N
line = "%s %s %s" % (fill, title, fill)
else:
# we want len(sepchar)*N <= fullwidth
# i.e. N <= fullwidth // len(sepchar)
line = sepchar * (fullwidth // len(sepchar))
# in some situations there is room for an extra sepchar at the right,
# in particular if we consider that with a sepchar like "_ " the
# trailing space is not important at the end of the line
if len(line) + len(sepchar.rstrip()) <= fullwidth:
line += sepchar.rstrip()
self.line(line, **kw)
def write(self, s, **kw):
if s:
s = self._getbytestring(s)
if self.hasmarkup and kw:
s = self.markup(s, **kw)
self._file.write(s)
self._file.flush()
def _getbytestring(self, s):
# XXX review this and the whole logic
if self.encoding and sys.version_info < (3,0) and isinstance(s, unicode):
return s.encode(self.encoding)
elif not isinstance(s, str):
return str(s)
return s
def line(self, s='', **kw):
self.write(s, **kw)
self.write('\n')
class WriteFile(object):
def __init__(self, writemethod, encoding=None):
self.encoding = encoding
self._writemethod = writemethod
def write(self, data):
if self.encoding:
data = data.encode(self.encoding)
self._writemethod(data)
def flush(self):
return

34
creole/lint/warning.py
View File

@ -1,34 +0,0 @@
# -*- coding: utf-8 -*-
import sys
from creole.lint import warnsymb
class Warn:
def __init__(self, write_level, itemname, warnno, comment, checks):
self.warnno = warnno
self.comment = comment
self.checks = checks
self.write_level = getattr(warnsymb, write_level)
def to_dict(self):
"""
formats a msg warn directly from a warning message
"""
dico_loc = {}
for var in self.checks:
if hasattr(var, 'location'):
locs = var.location
for vfile, vline in locs:
if vfile == 'dictionnaire':
raise Exception('vfile ne doit pas se nommer dictionnaire !!!')
if not dico_loc.has_key(vfile):
dico_loc[vfile] = []
dico_loc[vfile].append((vline, var))
else:
if not dico_loc.has_key('dictionnaire'):
dico_loc['dictionnaire'] = []
dico_loc['dictionnaire'].append((None, var))
# ret = ["[%s:%s:%s] %s : %s (dictionnaire)" %(level, name, self.itemname, self.comment, vname)]
return dico_loc

33
creole/lint/warnsymb.py
View File

@ -1,33 +0,0 @@
# -*- coding: utf-8 -*-
"""
Standard errno symbols
"""
"""Dictionary providing a mapping from the errno value to the string
name in the underlying waring. For instance,
errno.errorcode[errno.EPERM] maps to 'EPERM'."""
errorlevel = {
1: 'error',
2: 'warning',
3: 'info',
}
errorcode = {
1: ('ERROR', 1),
2: ('WARNING', 2),
3: ('INFO', 3),
4: ('NAME', 1),
5: ('NAME', 2),
6: ('NAME', 3),
7: ('UNUSED', 1),
8: ('UNUSED', 2),
9: ('UNUSED', 3),
}
globs = globals()
for key, value in errorlevel.items():
globs[value] = key

607
creole/template.py
View File

@ -1,607 +0,0 @@
# -*- coding: utf-8 -*-
"""
Gestion du mini-langage de template
On travaille sur les fichiers cibles
"""
import sys
import shutil
import logging
import traceback
import os
from os import listdir, unlink
from os.path import basename, join
from tempfile import mktemp
from Cheetah import Parser
# l'encoding du template est déterminé par une regexp (encodingDirectiveRE dans Parser.py)
# il cherche un ligne qui ressemble à '#encoding: utf-8
# cette classe simule le module 're' et retourne toujours l'encoding utf-8
# 6224
class FakeEncoding():
def groups(self):
return ('utf-8',)
def search(self, *args):
return self
Parser.encodingDirectiveRE = FakeEncoding()
from Cheetah.Template import Template as ChtTemplate
from Cheetah.NameMapper import NotFound as CheetahNotFound
from config import patch_dir, templatedir, distrib_dir
from .client import CreoleClient, CreoleClientError
from .error import FileNotFound, TemplateError, TemplateDisabled
import eosfunc
from .i18n import _
import pyeole
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
class IsDefined(object):
"""
filtre permettant de ne pas lever d'exception au cas où
la variable Creole n'est pas définie
"""
def __init__(self, context):
self.context = context
def __call__(self, varname):
if '.' in varname:
splitted_var = varname.split('.')
if len(splitted_var) != 2:
msg = _(u"Group variables must be of type master.slave")
raise KeyError(msg)
master, slave = splitted_var
if master in self.context:
return slave in self.context[master].slave.keys()
return False
else:
return varname in self.context
class CreoleGet(object):
def __init__(self, context):
self.context = context
def __call__(self, varname):
return self.context[varname]
def __getitem__(self, varname):
"""For bracket and dotted notation
"""
return self.context[varname]
def __contains__(self, varname):
"""Check variable existence in context
"""
return varname in self.context
@classmethod
def cl_compile(kls, *args, **kwargs):
kwargs['compilerSettings'] = {'directiveStartToken' : u'%',
'cheetahVarStartToken' : u'%%',
'EOLSlurpToken' : u'%',
'PSPStartToken' : u'µ' * 10,
'PSPEndToken' : u'µ' * 10,
'commentStartToken' : u'µ' * 10,
'commentEndToken' : u'µ' * 10,
'multiLineCommentStartToken' : u'µ' * 10,
'multiLineCommentEndToken' : u'µ' * 10}
return kls.old_compile(*args, **kwargs)
ChtTemplate.old_compile = ChtTemplate.compile
ChtTemplate.compile = cl_compile
class CheetahTemplate(ChtTemplate):
"""classe pour personnaliser et faciliter la construction
du template Cheetah
"""
def __init__(self, filename, context, current_container):
"""Initialize Creole CheetahTemplate
@param filename: name of the file to process
@type filename: C{str}
@param context: flat dictionary of creole variables as 'name':'value',
@type context: C{dict}
@param current_container: flat dictionary describing the current container
@type current_container: C{dict}
"""
eos = {}
for func in dir(eosfunc):
if not func.startswith('_'):
eos[func] = getattr(eosfunc, func)
# ajout des variables decrivant les conteneurs
#FIXME chercher les infos dans le client !
ChtTemplate.__init__(self, file=filename,
searchList=[context, eos, {u'is_defined' : IsDefined(context),
u'creole_client' : CreoleClient(),
u'current_container':CreoleGet(current_container),
}])
class CreoleMaster(object):
def __init__(self, value, slave=None, index=None):
"""
On rend la variable itérable pour pouvoir faire:
for ip in iplist:
print ip.network
print ip.netmask
print ip
index is used for CreoleLint
"""
self._value = value
if slave is not None:
self.slave = slave
else:
self.slave = {}
self._index = index
def __getattr__(self, name):
"""Get slave variable or attribute of master value.
If the attribute is a name of a slave variable, return its value.
Otherwise, returns the requested attribute of master value.
"""
if name in self.slave:
value = self.slave[name]
if isinstance(value, Exception):
raise value
return value
else:
return getattr(self._value, name)
def __getitem__(self, index):
"""Get a master.slave at requested index.
"""
ret = {}
for key, values in self.slave.items():
ret[key] = values[index]
return CreoleMaster(self._value[index], ret, index)
def __iter__(self):
"""Iterate over master.slave.
Return synchronised value of master.slave.
"""
for i in range(len(self._value)):
ret = {}
for key, values in self.slave.items():
ret[key] = values[i]
yield CreoleMaster(self._value[i], ret, i)
def __len__(self):
"""Delegate to master value
"""
return len(self._value)
def __repr__(self):
"""Show CreoleMaster as dictionary.
The master value is stored under 'value' key.
The slaves are stored under 'slave' key.
"""
return repr({u'value': self._value, u'slave': self.slave})
def __eq__(self, value):
return value == self._value
def __ne__(self, value):
return value != self._value
def __lt__(self, value):
return self._value < value
def __le__(self, value):
return self._value <= value
def __gt__(self, value):
return self._value > value
def __ge__(self, value):
return self._value >= value
def __str__(self):
"""Delegate to master value
"""
return str(self._value)
def __add__(self, val):
return self._value.__add__(val)
def __radd__(self, val):
return val + self._value
def __contains__(self, item):
return item in self._value
def add_slave(self, name, value):
"""Add a slave variable
Minimal check on type and value of the slave in regards to the
master one.
@param name: name of the slave variable
@type name: C{str}
@param value: value of the slave variable
"""
if isinstance(self._value, list):
if not isinstance(value, list):
raise TypeError
elif len(value) != len(self._value):
raise ValueError(_(u'length mismatch'))
new_value = []
for val in value:
if isinstance(val, dict):
new_value.append(ValueError(val['err']))
else:
new_value.append(val)
value = new_value
elif isinstance(value, list):
raise TypeError
self.slave[name] = value
class CreoleTemplateEngine(object):
"""Engine to process Creole cheetah template
"""
def __init__(self, force_values=None):
#force_values permit inject value and not used CreoleClient (used by CreoleLint)
self.client = CreoleClient()
self.creole_variables_dict = {}
self.force_values = force_values
self.load_eole_variables()
def load_eole_variables(self):
# remplacement des variables EOLE
self.creole_variables_dict = {}
if self.force_values is not None:
values = self.force_values
else:
values = self.client.get_creole()
for varname, value in values.items():
if varname in self.creole_variables_dict:
# Creation of a slave create the master
continue
if varname.find('.') != -1:
#support des groupes
mastername, slavename = varname.split('.')
if not mastername in self.creole_variables_dict or not \
isinstance(self.creole_variables_dict [mastername],
CreoleMaster):
# Create the master variable
if mastername in values:
self.creole_variables_dict[mastername] = CreoleMaster(values[mastername])
else:
#only for CreoleLint
self.creole_variables_dict[mastername] = CreoleMaster(value)
#test only for CreoleLint
if mastername != slavename:
self.creole_variables_dict[mastername].add_slave(slavename, value)
else:
self.creole_variables_dict[varname] = value
def patch_template(self, filevar, force_no_active=False):
"""Apply patch to a template
"""
var_dir = os.path.join(patch_dir,'variante')
patch_cmd = ['patch', '-d', templatedir, '-N', '-p1']
patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']
tmpl_filename = os.path.split(filevar[u'source'])[1]
# patches variante + locaux
for directory in [var_dir, patch_dir]:
patch_file = os.path.join(directory, '{0}.patch'.format(tmpl_filename))
if os.access(patch_file, os.F_OK):
msg = _(u"Patching template '{0}' with '{1}'")
log.info(msg.format(filevar[u'source'], patch_file))
ret, out, err = pyeole.process.system_out(patch_cmd + patch_no_debug + ['-i', patch_file])
if ret != 0:
msg = _(u"Error applying patch: '{0}'\nTo reproduce and fix this error {1}")
log.error(msg.format(patch_file, ' '.join(patch_cmd + ['-i', patch_file])))
#8307 : recopie le template original et n'arrête pas le processus
self._copy_to_template_dir(filevar, force_no_active)
#raise TemplateError(msg.format(patch_file, err))
def strip_template_comment(self, filevar):
"""Strip comment from template
This apply if filevar has a del_comment attribut
"""
# suppression des commentaires si demandé (attribut del_comment)
strip_cmd = ['sed', '-i']
if u'del_comment' in filevar and filevar[u'del_comment'] != '':
log.info(_(u"Cleaning file '{0}'").format( filevar[u'source'] ))
ret, out, err = pyeole.process.system_out(strip_cmd
+ ['/^\s*{0}/d ; /^$/d'.format(filevar[u'del_comment']),
filevar[u'source'] ])
if ret != 0:
msg = _(u"Error removing comments '{0}': {1}")
raise TemplateError(msg.format(filevar[u'del_comment'], err))
def _check_filevar(self, filevar, force_no_active=False):
"""Verify that filevar is processable
:param filevar: template file informations
:type filevar: `dict`
:raise CreoleClientError: if :data:`filevar` is disabled
inexistant or unknown.
"""
if not force_no_active and (u'activate' not in filevar or not filevar[u'activate']):
raise CreoleClientError(_(u"Template file not enabled:"
u" {0}").format(basename(filevar[u'source'])))
if u'source' not in filevar or filevar[u'source'] is None:
raise CreoleClientError(_(u"Template file not set:"
u" {0}").format(basename(filevar['source'])))
if u'name' not in filevar or filevar[u'name'] is None:
raise CreoleClientError(_(u"Template target not set:"
u" {0}").format(basename(filevar[u'source'])))
def _copy_to_template_dir(self, filevar, force_no_active=False):
"""Copy template to processing temporary directory.
:param filevar: template file informations
:type filevar: `dict`
:param force_no_active: copy disabled template if `True`
:type filevar: `bool`
:raise FileNotFound: if source template does not exist
"""
self._check_filevar(filevar, force_no_active)
tmpl_source_name = os.path.split(filevar[u'source'])[1]
tmpl_source_file = os.path.join(distrib_dir, tmpl_source_name)
if not os.path.isfile(tmpl_source_file):
msg = _(u"Template {0} unexistent").format(tmpl_source_file)
raise FileNotFound(msg)
else:
log.info(_(u"Copy template: '{0}' -> '{1}'").format(tmpl_source_file, templatedir))
shutil.copy(tmpl_source_file, templatedir)
def prepare_template(self, filevar, force_no_active=False):
"""Prepare template source file
"""
self._copy_to_template_dir(filevar, force_no_active)
self.patch_template(filevar, force_no_active)
self.strip_template_comment(filevar)
def verify(self, filevar):
"""
verifie que les fichiers existent
@param mkdir : création du répertoire si nécessaire
"""
if not os.path.isfile(filevar[u'source']):
raise FileNotFound(_(u"File {0} does not exist.").format(filevar[u'source']))
destfilename = filevar[u'full_name']
dir_target = os.path.dirname(destfilename)
if dir_target != '' and not os.path.isdir(dir_target):
if not filevar[u'mkdir']:
raise FileNotFound(_(u"Folder {0} does not exist but is required by {1}").format(dir_target, destfilename))
os.makedirs(dir_target)
# FIXME: pose plus de problème qu'autre chose (cf. #3048)
#if not isfile(target):
# system('cp %s %s' % (source, target))
def process(self, filevar, container):
"""Process a cheetah template
Process a cheetah template and copy the file to destination.
@param filevar: dictionary describing the file to process
@type filevar: C{dict}
@param container: dictionary describing the container
@type container: C{dict}
"""
UTF = "#encoding: utf-8"
self._check_filevar(filevar)
# full path of the destination file
destfilename = filevar[u'full_name']
log.info(_(u"Cheetah processing: '{0}' -> '{1}'").format(filevar[u'source'],
destfilename))
# utilisation d'un fichier temporaire
# afin de ne pas modifier l'original
tmpfile = mktemp()
shutil.copy(filevar[u'source'], tmpfile)
# ajout de l'en-tête pour le support de l'UTF-8
# FIXME: autres encodages ?
#os.system("sed -i '1i{0}' {1}".format(UTF, tmpfile)) (supprimé depuis #6224)
try:
cheetah_template = CheetahTemplate(tmpfile, self.creole_variables_dict, container)
os.unlink(tmpfile)
# suppression de l'en-tête UTF-8 ajouté !!! (supprimé depuis #6224)
data = str(cheetah_template) # .replace("{0}\n".format(UTF), '', 1)
except CheetahNotFound, err:
varname = err.args[0][13:-1]
msg = _(u"Error: unknown variable used in template {0} : {1}").format(filevar[u'name'], varname)
raise TemplateError, msg
except UnicodeDecodeError, err:
msg = _(u"Encoding issue detected in template {0}").format(filevar[u'name'])
raise TemplateError, msg
except Exception, err:
msg = _(u"Error while instantiating template {0}: {1}").format(filevar[u'name'], err)
raise TemplateError, msg
# écriture du fichier cible
if destfilename == '':
# CreoleCat may need to write on stdout (#10065)
sys.stdout.write(data)
else:
try:
file_h = file(destfilename, 'w')
file_h.write(data)
file_h.close()
except IOError, e:
msg = _(u"Unable to write in file '{0}': '{1}'").format(destfilename, e)
raise FileNotFound, msg
def change_properties(self, filevar, container=None, force_full_name=False):
chowncmd = [u'chown']
chownarg = ''
chmodcmd = [u'chmod']
chmodarg = ''
if not force_full_name:
destfilename = filevar[u'name']
else:
destfilename = filevar[u'full_name']
if u'owner' in filevar and filevar[u'owner']:
chownarg = filevar[u'owner']
else:
chownarg = u'root'
if u'group' in filevar and filevar[u'group']:
chownarg += ":" + filevar[u'group']
else:
chownarg += u':root'
if u'mode' in filevar and filevar[u'mode']:
chmodarg = filevar[u'mode']
else:
chmodarg = u'0644'
chowncmd.extend( [chownarg, destfilename] )
chmodcmd.extend( [chmodarg, destfilename] )
log.info(_(u'Changing properties: {0}').format(' '.join(chowncmd)) )
ret, out, err = pyeole.process.creole_system_out( chowncmd, container=container, context=False )
if ret != 0:
log.error(_(u'Error changing properties {0}: {1}').format(ret, err) )
log.info(_(u'Changing properties: {0}').format(' '.join(chmodcmd)) )
ret, out, err = pyeole.process.creole_system_out( chmodcmd, container=container, context=False )
if ret != 0:
log.error(_(u'Error changing properties {0}: {1}').format(ret, err) )
def remove_destfile(self, filevar):
"""
suppression du fichier de destination
"""
destfilename = filevar[u'full_name']
if os.path.isfile(destfilename):
os.unlink(destfilename)
else:
log.debug(_(u"File '{0}' unexistent.").format(destfilename))
def _instance_file(self, filevar, container=None):
"""Run templatisation on one file of one container
@param filevar: Dictionary describing the file
@type filevar: C{dict}
@param container: Dictionary describing a container
@type container: C{dict}
"""
if not filevar.get(u'activate', False):
try:
# copy and patch disabled templates too (#11029)
self.prepare_template(filevar, force_no_active=True)
except FileNotFound:
pass
if u'rm' in filevar and filevar[u'rm']:
log.info(_(u"Removing file '{0}'"
u" from container '{1}'").format(filevar[u'name'],
container[u'name']))
self.remove_destfile(filevar)
# The caller handles if it's an error
raise TemplateDisabled(_(u"Instantiation of file '{0}' disabled").format(filevar[u'name']))
log.info(_(u"Instantiating file '{0}'"
u" from '{1}'").format(filevar[u'name'], filevar[u'source']))
self.prepare_template(filevar)
self.verify(filevar)
self.process(filevar, container)
if filevar['name'].startswith('..') and container not in [None, 'root']:
self.change_properties(filevar, None, True)
else:
self.change_properties(filevar, container)
def instance_file(self, filename=None, container='root', ctx=None):
"""Run templatisation on one file
@param filename: name of a file
@type filename: C{str}
@param container: name of a container
@type container: C{str}
"""
if container == 'all':
if ctx is None:
groups = self.client.get_groups()
else:
groups = ctx.keys()
for group in groups:
if group in ['all', 'root']:
continue
if ctx is None:
lctx = None
else:
lctx = ctx[group]
self.instance_file(filename=filename, container=group, ctx=lctx)
else:
if ctx is None:
ctx = self.client.get_container_infos(container)
filevars = [f for f in ctx[u'files'] if f[u'name'] == filename]
for f in filevars:
self._instance_file(f, ctx)
def instance_files(self, filenames=None, container=None, containers_ctx=None):
"""Run templatisation on all files of all containers
@param filenames: names of files
@type filename: C{list}
@param container: name of a container
@type container: C{str}
"""
if containers_ctx is None:
containers_ctx = []
if container is not None:
containers_ctx = [self.client.get_container_infos(container)]
else:
for group_name in self.client.get_groups():
containers_ctx.append(self.client.get_group_infos(group_name))
if filenames is None:
all_files = set(listdir(distrib_dir))
prev_files = set(listdir(templatedir))
all_declared_files = set()
for ctx in containers_ctx:
for fdict in ctx[u'files']:
all_declared_files.add(basename(fdict['source']))
undeclared_files = all_files - all_declared_files
toremove_files = prev_files - all_files
# delete old templates (#6600)
for fname in toremove_files:
rm_file = join(templatedir, fname)
log.debug(_(u"Removing file '{0}'").format(rm_file))
unlink(rm_file)
# copy template not referenced in a dictionary (#6303)
for fname in undeclared_files:
fobj = {'source': join(templatedir, fname), 'name': ''}
self.prepare_template(fobj, True)
for ctx in containers_ctx:
for fdict in ctx[u'files']:
if not filenames or fdict[u'name'] in filenames:
try:
self._instance_file(fdict, container=ctx)
except TemplateDisabled, err:
# Information on disabled template only useful
# in debug
log.debug(err, exc_info=True)

64
data/diag.py
View File

@ -1,64 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
###########################################################################
# Eole NG - 2009
# Copyright Pole de Competence Eole (Ministere Education - Academie Dijon)
# http://eole.orion.education.fr - eole@ac-dijon.fr
#
# Licence CeCill
# cf: http://www.cecill.info/licences.fr.html
###########################################################################
import sys
import socket
from os.path import isfile
from os import system, stat
from pyeole.httprequest import HTTPRequest
from creole.config import configeol
from creole.client import CreoleClient
client = CreoleClient()
# adresse IP et port du serveur d'enregistrement
server = "http://194.167.18.21/apps/AutoDiag/index.n/diagnose"
md5file = "/etc/eole/.server.MD5"
module = "%s-%s" % (client.get_creole('eole_module'), client.get_creole('eole_version'))
def get_md5():
""" calcul de l'identifiant md5 """
if not isfile(md5file) or stat(md5file).st_size == 0:
system("md5sum %s | awk '{print $1}' > %s" % (configeol, md5file))
fp = file(md5file)
return (fp.read().split()[0])
def get_proxy():
""" récupération du proxy à utiliser """
if client.get_creole('activer_proxy_client') == 'oui':
return "http://{0}:{1}".format(
client.get_creole('proxy_client_adresse'),
client.get_creole('proxy_client_port'))
return ''
if __name__ == "__main__":
id5 = get_md5()
rne = client.get_creole('numero_etab')
data = {"ID5":id5, "module":module, "rne":rne, "dep":rne[0:3]}
socket.setdefaulttimeout(5)
proxy = get_proxy()
if proxy != '':
# essai avec proxy
try:
req = HTTPRequest(proxy={'http':proxy})
req.request(server, post_datas=data)
except:
pass
else:
sys.exit(0)
# essai sans proxy
try:
req = HTTPRequest()
req.request(server, post_datas=data)
except:
sys.exit(1)
else:
sys.exit(0)

2
data/funcs/__init__.py
View File

@ -1,2 +0,0 @@
"""Module de fonctions supplémentaires accessibles à creole. Tous les fichiers python
contenus dans ce répertoire sont lus par le module eosfunc de creole"""

69
data/gen_certif.py
View File

@ -1,69 +0,0 @@
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
script de generation d'un certificat ssl
prend un nom de fichier facultatif en argument (destination du certificat)
usage::
soit
%prog (-fc) [nom_certif]
soit
%prog (-f)
si [nom_certif] non renseigne, regenere tous les certificats par defaut ainsi que la ca locale.
Sinon, ne genere que [nom_certif]
-f :force la regeneration du (ou des) certificat(s) s'il(s) existe(nt)
-c : dans le cas de la generation d'un seul certificat, on copie la clef
"""
import sys, os
from optparse import OptionParser
from creole import cert
from pyeole.encode import normalize
def parse_command_line():
parser = OptionParser(__doc__)
parser.add_option("-c",
action="store_true", dest="copy", default=False,
help="copie de la clef")
parser.add_option("-f",
action="store_true", dest="regen", default=False,
help="force la regeneration de la clef")
options, args = parser.parse_args()
if len(args) > 1:
parser.error("Il faut au maximum un certificat")
return options, args
options, args = parse_command_line()
regen = options.regen
copy = options.copy
if len(args) == 1:
certfile = args[0]
else:
certfile = None
try:
cert.rehash_if_needed()
if certfile != None:
certfile = os.path.abspath(certfile)
dest_dir = os.path.dirname(certfile)
if not os.path.isdir(dest_dir):
print "Répertoire de destination inexistant (%s)" % dest_dir
sys.exit(1)
print "Generation du certificat machine"
cert.gen_certif(certfile, regen=regen, copy_key=copy)
else:
# génération de tous les certificats (CA, eole, scribe...)
cert.gen_certs(regen=regen)
sys.exit(0)
except Exception, err:
print "Erreur : "
print u'{0}'.format(normalize(err))
sys.exit(1)

26
data/testpatches.py
View File

@ -1,26 +0,0 @@
#! /usr/bin/env python
# -*- coding: UTF-8 -*-
"""
Test des patches pour diagnose
réutilisation du code de zephir-client
"""
import sys
from glob import glob
from os.path import basename
from creole import utils
from creole.config import patch_dir
from zephir.monitor.agents import patches
from os.path import join
patchs = glob(join(patch_dir, '*.patch'))
patchs.extend(glob(join(patch_dir, 'variante', '*.patch')))
err = []
for patch in patchs:
verif = patches.verify_patch(patch).values()
if len(verif) > 0 and len(verif[0]) > 0:
err.append(basename(patch))
if len(err) != 0:
utils.print_red('Erreur')
print "fichiers : %s" % (", ".join(err),)
else:
utils.print_green('Ok')

4
src/rougail/__init__.py Normal file
View File

@ -0,0 +1,4 @@
from .loader import load
from .annotator import modes
__ALL__ = ('load', 'modes')

0
creole/annotator.py → src/rougail/annotator.py
View File

0
creole/config.py → src/rougail/config.py
View File

0
creole/error.py → src/rougail/error.py
View File

0
creole/i18n.py → src/rougail/i18n.py
View File

41
creole/loader.py → src/rougail/loader.py
View File

@ -256,36 +256,6 @@ class PopulateTiramisuObjects(object):
self.separators[separator.attrib['name']] = info
elt.add_information('separator', info)
def build(self, persistent=False, session_id=None, meta_config=False):
if meta_config:
optiondescription = self.storage.paths['.'].get()
config = MetaConfig([],
optiondescription=optiondescription,
persistent=persistent,
session_id=session_id)
mixconfig = MixConfig(children=[],
optiondescription=optiondescription,
persistent=persistent,
session_id='m_' + session_id)
config.config.add(mixconfig)
else:
config = Config(self.storage.paths['.'].get(),
persistent=persistent,
session_id=session_id)
config.information.set('force_store_vars', self.force_store_values)
config.information.set('force_store_values', list(self.force_store_values))
# XXX really usefull?
ro_append = frozenset(config.property.getdefault('read_only', 'append') - {'force_store_value'})
rw_append = frozenset(config.property.getdefault('read_write', 'append') - {'force_store_value'})
config.property.setdefault(ro_append, 'read_only', 'append')
config.property.setdefault(rw_append, 'read_write', 'append')
config.property.read_only()
config.permissive.add('basic')
config.permissive.add('normal')
config.permissive.add('expert')
return config
class ElementStorage:
def __init__(self):
@ -491,7 +461,6 @@ class Variable(Common):
if not self.attrib['validators']:
del self.attrib['validators']
try:
print(self.attrib)
option = self.object_type(**self.attrib)
except Exception as err:
import traceback
@ -573,3 +542,13 @@ class Family(Common):
# self.option.impl_set_group_type(groups.leader)
return self.option
def load(xmlroot: str,
dtd_path: str,
funcs_path: str):
tiramisu_objects = PopulateTiramisuObjects()
tiramisu_objects.parse_dtd(dtd_path)
tiramisu_objects.make_tiramisu_objects(xmlroot,
funcs_path)
return tiramisu_objects.storage.paths['.'].get()

0
creole/objspace.py → src/rougail/objspace.py
View File

442
src/rougail/template.py Normal file
View File

@ -0,0 +1,442 @@
# -*- coding: utf-8 -*-
"""
Gestion du mini-langage de template
On travaille sur les fichiers cibles
"""
import imp
import sys
from shutil import copy
import logging
from typing import Dict
from subprocess import call
from os import listdir, unlink
from os.path import basename, join, split, isfile
from tempfile import mktemp
from Cheetah import Parser
# l'encoding du template est déterminé par une regexp (encodingDirectiveRE dans Parser.py)
# il cherche un ligne qui ressemble à '#encoding: utf-8
# cette classe simule le module 're' et retourne toujours l'encoding utf-8
# 6224
class FakeEncoding():
def groups(self):
return ('utf-8',)
def search(self, *args):
return self
Parser.encodingDirectiveRE = FakeEncoding()
from Cheetah.Template import Template as ChtTemplate
from Cheetah.NameMapper import NotFound as CheetahNotFound
from tiramisu import Config
from .config import patch_dir, templatedir, distrib_dir
from .error import FileNotFound, TemplateError, TemplateDisabled
from .i18n import _
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
class IsDefined(object):
"""
filtre permettant de ne pas lever d'exception au cas où
la variable Creole n'est pas définie
"""
def __init__(self, context):
self.context = context
def __call__(self, varname):
if '.' in varname:
splitted_var = varname.split('.')
if len(splitted_var) != 2:
msg = _("Group variables must be of type master.slave")
raise KeyError(msg)
master, slave = splitted_var
if master in self.context:
return slave in self.context[master].slave.keys()
return False
else:
return varname in self.context
class CreoleGet(object):
def __init__(self, context):
self.context = context
def __call__(self, varname):
return self.context[varname]
def __getitem__(self, varname):
"""For bracket and dotted notation
"""
return self.context[varname]
def __contains__(self, varname):
"""Check variable existence in context
"""
return varname in self.context
@classmethod
def cl_compile(kls, *args, **kwargs):
kwargs['compilerSettings'] = {'directiveStartToken' : '%',
'cheetahVarStartToken' : '%%',
'EOLSlurpToken' : '%',
'PSPStartToken' : 'µ' * 10,
'PSPEndToken' : 'µ' * 10,
'commentStartToken' : 'µ' * 10,
'commentEndToken' : 'µ' * 10,
'multiLineCommentStartToken' : 'µ' * 10,
'multiLineCommentEndToken' : 'µ' * 10}
return kls.old_compile(*args, **kwargs)
ChtTemplate.old_compile = ChtTemplate.compile
ChtTemplate.compile = cl_compile
class CreoleClient:
def __init__(self,
config: Config):
self.config = config
class CheetahTemplate(ChtTemplate):
"""classe pour personnaliser et faciliter la construction
du template Cheetah
"""
def __init__(self,
filename: str,
context,
eosfunc: Dict,
config: Config,
current_container: str):
"""Initialize Creole CheetahTemplate
"""
ChtTemplate.__init__(self, file=filename,
searchList=[context, eosfunc, {'is_defined' : IsDefined(context),
'creole_client' : CreoleClient(config),
'current_container':CreoleGet(current_container),
}])
class CreoleMaster(object):
def __init__(self, value, slave=None, index=None):
"""
On rend la variable itérable pour pouvoir faire:
for ip in iplist:
print ip.network
print ip.netmask
print ip
index is used for CreoleLint
"""
self._value = value
if slave is not None:
self.slave = slave
else:
self.slave = {}
self._index = index
def __getattr__(self, name):
"""Get slave variable or attribute of master value.
If the attribute is a name of a slave variable, return its value.
Otherwise, returns the requested attribute of master value.
"""
if name in self.slave:
value = self.slave[name]
if isinstance(value, Exception):
raise value
return value
else:
return getattr(self._value, name)
def __getitem__(self, index):
"""Get a master.slave at requested index.
"""
ret = {}
for key, values in self.slave.items():
ret[key] = values[index]
return CreoleMaster(self._value[index], ret, index)
def __iter__(self):
"""Iterate over master.slave.
Return synchronised value of master.slave.
"""
for i in range(len(self._value)):
ret = {}
for key, values in self.slave.items():
ret[key] = values[i]
yield CreoleMaster(self._value[i], ret, i)
def __len__(self):
"""Delegate to master value
"""
return len(self._value)
def __repr__(self):
"""Show CreoleMaster as dictionary.
The master value is stored under 'value' key.
The slaves are stored under 'slave' key.
"""
return repr({'value': self._value, 'slave': self.slave})
def __eq__(self, value):
return value == self._value
def __ne__(self, value):
return value != self._value
def __lt__(self, value):
return self._value < value
def __le__(self, value):
return self._value <= value
def __gt__(self, value):
return self._value > value
def __ge__(self, value):
return self._value >= value
def __str__(self):
"""Delegate to master value
"""
return str(self._value)
def __add__(self, val):
return self._value.__add__(val)
def __radd__(self, val):
return val + self._value
def __contains__(self, item):
return item in self._value
def add_slave(self, name, value):
"""Add a slave variable
Minimal check on type and value of the slave in regards to the
master one.
@param name: name of the slave variable
@type name: C{str}
@param value: value of the slave variable
"""
if isinstance(self._value, list):
if not isinstance(value, list):
raise TypeError
elif len(value) != len(self._value):
raise ValueError(_('length mismatch'))
new_value = []
for val in value:
if isinstance(val, dict):
new_value.append(ValueError(val['err']))
else:
new_value.append(val)
value = new_value
elif isinstance(value, list):
raise TypeError
self.slave[name] = value
class CreoleTemplateEngine:
"""Engine to process Creole cheetah template
"""
def __init__(self,
config: Config,
eosfunc_file: str):
self.config = config
eos = {}
eosfunc = imp.load_source('eosfunc', eosfunc_file)
for func in dir(eosfunc):
if not func.startswith('_'):
eos[func] = getattr(eosfunc, func)
self.eosfunc = eos
self.creole_variables_dict = {}
self.load_eole_variables(self.config.option('creole'))
def load_eole_variables(self, optiondescription):
# remplacement des variables EOLE
for option in optiondescription.list('all'):
if option.option.isoptiondescription():
if option.option.isleadership():
print('leadership')
raise Exception('a faire')
else:
self.load_eole_variables(option)
else:
self.creole_variables_dict[option.option.name()] = option.value.get()
#if varname.find('.') != -1:
# #support des groupes
# mastername, slavename = varname.split('.')
# if not mastername in self.creole_variables_dict or not \
# isinstance(self.creole_variables_dict [mastername],
# CreoleMaster):
# # Create the master variable
# if mastername in values:
# self.creole_variables_dict[mastername] = CreoleMaster(values[mastername])
# else:
# #only for CreoleLint
# self.creole_variables_dict[mastername] = CreoleMaster(value)
# #test only for CreoleLint
# if mastername != slavename:
# self.creole_variables_dict[mastername].add_slave(slavename, value)
#else:
# self.creole_variables_dict[varname] = value
def patch_template(self,
filename: str):
"""Apply patch to a template
"""
patch_cmd = ['patch', '-d', templatedir, '-N', '-p1']
patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']
# patches variante + locaux
for directory in [join(patch_dir, 'variante'), patch_dir]:
patch_file = join(directory, f'{filename}.patch')
if isfile(patch_file):
log.info(_("Patching template '{filename}' with '{patch_file}'"))
ret = call(patch_cmd + patch_no_debug + ['-i', patch_file])
if ret:
patch_cmd_err = ' '.join(patch_cmd + ['-i', patch_file])
log.error(_(f"Error applying patch: '{patch_file}'\nTo reproduce and fix this error {patch_cmd_err}"))
copy(filename, templatedir)
def strip_template_comment(self,
filename: str):
"""Strip comment from template
This apply if filevar has a del_comment attribut
"""
# suppression des commentaires si demandé (attribut del_comment)
if 'del_comment' in filevar and filevar['del_comment'] != '':
strip_cmd = ['sed', '-i']
log.info(_("Cleaning file '{0}'").format( filevar['source'] ))
raise Exception('hu')
#ret, out, err = pyeole.process.system_out(strip_cmd
# + ['/^\s*{0}/d ; /^$/d'.format(filevar['del_comment']),
# filevar['source'] ])
#if ret != 0:
# msg = _("Error removing comments '{0}': {1}")
# raise TemplateError(msg.format(filevar['del_comment'], err))
def prepare_template(self,
filename: str):
"""Prepare template source file
"""
log.info(_("Copy template: '{filename}' -> '{templatedir}'"))
copy(filename, templatedir)
self.patch_template(filename)
# self.strip_template_comment(filename)
def process(self,
filevar: Dict,
container: str):
"""Process a cheetah template
"""
# full path of the destination file
destfilename = join(dest_dir, filevar['source'])
log.info(_(f"Cheetah processing: '{destfilename}'"))
try:
cheetah_template = CheetahTemplate(join(templatedir, filevar['source']),
self.creole_variables_dict,
self.eosfunc,
self.config.config.copy(),
container)
data = str(cheetah_template)
except CheetahNotFound as err:
varname = err.args[0][13:-1]
raise TemplateError(_(f"Error: unknown variable used in template {destfilename} : {varname}"))
except Exception as err:
raise TemplateError(_(f"Error while instantiating template {destfilename}: {err}"))
with open(destfilename, 'w') as file_h:
file_h.write(data)
def change_properties(self,
filevar: Dict):
destfilename = join(dest_dir, filevar['source'])
#chowncmd = ['chown']
#chownarg = ''
chmodcmd = ['chmod']
chmodarg = ''
#if 'owner' in filevar and filevar['owner']:
# chownarg = filevar['owner']
#else:
# chownarg = 'root'
#if 'group' in filevar and filevar['group']:
# chownarg += ":" + filevar['group']
#else:
# chownarg += ':root'
if 'mode' in filevar and filevar['mode']:
chmodarg = filevar['mode']
else:
chmodarg = '0644'
#chowncmd.extend( [chownarg, destfilename] )
chmodcmd.extend([chmodarg, destfilename])
#log.info(_('Changing properties: {0}').format(' '.join(chowncmd)) )
#ret = call(chowncmd)
#if ret:
# log.error(_('Error changing properties {0}: {1}').format(ret, err) )
log.info(_('Changing properties: {0}').format(' '.join(chmodcmd)) )
ret = call(chmodcmd)
if ret:
chmod_cmd = ' '.join(chmodcmd)
log.error(_(f'Error changing properties: {chmodcmd}'))
def instance_file(self,
filevar: Dict,
container: str):
"""Run templatisation on one file of one container
"""
log.info(_("Instantiating file '{filename}'"))
self.process(filevar,
container)
self.change_properties(filevar)
def instance_files(self,
container=None):
"""Run templatisation on all files of all containers
@param container: name of a container
@type container: C{str}
"""
for template in listdir(distrib_dir):
self.prepare_template(join(distrib_dir, template))
for container_obj in self.config.option('containers').list('all'):
current_container = container_obj.option.doc()
if container is not None and container != current_container:
continue
for fills in container_obj.list('all'):
if fills.option.name() == 'files':
for fill_obj in fills.list('all'):
fill = fill_obj.value.dict()
filename = fill['source']
if not isfile(join(distrib_dir, filename)):
raise FileNotFound(_(f"File {filename} does not exist."))
print(fill)
if fill['activate']:
self.instance_file(fill, current_container)
else:
log.debug(_("Instantiation of file '{filename}' disabled"))
def generate(config: Config,
eosfunc_file: str,
container: str=None):
engine = CreoleTemplateEngine(config,
eosfunc_file)
engine.instance_files(container=container)

0
creole/utils.py → src/rougail/utils.py
View File

0
creole/xml_compare.py → src/rougail/xml_compare.py
View File

0
creole/xmlreflector.py → src/rougail/xmlreflector.py
View File

36
tests/test_makedict.py
View File

@ -4,13 +4,11 @@ from pytest import fixture, raises
from os import listdir, mkdir
from json import dump, load, dumps, loads
#from creole.xmlreflector import CreoleObjSpace, CreoleDictConsistencyError
#from creole import xmlreflector
from creole import objspace, annotator
from creole.xml_compare import xml_compare
from creole.error import CreoleDictConsistencyError
from creole.loader import PopulateTiramisuObjects
from creole.config import dtdfilename
from tiramisu import Config
from rougail import load as rougail_load
from rougail.xml_compare import xml_compare
from rougail.error import CreoleDictConsistencyError
from rougail.config import dtdfilename
dico_dirs = 'tests/flattener_dicos'
@ -43,10 +41,10 @@ def launch_flattener(test_dir):
cache_file = test_dir + '/result/00-base.xml'
fileio = open(cache_file)
xmlroot = etree.parse(fileio).getroot()
tiramisu_objects = PopulateTiramisuObjects()
tiramisu_objects.parse_dtd(dtdfilename)
tiramisu_objects.make_tiramisu_objects(xmlroot, eosfunc)
config = tiramisu_objects.build()
tiramisu_objects = rougail_load(xmlroot,
dtdfilename,
eosfunc)
config = Config(tiramisu_objects)
config.property.read_write()
makedict_dir = join(test_dir, 'makedict')
makedict_file = join(makedict_dir, 'base.json')
@ -64,22 +62,6 @@ def launch_flattener(test_dir):
assert load(fh) == loads(dumps(config_dict))
def fake_traduc(txt):
return txt
def setup_module(module):
module.traduc_ori = objspace._
objspace._ = fake_traduc
annotator._ = fake_traduc
objspace.ContainerAnnotator = getattr(annotator, 'ContainerAnnotator')
def teardown_module(module):
objspace._ = module.traduc_ori
annotator._ = module.traduc_ori
def test_dictionary(test_dir):
test_dir = join(dico_dirs, test_dir)
launch_flattener(test_dir)