Infra/rougail
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

separate engine from template

Browse Source
This commit is contained in:
Emmanuel Garette
2021-02-20 10:41:53 +01:00
parent 99d9fb7e70
commit ba41b27dbf
125 changed files with 793 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/rougail/__init__.py
View File

@ -25,7 +25,7 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from .convert import RougailConvert
from .template import RougailTemplate
from .template.base import RougailTemplate
from .config import RougailConfig
from .annotator import modes

28
src/rougail/annotator/service.py
View File

@ -30,6 +30,7 @@ from typing import Tuple
from ..i18n import _
from ..utils import normalize_family
from ..error import DictConsistencyError
from ..config import RougailConfig
# a CreoleObjSpace's attribute has some annotations
# that shall not be present in the exported (flatened) XML
ERASED_ATTRIBUTES = ('redefine', 'exists', 'optional', 'remove_check', 'namespace',
@ -52,6 +53,9 @@ class ServiceAnnotator:
"""
def __init__(self, objectspace):
self.objectspace = objectspace
self.uniq_ip = []
if 'network_type' not in self.objectspace.types:
self.objectspace.types['network_type'] = self.objectspace.types['ip_type']
if hasattr(self.objectspace.space, 'services'):
if not hasattr(self.objectspace.space.services, 'service'):
del self.objectspace.space.services
@ -162,7 +166,7 @@ class ServiceAnnotator:
path: str,
) -> Tuple[str, str]:
# create element name, if already exists, add _xx to be uniq
if hasattr(elt, 'source'):
if hasattr(elt, 'source') and elt.source:
name = elt.source
else:
name = elt.name
@ -254,11 +258,20 @@ class ServiceAnnotator:
msg = _(f'attribute "source" is mandatory for the file "{file_.name}" '
f'"({service_name})"')
raise DictConsistencyError(msg, 34, file_.xmlfiles)
if not file_.source:
del file_.source
if not hasattr(file_, 'engine'):
file_.engine = RougailConfig['default_engine']
def _update_ip(self,
ip,
service_name,
) -> None:
if service_name in self.uniq_ip:
msg = _(f'only one IP is allowed by IP, '
f'please use a variable multiple if you want have more than one IP')
raise DictConsistencyError(msg, 67, ip.xmlfiles)
self.uniq_ip.append(service_name)
variable = self.objectspace.paths.get_variable(ip.name)
if variable.type in ['ip', 'network_cidr'] and hasattr(ip, 'netmask'):
msg = _(f'ip with ip_type "{variable.type}" must not have netmask')
@ -271,3 +284,16 @@ class ServiceAnnotator:
if netmask.type != 'netmask':
msg = _(f'netmask in ip must have type "netmask", not "{netmask.type}"')
raise DictConsistencyError(msg, 65, ip.xmlfiles)
# Convert IP to file
ip.network_type = ip.ip_type
ip.network = ip.name
ip.ip_type = 'filename'
ip.name = f'/systemd/system/{service_name}.service.d/rougail_ip.conf'
# retrieve default value from File object
for attr in ['owner', 'group', 'mode']:
setattr(ip, attr, getattr(self.objectspace.file, attr))
ip.source = None
ip.engine = 'creole'
self._update_file(ip,
service_name,
)

7
src/rougail/annotator/value.py
View File

@ -26,6 +26,9 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from .variable import Walk
from ..i18n import _
from ..error import DictConsistencyError
class ValueAnnotator(Walk): # pylint: disable=R0903
"""Annotate value
"""
@ -87,5 +90,9 @@ class ValueAnnotator(Walk): # pylint: disable=R0903
else:
variable.default_multi = variable.value[0].name
else:
if len(variable.value) > 1:
msg = _(f'the non multi variable "{variable.name}" cannot have '
'more than one value')
raise DictConsistencyError(msg, 68, variable.xmlfiles)
variable.default = variable.value[0].name
del variable.value

1
src/rougail/config.py
View File

@ -42,4 +42,5 @@ RougailConfig = {'dictionaries_dir': [join(ROUGAILROOT, 'dictionaries')],
'functions_file': join(ROUGAILROOT, 'functions.py'),
'variable_namespace': 'rougail',
'auto_freeze_variable': 'instanciated_module',
'default_engine': 'creole',
}

4
src/rougail/data/rougail.dtd
View File

@ -71,11 +71,11 @@
<!ATTLIST file group CDATA "root">
<!ATTLIST file filelist CDATA #IMPLIED>
<!ATTLIST file redefine (True|False) "False">
<!ATTLIST file templating (none|creole) "creole">
<!ATTLIST file engine (none|creole) #IMPLIED>
<!ELEMENT override EMPTY>
<!ATTLIST override source CDATA #IMPLIED>
<!ATTLIST override templating (creole|none) "creole">
<!ATTLIST override engine (none|creole) #IMPLIED>
<!ELEMENT variables ((variable*|family*)*)>

179
src/rougail/template.py → src/rougail/template/base.py
View File

@ -30,10 +30,8 @@ import logging
from typing import Dict, Any
from subprocess import call
from os import listdir, makedirs, getcwd, chdir
from os.path import dirname, join, isfile, abspath, normpath, isdir
from Cheetah.Template import Template as ChtTemplate
from Cheetah.NameMapper import NotFound as CheetahNotFound
from os.path import dirname, join, isfile, isdir, abspath
from ipaddress import ip_network
try:
from tiramisu3 import Config
@ -42,68 +40,28 @@ except ModuleNotFoundError: # pragma: no cover
from tiramisu import Config
from tiramisu.error import PropertiesOptionError
from .config import RougailConfig
from .error import FileNotFound, TemplateError
from .i18n import _
from .utils import normalize_family, load_modules
from ..config import RougailConfig
from ..error import FileNotFound, TemplateError
from ..i18n import _
from ..utils import load_modules
from . import engine as engines
ENGINES = {}
for engine in engines.__all__:
ENGINES[engine] = getattr(engines, engine)
ROUGAIL_IP_TEMPLATE = """[Service]
%for %%ip in %%rougail_variable
IPAddressAllow=%%ip
%end for
IPAddressDeny=any
"""
log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
@classmethod
def cl_compile(kls, *args, **kwargs):
"""Rewrite compile methode to force some settings
"""
kwargs['compilerSettings'] = {'directiveStartToken' : '%',
'cheetahVarStartToken' : '%%',
'EOLSlurpToken' : '%',
'PSPStartToken' : 'µ' * 10,
'PSPEndToken' : 'µ' * 10,
'commentStartToken' : 'µ' * 10,
'commentEndToken' : 'µ' * 10,
'multiLineCommentStartToken' : 'µ' * 10,
'multiLineCommentEndToken' : 'µ' * 10}
return kls.old_compile(*args, **kwargs) # pylint: disable=E1101
ChtTemplate.old_compile = ChtTemplate.compile
ChtTemplate.compile = cl_compile
class CheetahTemplate(ChtTemplate): # pylint: disable=W0223
"""Construct a cheetah templating object
"""
def __init__(self,
filename: str,
context,
eosfunc: Dict,
extra_context: Dict,
):
"""Initialize Creole CheetahTemplate
"""
ChtTemplate.__init__(self,
file=filename,
searchList=[context, eosfunc, extra_context])
# FORK of Cheetah function, do not replace '\\' by '/'
def serverSidePath(self,
path=None,
normpath=normpath,
abspath=abspath
): # pylint: disable=W0621
# strange...
if path is None and isinstance(self, str):
path = self
if path: # pylint: disable=R1705
return normpath(abspath(path))
# original code return normpath(abspath(path.replace("\\", '/')))
elif hasattr(self, '_filePath') and self._filePath: # pragma: no cover
return normpath(abspath(self._filePath))
else: # pragma: no cover
return None
class RougailLeaderIndex:
"""This object is create when access to a specified Index of the variable
"""
@ -273,41 +231,9 @@ class RougailTemplate:
copy(filename, self.tmp_dir)
self.patch_template(filename)
def process(self,
source: str,
true_destfilename: str,
destfilename: str,
variable: Any,
):
"""Process a cheetah template
"""
# full path of the destination file
log.info(_(f"Cheetah processing: '{destfilename}'"))
try:
extra_context = {'normalize_family': normalize_family,
'rougail_filename': true_destfilename
}
if variable:
extra_context['rougail_variable'] = variable
cheetah_template = CheetahTemplate(source,
self.rougail_variables_dict,
self.eosfunc,
extra_context,
)
data = str(cheetah_template)
except CheetahNotFound as err: # pragma: no cover
varname = err.args[0][13:-1]
msg = f"Error: unknown variable used in template {source} to {destfilename}: {varname}"
raise TemplateError(_(msg)) from err
except Exception as err: # pragma: no cover
msg = _(f"Error while instantiating template {source} to {destfilename}: {err}")
raise TemplateError(msg) from err
with open(destfilename, 'w') as file_h:
file_h.write(data)
def instance_file(self,
filevar: Dict,
object_type: str,
) -> None:
"""Run templatisation on one file
"""
@ -316,29 +242,51 @@ class RougailTemplate:
variable = filevar['variable']
else:
variable = None
if 'network' in filevar:
if 'netmask' in filevar:
if isinstance(filevar['network'], list):
variable = [str(ip_network(f'{net}/{mask}'))
for net, mask in zip(filevar['network'], filevar['netmask'])]
else:
variable = str(ip_network(f'{filevar["network"]}/{filevar["netmask"]}'))
else:
variable = filevar['network']
if not isinstance(variable, list):
if variable is None:
variable = []
else:
variable = [variable]
filenames = filevar['name']
if not isinstance(filenames, list):
filenames = [filenames]
if variable:
if variable and not isinstance(variable, list):
variable = [variable]
if not isdir(self.destinations_dir):
raise TemplateError(_(f'cannot find destinations_dir {self.destinations_dir}'))
for idx, filename in enumerate(filenames):
destfilename = join(self.destinations_dir, filename[1:])
for idx, destfile in enumerate(filenames):
destfilename = join(self.destinations_dir, destfile[1:])
makedirs(dirname(destfilename), exist_ok=True)
if variable:
if object_type == 'ip':
var = variable
elif variable:
var = variable[idx]
else:
var = None
source = join(self.tmp_dir, filevar['source'])
if filevar['templating'] == 'creole':
self.process(source,
filename,
destfilename,
var,
)
if object_type == 'ip':
filename = None
source = ROUGAIL_IP_TEMPLATE
else:
copy(source, destfilename)
filename = join(self.tmp_dir, filevar['source'])
source = None
log.info(_(f"{filevar['engine']} processing: '{destfilename}'"))
ENGINES[filevar['engine']].process(filename=filename,
source=source,
true_destfilename=destfile,
destfilename=destfilename,
variable=var,
rougail_variables_dict=self.rougail_variables_dict,
eosfunc=self.eosfunc,
)
async def instance_files(self) -> None:
"""Run templatisation on all files
@ -355,14 +303,16 @@ class RougailTemplate:
self.prepare_template(template)
for service_obj in await self.config.option('services').list('all'):
for fills in await service_obj.list('all'):
if await fills.option.name() in ['files', 'overrides']:
type_ = await fills.option.name()
if type_ in ['files', 'overrides', 'ip']:
for fill_obj in await fills.list('all'):
fill = await fill_obj.value.dict()
filename = fill['source']
if not isfile(filename): # pragma: no cover
raise FileNotFound(_(f"File {filename} does not exist."))
if type_ != 'ip':
filename = fill['source']
if not isfile(filename): # pragma: no cover
raise FileNotFound(_(f"File {filename} does not exist."))
if fill['activate']:
self.instance_file(fill)
self.instance_file(fill, type_)
else:
log.debug(_("Instantiation of file '{filename}' disabled"))
chdir(ori_dir)
@ -398,5 +348,12 @@ class RougailTemplate:
if is_variable_namespace:
value = await option.value.get()
self.rougail_variables_dict[await option.option.name()] = value
variables[await option.option.name()] = await option.value.get()
if await option.option.issymlinkoption() and await option.option.isfollower():
value = []
path = await option.option.path()
for index in range(await option.value.len()):
value.append(await self.config.option(path, index).value.get())
else:
value = await option.value.get()
variables[await option.option.name()] = value
return RougailExtra(variables)

4
src/rougail/template/engine/__init__.py Normal file
View File

@ -0,0 +1,4 @@
from . import none, creole
__all__ = ('none', 'creole')

127
src/rougail/template/engine/creole.py Normal file
View File

@ -0,0 +1,127 @@
"""Creole engine
Created by:
EOLE (http://eole.orion.education.fr)
Copyright (C) 2005-2018
Forked by:
Cadoles (http://www.cadoles.com)
Copyright (C) 2021
distribued with GPL-2 or later license
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from os.path import abspath, normpath
from Cheetah.Template import Template
from Cheetah.NameMapper import NotFound
from typing import Dict, Any
from ...utils import normalize_family
from ...error import TemplateError
@classmethod
def cl_compile(kls, *args, **kwargs):
"""Rewrite compile methode to force some settings
"""
kwargs['compilerSettings'] = {'directiveStartToken' : '%',
'cheetahVarStartToken' : '%%',
'EOLSlurpToken' : '%',
'commentStartToken' : '#',
'multiLineCommentStartToken' : '#*',
'multiLineCommentEndToken' : '*#',
}
return kls.old_compile(*args, **kwargs) # pylint: disable=E1101
Template.old_compile = Template.compile
Template.compile = cl_compile
class CheetahTemplate(Template): # pylint: disable=W0223
"""Construct a cheetah templating object
"""
def __init__(self,
filename: str,
source: str,
context,
eosfunc: Dict,
extra_context: Dict,
):
"""Initialize Creole CheetahTemplate
"""
if filename is not None:
Template.__init__(self,
file=filename,
searchList=[context, eosfunc, extra_context],
)
else:
Template.__init__(self,
source=source,
searchList=[context, eosfunc, extra_context],
)
# FORK of Cheetah function, do not replace '\\' by '/'
def serverSidePath(self,
path=None,
normpath=normpath,
abspath=abspath
): # pylint: disable=W0621
# strange...
if path is None and isinstance(self, str):
path = self
if path: # pylint: disable=R1705
return normpath(abspath(path))
# original code return normpath(abspath(path.replace("\\", '/')))
elif hasattr(self, '_filePath') and self._filePath: # pragma: no cover
return normpath(abspath(self._filePath))
else: # pragma: no cover
return None
def process(filename: str,
source: str,
true_destfilename: str,
destfilename: str,
variable: Any,
rougail_variables_dict: Dict,
eosfunc: Dict,
):
"""Process a cheetah template
"""
# full path of the destination file
try:
extra_context = {'normalize_family': normalize_family,
'rougail_filename': true_destfilename
}
if variable is not None:
extra_context['rougail_variable'] = variable
cheetah_template = CheetahTemplate(filename,
source,
rougail_variables_dict,
eosfunc,
extra_context,
)
data = str(cheetah_template)
except NotFound as err: # pragma: no cover
varname = err.args[0][13:-1]
msg = f"Error: unknown variable used in template {filename} to {destfilename}: {varname}"
raise TemplateError(_(msg)) from err
except Exception as err: # pragma: no cover
msg = _(f"Error while instantiating template {filename} to {destfilename}: {err}")
raise TemplateError(msg) from err
with open(destfilename, 'w') as file_h:
file_h.write(data)

35
src/rougail/template/engine/none.py Normal file
View File

@ -0,0 +1,35 @@
"""None engine
Created by:
EOLE (http://eole.orion.education.fr)
Copyright (C) 2005-2018
Forked by:
Cadoles (http://www.cadoles.com)
Copyright (C) 2021
distribued with GPL-2 or later license
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""
from typing import Any
from shutil import copy
def process(filename: str,
destfilename: str,
**kwargs
):
copy(filename, destfilename)