Infra
/
rougail
9
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update improvment

This commit is contained in:
Emmanuel Garette 2021-03-27 07:45:45 +01:00
parent 2a77093187
commit 17e7f4caa0
1 changed files with 105 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
src/rougail/update.py
View File

@ -25,10 +25,13 @@ from os.path import join, isfile, basename
from os import listdir from os import listdir
from lxml.etree import DTD, parse, XMLParser, XMLSyntaxError # pylint: disable=E0611 from lxml.etree import DTD, parse, XMLParser, XMLSyntaxError # pylint: disable=E0611
from lxml.etree import Element, SubElement, tostring from lxml.etree import Element, SubElement, tostring
from ast import parse as ast_parse
from .i18n import _ from .i18n import _
from .error import UpgradeError from .error import UpgradeError
from .utils import normalize_family
VERSIONS = {'creole': ['1'], VERSIONS = {'creole': ['1'],
'rougail': ['0.9'], 'rougail': ['0.9'],
@ -40,7 +43,7 @@ def get_function_name(root, version):
return f'update_{root}_{version}' return f'update_{root}_{version}'
FUNCTION_VERSIONS = [get_function_name(root, version) for root, versions in VERSIONS.items() for version in versions] FUNCTION_VERSIONS = [(root, version, get_function_name(root, version)) for root, versions in VERSIONS.items() for version in versions]
class RougailUpgrade: class RougailUpgrade:
@ -53,6 +56,7 @@ class RougailUpgrade:
def load_xml_from_folders(self, def load_xml_from_folders(self,
srcfolder: str, srcfolder: str,
dstfolder: str, dstfolder: str,
namespace: str,
): ):
"""Loads all the XML files located in the xmlfolders' list """Loads all the XML files located in the xmlfolders' list
@ -71,10 +75,13 @@ class RougailUpgrade:
root = document.getroot() root = document.getroot()
search_function_name = get_function_name(root.tag, root.attrib.get('version', '1')) search_function_name = get_function_name(root.tag, root.attrib.get('version', '1'))
function_found = False function_found = False
for function_version in FUNCTION_VERSIONS: for root_name, version, function_version in FUNCTION_VERSIONS:
if function_found and hasattr(self, function_version): if function_found and hasattr(self, function_version):
print(f' - convert {filename} to version {version}')
upgrade_help = self.upgrade_help.get(function_version, {}).get(filename, {}) upgrade_help = self.upgrade_help.get(function_version, {}).get(filename, {})
root = getattr(self, function_version)(root, upgrade_help) if upgrade_help.get('remove') is True:
continue
root = getattr(self, function_version)(root, upgrade_help, namespace)
if function_version == search_function_name: if function_version == search_function_name:
function_found = True function_found = True
with open(xmldst, 'wb') as xmlfh: with open(xmldst, 'wb') as xmlfh:
@ -90,6 +97,7 @@ class RougailUpgrade:
def update_rougail_0_9(self, def update_rougail_0_9(self,
root: 'Element', root: 'Element',
upgrade_help: dict, upgrade_help: dict,
namespace: str,
) -> 'Element': ) -> 'Element':
# rename root # rename root
root.tag = 'rougail' root.tag = 'rougail'
@ -111,6 +119,9 @@ class RougailUpgrade:
if not isinstance(subelement.tag, str): if not isinstance(subelement.tag, str):
# XML comment # XML comment
continue continue
if subelement.tag == 'family_action':
root.remove(subelement)
continue
for subsubelement in subelement: for subsubelement in subelement:
if not isinstance(subsubelement.tag, str): if not isinstance(subsubelement.tag, str):
# XML comment # XML comment
@ -130,16 +141,27 @@ class RougailUpgrade:
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('hidden', {}).get('remove', []): if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('hidden', {}).get('remove', []):
self.remove(subsubsubelement, 'hidden', optional=True) self.remove(subsubsubelement, 'hidden', optional=True)
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mandatory', {}).get('remove', []): if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mandatory', {}).get('remove', []):
self.remove(subsubsubelement, 'mandatory', optional=True) self.remove(subsubsubelement, 'mandatory')
variables[subsubsubelement.attrib['name']] = subsubsubelement if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mandatory', {}).get('add', []):
subsubsubelement.attrib['mandatory'] = 'True'
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('type', {}):
subsubsubelement.attrib['type'] = upgrade_help.get('variables', {}).get('type', {})[subsubsubelement.attrib['name']]
if namespace == 'configuration':
path = subsubsubelement.attrib['name']
npath = normalize_family(subsubsubelement.attrib['name'])
else:
path = namespace + '.' + subsubelement.attrib['name'] + '.' + subsubsubelement.attrib['name']
npath = normalize_family(namespace) + '.' + normalize_family(subsubelement.attrib['name']) + '.' + normalize_family(subsubsubelement.attrib['name'])
variables[path] = subsubsubelement
variables[npath] = subsubsubelement
type = subsubsubelement.attrib.get('type') type = subsubsubelement.attrib.get('type')
if type in ['oui/non', 'yes/no', 'on/off']: if type in ['oui/non', 'yes/no', 'on/off']:
variables_auto_valid_enum.setdefault(subsubsubelement.attrib['type'], []).append(subsubsubelement.attrib['name']) variables_auto_valid_enum.setdefault(subsubsubelement.attrib['type'], []).append(path)
del subsubsubelement.attrib['type'] del subsubsubelement.attrib['type']
elif type == 'hostname_strict': elif type == 'hostname_strict':
subsubsubelement.attrib['type'] = 'hostname' subsubsubelement.attrib['type'] = 'hostname'
elif type == 'domain_strict': elif type in ['domain', 'domain_strict']:
subsubsubelement.attrib['type'] = 'domain' subsubsubelement.attrib['type'] = 'domainname'
elif type == 'string': elif type == 'string':
del subsubsubelement.attrib['type'] del subsubsubelement.attrib['type']
if self.test and subsubsubelement.attrib.get('auto_freeze') == 'True': if self.test and subsubsubelement.attrib.get('auto_freeze') == 'True':
@ -153,17 +175,36 @@ class RougailUpgrade:
subsubsubelement.attrib['remove_check'] = 'True' subsubsubelement.attrib['remove_check'] = 'True'
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mode', {}).get('modify', {}): if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mode', {}).get('modify', {}):
subsubsubelement.attrib['mode'] = upgrade_help.get('variables', {}).get('mode', {}).get('modify', {})[subsubsubelement.attrib['name']] subsubsubelement.attrib['mode'] = upgrade_help.get('variables', {}).get('mode', {}).get('modify', {})[subsubsubelement.attrib['name']]
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('type', {}).get('modify', {}):
subsubsubelement.attrib['type'] = upgrade_help.get('variables', {}).get('type', {}).get('modify', {})[subsubsubelement.attrib['name']]
type = subsubsubelement.attrib['type']
if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('test', {}): if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('test', {}):
subsubsubelement.attrib['test'] = upgrade_help.get('variables', {}).get('test', {})[subsubsubelement.attrib['name']] subsubsubelement.attrib['test'] = upgrade_help.get('variables', {}).get('test', {})[subsubsubelement.attrib['name']]
# for value in subsubsubelement: if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('remove_value', []):
# if value.text is None: for value in subsubsubelement:
# value.attrib['type'] = 'nil' subsubsubelement.remove(value)
if subsubsubelement.attrib['name'] != normalize_family(subsubsubelement.attrib['name']):
if "description" not in subsubsubelement.attrib:
subsubsubelement.attrib['description'] = subsubsubelement.attrib['name']
subsubsubelement.attrib['name'] = normalize_family(subsubsubelement.attrib['name'])
elif subsubsubelement.tag == 'param': elif subsubsubelement.tag == 'param':
if subsubelement.tag == 'check' and subsubelement.attrib['target'] in upgrade_help.get('check', {}).get('remove', []):
continue
type = subsubsubelement.attrib.get('type') type = subsubsubelement.attrib.get('type')
if type == 'eole': if type == 'eole':
subsubsubelement.attrib['type'] = 'variable' subsubsubelement.attrib['type'] = 'variable'
type = 'variable' type = 'variable'
elif type in ('container', 'context', 'python'): if type == 'python':
subsubsubelement.attrib['type'] = 'function'
if subsubsubelement.text.startswith('range('):
func_ast = ast_parse(subsubsubelement.text)
subsubsubelement.text = 'range'
for arg in func_ast.body[0].value.args:
SubElement(subsubelement, 'param', type='number').text = str(arg.value)
else:
raise Exception(f'{subsubsubelement.text} is not a supported function')
type = 'function'
elif type in ('container', 'context'):
raise UpgradeError(_(f'cannot convert param with type "{type}"')) raise UpgradeError(_(f'cannot convert param with type "{type}"'))
if subsubelement.attrib['name'] == 'valid_entier' and not 'type' in subsubsubelement.attrib: if subsubelement.attrib['name'] == 'valid_entier' and not 'type' in subsubsubelement.attrib:
subsubsubelement.attrib['type'] = 'number' subsubsubelement.attrib['type'] = 'number'
@ -172,10 +213,14 @@ class RougailUpgrade:
continue continue
if subsubelement.attrib['name'] == 'valid_enum' and not type: if subsubelement.attrib['name'] == 'valid_enum' and not type:
if subsubsubelement.attrib.get('name') == 'checkval': if subsubsubelement.attrib.get('name') == 'checkval':
if subsubelement.attrib['target'] in upgrade_help.get('check', {}).get('valid_enums', {}).get('checkval', {}).get('remove', []):
subsubelement.remove(subsubsubelement)
continue
raise UpgradeError(_('checkval in valid_enum is no more supported')) raise UpgradeError(_('checkval in valid_enum is no more supported'))
for val in eval(subsubsubelement.text): if subsubsubelement.text.startswith('['):
SubElement(subsubelement, 'param').text = str(val) for val in eval(subsubsubelement.text):
subsubelement.remove(subsubsubelement) SubElement(subsubelement, 'param').text = str(val)
subsubelement.remove(subsubsubelement)
self.move(subsubsubelement, 'hidden', 'propertyerror', optional=True) self.move(subsubsubelement, 'hidden', 'propertyerror', optional=True)
if type == 'variable' and subsubsubelement.text in upgrade_help.get('variables', {}).get('rename', []): if type == 'variable' and subsubsubelement.text in upgrade_help.get('variables', {}).get('rename', []):
subsubsubelement.text = upgrade_help.get('variables', {}).get('rename', [])[subsubsubelement.text] subsubsubelement.text = upgrade_help.get('variables', {}).get('rename', [])[subsubsubelement.text]
@ -189,16 +234,18 @@ class RougailUpgrade:
type = subsubsubelement.attrib.get('type') type = subsubsubelement.attrib.get('type')
if type in ['service_accesslist', 'service_restrictionlist', 'interfacelist', 'hostlist']: if type in ['service_accesslist', 'service_restrictionlist', 'interfacelist', 'hostlist']:
subsubelement.remove(subsubsubelement) subsubelement.remove(subsubsubelement)
# elif self.test and type in ['filelist', 'variable', 'servicelist']:
# subsubsubelement.attrib['optional'] = "True"
elif (not type or type == 'variable') and subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []): elif (not type or type == 'variable') and subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []):
subsubelement.remove(subsubsubelement) subsubelement.remove(subsubsubelement)
elif type == 'family' and subsubsubelement.text in upgrade_help.get('families', {}).get('remove', []): elif type == 'family' and subsubsubelement.text in upgrade_help.get('families', {}).get('remove', []):
subsubelement.remove(subsubsubelement) subsubelement.remove(subsubsubelement)
up = upgrade_help.get('targets', {}).get(subsubsubelement.text) elif type == 'actionlist':
if up: # for family in root.find('variables'):
if 'optional' in up: # family_list = SubElement(subsubelement, 'target')
subsubsubelement.attrib['optional'] = up['optional'] # family_list.attrib['type'] = 'family'
# family_list.text = namespace + '.' + family.attrib['name']
subsubelement.remove(subsubsubelement)
if upgrade_help.get('targets', {}).get(subsubsubelement.text, {}).get('optional'):
subsubsubelement.attrib['optional'] = upgrade_help.get('targets', {}).get(subsubsubelement.text, {}).get('optional')
has_target = False has_target = False
for target in subsubelement: for target in subsubelement:
if target.tag == 'target': if target.tag == 'target':
@ -208,7 +255,11 @@ class RougailUpgrade:
subelement.remove(subsubelement) subelement.remove(subsubelement)
continue continue
elif subsubsubelement.tag == 'slave': elif subsubsubelement.tag == 'slave':
if subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []):
subsubelement.remove(subsubsubelement)
continue
subsubsubelement.tag = 'follower' subsubsubelement.tag = 'follower'
subsubsubelement.text = normalize_family(subsubsubelement.text)
if subelement.tag == 'containers': if subelement.tag == 'containers':
current_service = self.upgrade_container(subsubsubelement, current_service, files, ips, servicelists, upgrade_help) current_service = self.upgrade_container(subsubsubelement, current_service, files, ips, servicelists, upgrade_help)
subsubelement.remove(subsubsubelement) subsubelement.remove(subsubsubelement)
@ -283,6 +334,12 @@ class RougailUpgrade:
subelement.remove(subsubelement) subelement.remove(subsubelement)
else: else:
self.move(subsubelement, 'master', 'leader') self.move(subsubelement, 'master', 'leader')
for follower in subsubelement:
if '.' in subsubelement.attrib['leader']:
path = subsubelement.attrib['leader'].rsplit('.', 1)[0] + '.' + follower.text
else:
path = follower.text
self.remove(variables[path], 'multi', optional=True)
if subsubelement.attrib['leader'] in upgrade_help.get('groups', {}).get('reorder', {}): if subsubelement.attrib['leader'] in upgrade_help.get('groups', {}).get('reorder', {}):
for follower in subsubelement: for follower in subsubelement:
subsubelement.remove(follower) subsubelement.remove(follower)
@ -302,6 +359,10 @@ class RougailUpgrade:
continue continue
if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('hidden', {}).get('add', []): if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('hidden', {}).get('add', []):
subsubelement.attrib['hidden'] = 'True' subsubelement.attrib['hidden'] = 'True'
if subsubelement.attrib['name'] != normalize_family(subsubelement.attrib['name']):
if "description" not in subsubelement.attrib:
subsubelement.attrib['description'] = subsubelement.attrib['name']
subsubelement.attrib['name'] = normalize_family(subsubelement.attrib['name'])
families[subsubelement.attrib['name']] = subsubelement families[subsubelement.attrib['name']] = subsubelement
# if empty, remove # if empty, remove
if is_empty(subelement) or subelement.tag in ['containers', 'files', 'help']: if is_empty(subelement) or subelement.tag in ['containers', 'files', 'help']:
@ -394,7 +455,9 @@ class RougailUpgrade:
for name, text in variables_help.items(): for name, text in variables_help.items():
variables[name].attrib['help'] = text variables[name].attrib['help'] = text
for name, text in families_help.items(): for name, text in families_help.items():
families[name].attrib['help'] = text if name in upgrade_help.get('families', {}).get('rename', {}):
name = upgrade_help.get('families', {}).get('rename', {})[name]
families[normalize_family(name)].attrib['help'] = text
for auto in autos: for auto in autos:
if auto in variables: if auto in variables:
variables[auto].attrib['hidden'] = 'True' variables[auto].attrib['hidden'] = 'True'
@ -436,6 +499,8 @@ class RougailUpgrade:
service.attrib['name'] = service_name service.attrib['name'] = service_name
if service_name == 'unknown': if service_name == 'unknown':
service.attrib['manage'] = 'False' service.attrib['manage'] = 'False'
if service_name in upgrade_help.get('services', {}).get('unmanage', []):
service.attrib['manage'] = 'False'
service_elt[service_name] = service service_elt[service_name] = service
if upgrade_help.get('servicelists', {}).get(service_name): if upgrade_help.get('servicelists', {}).get(service_name):
service.attrib['servicelist'] = upgrade_help.get('servicelists', {}).get(service_name) service.attrib['servicelist'] = upgrade_help.get('servicelists', {}).get(service_name)
@ -450,7 +515,15 @@ class RougailUpgrade:
elt.attrib['engine'] = 'creole_legacy' elt.attrib['engine'] = 'creole_legacy'
if (not 'instance_mode' in elt.attrib or elt.attrib['instance_mode'] != 'when_container') and \ if (not 'instance_mode' in elt.attrib or elt.attrib['instance_mode'] != 'when_container') and \
elt.text not in upgrade_help.get('files', {}).get('remove', {}): elt.text not in upgrade_help.get('files', {}).get('remove', {}):
files[current_service][elt.text] = elt if elt.attrib.get('filelist') in upgrade_help.get('services', {}).get('filelist_service', {}):
elt_service = upgrade_help.get('services', {}).get('filelist_service', {})[elt.attrib['filelist']]
if elt_service in files:
service = elt_service
else:
service = current_service
else:
service = current_service
files[service][elt.text] = elt
elif elt.tag in ['host', 'disknod', 'fstab', 'interface', 'package', 'service_access']: elif elt.tag in ['host', 'disknod', 'fstab', 'interface', 'package', 'service_access']:
pass pass
elif elt.tag == 'service_restriction': elif elt.tag == 'service_restriction':
@ -458,7 +531,15 @@ class RougailUpgrade:
if restriction.tag == 'ip' and restriction.text != '0.0.0.0': if restriction.tag == 'ip' and restriction.text != '0.0.0.0':
self.remove(restriction, 'ip_type', optional=True) self.remove(restriction, 'ip_type', optional=True)
self.remove(restriction, 'netmask_type', optional=True) self.remove(restriction, 'netmask_type', optional=True)
ips[current_service].append(restriction) if elt.attrib['service'] in upgrade_help.get('services', {}).get('rename', {}):
elt_service = upgrade_help.get('services', {}).get('rename', {})[elt.attrib['service']]
else:
elt_service = elt.attrib['service']
if elt_service in ips:
service = elt_service
else:
service = current_service
ips[service].append(restriction)
elif elt.tag == 'service': elif elt.tag == 'service':
new_name = elt.text new_name = elt.text
if current_service == 'unknown': if current_service == 'unknown':