add tmpfile_dest_dir option

Reviewed-on: #5

README.md

@@ -0,0 +1,27 @@
+Rougail
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+[Documentation (french)](doc/README.md)

debian/agpl-3

@@ -1,662 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.
-

debian/changelog

@@ -1,5 +0,0 @@
-rougail (0.1) unstable; urgency=low
-
-  * first version
-
- -- Cadoles <contact@cadoles.com>  Tue, 31 Mar 2020 10:40:42 +0200

debian/compat

@@ -1 +0,0 @@
-11

debian/control

@@ -1,14 +0,0 @@
-Source: rougail
-Section: admin
-Priority: extra
-Maintainer: Cadoles <contact@cadoles.com>
-Build-depends: debhelper (>=11), python3-all, python3-setuptools
-Standards-Version: 3.9.4
-Homepage: https://forge.cadoles.com/Infra/rougail
-
-Package: rougail
-Architecture: any
-Pre-Depends: dpkg, python3, ${misc:Pre-Depends}
-Depends: ${python:Depends}, ${misc:Depends}
-Description: configuration manager
-

debian/copyright

@@ -1,22 +0,0 @@
-Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: risotto
-Upstream-Contact: Cadoles <contact@cadoles.com>
-Source: https://forge.cadoles.com/Infra/rougail
-
-Files: *
-Copyright: 2019-2020 Cadoles <contact@cadoles.com>
-License: AGPL-3+
-
-License: AGPL-3+
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
- .
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU Affero General Public License for more details.
- .
- You should have received a copy of the GNU Affero General Public License
- along with this program.  If not, see <https://www.gnu.org/licenses/>.

debian/rougail.dirs

@@ -1,3 +0,0 @@
-/var/rougail/patches
-/var/rougail/templates
-/var/rougail/manifests

debian/rules

@@ -1,10 +0,0 @@
-#!/usr/bin/make -f
-# See debhelper(7) (uncomment to enable)
-# output every command that modifies files on the build system.
-#DH_VERBOSE = 1
-
-export PYBUILD_NAME = rougail
-export PYBUILD_DISABLE_python3 = test
-
-%:
-	dh $@ --with python3 --buildsystem=pybuild

debian/source/format

@@ -1 +0,0 @@
-3.0 (quilt)

doc/README.md

@@ -0,0 +1,37 @@
+# Rougail
+
+Rougail est un bibliothèque python3 qui permet de charger des dictionnaires (fichiers au format XML), de charger les variables dans Tiramisu et de générer des templates.
+
+## La bibliothèque
+
+  - [Utiliser la bibliothèque](dev/README.md)
+  - [Personnaliser la configration de la bibliothèque](dev/config.md)
+
+## Les dictionnaires
+
+  - [Les dictionnaires](dictionary/rougail.md)
+  - [Les dictionnaires extra](dictionary/extra.md)
+  - [Convention d'écriture d'un dictionnaire](dictionary/convention.md)
+
+### Les variables
+
+  - [Les familles](family/README.md)
+  - [Les variables](variable/README.md)
+
+### Les services
+
+  - [La gestion d'un fichier](service/file.md)
+  - [La gestion d'un fichier de service systemd](service/override.md)
+  - [La gestion d'une ip](service/ip.md)
+
+### Les contraintes
+
+  - [Les calcules automatiques](fill/README.md)
+  - [Les vérifications des valeurs](check/README.md)
+  - [Les conditions](condition/README.md)
+
+## Les templates
+
+  - Type creole
+  - Type jinja2
+FIXME ^^

doc/check/README.md

@@ -0,0 +1,4 @@
+# Les vérifications des valeurs
+
+  - [Fonction de vérification](function.md)
+  - [Réfinition](redefine.md)

doc/check/function.md

@@ -0,0 +1,53 @@
+# Fonction de vérification
+
+## Vérification stricte des valeurs
+
+Une fonction de vérification est une fonction complémentaire au type qui permet de valider plus précisement le contenu d'une variable.
+
+Voici un exemple simple de validation des valeurs :
+
+```
+<variables>
+  <variable name="my_variable"/>
+</variables>
+<constraints>
+  <check name="islower">
+    <target>my_variable</target>
+  </check>
+</constraints>
+```
+
+La [cible (de type variable)](../target/variable.md) de la fonction de vérification est ici "my_variable".
+
+Dans cette exemple, la valeur de la variable "my_variable" va être validé par la fonction islower.
+
+Voici le contenu de la fonction :
+
+```
+def islower(value):
+    if value is None:
+        return
+    if not value.islower():
+        raise ValueError(f'"{value}" is not lowercase string')
+```
+
+Une fonction de vérification doit prendre en compte 2 aspects important :
+
+- la valeur peut ne pas être renseigné (même si la variable est obligatoire), la valeur None doit être prise en compte
+- si la valeur est invalide, il faut faire un raise de type ValueError avec, si possible, un message explicite.
+
+À partir de maintenant seule None et des valeurs en minuscule seront autorisés.
+
+Il est possible de définir des [paramètres](../param/README.md) à cette fonction.
+
+## Vérification des valeurs avec avertissement
+
+Dans la contrainte, il est possible de spécifier le niveau d'erreur et le mettre en avertissement :
+
+```
+<check name="islower" level="warning">
+  <target>my_variable</target>
+</check>
+```
+
+Dans ce cas une valeur avec une majuscule sera accepté, mais un message d'avertissement apparaitra.

doc/check/redefine.md

@@ -0,0 +1,61 @@
+# Rédéfinition
+
+## Redéfinition des vérification
+
+Dans un premier dictionnaire déclarons notre variable et sa fonction de vérification :
+
+```
+<variables>
+  <variable name="my_variable"/>
+</variables>
+<constraints>
+  <check name="islower">
+    <target>my_variable</target>
+  </check>
+</constraints>
+```
+
+Dans un second dictionnaire il est possible de redéfinir le calcul :
+
+```
+<variables>
+  <variable name="my_variable" redefine="True"/>
+</variables>
+<constraints>
+  <check name="isspace">
+    <target>my_variable</target>
+  </check>
+</constraints>
+```
+
+Dans ce cas, la fonction "islower" exécuté. Si cette fonction ne retourne pas d'erreur, la seconde fonction "isspace" sera exécuté.
+
+## Redéfinition avec suppression d'un calcul
+
+Il se peut que dans un dictionnaire on décide de vérifier la valeur d'une variable.
+
+Dans un second dictionnaire il est possible de supprimer cette vérification.
+
+Dans un premier dictionnaire déclarons notre variable et notre fonction de vérification :
+
+```
+<variables>
+  <variable name="my_variable"/>
+</variables>
+<constraints>
+  <check name="islower">
+    <target>my_variable</target>
+  </check>
+</constraints>
+```
+
+Dans un second dictionnaire supprimer cette vérification :
+
+```
+<variables>
+  <family name="family">
+    <variable name="my_variable" redefine="True" remove_check="True"/>
+  </family>
+</variables>
+```
+

doc/condition/README.md

@@ -0,0 +1,5 @@
+# Les conditions
+
+  - [Déclaration d'une condition](condition.md)
+  - [Les différentes conditions](conditions.md)
+  - [Réfinition](redefine.md)

doc/condition/condition.md

@@ -0,0 +1,76 @@
+# Les conditions
+
+## Un condition
+
+Les conditions permettent d'ajouter ou de supprimer des propriétés à une [variable](../variable/README.md), une [famille](../family/README.md), un [service](../service/README.md), un [fichier](../service/file.md) ou une [ip](../service/ip.md) suivant le contexte.
+
+Nous allons nous concentrer ici sur la condition hidden_if_in, mais [il existe d'autre conditions](conditions.md).
+
+La condition hidden_if_in permet de cacher une variable où une famille à l'utilisateur, mais cette variable est toujours accessible dans un calcul, un vérification ou dans un template.
+
+```
+<variables>
+  <variable name="condition" type="boolean"/>
+  <variable name="my_variable"/>
+</variables>
+
+<constraints>
+  <condition name="hidden_if_in" source="condition">
+    <param>True</param>
+    <target>my_variable</target>
+  </condition>
+</constraints>
+```
+
+Le [paramètres](../param/README.md) de la condition permet de définir les valeurs que doit avoir la source pour appliquer l'action.
+
+La [cible](../target/README.md) de la condition est ici "my_variable".
+
+Donc ici la variable est caché à l'utilisateur si la variable "condition" est à True (le paramètre).
+
+## Un condition avec plusieurs paramètres
+
+Il est également possible de mettre plusieurs paramètre :
+
+```
+<variables>
+  <variable name="condition"/>
+  <variable name="my_variable"/>
+</variables>
+
+<constraints>
+  <condition name="hidden_if_in" source="condition">
+    <param>yes</param>
+    <param>maybe</param>
+    <target>my_variable</target>
+  </condition>
+</constraints>
+```
+
+## Une condition optionnelle
+
+Il est possible de définir une condition avec une variable source qui n'existe pas dans toutes les contextes.
+
+Dans ce cas, on met la condition en "optionnelle".
+
+Si la variable source existe, la condition s'applique.
+
+Si la variable source n'existe pas :
+
+- si le nom fini en _if_in (par exemple hidden_if_in), l'action est forcée sans condition (les cibles sont hidden)
+- si le nom fini en _if_not_in (par exemple hidden_if_not_in), la condition est totalement ignorée
+
+Ces deux comportements peuvent être changé à tout moment avec l'attribut "apply_on_fallback". Dans ce cas :
+
+- si la valeur de "apply_on_fallback" est "True", l'action est forcée sans condition
+- si la valeur de "apply_on_fallback" est "False", la condition est totalement ignorée 
+
+Exemple :
+
+```
+<condition name="hidden_if_in" source="condition" optional="True", apply_on_fallback="False">
+  <param>yes</param>
+  <param>maybe</param>
+  <target>my_variable</target>
+</condition>
+```

doc/condition/conditions.md

@@ -0,0 +1,29 @@
+# Les conditions
+
+## Les conditions \_if_in et \_if_not_in
+
+Il existe deux types de conditions :
+
+- les conditions dont le nom fini par \_if_in : dans ce cas si la variable source est égal à un des paramètres, l'action est effective
+- les conditions dont le nom fini par \_if_not_in : dans ce cas si la variable source est différents de tous les paramètres, l'action est effective
+
+## Désactivation
+
+Il est possible de désactiver une [variable](../variable/README.md) ou une [famille](../family/README.md) avec les conditions :
+
+- disabled_if_in
+- disabled_if_not_in
+
+## Caché
+
+Il est possible de cacher une [variable](../variable/README.md), une [famille](../family/README.md), un [service](../service/README.md), un [fichier](../service/file.md) ou une [ip](../service/ip.md) avec les conditions :
+
+- hidden_if_in
+- hidden_if_not_in
+
+## Obligatoire
+
+Il est possible de rendre obligatoire une [variable](../variable/README.md) avec les conditions :
+
+- mandatory_if_in
+- mandatory_if_not_in

doc/condition/redefine.md

@@ -0,0 +1,29 @@
+# Rédéfinition
+
+Il se peut que dans un dictionnaire on décide de définir une condition.
+
+Dans un second dictionnaire il est possible de supprimer cette condition.
+
+Dans un premier dictionnaire déclarons notre variable et notre calcule :
+
+```
+<variables>
+  <variable name="condition" type="boolean"/>
+  <variable name="my_variable"/>
+</variables>
+
+<constraints>
+  <condition name="hidden_if_in" source="condition">
+    <param>True</param>
+    <target>my_variable</target>
+  </condition>
+</constraints>
+```
+
+Dans un second dictionnaire supprimer ce calcul :
+
+```
+<variables>
+  <variable name="condition" redefine="True" remove_condition="True"/>
+</variables>
+```

doc/dev/README.md

@@ -0,0 +1,191 @@
+# La bibliothèque rougail
+
+Rougail est une bibliothèque simple a utiliser.
+
+Dans les exemples suivant, nous utilisons la configuration par défaut de Rougail. Vous pouvez [personnaliser les répertoires utilisés](config.md).
+
+## Convertisons un dictionnaire en objet tiramisu
+
+Commençons par créer un dictionnaire simple.
+
+Voici un premier dictionnaire /srv/rougail/dictionaries/00-base.xml :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <variables>
+    <variable name="my_variable">
+      <value>my_value</value>
+    </variable>
+  </variables>
+</rougail>
+```
+
+Construisons les objets tiramisu :
+
+```python
+from rougail import RougailConvert
+
+rougail = RougailConvert()
+rougail.save('example.py')
+```
+
+Un nouveau fichier 'example.py' va être créé dans le répertoire local
+
+## Convertisons un dictionnaire extra en objet tiramisu
+
+En plus du dictionnaire précédent, créons un dictionnaire extra /srv/rougail/extra_dictionaries/00-base.xml
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <variables>
+    <variable name="my_variable_extra">
+      <value>my_value_extra</value>
+    </variable>
+  </variables>
+</rougail>
+```
+
+Construisons les objets tiramisu :
+
+```python
+from rougail import RougailConvert, RougailConfig
+
+RougailConfig['extra_dictionaries']['example'] = ['/srv/rougail/extra_dictionaries/']
+
+rougail = RougailConvert()
+rougail.save('example.py')
+```
+
+## Templatisons un template
+
+Nous créons un dictionnaire complémentaire pour ajouter notre template /srv/rougail/dictionaries/00-template.xml :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <services>
+    <service name="example">
+      <file name="/etc/example.conf"/>
+    </service>
+  </services>
+</rougail>
+```
+
+Et un template /srv/rougail/templates/example.conf :
+
+```
+The value: %%my_variable
+
+The extra value: %%example.my_variable_extra
+```
+
+Générons le fichier tiramisu :
+
+```python
+from rougail import RougailConvert, RougailConfig
+
+RougailConfig['extra_dictionaries']['example'] = ['/srv/rougail/extra_dictionaries/']
+
+rougail = RougailConvert()
+rougail.save('example.py')
+```
+
+Créer les répertoires utils pour la templatisation :
+
+```bash
+mkdir /srv/rougail/destinations /srv/rougail/tmp
+```
+
+Générons le template :
+
+```python
+import asyncio
+from example import option_0
+from tiramisu import Config
+from rougail import RougailSystemdTemplate
+
+async def template():
+    config = await Config(option_0)
+    engine = RougailSystemdTemplate(config)
+    await engine.instance_files()
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(template())
+loop.close()
+```
+
+Le fichier /srv/rougail/destinations/etc/example.conf est maintenant créé avec le contenu attendu suivant :
+
+```
+The value: my_value
+
+The extra value: my_value_extra
+```
+
+## Appliquons un patch au template
+
+Il peut être intéressant de réaliser un patch à un template pour corriger un problème spécifique à notre environnement, sans attendre que le mainteneur du template n'est fait la correction.
+
+Testons en créant le patch /srv/rougail/patches/example.conf.patch :
+
+```
+--- /srv/rougail/templates/example.conf	2021-02-13 19:41:38.677491087 +0100
++++ tmp/example.conf	2021-02-13 20:12:55.525089820 +0100
+@@ -1,3 +1,5 @@
+ The value: %%my_variable
+ 
+ The extra value: %%example.my_variable_extra
++
++Add by a patch
+```
+
+Le patch est bien appliquer sur notre fichier /srv/rougail/destinations/etc/example.conf :
+
+```
+The value: my_value
+
+The extra value: my_value_extra
+
+Add by a patch
+```
+
+Deux choses importantes à savoir sur les patchs :
+
+- le nom du patch est obligatoire le nom du template source + ".patch"
+- la deuxième ligne doit toujours commencer par "+++ tmp/" + le nom du template source
+
+## Créons une fonction personnalisé
+
+Nous créons un dictionnaire complémentaire pour ajouter un calcul à la variable "my_variable" dans /srv/rougail/dictionaries/00-fill.xml :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <constraints>
+    <fill name="return_no">
+      <target>my_variable</target>
+    </fill>
+  </constraints>
+</rougail>
+```
+
+Puis créons la fonction "return_no" dans /srv/rougail/functions.py :
+
+```python
+def return_no():
+    return 'no'
+```
+
+Après avoir reconverti les dictionnaires et regénérer le template nous avons donc le contenu du fichier /srv/rougail/destinations/etc/example.conf :
+
+```
+The value: no
+
+The extra value: my_value_extra
+
+Add by a patch
+```
+
+La valeur de la variable "my_variable" est bien calculé à partir de la fonction "return_no".

doc/dev/config.md

@@ -0,0 +1,132 @@
+# Personnalisons la configuration de Rougail
+
+La configuration de rougail se trouve dans l'objet RougailConfig :
+
+```python
+from rougail import RougailConfig
+```
+
+C'est un simple dictionnaire python avec différentes clefs.
+
+Pour modifier il suffit de faire :
+
+```python
+RougailConfig[key] = value
+```
+
+## Ajout d'une fonction de conversion
+
+Les fonctions de conversion fait parti du moteur de rougail. Il converti les informations des dictionnaires pour créer des variables Tiramisu.
+
+La clef "extra_annotators" permet d'ajouter des fonctions complémentaires.
+
+## Configuration de chargement des dictionnaires
+
+### Les répertoires des dictionnaires
+
+Il existe deux types de répertoires de dictionnaires :
+
+- les dictionnaires principaux avec la clef "dictionaries_dir". La valeur par défaut est ['/srv/rougail/dictionaries']. Cette variable doit contenir la liste des répertoires contenants des dictionnaires.
+
+Les dictionnaires sont chargés dans l'ordre des répertoires. Chaque répertoire est chargé les uns après les autres. A l'intérieur de ces répertoires les fichiers XML seront classés par ordre alphabétique.
+
+Il n'y a pas de classement par ordre alphabétique de l'ensemble des fichiers XML de tous les répertoires.
+
+Les familles et variables de ces dictionnaires sont classés, par défaut, dans l'espace de nom "rougail". Il est possible de changer le nom de cet espace de nom avec la clef "variable_namespace".
+
+- les dictionnaires extra avec la clef "extra_dictionaries". La valeur est un dictionnaire avec l'ensemble des espaces de nom. La clef étant l'espace de nom et la valeur étant une liste de répertoire.
+
+Par exemple pour ajouter l'extra "example" il faut faire :
+
+```
+RougailConfig['extra_dictionaries']['example'] = ['/dir1', '/dir2']
+```
+
+Les dictionnaires sont chargés dans le même ordre que les dictionnaires principaux.
+
+### La DTD
+
+Rougail a besoin du fichier de DTD pour lire les fichiers dictionnaire.
+
+Par défaut le fichier de la DTD est dans le sous répertoire "data" du répertoire de code. Le nom du fichier est rougail.dtd.
+
+Pour pouvez changer le répertoire de destination de la DTD et le nom du fichier avec la clef "dtdfilename".
+
+### Le fichier de fonction
+
+Le fichier qui contient les fonctions personnalisés est géré dans la clef "functions_file" et a comme valeur par défaut "/srv/rougail/functions.py". Cette clef peut contenir une liste s'il y a plusieurs fichiers.
+
+Les fonctions doivent retourner une valeur, même si la variable que l'on calcul est une variable multiple.
+Si la fonction peut retourner une valeur multiple (une liste), il faut mettre le nom de la fonction dans la clef "multi_functions".
+
+### La variable auto_freeze
+
+La propriété auto_freeze n'est appliqué que une variable spécifique passe à True. Par défaut le nom de la variable est "instancied_module", mais il est possible de changer le nom de cette variable via la clef "auto_freeze_variable".
+
+### Les modes
+
+Les modes sont personnalisables dans Rougail. Par défaut les modes sont "basic", "normal" et "expert".
+Il est possible de changer cette liste via la clef "modes_level".
+
+Si vous changer ces valeurs, penser à changer les modes par défaut des familles et des variables.
+
+### Le mode par défaut pour une famille
+
+Le mode par défaut d'une famille est "basic". Il est possible de changer le mode par défaut d'une famille via la clef "default_family_mode".
+
+### Le mode par défaut pour une variable
+
+Le mode par défaut d'une variable est "normal". Il est possible de changer le mode par défaut d'une variable via la clef "default_variable_mode".
+
+### Le nom des fonctions internes
+
+Il est possible d'ajouter des fonctions interne via la clef "internal_functions".
+
+## Configuration de la templatisation
+
+### Le répertoire des templates
+
+Le répertoire des templates est géré dans la clef "templates_dir" et a comme valeur par défaut : "/srv/rougail/templates". Cette clef peut contenir une liste s'il y a plusieurs répertoires.
+
+### Le répertoire des patchs
+
+Le répertoire des patches est géré dans la clef "patches_dir" et a comme valeur par défaut : "/srv/rougail/patches".
+
+### Le répertoire temporaire
+
+Le répertoire temporaire est utile lors de la génération de template. Il contient une copie des templates avec, éventuellement, les patches appliqués sur les templates.
+
+Le répertoire de temporaire est géré dans la clef "tmp_dir" et a comme valeur par défaut : "/srv/rougail/tmp".
+
+### Le répertoire de destination des fichiers générés
+
+Le répertoire de destination des fichiers générés est géré dans la clef "destinations_dir" et a comme valeur par défaut : "/srv/rougail/destinations".
+
+### Le nom du répertoire où se trouve les fichiers pour tmpfile.d
+
+Le répertoire où se trouve les fichiers tmpfile.d sont par défaut dans "/usr/local/lib", il est possible de changer ce répertoire avec la clef "tmpfile_dest_dir".
+
+
+## La configuration par défaut des fichiers
+
+### Le moteur de templates par défaut
+
+Le moteur de template est géré dans la clef "default_files_engine" et a comme valeur par défaut : "creole". Les valeurs possible sont "none", "creole" ou "jinja2".
+
+### Les droits par défaut des fichiers
+
+Les droits des fichiers générés est géré dans la clef "default_files_mode" et a comme valeur par défaut : "0644".
+
+### Le propriétaire par défaut des fichiers
+
+Le propriétaire des fichiers générés est géré dans la clef "default_files_owner" et a comme valeur par défaut : "root".
+Le groupe propriétaire des fichiers générés est géré dans la clef "default_files_group" et a comme valeur par défaut : "root".
+
+### La méthode d'inclusion par défaut des fichiers
+
+La méthode d'inclusion des fichiers générés est géré dans la clef "default_files_included" et a comme valeur par défaut : "no". Les valeurs possible sont "no", "name" et "content".
+                 'default_files_included': 'no',
+
+## La configuration du moteur de templates
+
+Le moteur de template est géré dans la clef "default_overrides_engine" et a comme valeur par défaut : "creole". Les valeurs possible sont "none", "creole" ou "jinja2".

doc/dictionary/convention.md

@@ -0,0 +1,13 @@
+# Convention de rédaction d'un dictionnaire
+
+## Ordonnancement du dictionnaire
+
+L'ordre des informations mise dans le dictionnaire est idéalement :
+
+- services
+- variables
+- constraintes
+
+## Le nom d'espace
+
+Le nom d'espace dans un dictionnaire est de deux espaces.

doc/dictionary/extra.md

@@ -0,0 +1,12 @@
+# Les dictionnaires extra
+
+Un extra est un espace de nom différent. L'idée et de pouvoir classer les variables par thématique.
+
+Les espaces de nom extra doivent être déclaré au moment [de la configuration de Rougail](../dev/config.md).
+
+Dans cet espace de nom :
+
+- des variables et des familles peuvent avoir le même nom dans différentes familles
+- la valeur d'un cible, source, leader ou follower des contraintes doivent être avec un chemin complet
+- on ne peut pas déclarer des services dans cet espace de nom
+- dans un template il faut utiliser des chemins complet (%%my_extra.my_family.my_variable ou %%my_extra.my_family.leader.follower pour une variable suiveuse)

doc/dictionary/rougail.md

@@ -0,0 +1,22 @@
+# Les dictionnaires
+
+## Un dictionnaire ?
+
+Un dictionnaire est un fichier XML donc la structure est décrite dans cette documentation.
+
+Un dictionnaire contient en ensemble de variable chargé dans Tiramisu, utilisable à tout moment, notamment dans des templates.
+
+Les familles, les variables et les contraintes peuvent être défini dans plusieurs dictionnaires. Ces dictionnaires s'aggrège alors.
+
+Il est également possible de redéfinir des éléments pour changer les comportement d'une variable ou d'un service.
+
+## L'espace de nom par défaut
+
+L'espace de nom par défaut s'appelle "rougail" ([ce nom est personnalisable](../dev/config.md)).
+
+Cet espace de nom est un peu particulier :
+
+- le nom des variables et des familles doivent être unique pour l'ensemble de cet espace (même si ces variables ou familles sont dans des familles différentes)
+- la valeur d'un cible, source, leader ou follower des contraintes peuvent être avec nom de la variable ou de la famille ou leurs chemins complet
+- on peut déclarer des services dans cet espace de nom
+- dans un template on peut utiliser cette variable sans le chemin complet (%%my_variable) ou avec (%%rougail.my_family.my_variable)

doc/family/README.md

@@ -0,0 +1,6 @@
+# Famille
+
+  - [Une famille](simple.md)
+  - [Famille crée dynamiquement](auto.md)
+  - [Les variables meneuses ou suiveuses](leadership.md)
+

doc/family/auto.md

@@ -0,0 +1,27 @@
+# Famille crée dynamiquement
+
+Pour créer une famille dynamiquement, il faut créer une famille fictive lié à une variable.
+Le nom et la description de la famille et des variables qu'elle contient sera en réalité le prefix du nouveau nom/description. Le suffix viendra de la variable liée.
+
+Par exemple :
+
+```
+<variable name='varname' multi="True">
+  <value>val1</value>
+  <value>val2</value>
+</variable>
+<family name="my_dyn_family_" dynamic="varname" description="Describe ">
+  <variable name="my_dyn_var_"/>
+</family>
+```
+
+Créera deux familles :
+
+- la famille dynamique : "my_dyn_family_val1" avec la description "Describe val1"
+- la famille dynamique : "my_dyn_family_val2" avec la description "Describe val2"
+
+Dans la famille dynamique "my_dyn_family_val1" on retrouvera une variable "my_dyn_var_val1".
+
+Bien évidement si le contenu de "varname" venait a évolué, de nouvelles familles dynamiques pouvent apparaitre ou des familles dynamiques peuvent disparaître.
+
+Attention la variable lié à la famille doit être obligatoirement une variable multiple et il n'est pas possible de mettre une famille dans une famille dynamique.

doc/family/leadership.md

@@ -0,0 +1,56 @@
+# Variable meneuse ou suiveuse
+
+## Variable meneuse
+
+Une variable meneuse est une variable qui va guider la longueur d'autre variables (appelé variables suiveuse).
+
+Une variable meneuse est une [variable](../variable/README.md) qui est obligatoirement de type multiple.
+
+Une variable meneuse peut être obligatoire.
+
+Le [mode](../mode.md) par défaut correspond au plus petit mode définit par l'utilisateur des variables suiveuses.
+
+## Variable suiveuse
+
+Une variable suiveuse est une variable donc la longueur n'est pas déterminé par elle-même, mais est identique à celle de la variable meneuse dont elle dépend.
+
+Une variable suiveuse est une variable placer juste derrière une variable meneuse ou une autre variable suiveuse.
+
+L'ordre de définition des variables suiveuses est important.
+
+Cette variable peut être de type multiple. Dans ce cas, pour un index determiné, il est possible de mettre plusieurs valeurs à une même variable.
+
+Une variable suiveuse peut être obligatoire. Cela signifie que lorsqu'une variable meneuse est renseigné, il faut obligatoirement que la variable suiveuse est également une valeur à l'index considéré.
+Si aucune valeur n'est définit pour la variable meneuse, aucune valeur n'est a spécifié pour la variable suiveuse.
+
+Le [mode](../mode.md) par défaut d'une variable suiveuse correspond au [mode](../mode.md) de la variable meneuse.
+
+Si une variable meneuse est caché ou désactivé, les variables suiveuses le seront également.
+
+## Définition des variables meneuse et suiveuse
+
+Les variables meneuses et suiveuses doivent dans une famille de type "leadership".
+
+Voici un exemple de définition d'une variable meneuse et de deux variables meneuses :
+
+```
+<variables>
+  <family name="family" leadership='True'>
+    <variable name="leader" multi='True'/>
+    <variable name="follower1"/>
+    <variable name="follower2" multi='True'/>
+  </family>
+</variables>
+```
+
+## Ajout d'une nouvelle variable suiveuse
+
+Pour ajouter, dans un nouveau dictionnaire, une variable suiveuse à notre groupe, rien de plus simple, il suffit définir une ou des nouvelles variables dans la famille :
+
+```
+<variables>
+  <family name="family">
+    <variable name="follower3"/>
+  </family>
+</variables>
+```

doc/family/simple.md

@@ -0,0 +1,68 @@
+# Une famille
+
+Une famille est un conteneur de variables. Elle peut contenir également des familles.
+
+Pour décrire une famille il faut mettre au minimum un nom :
+
+```
+<family name="my_family"/>
+```
+
+Cette famille doit être placé dans une balise [variables](../variables.md) :
+
+```
+<variables>
+  <family name="my_family"/>
+</variables>
+```
+
+Ou dans une autre famille :
+
+```
+<variables>
+  <family name="my_family">
+    <family name="second_family"/>
+  </family>
+</variables>
+```
+
+Attention, une famille vide sera automatiquement supprimée.
+
+## Description et aide de la famille
+
+En plus d'un nom, il est possible de mettre une "description" à la famille. C'est une information "utilisateur" qui nous permettra d'avoir des informations complémentaires sur le contenu de cette famille :
+
+```
+<family name="my_family" description="This is a great family"/>
+```
+
+En plus de la description, il est possible de préciser une aide complémentaire :
+
+```
+<family name="my_family" help="This is a great family"/>
+```
+
+## Mode de la famille
+
+Le [mode](../mode.md) par défaut d'une famille correspond au [mode](../mode.md) du mode le plus petit entre la famille parente, les variables enfants ou des familles enfants qui sont contenu dans cette famille.
+
+Changer le [mode](../mode.md) d'une famille permet de définir le [mode](../mode.md) par défaut des variables ou des familles inclusent dans cette famille.
+
+Pour définir le [mode](../mode.md) :
+
+```
+<family name="my_family" mode="expert"/>
+```
+
+## Famille invisible
+
+Il est possible de cacher une famille, ainsi que toutes les variables et des familles inclusent dans cette famille.
+
+Cacher une famille signifie qu'elle ne sera pas visible lorsqu'on modifie la configuration du service.
+Par contre ces variables sont accessibles lorsqu'on va utiliser ces variables.
+
+Pour cacher une famille :
+
+```
+<family name="my_family" hidden="True"/>
+```

doc/fill/README.md

@@ -0,0 +1,7 @@
+# Les variables calculées
+
+Une variable calculée est une variable donc sa valeur est le résultat d'une fonction python.
+
+  - [Valeur calculée de la variable](value.md)
+  - [Réfinition](redefine.md)
+  - [Exemples de calcule](examples.md)

doc/fill/examples.md

@@ -0,0 +1,48 @@
+# Exemples de calcule
+
+## Calculer les noms de modèle à généré à partir d'une variable
+
+Créeons deux variables multiples, une pour accueillir la liste des éléments du nom variable (ici `zones_list`), la seconde étant le nom du fichier calculé (ici `netwokd_configurations`) :
+
+```
+<variables>
+  <variable name="zones_list" type="string" multi="True">
+    <value>zone1</value>
+    <value>zone2</value>
+    <value>zone3</value>
+  </variable>
+  <variable name="netwokd_configurations" type="filename" multi="True" hidden="True"/>
+</variables>
+```
+
+Calculons la valeur de la seconde variable à partir de la première :
+
+```
+<constraints>
+  <fill name="calc_value">
+    <param>/systemd/network/10-</param>
+    <param type="variable">zones_list</param>
+    <param>-risotto.network</param>
+    <param name="join"></param>
+    <param name="multi" type="boolean">True</param>
+    <target>netwokd_configurations</target>
+  </fill>
+</constraints>
+```
+
+Le contenu de la variable `netwokd_configurations` sera alors :
+
+- /systemd/netword/10-zone1/risotto.network
+- /systemd/netword/10-zone2/risotto.network
+- /systemd/netword/10-zone3/risotto.network
+
+Enfin déclarer une balise file en utilisant ces deux variables :
+
+```
+<services>
+  <service name="systemd-networkd">
+    <file file_type="variable" source="network" variable="zones_list">netwokd_configurations</file>
+  </service>
+</services>
+```
+

doc/fill/redefine.md

@@ -0,0 +1,58 @@
+# Rédéfinition
+
+## Redéfinition des calcules
+
+Dans un premier dictionnaire déclarons notre variable et notre calcule :
+
+```
+<variables>
+    <variable name="my_calculated_variable"/>
+</variables>
+<constraints>
+    <fill name="return_no">
+        <target>my_calculated_variable</target>
+    </fill>
+</constraints>
+```
+
+Dans un second dictionnaire il est possible de redéfinir le calcul :
+
+```
+<variables>
+    <variable name="my_calculated_variable" redefine="True"/>
+</variables>
+<constraints>
+    <fill name="return_yes">
+        <target>my_calculated_variable</target>
+    </fill>
+</constraints>
+```
+
+Dans ce cas, à aucun moment la fonction "return_no" ne sera exécuté. Seul la fonction "return_yes" le sera.
+
+## Redéfinition avec suppression d'un calcul
+
+Il se peut que dans un dictionnaire on décide de définir une valeur par défaut à une variable via un calcul.
+
+Dans un second dictionnaire il est possible de supprimer ce calcul.
+
+Dans un premier dictionnaire déclarons notre variable et notre calcule :
+
+```
+<variables>
+    <variable name="my_calculated_variable"/>
+</variables>
+<constraints>
+    <fill name="return_no">
+        <target>my_calculated_variable"</target>
+    </fill>
+</constraints>
+```
+
+Dans un second dictionnaire supprimer ce calcul :
+
+```
+<variables>
+    <variable name="my_calculated_variable" redefine="True" remove_fill="True"/>
+</variables>
+```

doc/fill/value.md

@@ -0,0 +1,112 @@
+# Valeur calculée de la variable
+
+## Variable avec une valeur par défaut calculée
+
+Créons une variable dont la valeur est retournée par la fonction "return_no" :
+
+```
+<variables>
+  <variable name="my_calculated_variable"/>
+</variables>
+<constraints>
+  <fill name="return_no">
+    <target>my_calculated_variable</target>
+  </fill>
+</constraints>
+```
+
+Puis créons la fonction "return_no" :
+
+```
+def return_no():
+    return 'no'
+```
+
+La [cible (de type variable)](../target/variable.md) du calcul est ici "my_calculated_variable".
+
+Dans ce cas, la valeur par défaut est la valeur retournée par la fonction (ici "no"), elle sera calculée tant que l'utilisateur n'a pas de spécifié de valeur à cette variable.
+
+Attention, si une valeur par défaut est définit dans la variable "my_calculated_variable" :
+
+```
+<variable name="my_calculated_variable">
+  <value>yes</value>
+</variable>
+```
+Cette valeur par défaut sera complètement ignorée. C'est le calcul qui en définira la valeur.
+
+Il est possible de définir des [paramètres](../param/README.md) à cette fonction.
+
+## Variable avec une valeur calculée
+
+En ajoutant le paramètre "hidden" à "True" dans la variable précédente, l'utilisateur n'aura plus la possibilité de modifié la valeur. La valeur de la variable sera donc systématiquement calculée :
+
+```
+<variable name="my_calculated_variable" hidden="True"/>
+```
+
+Si une condition "hidden_if_in" est spécifié à la variable, la valeur sera modifiable par l'utilisateur si elle n'est pas cachée mais elle sera systèmatiquement calculée (même si elle a déjà était modifiée) si la variable est cachée.
+
+## Variable meneuse ou suiveuse avec valeur calculé
+
+Une [variable suiveuse](../family/leadership.md) ne peut pas être calculé automatiquement.
+Une [variable meneuse](../family/leadership.md) peut être calculé automatiquement.
+Si la variable n'est pas multiple, il ne faut pas que le calcule retourne une liste.
+
+## Variable dynamique avec une valeur calculée
+
+Il est également possible de calculer [une variable d'une famille dynamique](../family/auto.md) à partir d'une variable standard :
+
+```
+<variables>
+  <variable name='suffixes' type='string' description="Suffixes of dynamic family" multi="True">
+    <value>val1</value>
+    <value>val2</value>
+  </variable>
+  <variable name="my_variable" type="string" description="My variable">
+    <value>val</value>
+  </variable>
+  <family name='dyn' dynamic="suffixes">
+    <variable name="my_calculated_variable_dyn\_" type="string" description="My calculated variable"/>
+      <value>val</value>
+    </variable>
+  </family>
+</variables>
+<constraints>
+  <fill name="return_value">
+    <param type="variable">my_variable</param>
+    <target>my_calculated_variable_dyn_</target>
+  </fill>
+</constraints>
+```
+
+Dans ce cas, les variables dynamiques "my_calculated_variable_dyn_" seront calculés à partir de la valeur de la variable "my_variable".
+Que cela soit pour la variable "my_calculated_variable_dyn_val1" et "my_calculated_variable_dyn_val2".
+
+Par contre, il n'est pas possible de faire un calcul pour une seule des deux variables issues de la variable dynamique.
+Si c'est ce que vous cherchez à faire, il faudra prévoir un traitement particulier dans votre fonction.
+
+Dans ce cas, il faut explicitement demander la valeur du suffix dans la fonction :
+
+```
+<constraints>
+  <fill name="return_value_suffix">
+    <param type="variable">my_variable</param>
+    <param type="suffix"/>
+    <target>my_calculated_variable_dyn_</target>
+  </fill>
+</constraints>
+```
+
+Et ainsi faire un traitement spécifique pour ce suffix :
+
+```
+def return_value_suffix(value, suffix):
+    if suffix == 'val1':
+        return value
+```
+
+## Variable avec valeur calculée obligatoire
+
+Par défaut les variables calculées ne sont pas des variables obligatoires.
+Dans ce cas un calcul peut retourner "None" ou "", mais surtout un utilisateur peut spécifier une valeur nulle à cette variable. Dans ce cas le calcul ne sera plus réalisé.

doc/mode.md

@@ -0,0 +1,10 @@
+Mode
+====
+
+Par défault, il existe trois "mode" dans Rougail :
+
+- basic : variables indispensables à la mise en place d'un service
+- normal : variables couramment modifié par l'utilisateur
+- expert : variables a manipuler avec précausion et en toute connaissance de cause
+
+Il est possible de personnaliser les modes dans la [configuration de rougail](dev/config.md)

doc/param/README.md

@@ -0,0 +1,7 @@
+# Paramètre de la fonction
+
+  - [Paramètre positionnel ou nommée](positional.md)
+  - [Type de paramètre simple](simple.md)
+  - [Type de paramètre "variable"](variable.md)
+  - [Type de paramètre "information"](information.md)
+

doc/param/information.md

@@ -0,0 +1,24 @@
+# Paramètre de type information
+
+## Les informations de la configuration
+
+Le paramètre peut être la valeur est issue d'une information de la configuration :
+
+```
+<param type="information">server_name</param>
+```
+
+Dans ce cas, l'information de la configuration "server_name" sera utilisé comme valeur du paramètre.
+Si l'information n'existe pas, la paramètre aura la valeur "None".
+
+## Les informations de la cible
+
+Le paramètre peut être la valeur est issue d'une information de la cible du calcul :
+
+```
+<param type="target_information">test</param>
+<param type="target_information">help</param>
+```
+
+Dans ce cas, l'information de la configuration "test" ou "help" sera utilisé comme valeur du paramètre.
+Si l'information n'existe pas, la paramètre aura la valeur "None".

doc/param/positional.md

@@ -0,0 +1,26 @@
+# Paramètre positionnel
+
+Déclarons un paramètre positionnel :
+
+```
+<param>no</param>
+```
+
+Créons la fonction correspondante :
+
+```
+def return_value(value):
+    return value
+```
+
+La variable "value" de la fonction "return_value" aura donc "no" comme valeur puisque le paramètre aura la valeur fixe "no".
+
+# Paramètre nommée
+
+Déclarons maintenant un paramètre nommée :
+
+```
+<param name="valeur">no</param>
+```
+
+Dans ce cas la fonction return_value sera exécuté avec le paramètre nommé "valeur" dont sa valeur sera "no".

doc/param/simple.md

@@ -0,0 +1,52 @@
+# Paramètre de type "texte"
+
+Déclarons un paramètre avec une string :
+
+```
+<param type="string">no</param>
+```
+
+C'est le type par défaut pour un paramètre.
+
+# Paramètre de type "nombre"
+
+Déclarons un paramètre avec un nombre :
+
+```
+<param type="number">1</param>
+```
+
+Créons la fonction correspondante :
+
+```
+def return_value_with_number(value):
+    if value == 1:
+        return 'no'
+    return 'yes'
+```
+
+La variable aura donc "no" comme valeur puisque le paramètre aura la valeur fixe "1".
+
+# Paramètre de type "booléen"
+
+Déclarons un paramètre avec un booléen :
+
+```
+<param type="boolean">True</param>
+```
+
+# Paramètre de type "nil"
+
+Le paramètre peut être une valeur null (None en python) :
+
+```
+<param type="nil"/>
+```
+
+# Paramètre de type "space"
+
+Les paramètres sont chargés en supprimer les espaces en début ou fin de chaîne. Ce qui rend impossible d'avoir un paramètre " ". Avec le type "space", le paramètre sera donc un simple espace :
+
+```
+<param type="space"/>
+```

doc/param/variable.md

@@ -0,0 +1,55 @@
+# Paramètre de type "variable"
+
+Imaginons que la variable "my_variable" pré-existe. La valeur de la variable sera utilisé comme paramètre :
+
+```
+<param type="variable">my_variable</param>
+```
+
+[Les variables meneuses ou suiveuses](../family/leadership.md) peuvent être utilisé sans soucis comme paramètre.
+
+## Paramètre avec variable potentiellement non existante
+
+Suivant le contexte une variable peut exister ou ne pas exister.
+
+Un paramètre de type "variable" peut être "optional" :
+
+```
+<param type="variable" optional="True">unknow_variable</param>
+```
+
+Si la variable "unknow_variable" n'existe pas, le paramètre ne sera pas passé à la fonction.
+
+Si maintenant on créé un nouveau dictionnaire en créant cette variable, la fonction sera exécuté avec le paramètre.
+
+## Paramètre avec variable potentiellement désactivée
+
+Si une variable est désactivé, l'utilisation de cette variable peut poser problème.
+
+Il est possible de ne pas générer d'erreur si une variable est désactivé en utilisant le paramètre "propertyerror" :
+
+```
+<param type="variable" propertyerror="False">variable1</param>
+```
+
+Dans ce cas, si la variable est désactivé, le paramètre n'est jamais donnée à la fonction de destination.
+
+## Paramètre avec variable dynamique
+
+Il est possible de faire un calcul avec comme paramètre [une variable d'une famille dynamique](../family/auto.md) mais pour une suffix particulier.
+
+Par exemple :
+
+```
+<param type="variable">vardynval1</param>
+```
+
+Dans ce cas, la valeur du paramètre de la fonction sera la valeur de la variable "vardyn" pour la famille ayant le suffix "val1".
+
+Il peut être utile de récupérer la valeur du suffix dans la fonction, pour cela il suffit de mettre un paramètre de type suffix :
+
+```
+<param type="suffix"/>
+```
+
+Dans l'exemple précédent la valeur de ce paramètre sera "val1".

doc/service/README.md

@@ -0,0 +1,107 @@
+# La gestion d'un service
+
+## La base service
+
+Un service est inclut dans un conteneur [services](../services.md).
+
+Cette balise permet de définir tous les éléments ([fichier](file.md), [IP](ip.md) et [réécriture](override.md)) liés à un service ou à démon.
+
+Il faut, à la création du service, préciser son nom :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <services>
+    <service name="squid"/>
+  </services>
+</rougail>
+```
+
+## Les types de service
+
+Les services peuvent être de plusieurs type :
+
+- service (valeur par défaut) : cela signifie que c'est un service systemd qui est activer au démarrage de la machine
+- mount : fichier utilisé par systemd-mount
+- swap : fichier utilisé par systemd-swap
+- timer : tâche planifié pour systemd
+
+```
+<service name="dev-disk-by\x2dpartlabel-swap" type="swap"/>
+```
+
+## Les targets de service
+
+Active le service systemd pour une target systemd déterminé. Par défaut, l'activation du service n'est pas gérer par rougail.
+
+```
+<service name="squid" target="multi-user"/>
+```
+
+## La génération du fichier service
+
+Le fichier de description du service peut être fourni directement par la distribution GNU/Linux utilisé, mais il peut également être fournit par l'administrateur.
+Dans ce cas, il est possible de créé un template, dont le nom est obligatoirement la valeur de la balise "name" + "." + la valeur de la base "type".
+
+Deux types de template sont aujourd'hui disponible :
+
+- creole
+- jinja2
+
+```
+<service name="dev-disk-by\x2dpartlabel-swap" type="swap" engine="creole"/>
+```
+
+Dans ce cas, rougail utilisera le template "dev-disk-by\x2dpartlabel-swap.swap" pour générer le fichier systemd de gestion de ce service.
+
+## Le service factice
+
+Un service peut être factice, donc non géré par le système de service du système :
+
+```
+<service name="ldap_client" manage="False"/>
+```
+
+Un service factice est généralement une service qui n'existe pas réellement (par exemple si on configure un client). Il n'est là que pour contenir des fichiers.
+
+## Désactiver le service
+
+Il est possible de désactiver un service. Pour cela il faut rajouter l'attribut "disabled" à True :
+
+```
+<services>
+  <service name="test" disabled="True"/>
+</services>
+```
+
+Dans ce cas, le service et les éléments qu'il compose ([fichier](file.md), [IP](ip.md) et [réécriture](override.md) seront désactivés.
+
+Il est possible de définir une [condition](../condition/README.md) de type "disabled_if_in" ou "disabled_if_not_in" sur une balise service :
+
+```
+<services>
+  <service name="test" servicelist="test">
+  </service>
+</services>
+<variables>
+  <variable name="condition" type="boolean"/>
+</variables>
+<constraints>
+  <condition name="disabled_if_in" source="condition">
+    <param>False</param>
+    <target type="servicelist">test</target>
+  </condition>
+</constraints>
+```
+
+Dans ce cas, tous les services et les éléments qu'il compose avec un attribut servicelist à "test" seront désactivés si la variable "condition" est False.
+
+## Ne pas désactiver le service dans systemd
+
+La désactivation du service va créé un lien symbolique vers /dev/null.
+
+Si vous ne voulez juste pas créé le fichier de service et ne pas faire de lien symbolique, il faut utiliser l'attribut undisable :
+
+```
+<service name="test" disabled="True" undisable="True"/>
+```

doc/service/file.md

@@ -0,0 +1,249 @@
+# La gestion d'un fichier
+
+## La balise file
+
+La gestion des fichiers se fait dans un conteneur de [service](README.md).
+
+La déclaration du fichier permet de générer un fichier à partir d'un template pour le déposer à l'endroit prévu dans la déclaration de cette élément.
+
+Il est nécessaire, au minimum, de spécifier le chemin complet du fichier :
+
+```
+<services>
+  <service name="squid">
+    <file>/etc/squid/squid.conf</file>
+  </service>
+</services>
+```
+
+Dans ce cas, le nom du template est déduit du nom du fichier, ici cela sera "squid.conf".
+
+Si le template a un nom différent (par exemple si plusieurs template se retrouve avec le même nom), il est possible de changer le nom du template avec l'attribut source :
+
+```
+<file source="template-squid.conf">/etc/squid/squid.conf</file>
+```
+
+## Les noms de fichiers dynamique
+
+Il est possible également de définir le nom du fichier dans une variable :
+
+```
+<services>
+  <service name="squid">
+    <file file_type="variable" source="squid.conf">my_variable</file>
+  </service>
+</services>
+<variables>
+  <variable name="my_variable">
+    <value>/etc/squid/squid.conf</value>
+  </variable>
+</variables>
+```
+
+Attention, la variable doit être de type "filename".
+
+Dans le cas des fichiers dynamique, la source est obligatoire.
+
+Il est même possible de définir une variable de type multiple, ce qui génèrera plusiers fichiers :
+
+```
+<services>
+  <service name="squid">
+    <file file_type="variable" source="squid.conf">my_variable</file>
+  </service>
+</services>
+<variables>
+  <variable name="my_variable" multi="True">
+    <value>/etc/squid1/squid.conf</value>
+    <value>/etc/squid2/squid.conf</value>
+  </variable>
+</variables>
+```
+
+Dans ce cas là, le fichier source est identique mais les fichiers de destination seront différent.
+
+Il peut être important de personnaliser le contenu du fichier suivant le fichier de destination.
+Dans ce cas il y a deux possibilités :
+
+- la variable "rougail_filename" contient le nom de fichier de destination
+- l'utilisateur de l'attribut "variable" 
+
+En effet, il est possible de passer le contenu d'une variable au template :
+
+```
+<services>
+  <service name="squid">
+    <file file_type="variable" source="squid.conf" variable="my_variable2">my_variable1</file>
+  </service>
+</services>
+<variables>
+  <variable name="my_variable1" multi="True">
+    <value>/etc/squid1/squid.conf</value>
+    <value>/etc/squid2/squid.conf</value>
+  </variable>
+  <variable name="my_variable2" multi="True">
+    <value>squid1</value>
+    <value>squid2</value>
+  </variable>
+</variables>
+```
+
+Dans ce cas, lors de la génération du fichier /etc/squid1/squid.conf on retrouvera la variable "rougail_variable" avec la valeur "squid1" et la variable "rougail_index" avec la valeur "0". Lors de la génération du fichier /etc/squid2/squid.conf on retrouvera la variable "rougail_variable" avec la valeur "squid2" et la variable "rougail_index" avec la valeur "1".
+
+Attention : les deux variables "my_variable1" et "my_variable2" doivent être multiple et de même longueur.
+
+## Les droits des fichiers
+
+Par défaut les droits du fichier généré sont "0644" avec comme utilisateur "root" et groupe "root".
+
+Il est possible de définir les droits, l'utilisateur ou le groupe d'un fichier généré :
+
+```
+<file mode="0640" owner="nobody" group="squid">/etc/squid/squid.conf</file>
+```
+
+Il est possible de personnaliser les droits par défaut dans la [configuration de rougail](../dev/config.md)
+
+## Désactiver la génération d'un fichier
+
+Il est possible de désactiver la génération d'un fichier avec l'attribut "disabled" :
+
+```
+<file disabled="True">/etc/squid/squid.conf</file>
+```
+
+Il est aussi possible de définir une [condition](../condition/README.md) de type "disabled_if_in" ou "disabled_if_not_in" sur une balise fichier :
+
+```
+<services>
+  <service name="test">
+    <file filelist="squid">/etc/squid/squid.conf</file>
+  </service>
+</services>
+<variables>
+  <variable name="condition" type="boolean"/>
+</variables>
+<constraints>
+  <condition name="disabled_if_in" source="condition">
+    <param>False</param>
+    <target type="filelist">squid</target>
+  </condition>
+</constraints>
+```
+
+Dans ce cas, tous les fichiers avec un attribut filelist à "squid" seront désactivés si la variable "condition" est False.
+
+## Redéfinir une fichier
+
+Il est possible de redéfinir les éléments d'un fichier dans un dictionnaire différent en utilisant l'attribut redefine :
+
+```
+<file source="template-squid.conf" redefine="True">/etc/squid/squid.conf</file>
+```
+
+## Choix du moteur de templating
+
+Par défaut, le moteur de templating est le moteur de templating compatible avec "creole".
+
+Il est possible de désactiver la templatisation du fichier (il sera alors uniquement copié) :
+
+```
+<file engine="none">/etc/squid/squid.conf</file>
+```
+
+Ou d'utiliser le moteur "jinja2" :
+
+```
+<file engine="jinja2">/etc/squid/squid.conf</file>
+```
+
+Il est possible de personnaliser le moteur par défaut dans la [configuration de rougail](../dev/config.md)
+
+## Inclusion de template
+
+Un attribut "included" permet de définir la nature du fichier. Cet attribut peut avoir trois valeurs :
+
+- "no" : c'est un fichier normal
+- "name" : le répertoire de destination est listé dans un autre template, il faut que le fichier soit généré avant cet autre template
+- "content" : le contenu du fichier est copié dans un autre template, il faut que le fichier soit généré avant cet autre template et ce fichier n'a pas besoin d'être installé sur le serveur cible.
+
+Bien entendu, c'est au développeur de lister ou d'inclure le contenu de ce template dans le fichier de destination. Cet attribut permet juste de garantir que le fichier sera fait avant l'autre et de ne pas l'installer sur le serveur si ce n'est pas nécessaire.
+
+Il est possible de personnaliser les methodes d'inclusion par défaut dans la [configuration de rougail](../dev/config.md)
+
+Exemples :
+
+- créons un template qui inclut des noms de fichiers :
+
+Le contenu de ce template (squid.conf) est :
+
+```
+%import os
+%set %%confdir = 'etc/squid/squid.d/'
+%if %%os.path.exists(%%confdir)
+ %set %%files = %%os.listdir(%%confdir)
+ %%files.sort()
+ %for %%file in %%files
+  %if %%file.endswith('.cfg')
+include '/' + %%confdir + file
+  %end if
+ %end for
+%end if
+```
+
+Ajoutons un second template (squid.included.conf) qui sera copié dans ce répertoire :
+
+```
+template content
+```
+
+Et déclaront ces deux templates :
+
+```
+<file>/etc/squid/squid.conf</file>
+<file included="name" engine="none">/etc/squid/squid.d/squid.conf</file>
+```
+
+Le contenu du fichier généré (/etc/squid/squid.conf) sera donc :
+
+```
+include squid.d/squid.conf
+```
+
+- créons un template qui inclut le contenu de fichiers :
+
+Le contenu de ce template (squid.conf) est :
+
+```
+%import os
+%set %%confdir = 'squid.d/'
+%if %%os.path.exists(%%confdir)
+ %set %%files = %%os.listdir(%%confdir)
+ %%files.sort()
+ %for %%file in %%files
+  %if %%file.endswith('.cfg')
+%include raw %%confdir + file
+  %end if
+ %end for
+%end if
+```
+
+Ajoutons un second template (squid.included.conf) qui sera copié dans ce répertoire :
+
+```
+template content
+```
+
+Et déclaront ces deux templates :
+
+```
+<file>/etc/squid/squid.conf</file>
+<file included="content" engine="none">squid.d/squid.conf</file>
+```
+
+Le contenu du fichier généré (/etc/squid/squid.conf) sera donc maintenant :
+
+```
+template content
+```

doc/service/ip.md

@@ -0,0 +1,47 @@
+# La gestion d'une IP
+
+## La balise IP
+
+La gestion des IP se fait dans un conteneur de [service](README.md).
+
+La déclaration de l'attribut permet d'associer une IP autorisé à accéder au service.
+
+Il est nécessaire, au minimum, de spécifier le nom d'une variable de type "IP" :
+
+```
+<ip ip_type="variable">variable_ip</ip>
+```
+
+## La gestion d'un réseau
+
+L'adresse peut être de type réseau ("network") :
+
+```
+<ip netmask="variable_netmask">variable_ip</ip>
+```
+
+Attention, dans ce cas il faut préciser une variable de type "netmask" dans l'attribut netmask.
+
+## Désactiver la génération d'une IP
+
+Il est possible de définir une [condition](../condition/README.md) de type "disabled_if_in" ou "disabled_if_not_in" sur une balise IP :
+
+```
+<services>
+  <service name="test">
+    <ip iplist="test_ip">variable_ip</ip>
+  </service>
+</services>
+<variables>
+  <variable name="condition" type="boolean"/>
+  <variable name="variable_ip" type="ip"/>
+</variables>
+<constraints>
+  <condition name="disabled_if_in" source="condition">
+    <param>False</param>
+    <target type="iplist">test_ip</target>
+  </condition>
+</constraints>
+```
+
+Dans ce cas, tous les IP avec un attribut iplist à "test_ip" seront désactivé si la variable "condition" est False.

doc/service/override.md

@@ -0,0 +1,45 @@
+# Réécriture du service
+
+## La balise override
+
+La gestion des overrides se fait dans un conteneur de [service](README.md).
+
+La balise override permet de redéfinir facilement un service systemd.
+
+Il suffit d'avoir un template dont le nom est par défaut le nom du service avec l'extension "service" et de déclarer la balise :
+
+```
+<services>
+  <service name="squid">
+    <override/>
+  </service>
+</services>
+```
+
+Dans cette exemple, le template associé doit s'appeler squid.service
+
+Si le fichier service a un nom différent (par exemple si plusieurs template se retrouve avec le même nom), il est possible de changer le nom du template avec l'attribut source :
+
+```
+<override source="test.service"/>
+```
+
+Dans ce cas le fichier de destination aura le même nom.
+
+## Choix du moteur de templating
+
+Par défaut, le moteur de templating est le moteur de templating compatible avec "creole".
+
+Il est possible de désactiver la templatisation du fichier (il sera alors uniquement copié) :
+
+```
+<override engine="none"/>
+```
+
+Ou d'utiliser le moteur "jinja2" :
+
+```
+<override engine="jinja2"/>
+```
+
+Il est possible de personnaliser le moteur par défaut dans la [configuration de rougail](../dev/config.md)

doc/services.md

@@ -0,0 +1,14 @@
+# Le conteneur des services
+
+La balise "services" est le conteneur de l'ensemble des [services](service/README.md).
+
+Il est placé à la racine du dictionnaire :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <services/>
+</rougail>
+```
+
+Attention, cette balise ne peut pas être placé dans un dictionnaire "extra".

doc/target/README.md

@@ -0,0 +1,5 @@
+# La cible
+
+  - [De type variable](../target/variable.md)
+  - [De type famille](../target/family.md)
+  - [De type \*list](../target/list.md)

doc/target/family.md

@@ -0,0 +1,13 @@
+# Cible de type "variable"
+
+Une cible peut être de type famille :
+
+```
+<target type="family">my_family</target>
+```
+
+Mais une target peut être optionnelle. C'est à dire que si la famille n'existe pas, l'action ne sera pas associé à cette famille.
+
+```
+<target type="family" optional="True">my_family</target>
+```

doc/target/list.md

@@ -0,0 +1,35 @@
+# Cible de type \*list
+
+## Les différences cible de type \*.list
+
+### servicelist
+
+Une cible peut être de type [service](../service/README.md) :
+
+```
+<target type="servicelist">example</target>
+```
+
+### filelist
+
+Une cible peut être de type [fichier](../service/file.md) :
+
+```
+<target type="filelist">example</target>
+```
+
+### iplist
+
+Une cible peut être de type [ip](../service/ip.md) :
+
+```
+<target type="iplist">example</target>
+```
+
+## La cible optionnelle
+
+Mais une target peut être optionnelle. C'est à dire que si la \*list n'existe pas, l'action ne sera pas associé.
+
+```
+<target type="filelist" optional="True">unknown</target>
+```

doc/target/variable.md

@@ -0,0 +1,13 @@
+# Cible de type "variable"
+
+Par défaut une cible est de type variable.
+
+```
+<target>my_variable</target>
+```
+
+Mais une target peut être optionnelle. C'est à dire que si la variable n'existe pas, l'action ne sera pas associé à cette variable.
+
+```
+<target optional='True'>my_variable</target>
+```

doc/variable/README.md

@@ -0,0 +1,275 @@
+# Variable
+
+## Un variable
+
+Une variable est forcement dans [variables](../variables.md) ou dans une [famille](../family/README.md).
+
+Une variable est déjà un nom. C'est à dire qu'on pourra utiliser plus tard la variable via ce nom.
+
+```
+<variables>
+  <variable name="my_variable"/>
+  <family name="my_family">
+    <variable name="my_family_variable"/>
+  </variable>
+</variables>
+```
+
+## Description et aide sur la variable
+
+En plus d'un nom, il est possible de mettre une "description" à la variable. C'est une information "utilisateur" qui nous permettra d'avoir des informations complémentaires sur le contenu de cette variable :
+
+```
+<variable name="my_variable" description="This is a greate variable"/>
+```
+
+En plus de la description, il est possible de préciser une aide complémentaire :
+
+```
+<variable name="my_variable" help="This is a greate variable"/>
+```
+
+Cette aide peut être utilisé à tout moment comme valeur [d'un paramètre](../param/information.md).
+
+## Le type de la variable
+
+Une variable a un type. Ce type permet de définir les valeurs acceptées par cette variable :
+
+- string : chaine de caractère (type par défaut)
+- number : un nombre
+- float : un chiffre flottant
+- boolean : "True" ou "False", si aucune valeur n'est défini la valeur par défaut de cette variable sera "True", ces variables sont également obligatoire par défaut
+- secret (ou password mais est obsolète) : un secret (comme un mot de passe, une clef privée, ...)
+- mail : une adresse mail
+- filename : nom de fichier au sens Unix (exemple : "/etc/passwd")
+- date : une date au format "%Y-%m-%d" (exemple : "2021-01-30")
+- unix_user : nom d'utilisateur au sens Unix
+- ip : n'importe quelle adresse IPv4
+- cidr : n'importe quelle adresse IPv4 au format CIDR
+- local_ip : adresse IPv4 sur un réseau local, si l'adresse IPv4 n'est pas local, un warning sera afficher mais la valeur sera accepté tout de même
+- netmask : masque d'une adresse IPv4
+- network : adresse réseau
+- network_cidr : adresse réseau au format CIDR
+- broadcast : adresse de diffusion
+- netbios : nom netbios
+- domain : nom de domaine
+- hostname : nom d'hôte
+- web_address : adresse web (http://www.cadoles.com/)
+- port : port
+- mac : adresse MAC
+- schedule : périodicité du schedule, les valeurs possibles sont "none", "daily", "weekly" ou "monthly"
+- schedulemod : type de schedule, les valeurs possibles sont "pre" ou "post"
+
+Pour définir le type d'une variable :
+
+```
+<variable name="my_variable" type="number"/>
+```
+
+## Variable à valeur multiple
+
+Par défaut une variable ne peut acceuillir qu'une seule valeur. Il peut être utile de pouvoir spécifier plusieurs valeurs à une même variable.
+
+Pour définir une variable à valeur multiple :
+
+```
+<variable name="my_variable" multi="True"/>
+```
+
+## Variable invisible
+
+Il est possible de cacher une variable.
+
+Cacher une variable signifie qu'elle ne sera pas visible lorsqu'on modifie la configuration du service.
+Par contre cette variable sera accessibles lorsqu'on va l'utiliser.
+
+Pour cacher une variable :
+
+```
+<variable name="my_variable" hidden="True"/>
+```
+
+## Variable désactive
+
+Il est possible de désactiver une variable.
+
+Désactiver une variable signifie qu'elle ne sera pas visible lorsqu'on modifie la configuration du service mais également lorsqu'on va l'utiliser.
+
+Pour désactiver une variable :
+
+```
+<variable name="my_variable" disabled="True"/>
+```
+
+## Variable obligatoire
+
+Variable dont une valeur est requise :
+
+```
+<variable name="my_variable" mandatory="True"/>
+```
+
+Les variables booléans sont par défaut obligatoire. Pour qu'une variable booléan ne soit pas obligatoire il faut le préciser explicitement :
+
+```
+<variable name="my_variable" type="boolean" mandatory="False"/>
+```
+
+Les variables avec une valeur par défaut (non calculée) sont également automatiquement obligatoire.
+
+## Valeur par défaut d'une variable
+
+Il est possible de fixer les valeurs par défaut d'une variable :
+
+```
+<variable name="my_variable">
+  <value>value</value>
+</variable>
+```
+
+Pour une variable multiple, il est possible de préciser plusieurs valeurs :
+
+```
+<variable name="my_variable" multi="True">
+  <value>value 1</value>
+  <value>value 2</value>
+</variable>
+```
+
+Si la variable n'est pas pas une [variable meneuse](../family/leadership.md), la première valeur défini dans cette liste sera également la valeur par défaut proposé si on ajoute une nouvelle valeur à cette variable.
+
+Une valeur par défaut peut également être [une valeur calculer](../fill/README.md).
+
+## Redéfinir une variable
+
+Il est possible de définir une variable dans un dictionnaire et de changer son comportement dans une second dictionnaire.
+
+Attention trois attributs ne sont redéfinisable :
+
+- name
+- type
+- multi
+
+Créons notre variable :
+
+```
+<variable name="my_variable"/>
+```
+
+Et redéfinisons là :
+
+```
+<variable name="my_variable" redefine="True" description="New description"/>
+```
+
+## Créer une variable inexistante
+
+Il est parfois utile de créer une variable si elle n'existe pas dans un autre dictionnaire :
+
+```
+<variable name="my_variable" exists="False"/>
+```
+
+Si cette variable existe dans un autre dictionnaire, elle ne sera pas modifié ni recréé
+
+## Redéfinir une variable si elle existe
+
+Parfois on veut pouvoir redéfinir une variable mais seulement dans le cas où elle existe déjà :
+
+```
+<variable name="my_variable" redefine="True" exists="True" hidden="True"/>
+```
+
+## Variable à valeur automatiquement modifiée
+
+Une variable avec valeur automatiquement modifiée est une variable dont la valeur sera considéré comme modifié quand la propriété global "force_store_value" de Tiramisu est mise.
+
+Voici une variable a valeur automatiquement modifiée :
+
+```
+<variable name="my_variable" auto_save="True">
+  <value>my_value</value>
+</variable>
+```
+
+Dans ce cas la valeur est fixée à la valeur actuelle.
+Par exemple, si la valeur de cette variable est issue d'un calcul, la valeur ne sera plus recalculée.
+
+Ces variables sont généralement des variables obligatoires. En effet ces variable ne sont automatiquement modifiées que si elles ont une valeurs.
+
+Une [variable meneuse ou suiveuse](../family/leadership.md) ne peut pas avoir la propriété auto_save.
+
+## Variable à valeur en lecture seule automatique
+
+Une variable avec valeur en lecture seule automatique est une variable dont la valeur ne sera plus modifiable par l'utilisateur quand la [variable "server_deployed" passe à "True"](../dev/config.md).
+
+Voici un variable à valeur en lecture seule automatique :
+
+```
+<variable name="server_deployed" type="boolean">
+  <value>False</value>
+</variable>
+<variable name="my_variable" auto_freeze="True"/>
+```
+
+Dans ce cas la valeur est fixée à la valeur actuelle et elle ne sera plus modifiable par l'utilisateur.
+Par exemple, si la valeur de cette variable est issue d'un calcul, la valeur ne sera plus recalculée.
+
+Ces variables sont généralement des variables obligatoires. En effet ces variable ne sont en lecteur seul que si elles ont une valeurs.
+
+Une [variable meneuse ou suiveuse](../family/leadership.md) ne peut pas avoir la propriété auto_freeze.
+
+## Information "test"
+
+L'attribut "test" est un attribut spécial qui permet aux concepteurs d'un dictionnaire d'influancer le robot de test en précisant de valeurs utile à tester.
+
+Concrêtement, le contenu de cet attribut est enregister dans une "information" de l'option Tiramisu correspondante.
+
+Exemple :
+
+```
+<variable name="my_variable" test="yes"/>
+```
+
+Il est possible de préciser plusieurs valeurs avec le séparateur "|" :
+
+```
+<variable name="my_variable" test="yes|no"/>
+```
+
+Cette valeur peut être utilisé à tout moment comme valeur [d'un paramètre](../param/information.md).
+
+## Mode de la variable
+
+Le [mode](../mode.md) par défaut d'une variable correspond au [mode](../mode.md) de la [famille](../family/README.md).
+
+Cas particuliers :
+
+- une variable à valeur automatiquement modifiée ou une variable en lecture seule automatique est par défaut en mode "basic".
+- si la variable n'est pas dans une famille, la variable aura le mode "normal" par défaut.
+- une variable obligatoire sans valeur par défaut (calculer ou non) aura le mode "basic".
+
+Pour définir le [mode](../mode.md) :
+
+```
+<variable name="my_variable" mode="expert"/>
+```
+
+## Les variables qui fournissent des valeurs
+
+Il peut être intéressant de retrouver facilement des variables sans connaitre le chemin complet mais en utilisant le contenu du paramètre "provider".
+C'est particulièrement utile si un service peut être fournit par plusieurs services. Les variables n'auront donc pas le même nom. Utiliser ce paramètre, permet donc de retrouver facilement la variable.
+
+Pour déclarer :
+
+```
+<variable name="my_variable" provider="my_function"/>
+```
+
+Dans le code de l'application, on pourra retrouver le chemin de la variable en faisant :
+
+```
+print(await config.information.get('provider:my_function'))
+```
+
+Pour les variables inclusent dans une famille dynamique, le chemin de la variable sera un template comme ceci "rougail.family_{suffix}.my_variable_{suffix}". Il vous suffit de remplacer "{suffix}" par le suffix voulu de la famille dynamique.

doc/variable/choice.md

@@ -0,0 +1,91 @@
+# Les variables à choix
+
+## Une variable à choix
+
+Il est possible d'imposer une liste de valeur pour une variable particulière :
+
+```
+<variable name="my_variable" type="choice">
+  <choice>val1</choice>
+  <choice>val2</choice>
+  <choice>val3</choice>
+</variable>
+```
+
+Dans ce cas, seules les valeurs proposées sont possibles pour cette variable.
+Cette variable n'est pas obligatoire dont il est possible de mettre la valeur "None".
+
+Si la variable est obligatoire ou si une valeur est précisée (la variable passe obligatoire) alors la valeur "None" n'est plus autorisé :
+
+```
+<variable name="my_variable" type="choice">
+  <choice>val1</choice>
+  <choice>val2</choice>
+  <choice>val3</choice>
+  <value>val1</value>
+</variable>
+```
+
+## Un variable à choix typée
+
+Par défaut les choix sont de type "string". Il est possible de préciser des nombres, des booléens ou la valeur None :
+
+```
+<variable name="my_variable" type="choice">
+  <choice>val1</choice>
+  <choice type="string">val2</choice>
+  <choice type="number">3</choice>
+  <choice type="boolean">True</choice>
+  <choice type="nil"/>
+</variable>
+```
+
+Comme vu précédement ajouter la valeur None n'est pas utile parce qu'elle est automatiquement ajouté si la variable n'est pas obligatoire.
+
+## Ajouter une option à une variable à choix existante
+
+Pour ajouter un choix à une variable à choix existante, rien de plus simple, juste redéfinir la variable en ajoutant le choix voulu :
+
+```
+<variable name="my_variable" redefine="True">
+  <choice>val4</choice>
+</variable>
+```
+
+## Redéfinir une option à choix
+
+Si on veut supprimer un choix ou redéfinir complètement la liste, il faut redéfinir cette variable et ajouter l'attribut "remove_choice" à "True" :
+
+```
+<variable name="my_variable" redefine="True" remove_choice="True">
+  <choice>val1</choice>
+  <choice>val2</choice>
+</variable>
+```
+
+Dans ce cas toutes les anciens choix ne seront plus possible. Seuls les nouveaux le seront.
+
+## Un variable à choix provenant d'une variable
+
+Une variable à valeur multiple peut servir de source des choix :
+
+```
+<variable name="my_variable" type="choice">
+  <choice type="variable">other_variable</choice>
+</variable>
+```
+
+Dans ce cas, toutes les valeurs de la variable seront des choix utilisables par l'utilisateur.
+Seul un choice de type "variable" est possible par variable.
+
+## Un variable à choix provenant d'une fonction
+
+```
+<variable name="my_variable" type="choice">
+  <choice type="function" name="trange">
+    <param type="number">0</param>
+    <param type="number">10</param>
+  </choice>
+  <value type="number">9</value>
+</variable>
+```

doc/variable/leadership.md

@@ -0,0 +1,113 @@
+# Variable meneuse ou suiveuse
+
+## Variable meneuse
+
+Une variable meneuse est une variable qui va guider la longueur d'autre variables (appelé variables suiveuse).
+
+Une variable meneuse est une [variable](simple.md) qui est obligatoirement de type multiple.
+
+Une variable meneuse peut être obligatoire.
+
+Le [mode](../mode.md) par défaut correspond au plus petit mode définit par l'utilisateur des variables suiveuses.
+
+## Variable suiveuse
+
+Une variable suiveuse est une variable donc la longueur n'est pas déterminé par elle-même, mais est identique à celle de la variable meneuse dont elle dépend.
+
+Une variable suiveuse est une variable placer juste derrière une variable meneuse ou une autre variable suiveuse.
+
+L'ordre de définition des variables suiveuses est important.
+
+Cette variable peut être de type multiple. Dans ce cas, pour un index determiné, il est possible de mettre plusieurs valeurs à une même variable.
+
+Une variable suiveuse peut être obligatoire. Cela signifie que lorsqu'une variable meneuse est renseigné, il faut obligatoirement que la variable suiveuse est également une valeur à l'index considéré.
+Si aucune valeur n'est définit pour la variable meneuse, aucune valeur n'est a spécifié pour la variable suiveuse.
+
+Le [mode](../mode.md) par défaut d'une variable suiveuse correspond au [mode](../mode.md) de la variable meneuse.
+
+Si une variable meneuse est caché ou désactivé, les variables suiveuses le seront également.
+
+## Définition des variables meneuse et suiveuse
+
+Voici un exemple de définition d'une variable meneuse et de deux variables meneuses :
+
+```
+<variables>
+  <family name="family">
+    <variable name="leader" multi='True'/>
+    <variable name="follower1"/>
+    <variable name="follower2" multi='True'/>
+  </family>
+</variables>
+<constraints>
+  <group leader="leader">
+    <follower>follower1</follower>
+    <follower>follower2</follower>
+  </group>
+</constraints>
+```
+
+Le nom et la description du groupe prendra par défaut le nom et la description de la variable meneuse.
+
+Il est possible d'en définit d'autres :
+
+```
+<constraints>
+  <group leader="leader" name="leadership" description="My leadership">
+    <follower>follower1</follower>
+    <follower>follower2</follower>
+  </group>
+</constraints>
+```
+
+## Faire un calcul avec l'index de la variable meneuse
+
+Dans un calcul sur un variable suiveuse il est possible de récupérer l'index de la variable meneuse :
+
+```
+<fill name="calc_val">
+  <param type="index"/>
+  <target>follower</target>
+</fill>
+```
+
+## Définition des variables meneuse et suiveuse dans un dictionnaire extra
+
+Voici un exemple de définition d'une variable meneuse et de deux variables meneuses dans un espace de nom "example" :
+
+```
+<variables>
+  <family name="family">
+    <variable name="leader" multi='True'/>
+    <variable name="follower1"/>
+    <variable name="follower2" multi='True'/>
+  </family>
+</variables>
+<constraints>
+  <group leader="example.family.leader">
+    <follower>follower1</follower>
+    <follower>follower2</follower>
+  </group>
+</constraints>
+```
+
+Le chemin de la variable meneuse est traditionnel, par contre le chemin des variables suiveuses est le chemin relatif de la variable.
+
+Le chemin d'une variable suiveuse est normalement "example.family.leader.follower1" mais la variable n'est pas encore une variable suiveuse à ce moment là du traitement. C'est pour cela qu'il ne faut, uniquement dans les groupes, mettre le nom de la variable suiveuse.
+
+## Ajout d'une nouvelle variable suiveuse
+
+Pour ajouter, dans un nouveau dictionnaire, une variable suiveuse à notre groupe, rien de plus simple, il suffit de redéfinir un groupe avec cette (ou ces) nouvelle variable :
+
+```
+<variables>
+  <family name="family">
+    <variable name="follower3"/>
+  </family>
+</variables>
+<constraints>
+  <group leader="leader">
+    <follower>follower3</follower>
+  </group>
+</constraints>
+```

doc/variables.md

@@ -0,0 +1,12 @@
+# Le conteneur des variables
+
+La balise "variables" est le conteneur de l'ensemble des [familles](family/README.md) et des [variables](variable/README.md).
+
+Il est placé à la racine du dictionnaire :
+
+```
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail>
+  <variables/>
+</rougail>
+```

src/rougail/__init__.py

@@ -1,5 +1,32 @@
-#from .loader import load
-from .objspace import CreoleObjSpace
-from .annotator import modes
+"""Rougail method
 
-__ALL__ = ('CreoleObjSpace', 'modes')
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from .convert import RougailConvert
+from .template.systemd import RougailSystemdTemplate
+from .config import RougailConfig
+from rougail.update import RougailUpgrade
+
+__ALL__ = ('RougailConvert', 'RougailSystemdTemplate', 'RougailConfig', 'RougailUpgrade')

src/rougail/annotator.py

@@ -1,1237 +0,0 @@
-# coding: utf-8
-from copy import copy
-from typing import List
-
-from collections import OrderedDict
-from os.path import join, basename
-from ast import literal_eval
-import imp
-
-
-from .i18n import _
-from .utils import normalize_family
-from .error import DictConsistencyError
-
-#mode order is important
-modes_level = ('basic', 'normal', 'expert')
-
-
-class Mode(object):
-    def __init__(self, name, level):
-        self.name = name
-        self.level = level
-    def __gt__(self, other):
-        return other.level < self.level
-
-
-def mode_factory():
-    mode_obj = {}
-    for idx in range(len(modes_level)):
-        name = modes_level[idx]
-        mode_obj[name] = Mode(name, idx)
-    return mode_obj
-
-modes = mode_factory()
-
-
-# a CreoleObjSpace's attribute has some annotations
-# that shall not be present in the exported (flatened) XML
-ERASED_ATTRIBUTES = ('redefine', 'exists', 'fallback', 'optional', 'remove_check', 'namespace',
-                     'remove_condition', 'path', 'instance_mode', 'index', 'is_in_leadership',
-                     'level')  # , '_real_container')
-ERASED_CONTAINER_ATTRIBUTES = ('id', 'container', 'group_id', 'group', 'container_group')
-
-FORCE_CHOICE = {'oui/non': ['oui', 'non'],
-                'on/off': ['on', 'off'],
-                'yes/no': ['yes', 'no'],
-                'schedule': ['none', 'daily', 'weekly', 'monthly'],
-                'schedulemod': ['pre', 'post']}
-
-KEY_TYPE = {'variable': 'symlink',
-            'SymLinkOption': 'symlink',
-            'PortOption': 'port',
-            'UnicodeOption': 'string',
-            'NetworkOption': 'network',
-            'NetmaskOption': 'netmask',
-            'URLOption': 'web_address',
-            'FilenameOption': 'filename'}
-
-TYPE_PARAM_CHECK = ('string', 'python', 'variable')
-TYPE_PARAM_CONDITION = ('string', 'python', 'number', 'variable')
-TYPE_PARAM_FILL = ('string', 'number', 'variable')
-CONVERSION = {'number': int}
-
-FREEZE_AUTOFREEZE_VARIABLE = 'module_instancie'
-
-PROPERTIES = ('hidden', 'frozen', 'auto_freeze', 'auto_save', 'force_default_on_freeze',
-              'force_store_value', 'disabled', 'mandatory')
-CONVERT_PROPERTIES = {'auto_save': ['force_store_value'], 'auto_freeze': ['force_store_value', 'auto_freeze']}
-
-RENAME_ATTIBUTES = {'description': 'doc'}
-INTERNAL_FUNCTIONS = ['valid_enum', 'valid_in_network', 'valid_differ', 'valid_entier']
-
-class SpaceAnnotator:
-    """Transformations applied on a CreoleObjSpace instance
-    """
-    def __init__(self, objectspace, eosfunc_file):
-        self.objectspace = objectspace
-        GroupAnnotator(objectspace)
-        ServiceAnnotator(objectspace)
-        VariableAnnotator(objectspace)
-        ConstraintAnnotator(objectspace,
-                            eosfunc_file,
-                            )
-        FamilyAnnotator(objectspace)
-        PropertyAnnotator(objectspace)
-
-
-class GroupAnnotator:
-    def __init__(self,
-                 objectspace,
-                 ):
-        self.objectspace = objectspace
-        if not hasattr(self.objectspace.space, 'constraints') or not hasattr(self.objectspace.space.constraints, 'group'):
-            return
-        self.convert_groups()
-
-    def convert_groups(self):  # pylint: disable=C0111
-        for group in self.objectspace.space.constraints.group:
-            leader_fullname = group.leader
-            leader_family_name = self.objectspace.paths.get_variable_family_name(leader_fullname)
-            leader_name = self.objectspace.paths.get_variable_name(leader_fullname)
-            namespace = self.objectspace.paths.get_variable_namespace(leader_fullname)
-            if '.' not in leader_fullname:
-                leader_fullname = '.'.join([namespace, leader_family_name, leader_fullname])
-            follower_names = list(group.follower.keys())
-            has_a_leader = False
-            ori_leader_family = self.objectspace.paths.get_family_obj(leader_fullname.rsplit('.', 1)[0])
-            for variable in list(ori_leader_family.variable.values()):
-                if has_a_leader:
-                    # it's a follower
-                    self.manage_follower(namespace,
-                                         leader_family_name,
-                                         variable,
-                                         leader_name,
-                                         follower_names,
-                                         leader_space,
-                                         leader_is_hidden,
-                                         )
-                    ori_leader_family.variable.pop(variable.name)
-                    if follower_names == []:
-                        # no more follower
-                        break
-                elif variable.name == leader_name:
-                    # it's a leader
-                    if isinstance(variable, self.objectspace.Leadership):
-                        # append follower to an existed leadership
-                        leader_space = variable
-                        # if variable.hidden:
-                        #     leader_is_hidden = True
-                    else:
-                        leader_space = self.objectspace.Leadership()
-                        leader_is_hidden = self.manage_leader(leader_space,
-                                                              leader_family_name,
-                                                              leader_name,
-                                                              namespace,
-                                                              variable,
-                                                              group,
-                                                              leader_fullname,
-                                                              )
-                    has_a_leader = True
-            else:
-                raise DictConsistencyError(_('cannot found followers {}').format(follower_names))
-        del self.objectspace.space.constraints.group
-
-    def manage_leader(self,
-                      leader_space: 'Leadership',
-                      leader_family_name: str,
-                      leader_name: str,
-                      namespace: str,
-                      variable: 'Variable',
-                      group: 'Group',
-                      leader_fullname: str,
-                      ) -> None:
-        # manage leader's variable
-        if variable.multi is not True:
-            raise DictConsistencyError(_('the variable {} in a group must be multi').format(variable.name))
-        leader_space.variable = []
-        leader_space.name = leader_name
-        leader_space.hidden = variable.hidden
-        if variable.hidden:
-            leader_is_hidden = True
-            variable.frozen = True
-            variable.force_default_on_freeze = True
-        else:
-            leader_is_hidden = False
-        variable.hidden = None
-        if hasattr(group, 'description'):
-            leader_space.doc = group.description
-        elif hasattr(variable, 'description'):
-            leader_space.doc = variable.description
-        else:
-            leader_space.doc = variable.name
-        leader_path = namespace + '.' + leader_family_name + '.' + leader_name
-        self.objectspace.paths.add_family(namespace,
-                                          leader_path,
-                                          leader_space,
-                                          )
-        leader_family = self.objectspace.space.variables[namespace].family[leader_family_name]
-        leader_family.variable[leader_name] = leader_space
-        leader_space.variable.append(variable)
-        self.objectspace.paths.set_leader(namespace,
-                                          leader_family_name,
-                                          leader_name,
-                                          leader_name,
-                                          )
-        leader_space.path = leader_fullname
-        return leader_is_hidden
-
-    def manage_follower(self,
-                        namespace: str,
-                        leader_family_name: str,
-                        variable: 'Variable',
-                        leader_name: str,
-                        follower_names: List[str],
-                        leader_space: 'Leadership',
-                        leader_is_hidden: bool,
-                        ) -> None:
-        if variable.name != follower_names[0]:
-            raise DictConsistencyError(_('cannot found this follower {}').format(follower_names[0]))
-        follower_names.remove(variable.name)
-        if leader_is_hidden:
-            variable.frozen = True
-            variable.force_default_on_freeze = True
-        leader_space.variable.append(variable)  # pylint: disable=E1101
-        self.objectspace.paths.set_leader(namespace,
-                                          leader_family_name,
-                                          variable.name,
-                                          leader_name,
-                                          )
-
-
-class ServiceAnnotator:
-    """Manage service's object
-    for example::
-        <services>
-            <service name="test">
-                <service_access service='ntp'>
-                    <port protocol='udp' service_accesslist='ntp_udp'>123</port>
-                </service_access>
-            </service>
-        </services>
-    """
-    def __init__(self, objectspace):
-        self.objectspace = objectspace
-        self.convert_services()
-
-    def convert_services(self):
-        if not hasattr(self.objectspace.space, 'services'):
-            return
-        if not hasattr(self.objectspace.space.services, 'service'):
-            del self.objectspace.space.services
-            return
-        self.objectspace.space.services.hidden = True
-        self.objectspace.space.services.name = 'services'
-        self.objectspace.space.services.doc = 'services'
-        families = {}
-        for idx, service_name in enumerate(self.objectspace.space.services.service.keys()):
-            service = self.objectspace.space.services.service[service_name]
-            new_service = self.objectspace.service()
-            for elttype, values in vars(service).items():
-                if not isinstance(values, (dict, list)) or elttype in ERASED_ATTRIBUTES:
-                    setattr(new_service, elttype, values)
-                    continue
-                eltname = elttype + 's'
-                path = '.'.join(['services', service_name, eltname])
-                family = self.gen_family(eltname,
-                                         path,
-                                         )
-                if isinstance(values, dict):
-                    values = list(values.values())
-                family.family = self.make_group_from_elts(service_name,
-                                                          elttype,
-                                                          values,
-                                                          path,
-                                                          )
-                setattr(new_service, elttype, family)
-            new_service.doc = new_service.name
-            families[service_name] = new_service
-        self.objectspace.space.services.service = families
-
-    def gen_family(self,
-                   name,
-                   path,
-                   ):
-        family = self.objectspace.family()
-        family.name = normalize_family(name)
-        family.doc = name
-        family.mode = None
-        self.objectspace.paths.add_family('services',
-                                          path,
-                                          family,
-                                          )
-        return family
-
-    def make_group_from_elts(self,
-                             service_name,
-                             name,
-                             elts,
-                             path,
-                             ):
-        """Splits each objects into a group (and `OptionDescription`, in tiramisu terms)
-        and build elements and its attributes (the `Options` in tiramisu terms)
-        """
-        families = []
-        new_elts = self._reorder_elts(name,
-                                      elts,
-                                      )
-        for index, elt_info in enumerate(new_elts):
-            elt = elt_info['elt']
-            elt_name = elt_info['elt_name']
-
-            # try to launch _update_xxxx() function
-            update_elt = '_update_' + elt_name
-            if hasattr(self, update_elt):
-                getattr(self, update_elt)(elt,
-                                          index,
-                                          path,
-                                          service_name,
-                                          )
-
-            idx = 0
-            while True:
-                if hasattr(elt, 'source'):
-                    c_name = elt.source
-                else:
-                    c_name = elt.name
-                if idx:
-                    c_name += f'_{idx}'
-                subpath = '{}.{}'.format(path, c_name)
-                if not self.objectspace.paths.family_is_defined(subpath):
-                    break
-                idx += 1
-            family = self.gen_family(c_name, subpath)
-            family.variable = []
-            listname = '{}list'.format(name)
-            activate_path = '.'.join([subpath, 'activate'])
-            for key in dir(elt):
-                if key.startswith('_') or key.endswith('_type') or key in ERASED_ATTRIBUTES:
-                    continue
-                value = getattr(elt, key)
-                if key == listname:
-                    self.objectspace.list_conditions.setdefault(listname,
-                                                                {}).setdefault(
-                                                                    value,
-                                                                    []).append(activate_path)
-                    continue
-                family.variable.append(self._generate_element(elt_name,
-                                                              key,
-                                                              value,
-                                                              elt,
-                                                              f'{subpath}.{key}'
-                                                              ))
-            # FIXME ne devrait pas etre True par défaut
-            # devrait etre un calcule
-            family.variable.append(self._generate_element(elt_name,
-                                                         'activate',
-                                                          True,
-                                                          elt,
-                                                          activate_path,
-                                                          ))
-            families.append(family)
-        return families
-
-    def _generate_element(self,
-                          elt_name,
-                          key,
-                          value,
-                          elt,
-                          path,
-                          ):
-        variable = self.objectspace.variable()
-        variable.name = normalize_family(key)
-        variable.mode = None
-        if key == 'name':
-            true_key = elt_name
-        else:
-            true_key = key
-        dtd_key_type = true_key + '_type'
-        if key == 'activate':
-            type_ = 'boolean'
-        elif hasattr(elt, dtd_key_type):
-            type_ = KEY_TYPE[getattr(elt, dtd_key_type)]
-        elif key in self.objectspace.booleans_attributs:
-            type_ = 'boolean'
-        else:
-            type_ = 'string'
-        variable.type = type_
-        if type_ == 'symlink':
-            variable.opt = self.objectspace.paths.get_variable_path(value,
-                                                                    'services',
-                                                                    )
-#            variable.opt = value
-            variable.multi = None
-        else:
-            variable.doc = key
-            val = self.objectspace.value()
-            val.type = type_
-            val.name = value
-            variable.value = [val]
-        self.objectspace.paths.add_variable('services',
-                                            path,
-                                            'service',
-                                            False,
-                                            variable,
-                                            )
-        return variable
-
-    def _reorder_elts(self,
-                      name,
-                      elts,
-                      ):
-        """Reorders by index the elts
-        """
-        new_elts = {}
-        # reorder elts by index
-        for idx, elt in enumerate(elts):
-            new_elts.setdefault(idx, []).append(elt)
-        idxes = list(new_elts.keys())
-        idxes.sort()
-        result_elts = []
-        for idx in idxes:
-            for elt in new_elts[idx]:
-                result_elts.append({'elt_name': name, 'elt': elt})
-        return result_elts
-
-    def _update_override(self,
-                         file_,
-                         index,
-                         service_path,
-                         service_name,
-                         ):
-        file_.name = f'/systemd/system/{service_name}.service.d/rougail.conf'
-        # retrieve default value from File object
-        for attr in ['owner', 'group', 'mode']:
-            setattr(file_, attr, getattr(self.objectspace.file, attr))
-        if not hasattr(file_, 'source'):
-            file_.source = f'{service_name}.service'
-        self._update_file(file_,
-                          index,
-                          service_path,
-                          service_name,
-                          )
-
-    def _update_file(self,
-                     file_,
-                     index,
-                     service_path,
-                     service_name,
-                     ):
-        if not hasattr(file_, 'file_type') or file_.file_type == "UnicodeOption":
-            if not hasattr(file_, 'source'):
-                file_.source = basename(file_.name)
-        elif not hasattr(file_, 'source'):
-            raise DictConsistencyError(_('attribute source mandatory for file with variable name '
-                                               'for {}').format(file_.name))
-
-
-class VariableAnnotator:
-    def __init__(self,
-                 objectspace,
-                 ):
-        self.objectspace = objectspace
-        self.convert_variable()
-        self.convert_helps()
-        self.convert_auto_freeze()
-        self.convert_separators()
-
-    def convert_variable(self):
-        def _convert_variable(variable,
-                              variable_type,
-                              ):
-            if not hasattr(variable, 'type'):
-                variable.type = 'string'
-            if variable.type != 'symlink' and not hasattr(variable, 'description'):
-                variable.description = variable.name
-            if hasattr(variable, 'value'):
-                for value in variable.value:
-                    if not hasattr(value, 'type'):
-                        value.type = variable.type
-                    value.name = CONVERSION.get(value.type, str)(value.name)
-            for key, value in RENAME_ATTIBUTES.items():
-                setattr(variable, value, getattr(variable, key))
-                setattr(variable, key, None)
-            if variable_type == 'follower':
-                if variable.multi is True:
-                    variable.multi = 'submulti'
-                else:
-                    variable.multi = True
-
-        def _convert_valid_enum(namespace,
-                                variable,
-                                path,
-                                ):
-            if variable.type in FORCE_CHOICE:
-                check = self.objectspace.check()
-                check.name = 'valid_enum'
-                check.target = path
-                check.namespace = namespace
-                param = self.objectspace.param()
-                param.text = str(FORCE_CHOICE[variable.type])
-                check.param = [param]
-                if not hasattr(self.objectspace.space, 'constraints'):
-                    self.objectspace.space.constraints = self.objectspace.constraints()
-                    self.objectspace.space.constraints.namespace = namespace
-                if not hasattr(self.objectspace.space.constraints, 'check'):
-                    self.objectspace.space.constraints.check = []
-                self.objectspace.space.constraints.check.append(check)
-                variable.type = 'string'
-
-        if not hasattr(self.objectspace.space, 'variables'):
-            return
-        for families in self.objectspace.space.variables.values():
-            namespace = families.name
-            if hasattr(families, 'family'):
-                families.doc = families.name
-                for family in families.family.values():
-                    family.doc = family.name
-                    for key, value in RENAME_ATTIBUTES.items():
-                        if hasattr(family, key):
-                            setattr(family, value, getattr(family, key))
-                            setattr(family, key, None)
-                    family.name = normalize_family(family.name)
-                    if hasattr(family, 'variable'):
-                        for variable in family.variable.values():
-                            if isinstance(variable, self.objectspace.Leadership):
-                                for idx, follower in enumerate(variable.variable):
-                                    if idx == 0:
-                                        variable_type = 'master'
-                                    else:
-                                        variable_type = 'follower'
-                                    path = '{}.{}.{}.{}'.format(namespace, normalize_family(family.name), variable.name, follower.name)
-                                    _convert_variable(follower,
-                                                      variable_type,
-                                                      )
-                                    _convert_valid_enum(namespace,
-                                                        follower,
-                                                        path,
-                                                        )
-                            else:
-                                path = '{}.{}.{}'.format(namespace, normalize_family(family.name), variable.name)
-                                _convert_variable(variable,
-                                                  'variable',
-                                                  )
-                                _convert_valid_enum(namespace,
-                                                    variable,
-                                                    path,
-                                                    )
-
-    def convert_helps(self):
-        if not hasattr(self.objectspace.space, 'help'):
-            return
-        helps = self.objectspace.space.help
-        if hasattr(helps, 'variable'):
-            for hlp in helps.variable.values():
-                variable = self.objectspace.paths.get_variable_obj(hlp.name)
-                variable.help = hlp.text
-        if hasattr(helps, 'family'):
-            for hlp in helps.family.values():
-                variable = self.objectspace.paths.get_family_obj(hlp.name)
-                variable.help = hlp.text
-        del self.objectspace.space.help
-
-    def convert_auto_freeze(self):  # pylint: disable=C0111
-        def _convert_auto_freeze(variable, namespace):
-            if variable.auto_freeze:
-                new_condition = self.objectspace.condition()
-                new_condition.name = 'auto_hidden_if_not_in'
-                new_condition.namespace = namespace
-                new_condition.source = FREEZE_AUTOFREEZE_VARIABLE
-                new_param = self.objectspace.param()
-                new_param.text = 'oui'
-                new_condition.param = [new_param]
-                new_target = self.objectspace.target()
-                new_target.type = 'variable'
-                path = variable.namespace + '.' + normalize_family(family.name) + '.' + variable.name
-                new_target.name = path
-                new_condition.target = [new_target]
-                if not hasattr(self.objectspace.space.constraints, 'condition'):
-                    self.objectspace.space.constraints.condition = []
-                self.objectspace.space.constraints.condition.append(new_condition)
-        if hasattr(self.objectspace.space, 'variables'):
-            for variables in self.objectspace.space.variables.values():
-                if hasattr(variables, 'family'):
-                    namespace = variables.name
-                    for family in variables.family.values():
-                        if hasattr(family, 'variable'):
-                            for variable in family.variable.values():
-                                if isinstance(variable, self.objectspace.Leadership):
-                                    for follower in variable.variable:
-                                        _convert_auto_freeze(follower, namespace)
-                                else:
-                                    _convert_auto_freeze(variable, namespace)
-
-    def convert_separators(self):  # pylint: disable=C0111,R0201
-        if not hasattr(self.objectspace.space, 'variables'):
-            return
-        for family in self.objectspace.space.variables.values():
-            if not hasattr(family, 'separators'):
-                continue
-            if hasattr(family.separators, 'separator'):
-                for idx, separator in enumerate(family.separators.separator):
-                    option = self.objectspace.paths.get_variable_obj(separator.name)
-                    if hasattr(option, 'separator'):
-                        subpath = self.objectspace.paths.get_variable_path(separator.name,
-                                                                           separator.namespace,
-                                                                           )
-                        raise DictConsistencyError(_('{} already has a separator').format(subpath))
-                    option.separator = separator.text
-            del family.separators
-
-
-class ConstraintAnnotator:
-    def __init__(self,
-                 objectspace,
-                 eosfunc_file,
-                 ):
-        if not hasattr(objectspace.space, 'constraints'):
-            return
-        self.objectspace = objectspace
-        self.eosfunc = imp.load_source('eosfunc', eosfunc_file)
-        self.valid_enums = {}
-        if hasattr(self.objectspace.space.constraints, 'check'):
-            self.check_check()
-            self.check_replace_text()
-            self.check_valid_enum()
-            self.check_change_warning()
-            self.convert_check()
-        if hasattr(self.objectspace.space.constraints, 'condition'):
-            self.check_params_target()
-            self.filter_targets()
-            self.convert_xxxlist_to_variable()
-            self.check_condition_fallback_optional()
-            self.check_choice_option_condition()
-            self.remove_condition_with_empty_target()
-            self.convert_condition()
-        if hasattr(self.objectspace.space.constraints, 'fill'):
-            self.convert_fill()
-        self.remove_constraints()
-
-    def check_check(self):
-        remove_indexes = []
-        functions = dir(self.eosfunc)
-        functions.extend(INTERNAL_FUNCTIONS)
-        for check_idx, check in enumerate(self.objectspace.space.constraints.check):
-            if not check.name in functions:
-                raise DictConsistencyError(_('cannot find check function {}').format(check.name))
-            if hasattr(check, 'param'):
-                param_option_indexes = []
-                for idx, param in enumerate(check.param):
-                    if param.type not in TYPE_PARAM_CHECK:
-                        raise DictConsistencyError(_('cannot use {} type as a param in check for {}').format(param.type, check.target))
-                    if param.type == 'variable' and not self.objectspace.paths.path_is_defined(param.text):
-                        if param.optional is True:
-                            param_option_indexes.append(idx)
-                        else:
-                            raise DictConsistencyError(_(f'unknown param {param.text} in check'))
-                    if param.type != 'variable':
-                        param.notraisepropertyerror = None
-                param_option_indexes = list(set(param_option_indexes))
-                param_option_indexes.sort(reverse=True)
-                for idx in param_option_indexes:
-                    check.param.pop(idx)
-                if check.param == []:
-                    remove_indexes.append(check_idx)
-        remove_indexes.sort(reverse=True)
-        for idx in remove_indexes:
-            del self.objectspace.space.constraints.check[idx]
-
-    def check_replace_text(self):
-        for check_idx, check in enumerate(self.objectspace.space.constraints.check):
-            namespace = check.namespace
-            if hasattr(check, 'param'):
-                for idx, param in enumerate(check.param):
-                    if param.type == 'variable':
-                        param.text = self.objectspace.paths.get_variable_path(param.text, namespace)
-            check.is_in_leadership = self.objectspace.paths.get_leader(check.target) != None
-            # let's replace the target by the path
-            check.target = self.objectspace.paths.get_variable_path(check.target, namespace)
-
-    def check_valid_enum(self):
-        remove_indexes = []
-        for idx, check in enumerate(self.objectspace.space.constraints.check):
-            if check.name == 'valid_enum':
-                if len(check.param) != 1:
-                    raise DictConsistencyError(_(f'cannot set more than one param for valid_enum for variable {check.target}'))
-                param = check.param[0]
-                if check.target in self.valid_enums:
-                    raise DictConsistencyError(_(f'valid_enum already set for {check.target}'))
-                if param.type not in ['string', 'python', 'number']:
-                    raise DictConsistencyError(_(f'unknown type {param.type} for param in valid_enum for {check.target}'))
-                variable = self.objectspace.paths.get_variable_obj(check.target)
-                values = self.load_params_in_validenum(param,
-                                                       variable.name,
-                                                       variable.type,
-                                                       )
-                self.valid_enums[check.target] = {'type': param.type,
-                                                  'values': values}
-                self._set_valid_enum(variable,
-                                     values,
-                                     variable.type,
-                                     )
-                remove_indexes.append(idx)
-        remove_indexes.sort(reverse=True)
-        for idx in remove_indexes:
-            del self.objectspace.space.constraints.check[idx]
-
-    def load_params_in_validenum(self,
-                                 param,
-                                 variable_name,
-                                 variable_type,
-                                 ):
-        if not hasattr(param, 'text') and (param.type == 'python' or param.type == 'number'):
-            raise DictConsistencyError(_(f"All '{param.type}' variables shall be set in order to calculate valid_enum for variable {variable_name}"))
-        if variable_type == 'string' and param.type == 'number':
-            raise DictConsistencyError(_(f'Unconsistency valid_enum type ({param.type}), for variable {variable_name}'))
-        if param.type == 'python':
-            try:
-                values = eval(param.text, {'eosfunc': self.eosfunc, '__builtins__': {'range': range, 'str': str}})
-            except NameError:
-                raise DictConsistencyError(_('The function {} is unknown').format(param.text))
-        else:
-            try:
-                values = literal_eval(param.text)
-            except ValueError:
-                raise DictConsistencyError(_(f'Cannot load {param.text} in valid_enum'))
-        if not isinstance(values, list):
-            raise DictConsistencyError(_('Function {} shall return a list').format(param.text))
-        for value in values:
-            if variable_type == 'string' and not isinstance(value, str):
-                raise DictConsistencyError(_(f'Cannot load "{param.text}", "{value}" is not a string'))
-            if variable_type == 'number' and not isinstance(value, int):
-                raise DictConsistencyError(_(f'Cannot load "{param.text}", "{value}" is not a number'))
-        return values
-
-    def check_change_warning(self):
-        #convert level to "warnings_only"
-        for check in self.objectspace.space.constraints.check:
-            if check.level == 'warning':
-                check.warnings_only = True
-            else:
-                check.warnings_only = False
-            check.level = None
-
-    def _get_family_variables_from_target(self,
-                                          target,
-                                          ):
-        if target.type == 'variable':
-            variable = self.objectspace.paths.get_variable_obj(target.name)
-            family = self.objectspace.paths.get_family_obj(target.name.rsplit('.', 1)[0])
-            if isinstance(family, self.objectspace.Leadership) and family.name == variable.name:
-                return family, family.variable
-            return variable, [variable]
-        # it's a family
-        variable = self.objectspace.paths.get_family_obj(target.name)
-        return variable, list(variable.variable.values())
-
-    def check_params_target(self):
-        for condition in self.objectspace.space.constraints.condition:
-            for param in condition.param:
-                if param.type not in TYPE_PARAM_CONDITION:
-                    raise DictConsistencyError(_(f'cannot use {param.type} type as a param in a condition'))
-            if not hasattr(condition, 'target'):
-                raise DictConsistencyError(_('target is mandatory in condition'))
-            for target in condition.target:
-                if target.type.endswith('list') and condition.name not in ['disabled_if_in', 'disabled_if_not_in']:
-                    raise DictConsistencyError(_(f'target in condition for {target.type} not allow in {condition.name}'))
-
-    def filter_targets(self):  # pylint: disable=C0111
-        for condition_idx, condition in enumerate(self.objectspace.space.constraints.condition):
-            namespace = condition.namespace
-            for idx, target in enumerate(condition.target):
-                if target.type == 'variable':
-                    if condition.source == target.name:
-                        raise DictConsistencyError(_('target name and source name must be different: {}').format(condition.source))
-                    try:
-                        target.name = self.objectspace.paths.get_variable_path(target.name, namespace)
-                    except DictConsistencyError:
-                        # for optional variable
-                        pass
-                elif target.type == 'family':
-                    try:
-                        target.name = self.objectspace.paths.get_family_path(target.name, namespace)
-                    except KeyError:
-                        raise DictConsistencyError(_('cannot found family {}').format(target.name))
-
-    def convert_xxxlist_to_variable(self):  # pylint: disable=C0111
-        # transform *list to variable or family
-        for condition_idx, condition in enumerate(self.objectspace.space.constraints.condition):
-            new_targets = []
-            remove_targets = []
-            for target_idx, target in enumerate(condition.target):
-                if target.type.endswith('list'):
-                    listname = target.type
-                    listvars = self.objectspace.list_conditions.get(listname,
-                                                                    {}).get(target.name)
-                    if listvars:
-                        for listvar in listvars:
-                            variable = self.objectspace.paths.get_variable_obj(listvar)
-                            type_ = 'variable'
-                            new_target = self.objectspace.target()
-                            new_target.type = type_
-                            new_target.name = listvar
-                            new_target.index = target.index
-                            new_targets.append(new_target)
-                    remove_targets.append(target_idx)
-            remove_targets.sort(reverse=True)
-            for target_idx in remove_targets:
-                condition.target.pop(target_idx)
-            condition.target.extend(new_targets)
-
-    def check_condition_fallback_optional(self):
-        # a condition with a fallback **and** the source variable doesn't exist
-        remove_conditions = []
-        for idx, condition in enumerate(self.objectspace.space.constraints.condition):
-            # fallback
-            if condition.fallback is True and not self.objectspace.paths.path_is_defined(condition.source):
-                apply_action = False
-                if condition.name in ['disabled_if_in', 'mandatory_if_in', 'hidden_if_in']:
-                    apply_action = not condition.force_condition_on_fallback
-                else:
-                    apply_action = condition.force_inverse_condition_on_fallback
-                if apply_action:
-                    actions = self._get_condition_actions(condition.name)
-                    for target in condition.target:
-                        leader_or_variable, variables = self._get_family_variables_from_target(target)
-                        for action_idx, action in enumerate(actions):
-                            if action_idx == 0:
-                                setattr(leader_or_variable, action, True)
-                            else:
-                                for variable in variables:
-                                    setattr(variable, action, True)
-                remove_conditions.append(idx)
-                continue
-
-            remove_targets = []
-            # optional
-            for idx, target in enumerate(condition.target):
-                if target.optional is True and not self.objectspace.paths.path_is_defined(target.name):
-                    remove_targets.append(idx)
-            remove_targets = list(set(remove_targets))
-            remove_targets.sort(reverse=True)
-            for idx in remove_targets:
-                condition.target.pop(idx)
-        remove_conditions = list(set(remove_conditions))
-        remove_conditions.sort(reverse=True)
-        for idx in remove_conditions:
-            self.objectspace.space.constraints.condition.pop(idx)
-
-    def _get_condition_actions(self, condition_name):
-        if condition_name.startswith('disabled_if_'):
-            return ['disabled']
-        elif condition_name.startswith('hidden_if_'):
-            return ['hidden', 'frozen', 'force_default_on_freeze']
-        elif condition_name.startswith('mandatory_if_'):
-            return ['mandatory']
-        elif condition_name == 'auto_hidden_if_not_in':
-            return ['auto_frozen']
-
-    def check_choice_option_condition(self):
-        # remove condition for ChoiceOption that don't have param
-        remove_conditions = []
-        for condition_idx, condition in enumerate(self.objectspace.space.constraints.condition):
-            namespace = condition.namespace
-            condition.source = self.objectspace.paths.get_variable_path(condition.source, namespace, allow_source=True)
-            src_variable = self.objectspace.paths.get_variable_obj(condition.source)
-            valid_enum = None
-            if condition.source in self.valid_enums and self.valid_enums[condition.source]['type'] == 'string':
-                valid_enum = self.valid_enums[condition.source]['values']
-            if valid_enum is not None:
-                remove_param = []
-                for param_idx, param in enumerate(condition.param):
-                    if param.text not in valid_enum:
-                        remove_param.append(param_idx)
-                remove_param.sort(reverse=True)
-                for idx in remove_param:
-                    del condition.param[idx]
-                if condition.param == []:
-                    remove_targets = []
-                    for target in condition.target:
-                        leader_or_variable, variables = self._get_family_variables_from_target(target)
-                        if condition.name == 'disabled_if_not_in':
-                            leader_or_variable.disabled = True
-                        elif condition.name == 'hidden_if_not_in':
-                            leader_or_variable.hidden = True
-                            for variable in variables:
-                                variable.frozen = True
-                                variable.force_default_on_freeze = True
-                        elif condition.name == 'mandatory_if_not_in':
-                            variable.mandatory = True
-                    remove_targets = list(set(remove_targets))
-                    remove_targets.sort(reverse=True)
-                    for target_idx in remove_targets:
-                        condition.target.pop(target_idx)
-                    remove_conditions.append(condition_idx)
-        remove_conditions = list(set(remove_conditions))
-        remove_conditions.sort(reverse=True)
-        for idx in remove_conditions:
-            self.objectspace.space.constraints.condition.pop(idx)
-
-    def remove_condition_with_empty_target(self):
-        remove_conditions = []
-        for condition_idx, condition in enumerate(self.objectspace.space.constraints.condition):
-            if not condition.target:
-                remove_conditions.append(condition_idx)
-        remove_conditions = list(set(remove_conditions))
-        remove_conditions.sort(reverse=True)
-        for idx in remove_conditions:
-            self.objectspace.space.constraints.condition.pop(idx)
-
-    def convert_condition(self):
-        for condition in self.objectspace.space.constraints.condition:
-            inverse = condition.name.endswith('_if_not_in')
-            actions = self._get_condition_actions(condition.name)
-            for param in condition.param:
-                if hasattr(param, 'text'):
-                    param = param.text
-                else:
-                    param = None
-                for target in condition.target:
-                    leader_or_variable, variables = self._get_family_variables_from_target(target)
-                    # if option is already disable, do not apply disable_if_in
-                    if hasattr(leader_or_variable, actions[0]) and getattr(leader_or_variable, actions[0]) is True:
-                        continue
-                    for idx, action in enumerate(actions):
-                        prop = self.objectspace.property_()
-                        prop.type = 'calculation'
-                        prop.inverse = inverse
-                        prop.source = condition.source
-                        prop.expected = param
-                        prop.name = action
-                        if idx == 0:
-                            if not hasattr(leader_or_variable, 'property'):
-                                leader_or_variable.property = []
-                            leader_or_variable.property.append(prop)
-                        else:
-                            for variable in variables:
-                                if not hasattr(variable, 'property'):
-                                    variable.property = []
-                                variable.property.append(prop)
-        del self.objectspace.space.constraints.condition
-
-    def _set_valid_enum(self, variable, values, type_):
-        # value for choice's variable is mandatory
-        variable.mandatory = True
-        # build choice
-        variable.choice = []
-        choices = []
-        for value in values:
-            choice = self.objectspace.choice()
-            try:
-                choice.name = CONVERSION.get(type_, str)(value)
-            except:
-                raise DictConsistencyError(_(f'unable to change type of a valid_enum entry "{value}" is not a valid "{type_}" for "{variable.name}"'))
-            if choice.name == '':
-                choice.name = None
-            choices.append(choice.name)
-            choice.type = type_
-            variable.choice.append(choice)
-        if not variable.choice:
-            raise DictConsistencyError(_('empty valid enum is not allowed for variable {}').format(variable.name))
-        # check value or set first choice value has default value
-        if hasattr(variable, 'value'):
-            for value in variable.value:
-                value.type = type_
-                try:
-                    cvalue = CONVERSION.get(type_, str)(value.name)
-                except:
-                    raise DictConsistencyError(_(f'unable to change type of value "{value}" is not a valid "{type_}" for "{variable.name}"'))
-                if cvalue not in choices:
-                    raise DictConsistencyError(_('value "{}" of variable "{}" is not in list of all expected values ({})').format(value.name, variable.name, choices))
-        else:
-            new_value = self.objectspace.value()
-            new_value.name = values[0]
-            new_value.type = type_
-            variable.value = [new_value]
-        variable.type = 'choice'
-
-    def convert_check(self):
-        for check in self.objectspace.space.constraints.check:
-            variable = self.objectspace.paths.get_variable_obj(check.target)
-            name = check.name
-            if name == 'valid_entier':
-                if not hasattr(check, 'param'):
-                    raise DictConsistencyError(_('{} must have, at least, 1 param').format(name))
-                for param in check.param:
-                    if param.type not in ['string', 'number']:
-                        raise DictConsistencyError(_(f'param in "valid_entier" must not be a "{param.type}"'))
-                    if param.name == 'mini':
-                        variable.min_number = int(param.text)
-                    elif param.name == 'maxi':
-                        variable.max_number = int(param.text)
-                    else:
-                        raise DictConsistencyError(_(f'unknown parameter {param.text} in check "valid_entier" for variable {check.target}'))
-            else:
-                check_ = self.objectspace.check()
-                if name == 'valid_differ':
-                    name = 'valid_not_equal'
-                elif name == 'valid_network_netmask':
-                    params_len = 1
-                    if len(check.param) != params_len:
-                        raise DictConsistencyError(_('{} must have {} param').format(name, params_len))
-                elif name == 'valid_ipnetmask':
-                    params_len = 1
-                    if len(check.param) != params_len:
-                        raise DictConsistencyError(_('{} must have {} param').format(name, params_len))
-                    name = 'valid_ip_netmask'
-                elif name == 'valid_broadcast':
-                    params_len = 2
-                    if len(check.param) != params_len:
-                        raise DictConsistencyError(_('{} must have {} param').format(name, params_len))
-                elif name == 'valid_in_network':
-                    params_len = 2
-                    if len(check.param) != params_len:
-                        raise DictConsistencyError(_('{} must have {} param').format(name, params_len))
-                check_.name = name
-                check_.warnings_only = check.warnings_only
-                if hasattr(check, 'param'):
-                    check_.param = check.param
-                if not hasattr(variable, 'check'):
-                    variable.check = []
-                variable.check.append(check_)
-        del self.objectspace.space.constraints.check
-
-    def convert_fill(self):  # pylint: disable=C0111,R0912
-        # sort fill/auto by index
-        fills = {fill.index: fill for idx, fill in enumerate(self.objectspace.space.constraints.fill)}
-        indexes = list(fills.keys())
-        indexes.sort()
-        targets = []
-        eosfunc = dir(self.eosfunc)
-        for idx in indexes:
-            fill = fills[idx]
-            # test if it's redefined calculation
-            if fill.target in targets and not fill.redefine:
-                raise DictConsistencyError(_(f"A fill already exists for the target: {fill.target}"))
-            targets.append(fill.target)
-            #
-            if fill.name not in eosfunc:
-                raise DictConsistencyError(_('cannot find fill function {}').format(fill.name))
-
-            namespace = fill.namespace
-            # let's replace the target by the path
-            fill.target = self.objectspace.paths.get_variable_path(fill.target,
-                                                                   namespace,
-                                                                   )
-
-            value = self.objectspace.value()
-            value.type = 'calculation'
-            value.name = fill.name
-            if hasattr(fill, 'param'):
-                param_to_delete = []
-                for fill_idx, param in enumerate(fill.param):
-                    if param.type not in TYPE_PARAM_FILL:
-                        raise DictConsistencyError(_(f'cannot use {param.type} type as a param in a fill/auto'))
-                    if param.type != 'string' and not hasattr(param, 'text'):
-                        raise DictConsistencyError(_(f"All '{param.type}' variables shall have a value in order to calculate {fill.target}"))
-                    if param.type == 'variable':
-                        try:
-                            param.text, suffix = self.objectspace.paths.get_variable_path(param.text,
-                                                                                          namespace,
-                                                                                          with_suffix=True,
-                                                                                          )
-                            if suffix:
-                                param.suffix = suffix
-                        except DictConsistencyError as err:
-                            if param.optional is False:
-                                raise err
-                            param_to_delete.append(fill_idx)
-                            continue
-                    else:
-                        param.notraisepropertyerror = None
-                param_to_delete.sort(reverse=True)
-                for param_idx in param_to_delete:
-                    fill.param.pop(param_idx)
-                value.param = fill.param
-            variable = self.objectspace.paths.get_variable_obj(fill.target)
-            variable.value = [value]
-        del self.objectspace.space.constraints.fill
-
-    def remove_constraints(self):
-        if hasattr(self.objectspace.space.constraints, 'index'):
-            del self.objectspace.space.constraints.index
-        del self.objectspace.space.constraints.namespace
-        if vars(self.objectspace.space.constraints):
-            raise Exception('constraints again?')
-        del self.objectspace.space.constraints
-
-
-class FamilyAnnotator:
-    def __init__(self,
-                 objectspace,
-                 ):
-        self.objectspace = objectspace
-        self.remove_empty_families()
-        self.change_variable_mode()
-        self.change_family_mode()
-        self.dynamic_families()
-
-    def remove_empty_families(self):  # pylint: disable=C0111,R0201
-        if hasattr(self.objectspace.space, 'variables'):
-            for family in self.objectspace.space.variables.values():
-                if hasattr(family, 'family'):
-                    space = family.family
-                    removed_families = []
-                    for family_name, family in space.items():
-                        if not hasattr(family, 'variable') or len(family.variable) == 0:
-                            removed_families.append(family_name)
-                    for family_name in removed_families:
-                        del space[family_name]
-
-    def change_family_mode(self):  # pylint: disable=C0111
-        if not hasattr(self.objectspace.space, 'variables'):
-            return
-        for family in self.objectspace.space.variables.values():
-            if hasattr(family, 'family'):
-                for family in family.family.values():
-                    mode = modes_level[-1]
-                    for variable in family.variable.values():
-                        if isinstance(variable, self.objectspace.Leadership):
-                            variable_mode = variable.variable[0].mode
-                            variable.variable[0].mode = None
-                            variable.mode = variable_mode
-                        else:
-                            variable_mode = variable.mode
-                        if variable_mode is not None and modes[mode] > modes[variable_mode]:
-                            mode = variable_mode
-                    family.mode = mode
-
-    def dynamic_families(self):  # pylint: disable=C0111
-        if not hasattr(self.objectspace.space, 'variables'):
-            return
-        for family in self.objectspace.space.variables.values():
-            if hasattr(family, 'family'):
-                for family in family.family.values():
-                    if 'dynamic' in vars(family):
-                        namespace = self.objectspace.paths.get_variable_namespace(family.dynamic)
-                        varpath = self.objectspace.paths.get_variable_path(family.dynamic, namespace)
-                        family.dynamic = varpath
-
-    def annotate_variable(self, variable, family_mode, path, is_follower=False):
-        # if the variable is mandatory and doesn't have any value
-        # then the variable's mode is set to 'basic'
-        if not hasattr(variable, 'value') and variable.type == 'boolean':
-            new_value = self.objectspace.value()
-            new_value.name = True
-            new_value.type = 'boolean'
-            variable.value = [new_value]
-        if hasattr(variable, 'value') and variable.value:
-            has_value = True
-            for value in variable.value:
-                if value.type == 'calculation':
-                    has_value = False
-                    has_variable = False
-                    if hasattr(value, 'param'):
-                        for param in value.param:
-                            if param.type == 'variable':
-                                has_variable = True
-                                break
-                    if not has_variable:
-                        # if one parameter is a variable, let variable choice if it's mandatory
-                        variable.mandatory = True
-            if has_value:
-                # if has value but without any calculation
-                variable.mandatory = True
-        if variable.mandatory is True and (not hasattr(variable, 'value') or is_follower):
-            variable.mode = modes_level[0]
-        if variable.mode != None and modes[variable.mode] < modes[family_mode] and (not is_follower or variable.mode != modes_level[0]):
-            variable.mode = family_mode
-        if variable.hidden is True:
-            variable.frozen = True
-            if not variable.auto_save is True and 'force_default_on_freeze' not in vars(variable):
-                variable.force_default_on_freeze = True
-
-    def change_variable_mode(self):  # pylint: disable=C0111
-        if not hasattr(self.objectspace.space, 'variables'):
-            return
-        for variables in self.objectspace.space.variables.values():
-            namespace = variables.name
-            if hasattr(variables, 'family'):
-                for family in variables.family.values():
-                    family_mode = family.mode
-                    if hasattr(family, 'variable'):
-                        for variable in family.variable.values():
-
-                            if isinstance(variable, self.objectspace.Leadership):
-                                mode = modes_level[-1]
-                                for idx, follower in enumerate(variable.variable):
-                                    if follower.auto_save is True:
-                                        raise DictConsistencyError(_(f'leader/followers {follower.name} could not be auto_save'))
-                                    if follower.auto_freeze is True:
-                                        raise DictConsistencyError(_('leader/followers {follower.name} could not be auto_freeze'))
-                                    is_follower = idx != 0
-                                    path = '{}.{}.{}'.format(family.path, variable.name, follower.name)
-                                    self.annotate_variable(follower, family_mode, path, is_follower)
-                                    # leader's mode is minimum level
-                                    if modes[variable.variable[0].mode] > modes[follower.mode]:
-                                        follower.mode = variable.variable[0].mode
-                                variable.mode = variable.variable[0].mode
-                            else:
-                                # auto_save's variable is set in 'basic' mode if its mode is 'normal'
-                                if variable.auto_save is True and variable.mode != modes_level[-1]:
-                                    variable.mode = modes_level[0]
-                                # auto_freeze's variable is set in 'basic' mode if its mode is 'normal'
-                                if variable.auto_freeze is True and variable.mode != modes_level[-1]:
-                                    variable.mode = modes_level[0]
-                                path = '{}.{}'.format(family.path, variable.name)
-                                self.annotate_variable(variable, family_mode, path)
-
-class PropertyAnnotator:
-    def __init__(self, objectspace):
-        self.objectspace = objectspace
-        self.convert_annotator()
-
-    def convert_property(self,
-                         variable,
-                         ):
-        properties = []
-        for prop in PROPERTIES:
-            if hasattr(variable, prop):
-                if getattr(variable, prop) == True:
-                    for subprop in CONVERT_PROPERTIES.get(prop, [prop]):
-                        properties.append(subprop)
-                setattr(variable, prop, None)
-        if hasattr(variable, 'mode') and variable.mode:
-            properties.append(variable.mode)
-            variable.mode = None
-        if properties:
-            variable.properties = frozenset(properties)
-
-    def convert_annotator(self):  # pylint: disable=C0111
-        if hasattr(self.objectspace.space, 'services'):
-            self.convert_property(self.objectspace.space.services)
-            for services in self.objectspace.space.services.service.values():
-                self.convert_property(services)
-                for service in vars(services).values():
-                    if isinstance(service, self.objectspace.family):
-                        self.convert_property(service)
-                        if hasattr(service, 'family'):
-                            self.convert_property(service)
-                            for family in service.family:
-                                self.convert_property(family)
-                                if hasattr(family, 'variable'):
-                                    for variable in family.variable:
-                                        self.convert_property(variable)
-        if hasattr(self.objectspace.space, 'variables'):
-            for variables in self.objectspace.space.variables.values():
-                if hasattr(variables, 'family'):
-                    for family in variables.family.values():
-                        self.convert_property(family)
-                        if hasattr(family, 'variable'):
-                            for variable in family.variable.values():
-                                if isinstance(variable, self.objectspace.Leadership):
-                                    self.convert_property(variable)
-                                    for follower in variable.variable:
-                                        self.convert_property(follower)
-                                else:
-                                    self.convert_property(variable)

src/rougail/annotator/__init__.py

@@ -0,0 +1,93 @@
+"""Annotate dictionaries
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from .variable import CONVERT_OPTION
+import importlib.resources
+from os.path import dirname, join
+from ..utils import load_modules
+
+
+ANNOTATORS = None
+
+
+if not 'files' in dir(importlib.resources):
+    # old python version
+    class fake_files:
+        def __init__(self, package):
+            self.mod = []
+            dir_package = dirname(importlib.resources._get_package(package).__file__)
+            for mod in importlib.resources.contents(package):
+                self.mod.append(join(dir_package, mod))
+
+        def iterdir(self):
+            return self.mod
+    importlib.resources.files = fake_files
+
+
+def get_level(module):
+    return module.level
+
+
+def get_annotators(annotators, module_name):
+    annotators[module_name] = []
+    for pathobj in importlib.resources.files(module_name).iterdir():
+        path = str(pathobj)
+        if not path.endswith('__') and not path.endswith('__.py'):
+            module = load_modules(path)
+            if 'Annotator' in dir(module):
+                annotators[module_name].append(module.Annotator)
+
+
+class SpaceAnnotator:  # pylint: disable=R0903
+    """Transformations applied on a object instance
+    """
+    def __init__(self,
+                 objectspace,
+                 eosfunc_files,
+                 ):
+        global ANNOTATORS
+        if ANNOTATORS is None:
+            ANNOTATORS = {}
+            get_annotators(ANNOTATORS, 'rougail.annotator')
+            for extra_annotator in objectspace.rougailconfig['extra_annotators']:
+                get_annotators(ANNOTATORS, extra_annotator)
+        for extra_annotator in objectspace.rougailconfig['extra_annotators']:
+            if extra_annotator not in ANNOTATORS:
+                get_annotators(ANNOTATORS, extra_annotator)
+        annotators = ANNOTATORS['rougail.annotator'].copy()
+        for extra_annotator in objectspace.rougailconfig['extra_annotators']:
+            annotators.extend(ANNOTATORS[extra_annotator])
+        annotators = sorted(annotators, key=get_level)
+        functions = []
+        for eosfunc_file in eosfunc_files:
+            functions.extend(dir(load_modules(eosfunc_file)))
+        for annotator in annotators:
+            annotator(objectspace,
+                      functions,
+                      )
+
+
+__all__ = ('SpaceAnnotator', 'CONVERT_OPTION')

src/rougail/annotator/check.py

@@ -0,0 +1,123 @@
+"""Annotate check
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import List, Any
+from copy import copy
+
+from rougail.annotator.target import TargetAnnotator
+from rougail.annotator.param import ParamAnnotator
+
+from rougail.i18n import _
+from rougail.error import DictConsistencyError, display_xmlfiles
+
+INTERNAL_FUNCTIONS = ['valid_in_network', 'valid_differ', 'valid_entier']
+
+class Annotator(TargetAnnotator, ParamAnnotator):
+    """Annotate check
+    """
+    level = 40
+    def __init__(self,
+                 objectspace,
+                 functions,
+                 *args,
+                 ):
+        if not hasattr(objectspace.space, 'constraints') or \
+                not hasattr(objectspace.space.constraints, 'check'):
+            return
+        self.objectspace = objectspace
+        self.let_none = True
+        self.only_variable = True
+        self.functions = copy(functions)
+        self.functions.extend(INTERNAL_FUNCTIONS)
+        self.functions.extend(self.objectspace.rougailconfig['internal_functions'])
+        self.target_is_uniq = False
+        self.allow_function = True
+        self.convert_target(self.objectspace.space.constraints.check)
+        self.convert_param(self.objectspace.space.constraints.check)
+        self.check_check()
+        self.check_change_warning()
+        self.convert_valid_entier()
+        self.convert_check()
+        del objectspace.space.constraints.check
+
+    def check_check(self):  # pylint: disable=R0912
+        """valid and manage <check>
+        """
+        remove_indexes = []
+        for check_idx, check in enumerate(self.objectspace.space.constraints.check):
+            if not check.name in self.functions:
+                msg = _(f'cannot find check function "{check.name}"')
+                raise DictConsistencyError(msg, 1, check.xmlfiles)
+            if hasattr(check, 'param') and check.param == []:
+                remove_indexes.append(check_idx)
+        remove_indexes.sort(reverse=True)
+        for idx in remove_indexes:
+            del self.objectspace.space.constraints.check[idx]
+
+    def check_change_warning(self):
+        """convert level to "warnings_only"
+        """
+        for check in self.objectspace.space.constraints.check:
+            check.warnings_only = check.level == 'warning'
+            check.level = None
+
+    def convert_valid_entier(self) -> None:
+        """valid and manage <check>
+        """
+        remove_indexes = []
+        for check_idx, check in enumerate(self.objectspace.space.constraints.check):
+            if not check.name == 'valid_entier':
+                continue
+            remove_indexes.append(check_idx)
+            if not hasattr(check, 'param'):
+                msg = _(f'{check.name} must have, at least, 1 param')
+                raise DictConsistencyError(msg, 17, check.xmlfiles)
+            for param in check.param:
+                if param.type != 'number':
+                    msg = _(f'param in "valid_entier" must be an "integer", not "{param.type}"')
+                    raise DictConsistencyError(msg, 18, check.xmlfiles)
+                if not hasattr(param, 'name'):
+                    continue
+                for target in check.target:
+                    if param.name == 'mini':
+                        target.name.min_number = int(param.text)
+                    elif param.name == 'maxi':
+                        target.name.max_number = int(param.text)
+                    else:
+                        msg = _(f'unknown parameter "{param.name}" in check "valid_entier"')
+                        raise DictConsistencyError(msg, 19, check.xmlfiles)
+        remove_indexes.sort(reverse=True)
+        for idx in remove_indexes:
+            del self.objectspace.space.constraints.check[idx]
+
+    def convert_check(self) -> None:
+        """valid and manage <check>
+        """
+        for check in self.objectspace.space.constraints.check:
+            for target in check.target:
+                if not hasattr(target.name, 'validators'):
+                    target.name.validators = []
+                target.name.validators.append(check)

src/rougail/annotator/condition.py

@@ -0,0 +1,370 @@
+"""Annotate condition
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import List
+
+
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+
+from rougail.annotator.target import TargetAnnotator
+from rougail.annotator.param import ParamAnnotator
+from rougail.annotator.variable import Walk
+
+
+class Annotator(TargetAnnotator, ParamAnnotator, Walk):
+    """Annotate condition
+    """
+    level = 50
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ):
+        self.objectspace = objectspace
+        self.force_service_value = {}
+        if hasattr(objectspace.space, 'variables'):
+            self.convert_auto_freeze()
+        if not hasattr(objectspace.space, 'constraints') or \
+                not hasattr(self.objectspace.space.constraints, 'condition'):
+            return
+        self.target_is_uniq = False
+        self.only_variable = False
+        self.allow_function = False
+        self.convert_target(self.objectspace.space.constraints.condition)
+        self.check_condition_optional()
+        self.convert_condition_source()
+        self.convert_param(self.objectspace.space.constraints.condition)
+        self.check_source_target()
+        self.convert_xxxlist()
+        self.check_choice_option_condition()
+        self.remove_condition_with_empty_target()
+        self.convert_condition()
+
+    def valid_type_validation(self,
+                              obj,
+                              ) -> None:
+        if obj.source.type == 'choice':
+            return None
+        return obj.source.type
+
+    def convert_auto_freeze(self):
+        """convert auto_freeze
+        only if auto_freeze_variable is True this variable is frozen
+        """
+        for variable in self.get_variables():
+            if not variable.auto_freeze and not variable.auto_save:
+                continue
+            #if variable.namespace != self.objectspace.rougailconfig['variable_namespace']:
+            #    msg = _(f'auto_freeze is not allowed in extra "{variable.namespace}"')
+            #    raise DictConsistencyError(msg, 49, variable.xmlfiles)
+            variable.force_store_value = True
+            if variable.auto_save:
+                continue
+            new_condition = self.objectspace.condition(variable.xmlfiles)
+            new_condition.name = 'auto_frozen_if_in'
+            new_condition.namespace = variable.namespace
+            new_condition.source = self.objectspace.rougailconfig['auto_freeze_variable']
+            new_param = self.objectspace.param(variable.xmlfiles)
+            new_param.text = True
+            new_condition.param = [new_param]
+            new_target = self.objectspace.target(variable.xmlfiles)
+            new_target.type = 'variable'
+            new_target.name = variable.path
+            new_condition.target = [new_target]
+            if not hasattr(self.objectspace.space, 'constraints'):
+                self.objectspace.space.constraints = self.objectspace.constraints(variable.xmlfiles)
+            if not hasattr(self.objectspace.space.constraints, 'condition'):
+                self.objectspace.space.constraints.condition = []
+            self.objectspace.space.constraints.condition.append(new_condition)
+
+    def check_source_target(self):
+        """verify that source != target in condition
+        """
+        for condition in self.objectspace.space.constraints.condition:
+            for target in condition.target:
+                if target.type == 'variable' and \
+                        condition.source.path == target.name.path:
+                    msg = _('target name and source name must be different: '
+                            f'{condition.source.path}')
+                    raise DictConsistencyError(msg, 11, condition.xmlfiles)
+
+    def check_condition_optional(self):
+        """a condition with a optional **and** the source variable doesn't exist
+        """
+        remove_conditions = []
+        for idx, condition in enumerate(self.objectspace.space.constraints.condition):
+            if condition.optional is False or \
+                    self.objectspace.paths.path_is_defined(condition.source):
+                continue
+            remove_conditions.append(idx)
+            if (hasattr(condition, 'apply_on_fallback') and condition.apply_on_fallback) or \
+                    (not hasattr(condition, 'apply_on_fallback') and \
+                     condition.name.endswith('_if_in')):
+                self.force_actions_to_variable(condition)
+        remove_conditions.sort(reverse=True)
+        for idx in remove_conditions:
+            self.objectspace.space.constraints.condition.pop(idx)
+
+    def force_actions_to_variable(self,
+                                  condition: 'self.objectspace.condition',
+                                  ) -> None:
+        """force property to a variable
+        for example disabled_if_not_in => variable.disabled = True
+        """
+        actions = self.get_actions_from_condition(condition.name)
+        for target in condition.target:
+            main_action = actions[0]
+            if target.type.endswith('list'):
+                self.force_service_value[target.name] = main_action != 'disabled'
+                continue
+            leader_or_var, variables = self._get_family_variables_from_target(target)
+            setattr(leader_or_var, main_action, True)
+            for action in actions[1:]:
+                for variable in variables:
+                    setattr(variable, action, True)
+
+    @staticmethod
+    def get_actions_from_condition(condition_name: str) -> List[str]:
+        """get action's name from a condition
+        """
+        if condition_name.startswith('hidden_if_'):
+            return ['hidden', 'frozen', 'force_default_on_freeze']
+        if condition_name == 'auto_frozen_if_in':
+            return ['frozen']
+        return [condition_name.split('_', 1)[0]]
+
+    def _get_family_variables_from_target(self,
+                                          target,
+                                          ):
+        if target.type == 'variable':
+            if not self.objectspace.paths.is_leader(target.name.path):
+                return target.name, [target.name]
+            # it's a leader, so apply property to leadership
+            family_name = self.objectspace.paths.get_variable_family_path(target.name.path)
+            namespace = target.name.namespace
+        else:
+            family_name = target.name.path
+            namespace = target.namespace
+        variable = self.objectspace.paths.get_family(family_name,
+                                                     namespace,
+                                                     )
+        if hasattr(variable, 'variable'):
+            return variable, list(variable.variable.values())
+        return variable, []
+
+    def convert_xxxlist(self):
+        """transform *list to variable or family
+        """
+        fills = {}
+        for condition in self.objectspace.space.constraints.condition:
+            remove_targets = []
+            for target_idx, target in enumerate(condition.target):
+                if target.type.endswith('list'):
+                    listvars = self.objectspace.list_conditions.get(target.type,
+                                                                    {}).get(target.name)
+                    if listvars:
+                        self._convert_xxxlist_to_fill(condition,
+                                                      target,
+                                                      listvars,
+                                                      fills,
+                                                      )
+                    elif not target.optional:
+                        msg = f'cannot found target "{target.type}" "{target.name}"'
+                        raise DictConsistencyError(_(msg), 2, target.xmlfiles)
+                    remove_targets.append(target_idx)
+            remove_targets.sort(reverse=True)
+            for target_idx in remove_targets:
+                condition.target.pop(target_idx)
+
+    def _convert_xxxlist_to_fill(self,
+                                 condition: 'self.objectspace.condition',
+                                 target: 'self.objectspace.target',
+                                 listvars: list,
+                                 fills: dict,
+                                 ):
+        for listvar in listvars:
+            if target.name in self.force_service_value:
+                listvar.default = self.force_service_value[target.name]
+                continue
+            if listvar.path in fills:
+                fill = fills[listvar.path]
+                or_needed = True
+                for param in fill.param:
+                    if hasattr(param, 'name') and \
+                            param.name == 'condition_operator':
+                        or_needed = False
+                        break
+                fill.index += 1
+            else:
+                fill = self.objectspace.fill(target.xmlfiles)
+                new_target = self.objectspace.target(target.xmlfiles)
+                new_target.name = listvar.path
+                fill.target = [new_target]
+                fill.name = 'calc_value'
+                fill.namespace = 'services'
+                fill.index = 0
+                if not hasattr(self.objectspace.space.constraints, 'fill'):
+                    self.objectspace.space.constraints.fill = []
+                self.objectspace.space.constraints.fill.append(fill)
+                fills[listvar.path] = fill
+                param1 = self.objectspace.param(target.xmlfiles)
+                param1.text = False
+                param1.type = 'boolean'
+                param2 = self.objectspace.param(target.xmlfiles)
+                param2.name = 'default'
+                param2.text = True
+                param2.type = 'boolean'
+                fill.param = [param1, param2]
+                or_needed = len(condition.param) != 1
+            if len(condition.param) == 1:
+                values = getattr(condition.param[0], 'text', None)
+                param_type = condition.param[0].type
+            else:
+                values = tuple([getattr(param, 'text', None) for param in condition.param])
+                param_type = None
+                for param in condition.param:
+                    if param_type is None or param_type == 'nil':
+                        param_type = param.type
+            param3 = self.objectspace.param(target.xmlfiles)
+            param3.name = f'condition_{fill.index}'
+            param3.type = 'variable'
+            param3.text = condition.source.path
+            param3.propertyerror = False
+            fill.param.append(param3)
+            param4 = self.objectspace.param(target.xmlfiles)
+            param4.name = f'expected_{fill.index}'
+            param4.text = values
+            param4.type = param_type
+            fill.param.append(param4)
+            if condition.name != 'disabled_if_in':
+                param5 = self.objectspace.param(target.xmlfiles)
+                param5.name = f'reverse_condition_{fill.index}'
+                param5.text = True
+                param5.type = 'boolean'
+                fill.param.append(param5)
+            if or_needed:
+                param6 = self.objectspace.param(target.xmlfiles)
+                param6.name = 'condition_operator'
+                param6.text = 'OR'
+                fill.param.append(param6)
+
+    def convert_condition_source(self):
+        """remove condition for ChoiceOption that don't have param
+        """
+        for condition in self.objectspace.space.constraints.condition:
+            try:
+                condition.source = self.objectspace.paths.get_variable(condition.source)
+            except DictConsistencyError as err:
+                if err.errno == 36:
+                    msg = _(f'the source "{condition.source}" in condition cannot be a dynamic '
+                            f'variable')
+                    raise DictConsistencyError(msg, 20, condition.xmlfiles) from err
+                if err.errno == 42:
+                    msg = _(f'the source "{condition.source}" in condition is an unknown variable')
+                    raise DictConsistencyError(msg, 23, condition.xmlfiles) from err
+                raise err from err  # pragma: no cover
+
+    def check_choice_option_condition(self):
+        """remove condition of ChoiceOption that doesn't match
+        """
+        remove_conditions = []
+        for condition_idx, condition in enumerate(self.objectspace.space.constraints.condition):
+            if condition.source.path in self.objectspace.valid_enums:
+                valid_enum = self.objectspace.valid_enums[condition.source.path]['values']
+                remove_param = [param_idx for param_idx, param in enumerate(condition.param) \
+                                if param.type != 'variable' and param.text not in valid_enum]
+                if not remove_param:
+                    continue
+                remove_param.sort(reverse=True)
+                for idx in remove_param:
+                    del condition.param[idx]
+                if not condition.param and condition.name.endswith('_if_not_in'):
+                    self.force_actions_to_variable(condition)
+                    remove_conditions.append(condition_idx)
+        remove_conditions.sort(reverse=True)
+        for idx in remove_conditions:
+            self.objectspace.space.constraints.condition.pop(idx)
+
+    def remove_condition_with_empty_target(self):
+        """remove condition with empty target
+        """
+        # optional target are remove, condition could be empty
+        remove_conditions = [condition_idx for condition_idx, condition in \
+                             enumerate(self.objectspace.space.constraints.condition) \
+                             if not condition.target]
+        remove_conditions.sort(reverse=True)
+        for idx in remove_conditions:
+            self.objectspace.space.constraints.condition.pop(idx)
+
+    def convert_condition(self):
+        """valid and manage <condition>
+        """
+        for condition in self.objectspace.space.constraints.condition:
+            actions = self.get_actions_from_condition(condition.name)
+            for param in condition.param:
+                for target in condition.target:
+                    leader_or_variable, variables = self._get_family_variables_from_target(target)
+                    # if option is already disable, do not apply disable_if_in
+                    # check only the first action (example of multiple actions:
+                    # 'hidden', 'frozen', 'force_default_on_freeze')
+                    main_action = actions[0]
+                    if getattr(leader_or_variable, main_action, False) is True:
+                        continue
+                    self.build_property(leader_or_variable,
+                                        param,
+                                        condition,
+                                        main_action,
+                                        )
+                    if isinstance(leader_or_variable, self.objectspace.variable) and \
+                            (leader_or_variable.auto_save or leader_or_variable.auto_freeze) and \
+                            'force_default_on_freeze' in actions:
+                        continue
+                    for action in actions[1:]:
+                        # other actions are set to the variable or children of family
+                        for variable in variables:
+                            self.build_property(variable,
+                                                param,
+                                                condition,
+                                                action,
+                                                )
+
+    def build_property(self,
+                       obj,
+                       param: 'self.objectspace.param',
+                       condition: 'self.objectspace.condition',
+                       action: str,
+                       ) -> 'self.objectspace.property_':
+        """build property_ for a condition
+        """
+        prop = self.objectspace.property_(obj.xmlfiles)
+        prop.type = 'calculation'
+        prop.inverse = condition.name.endswith('_if_not_in')
+        prop.source = condition.source
+        prop.expected = param
+        prop.name = action
+        if not hasattr(obj, 'properties'):
+            obj.properties = []
+        obj.properties.append(prop)

src/rougail/annotator/family.py

@@ -0,0 +1,291 @@
+"""Annotate family
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+from rougail.annotator.variable import Walk
+
+
+class Mode:  # pylint: disable=R0903
+    """Class to manage mode level
+    """
+    def __init__(self,
+                 level: int,
+                 ) -> None:
+        self.level = level
+
+    def __gt__(self,
+               other: int,
+               ) -> bool:
+        return other.level < self.level
+
+
+class Annotator(Walk):
+    """Annotate family
+    """
+    level = 80
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ):
+        self.objectspace = objectspace
+        if not hasattr(self.objectspace.space, 'variables'):
+            return
+        self.modes = {name: Mode(idx) for idx, name in enumerate(self.objectspace.rougailconfig['modes_level'])}
+        self.remove_empty_families()
+        self.family_names()
+        self.change_modes()
+        self.dynamic_families()
+        self.convert_help()
+
+    def _has_variable(self,
+                      family: 'self.objectspace.family',
+                      ) -> bool:
+        if hasattr(family, 'variable'):
+            for variable in family.variable.values():
+                if isinstance(variable, self.objectspace.family):
+                    if self._has_variable(variable):
+                        return True
+                else:
+                    return True
+        return False
+
+    def remove_empty_families(self) -> None:
+        """Remove all families without any variable
+        """
+        #FIXME pas sous family
+        for families in self.objectspace.space.variables.values():
+            removed_families = []
+            for family_name, family in families.variable.items():
+                if isinstance(family, self.objectspace.family) and not self._has_variable(family):
+                    removed_families.append(family_name)
+            for family_name in removed_families:
+                del families.variable[family_name]
+
+    def family_names(self) -> None:
+        """Set doc, path, ... to family
+        """
+        for family in self.get_families():
+            if not hasattr(family, 'description'):
+                family.description = family.name
+            family.doc = family.description
+            del family.description
+
+    def change_modes(self):
+        """change the mode of variables
+        """
+        modes_level = self.objectspace.rougailconfig['modes_level']
+        default_variable_mode = self.objectspace.rougailconfig['default_variable_mode']
+        if default_variable_mode not in modes_level:
+            msg = _(f'default variable mode "{default_variable_mode}" is not a valid mode, '
+                    f'valid modes are {modes_level}')
+            raise DictConsistencyError(msg, 72, None)
+        default_family_mode = self.objectspace.rougailconfig['default_family_mode']
+        if default_family_mode not in modes_level:
+            msg = _(f'default family mode "{default_family_mode}" is not a valid mode, '
+                    f'valid modes are {modes_level}')
+            raise DictConsistencyError(msg, 73, None)
+        families = list(self.get_families())
+        for family in families:
+            self.valid_mode(family)
+            self._set_default_mode(family)
+        families.reverse()
+        for family in families:
+            self._change_family_mode(family)
+
+    def valid_mode(self,
+                   obj,
+                   ) -> None:
+        modes_level = self.objectspace.rougailconfig['modes_level']
+        if self._has_mode(obj) and obj.mode not in modes_level:
+            msg = _(f'mode "{obj.mode}" for "{obj.name}" is not a valid mode, '
+                    f'valid modes are {modes_level}')
+            raise DictConsistencyError(msg, 71, obj.xmlfiles)
+
+    def _set_default_mode(self,
+                         family: 'self.objectspace.family',
+                         ) -> None:
+        if not hasattr(family, 'variable'):
+            return
+        if self._has_mode(family):
+            family_mode = family.mode
+        else:
+            family_mode = None
+        leader = None
+        for variable in family.variable.values():
+            if leader is None and hasattr(family, 'leadership') and family.leadership:
+                leader = variable
+            if isinstance(variable, self.objectspace.family):
+                # set default mode a subfamily
+                if family_mode and not self._has_mode(variable):
+                    self._set_auto_mode(variable, family_mode)
+            else:
+                # set default mode to a variable
+                self.valid_mode(variable)
+                if leader:
+                    self._set_default_mode_leader(leader, variable)
+                self._set_default_mode_variable(variable, family_mode)
+        if leader:
+            # here because follower can change leader mode
+            self._set_auto_mode(family, leader.mode)
+
+    @staticmethod
+    def _has_mode(obj) -> bool:
+        return 'mode' in vars(obj) and not hasattr(obj, 'mode_auto')
+
+    def _set_default_mode_variable(self,
+                                   variable: 'self.objectspace.variable',
+                                   family_mode: str,
+                                   ) -> None:
+        # auto_save or auto_freeze variable is set to 'basic' mode
+        # if its mode is not defined by the user
+        if not self._has_mode(variable) and \
+                (variable.auto_save is True or variable.auto_freeze is True):
+            variable.mode = self.objectspace.rougailconfig['modes_level'][0]
+        # mandatory variable without value is a basic variable
+        elif not self._has_mode(variable) and \
+                variable.mandatory is True and \
+                not hasattr(variable, 'default') and \
+                not hasattr(variable, 'default_multi'):
+            variable.mode = self.objectspace.rougailconfig['modes_level'][0]
+        elif family_mode and not self._has_mode(variable):
+            self._set_auto_mode(variable, family_mode)
+
+    @staticmethod
+    def _set_auto_mode(obj, mode: str) -> None:
+        obj.mode = mode
+        obj.mode_auto = True
+
+    def _set_default_mode_leader(self,
+                                 leader: 'self.objectspace.variable',
+                                 follower: 'self.objectspace.variable',
+                                 ) -> None:
+        if follower.auto_save is True:
+            msg = _(f'leader/followers "{follower.name}" could not be auto_save')
+            raise DictConsistencyError(msg, 29, follower.xmlfiles)
+        if follower.auto_freeze is True:
+            msg = f'leader/followers "{follower.name}" could not be auto_freeze'
+            raise DictConsistencyError(_(msg), 30, follower.xmlfiles)
+        if leader == follower:
+            # it's a leader
+            if not hasattr(leader, 'mode'):
+                self._set_auto_mode(leader, self.objectspace.rougailconfig['default_variable_mode'])
+            return
+        if self._has_mode(follower):
+            follower_mode = follower.mode
+        else:
+            follower_mode = self.objectspace.rougailconfig['default_variable_mode']
+        if self.modes[leader.mode] > self.modes[follower_mode]:
+            if self._has_mode(follower) and not self._has_mode(leader):
+                # if follower has mode but not the leader
+                self._set_auto_mode(leader, follower_mode)
+            else:
+                # leader's mode is minimum level
+                if self._has_mode(follower):
+                    msg = _(f'the follower "{follower.name}" is in "{follower_mode}" mode '
+                            f'but leader have the higher mode "{leader.mode}"')
+                    raise DictConsistencyError(msg, 63, follower.xmlfiles)
+                self._set_auto_mode(follower, leader.mode)
+
+    def _change_family_mode(self,
+                            family: 'self.objectspace.family',
+                            ) -> None:
+        if hasattr(family, 'mode'):
+            family_mode = family.mode
+        else:
+            family_mode = self.objectspace.rougailconfig['default_family_mode']
+        min_variable_mode = self.objectspace.rougailconfig['modes_level'][-1]
+        # change variable mode, but not if variables are not in a family
+        is_leadership = hasattr(family, 'leadership') and family.leadership
+        if hasattr(family, 'variable'):
+            for idx, variable in enumerate(family.variable.values()):
+                if isinstance(variable, self.objectspace.family):
+                    if not hasattr(variable, 'mode'):
+                        variable.mode = self.objectspace.rougailconfig['default_family_mode']
+                elif idx == 0 and is_leadership:
+                    variable.mode = None
+                    continue
+                else:
+                    self._change_variable_mode(variable, family_mode, is_leadership)
+                if self.modes[min_variable_mode] > self.modes[variable.mode]:
+                    min_variable_mode = variable.mode
+        if not isinstance(family, self.objectspace.family) or is_leadership:
+            # it's Variable, Service, ... and leadership
+            return
+        if not hasattr(family, 'mode'):
+            # set the lower variable mode to family
+            self._set_auto_mode(family, min_variable_mode)
+        if family.mode != min_variable_mode:
+            msg = _(f'the family "{family.name}" is in "{family.mode}" mode but variables and '
+                    f'families inside have the higher modes "{min_variable_mode}"')
+            raise DictConsistencyError(msg, 62, family.xmlfiles)
+
+    def _change_variable_mode(self,
+                             variable,
+                             family_mode: str,
+                             is_follower: bool,
+                             ) -> None:
+        if hasattr(variable, 'mode'):
+            variable_mode = variable.mode
+        else:
+            variable_mode = self.objectspace.rougailconfig['default_variable_mode']
+        # none basic variable in high level family has to be in high level
+        if not is_follower and self.modes[variable_mode] < self.modes[family_mode]:
+            if self._has_mode(variable):
+                msg = _(f'the variable "{variable.name}" is in "{variable_mode}" mode '
+                        f'but family has the higher family mode "{family_mode}"')
+                raise DictConsistencyError(msg, 61, variable.xmlfiles)
+            self._set_auto_mode(variable, family_mode)
+        if not hasattr(variable, 'mode'):
+            variable.mode = variable_mode
+
+    def dynamic_families(self):
+        """link dynamic families to object
+        """
+        for family in self.get_families():
+            if 'dynamic' not in vars(family):
+                continue
+            family.suffixes = self.objectspace.paths.get_variable(family.dynamic, family.xmlfiles)
+            del family.dynamic
+            if not family.suffixes.multi:
+                msg = _(f'dynamic family "{family.name}" must be linked '
+                        f'to multi variable')
+                raise DictConsistencyError(msg, 16, family.xmlfiles)
+            for variable in family.variable.values():
+                if isinstance(variable, self.objectspace.family) and not variable.leadership:
+                    msg = _(f'dynamic family "{family.name}" cannot contains another family')
+                    raise DictConsistencyError(msg, 22, family.xmlfiles)
+
+    def convert_help(self):
+        """Convert variable help
+        """
+        for family in self.get_families():
+            if not hasattr(family, 'help'):
+                continue
+            if not hasattr(family, 'information'):
+                family.information = self.objectspace.information(family.xmlfiles)
+            family.information.help = family.help
+            del family.help

src/rougail/annotator/fill.py

@@ -0,0 +1,95 @@
+"""Fill annotator
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from copy import copy
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+
+from rougail.annotator.target import TargetAnnotator
+from rougail.annotator.param import ParamAnnotator
+
+
+CALC_MULTI = ('calc_value', 'calc_list', 'get_range', 'calc_val_first_value', 'unbound_filename', 'zone_information', 'get_certificates', 'nsd_filename', 'get_linked_configuration', 'get_internal_zones', 'nsd_concat_lists', 'get_internal_info_in_zone')
+
+
+class Annotator(TargetAnnotator, ParamAnnotator):
+    """Fill annotator
+    """
+    level = 60
+    def __init__(self,
+                 objectspace,
+                 functions,
+                 *args,
+                 ):
+        if not hasattr(objectspace.space, 'constraints') or \
+                not hasattr(objectspace.space.constraints, 'fill'):
+            return
+        self.objectspace = objectspace
+        self.functions = copy(functions)
+        self.functions.extend(self.objectspace.rougailconfig['internal_functions'])
+        self.calc_multi = list(CALC_MULTI)
+        self.calc_multi.extend(self.objectspace.rougailconfig['multi_functions'])
+        self.target_is_uniq = True
+        self.only_variable = True
+        self.allow_function = False
+        self.convert_target(self.objectspace.space.constraints.fill)
+        self.convert_param(self.objectspace.space.constraints.fill)
+        self.fill_to_value()
+        del self.objectspace.space.constraints.fill
+
+    def calc_is_multi(self, variable: 'self.objectspace.variable') -> bool:
+        multi = variable.multi
+        if multi is False:
+            return multi
+        if multi == 'submulti':
+            return True
+        return not self.objectspace.paths.is_follower(variable.path)
+
+    def fill_to_value(self) -> None:
+        """valid and manage <fill>
+        """
+        for fill in self.objectspace.space.constraints.fill:
+            # test if the function exists
+            if fill.name not in self.functions:
+                msg = _(f'cannot find fill function "{fill.name}"')
+                raise DictConsistencyError(msg, 25, fill.xmlfiles)
+            for target in fill.target:
+                # create an object value
+                value = self.objectspace.value(fill.xmlfiles)
+                value.type = 'calculation'
+                value.name = fill.name
+                if fill.name not in self.calc_multi:
+                    is_calc_multi = self.calc_is_multi(target.name)
+                else:
+                    is_calc_multi = False
+                value.calc_multi = is_calc_multi
+                if target.name.namespace == 'services':
+                    target.name.default = value
+                else:
+                    target.name.value = [value]
+                # manage params
+                if hasattr(fill, 'param') and fill.param:
+                    value.param = fill.param

src/rougail/annotator/group.py

@@ -0,0 +1,77 @@
+"""Annotate group
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+from rougail.utils import normalize_family
+from rougail.annotator.variable import Walk
+
+
+class Annotator(Walk):
+    """Annotate group
+    """
+    level = 10
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ):
+        if not hasattr(objectspace.space, 'variables'):
+            return
+        self.objectspace = objectspace
+        self.convert_groups()
+
+    def convert_groups(self):  # pylint: disable=C0111
+        """convert groups
+        """
+        # store old leaders family name
+        for family in self.get_families():
+            if not isinstance(family, self.objectspace.family):
+                continue
+            if not family.leadership:
+                continue
+            if hasattr(family, 'dynamic'):
+                msg = _(f'the family "{family.name}" cannot be leadership and dynamic together')
+                raise DictConsistencyError(msg, 31, family.xmlfiles)
+            for idx, variable in enumerate(family.variable.values()):
+                if idx == 0:
+                    # it's a leader
+                    if variable.multi is not True:
+                        msg = _(f'the variable "{variable.name}" in a leadership must be multi')
+                        raise DictConsistencyError(msg, 32, variable.xmlfiles)
+                    family.hidden = variable.hidden
+                    if variable.hidden:
+                        variable.frozen = True
+                        variable.force_default_on_freeze = True
+                    variable.hidden = None
+                else:
+                    # it's a follower
+                    if family.hidden:
+                        variable.frozen = True
+                        variable.force_default_on_freeze = True
+                    if variable.multi is True:
+                        variable.multi = 'submulti'
+                    else:
+                        variable.multi = True

src/rougail/annotator/param.py

@@ -0,0 +1,154 @@
+"""Param annotator
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+try:
+    import tiramisu3 as tiramisu
+except ModuleNotFoundError:
+    import tiramisu
+
+from rougail.annotator.variable import CONVERT_OPTION
+
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+
+
+class ParamAnnotator:
+    """Param annotator
+    """
+    objectspace = None
+
+    @staticmethod
+    def valid_type_validation(obj) -> None:
+        """Function to valid type (redefine in fill/condition/check classes)
+        """
+        return None
+
+    def convert_param(self, objects) -> None:
+        """ valid and convert param
+        """
+        for obj in objects:
+            if not hasattr(obj, 'param'):
+                continue
+            param_to_delete = []
+            variable_type = self.valid_type_validation(obj)
+            for param_idx, param in enumerate(obj.param):
+                if hasattr(param, 'text'):
+                    if param.type in ['suffix', 'index']:
+                        msg = _(f'"{param.type}" parameter must not have a value')
+                        raise DictConsistencyError(msg, 28, obj.xmlfiles)
+                    if param.type in ['nil', 'space']:
+                        if param.text is not None:
+                            msg = _(f'"{param.type}" parameter must not have a value')
+                            raise DictConsistencyError(msg, 40, obj.xmlfiles)
+                    elif param.type == 'variable':
+                        try:
+                            path, suffix = self.objectspace.paths.get_variable_path(param.text,
+                                                                                    obj.namespace,
+                                                                                    param.xmlfiles,
+                                                                                    )
+                            param.text = self.objectspace.paths.get_variable(path)
+                            if variable_type and param.text.type != variable_type:
+                                msg = _(f'"{obj.name}" has type "{variable_type}" but param '
+                                        f'has type "{param.text.type}"')
+                                raise DictConsistencyError(msg, 26, param.xmlfiles)
+                            if suffix:
+                                param.suffix = suffix
+                                family_path = self.objectspace.paths.get_variable_family_path(path)
+                                namespace = param.text.namespace
+                                param.family = self.objectspace.paths.get_family(family_path,
+                                                                                 namespace,
+                                                                                 )
+                        except DictConsistencyError as err:
+                            if err.errno != 42 or not param.optional:
+                                raise err
+                            param_to_delete.append(param_idx)
+                    elif param.type == 'function':
+                        if not self.allow_function:
+                            msg = _(f'cannot use "function" type')
+                            raise DictConsistencyError(msg, 74, param.xmlfiles)
+                        if not param.text in self.functions:
+                            msg = _(f'cannot find function "{param.text}"')
+                            raise DictConsistencyError(msg, 67, param.xmlfiles)
+                        if param_idx != 0:
+                            msg = _(f'function "{param.text}" must only set has first parameter')
+                            raise DictConsistencyError(msg, 75, param.xmlfiles)
+                    elif variable_type:
+                        self._convert_with_variable_type(variable_type, param)
+                    continue
+                # no param.text
+                if param.type == 'suffix':
+                    for target in obj.target:
+                        if not self.objectspace.paths.variable_is_dynamic(target.name.path):
+                            target_name = target.name
+                            if isinstance(target_name, self.objectspace.variable):
+                                target_name = target_name.name
+                            msg = _(f'"{param.type}" parameter cannot be set with target '
+                                    f'"{target_name}" which is not a dynamic variable')
+                            raise DictConsistencyError(msg, 53, obj.xmlfiles)
+                elif param.type == 'index':
+                    for target in obj.target:
+                        if not self.objectspace.paths.is_follower(target.name.path):
+                            msg = _(f'"{param.type}" parameter cannot be set with target '
+                                    f'"{target.name.name}" which is not a follower variable')
+                            raise DictConsistencyError(msg, 60, obj.xmlfiles)
+                elif param.type == 'nil':
+                    param.text = None
+                elif param.type == 'space':
+                    param.type = 'string'
+                    param.text = ' '
+                elif param.type == 'string':
+                    param.text = ''
+                    if variable_type:
+                        self._convert_with_variable_type(variable_type, param)
+                else:
+                    msg = _(f'"{param.type}" parameter must have a value')
+                    raise DictConsistencyError(msg, 27, obj.xmlfiles)
+            param_to_delete.sort(reverse=True)
+            for param_idx in param_to_delete:
+                obj.param.pop(param_idx)
+
+    @staticmethod
+    def _convert_with_variable_type(variable_type: str,
+                                    param: 'self.objectspace.param',
+                                    ) -> None:
+        if 'type' in vars(param) and variable_type != param.type:
+            msg = _(f'parameter has incompatible type "{param.type}" '
+                    f'with type "{variable_type}"')
+            raise DictConsistencyError(msg, 7, param.xmlfiles)
+        if variable_type != 'choice':
+            try:
+                option = CONVERT_OPTION[variable_type]
+                param.text = option.get('func', str)(param.text)
+                getattr(tiramisu, option['opttype'])('test',
+                                                     'Object to valid value',
+                                                     param.text,
+                                                     **option.get('initkwargs', {}),
+                                                     )
+            except ValueError as err:
+                msg = _(f'unable to change type of value "{param.text}" '
+                        f'is not a valid "{variable_type}"')
+                raise DictConsistencyError(msg, 13, param.xmlfiles) from err
+        param.type = variable_type

src/rougail/annotator/property.py

@@ -0,0 +1,112 @@
+"""Annotate properties
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+from rougail.annotator.variable import Walk
+
+
+PROPERTIES = ('hidden', 'frozen', 'force_default_on_freeze',
+              'force_store_value', 'disabled', 'mandatory')
+
+
+class Annotator(Walk):
+    """Annotate properties
+    """
+    level = 90
+    def __init__(self,
+                 objectspace,
+                 *args
+                 ) -> None:
+        self.objectspace = objectspace
+        if hasattr(self.objectspace.space, 'services'):
+            self.convert_services()
+        if hasattr(self.objectspace.space, 'variables'):
+            self.convert_family()
+            self.convert_variable()
+
+    def convert_property(self,
+                         variable,
+                         ) -> None:
+        """convert properties
+        """
+        # hidden variable is also frozen
+        if isinstance(variable, self.objectspace.variable) and variable.hidden is True and \
+                variable.name != self.objectspace.rougailconfig['auto_freeze_variable']:
+            if not variable.auto_freeze and \
+                    not hasattr(variable, 'provider'):
+                variable.frozen = True
+            if not variable.auto_save and \
+                    not variable.auto_freeze and \
+                    'force_default_on_freeze' not in vars(variable) and \
+                    not hasattr(variable, 'provider'):
+                variable.force_default_on_freeze = True
+        if not hasattr(variable, 'properties'):
+            variable.properties = []
+        for prop in PROPERTIES:
+            if hasattr(variable, prop):
+                if getattr(variable, prop) is True:
+#                    for subprop in CONVERT_PROPERTIES.get(prop, [prop]):
+                    variable.properties.append(prop)
+                setattr(variable, prop, None)
+        if hasattr(variable, 'mode') and variable.mode:
+            variable.properties.append(variable.mode)
+            variable.mode = None
+        if 'force_store_value' in variable.properties and \
+                'force_default_on_freeze' in variable.properties:  # pragma: no cover
+            # should not appened
+            msg = _('cannot have auto_freeze or auto_save with the hidden '
+                    f'variable "{variable.name}"')
+            raise DictConsistencyError(msg, 50, variable.xmlfiles)
+        if not variable.properties:
+            del variable.properties
+
+    def convert_services(self) -> None:
+        """convert services
+        """
+        self.convert_property(self.objectspace.space.services)
+        for services in self.objectspace.space.services.service.values():
+            self.convert_property(services)
+            for service in vars(services).values():
+                if not isinstance(service, self.objectspace.family):
+                    continue
+                self.convert_property(service)
+                for family in service.family:
+                    self.convert_property(family)
+                    for variable in family.variable:
+                        self.convert_property(variable)
+
+    def convert_family(self) -> None:
+        """convert families
+        """
+        for family in self.get_families():
+            self.convert_property(family)
+
+    def convert_variable(self) -> None:
+        """convert variables
+        """
+        for variable in self.get_variables():
+            self.convert_property(variable)

src/rougail/annotator/service.py

@@ -0,0 +1,320 @@
+"""Annotate services
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from os.path import basename
+from typing import Tuple
+
+from rougail.i18n import _
+from rougail.utils import normalize_family
+from rougail.error import DictConsistencyError
+# a object's attribute has some annotations
+# that shall not be present in the exported (flatened) XML
+ERASED_ATTRIBUTES = ('redefine', 'namespace', 'xmlfiles', 'disabled', 'name', 'manage')
+ERASED_ATTRIBUTES2 = ('redefine', 'namespace', 'xmlfiles', 'disabled')
+ALLOW_ATTRIBUT_NOT_MANAGE = ['file', 'engine', 'target']
+
+
+class Annotator:
+    """Manage service's object
+    for example::
+        <services>
+            <service name="test">
+                <service_access service='ntp'>
+                </service_access>
+            </service>
+        </services>
+    """
+    level = 20
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ) -> None:
+        self.objectspace = objectspace
+        self.uniq_overrides = []
+        if 'network_type' not in self.objectspace.types:
+            self.objectspace.types['network_type'] = self.objectspace.types['ip_type']
+        if hasattr(self.objectspace.space, 'services'):
+            if not hasattr(self.objectspace.space.services, 'service'):
+                del self.objectspace.space.services
+            else:
+                self.convert_services()
+
+    def convert_services(self):
+        """convert services to variables
+        """
+        self.objectspace.space.services.hidden = True
+        self.objectspace.space.services.name = 'services'
+        self.objectspace.space.services.doc = 'services'
+        self.objectspace.space.services.path = 'services'
+        for service_name, service in self.objectspace.space.services.service.items():
+            service.name = normalize_family(service_name)
+            activate_obj = self._generate_element('boolean',
+                                                  None,
+                                                  None,
+                                                  'activate',
+                                                  not service.disabled,
+                                                  service,
+                                                  '.'.join(['services', service.name, 'activate']),
+                                                  )
+            service.disabled = None
+            for elttype, values in dict(vars(service)).items():
+                if elttype == 'servicelist':
+                    self.objectspace.list_conditions.setdefault('servicelist',
+                                                                {}).setdefault(
+                                                                    values,
+                                                                    []).append(activate_obj)
+                    continue
+                if elttype in ERASED_ATTRIBUTES:
+                    continue
+                if not service.manage and elttype not in ALLOW_ATTRIBUT_NOT_MANAGE:
+                    msg = _(f'unmanage service cannot have "{elttype}"')
+                    raise DictConsistencyError(msg, 66, service.xmlfiles)
+                if isinstance(values, (dict, list)):
+                    if elttype != 'ip':
+                        eltname = elttype + 's'
+                    else:
+                        eltname = elttype
+                    path = '.'.join(['services', service.name, eltname])
+                    family = self._gen_family(eltname,
+                                              path,
+                                              service.xmlfiles,
+                                              with_informations=False,
+                                              )
+                    if isinstance(values, dict):
+                        values = list(values.values())
+                    family.family = self.make_group_from_elts(service_name,
+                                                              elttype,
+                                                              values,
+                                                              path,
+                                                              )
+                    setattr(service, elttype, family)
+                else:
+                    if not hasattr(service, 'information'):
+                        service.information = self.objectspace.information(service.xmlfiles)
+                    setattr(service.information, elttype, values)
+            service.path = '.'.join(['services', service.name])
+            manage = self._generate_element('boolean',
+                                            None,
+                                            None,
+                                            'manage',
+                                            service.manage,
+                                            service,
+                                            '.'.join(['services', service.name, 'manage']),
+                                                  )
+            service.variable = [activate_obj, manage]
+            service.doc = service_name
+
+    def make_group_from_elts(self,
+                             service_name,
+                             elttype,
+                             elts,
+                             path,
+                             ):
+        """Splits each objects into a group (and `OptionDescription`, in tiramisu terms)
+        and build elements and its attributes (the `Options` in tiramisu terms)
+        """
+        families = []
+        listname = '{}list'.format(elttype)
+        for elt in elts:
+            # try to launch _update_xxxx() function
+            update_elt = '_update_' + elttype
+            if hasattr(self, update_elt):
+                getattr(self, update_elt)(elt,
+                                          service_name,
+                                          )
+            c_name, subpath = self._get_name_path(elt,
+                                                  path,
+                                                  )
+            family = self._gen_family(c_name,
+                                      subpath,
+                                      elt.xmlfiles,
+                                      )
+            family.variable = []
+            if hasattr(elt, 'disabled'):
+                disabled = elt.disabled
+            else:
+                disabled = False
+            activate_obj = self._generate_element('boolean',
+                                                  None,
+                                                  None,
+                                                  'activate',
+                                                  not disabled,
+                                                  elt,
+                                                  '.'.join([subpath, 'activate']),
+                                                  )
+            for key in dir(elt):
+                if key.startswith('_') or key.endswith('_type') or key in ERASED_ATTRIBUTES2:
+                    continue
+                value = getattr(elt, key)
+                if key == listname:
+                    self.objectspace.list_conditions.setdefault(listname,
+                                                                {}).setdefault(
+                                                                    value,
+                                                                    []).append(activate_obj)
+                    continue
+                if key == 'name':
+                    dtd_key_type = elttype + '_type'
+                else:
+                    dtd_key_type = key + '_type'
+                elt_type = getattr(elt, dtd_key_type, None)
+                if elt_type:
+                    if elt_type == 'variable':
+                        elt_type = 'symlink'
+                    family.variable.append(self._generate_element(elt_type,
+                                                                  dtd_key_type,
+                                                                  elttype,
+                                                                  key,
+                                                                  value,
+                                                                  elt,
+                                                                  f'{subpath}.{key}'
+                                                                  ))
+                else:
+                    setattr(family.information, key, value)
+
+            family.variable.append(activate_obj)
+            families.append(family)
+        return families
+
+    def _get_name_path(self,
+                       elt,
+                       path: str,
+                       ) -> Tuple[str, str]:
+        # create element name, if already exists, add _xx to be uniq
+        if hasattr(elt, 'source') and elt.source:
+            name = elt.source
+        else:
+            name = elt.name
+        idx = 0
+        while True:
+            c_name = name
+            if idx:
+                c_name += f'_{idx}'
+            subpath = '{}.{}'.format(path, normalize_family(c_name))
+            try:
+                self.objectspace.paths.get_family(subpath, 'services')
+            except DictConsistencyError as err:
+                if err.errno == 42:
+                    return c_name, subpath
+            idx += 1
+
+    def _gen_family(self,
+                    name,
+                    path,
+                    xmlfiles,
+                    with_informations=True,
+                    ):
+        family = self.objectspace.family(xmlfiles)
+        family.name = normalize_family(name)
+        family.doc = name
+        family.mode = None
+        self.objectspace.paths.add_family('services',
+                                          path,
+                                          family,
+                                          None,
+                                          )
+        if with_informations:
+            family.information = self.objectspace.information(xmlfiles)
+        return family
+
+    def _generate_element(self,
+                          type_,
+                          dtd_key_type,
+                          elttype,
+                          key,
+                          value,
+                          elt,
+                          path,
+                          ):  # pylint: disable=R0913
+        variable = self.objectspace.variable(elt.xmlfiles)
+        variable.name = normalize_family(key)
+        variable.mode = None
+        variable.type = type_
+        if type_ == 'symlink':
+            variable.opt = self.objectspace.paths.get_variable(value, xmlfiles=elt.xmlfiles)
+            variable.multi = None
+            needed_type = self.objectspace.types[dtd_key_type]
+            if needed_type not in ('variable', variable.opt.type):
+                msg = _(f'"{value}" in "{elttype}" must be a variable with type '
+                        f'"{needed_type}" not "{variable.opt.type}"')
+                raise DictConsistencyError(msg, 58, elt.xmlfiles)
+
+        else:
+            variable.doc = key
+            variable.default = value
+            variable.namespace = 'services'
+        self.objectspace.paths.add_variable('services',
+                                            path,
+                                            'service',
+                                            False,
+                                            variable,
+                                            )
+        return variable
+
+    def _update_override(self,
+                         override,
+                         service_name,
+                         ):
+
+        if service_name in self.uniq_overrides:
+            msg = _('only one override is allowed by service, '
+                    'please use a variable multiple if you want have more than one IP')
+            raise DictConsistencyError(msg, 69, override.xmlfiles)
+        self.uniq_overrides.append(service_name)
+        override.name = service_name
+        if not hasattr(override, 'source'):
+            override.source = service_name
+
+    @staticmethod
+    def _update_file(file_,
+                     service_name,
+                     ):
+        if not hasattr(file_, 'file_type') or file_.file_type == "filename":
+            if not hasattr(file_, 'source'):
+                file_.source = basename(file_.name)
+        elif not hasattr(file_, 'source'):
+            msg = _(f'attribute "source" is mandatory for the file "{file_.name}" '
+                    f'"({service_name})"')
+            raise DictConsistencyError(msg, 34, file_.xmlfiles)
+
+    def _update_ip(self,
+                   ip,
+                   service_name,
+                   ) -> None:
+        variable = self.objectspace.paths.get_variable(ip.name, ip.xmlfiles)
+        if variable.type not in ['ip', 'network', 'network_cidr']:
+            msg = _(f'ip cannot be linked to "{variable.type}" variable "{ip.name}"')
+            raise DictConsistencyError(msg, 70, ip.xmlfiles)
+        if variable.type in ['ip', 'network_cidr'] and hasattr(ip, 'netmask'):
+            msg = _(f'ip with ip_type "{variable.type}" must not have netmask')
+            raise DictConsistencyError(msg, 59, ip.xmlfiles)
+        if variable.type == 'network' and not hasattr(ip, 'netmask'):
+            msg = _(f'ip with ip_type "{variable.type}" must have netmask')
+            raise DictConsistencyError(msg, 64, ip.xmlfiles)
+        if hasattr(ip, 'netmask'):
+            netmask = self.objectspace.paths.get_variable(ip.netmask, ip.xmlfiles)
+            if netmask.type != 'netmask':
+                msg = _(f'netmask in ip must have type "netmask", not "{netmask.type}"')
+                raise DictConsistencyError(msg, 65, ip.xmlfiles)

src/rougail/annotator/target.py

@@ -0,0 +1,91 @@
+"""Target annotator
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+
+
+class TargetAnnotator:
+    """Target annotator
+    """
+    def convert_target(self,
+                       objects,
+                       ) -> None:
+        """ valid and convert param
+        """
+        targets = []
+        for obj in objects:
+            if not hasattr(obj, 'target'):
+                msg = _('target is mandatory')
+                raise DictConsistencyError(msg, 9, obj.xmlfiles)
+            remove_targets = []
+            for index, target in enumerate(obj.target):
+                # test if it's redefined calculation
+                if self.target_is_uniq and target.name in targets:
+                    msg = _(f'A fill already exists for the target of "{target.name}" created')
+                    raise DictConsistencyError(msg, 24, obj.xmlfiles)
+                targets.append(target.name)
+
+                # let's replace the target by the path
+                try:
+                    if target.type == 'variable':
+                        path, suffix = self.objectspace.paths.get_variable_path(target.name,
+                                                                                obj.namespace,
+                                                                                )
+                        target.name = self.objectspace.paths.get_variable(path)
+                        if suffix:
+                            msg = _(f'target to {target.name.path} with suffix is not allowed')
+                            raise DictConsistencyError(msg, 35, obj.xmlfiles)
+                    elif self.only_variable:
+                        msg = _(f'target to "{target.name}" with param type "{target.type}" '
+                                f'is not allowed')
+                        raise DictConsistencyError(msg, 8, obj.xmlfiles)
+                    if target.type == 'family':
+                        if obj.name not in ['disabled_if_in',
+                                            'disabled_if_not_in',
+                                            'hidden_if_in',
+                                            'hidden_if_not_in',
+                                            ]:
+                            msg = _(f'target "{target.type}" not allow with "{obj.name}"')
+                            raise DictConsistencyError(msg, 51, target.xmlfiles)
+                        target.name = self.objectspace.paths.get_family(target.name,
+                                                                        obj.namespace,
+                                                                        )
+                    elif target.type.endswith('list') and \
+                            obj.name not in ['disabled_if_in', 'disabled_if_not_in']:
+                        msg = _(f'target "{target.type}" not allow with "{obj.name}"')
+                        raise DictConsistencyError(msg, 10, target.xmlfiles)
+                except DictConsistencyError as err:
+                    if err.errno != 42:
+                        raise err
+                    # for optional variable
+                    if not target.optional:
+                        msg = f'cannot found target "{target.type}" "{target.name}"'
+                        raise DictConsistencyError(_(msg), 12, target.xmlfiles) from err
+                    remove_targets.append(index)
+            remove_targets.sort(reverse=True)
+            for index in remove_targets:
+                obj.target.pop(index)

src/rougail/annotator/value.py

@@ -0,0 +1,105 @@
+"""Annotate value
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from rougail.annotator.variable import Walk
+
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+
+class Annotator(Walk):  # pylint: disable=R0903
+    """Annotate value
+    """
+    level = 70
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ) -> None:
+        if not hasattr(objectspace.space, 'variables'):
+            return
+        self.objectspace = objectspace
+        self.convert_value()
+        self.add_choice_nil()
+
+    def convert_value(self) -> None:
+        """convert value
+        """
+        for variable in self.get_variables():
+            self._convert_value(variable)
+
+    def _convert_value(self,
+                       variable,
+                       ) -> None:
+        # a boolean must have value, the default value is "True"
+        if not hasattr(variable, 'value') and variable.type == 'boolean':
+            new_value = self.objectspace.value(variable.xmlfiles)
+            new_value.name = True
+            new_value.type = 'boolean'
+            variable.value = [new_value]
+        # if the variable is mandatory and doesn't have any value
+        # then the variable's mode is set to 'basic'
+        # variable with default value is mandatory
+        if hasattr(variable, 'value') and variable.value:
+            has_value = True
+            for value in variable.value:
+                if value.type == 'calculation':
+                    has_value = False
+                    break
+            if has_value and 'mandatory' not in vars(variable):
+                # if has value without any calculation
+                variable.mandatory = True
+        if not hasattr(variable, 'value'):
+            return
+        if variable.value[0].type == 'calculation':
+            variable.default = variable.value[0]
+        elif variable.multi:
+            if not self.objectspace.paths.is_follower(variable.path):
+                variable.default = [value.name for value in variable.value]
+            if not self.objectspace.paths.is_leader(variable.path):
+                if variable.multi == 'submulti':
+                    variable.default_multi = [value.name for value in variable.value]
+                else:
+                    variable.default_multi = variable.value[0].name
+        else:
+            if len(variable.value) > 1:
+                msg = _(f'the none multi variable "{variable.name}" cannot have '
+                        'more than one value')
+                raise DictConsistencyError(msg, 68, variable.xmlfiles)
+            variable.default = variable.value[0].name
+        del variable.value
+
+    def add_choice_nil(self) -> None:
+        for variable in self.get_variables():
+            if variable.type != 'choice':
+                continue
+            is_none = False
+            for choice in variable.choice:
+                if choice.type == 'nil':
+                    is_none = True
+            if not variable.mandatory and not is_none:
+                choice = self.objectspace.choice(variable.xmlfiles)
+                choice.name = None
+                choice.type = 'nil'
+                variable.choice.append(choice)

src/rougail/annotator/variable.py

@@ -0,0 +1,226 @@
+"""Annotate variable
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from rougail.i18n import _
+from rougail.error import DictConsistencyError
+from rougail.objspace import convert_boolean
+
+
+CONVERT_OPTION = {'number': dict(opttype="IntOption", func=int),
+                  'float': dict(opttype="FloatOption", func=float),
+                  'choice': dict(opttype="ChoiceOption"),
+                  'string': dict(opttype="StrOption"),
+                  'password': dict(opttype="PasswordOption"),
+                  'secret': dict(opttype="PasswordOption"),
+                  'mail': dict(opttype="EmailOption"),
+                  'boolean': dict(opttype="BoolOption", func=convert_boolean),
+                  'symlink': dict(opttype="SymLinkOption"),
+                  'filename': dict(opttype="FilenameOption"),
+                  'date': dict(opttype="DateOption"),
+                  'unix_user': dict(opttype="UsernameOption"),
+                  'ip': dict(opttype="IPOption", initkwargs={'allow_reserved': True}),
+                  'local_ip': dict(opttype="IPOption", initkwargs={'private_only': True,
+                                   'warnings_only': True}),
+                  'netmask': dict(opttype="NetmaskOption"),
+                  'network': dict(opttype="NetworkOption"),
+                  'broadcast': dict(opttype="BroadcastOption"),
+                  'netbios': dict(opttype="DomainnameOption", initkwargs={'type': 'netbios',
+                                  'warnings_only': True}),
+                  'domainname': dict(opttype="DomainnameOption", initkwargs={'type': 'domainname',
+                                 'allow_ip': False}),
+                  'hostname': dict(opttype="DomainnameOption", initkwargs={'type': 'hostname',
+                                   'allow_ip': False}),
+                  'web_address': dict(opttype="URLOption", initkwargs={'allow_ip': True,
+                                      'allow_without_dot': True}),
+                  'port': dict(opttype="PortOption", initkwargs={'allow_private': True}),
+                  'mac': dict(opttype="MACOption"),
+                  'cidr': dict(opttype="IPOption", initkwargs={'cidr': True}),
+                  'network_cidr': dict(opttype="NetworkOption", initkwargs={'cidr': True}),
+                  }
+
+
+class Walk:
+    """Walk to objectspace to find variable or family
+    """
+    objectspace = None
+
+    def get_variables(self):
+        """Iter all variables from the objectspace
+        """
+        for family in self.objectspace.space.variables.values():
+            yield from self._get_variables(family)
+
+    def _get_variables(self,
+                       family: 'self.objectspace.family',
+                       ):
+        if not hasattr(family, 'variable'):
+            return
+        for variable in family.variable.values():
+            if isinstance(variable, self.objectspace.family):
+                yield from self._get_variables(variable)
+            else:
+                yield variable
+
+    def get_families(self):
+        """Iter all families from the objectspace
+        """
+        for family in self.objectspace.space.variables.values():
+            yield from self._get_families(family)
+
+    def _get_families(self,
+                      family: 'self.objectspace.family',
+                      ):
+        yield family
+        if not hasattr(family, 'variable'):
+            return
+        for fam in family.variable.values():
+            if isinstance(fam, self.objectspace.family):
+                yield from self._get_families(fam)
+
+
+class Annotator(Walk):  # pylint: disable=R0903
+    """Annotate variable
+    """
+    level = 30
+    def __init__(self,
+                 objectspace,
+                 *args,
+                 ):
+        if not hasattr(objectspace.space, 'variables'):
+            return
+        self.objectspace = objectspace
+        self.forbidden_name = ['services', self.objectspace.rougailconfig['variable_namespace']]
+        for extra in self.objectspace.rougailconfig['extra_dictionaries']:
+            self.forbidden_name.append(extra)
+        self.convert_variable()
+        self.convert_test()
+        self.convert_help()
+
+    def convert_variable(self):
+        """convert variable
+        """
+        for variable in self.get_variables():
+            self._convert_variable(variable)
+
+    def _convert_variable(self,
+                          variable,
+                          ) -> None:
+        if variable.namespace == self.objectspace.rougailconfig['variable_namespace'] and \
+                variable.name in self.forbidden_name:
+            msg = _(f'the name of the variable "{variable.name}" cannot be the same as the name'
+                    ' of a namespace')
+            raise DictConsistencyError(msg, 54, variable.xmlfiles)
+        if variable.type != 'symlink' and not hasattr(variable, 'description'):
+            variable.description = variable.name
+        if hasattr(variable, 'value'):
+            value_to_del = []
+            for idx, value in enumerate(variable.value):
+                if not hasattr(value, 'name'):
+                    value_to_del.append(idx)
+                else:
+                    if not hasattr(value, 'type'):
+                        value.type = variable.type
+                    value.name = CONVERT_OPTION.get(value.type, {}).get('func', str)(value.name)
+            value_to_del.sort(reverse=True)
+            for idx in value_to_del:
+                del variable.value[idx]
+            if not variable.value:
+                del variable.value
+        if hasattr(variable, 'choice'):
+            if variable.type != 'choice':
+                msg = _(f'choice for the variable "{variable.name}" not allowed with "{variable.type}" type')
+                raise DictConsistencyError(msg, 3, variable.xmlfiles)
+            values = []
+            choice_type = None
+            for choice in variable.choice:
+                if choice_type == 'variable':
+                    msg = _(f'only one "variable" choice is allowed '
+                            f'the variable "{variable.name}"')
+                    raise DictConsistencyError(msg, 5, choice.xmlfiles)
+                if choice.type == 'nil':
+                    choice.name = None
+                elif choice.type == 'space':
+                    choice.name = ' '
+                elif choice.type == 'variable':
+                    choice.name = self.objectspace.paths.get_variable(choice.name)
+                    if not choice.name.multi:
+                        msg = _(f'only multi "variable" is allowed for a choice '
+                                f'of variable "{variable.name}"')
+                        raise DictConsistencyError(msg, 6, choice.xmlfiles)
+                else:
+                    if not hasattr(choice, 'name'):
+                        msg = _(f'choice for variable "{variable.name}" must have a value')
+                        raise DictConsistencyError(msg, 14, choice.xmlfiles)
+                    choice.name = CONVERT_OPTION.get(choice.type, {}).get('func', str)(choice.name)
+                if choice_type is None:
+                    choice_type = choice.type
+                values.append(choice.name)
+            if choice_type not in ['function', 'variable'] and hasattr(variable, 'value'):
+                for value in variable.value:
+                    if value.name not in values:
+                        msg = _(f'value "{value.name}" of variable "{variable.name}" is not in list '
+                                f'of all expected values ({values})')
+                        raise DictConsistencyError(msg, 15, value.xmlfiles)
+            ref_choice = variable.choice[0]
+            self.objectspace.valid_enums[variable.path] = {'type': ref_choice.type,
+                                                           'values': values,
+                                                           'xmlfiles': ref_choice.xmlfiles,
+                                                           }
+        elif variable.type == 'choice':
+            msg = _(f'choice is mandatory for the variable "{variable.name}" with choice type')
+            raise DictConsistencyError(msg, 4, variable.xmlfiles)
+        variable.doc = variable.description
+        del variable.description
+
+    def convert_test(self):
+        """Convert variable tests value
+        """
+        for variable in self.get_variables():
+            if not hasattr(variable, 'test') or not variable.test:
+                # with we want remove test, we set "" has test value
+                continue
+            new_values = []
+            for value in variable.test.split('|'):
+                if value == '':
+                    value = None
+                else:
+                    value = CONVERT_OPTION.get(variable.type, {}).get('func', str)(value)
+                new_values.append(value)
+            if not hasattr(variable, 'information'):
+                variable.information = self.objectspace.information(variable.xmlfiles)
+            variable.information.test = tuple(new_values)
+
+    def convert_help(self):
+        """Convert variable help
+        """
+        for variable in self.get_variables():
+            if not hasattr(variable, 'help'):
+                continue
+            if not hasattr(variable, 'information'):
+                variable.information = self.objectspace.information(variable.xmlfiles)
+            variable.information.help = variable.help
+            del variable.help

src/rougail/config.py

@@ -1,19 +1,58 @@
-# -*- coding: utf-8 -*-
 """
-fichier de configuration pour rougail
+Config file for Rougail
 
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
-from os.path import join, isfile, abspath, dirname
-from pathlib import Path
+from os.path import join, abspath, dirname
 
-rougailroot = '/var/rougail' 
-patch_dir = join(rougailroot, 'patches')
-manifests_dir = join(rougailroot, 'manifests')
-templates_dir = join(rougailroot, 'templates')
-dtddir = join(dirname(abspath(__file__)), 'data')
-dtdfilename = join(dtddir, 'rougail.dtd')
 
-# chemin du répertoire source des fichiers templates
-patch_dir = '/srv/rougail/patch'
+ROUGAILROOT = '/srv/rougail'
+DTDDIR = join(dirname(abspath(__file__)), 'data')
 
-variable_namespace = 'rougail'
+
+RougailConfig = {'dictionaries_dir': [join(ROUGAILROOT, 'dictionaries')],
+                 'extra_dictionaries': {},
+                 'patches_dir': join(ROUGAILROOT, 'patches'),
+                 'templates_dir': join(ROUGAILROOT, 'templates'),
+                 'destinations_dir': join(ROUGAILROOT, 'destinations'),
+                 'tmp_dir': join(ROUGAILROOT, 'tmp'),
+                 'dtdfilename': join(DTDDIR, 'rougail.dtd'),
+                 'functions_file': join(ROUGAILROOT, 'functions.py'),
+                 'tmpfile_dest_dir': '/usr/local/lib',
+                 'variable_namespace': 'rougail',
+                 'auto_freeze_variable': 'server_deployed',
+                 'internal_functions': [],
+                 'multi_functions': [],
+                 'extra_annotators': [],
+                 'modes_level': ['basic', 'normal', 'expert'],
+                 'default_family_mode': 'basic',
+                 'default_variable_mode': 'normal',
+                 'default_files_engine': 'creole',
+                 'default_files_mode': '0644',
+                 'default_files_owner': 'root',
+                 'default_files_group': 'root',
+                 'default_files_included': 'no',
+                 'default_overrides_engine': 'creole',
+                 }

src/rougail/convert.py

@@ -0,0 +1,112 @@
+"""Takes a bunch of Rougail XML dispatched in differents folders
+as an input and outputs a Tiramisu's file.
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+Sample usage::
+
+    >>> from rougail import RougailConvert
+    >>> rougail = RougailConvert()
+    >>> tiramisu = rougail.save('tiramisu.py')
+
+The Rougail
+
+- loads the XML into an internal RougailObjSpace representation
+- visits/annotates the objects
+- dumps the object space as Tiramisu string
+
+The visit/annotation stage is a complex step that corresponds to the Rougail
+procedures.
+"""
+
+from typing import List
+
+from .i18n import _
+from .config import RougailConfig
+from .objspace import RougailObjSpace
+from .xmlreflector import XMLReflector
+from .tiramisureflector import TiramisuReflector
+from .annotator import SpaceAnnotator
+from .error import DictConsistencyError
+
+
+class RougailConvert:
+    """Rougail object
+    """
+    def __init__(self,
+                 rougailconfig: RougailConfig=None,
+                 ) -> None:
+        if rougailconfig is None:
+            rougailconfig = RougailConfig
+        xmlreflector = XMLReflector(rougailconfig)
+        rougailobjspace = RougailObjSpace(xmlreflector,
+                                          rougailconfig,
+                                          )
+        self._load_dictionaries(xmlreflector,
+                                rougailobjspace,
+                                rougailconfig['variable_namespace'],
+                                rougailconfig['dictionaries_dir'],
+                                )
+        for namespace, extra_dir in rougailconfig['extra_dictionaries'].items():
+            if namespace in ['services', rougailconfig['variable_namespace']]:
+                msg = _(f'Namespace name "{namespace}" is not allowed')
+                raise DictConsistencyError(msg, 21, None)
+            self._load_dictionaries(xmlreflector,
+                                    rougailobjspace,
+                                    namespace,
+                                    extra_dir,
+                                    )
+        functions_file = rougailconfig['functions_file']
+        if not isinstance(functions_file, list):
+            functions_file = [functions_file]
+        SpaceAnnotator(rougailobjspace,
+                       functions_file,
+                       )
+        self.output = TiramisuReflector(rougailobjspace,
+                                        functions_file,
+                                        rougailconfig['internal_functions'],
+                                        ).get_text() + '\n'
+
+    @staticmethod
+    def _load_dictionaries(xmlreflector: XMLReflector,
+                           rougailobjspace: RougailObjSpace,
+                           namespace: str,
+                           xmlfolders: List[str],
+                           ) -> List[str]:
+        for xmlfile, document in xmlreflector.load_xml_from_folders(xmlfolders):
+            rougailobjspace.xml_parse_document(xmlfile,
+                                               document,
+                                               namespace,
+                                               )
+
+    def save(self,
+             filename: str,
+             ) -> str:
+        """Return tiramisu object declaration as a string
+        """
+        if filename:
+            with open(filename, 'w') as tiramisu:
+                tiramisu.write(self.output)
+        return self.output

src/rougail/data/rougail.dtd

@@ -13,7 +13,7 @@
 
 #    Forked by:
 #    Cadoles (http://www.cadoles.com)
-#    Copyright (C) 2019-2020
+#    Copyright (C) 2019-2021
 
 #    distribued with GPL-2 or later license
 
@@ -37,7 +37,8 @@
 <!-- root element    -->
 <!-- =============== -->
 
-<!ELEMENT rougail (services | variables | constraints | help)*>
+<!ELEMENT rougail (services|variables|constraints)*>
+<!ATTLIST rougail version (0.10) #REQUIRED>
 
 <!-- ============== -->
 <!-- files element  -->
@@ -45,60 +46,58 @@
 
 <!ELEMENT services (service*)>
 
-<!ELEMENT service ((port* | tcpwrapper* | ip* | interface* | package* | file* | digitalcertificate* | override*)*) >
+<!ELEMENT service ((ip*|file*|override*)*)>
 <!ATTLIST service name CDATA #REQUIRED>
 <!ATTLIST service manage (True|False) "True">
-
-<!ELEMENT port (#PCDATA)>
-<!ATTLIST port port_type (PortOption|SymLinkOption|variable) "PortOption">
-<!ATTLIST port portlist CDATA #IMPLIED >
-<!ATTLIST port protocol (tcp|udp) "tcp">
+<!ATTLIST service servicelist CDATA #IMPLIED>
+<!ATTLIST service disabled (True|False) "False">
+<!ATTLIST service engine (none|creole|jinja2) #IMPLIED>
+<!ATTLIST service target CDATA #IMPLIED>
+<!ATTLIST service type (service|mount|swap|timer) "service">
+<!ATTLIST service undisable (True|False) "False">
 
 <!ELEMENT ip (#PCDATA)>
-<!ATTLIST ip iplist CDATA #IMPLIED >
-<!ATTLIST ip ip_type (NetworkOption|SymLinkOption|variable) "NetworkOption">
-<!ATTLIST ip interface_type (UnicodeOption|SymLinkOption|variable) "UnicodeOption">
-<!ATTLIST ip interface CDATA #REQUIRED>
-<!ATTLIST ip netmask_type (NetmaskOption|SymLinkOption|variable) "NetmaskOption">
-<!ATTLIST ip netmask CDATA "255.255.255.255">
+<!ATTLIST ip iplist CDATA #IMPLIED>
+<!ATTLIST ip ip_type (variable) "variable">
+<!ATTLIST ip netmask_type (variable) "variable">
+<!ATTLIST ip netmask CDATA #IMPLIED>
 
-<!ELEMENT file EMPTY>
-<!ATTLIST file name CDATA #REQUIRED >
-<!ATTLIST file file_type (UnicodeOption|SymLinkOption|variable) "UnicodeOption">
+<!ELEMENT file (#PCDATA)>
+<!ATTLIST file file_type (filename|variable) "filename">
 <!ATTLIST file variable CDATA #IMPLIED>
 <!ATTLIST file variable_type (variable) "variable">
 <!ATTLIST file source CDATA #IMPLIED>
-<!ATTLIST file mode CDATA "0644">
-<!ATTLIST file owner CDATA "root">
-<!ATTLIST file group CDATA "root">
-<!ATTLIST file filelist CDATA #IMPLIED >
+<!ATTLIST file mode CDATA #IMPLIED>
+<!ATTLIST file owner CDATA #IMPLIED>
+<!ATTLIST file owner_type (unix_user|variable) "unix_user">
+<!ATTLIST file group CDATA #IMPLIED>
+<!ATTLIST file group_type (unix_user|variable) "unix_user">
+<!ATTLIST file filelist CDATA #IMPLIED>
 <!ATTLIST file redefine (True|False) "False">
-<!ATTLIST file templating (True|False) "True">
-
-<!ELEMENT digitalcertificate EMPTY>
-<!ATTLIST digitalcertificate name CDATA #REQUIRED >
-<!ATTLIST digitalcertificate digitalcertificate_type (variable) "variable">
-<!ATTLIST digitalcertificate certificate CDATA #REQUIRED >
-<!ATTLIST digitalcertificate certificate_type (variable) "variable">
-<!ATTLIST digitalcertificate type CDATA #REQUIRED >
-<!ATTLIST digitalcertificate ca CDATA #REQUIRED >
+<!ATTLIST file engine (none|creole|jinja2|creole_legacy) #IMPLIED>
+<!ATTLIST file included (no|name|content) #IMPLIED>
+<!ATTLIST file disabled (True|False) "False">
 
 <!ELEMENT override EMPTY>
-<!ATTLIST override source CDATA #IMPLIED >
-<!ATTLIST override templating (True|False) "True">
+<!ATTLIST override source CDATA #IMPLIED>
+<!ATTLIST override engine (none|creole|jinja2) #IMPLIED>
 
-<!ELEMENT variables (family*, separators*)>
-<!ELEMENT family (#PCDATA | variable)*>
+<!ELEMENT variables ((variable*|family*)*)>
+
+<!ELEMENT family ((variable*|family*)*)>
 <!ATTLIST family name CDATA #REQUIRED>
 <!ATTLIST family description CDATA #IMPLIED>
-<!ATTLIST family mode (basic|normal|expert) "basic">
+<!ATTLIST family help CDATA #IMPLIED>
+<!ATTLIST family mode CDATA #IMPLIED>
 <!ATTLIST family hidden (True|False) "False">
 <!ATTLIST family dynamic CDATA #IMPLIED>
+<!ATTLIST family leadership (True|False) "False">
 
-<!ELEMENT variable (#PCDATA | value)*>
+<!ELEMENT variable ((choice*|value*)*)>
 <!ATTLIST variable name CDATA #REQUIRED>
-<!ATTLIST variable type CDATA #IMPLIED>
+<!ATTLIST variable type (number|float|string|password|secret|mail|boolean|filename|date|unix_user|ip|local_ip|netmask|network|broadcast|netbios|domainname|hostname|web_address|port|mac|cidr|network_cidr|choice) "string">
 <!ATTLIST variable description CDATA #IMPLIED>
+<!ATTLIST variable help CDATA #IMPLIED>
 <!ATTLIST variable hidden (True|False) "False">
 <!ATTLIST variable disabled (True|False) "False">
 <!ATTLIST variable multi (True|False) "False">
@@ -107,49 +106,42 @@
 <!ATTLIST variable mandatory (True|False) "False">
 <!ATTLIST variable auto_freeze (True|False) "False">
 <!ATTLIST variable auto_save (True|False) "False">
-<!ATTLIST variable mode (basic|normal|expert) "normal">
+<!ATTLIST variable mode CDATA #IMPLIED>
+<!ATTLIST variable remove_choice (True|False) "False">
 <!ATTLIST variable remove_check (True|False) "False">
 <!ATTLIST variable remove_condition (True|False) "False">
+<!ATTLIST variable remove_fill (True|False) "False">
+<!ATTLIST variable provider CDATA #IMPLIED>
 <!ATTLIST variable test CDATA #IMPLIED>
 
-<!ELEMENT separators (separator*)>
-
-<!ELEMENT separator (#PCDATA)>
-<!ATTLIST separator name CDATA #REQUIRED>
-
 <!ELEMENT value (#PCDATA)>
+<!ATTLIST value type (string|number|nil|space|boolean) #IMPLIED>
 
-<!ELEMENT constraints ((fill* | check* | condition* | group*)*)>
-<!ELEMENT fill (param*)>
+<!ELEMENT choice (#PCDATA | param)*>
+<!ATTLIST choice type (string|number|nil|space|boolean|function|variable) "string">
+<!ATTLIST choice name CDATA #IMPLIED>
+
+<!ELEMENT constraints ((fill*|check*|condition*)*)>
+
+<!ELEMENT fill ((target|param)+)>
 <!ATTLIST fill name CDATA #REQUIRED>
-<!ATTLIST fill target CDATA #REQUIRED>
 
-<!ELEMENT check (param*)>
+<!ELEMENT check ((target|param)+)>
 <!ATTLIST check name CDATA #REQUIRED>
-<!ATTLIST check target CDATA #REQUIRED>
 <!ATTLIST check level (error|warning) "error">
 
-<!ELEMENT condition ((target | param)+ )>
-<!ATTLIST condition name (disabled_if_in|disabled_if_not_in|hidden_if_in|auto_hidden_if_not_in|hidden_if_not_in|mandatory_if_in|mandatory_if_not_in) #REQUIRED>
+<!ELEMENT condition ((target|param)+)>
+<!ATTLIST condition name (disabled_if_in|disabled_if_not_in|hidden_if_in|hidden_if_not_in|mandatory_if_in|mandatory_if_not_in) #REQUIRED>
 <!ATTLIST condition source CDATA #REQUIRED>
-<!ATTLIST condition fallback (True|False) "False">
-<!ATTLIST condition force_condition_on_fallback (True|False) "False">
-<!ATTLIST condition force_inverse_condition_on_fallback (True|False) "False">
-
-<!ELEMENT group (follower+)>
-<!ATTLIST group leader CDATA #REQUIRED>
-<!ATTLIST group description CDATA #IMPLIED>
+<!ATTLIST condition optional (True|False) "False">
+<!ATTLIST condition apply_on_fallback (True|False) #IMPLIED>
 
 <!ELEMENT param (#PCDATA)>
-<!ATTLIST param type (string|variable|number|python) "string">
+<!ATTLIST param type (string|number|nil|space|boolean|variable|function|information|target_information|suffix|index) "string">
 <!ATTLIST param name CDATA #IMPLIED>
-<!ATTLIST param notraisepropertyerror (True|False) "False">
+<!ATTLIST param propertyerror (True|False) "True">
 <!ATTLIST param optional (True|False) "False">
 
 <!ELEMENT target (#PCDATA)>
-<!ATTLIST target type (family|variable|filelist|iplist|portlist) "variable">
+<!ATTLIST target type (variable|family|servicelist|filelist|iplist) "variable">
 <!ATTLIST target optional (True|False) "False">
-
-<!ELEMENT follower (#PCDATA)>
-
-<!ELEMENT help ((variable* | family*)*)>

src/rougail/error.py

@@ -1,22 +1,58 @@
-# -*- coding: utf-8 -*-
+"""Standard error classes
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from .i18n import _
+
+
+def display_xmlfiles(xmlfiles: list) -> str:
+    """The function format xmlfiles informations to generate errors
+    """
+    if len(xmlfiles) == 1:
+        return '"' + xmlfiles[0] + '"'
+    return '"' + '", "'.join(xmlfiles[:-1]) + '"' + ' and ' + '"' + xmlfiles[-1] + '"'
+
+
 class ConfigError(Exception):
-    pass
+    """Standard error for templating
+    """
+
 
 class FileNotFound(ConfigError):
-    pass
+    """Template file is not found
+    """
+
 
 class TemplateError(ConfigError):
-    pass
+    """Templating generate an error
+    """
+
 
 class TemplateDisabled(TemplateError):
     """Template is disabled.
     """
-    pass
-
-
-class OperationError(Exception):
-    """Type error or value Error for Creole variable's type or values
-    """
 
 
 class SpaceObjShallNotBeUpdated(Exception):
@@ -29,7 +65,13 @@ class DictConsistencyError(Exception):
     """It's not only that the Creole XML is valid against the Creole DTD
     it's that it is not consistent.
     """
+    def __init__(self, msg, errno, xmlfiles):
+        if xmlfiles:
+            msg = _(f'{msg} in {display_xmlfiles(xmlfiles)}')
+        super().__init__(msg)
+        self.errno = errno
 
 
-class LoaderError(Exception):
-    pass
+class UpgradeError(Exception):
+    """Error during XML upgrade
+    """

src/rougail/i18n.py

@@ -1,23 +1,28 @@
-# -*- coding: UTF-8 -*-
-# Copyright (C) 2012-2013 Team tiramisu (see AUTHORS for all contributors)
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by the
-# Free Software Foundation, either version 3 of the License, or (at your
-# option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# The original `Config` design model is unproudly borrowed from
-# the rough gus of pypy: pypy: http://codespeak.net/svn/pypy/dist/pypy/config/
-# the whole pypy projet is under MIT licence
-"internationalisation utilities"
+"""Internationalisation utilities
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
 import gettext
 import os
 import sys

src/rougail/objspace.py

@@ -1,529 +1,534 @@
+"""parse XML files and build a space with objects
+it aggregates this files and manage redefine and exists attributes
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
-Creole flattener. Takes a bunch of Creole XML dispatched in differents folders
-as an input and outputs a human readable flatened XML
-
-Sample usage::
-
-    >>> from rougail.objspace import CreoleObjSpace
-    >>> eolobj = CreoleObjSpace('/usr/share/rougail/rougail.dtd')
-    >>> eolobj.create_or_populate_from_xml('rougail', ['/usr/share/eole/rougail/dicos'])
-    >>> eolobj.space_visitor()
-    >>> eolobj.save('/tmp/rougail_flatened_output.xml')
-
-The CreoleObjSpace
-
-- loads the XML into an internal CreoleObjSpace representation
-- visits/annotates the objects
-- dumps the object space as XML output into a single XML target
-
-The visit/annotation stage is a complex step that corresponds to the Creole
-procedures.
-
-For example: a variable is redefined and shall be moved to another family
-means that a variable1 = Variable() object in the object space who lives in the family1 parent
-has to be moved in family2. The visit procedure changes the varable1's object space's parent.
-"""
-from lxml.etree import Element, SubElement  # pylint: disable=E0611
+from typing import Optional
 
 from .i18n import _
 from .xmlreflector import XMLReflector
-from .annotator import ERASED_ATTRIBUTES, SpaceAnnotator
-from .tiramisureflector import TiramisuReflector
-from .utils import normalize_family
-from .error import OperationError, SpaceObjShallNotBeUpdated, DictConsistencyError
+from .utils import valid_variable_family_name
+from .error import SpaceObjShallNotBeUpdated, DictConsistencyError
 from .path import Path
-from .config import variable_namespace
 
-# CreoleObjSpace's elements like 'family' or 'follower', that shall be forced to the Redefinable type
+# RougailObjSpace's elements that shall be forced to the Redefinable type
 FORCE_REDEFINABLES = ('family', 'follower', 'service', 'disknod', 'variables')
-# CreoleObjSpace's elements that shall be forced to the UnRedefinable type
+# RougailObjSpace's elements that shall be forced to the UnRedefinable type
 FORCE_UNREDEFINABLES = ('value',)
-# CreoleObjSpace's elements that shall be set to the UnRedefinable type
-UNREDEFINABLE = ('multi', 'type')
+# RougailObjSpace's elements that shall not be modify
+UNREDEFINABLE = ('multi', 'type',)
+# RougailObjSpace's elements that did not created automaticly
+FORCE_ELEMENTS = ('property_', 'information')
+# XML text are convert has name
+FORCED_TEXT_ELTS_AS_NAME = ('choice', 'property', 'value',)
 
-CONVERT_PROPERTIES = {'auto_save': ['force_store_value'], 'auto_freeze': ['force_store_value', 'auto_freeze']}
+FORCE_TAG = {'family': 'variable'}
 
-RENAME_ATTIBUTES = {'description': 'doc'}
-
-FORCED_TEXT_ELTS_AS_NAME = ('choice', 'property', 'value', 'target')
-
-CONVERT_EXPORT = {'Leadership': 'leader',
-                  'Variable': 'variable',
-                  'Value': 'value',
-                  'Property': 'property',
-                  'Choice': 'choice',
-                  'Param': 'param',
-                  'Check': 'check',
-                  }
 
 # _____________________________________________________________________________
 # special types definitions for the Object Space's internal representation
-class RootCreoleObject:
-    ""
-
-
-class CreoleObjSpace:
-    """DOM XML reflexion free internal representation of a Creole Dictionary
+class RootRougailObject:  # pylint: disable=R0903
+    """Root object
     """
-    choice = type('Choice', (RootCreoleObject,), dict())
-    property_ = type('Property', (RootCreoleObject,), dict())
-    # Creole ObjectSpace's Leadership variable class type
-    Leadership = type('Leadership', (RootCreoleObject,), dict())
-    """
-    This Atom type stands for singleton, that is
-    an Object Space's atom object is present only once in the
-    object space's tree
-    """
-    Atom = type('Atom', (RootCreoleObject,), dict())
-    "A variable that can't be redefined"
-    Redefinable = type('Redefinable', (RootCreoleObject,), dict())
-    "A variable can be redefined"
-    UnRedefinable = type('UnRedefinable', (RootCreoleObject,), dict())
+    def __init__(self,
+                 xmlfiles,
+                 name=None,
+                 ):
+        if not isinstance(xmlfiles, list):
+            xmlfiles = [xmlfiles]
+        self.xmlfiles = xmlfiles
+        if name:
+            self.name = name
 
 
-    def __init__(self, dtdfilename):  # pylint: disable=R0912
-        self.index = 0
-        class ObjSpace:  # pylint: disable=R0903
-            """
-            Base object space
-            """
+class Atom(RootRougailObject):  # pylint: disable=R0903
+    """Atomic object (means can only define one time)
+    """
+
+
+class Redefinable(RootRougailObject):  # pylint: disable=R0903
+    """Object that could be redefine
+    """
+
+
+class UnRedefinable(RootRougailObject):  # pylint: disable=R0903
+    """Object that could not be redefine
+    """
+
+
+class ObjSpace:  # pylint: disable=R0903
+    """
+    Base object space
+    """
+
+
+def convert_boolean(value: str) -> bool:
+    """Boolean coercion. The Rougail XML may contain srings like `True` or `False`
+    """
+    if isinstance(value, bool):
+        return value
+    if value == 'True':
+        return True
+    return False
+
+
+class RougailObjSpace:
+    """Rougail ObjectSpace is an object's reflexion of the XML elements
+    """
+
+    def __init__(self,
+                 xmlreflector: XMLReflector,
+                 rougailconfig: 'RougailConfig',
+                 ) -> None:
         self.space = ObjSpace()
-        self.paths = Path()
-        self.xmlreflector = XMLReflector()
-        self.xmlreflector.parse_dtd(dtdfilename)
-        self.redefine_variables = None
-        self.check_removed = None
-        self.condition_removed = None
+        self.paths = Path(rougailconfig)
 
-        # ['variable', 'separator', 'family']
-        self.forced_text_elts = set()
         self.forced_text_elts_as_name = set(FORCED_TEXT_ELTS_AS_NAME)
         self.list_conditions = {}
+        self.valid_enums = {}
         self.booleans_attributs = []
+        self.has_dyn_option = False
+        self.types = {}
 
-        self.make_object_space_class()
+        self.make_object_space_classes(xmlreflector)
+        self.rougailconfig = rougailconfig
 
-    def make_object_space_class(self):
-        """Create Rougail ObjectSpace class types, it enables us to create objects like:
+    def make_object_space_classes(self,
+                                  xmlreflector: XMLReflector,
+                                  ) -> None:
+        """Create Rougail ObjectSpace class types from DDT file
+        It enables us to create objects like:
         File(), Variable(), Ip(), Family(), Constraints()... and so on.
-        
-        Creole ObjectSpace is an object's reflexion of the XML elements"""
+        """
 
-        for dtd_elt in self.xmlreflector.dtd.iterelements():
+        for dtd_elt in xmlreflector.dtd.iterelements():
             attrs = {}
             if dtd_elt.name in FORCE_REDEFINABLES:
-                clstype = self.Redefinable
+                clstype = Redefinable
+            elif not dtd_elt.attributes() and dtd_elt.name not in FORCE_UNREDEFINABLES:
+                clstype = Atom
             else:
-                clstype = self.UnRedefinable
-            atomic = dtd_elt.name not in FORCE_UNREDEFINABLES and dtd_elt.name not in FORCE_REDEFINABLES
+                clstype = UnRedefinable
             forced_text_elt = dtd_elt.type == 'mixed'
             for dtd_attr in dtd_elt.iterattributes():
-                atomic = False
-                if set(dtd_attr.itervalues()) == set(['True', 'False']):
+                if set(dtd_attr.itervalues()) == {'True', 'False'}:
                     # it's a boolean
                     self.booleans_attributs.append(dtd_attr.name)
                 if dtd_attr.default_value:
                     # set default value for this attribute
                     default_value = dtd_attr.default_value
                     if dtd_attr.name in self.booleans_attributs:
-                        default_value = self.convert_boolean(dtd_attr.default_value)
+                        default_value = convert_boolean(default_value)
                     attrs[dtd_attr.name] = default_value
+                    if dtd_attr.name.endswith('_type'):
+                        self.types[dtd_attr.name] = default_value
                 if dtd_attr.name == 'redefine':
                     # has a redefine attribute, so it's a Redefinable object
-                    clstype = self.Redefinable
+                    clstype = Redefinable
                 if dtd_attr.name == 'name' and forced_text_elt:
                     # child.text should be transform has a "name" attribute
-                    self.forced_text_elts.add(dtd_elt.name)
                     forced_text_elt = False
 
             if forced_text_elt is True:
                 self.forced_text_elts_as_name.add(dtd_elt.name)
-            if atomic:
-                # has any attribute so it's an Atomic object
-                clstype = self.Atom
 
             # create ObjectSpace object
             setattr(self, dtd_elt.name, type(dtd_elt.name.capitalize(), (clstype,), attrs))
+        for elt in FORCE_ELEMENTS:
+            setattr(self, elt, type(self._get_elt_name(elt), (RootRougailObject,), dict()))
 
-    def create_or_populate_from_xml(self,
-                                    namespace,
-                                    xmlfolders):
-        """Parses a bunch of XML files
-        populates the CreoleObjSpace
-        """
-        for xmlfile, document in self.xmlreflector.load_xml_from_folders(xmlfolders):
-            self.redefine_variables = []
-            self.check_removed = []
-            self.condition_removed = []
-            self.xml_parse_document(document,
-                                    self.space,
-                                    namespace,
-                                    )
+    @staticmethod
+    def _get_elt_name(elt) -> str:
+        name = elt.capitalize()
+        if name.endswith('_'):
+            name = name[:-1]
+        return name
 
     def xml_parse_document(self,
+                           xmlfile,
                            document,
-                           space,
                            namespace,
                            ):
-        """Parses a Creole XML file
-        populates the CreoleObjSpace
+        """Parses a Rougail XML file and populates the RougailObjSpace
         """
+        redefine_variables = []
+        self._xml_parse(xmlfile,
+                        document,
+                        self.space,
+                        namespace,
+                        redefine_variables,
+                        False,
+                        )
+
+    def _xml_parse(self,  # pylint: disable=R0913
+                   xmlfile,
+                   document,
+                   space,
+                   namespace,
+                   redefine_variables,
+                   is_dynamic,
+                   ) -> None:
+        # var to check unique family name in a XML file
         family_names = []
         for child in document:
-            # this index enables us to reorder objects
-            self.index += 1
-            # doesn't proceed the XML commentaries
+            if is_dynamic:
+                is_sub_dynamic = True
+            else:
+                is_sub_dynamic = document.attrib.get('dynamic') is not None
             if not isinstance(child.tag, str):
+                # doesn't proceed the XML commentaries
                 continue
             if child.tag == 'family':
                 if child.attrib['name'] in family_names:
-                    raise DictConsistencyError(_('Family {} is set several times').format(child.attrib['name']))
+                    msg = _(f'Family "{child.attrib["name"]}" is set several times')
+                    raise DictConsistencyError(msg, 44, [xmlfile])
                 family_names.append(child.attrib['name'])
-            if child.tag == 'variables':
-                child.attrib['name'] = namespace
-            if child.tag == 'value' and child.text == None:
-                # FIXME should not be here
-                continue
-            # variable objects creation
             try:
-                variableobj = self.generate_variableobj(child,
-                                                    space,
-                                                    namespace,
-                                                    )
+                # variable objects creation
+                exists, variableobj = self.get_variableobj(xmlfile,
+                                                           child,
+                                                           space,
+                                                           namespace,
+                                                           redefine_variables,
+                                                           )
             except SpaceObjShallNotBeUpdated:
                 continue
-            self.set_text_to_obj(child,
-                                 variableobj,
-                                 )
-            self.set_xml_attributes_to_obj(child,
-                                           variableobj,
-                                           )
-            self.variableobj_tree_visitor(child,
-                                          variableobj,
-                                          namespace,
-                                          )
-            self.fill_variableobj_path_attribute(space,
-                                                 child,
-                                                 namespace,
-                                                 document,
-                                                 variableobj,
-                                                 )
+            self.set_text(child,
+                          variableobj,
+                          )
+            self.set_attributes(xmlfile,
+                                child,
+                                variableobj,
+                                )
+            self.remove(child,
+                        variableobj,
+                        redefine_variables,
+                        )
+            if not exists:
+                self.set_path(namespace,
+                              document,
+                              variableobj,
+                              space,
+                              is_sub_dynamic,
+                              )
             self.add_to_tree_structure(variableobj,
                                        space,
                                        child,
+                                       namespace,
                                        )
             if list(child) != []:
-                self.xml_parse_document(child,
-                                        variableobj,
-                                        namespace,
-                                        )
+                self._xml_parse(xmlfile,
+                                child,
+                                variableobj,
+                                namespace,
+                                redefine_variables,
+                                is_sub_dynamic,
+                                )
 
-    def generate_variableobj(self,
-                           child,
-                           space,
-                           namespace,
-                           ):
+    def get_variableobj(self,
+                        xmlfile: str,
+                        child: list,
+                        space,
+                        namespace,
+                        redefine_variables,
+                        ):  # pylint: disable=R0913
         """
-        instanciates or creates Creole Object Subspace objects
+        retrieves or creates Rougail Object Subspace objects
         """
-        variableobj = getattr(self, child.tag)()
-        if isinstance(variableobj, self.Redefinable):
-            variableobj = self.create_or_update_redefinable_object(child.attrib,
-                                                                 space,
-                                                                 child,
-                                                                 namespace,
-                                                                 )
-        elif isinstance(variableobj, self.Atom) and child.tag in vars(space):
-            # instanciates an object from the CreoleObjSpace's builtins types
-            # example : child.tag = constraints -> a self.Constraints() object is created
-            # this Atom instance has to be a singleton here
-            # we do not re-create it, we reuse it
-            variableobj = getattr(space, child.tag)
-        self.create_tree_structure(space,
-                                   child,
-                                   variableobj,
-                                   )
-        return variableobj
+        tag = FORCE_TAG.get(child.tag, child.tag)
+        obj = getattr(self, tag)
+        name = self._get_name(child, namespace)
+        if Redefinable in obj.__mro__:
+            return self.create_or_update_redefinable_object(xmlfile,
+                                                            child.attrib,
+                                                            space,
+                                                            child,
+                                                            name,
+                                                            namespace,
+                                                            redefine_variables,
+                                                            )
+        if Atom in obj.__mro__:
+            if child.tag in vars(space):
+                # Atom instance has to be a singleton here
+                # we do not re-create it, we reuse it
+                return False, getattr(space, child.tag)
+            return False, obj(xmlfile, name)
+        # UnRedefinable object
+        if child.tag not in vars(space):
+            setattr(space, child.tag, [])
+        return False, obj(xmlfile, name)
+
+    def _get_name(self,
+                  child,
+                  namespace: str,
+                  ) -> Optional[str]:
+        if child.tag == 'variables':
+            return namespace
+        if child.tag == 'service':
+            return child.attrib['name'] + '.' + child.attrib.get('type', 'service')
+        if 'name' in child.attrib:
+            return child.attrib['name']
+        if child.text and child.tag in self.forced_text_elts_as_name:
+            return child.text.strip()
+        return None
 
     def create_or_update_redefinable_object(self,
+                                            xmlfile,
                                             subspace,
                                             space,
                                             child,
+                                            name,
                                             namespace,
-                                            ):
-        """Creates or retrieves the space object that corresponds
-        to the `child` XML object
-
-        Two attributes of the `child` XML object are important:
-
-        - with the `redefine` boolean flag attribute we know whether
-          the corresponding space object shall be created or updated
-
-            - `True` means that the corresponding space object shall be updated
-            - `False` means that the corresponding space object shall be created
-
-        - with the `exists` boolean flag attribute we know whether
-          the corresponding space object shall be created
-          (or nothing -- that is the space object isn't modified)
-
-            - `True` means that the corresponding space object shall be created
-            - `False` means that the corresponding space object is not updated
-
-        In the special case `redefine` is True and `exists` is False,
-        we create the corresponding space object if it doesn't exist
-        and we update it if it exists.
-
-        :return: the corresponding space object of the `child` XML object
+                                            redefine_variables,
+                                            ):  # pylint: disable=R0913
+        """A redefinable object could be created or updated
         """
-        if child.tag in self.forced_text_elts_as_name:
-            name = child.text
-        else:
-            name = subspace['name']
-        if self.is_already_exists(name,
-                                  space,
-                                  child,
-                                  namespace,
-                                  ):
+        existed_var = self.get_existed_obj(name,
+                                           xmlfile,
+                                           space,
+                                           child,
+                                           namespace,
+                                           )
+        if existed_var:
+            # if redefine is set to object, default value is False
+            # otherwise it's always a redefinable object
             default_redefine = child.tag in FORCE_REDEFINABLES
-            redefine = self.convert_boolean(subspace.get('redefine', default_redefine))
-            exists = self.convert_boolean(subspace.get('exists', True))
+            redefine = convert_boolean(subspace.get('redefine', default_redefine))
             if redefine is True:
-                return self.translate_in_space(name,
-                                               space,
-                                               child,
-                                               namespace,
-                                               )
-            elif exists is False:
+                if isinstance(existed_var, self.variable):  # pylint: disable=E1101
+                    if namespace == self.rougailconfig['variable_namespace']:
+                        redefine_variables.append(name)
+                    else:
+                        redefine_variables.append(space.path + '.' + name)
+                existed_var.xmlfiles.append(xmlfile)
+                return True, existed_var
+            exists = convert_boolean(subspace.get('exists', True))
+            if exists is False:
                 raise SpaceObjShallNotBeUpdated()
-            raise DictConsistencyError(_(f'Already present in another XML file, {name} cannot be re-created'))
-        redefine = self.convert_boolean(subspace.get('redefine', False))
-        exists = self.convert_boolean(subspace.get('exists', False))
-        if redefine is False or exists is True:
-            return getattr(self, child.tag)()
-        raise DictConsistencyError(_(f'Redefined object: {name} does not exist yet'))
+            msg = _(f'"{child.tag}" named "{name}" cannot be re-created in "{xmlfile}", '
+                    f'already defined')
+            raise DictConsistencyError(msg, 45, existed_var.xmlfiles)
+        # object deos not exists
+        exists = convert_boolean(subspace.get('exists', False))
+        if exists is True:
+            # manage object only if already exists, so cancel
+            raise SpaceObjShallNotBeUpdated()
+        redefine = convert_boolean(subspace.get('redefine', False))
+        if redefine is True:
+            # cannot redefine an inexistant object
+            msg = _(f'Redefined object: "{name}" does not exist yet')
+            raise DictConsistencyError(msg, 46, [xmlfile])
+        tag = FORCE_TAG.get(child.tag, child.tag)
+        if tag not in vars(space):
+            setattr(space, tag, {})
+        obj = getattr(self, child.tag)(xmlfile, name)
+        return False, obj
 
-    def create_tree_structure(self,
-                              space,
-                              child,
-                              variableobj,
-                              ):  # pylint: disable=R0201
-        """
-        Builds the tree structure of the object space here
-        we set services attributes in order to be populated later on
-        for example::
-
-            space = Family()
-            space.variable = dict()
-            another example:
-            space = Variable()
-            space.value = list()
-        """
-        if child.tag not in vars(space):
-            if isinstance(variableobj, self.Redefinable):
-                setattr(space, child.tag, dict())
-            elif isinstance(variableobj, self.UnRedefinable):
-                setattr(space, child.tag, [])
-            elif not isinstance(variableobj, self.Atom):  # pragma: no cover
-                raise OperationError(_("Creole object {} "
-                                             "has a wrong type").format(type(variableobj)))
-
-    def is_already_exists(self, name, space, child, namespace):
-        if isinstance(space, self.family):  # pylint: disable=E1101
-            if namespace != variable_namespace:
-                name = space.path + '.' + name
-            return self.paths.path_is_defined(name)
-        if child.tag == 'family':
-            norm_name = normalize_family(name)
-        else:
-            norm_name = name
-        return norm_name in getattr(space, child.tag, {})
-
-    def convert_boolean(self, value):  # pylint: disable=R0201
-        """Boolean coercion. The Creole XML may contain srings like `True` or `False`
-        """
-        if isinstance(value, bool):
-            return value
-        if value == 'True':
-            return True
-        elif value == 'False':
-            return False
-        else:
-            raise TypeError(_('{} is not True or False').format(value))  # pragma: no cover
-
-    def translate_in_space(self,
-                           name,
-                           family,
-                           variable,
-                           namespace,
-                           ):
-        if not isinstance(family, self.family):  # pylint: disable=E1101
-            if variable.tag == 'family':
-                norm_name = normalize_family(name)
-            else:
-                norm_name = name
-            return getattr(family, variable.tag)[norm_name]
-        if namespace == variable_namespace:
-            path = name
-        else:
-            path = family.path + '.' + name
-        old_family_name = self.paths.get_variable_family_name(path)
-        if normalize_family(family.name) == old_family_name:
-            return getattr(family, variable.tag)[name]
-        old_family = self.space.variables[variable_namespace].family[old_family_name]  # pylint: disable=E1101
-        variable_obj = old_family.variable[name]
-        del old_family.variable[name]
-        if 'variable' not in vars(family):
-            family.variable = dict()
-        family.variable[name] = variable_obj
-        self.paths.add_variable(namespace,
-                                name,
-                                family.name,
-                                False,
-                                variable_obj,
-                                )
-        return variable_obj
-
-    def remove_check(self, name):  # pylint: disable=C0111
-        if hasattr(self.space, 'constraints') and hasattr(self.space.constraints, 'check'):
-            remove_checks = []
-            for idx, check in enumerate(self.space.constraints.check):  # pylint: disable=E1101
-                if hasattr(check, 'target') and check.target == name:
-                    remove_checks.append(idx)
-
-            remove_checks = list(set(remove_checks))
-            remove_checks.sort(reverse=True)
-            for idx in remove_checks:
-                self.space.constraints.check.pop(idx)  # pylint: disable=E1101
-
-    def remove_condition(self, name):  # pylint: disable=C0111
-        for idx, condition in enumerate(self.space.constraints.condition):  # pylint: disable=E1101
-            remove_targets = []
-            if hasattr(condition, 'target'):
-                for target_idx, target in enumerate(condition.target):
-                    if target.name == name:
-                        remove_targets.append(target_idx)
-            remove_targets = list(set(remove_targets))
-            remove_targets.sort(reverse=True)
-            for idx in remove_targets:
-                del condition.target[idx]
-
-    def add_to_tree_structure(self,
-                              variableobj,
-                              space,
-                              child,
-                              ):  # pylint: disable=R0201
-        if isinstance(variableobj, self.Redefinable):
-            name = variableobj.name
-            if child.tag == 'family':
-                name = normalize_family(name)
-            getattr(space, child.tag)[name] = variableobj
-        elif isinstance(variableobj, self.UnRedefinable):
-            getattr(space, child.tag).append(variableobj)
-        else:
-            setattr(space, child.tag, variableobj)
-
-    def set_text_to_obj(self,
+    def get_existed_obj(self,
+                        name: str,
+                        xmlfile: str,
+                        space: str,
                         child,
-                        variableobj,
-                        ):
-        if child.text is None:
-            text = None
-        else:
-            text = child.text.strip()
-        if text:
-            if child.tag in self.forced_text_elts_as_name:
-                variableobj.name = text
-            else:
-                variableobj.text = text
+                        namespace: str,
+                        ) -> None:
+        """if an object exists, return it
+        """
+        if child.tag in ['variable', 'family']:
+            valid_variable_family_name(name, [xmlfile])
+        if child.tag == 'variable':  # pylint: disable=E1101
+            if namespace != self.rougailconfig['variable_namespace']:
+                name = space.path + '.' + name
+            if not self.paths.path_is_defined(name):
+                return None
+            old_family_name = self.paths.get_variable_family_path(name)
+            if space.path != old_family_name:
+                msg = _(f'Variable "{name}" was previously create in family "{old_family_name}", '
+                        f'now it is in "{space.path}"')
+                raise DictConsistencyError(msg, 47, space.xmlfiles)
+            return self.paths.get_variable(name)
+        # it's not a family
+        tag = FORCE_TAG.get(child.tag, child.tag)
+        children = getattr(space, tag, {})
+        if name in children and isinstance(children[name], getattr(self, child.tag)):
+            return children[name]
+        return None
 
-    def set_xml_attributes_to_obj(self,
-                                  child,
-                                  variableobj,
-                                  ):
-        redefine = self.convert_boolean(child.attrib.get('redefine', False))
-        has_value = hasattr(variableobj, 'value')
-        if redefine is True and child.tag == 'variable' and has_value and len(child) != 0:
-            del variableobj.value
+    def set_text(self,
+                 child,
+                 variableobj,
+                 ) -> None:
+        """set text
+        """
+        if child.text is None or child.tag in self.forced_text_elts_as_name:
+            return
+        text = child.text.strip()
+        if text:
+            variableobj.text = text
+
+    def set_attributes(self,
+                       xmlfile,
+                       child,
+                       variableobj,
+                       ):
+        """ set attributes to an object
+        """
+        redefine = convert_boolean(child.attrib.get('redefine', False))
+        if redefine and child.tag == 'variable':
+            # delete old values
+            has_value = hasattr(variableobj, 'value')
+            if has_value and len(child) != 0:
+                del variableobj.value
         for attr, val in child.attrib.items():
             if redefine and attr in UNREDEFINABLE:
-                # UNREDEFINABLE concerns only 'variable' node so we can fix name
-                # to child.attrib['name']
-                name = child.attrib['name']
-                raise DictConsistencyError(_(f'cannot redefine attribute {attr} for variable {name}'))
+                msg = _(f'cannot redefine attribute "{attr}" for variable "{child.attrib["name"]}"'
+                        f' already defined')
+                raise DictConsistencyError(msg, 48, variableobj.xmlfiles[:-1])
             if attr in self.booleans_attributs:
-                val = self.convert_boolean(val)
-            if not (attr == 'name' and getattr(variableobj, 'name', None) != None):
-                setattr(variableobj, attr, val)
-        keys = list(vars(variableobj).keys())
+                val = convert_boolean(val)
+            if attr == 'name' and getattr(variableobj, 'name', None):
+                # do not redefine name
+                continue
+            setattr(variableobj, attr, val)
 
-    def variableobj_tree_visitor(self,
-                                 child,
-                                 variableobj,
-                                 namespace,
-                                 ):
-        """Creole object tree manipulations
+    def remove(self,
+               child,
+               variableobj,
+               redefine_variables,
+               ):
+        """Rougail object tree manipulations
         """
         if child.tag == 'variable':
+            if child.attrib.get('remove_choice', False):
+                if variableobj.type != 'choice':
+                    msg = _(f'cannot remove choices for variable "{variableobj.path}"'
+                            f' the variable has type "{variableobj.type}"')
+                    raise DictConsistencyError(msg, 33, variableobj.xmlfiles)
+                variableobj.choice = []
             if child.attrib.get('remove_check', False):
                 self.remove_check(variableobj.name)
             if child.attrib.get('remove_condition', False):
                 self.remove_condition(variableobj.name)
+            if child.attrib.get('remove_fill', False):
+                self.remove_fill(variableobj.name)
         if child.tag == 'fill':
-            # if variable is a redefine in current dictionary
-            # XXX not working with variable not in variable and in leader/followers
-            variableobj.redefine = child.attrib['target'] in self.redefine_variables
-        if not hasattr(variableobj, 'index'):
-            variableobj.index = self.index
-        if child.tag == 'check' and child.attrib['target'] in self.redefine_variables and child.attrib['target'] not in self.check_removed:
-            self.remove_check(child.attrib['target'])
-            self.check_removed.append(child.attrib['target'])
-        if child.tag == 'condition' and child.attrib['source'] in self.redefine_variables and child.attrib['source'] not in self.check_removed:
-            self.remove_condition(child.attrib['source'])
-            self.condition_removed.append(child.attrib['source'])
-        variableobj.namespace = namespace
+            for target in child:
+                if target.tag == 'target' and target.text in redefine_variables:
+                    self.remove_fill(target.text)
 
-    def fill_variableobj_path_attribute(self,
-                                        space,
-                                        child,
-                                        namespace,
-                                        document,
-                                        variableobj,
-                                        ):  # pylint: disable=R0913
+    def remove_check(self, name):
+        """Remove a check with a specified target
+        """
+        if hasattr(self.space.constraints, 'check'):
+            remove_checks = []
+            for idx, check in enumerate(self.space.constraints.check):  # pylint: disable=E1101
+                for target in check.target:
+                    if target.name == name:
+                        remove_checks.append(idx)
+            remove_checks.sort(reverse=True)
+            for idx in remove_checks:
+                self.space.constraints.check.pop(idx)  # pylint: disable=E1101
+
+    def remove_condition(self,
+                         name: str,
+                         ) -> None:
+        """Remove a condition with a specified source
+        """
+        remove_conditions = []
+        for idx, condition in enumerate(self.space.constraints.condition):  # pylint: disable=E1101
+            if condition.source == name:
+                remove_conditions.append(idx)
+        remove_conditions.sort(reverse=True)
+        for idx in remove_conditions:
+            del self.space.constraints.condition[idx]  # pylint: disable=E1101
+
+    def remove_fill(self,
+                    name: str,
+                    ) -> None:
+        """Remove a fill with a specified target
+        """
+        remove_fills = []
+        for idx, fill in enumerate(self.space.constraints.fill):  # pylint: disable=E1101
+            for target in fill.target:
+                if target.name == name:
+                    remove_fills.append(idx)
+        remove_fills.sort(reverse=True)
+        for idx in remove_fills:
+            self.space.constraints.fill.pop(idx)  # pylint: disable=E1101
+
+    def set_path(self,
+                 namespace,
+                 document,
+                 variableobj,
+                 space,
+                 is_dynamic,
+                 ):
         """Fill self.paths attributes
         """
-        if isinstance(space, self.help):  # pylint: disable=E1101
-            return
-        if child.tag == 'variable':
-            family_name = normalize_family(document.attrib['name'])
-            self.paths.add_variable(namespace,
-                                    child.attrib['name'],
-                                    family_name,
-                                    document.attrib.get('dynamic') != None,
-                                    variableobj)
-            if child.attrib.get('redefine', 'False') == 'True':
-                if namespace == variable_namespace:
-                    self.redefine_variables.append(child.attrib['name'])
-                else:
-                    self.redefine_variables.append(namespace + '.' + family_name + '.' +
-                                                   child.attrib['name'])
+        if isinstance(variableobj, self.variable):  # pylint: disable=E1101
+            if 'name' in document.attrib:
+                family_name = document.attrib['name']
+            else:
+                family_name = namespace
 
-        elif child.tag == 'family':
-            family_name = normalize_family(child.attrib['name'])
-            if namespace != variable_namespace:
-                family_name = namespace + '.' + family_name
+            if isinstance(space, self.family) and space.leadership:
+                leader = space.path
+            else:
+                leader = None
+            self.paths.add_variable(namespace,
+                                    variableobj.name,
+                                    space.path,
+                                    is_dynamic,
+                                    variableobj,
+                                    leader,
+                                    )
+        elif isinstance(variableobj, self.family):  # pylint: disable=E1101
+            family_name = variableobj.name
+            if namespace != self.rougailconfig['variable_namespace']:
+                family_name = space.path + '.' + family_name
             self.paths.add_family(namespace,
                                   family_name,
                                   variableobj,
+                                  space.path,
                                   )
-            variableobj.path = self.paths.get_family_path(family_name, namespace)
+        elif isinstance(variableobj, self.variables):
+            variableobj.path = variableobj.name
 
-    def space_visitor(self, eosfunc_file):  # pylint: disable=C0111
-        self.funcs_path = eosfunc_file
-        SpaceAnnotator(self, eosfunc_file)
-
-    def save(self,
-             ):
-        tiramisu_objects = TiramisuReflector(self.space,
-                                             self.funcs_path,
-                                             )
-        return tiramisu_objects.get_text() + '\n'
+    @staticmethod
+    def add_to_tree_structure(variableobj,
+                              space,
+                              child,
+                              namespace: str,
+                              ) -> None:
+        """add a variable to the tree
+        """
+        variableobj.namespace = namespace
+        if isinstance(variableobj, Redefinable):
+            name = variableobj.name
+            tag = FORCE_TAG.get(child.tag, child.tag)
+            getattr(space, tag)[name] = variableobj
+        elif isinstance(variableobj, UnRedefinable):
+            getattr(space, child.tag).append(variableobj)
+        else:
+            setattr(space, child.tag, variableobj)

src/rougail/path.py

@@ -1,204 +1,221 @@
+"""Manage path to find objects
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import List
 from .i18n import _
+from .error import DictConsistencyError
 from .utils import normalize_family
-from .error import OperationError, DictConsistencyError
-from .config import variable_namespace
 
 
 class Path:
-    """Helper class to handle the `path` attribute of a CreoleObjSpace
-    instance.
+    """Helper class to handle the `path` attribute.
 
     sample: path="creole.general.condition"
     """
-    def __init__(self):
+    def __init__(self,
+                 rougailconfig: 'RougailConfig',
+                 ) -> None:
         self.variables = {}
         self.families = {}
-        self.full_paths = {}
+        self.full_paths_families = {}
+        self.full_paths_variables = {}
+        self.variable_namespace = rougailconfig['variable_namespace']
 
     # Family
     def add_family(self,
                    namespace: str,
                    name: str,
                    variableobj: str,
+                   subpath: str,
                    ) -> str:  # pylint: disable=C0111
-        if '.' not in name and namespace == variable_namespace:
-            full_name = '.'.join([namespace, name])
-            self.full_paths[name] = full_name
+        """Add a new family
+        """
+        if namespace == self.variable_namespace:
+            full_name = '.'.join([subpath, name])
+            if name in self.full_paths_families:
+                msg = _(f'Duplicate family name "{name}"')
+                raise DictConsistencyError(msg, 55, variableobj.xmlfiles)
+            self.full_paths_families[name] = full_name
         else:
+            if '.' not in name:  # pragma: no cover
+                msg = _(f'Variable "{name}" in namespace "{namespace}" must have dot')
+                raise DictConsistencyError(msg, 39, variableobj.xmlfiles)
             full_name = name
-        if full_name in self.families and self.families[full_name]['variableobj'] != variableobj:
-            raise DictConsistencyError(_(f'Duplicate family name {name}'))
+        if full_name in self.families and \
+                self.families[full_name]['variableobj'] != variableobj:  # pragma: no cover
+            msg = _(f'Duplicate family name "{name}"')
+            raise DictConsistencyError(msg, 37, variableobj.xmlfiles)
+        if full_name in self.variables:
+            msg = _(f'A variable and a family has the same path "{full_name}"')
+            raise DictConsistencyError(msg, 56, variableobj.xmlfiles)
         self.families[full_name] = dict(name=name,
                                         namespace=namespace,
                                         variableobj=variableobj,
                                         )
+        variableobj.path = full_name
 
-    def get_family_path(self,
-                        name: str,
-                        current_namespace: str,
-                        ) -> str:  # pylint: disable=C0111
-        name = normalize_family(name,
-                                check_name=False,
-                                allow_dot=True,
-                                )
-        if '.' not in name and current_namespace == variable_namespace and name in self.full_paths:
-            name = self.full_paths[name]
-        if current_namespace is None:  # pragma: no cover
-            raise OperationError('current_namespace must not be None')
-        dico = self.families[name]
-        if dico['namespace'] != variable_namespace and current_namespace != dico['namespace']:
-            raise DictConsistencyError(_('A family located in the {} namespace '
-                                               'shall not be used in the {} namespace').format(
-                                                   dico['namespace'], current_namespace))
-        return dico['name']
-
-    def get_family_obj(self,
-                       name: str,
-                       ) -> 'Family':  # pylint: disable=C0111
-        if '.' not in name and name in self.full_paths:
-            name = self.full_paths[name]
+    def get_family(self,
+                   name: str,
+                   current_namespace: str,
+                   ) -> 'Family':  # pylint: disable=C0111
+        """Get a family
+        """
+        name = '.'.join([normalize_family(subname) for subname in name.split('.')])
+        if name not in self.families and name in self.full_paths_families:
+            name = self.full_paths_families[name]
         if name not in self.families:
-            raise DictConsistencyError(_('unknown family {}').format(name))
+            raise DictConsistencyError(_(f'unknown option {name}'), 42, [])
         dico = self.families[name]
+        if current_namespace not in [self.variable_namespace, 'services'] and \
+                current_namespace != dico['namespace']:
+            msg = _(f'A family located in the "{dico["namespace"]}" namespace '
+                    f'shall not be used in the "{current_namespace}" namespace')
+            raise DictConsistencyError(msg, 38, [])
         return dico['variableobj']
 
-    def family_is_defined(self,
-                        name: str,
-                        ) -> str:  # pylint: disable=C0111
-        if '.' not in name and name not in self.families and name in self.full_paths:
-            return True
-        return name in self.families
+    def is_leader(self, path):  # pylint: disable=C0111
+        """Is the variable is a leader
+        """
+        variable = self._get_variable(path)
+        if not variable['leader']:
+            return False
+        leadership = self.get_family(variable['leader'], variable['variableobj'].namespace)
+        return next(iter(leadership.variable.values())).path == path
 
-    # Leadership
-    def set_leader(self,
-                   namespace: str,
-                   leader_family_name: str,
-                   name: str,
-                   leader_name: str,
-                   ) -> None:  # pylint: disable=C0111
-        # need rebuild path and move object in new path
-        old_path = namespace + '.' + leader_family_name + '.' + name
-        dico = self._get_variable(old_path)
-        del self.variables[old_path]
-        new_path = namespace + '.' + leader_family_name + '.' + leader_name + '.' + name
-        self.add_variable(namespace,
-                          new_path,
-                          dico['family'],
-                          False,
-                          dico['variableobj'],
-                          )
-        if namespace == variable_namespace:
-            self.full_paths[name] = new_path
-        else:
-            name = new_path
-        dico = self._get_variable(name)
-        if dico['leader'] != None:
-            raise DictConsistencyError(_('Already defined leader {} for variable'
-                                         ' {}'.format(dico['leader'], name)))
-        dico['leader'] = leader_name
-
-    def get_leader(self, name):  # pylint: disable=C0111
-        return self._get_variable(name)['leader']
+    def is_follower(self, path):
+        """Is the variable is a follower
+        """
+        variable = self._get_variable(path)
+        if not variable['leader']:
+            return False
+        leadership = self.get_family(variable['leader'], variable['variableobj'].namespace)
+        return next(iter(leadership.variable.values())).path != path
 
     # Variable
-    def add_variable(self,
+    def add_variable(self,  # pylint: disable=R0913
                      namespace: str,
                      name: str,
                      family: str,
                      is_dynamic: bool,
                      variableobj,
+                     leader: 'self.objectspace.family'=None,
                      ) -> str:  # pylint: disable=C0111
+        """Add a new variable (with path)
+        """
+        if name == 'hostname_' and not is_dynamic:
+            raise Exception('pffff')
         if '.' not in name:
-            full_name = '.'.join([namespace, family, name])
-            self.full_paths[name] = full_name
+            full_path = '.'.join([family, name])
+            if namespace == self.variable_namespace:
+                self.full_paths_variables[name] = full_path
         else:
-            full_name = name
-            if namespace == variable_namespace:
-                name = name.rsplit('.', 1)[1]
-        self.variables[full_name] = dict(name=name,
+            full_path = name
+        variableobj.path = full_path
+        if full_path in self.families:
+            msg = _(f'A family and a variable has the same path "{full_path}"')
+            raise DictConsistencyError(msg, 57, variableobj.xmlfiles)
+        self.variables[full_path] = dict(name=name,
                                          family=family,
-                                         namespace=namespace,
-                                         leader=None,
+                                         leader=leader,
                                          is_dynamic=is_dynamic,
-                                         variableobj=variableobj)
+                                         variableobj=variableobj,
+                                         )
 
-    def get_variable_name(self,
-                          name,
-                          ):  # pylint: disable=C0111
-        return self._get_variable(name)['name']
+    def get_variable(self,
+                     name: str,
+                     xmlfiles: List[str]=[],
+                     ) -> 'Variable':  # pylint: disable=C0111
+        """Get variable object from a path
+        """
+        variable, suffix = self._get_variable(name, with_suffix=True, xmlfiles=xmlfiles)
+        if suffix:
+            raise DictConsistencyError(_(f"{name} is a dynamic variable"), 36, [])
+        return variable['variableobj']
 
-    def get_variable_obj(self,
-                         name:str,
-                         ) -> 'Variable':  # pylint: disable=C0111
-        return self._get_variable(name)['variableobj']
-
-    def get_variable_family_name(self,
+    def get_variable_family_path(self,
                                  name: str,
+                                 xmlfiles: List[str]=False,
                                  ) -> str:  # pylint: disable=C0111
-        return self._get_variable(name)['family']
-
-    def get_variable_namespace(self,
-                               name: str,
-                               ) -> str:  # pylint: disable=C0111
-        return self._get_variable(name)['namespace']
+        """Get the full path of a family
+        """
+        return self._get_variable(name, xmlfiles=xmlfiles)['family']
 
     def get_variable_path(self,
                           name: str,
                           current_namespace: str,
-                          allow_source: str=False,
-                          with_suffix: bool=False,
+                          xmlfiles: List[str]=[],
                           ) -> str:  # pylint: disable=C0111
-        if current_namespace is None:  # pragma: no cover
-            raise OperationError('current_namespace must not be None')
-        if with_suffix:
-            dico, suffix = self._get_variable(name,
-                                              with_suffix=True,
-                                              )
-        else:
-            dico = self._get_variable(name)
-        if not allow_source:
-            if dico['namespace'] not in [variable_namespace, 'services'] and current_namespace != dico['namespace']:
-                raise DictConsistencyError(_('A variable located in the {} namespace '
-                                                   'shall not be used in the {} namespace').format(
-                                                       dico['namespace'], current_namespace))
-        if '.' in dico['name']:
-            value = dico['name']
-        else:
-            list_path = [dico['namespace'], dico['family']]
-            if dico['leader'] is not None:
-                list_path.append(dico['leader'])
-            list_path.append(dico['name'])
-            value = '.'.join(list_path)
-        if with_suffix:
-            return value, suffix
-        return value
+        """get full path of a variable
+        """
+        dico, suffix = self._get_variable(name,
+                                          with_suffix=True,
+                                          xmlfiles=xmlfiles,
+                                          )
+        namespace = dico['variableobj'].namespace
+        if namespace not in [self.variable_namespace, 'services'] and \
+                current_namespace != 'services' and \
+                current_namespace != namespace:
+            msg = _(f'A variable located in the "{namespace}" namespace shall not be used '
+                    f'in the "{current_namespace}" namespace')
+            raise DictConsistencyError(msg, 41, xmlfiles)
+        return dico['variableobj'].path, suffix
 
     def path_is_defined(self,
-                        name: str,
+                        path: str,
                         ) -> str:  # pylint: disable=C0111
-        if '.' not in name and name not in self.variables and name in self.full_paths:
+        """The path is a valid path
+        """
+        if '.' not in path and path not in self.variables and path in self.full_paths_variables:
             return True
-        return name in self.variables
+        return path in self.variables
+
+    def variable_is_dynamic(self,
+                            name: str,
+                            ) -> bool:
+        """This variable is in dynamic family
+        """
+        return self._get_variable(name)['is_dynamic']
 
     def _get_variable(self,
                       name: str,
                       with_suffix: bool=False,
+                      xmlfiles: List[str]=[],
                       ) -> str:
+        name = '.'.join([normalize_family(subname) for subname in name.split('.')])
         if name not in self.variables:
-            if name not in self.variables:
-                if '.' not in name and name in self.full_paths:
-                    name = self.full_paths[name]
-            if name not in self.variables:
-                for var_name, variable in self.variables.items():
-                    if variable['is_dynamic'] and name.startswith(var_name):
-                        return variable, name[len(var_name):]
-                if '.' not in name:
-                    for var_name, path in self.full_paths.items():
-                        if name.startswith(var_name):
-                            variable = self.variables[self.full_paths[var_name]]
-                            if variable['is_dynamic']:
-                                return variable, name[len(var_name):]
-                raise DictConsistencyError(_('unknown option {}').format(name))
+            if '.' not in name and name in self.full_paths_variables:
+                name = self.full_paths_variables[name]
+            elif with_suffix:
+                for var_name, full_path in self.full_paths_variables.items():
+                    if name.startswith(var_name):
+                        variable = self._get_variable(full_path)
+                        if variable['is_dynamic']:
+                            return variable, name[len(var_name):]
+        if name not in self.variables:
+            raise DictConsistencyError(_(f'unknown option "{name}"'), 42, xmlfiles)
         if with_suffix:
             return self.variables[name], None
         return self.variables[name]

src/rougail/template.py

@@ -1,413 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Gestion du mini-langage de template
-On travaille sur les fichiers cibles
-"""
-
-import imp
-from shutil import copy
-import logging
-from typing import Dict, Any
-from subprocess import call
-from os import listdir, makedirs
-from os.path import dirname, join, isfile
-
-from Cheetah.Template import Template as ChtTemplate
-from Cheetah.NameMapper import NotFound as CheetahNotFound
-
-from tiramisu import Config
-from tiramisu.error import PropertiesOptionError
-
-from .config import patch_dir, variable_namespace
-from .error import FileNotFound, TemplateError
-from .i18n import _
-from .utils import normalize_family
-
-
-log = logging.getLogger(__name__)
-log.addHandler(logging.NullHandler())
-
-
-class IsDefined:
-    """
-    filtre permettant de ne pas lever d'exception au cas où
-    la variable Creole n'est pas définie
-     """
-    def __init__(self, context):
-        self.context = context
-
-    def __call__(self, varname):
-        if '.' in varname:
-            splitted_var = varname.split('.')
-            if len(splitted_var) != 2:
-                msg = _("Group variables must be of type leader.follower")
-                raise KeyError(msg)
-            leader, follower = splitted_var
-            if leader in self.context:
-                return follower in self.context[leader].follower.keys()
-            return False
-        else:
-            return varname in self.context
-
-
-@classmethod
-def cl_compile(kls, *args, **kwargs):
-    kwargs['compilerSettings'] = {'directiveStartToken' : '%',
-                                  'cheetahVarStartToken' : '%%',
-                                  'EOLSlurpToken' : '%',
-                                  'PSPStartToken' : 'µ' * 10,
-                                  'PSPEndToken' : 'µ' * 10,
-                                  'commentStartToken' : 'µ' * 10,
-                                  'commentEndToken' : 'µ' * 10,
-                                  'multiLineCommentStartToken' : 'µ' * 10,
-                                  'multiLineCommentEndToken' : 'µ' * 10}
-    return kls.old_compile(*args, **kwargs)
-ChtTemplate.old_compile = ChtTemplate.compile
-ChtTemplate.compile = cl_compile
-
-
-class CheetahTemplate(ChtTemplate):
-    """classe pour personnaliser et faciliter la construction
-    du template Cheetah
-    """
-    def __init__(self,
-                 filename: str,
-                 context,
-                 eosfunc: Dict,
-                 destfilename,
-                 variable):
-        """Initialize Creole CheetahTemplate
-        """
-        extra_context = {'is_defined' : IsDefined(context),
-                         'normalize_family': normalize_family,
-                         'rougail_filename': destfilename
-                         }
-        if variable:
-            extra_context['rougail_variable'] = variable
-        ChtTemplate.__init__(self,
-                             file=filename,
-                             searchList=[context, eosfunc, extra_context])
-
-
-class CreoleLeader:
-    def __init__(self, value, follower=None, index=None):
-        """
-        On rend la variable itérable pour pouvoir faire:
-        for ip in iplist:
-        print(ip.network)
-        print(ip.netmask)
-        print(ip)
-        index is used for CreoleLint
-        """
-        self._value = value
-        if follower is not None:
-            self.follower = follower
-        else:
-            self.follower = {}
-        self._index = index
-
-    def __getattr__(self, name):
-        """Get follower variable or attribute of leader value.
-
-        If the attribute is a name of a follower variable, return its value.
-        Otherwise, returns the requested attribute of leader value.
-        """
-        if name in self.follower:
-            value = self.follower[name]
-            if isinstance(value, PropertiesOptionError):
-                raise AttributeError()
-            return value
-        else:
-            return getattr(self._value, name)
-
-    def __getitem__(self, index):
-        """Get a leader.follower at requested index.
-        """
-        ret = {}
-        for key, values in self.follower.items():
-            ret[key] = values[index]
-        return CreoleLeader(self._value[index], ret, index)
-
-    def __iter__(self):
-        """Iterate over leader.follower.
-
-        Return synchronised value of leader.follower.
-        """
-        for i in range(len(self._value)):
-            ret = {}
-            for key, values in self.follower.items():
-                ret[key] = values[i]
-            yield CreoleLeader(self._value[i], ret, i)
-
-    def __len__(self):
-        """Delegate to leader value
-        """
-        return len(self._value)
-
-    def __repr__(self):
-        """Show CreoleLeader as dictionary.
-
-        The leader value is stored under 'value' key.
-        The followers are stored under 'follower' key.
-        """
-        return repr({'value': self._value, 'follower': self.follower})
-
-    def __eq__(self, value):
-        return value == self._value
-
-    def __ne__(self, value):
-        return value != self._value
-
-    def __lt__(self, value):
-        return self._value < value
-
-    def __le__(self, value):
-        return self._value <= value
-
-    def __gt__(self, value):
-        return self._value > value
-
-    def __ge__(self, value):
-        return self._value >= value
-
-    def __str__(self):
-        """Delegate to leader value
-        """
-        return str(self._value)
-
-    def __add__(self, val):
-        return self._value.__add__(val)
-
-    def __radd__(self, val):
-        return val + self._value
-
-    def __contains__(self, item):
-        return item in self._value
-
-    async def add_follower(self, config, name, path):
-        if isinstance(self._value, list):
-            values = []
-            for idx in range(len(self._value)):
-                try:
-                    values.append(await config.option(path, idx).value.get())
-                except PropertiesOptionError as err:
-                    values.append(err)
-        else:
-            raise Exception('hu?')
-        self.follower[name] = values
-
-
-class CreoleExtra:
-    def __init__(self,
-                 suboption: Dict) -> None:
-        self.suboption = suboption
-
-    def __getattr__(self,
-                    key: str) -> Any:
-        return self.suboption[key]
-
-    def __repr__(self):
-        return self.suboption.__str__()
-
-    def __iter__(self):
-        return iter(self.suboption.values())
-
-
-class CreoleTemplateEngine:
-    """Engine to process Creole cheetah template
-    """
-    def __init__(self,
-                 config: Config,
-                 eosfunc_file: str,
-                 distrib_dir: str,
-                 tmp_dir: str,
-                 dest_dir: str,
-                 ) -> None:
-        self.config = config
-        self.dest_dir = dest_dir
-        self.tmp_dir = tmp_dir
-        self.distrib_dir = distrib_dir
-        eos = {}
-        if eosfunc_file is not None:
-            eosfunc = imp.load_source('eosfunc', eosfunc_file)
-            for func in dir(eosfunc):
-                 if not func.startswith('_'):
-                     eos[func] = getattr(eosfunc, func)
-        self.eosfunc = eos
-        self.rougail_variables_dict = {}
-
-    async def load_eole_variables_rougail(self,
-                                          optiondescription):
-        for option in await optiondescription.list('all'):
-            if await option.option.isoptiondescription():
-                if await option.option.isleadership():
-                    for idx, suboption in enumerate(await option.list('all')):
-                        if idx == 0:
-                            leader = CreoleLeader(await suboption.value.get())
-                            self.rougail_variables_dict[await suboption.option.name()] = leader
-                        else:
-                            await leader.add_follower(self.config,
-                                                   await suboption.option.name(),
-                                                   await suboption.option.path())
-                else:
-                    await self.load_eole_variables_rougail(option)
-            else:
-                self.rougail_variables_dict[await option.option.name()] = await option.value.get()
-
-    async def load_eole_variables(self,
-                                  namespace,
-                                  optiondescription):
-        families = {}
-        for family in await optiondescription.list('all'):
-            variables = {}
-            for variable in await family.list('all'):
-                if await variable.option.isoptiondescription():
-                    if await variable.option.isleadership():
-                        for idx, suboption in enumerate(await variable.list('all')):
-                            if idx == 0:
-                                leader = CreoleLeader(await suboption.value.get())
-                                leader_name = await suboption.option.name()
-                            else:
-                                await leader.add_follower(self.config,
-                                                       await suboption.option.name(),
-                                                       await suboption.option.path())
-                        variables[leader_name] = leader
-                    else:
-                        subfamilies = await self.load_eole_variables(await variable.option.name(),
-                                                                     variable,
-                                                                     )
-                        variables[await variable.option.name()] = subfamilies
-                else:
-                    variables[await variable.option.name()] = await variable.value.get()
-            families[await family.option.name()] = CreoleExtra(variables)
-        return CreoleExtra(families)
-
-    def patch_template(self,
-                       filename: str):
-        """Apply patch to a template
-        """
-        patch_cmd = ['patch', '-d', self.tmp_dir, '-N', '-p1']
-        patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']
-
-        # patches variante + locaux
-        for directory in [join(patch_dir, 'variante'), patch_dir]:
-            patch_file = join(directory, f'{filename}.patch')
-            if isfile(patch_file):
-                log.info(_("Patching template '{filename}' with '{patch_file}'"))
-                ret = call(patch_cmd + patch_no_debug + ['-i', patch_file])
-                if ret:
-                    patch_cmd_err = ' '.join(patch_cmd + ['-i', patch_file])
-                    log.error(_(f"Error applying patch: '{patch_file}'\nTo reproduce and fix this error {patch_cmd_err}"))
-                    copy(filename, self.tmp_dir)
-
-    def prepare_template(self,
-                         filename: str):
-        """Prepare template source file
-        """
-        log.info(_("Copy template: '{filename}' -> '{self.tmp_dir}'"))
-        copy(filename, self.tmp_dir)
-        self.patch_template(filename)
-
-    def process(self,
-                source: str,
-                true_destfilename: str,
-                destfilename: str,
-                filevar: Dict,
-                variable: Any):
-        """Process a cheetah template
-        """
-        # full path of the destination file
-        log.info(_(f"Cheetah processing: '{destfilename}'"))
-        try:
-            cheetah_template = CheetahTemplate(source,
-                                               self.rougail_variables_dict,
-                                               self.eosfunc,
-                                               true_destfilename,
-                                               variable,
-                                               )
-            data = str(cheetah_template)
-        except CheetahNotFound as err:
-            varname = err.args[0][13:-1]
-            raise TemplateError(_(f"Error: unknown variable used in template {destfilename} : {varname}"))
-        except Exception as err:
-            raise TemplateError(_(f"Error while instantiating template {destfilename}: {err}"))
-
-        with open(destfilename, 'w') as file_h:
-            file_h.write(data)
-
-    def instance_file(self,
-                      filevar: Dict,
-                      service_name: str) -> None:
-        """Run templatisation on one file
-        """
-        log.info(_("Instantiating file '{filename}'"))
-        if 'variable' in filevar:
-            variable = filevar['variable']
-        else:
-            variable = None
-        filenames = filevar['name']
-        if not isinstance(filenames, list):
-            filenames = [filenames]
-            if variable:
-                variable = [variable]
-        for idx, filename in enumerate(filenames):
-            destfilename = join(self.dest_dir, filename[1:])
-            makedirs(dirname(destfilename), exist_ok=True)
-            if variable:
-                var = variable[idx]
-            else:
-                var = None
-            source = join(self.tmp_dir, filevar['source'])
-            if filevar['templating']:
-                self.process(source,
-                             filename,
-                             destfilename,
-                             filevar,
-                             var)
-            else:
-                copy(source, destfilename)
-
-    async def instance_files(self) -> None:
-        """Run templatisation on all files
-        """
-        for option in await self.config.option.list(type='all'):
-            namespace = await option.option.name()
-            if namespace == variable_namespace:
-                await self.load_eole_variables_rougail(option)
-            else:
-                families = await self.load_eole_variables(namespace,
-                                                          option)
-                self.rougail_variables_dict[namespace] = families
-        for template in listdir(self.distrib_dir):
-            self.prepare_template(join(self.distrib_dir, template))
-        for service_obj in await self.config.option('services').list('all'):
-            service_name = await service_obj.option.doc()
-            for fills in await service_obj.list('all'):
-                if await fills.option.name() in ['files', 'overrides']:
-                    for fill_obj in await fills.list('all'):
-                        fill = await fill_obj.value.dict()
-                        filename = fill['source']
-                        distib_file = join(self.distrib_dir, filename)
-                        if not isfile(distib_file):
-                            raise FileNotFound(_(f"File {distib_file} does not exist."))
-                        if fill.get('activate', False):
-                            self.instance_file(fill,
-                                               service_name,
-                                               )
-                        else:
-                            log.debug(_("Instantiation of file '{filename}' disabled"))
-
-
-async def generate(config: Config,
-                   eosfunc_file: str,
-                   distrib_dir: str,
-                   tmp_dir: str,
-                   dest_dir: str,
-                   ) -> None:
-    engine = CreoleTemplateEngine(config,
-                                  eosfunc_file,
-                                  distrib_dir,
-                                  tmp_dir,
-                                  dest_dir,
-                                  )
-    await engine.instance_files()

src/rougail/template/base.py

@@ -0,0 +1,541 @@
+"""Template langage for Rougail
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from shutil import copy
+import logging
+from typing import Dict, Any
+from subprocess import call
+from os import listdir, makedirs, getcwd, chdir, unlink, rmdir
+from os.path import dirname, join, isfile, isdir, abspath
+
+
+try:
+    from tiramisu3 import Config, undefined
+    from tiramisu3.error import PropertiesOptionError  # pragma: no cover
+except ModuleNotFoundError:  # pragma: no cover
+    from tiramisu import Config, undefined
+    from tiramisu.error import PropertiesOptionError
+
+from ..config import RougailConfig
+from ..error import FileNotFound, TemplateError
+from ..i18n import _
+from ..utils import load_modules
+
+from . import engine as engines
+ENGINES = {}
+for engine in engines.__all__:
+    ENGINES[engine] = getattr(engines, engine)
+
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+
+INFORMATIONS = {'files': ['source', 'mode', 'engine', 'included'],
+                'overrides': ['name', 'source', 'engine'],
+                }
+DEFAULT = {'files': ['owner', 'group'],
+           'overrides': [],
+           }
+
+
+class RougailLeaderIndex:
+    """This object is create when access to a specified Index of the variable
+    """
+    def __init__(self,
+                 value,
+                 follower,
+                 index,
+                 ) -> None:
+        self._value = value
+        self._follower = follower
+        self._index = index
+
+    def __getattr__(self, name):
+        if name not in self._follower:
+            raise AttributeError(f'unable to find follower "{name}"')
+        value = self._follower[name]
+        if isinstance(value, PropertiesOptionError):
+            raise AttributeError(f'unable to access to follower "{name}": {value}')
+        return value
+
+    def __getitem__(self, name):
+        return self.__getattr__(name)
+
+    def __contains__(self, name):
+        if self._follower.__contains__(name):
+            value = self._follower[name]
+            return not isinstance(value, PropertiesOptionError)
+        return False
+
+    def __str__(self):
+        return str(self._value)
+
+    def __lt__(self, value):
+        return self._value.__lt__(value)
+
+    def __le__(self, value):
+        return self._value.__le__(value)
+
+    def __eq__(self, value):
+        return self._value.__eq__(value)
+
+    def __ne__(self, value):
+        return self._value.__ne__(value)
+
+    def __gt__(self, value):
+        return self._value.__gt__(value)
+
+    def __ge__(self, value):
+        return self._value >= value
+
+    def __add__(self, value):
+        return self._value.__add__(value)
+
+    def __radd__(self, value):
+        return value + self._value
+
+
+class RougailLeader:
+    """Implement access to leader and follower variable
+    For examples: %%leader, %%leader[0].follower1
+    """
+    def __init__(self,
+                 leader_name,
+                 value,
+                 ) -> None:
+        self._value = value
+        self._follower = {leader_name: value}
+
+    def __getitem__(self, index):
+        """Get a leader.follower at requested index.
+        """
+        followers = {key: values[index] for key, values in self._follower.items()}
+        return RougailLeaderIndex(self._value[index],
+                                 followers,
+                                 index,
+                                 )
+
+    def __iter__(self):
+        """Iterate over leader.follower.
+
+        Return synchronised value of leader.follower.
+        """
+        for index in range(len(self._value)):
+            yield self.__getitem__(index)
+
+    def __len__(self):
+        return len(self._value)
+
+    def __contains__(self, value):
+        return self._value.__contains__(value)
+
+    async def _add_follower(self,
+                            config,
+                            name: str,
+                            path: str,
+                            ):
+        """Add a new follower
+        """
+        self._follower[name] = []
+        for index in range(len(self._value)):
+            try:
+                value = await config.option(path, index).value.get()
+            except PropertiesOptionError as err:
+                value = err
+            self._follower[name].append(value)
+
+
+class RougailExtra:
+    """Object that implement access to extra variable
+    For example %%extra1.family.variable
+    """
+    def __init__(self,
+                 suboption: Dict,
+                 ) -> None:
+        self._suboption = suboption
+
+    def __getattr__(self,
+                    key: str,
+                    ) -> Any:
+        try:
+            return self._suboption[key]
+        except KeyError:
+            raise AttributeError(f'unable to find extra "{key}"')
+
+    def __getitem__(self,
+                    key: str,
+                    ) -> Any:
+        return self.__getattr__(key)
+
+    def __iter__(self):
+        return iter(self._suboption.values())
+
+    def items(self):
+        return self._suboption.items()
+
+    def __str__(self):
+        suboptions = {}
+        for key, value in self._suboption.items():
+            suboptions[key] = str(value)
+        return f'<extra object with: {suboptions}>'
+
+
+class RougailBaseTemplate:
+    """Engine to process Creole cheetah template
+    """
+    def __init__(self,  # pylint: disable=R0913
+                 config: Config,
+                 rougailconfig: RougailConfig=None,
+                 ) -> None:
+        if rougailconfig is None:
+            rougailconfig = RougailConfig
+        self.config = config
+        self.destinations_dir = abspath(rougailconfig['destinations_dir'])
+        self.tmp_dir = abspath(rougailconfig['tmp_dir'])
+        self.templates_dir = []
+        templates_dir = rougailconfig['templates_dir']
+        if not isinstance(templates_dir, list):
+            templates_dir = [templates_dir]
+        for templ_dir in templates_dir:
+            self.templates_dir.append(abspath(templ_dir))
+        self.patches_dir = abspath(rougailconfig['patches_dir'])
+        eos = {}
+        functions_file = rougailconfig['functions_file']
+        if not isinstance(functions_file, list):
+            functions_file = [functions_file]
+        for function in functions_file:
+            if isfile(function):
+                eosfunc = load_modules(function)
+                for func in dir(eosfunc):
+                    if not func.startswith('_'):
+                        eos[func] = getattr(eosfunc, func)
+        self.eosfunc = eos
+        self.rougail_variables_dict = {}
+        self.rougailconfig = rougailconfig
+        self.log = log
+        self.engines = ENGINES
+
+    def patch_template(self,
+                       filename: str,
+                       templates_dir: str,
+                       ) -> None:
+        """Apply patch to a template
+        """
+        patch_cmd = ['patch', '-d', self.tmp_dir, '-N', '-p1', '-f']
+        patch_no_debug = ['-s', '-r', '-', '--backup-if-mismatch']
+
+        patch_file = join(self.patches_dir, f'{filename}.patch')
+        if isfile(patch_file):
+            self.log.info(_("Patching template '{filename}' with '{patch_file}'"))
+            ret = call(patch_cmd + patch_no_debug + ['-i', patch_file])
+            if ret:  # pragma: no cover
+                patch_cmd_err = ' '.join(patch_cmd + ['-i', patch_file])
+                msg = _(f"Error applying patch: '{patch_file}'\n"
+                        f"To reproduce and fix this error {patch_cmd_err}")
+                self.log.error(_(msg))
+                copy(join(templates_dir, filename), self.tmp_dir)
+
+    def prepare_template(self,
+                         filename: str,
+                         templates_dir: str,
+                         ) -> None:
+        """Prepare template source file
+        """
+        self.log.info(_("Copy template: '{filename}' -> '{self.tmp_dir}'"))
+        if not isdir(self.tmp_dir):
+            raise TemplateError(_(f'cannot find tmp_dir {self.tmp_dir}'))
+        copy(join(templates_dir, filename), self.tmp_dir)
+        self.patch_template(filename, templates_dir)
+
+    def instance_file(self,
+                      filevar: Dict,
+                      type_: str,
+                      service_name: str,
+                      ) -> str:
+        """Run templatisation on one file
+        """
+        self.log.info(_("Instantiating file '{filename}'"))
+        if 'variable' in filevar:
+            variable = filevar['variable']
+        else:
+            variable = None
+        filenames = filevar.get('name')
+        if not isinstance(filenames, list):
+            filenames = [filenames]
+            if variable and not isinstance(variable, list):
+                variable = [variable]
+        if not isdir(self.destinations_dir):
+            raise TemplateError(_(f'cannot find destinations_dir {self.destinations_dir}'))
+        destfilenames = []
+        for idx, filename, in enumerate(filenames):
+            if variable:
+                var = variable[idx]
+            else:
+                var = None
+            func = f'get_data_{type_}'
+            data = getattr(self, func)(filevar,
+                                       filename,
+                                       service_name,
+                                       variable,
+                                       idx,
+                                       )
+            if data is None:
+                continue
+            filename, source, true_destfilename, var = data
+            destfilename = join(self.destinations_dir, true_destfilename[1:])
+            makedirs(dirname(destfilename), exist_ok=True)
+            self.log.info(_(f"{filevar['engine']} processing: '{destfilename}'"))
+            self.engines[filevar['engine']].process(filename=filename,
+                                                    source=source,
+                                                    true_destfilename=true_destfilename,
+                                                    destfilename=destfilename,
+                                                    destdir=self.destinations_dir,
+                                                    variable=var,
+                                                    index=idx,
+                                                    rougail_variables_dict=self.rougail_variables_dict,
+                                                    eosfunc=self.eosfunc,
+                                                    )
+            self.process(true_destfilename,
+                         destfilename,
+                         filevar.get('mode'),
+                         filevar.get('owner'),
+                         filevar.get('group'),
+                         )
+            destfilenames.append(destfilename)
+        return destfilenames
+
+    async def instance_files(self) -> None:
+        """Run templatisation on all files
+        """
+        try:
+            ori_dir = getcwd()
+        except FileNotFoundError:
+            ori_dir = None
+        chdir(self.tmp_dir)
+        for option in await self.config.option.list(type='all'):
+            namespace = await option.option.name()
+            is_variable_namespace = namespace == self.rougailconfig['variable_namespace']
+            if namespace == 'services':
+                is_service_namespace = 'root'
+            else:
+                is_service_namespace = False
+            self.rougail_variables_dict[namespace] = await self.load_variables(option,
+                                                                               is_variable_namespace,
+                                                                               is_service_namespace,
+                                                                               )
+        for templates_dir in self.templates_dir:
+            for template in listdir(templates_dir):
+                self.prepare_template(template,
+                                      templates_dir,
+                                      )
+        files_to_delete = []
+        for included in (True, False):
+            for service_obj in await self.config.option('services').list('all'):
+                service_name = await service_obj.option.description()
+                if await service_obj.option('activate').value.get() is False:
+                    if included is False and not await service_obj.information.get('undisable', False):
+                        self.desactive_service(service_name)
+                    continue
+                if not included:
+                    engine = await service_obj.information.get('engine', None)
+                    if engine:
+                        self.instance_file({'engine': engine},
+                                           'service',
+                                           service_name,
+                                           )
+                    target_name = await service_obj.information.get('target', None)
+                    if target_name:
+                        self.target_service(service_name,
+                                            target_name,
+                                            engine is None,
+                                            )
+                for fills in await service_obj.list('optiondescription'):
+                    type_ = await fills.option.name()
+                    for fill_obj in await fills.list('all'):
+                        fill = await fill_obj.value.dict()
+                        self.get_default(type_, fill, fill_obj)
+                        await self.get_informations(type_, fill, fill_obj)
+                        if 'included' in fill:
+                            if (fill['included'] == 'no' and included is True) or \
+                                    (fill['included'] != 'no' and included is False):
+                                continue
+                        elif included is True:
+                            continue
+                        if fill['activate']:
+                            destfilenames = self.instance_file(fill,
+                                                               type_,
+                                                               service_name,
+                                                               )
+                            if included and fill.get('included', 'no') == 'content':
+                                files_to_delete.extend(destfilenames)
+                        else:
+                            self.log.debug(_(f"Instantiation of file '{fill['name']}' disabled"))
+                self.post_instance_service(service_name)
+        for filename in files_to_delete:
+            unlink(filename)
+            parent = filename
+            while True:
+                parent = dirname(parent)
+                if listdir(parent):
+                    break
+                rmdir(parent)
+        self.post_instance()
+        if ori_dir is not None:
+            chdir(ori_dir)
+
+    def get_default(self,
+                    type_: str,
+                    dico: dict,
+                    obj: 'Option',
+                    ) -> None:
+        for key in DEFAULT.get(type_, []):
+            default_key = f'default_{type_}_{key}'
+            if default_key in RougailConfig:
+                default_value = RougailConfig[default_key]
+            else:
+                default_value = undefined
+            dico[key] = dico.get(key, default_value)
+
+    async def get_informations(self,
+                               type_: str,
+                               dico: dict,
+                               obj: 'Option',
+                               ) -> None:
+        for key in INFORMATIONS.get(type_, []):
+            default_key = f'default_{type_}_{key}'
+            if default_key in RougailConfig:
+                default_value = RougailConfig[default_key]
+            else:
+                default_value = undefined
+            dico[key] = await obj.information.get(key, default_value)
+
+    def desactive_service(self,
+                          *args,
+                          ):
+        raise NotImplementedError(_('cannot desactivate a service'))
+
+    def target_service(self,
+                       service_name: str,
+                       *args,
+                       ):
+        raise NotImplementedError(_('cannot use target for the service {service_name}'))
+
+    def post_instance_service(self,
+                              *args,
+                              ):  # pragma: no cover
+        pass
+
+    def process(self,
+                *args,
+                ):  # pragma: no cover
+        raise NotImplementedError(_('cannot processed'))
+
+    def post_instance(self):  # pragma: no cover
+        pass
+
+    def get_data_ip(self,
+                    *args,
+                    ) -> None:  # pragma: no cover
+        raise NotImplementedError(_('cannot instanciate this service type ip'))
+
+    def get_data_files(self,
+                       *args,
+                       ) -> None:  # pragma: no cover
+        raise NotImplementedError(_('cannot instanciate this service type file'))
+
+    def get_data_service(self,
+                         *args,
+                          ) -> None:  # pragma: no cover
+        raise NotImplementedError(_('cannot instanciate this service'))
+
+    def get_data_overrides(self,
+                           *args,
+                           ) -> None:  # pragma: no cover
+        raise NotImplementedError(_('cannot instanciate this service type override'))
+
+    async def load_variables(self,
+                             optiondescription,
+                             is_variable_namespace: str,
+                             is_service_namespace: str,
+                             ) -> RougailExtra:
+        """Load all variables and set it in RougailExtra objects
+        """
+        variables = {}
+        for option in await optiondescription.list('all'):
+            if await option.option.isoptiondescription():
+                if await option.option.isleadership():
+                    for idx, suboption in enumerate(await option.list('all')):
+                        if idx == 0:
+                            leader_name = await suboption.option.name()
+                            leader = RougailLeader(leader_name, await suboption.value.get())
+                            leadership_name = await option.option.name()
+                            if is_variable_namespace:
+                                self.rougail_variables_dict[await suboption.option.name()] = leader
+                        else:
+                            await leader._add_follower(self.config,
+                                                       await suboption.option.name(),
+                                                       await suboption.option.path(),
+                                                       )
+                    variables[leadership_name] = RougailExtra({leader_name: leader})
+                else:
+                    if is_service_namespace == 'root':
+                        new_is_service_namespace = 'service_name'
+                    elif is_service_namespace == 'service_name':
+                        new_is_service_namespace = await option.option.name()
+                    elif is_service_namespace in INFORMATIONS:
+                        # remove 's'
+                        new_is_service_namespace = is_service_namespace[:-1]
+                    else:
+                        new_is_service_namespace = is_service_namespace
+                    subfamilies = await self.load_variables(option,
+                                                            is_variable_namespace,
+                                                            new_is_service_namespace,
+                                                            )
+                    variables[await option.option.name()] = subfamilies
+            else:
+                if is_variable_namespace:
+                    value = await option.value.get()
+                    self.rougail_variables_dict[await option.option.name()] = value
+                if await option.option.issymlinkoption() and await option.option.isfollower():
+                    value = []
+                    path = await option.option.path()
+                    for index in range(await option.value.len()):
+                        value.append(await self.config.option(path, index).value.get())
+                else:
+                    value = await option.value.get()
+                variables[await option.option.name()] = value
+        if isinstance(is_service_namespace, str) and is_service_namespace + 's' in INFORMATIONS:
+            self.get_default(is_service_namespace + 's',
+                             variables,
+                             optiondescription,
+                             )
+            await self.get_informations(is_service_namespace + 's',
+                                        variables,
+                                        optiondescription,
+                                        )
+        return RougailExtra(variables)

src/rougail/template/engine/__init__.py

@@ -0,0 +1,4 @@
+from . import none, creole, jinja2, creole_legacy
+
+
+__all__ = ('none', 'creole', 'jinja2', 'creole_legacy')

src/rougail/template/engine/creole.py

@@ -0,0 +1,138 @@
+"""Creole engine
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from os.path import abspath, normpath
+from Cheetah.Template import Template
+from Cheetah.NameMapper import NotFound
+from typing import Dict, Any
+
+from ...i18n import _
+from ...utils import normalize_family
+from ...error import TemplateError
+
+
+@classmethod
+def cl_compile(kls, *args, **kwargs):
+    """Rewrite compile methode to force some settings
+    """
+    kwargs['compilerSettings'] = {'directiveStartToken': '%',
+                                  'cheetahVarStartToken': '%%',
+                                  'EOLSlurpToken': '%',
+                                  'commentStartToken': '#',
+                                  'multiLineCommentStartToken': '#*',
+                                  'multiLineCommentEndToken': '*#',
+                                  }
+    return kls.old_compile(*args, **kwargs)  # pylint: disable=E1101
+Template.old_compile = Template.compile
+Template.compile = cl_compile
+
+
+class CheetahTemplate(Template):  # pylint: disable=W0223
+    """Construct a cheetah templating object
+    """
+    def __init__(self,
+                 filename: str,
+                 source: str,
+                 context,
+                 eosfunc: Dict,
+                 extra_context: Dict,
+                 ):
+        """Initialize Creole CheetahTemplate
+        """
+        if filename is not None:
+            super().__init__(file=filename,
+                             searchList=[context, eosfunc, extra_context],
+                             )
+        else:
+            super().__init__(source=source,
+                             searchList=[context, eosfunc, extra_context],
+                             )
+
+    # FORK of Cheetah function, do not replace '\\' by '/'
+    def serverSidePath(self,
+                       path=None,
+                       normpath=normpath,
+                       abspath=abspath
+                       ):  # pylint: disable=W0621
+
+        # strange...
+        if path is None and isinstance(self, str):
+            path = self
+        if path:  # pylint: disable=R1705
+            return normpath(abspath(path))
+#           original code return normpath(abspath(path.replace("\\", '/')))
+        elif hasattr(self, '_filePath') and self._filePath:  # pragma: no cover
+            return normpath(abspath(self._filePath))
+        else:  # pragma: no cover
+            return None
+
+
+# Sync to creole_legacy.py
+def process(filename: str,
+            source: str,
+            true_destfilename: str,
+            destfilename: str,
+            destdir: str,
+            variable: Any,
+            index: int,
+            rougail_variables_dict: Dict,
+            eosfunc: Dict,
+            ):
+    """Process a cheetah template
+    """
+    # full path of the destination file
+    try:
+        extra_context = {'normalize_family': normalize_family,
+                         'rougail_filename': true_destfilename,
+                         'rougail_destination_dir': destdir,
+                         }
+        if variable is not None:
+            extra_context['rougail_variable'] = variable
+        if index is not None:
+            extra_context['rougail_index'] = index
+        cheetah_template = CheetahTemplate(filename,
+                                           source,
+                                           rougail_variables_dict,
+                                           eosfunc,
+                                           extra_context,
+                                           )
+        data = str(cheetah_template)
+    except NotFound as err:  # pragma: no cover
+        varname = err.args[0][13:].split(' ', 1)[0][:-1]
+        if filename:
+            msg = f"Error: unknown variable used in template {filename} to {destfilename}: {varname}"
+        else:
+            msg = f"Error: unknown variable used in file {destfilename}: {varname}"
+        raise TemplateError(_(msg)) from err
+    except Exception as err:  # pragma: no cover
+        if filename:
+            msg = _(f"Error while instantiating template {filename} to {destfilename}: {err}")
+        else:
+            msg = _(f"Error while instantiating filename {destfilename}: {err}")
+        raise TemplateError(msg) from err
+
+    with open(destfilename, 'w') as file_h:
+        file_h.write(data)

src/rougail/template/engine/creole_legacy.py

@@ -0,0 +1,164 @@
+"""Legacy Creole engine
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+
+from typing import Dict, Any
+from Cheetah.NameMapper import NotFound
+
+from .creole import CheetahTemplate as oriCheetahTemplate
+from ...i18n import _
+from ...utils import normalize_family
+from ...error import TemplateError
+
+
+@classmethod
+def cl_compile(kls, *args, **kwargs):
+    """Rewrite compile methode to force some settings
+    """
+    kwargs['compilerSettings'] = {'directiveStartToken' : u'%',
+                   'cheetahVarStartToken' : u'%%',
+                   'EOLSlurpToken' : u'%',
+                   'PSPStartToken' : u'µ' * 10,
+                   'PSPEndToken' : u'µ' * 10,
+                   'commentStartToken' : u'µ' * 10,
+                   'commentEndToken' : u'µ' * 10,
+                   'multiLineCommentStartToken' : u'µ' * 10,
+                   'multiLineCommentEndToken' : u'µ' * 10}
+    return kls.old_compile(*args, **kwargs)  # pylint: disable=E1101
+
+
+class IsDefined:
+    """
+    filtre permettant de ne pas lever d'exception au cas où
+    la variable Creole n'est pas définie
+     """
+    def __init__(self, context):
+        self.context = context
+
+    def __call__(self, varname):
+        if '.' in varname:
+            splitted_var = varname.split('.')
+            if len(splitted_var) != 2:
+                msg = u"Group variables must be of type master.slave"
+                raise KeyError(msg)
+            master, slave = splitted_var
+            if master in self.context:
+                return slave in self.context[master].slave.keys()
+            return False   
+        else:
+            return varname in self.context
+
+
+class CreoleClient():
+    def get(self, path):
+        path = path.replace('/', '.')
+        if path.startswith('.'):
+            path = path[1:]
+        if '.' not in path:
+            return self.context[path]
+        else:
+            root, path = path.split('.', 1)
+            obj = self.context[root]
+            for var in path.split('.'):
+                obj = getattr(obj, var)
+            return obj
+
+
+def is_empty(data):
+    if str(data) in ['', '""', "''", "[]", "['']", '[""]', "None"]:
+        return True
+    return False
+
+
+class CheetahTemplate(oriCheetahTemplate):
+    def __init__(self,
+                 filename: str,
+                 source: str,
+                 context,
+                 eosfunc: Dict,
+                 extra_context: Dict,
+                 ):
+        creole_client = CreoleClient()
+        creole_client.context=context
+        extra_context['is_defined'] = IsDefined(context)
+        extra_context['creole_client'] = creole_client
+        extra_context['is_empty'] = is_empty
+        extra_context['_creole_filename'] = extra_context['rougail_filename']
+        super().__init__(filename, source, context, eosfunc, extra_context)
+
+
+# Sync to creole.py
+def process(filename: str,
+            source: str,
+            true_destfilename: str,
+            destfilename: str,
+            destdir: str,
+            variable: Any,
+            index: int,
+            rougail_variables_dict: Dict,
+            eosfunc: Dict,
+            ):
+    """Process a cheetah template
+    """
+    # full path of the destination file
+    ori_compile = oriCheetahTemplate.compile
+    oriCheetahTemplate.compile = cl_compile
+    try:
+        extra_context = {'normalize_family': normalize_family,
+                         'rougail_filename': true_destfilename,
+                         'rougail_destination_dir': destdir,
+                         }
+        if variable is not None:
+            extra_context['rougail_variable'] = variable
+        if index is not None:
+            extra_context['rougail_index'] = index
+        cheetah_template = CheetahTemplate(filename,
+                                           source,
+                                           rougail_variables_dict,
+                                           eosfunc,
+                                           extra_context,
+                                           )
+        data = str(cheetah_template)
+    except NotFound as err:  # pragma: no cover
+        varname = err.args[0][13:-1]
+        if filename:
+            msg = f"Error: unknown variable used in template {filename} to {destfilename}: {varname}"
+        else:
+            msg = f"Error: unknown variable used in file {destfilename}: {varname}"
+        oriCheetahTemplate.compile = ori_compile
+        raise TemplateError(_(msg)) from err
+    except Exception as err:  # pragma: no cover
+        if filename:
+            msg = _(f"Error while instantiating template {filename} to {destfilename}: {err}")
+        else:
+            msg = _(f"Error while instantiating filename {destfilename}: {err}")
+        oriCheetahTemplate.compile = ori_compile
+        raise TemplateError(msg) from err
+
+    with open(destfilename, 'w') as file_h:
+        file_h.write(data)
+    oriCheetahTemplate.compile = ori_compile

src/rougail/template/engine/jinja2.py

@@ -0,0 +1,75 @@
+"""Jinja2 engine
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import Any, Dict
+from jinja2 import Environment, FileSystemLoader
+from jinja2.exceptions import UndefinedError
+
+from ...i18n import _
+from ...utils import normalize_family
+from ...error import TemplateError
+
+
+def process(filename: str,
+            source: str,
+            true_destfilename: str,
+            destfilename: str,
+            destdir: str,
+            variable: Any,
+            index: int,
+            rougail_variables_dict: Dict,
+            eosfunc: Dict,
+            ):
+    """Process a cheetah template
+    """
+    # full path of the destination file
+    dir_name, template_name = filename.rsplit('/', 1)
+    if source is not None:  # pragma: no cover
+        raise TemplateError(_('source is not supported for jinja2'))
+    var = {}
+    if variable is not None:
+        var['rougail_variable'] = variable
+    if index is not None:
+        var['rougail_index'] = index
+    try:
+        # extra_context = {'normalize_family': normalize_family,
+        # eosfunc
+        env = Environment(loader=FileSystemLoader([dir_name, destdir]))
+        template = env.get_template(template_name)
+        data = template.render(**rougail_variables_dict,
+                               rougail_filename=true_destfilename,
+                               rougail_destination_dir=destdir,
+                               **var,
+                               )
+    except UndefinedError as err:  # pragma: no cover
+        varname = err
+        msg = f"Error: unknown variable used in template {filename} to {destfilename}: {varname}"
+        raise TemplateError(_(msg)) from err
+
+    if not data.endswith('\n'):
+        data = data + '\n'
+    with open(destfilename, 'w') as file_h:
+        file_h.write(data)

src/rougail/template/engine/none.py

@@ -0,0 +1,35 @@
+"""None engine
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import Any
+from shutil import copy
+
+
+def process(filename: str,
+            destfilename: str,
+            **kwargs
+            ):
+    copy(filename, destfilename)

src/rougail/template/systemd.py

@@ -0,0 +1,212 @@
+"""Template langage for Rougail to create file and systemd file
+
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from typing import Dict, Any
+from os import makedirs, symlink, chmod
+from os.path import dirname, isfile, join
+from ipaddress import ip_network
+
+from .base import RougailBaseTemplate
+from ..i18n import _
+from ..error import FileNotFound, TemplateError
+
+
+ROUGAIL_IP_TEMPLATE = """[Service]
+%for %%ip in %%rougail_variable
+IPAddressAllow=%%ip
+%end for
+IPAddressDeny=any
+"""
+
+
+ROUGAIL_GLOBAL_SYSTEMD_FILE = '/usr/lib/systemd/system'
+ROUGAIL_DEST_FILE = '/tmpfiles.d/0rougail.conf'
+LOCAL_DIR = ('/etc/', '/var/', '/srv/')
+
+
+class RougailSystemdTemplate(RougailBaseTemplate):
+    def __init__(self,  # pylint: disable=R0913
+                 config: 'Config',
+                 rougailconfig: 'RougailConfig'=None,
+                 ) -> None:
+        self.ip_per_service = None
+        super().__init__(config, rougailconfig)
+        self.rougail_tmpl_template = f"""%def display(%%file, %%filename)
+"""
+        tmp_local_dir = (f"%%filename.startswith('{local_dir}')" for local_dir in LOCAL_DIR)
+        self.rougail_tmpl_template += '%if ' + ' or '.join(tmp_local_dir)
+        self.rougail_tmpl_template += f"""
+C %%filename %%file.mode %%file.owner %%file.group - {self.rougailconfig['tmpfile_dest_dir']}%%filename
+%end if
+%end def
+%for %%service in %%services
+  %if %%service.activate is True and %%hasattr(%%service, 'files')
+    %for %%file in %%service.files
+      %if %%file.activate is True and %%file.included != 'content'
+        %if %%isinstance(%%file.name, list)
+          %for %%filename in %%file.name
+%%display(%%file, %%filename)%slurp
+          %end for
+        %else
+%%display(%%file, %%file.name)%slurp
+        %end if
+      %end if
+    %end for
+  %end if
+%end for
+"""
+
+    def get_data_files(self,
+                       filevar: Dict,
+                       destfile: str,
+                       service_name: str,
+                       variable,
+                       idx: int,
+                       ) -> tuple:
+        source = filevar['source']
+        if not isfile(source):  # pragma: no cover
+            raise FileNotFound(_(f'Source file "{source}" does not exist in {", ".join(self.templates_dir)}'))
+        tmp_file = join(self.tmp_dir, source)
+        #self.instance_file(fill, 'files')
+        if variable:
+            var = variable[idx]
+        else:
+            var = None
+        return tmp_file, None, destfile, var
+
+    def get_data_overrides(self,
+                           filevar: Dict,
+                           destfile,
+                           service_name: str,
+                           *args,
+                           ) -> tuple:
+        source = filevar['source']
+        if not isfile(source):  # pragma: no cover
+            raise FileNotFound(_(f'Override source file "{source}" does not exist in {", ".join(self.templates_dir)}'))
+        tmp_file = join(self.tmp_dir, source)
+        service_name = filevar['name']
+        destfile = f'/systemd/system/{service_name}.d/rougail.conf'
+        return tmp_file, None, destfile, None
+
+    def get_data_ip(self,
+                    filevar: Dict,
+                    ip,
+                    service_name: str,
+                    var: Any,
+                    idx: int,
+                    *args,
+                    ) -> tuple:
+        if self.ip_per_service is None:
+            self.ip_per_service = []
+        if 'netmask' in filevar:
+            if isinstance(filevar["netmask"], list):
+                netmask = filevar['netmask'][idx]
+            else:
+                netmask = filevar['netmask']
+            self.ip_per_service.append(str(ip_network(f'{ip}/{netmask}')))
+        elif ip:
+            self.ip_per_service.append(ip)
+
+    def get_data_service(self,
+                         servicevar: Dict,
+                         info,
+                         service_name: str,
+                         *args,
+                         ):
+        tmp_file = join(self.tmp_dir, service_name)
+        var = None
+        destfile = f'/systemd/system/{service_name}'
+        return tmp_file, None, destfile, var
+
+
+    def desactive_service(self,
+                          service_name: str,
+                          ):
+        filename = f'{self.destinations_dir}/systemd/system/{service_name}'
+        makedirs(dirname(filename), exist_ok=True)
+        symlink('/dev/null', filename)
+
+    def target_service(self,
+                       service_name: str,
+                       target_name: str,
+                       global_service: str,
+                       ):
+        filename = f'{self.destinations_dir}/systemd/system/{target_name}.target.wants/{service_name}'
+        makedirs(dirname(filename), exist_ok=True)
+        if global_service:
+            source_filename = f'{ROUGAIL_GLOBAL_SYSTEMD_FILE}/{service_name}'
+        else:
+            source_filename = f"{self.rougailconfig['tmpfile_dest_dir']}/systemd/system/{service_name}"
+        symlink(source_filename, filename)
+
+    def post_instance_service(self,
+                              service_name: str,
+                              ) -> None:  # pragma: no cover
+        if self.ip_per_service is None:
+            return
+        destfile = f'/systemd/system/{service_name}.d/rougail_ip.conf'
+        destfilename = join(self.destinations_dir, destfile[1:])
+        makedirs(dirname(destfilename), exist_ok=True)
+        self.log.info(_(f"creole processing: '{destfilename}'"))
+        self.engines['creole'].process(filename=None,
+                                       source=ROUGAIL_IP_TEMPLATE,
+                                       true_destfilename=destfile,
+                                       destfilename=destfilename,
+                                       destdir=self.destinations_dir,
+                                       variable=self.ip_per_service,
+                                       index=None,
+                                       rougail_variables_dict=self.rougail_variables_dict,
+                                       eosfunc=self.eosfunc,
+                                       )
+        self.ip_per_service = None
+
+    def process(self,
+                filename: str,
+                destfilename: str,
+                mode: str,
+                owner: str,
+                group: str,
+                ) -> None:
+        for local_dir in LOCAL_DIR:
+            if filename.startswith(local_dir):
+                return
+        if owner not in [None, self.rougailconfig['default_files_owner']]:
+            raise TemplateError(_(f'cannot change owner of file {destfilename}'))
+        if group not in [None, self.rougailconfig['default_files_group']]:
+            raise TemplateError(_(f'cannot change group of file {destfilename}'))
+        if mode not in [None, self.rougailconfig['default_files_mode']]:
+            chmod(destfilename, eval(f'0o{mode}'))
+
+    def post_instance(self):
+        destfilename = join(self.destinations_dir, ROUGAIL_DEST_FILE[1:])
+        makedirs(dirname(destfilename), exist_ok=True)
+        self.log.info(_(f"creole processing: '{destfilename}'"))
+        self.engines['creole'].process(filename=None,
+                                       source=self.rougail_tmpl_template,
+                                       true_destfilename=ROUGAIL_DEST_FILE,
+                                       destfilename=destfilename,
+                                       destdir=self.destinations_dir,
+                                       variable=None,
+                                       index=None,
+                                       rougail_variables_dict=self.rougail_variables_dict,
+                                       eosfunc=self.eosfunc,
+                                       )

src/rougail/tiramisu.py

@@ -1,10 +1,43 @@
-from tiramisu import DynOptionDescription
+"""Redefine Tiramisu object
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+try:
+    from tiramisu3 import DynOptionDescription
+except ModuleNotFoundError:
+    from tiramisu import DynOptionDescription
 from .utils import normalize_family
 
 
 class ConvertDynOptionDescription(DynOptionDescription):
+    """Suffix could be an integer, we should convert it in str
+    Suffix could also contain invalid character, so we should "normalize" it
+    """
     def convert_suffix_to_path(self, suffix):
+        if suffix is None:
+            return suffix
         if not isinstance(suffix, str):
             suffix = str(suffix)
-        return normalize_family(suffix,
-                                check_name=False)
+        return normalize_family(suffix)

src/rougail/tiramisureflector.py

@@ -1,503 +1,452 @@
 """loader
 flattened XML specific
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
+from json import dumps
 from os.path import isfile
-from lxml.etree import DTD
 
-from .config import dtdfilename, variable_namespace
 from .i18n import _
-from .error import LoaderError
-from .annotator import ERASED_ATTRIBUTES
+from .annotator import CONVERT_OPTION
+from .objspace import RootRougailObject
+from .error import DictConsistencyError
 
 
-FUNC_TO_DICT = ['valid_not_equal']
-FORCE_INFORMATIONS = ['help', 'test', 'separator', 'manage']
-ATTRIBUTES_ORDER = ('name', 'doc', 'default', 'multi')
-
-
-CONVERT_OPTION = {'number': dict(opttype="IntOption", func=int),
-                  'choice': dict(opttype="ChoiceOption"),
-                  'string': dict(opttype="StrOption"),
-                  'password': dict(opttype="PasswordOption"),
-                  'mail': dict(opttype="EmailOption"),
-                  'boolean': dict(opttype="BoolOption"),
-                  'symlink': dict(opttype="SymLinkOption"),
-                  'filename': dict(opttype="FilenameOption"),
-                  'date': dict(opttype="DateOption"),
-                  'unix_user': dict(opttype="UsernameOption"),
-                  'ip': dict(opttype="IPOption", initkwargs={'allow_reserved': True}),
-                  'local_ip': dict(opttype="IPOption", initkwargs={'private_only': True, 'warnings_only': True}),
-                  'netmask': dict(opttype="NetmaskOption"),
-                  'network': dict(opttype="NetworkOption"),
-                  'broadcast': dict(opttype="BroadcastOption"),
-                  'netbios': dict(opttype="DomainnameOption", initkwargs={'type': 'netbios', 'warnings_only': True}),
-                  'domain': dict(opttype="DomainnameOption", initkwargs={'type': 'domainname', 'allow_ip': False}),
-                  'hostname': dict(opttype="DomainnameOption", initkwargs={'type': 'hostname', 'allow_ip': False}),
-                  'web_address': dict(opttype="URLOption", initkwargs={'allow_ip': True, 'allow_without_dot': True}),
-                  'port': dict(opttype="PortOption", initkwargs={'allow_private': True}),
-                  'mac': dict(opttype="MACOption"),
-                  'cidr': dict(opttype="IPOption", initkwargs={'cidr': True}),
-                  'network_cidr': dict(opttype="NetworkOption", initkwargs={'cidr': True}),
-                  }
+class BaseElt:  # pylint: disable=R0903
+    """Base element
+    """
+    name = 'baseoption'
+    doc = 'baseoption'
+    path = '.'
 
 
 class TiramisuReflector:
+    """Convert object to tiramisu representation
+    """
     def __init__(self,
-                 xmlroot,
-                 funcs_path,
+                 objectspace,
+                 funcs_paths,
+                 internal_functions,
                  ):
-        self.storage = ElementStorage()
-        self.storage.text = ["from tiramisu import *",
-                             "from rougail.tiramisu import ConvertDynOptionDescription",
-                             "import imp",
-                             f"func = imp.load_source('func', '{funcs_path}')",
-                             ]
-        self.make_tiramisu_objects(xmlroot)
-        # parse object
-        self.storage.get('.').get()
+        self.index = 0
+        self.text = []
+        if funcs_paths:
+            self.text.extend(["from importlib.machinery import SourceFileLoader as _SourceFileLoader",
+                              "from importlib.util import spec_from_loader as _spec_from_loader, module_from_spec as _module_from_spec",
+                              "class func:",
+                              "    pass",
+                              ])
+            for funcs_path in funcs_paths:
+                if not isfile(funcs_path):
+                    continue
+                self.text.extend([f"_loader = _SourceFileLoader('func', '{funcs_path}')",
+                                  "_spec = _spec_from_loader(_loader.name, _loader)",
+                                  "_func = _module_from_spec(_spec)",
+                                  "_loader.exec_module(_func)",
+                                  "for function in dir(_func):",
+                                  "    if function.startswith('_'):",
+                                  "        continue",
+                                  "    setattr(func, function, getattr(_func, function))",
+                                  ])
+        if internal_functions:
+            for func in internal_functions:
+                self.text.append(f"setattr(func, '{func}', {func})")
+        self.text.extend(["try:",
+                          "    from tiramisu3 import *",
+                          "except:",
+                          "    from tiramisu import *",
+                          ])
+        self.objectspace = objectspace
+        self.make_tiramisu_objects()
+        if self.objectspace.has_dyn_option is True:
+            self.text.append("from rougail.tiramisu import ConvertDynOptionDescription")
 
-    def make_tiramisu_objects(self,
-                              xmlroot,
-                              ):
-        family = self.get_root_family()
-        for xmlelt in self.reorder_family(xmlroot):
-            self.iter_family(xmlelt,
-                             family,
-                             None,
-                             )
+    def make_tiramisu_objects(self) -> None:
+        """make tiramisu objects
+        """
+        providers = {}
+        baseelt = BaseElt()
+        self.set_name(baseelt)
+        dynamic_path = ''
+        dynamic = False
+        basefamily = Family(baseelt,
+                            self.text,
+                            self.objectspace,
+                            )
+        for elt in self.reorder_family():
+            self.populate_family(basefamily,
+                                 elt,
+                                 providers,
+                                 dynamic,
+                                 dynamic_path,
+                                 )
+        basefamily.elt.information = providers
+        basefamily.populate_informations()
+        self.baseelt = baseelt
 
-    def get_root_family(self):
-        family = Family(BaseElt('baseoption',
-                                'baseoption',
-                                ),
-                        self.storage,
-                        False,
-                        '.',
-                        )
-        return family
-
-    def reorder_family(self, xmlroot):
-        # variable_namespace family has to be loaded before any other family
-        # because `extra` family could use `variable_namespace` variables.
-        if hasattr(xmlroot, 'variables'):
-            if variable_namespace in xmlroot.variables:
-                yield xmlroot.variables[variable_namespace]
-            for xmlelt, value in xmlroot.variables.items():
-                if xmlelt != variable_namespace:
+    def reorder_family(self):
+        """variable_namespace family has to be loaded before any other family
+        because `extra` family could use `variable_namespace` variables.
+        """
+        if hasattr(self.objectspace.space, 'variables'):
+            variable_namespace = self.objectspace.rougailconfig['variable_namespace']
+            if variable_namespace in self.objectspace.space.variables:
+                yield self.objectspace.space.variables[variable_namespace]
+            for elt, value in self.objectspace.space.variables.items():
+                if elt != self.objectspace.rougailconfig['variable_namespace']:
                     yield value
-        if hasattr(xmlroot, 'services'):
-            yield xmlroot.services
-
-    def get_attributes(self, space):  # pylint: disable=R0201
-        for attr in dir(space):
-            if not attr.startswith('_') and attr not in ERASED_ATTRIBUTES:
-                yield attr
-
-    def get_children(self,
-                     space,
-                     ):
-        for tag in self.get_attributes(space):
-            children = getattr(space, tag)
-            if children.__class__.__name__ == 'Family':
-                children = [children]
-            if isinstance(children, dict):
-                children = list(children.values())
-            if isinstance(children, list):
-                yield tag, children
-
-    def iter_family(self,
-                    child,
-                    family,
-                    subpath,
-                    ):
-        tag = child.__class__.__name__
-        if tag == 'Variable':
-            function = self.populate_variable
-        elif tag == 'Property':
-            # property already imported with family
-            return
-        else:
-            function = self.populate_family
-        #else:
-        #    raise Exception('unknown tag {}'.format(child.tag))
-        function(family,
-                 child,
-                 subpath,
-                 )
+        if hasattr(self.objectspace.space, 'services'):
+            yield self.objectspace.space.services
 
     def populate_family(self,
                         parent_family,
                         elt,
-                        subpath,
+                        providers,
+                        dynamic,
+                        dynamic_path,
                         ):
-        path = self.build_path(subpath,
-                               elt,
-                               )
-        tag = elt.__class__.__name__
+        """Populate family
+        """
+        self.set_name(elt)
         family = Family(elt,
-                        self.storage,
-                        tag == 'Leadership',
-                        path,
+                        self.text,
+                        self.objectspace,
                         )
+        if not dynamic_path:
+            dynamic_path = elt.name
+        else:
+            dynamic_path = dynamic_path + '.' + elt.name
+        if dynamic or hasattr(elt, 'suffixes'):
+            dynamic_path += '{suffix}'
+            dynamic = True
         parent_family.add(family)
-        for tag, children in self.get_children(elt):
-            for child in children:
-                self.iter_family(child,
-                                 family,
-                                 path,
-                                 )
+        for children in vars(elt).values():
+            if isinstance(children, self.objectspace.family):
+                self.populate_family(family,
+                                     children,
+                                     providers,
+                                     dynamic,
+                                     dynamic_path,
+                                     )
+                continue
+            if isinstance(children, dict):
+                children = list(children.values())
+            if isinstance(children, list):
+                for child in children:
+                    if isinstance(child, self.objectspace.property_) or \
+                            not isinstance(child, RootRougailObject):
+                        continue
+                    if isinstance(child, self.objectspace.variable):
+                        self.set_name(child)
+                        sub_dynamic_path = dynamic_path + '.' + child.name
+                        if dynamic:
+                            sub_dynamic_path += '{suffix}'
+                        family.add(Variable(child,
+                                            self.text,
+                                            self.objectspace,
+                                            providers,
+                                            sub_dynamic_path,
+                                            ))
+                    else:
+                        self.populate_family(family,
+                                             child,
+                                             providers,
+                                             dynamic,
+                                             dynamic_path,
+                                             )
 
-    def populate_variable(self,
-                          family,
-                          elt,
-                          subpath,
-                          ):
-        is_follower = False
-        is_leader = False
-        if family.is_leader:
-            if elt.name != family.elt.name:
-                is_follower = True
-            else:
-                is_leader = True
-        family.add(Variable(elt,
-                            self.storage,
-                            is_follower,
-                            is_leader,
-                            self.build_path(subpath,
-                                            elt,
-                                            )
-                            ))
-
-    def build_path(self,
-                   subpath,
-                   elt,
-                   ):
-        if subpath is None:
-            return elt.name
-        return subpath + '.' + elt.name
-
-    def get_text(self):
-        return '\n'.join(self.storage.get('.').get_text())
-
-
-class BaseElt:
-    def __init__(self,
-                 name,
-                 doc,
+    def set_name(self,
+                 elt,
                  ):
-        self.name = name
-        self.doc = doc
-
-    def __iter__(self):
-        return iter([])
-
-
-class ElementStorage:
-    def __init__(self,
-                 ):
-        self.paths = {}
-        self.text = []
-        self.index = 0
-
-    def add(self, path, elt):
-        self.paths[path] = (elt, self.index)
+        """Set name
+        """
+        elt.reflector_name = f'option_{self.index}'
         self.index += 1
 
-    def get(self, path):
-        if path not in self.paths or self.paths[path][0] is None:
-            raise LoaderError(_('there is no element for path {}').format(path))
-        return self.paths[path][0]
-
-    def get_name(self, path):
-        if path not in self.paths:
-            raise LoaderError(_('there is no element for index {}').format(path))
-        return f'option_{self.paths[path][1]}'
+    def get_text(self):
+        """Get text
+        """
+        self.baseelt.reflector_object.get([])  # pylint: disable=E1101
+        return '\n'.join(self.text)
 
 
 class Common:
+    """Common function for variable and family
+    """
     def __init__(self,
                  elt,
-                 storage,
-                 is_leader,
-                 path,
+                 text,
+                 objectspace,
                  ):
-        self.option_name = None
-        self.path = path
-        self.attrib = {}
-        self.informations = {}
-        self.storage = storage
-        self.is_leader = is_leader
-        self.storage.add(self.path, self)
-
-    def populate_properties(self, child):
-        if child.type == 'calculation':
-            action = f"ParamValue('{child.name}')"
-            option_name = self.storage.get(child.source).get()
-            kwargs = f"'condition': ParamOption({option_name}, todict=True), 'expected': ParamValue('{child.expected}')"
-            if child.inverse:
-                kwargs += ", 'reverse_condition': ParamValue(True)"
-            prop = 'Calculation(calc_value, Params(' + action + ', kwargs={' + kwargs + '}))'
-        else:
-            prop = "'" + child.text + "'"
-        if self.attrib['properties']:
-            self.attrib['properties'] += ', '
-        self.attrib['properties'] += prop
-
-    def get_attrib(self, attrib):
-        ret_list = []
-        for key, value in self.attrib.items():
-            if value is None:
-                continue
-            if key == 'properties':
-                if not self.attrib[key]:
-                    continue
-                value = "frozenset({" + self.attrib[key] + "})"
-            elif key in ['default', 'multi', 'suffixes', 'validators']:
-                value = self.attrib[key]
-            elif isinstance(value, str) and key != 'opt' and not value.startswith('['):
-                value = "'" + value.replace("'", "\\\'") + "'"
-            ret_list.append(f'{key}={value}')
-        return ', '.join(ret_list)
-
-    def populate_informations(self):
-        for key, value in self.informations.items():
-            if isinstance(value, str):
-                value = '"' + value.replace('"', '\"') + '"'
-            self.storage.text.append(f'{self.option_name}.impl_set_information("{key}", {value})')
-
-    def get_text(self,
-                 ):
-        return self.storage.text
-
-    def get_attributes(self, space):  # pylint: disable=R0201
-        attributes = dir(space)
-        for attr in ATTRIBUTES_ORDER:
-            if attr in attributes:
-                yield attr
-        for attr in dir(space):
-            if attr not in ATTRIBUTES_ORDER:
-                if not attr.startswith('_') and attr not in ERASED_ATTRIBUTES:
-                    value = getattr(space, attr)
-                    if not isinstance(value, (list, dict)) and not value.__class__.__name__ == 'Family':
-                        yield attr
-
-    def get_children(self,
-                     space,
-                     ):
-        for attr in dir(space):
-            if not attr.startswith('_') and attr not in ERASED_ATTRIBUTES:
-                value = getattr(space, attr)
-                if isinstance(value, (list, dict)):
-                    children = getattr(space, attr)
-                    if children.__class__.__name__ == 'Family':
-                        children = [children]
-                    if isinstance(children, dict):
-                        children = list(children.values())
-                    if children and isinstance(children, list):
-                        yield attr, children
-
-
-class Variable(Common):
-    def __init__(self,
-                 elt,
-                 storage,
-                 is_follower,
-                 is_leader,
-                 path,
-                 ):
-        super().__init__(elt,
-                         storage,
-                         is_leader,
-                         path,
-                         )
-        self.is_follower = is_follower
-        convert_option = CONVERT_OPTION[elt.type]
-        del elt.type
-        self.object_type = convert_option['opttype']
-        self.attrib.update(convert_option.get('initkwargs', {}))
-        if self.object_type != 'SymLinkOption':
-            self.attrib['properties'] = []
-            self.attrib['validators'] = []
         self.elt = elt
+        self.option_name = None
+        self.text = text
+        self.objectspace = objectspace
+        self.elt.reflector_object = self
+        self.object_type = None
 
-    def get(self):
+    def get(self, calls):
+        """Get tiramisu's object
+        """
+        self_calls = calls.copy()
+        if self.elt.path in self_calls:
+            msg = f'"{self.elt.path}" will make an infinite loop'
+            raise DictConsistencyError(msg, 80, self.elt.xmlfiles)
+        self_calls.append(self.elt.path)
+        self.calls = self_calls
         if self.option_name is None:
+            self.option_name = self.elt.reflector_name
             self.populate_attrib()
-            if self.object_type == 'SymLinkOption':
-                self.attrib['opt'] = self.storage.get(self.attrib['opt']).get()
-            else:
-                self.parse_children()
-            attrib = self.get_attrib(self.attrib)
-            self.option_name = self.storage.get_name(self.path)
-            self.storage.text.append(f'{self.option_name} = {self.object_type}({attrib})')
             self.populate_informations()
+            if hasattr(self.elt, 'provider'):
+                name = 'provider:' + self.elt.provider
+                if name in self.providers:
+                    msg = f'provider {name} declare multiple time'
+                    raise DictConsistencyError(msg, 79, self.elt.xmlfiles)
+                self.providers[name] = self.dynamic_path
         return self.option_name
 
     def populate_attrib(self):
-        for key in self.get_attributes(self.elt):
-            value = getattr(self.elt, key)
-            if key in FORCE_INFORMATIONS:
-                if key == 'test':
-                    value = value.split(',')
-                    if self.object_type == 'IntOption':
-                        value = [int(v) for v in value]
-                self.informations[key] = value
-            else:
-                self.attrib[key] = value
+        """Populate attributes
+        """
+        keys = {'name': self.convert_str(self.elt.name)}
+        if hasattr(self.elt, 'doc'):
+            keys['doc'] = self.convert_str(self.elt.doc)
+        self._populate_attrib(keys)
+        if hasattr(self.elt, 'properties'):
+            keys['properties'] = self.properties_to_string(self.elt.properties)
+        attrib = ', '.join([f'{key}={value}' for key, value in keys.items()])
+        self.text.append(f'{self.option_name} = {self.object_type}({attrib})')
 
-    def parse_children(self):
-        if not 'default' in self.attrib or self.attrib['multi']:
-            self.attrib['default'] = []
-        if self.attrib['multi'] == 'submulti' and self.is_follower:
-            self.attrib['default_multi'] = []
-        choices = []
-        if 'properties' in self.attrib:
-            if self.attrib['properties']:
-                self.attrib['properties'] = "'" + "', '".join(sorted(list(self.attrib['properties']))) + "'"
-            else:
-                self.attrib['properties'] = ''
-        for tag, children in self.get_children(self.elt):
-            for child in children:
-                if tag == 'property':
-                    self.populate_properties(child)
-                elif tag == 'value':
-                    if child.type == 'calculation':
-                        self.attrib['default'] = self.calculation_value(child, [])
-                    else:
-                        self.populate_value(child)
-                elif tag == 'check':
-                    self.attrib['validators'].append(self.calculation_value(child, ['ParamSelfOption()']))
-                elif tag == 'choice':
-                    choices.append(child.name)
-        if choices:
-            self.attrib['values'] = tuple(choices)
-        if self.attrib['default'] == []:
-            del self.attrib['default']
-        elif not self.attrib['multi'] and isinstance(self.attrib['default'], list):
-            self.attrib['default'] = self.attrib['default'][-1]
-        if self.attrib['validators'] == []:
-            del self.attrib['validators']
-        else:
-            self.attrib['validators'] = '[' + ', '.join(self.attrib['validators']) + ']'
+    def _populate_attrib(self,
+                         keys: dict,
+                         ) -> None:  # pragma: no cover
+        raise NotImplementedError()
 
-    def calculation_value(self, child, args):
-        kwargs = []
-        if hasattr(child, 'name'):
-            # has parameters
-            function = child.name
-            if hasattr(child, 'param'):
-                for param in child.param:
-                    value = self.populate_param(param)
-                    if not hasattr(param, 'name'):
-                        args.append(str(value))
-                    else:
-                        kwargs.append(f"'{param.name}': " + value)
+    @staticmethod
+    def convert_str(value):
+        """convert string
+        """
+        return dumps(value, ensure_ascii=False)
+
+    def properties_to_string(self,
+                             values: list,
+                             ) -> None:
+        """Change properties to string
+        """
+        properties = [self.convert_str(property_) for property_ in values
+                      if isinstance(property_, str)]
+        calc_properties = [self.calc_properties(property_) for property_ in values \
+                if isinstance(property_, self.objectspace.property_)]
+        return 'frozenset({' + ', '.join(sorted(properties) + calc_properties) + '})'
+
+    def calc_properties(self,
+                        child,
+                        ) -> str:
+        """Populate properties
+        """
+        option_name = child.source.reflector_object.get(self.calls)
+        kwargs = (f"'condition': ParamOption({option_name}, todict=True, notraisepropertyerror=True), "
+                  f"'expected': {self.populate_param(child.expected)}")
+        if child.inverse:
+            kwargs += ", 'reverse_condition': ParamValue(True)"
+        return (f"Calculation(func.calc_value, Params(ParamValue('{child.name}'), "
+                f"kwargs={{{kwargs}}}))")
+
+    def populate_informations(self):
+        """Populate Tiramisu's informations
+        """
+        if not hasattr(self.elt, 'information'):
+            return
+        if isinstance(self.elt.information, dict):
+            informations = self.elt.information
         else:
-            # function without any parameter
-            function = child.text.strip()
-        ret = f"Calculation(func.{function}, Params((" + ', '.join(args) + "), kwargs=" + "{" + ', '.join(kwargs) + "})"
-        if hasattr(child, 'warnings_only'):
-            ret += f', warnings_only={child.warnings_only}'
-        return ret + ')'
+            informations = vars(self.elt.information)
+        for key, value in informations.items():
+            if key == 'xmlfiles':
+                continue
+            if isinstance(value, str):
+                value = self.convert_str(value)
+            self.text.append(f"{self.option_name}.impl_set_information('{key}', {value})")
 
     def populate_param(self,
                        param,
                        ):
-        if param.type == 'string':
-            return f'ParamValue("{param.text}")'
-        elif param.type == 'number':
-            return f'ParamValue({param.text})'
-        elif param.type == 'variable':
-            value = {'option': param.text,
-                     'notraisepropertyerror': param.notraisepropertyerror,
-                     'todict': param.text in FUNC_TO_DICT,
-                     }
-            if hasattr(param, 'suffix'):
-                value['suffix'] = param.suffix
-            return self.build_param(value)
-        raise LoaderError(_('unknown param type {}').format(param.type))
+        """Populate variable parameters
+        """
+        if param.type in ['number', 'boolean', 'nil', 'string', 'port', 'choice', 'space']:
+            value = param.text
+            if param.type == 'string' and value is not None:
+                value = self.convert_str(value)
+            return f'ParamValue({value})'
+        if param.type == 'variable':
+            return self.build_option_param(param)
+        if param.type == 'information':
+            if hasattr(self.elt, 'multi') and self.elt.multi:
+                default = []
+            else:
+                default = None
+            return f'ParamInformation("{param.text}", {default})'
+        if param.type == 'target_information':
+            return f'ParamSelfInformation("{param.text}", None)'
+        if param.type == 'suffix':
+            return 'ParamSuffix()'
+        if param.type == 'index':
+            return 'ParamIndex()'
+        raise Exception(_(f'unknown type {param.type}'))  # pragma: no cover
 
-    def populate_value(self,
-                       child,
-                       ):
-        value = child.name
-        if self.attrib['multi'] == 'submulti':
-            self.attrib['default_multi'].append(value)
-        elif self.is_follower:
-            self.attrib['default_multi'] = value
-        elif self.attrib['multi']:
-            self.attrib['default'].append(value)
-            if not self.is_leader:
-                self.attrib['default_multi'] = value
-        elif isinstance(value, int):
-            self.attrib['default'].append(value)
+    def build_option_param(self,
+                           param,
+                           ) -> str:
+        """build variable parameters
+        """
+        option_name = param.text.reflector_object.get(self.calls)
+        params = [f'{option_name}']
+        if hasattr(param, 'suffix'):
+            param_type = 'ParamDynOption'
+            family = param.family.reflector_object.get(self.calls)
+            params.extend([f"'{param.suffix}'", f'{family}'])
         else:
-            self.attrib['default'].append("'" + value + "'")
+            param_type = 'ParamOption'
+        if not param.propertyerror:
+            params.append('notraisepropertyerror=True')
+        return "{}({})".format(param_type, ', '.join(params))
 
-    def build_param(self,
-                    param,
-                    ):
-        option_name = self.storage.get(param['option']).get()
-        if 'suffix' in param:
-            family = '.'.join(param['option'].split('.')[:-1])
-            family_option = self.storage.get_name(family)
-            return f"ParamDynOption({option_name}, '{param['suffix']}', {family_option}, notraisepropertyerror={param['notraisepropertyerror']}, todict={param['todict']})"
-        return f"ParamOption({option_name}, notraisepropertyerror={param['notraisepropertyerror']}, todict={param['todict']})"
+
+class Variable(Common):
+    """Manage variable
+    """
+    def __init__(self,
+                 elt,
+                 text,
+                 objectspace,
+                 providers,
+                 dynamic_path,
+                 ):
+        self.providers = providers
+        self.dynamic_path = dynamic_path
+        super().__init__(elt, text, objectspace)
+        self.object_type = CONVERT_OPTION[elt.type]['opttype']
+
+    def _populate_attrib(self,
+                         keys: dict,
+                         ):
+        if hasattr(self.elt, 'opt'):
+            keys['opt'] = self.elt.opt.reflector_object.get(self.calls)
+        if hasattr(self.elt, 'choice'):
+            values = self.elt.choice
+            if values[0].type == 'variable':
+                value = values[0].name.reflector_object.get(self.calls)
+                keys['values'] = f"Calculation(func.calc_value, Params((ParamOption({value}))))"
+            elif values[0].type == 'function':
+                keys['values'] = self.calculation_value(values[0], [])
+            else:
+                keys['values'] = str(tuple([val.name for val in values]))
+        if hasattr(self.elt, 'multi') and self.elt.multi:
+            keys['multi'] = self.elt.multi
+        for key in ['default', 'default_multi']:
+            if hasattr(self.elt, key) and getattr(self.elt, key) is not None:
+                value = getattr(self.elt, key)
+                if isinstance(value, str):
+                    value = self.convert_str(value)
+                elif isinstance(value, self.objectspace.value):
+                    value = self.calculation_value(value, [], calc_multi=value.calc_multi)
+                keys[key] = value
+        if hasattr(self.elt, 'validators'):
+            keys['validators'] = '[' + ', '.join([self.calculation_value(val,
+                                    ['ParamSelfOption(whole=False)']) for val in self.elt.validators]) + ']'
+        for key in ['min_number', 'max_number']:
+            if hasattr(self.elt, key):
+                keys[key] = getattr(self.elt, key)
+        for key, value in CONVERT_OPTION[self.elt.type].get('initkwargs', {}).items():
+            if isinstance(value, str):
+                value = f"'{value}'"
+            keys[key] = value
+
+    def calculation_value(self,
+                          child,
+                          args,
+                          calc_multi=False,
+                          ) -> str:
+        """Generate calculated value
+        """
+        kwargs = []
+        # has parameters
+        function = child.name
+        new_args = []
+        if hasattr(child, 'param'):
+            for param in child.param:
+                value = self.populate_param(param)
+                if not hasattr(param, 'name'):
+                    new_args.append(str(value))
+                else:
+                    kwargs.append(f"'{param.name}': " + value)
+        if function == 'valid_network_netmask':
+            new_args.extend(args)
+        else:
+            args.extend(new_args)
+            new_args = args
+        ret = f'Calculation(func.{function}, Params((' + ', '.join(new_args) + ')'
+        if kwargs:
+            ret += ', kwargs={' + ', '.join(kwargs) + '}'
+        ret += ')'
+        if hasattr(child, 'warnings_only'):
+            ret += f', warnings_only={child.warnings_only}'
+        ret = ret + ')'
+        if calc_multi:
+            ret = '[' + ret + ']'
+        return ret
 
 
 class Family(Common):
+    """Manage family
+    """
     def __init__(self,
                  elt,
-                 storage,
-                 is_leader,
-                 path,
+                 text,
+                 objectspace,
                  ):
-        super().__init__(elt,
-                         storage,
-                         is_leader,
-                         path,
-                         )
+        super().__init__(elt, text, objectspace)
+        if hasattr(self.elt, 'suffixes'):
+            self.objectspace.has_dyn_option = True
+            self.object_type = 'ConvertDynOptionDescription'
+        elif hasattr(self.elt, 'leadership') and self.elt.leadership:
+            self.object_type = 'Leadership'
+        else:
+            self.object_type = 'OptionDescription'
         self.children = []
-        self.elt = elt
 
     def add(self, child):
+        """Add a child
+        """
         self.children.append(child)
 
-    def get(self):
-        if not self.option_name:
-            self.populate_attrib()
-            self.parse_children()
-            self.option_name = self.storage.get_name(self.path)
-            object_name = self.get_object_name()
-            attrib = self.get_attrib(self.attrib) + ', children=[' + ', '.join([child.get() for child in self.children]) + ']'
-            self.storage.text.append(f'{self.option_name} = {object_name}({attrib})')
-            self.populate_informations()
-        return self.option_name
-
-    def populate_attrib(self):
-        for key in self.get_attributes(self.elt):
-            value = getattr(self.elt, key)
-            if key in FORCE_INFORMATIONS:
-                self.informations[key] = value
-            elif key == 'dynamic':
-                dynamic = self.storage.get(value).get()
-                self.attrib['suffixes'] = f"Calculation(func.calc_value, Params((ParamOption({dynamic}))))"
-            else:
-                self.attrib[key] = value
-
-    def parse_children(self):
-        if 'properties' in self.attrib:
-            self.attrib['properties'] = "'" + "', '".join(sorted(list(self.attrib['properties']))) + "'"
-        if hasattr(self.elt, 'property'):
-            #self.attrib['properties'] = ''
-            for child in self.elt.property:
-                self.populate_properties(child)
-            if not self.attrib['properties']:
-                del self.attrib['properties']
-
-    def get_object_name(self):
-        if 'suffixes' in self.attrib:
-            return 'ConvertDynOptionDescription'
-        elif not self.is_leader:
-            return 'OptionDescription'
-        return 'Leadership'
+    def _populate_attrib(self,
+                         keys: list,
+                         ) -> None:
+        if hasattr(self.elt, 'suffixes'):
+            dyn = self.elt.suffixes.reflector_object.get(self.calls)
+            keys['suffixes'] = f"Calculation(func.calc_value, Params((ParamOption({dyn}, notraisepropertyerror=True))))"
+        keys['children'] = '[' + ', '.join([child.get(self.calls) for child in self.children]) + ']'

src/rougail/update.py

@@ -0,0 +1,761 @@
+"""Update Rougail XML file to new version
+
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+
+from typing import List
+from os.path import join, isfile, basename
+from os import listdir
+from lxml.etree import DTD, parse, XMLParser, XMLSyntaxError  # pylint: disable=E0611
+from lxml.etree import Element, SubElement, tostring
+from ast import parse as ast_parse
+
+from .i18n import _
+from .error import UpgradeError
+
+from .utils import normalize_family
+from .config import RougailConfig
+
+
+VERSIONS = {'creole': ['1'],
+            'rougail': ['0.9', '0.10'],
+            }
+
+
+def get_function_name(root, version):
+    version = version.replace('.', '_')
+    return f'update_{root}_{version}'
+
+
+FUNCTION_VERSIONS = [(root, version, get_function_name(root, version)) for root, versions in VERSIONS.items() for version in versions]
+
+
+class RougailUpgrade:
+    def __init__(self,
+                 test=False,
+                 upgrade_help=None,
+                 rougailconfig: RougailConfig=None,
+                 ) -> None:
+        self.test = test
+        if upgrade_help is None:
+            upgrade_help = {}
+        self.upgrade_help = upgrade_help
+        if rougailconfig is None:
+            rougailconfig = RougailConfig
+        self.rougailconfig = rougailconfig
+
+    def load_xml_from_folders(self,
+                              srcfolder: str,
+                              dstfolder: str,
+                              namespace: str,
+                              display: bool=True,
+                              ):
+        """Loads all the XML files located in the xmlfolders' list
+
+        :param xmlfolders: list of full folder's name
+        """
+        filenames = [filename for filename in listdir(srcfolder) if filename.endswith('.xml')]
+        filenames.sort()
+        for filename in filenames:
+            xmlsrc = join(srcfolder, filename)
+            xmldst = join(dstfolder, filename)
+            if isfile(xmldst):
+                raise Exception(f'cannot update "{xmlsrc}" destination file "{xmldst}" already exists')
+            try:
+                parser = XMLParser(remove_blank_text=True)
+                document = parse(xmlsrc, parser)
+            except XMLSyntaxError as err:
+                raise Exception(_(f'not a XML file: {err}')) from err
+            root = document.getroot()
+            search_function_name = get_function_name(root.tag, root.attrib.get('version', '1'))
+            function_found = False
+            for root_name, version, function_version in FUNCTION_VERSIONS:
+                if function_found and hasattr(self, function_version):
+                    if display:
+                        print(f'  - convert {filename} to version {version}')
+                    upgrade_help = self.upgrade_help.get(function_version, {}).get(filename, {})
+                    if upgrade_help.get('remove') is True:
+                        continue
+                    root = getattr(self, function_version)(root, upgrade_help, namespace)
+                if function_version == search_function_name:
+                    function_found = True
+                root.attrib['version'] = version
+            with open(xmldst, 'wb') as xmlfh:
+                xmlfh.write(tostring(root, pretty_print=True, encoding="UTF-8", xml_declaration=True))
+    #            if
+    #            if not self.dtd.validate(document):
+    #                dtd_error = self.dtd.error_log.filter_from_errors()[0]
+    #                msg = _(f'not a valid XML file: {dtd_error}')
+    #                raise DictConsistencyError(msg, 43, [xmlfile])
+    #            yield xmlfile, document.getroot()
+
+
+    def update_rougail_0_10(self,
+                           root: 'Element',
+                           upgrade_help: dict,
+                           namespace: str,
+                           ) -> 'Element':
+        variables = root.find('variables')
+        if variables is None:
+            return root
+        paths = self._get_path_variables(variables,
+                                         namespace == 'configuration',
+                                         namespace,
+                                         )
+        constraints = root.find('constraints')
+        # convert schedule and schedulemod
+        for variable in paths.values():
+            variable = variable['variable']
+            if variable.tag != 'variable':
+                continue
+            if 'type' in variable.attrib and variable.attrib['type'] in ['schedule', 'schedulemod']:
+                if variable.attrib['type'] == 'schedule':
+                    choices = ('none', 'daily', 'weekly', 'monthly')
+                else:
+                    choices = ('pre', 'post')
+                variable.attrib['type'] = 'choice'
+                has_value = False
+                for value in variable:
+                    if value.tag == 'value':
+                        has_value = True
+                        break
+                for name in choices:
+                    choice = SubElement(variable, 'choice')
+                    choice.text = name
+                if not has_value:
+                    value = SubElement(variable, 'value')
+                    value.text = choices[0]
+                variable.attrib['mandatory'] = 'True'
+
+        # convert group to leadership
+        groups = []
+        if constraints is not None:
+            for constraint in constraints:
+                if constraint.tag == 'group':
+                    constraints.remove(constraint)
+                    groups.append(constraint)
+        for group in groups:
+            if group.attrib['leader'] in paths:
+                leader_obj = paths[group.attrib['leader']]
+                #FIXME name peut avoir "." il faut le virer
+                #FIXME si extra c'est un follower !
+                if 'name' in group.attrib:
+                    grpname = group.attrib['name']
+                    if 'description' in group.attrib:
+                        description = group.attrib['description']
+                    else:
+                        description = grpname
+                else:
+                    grpname = leader_obj['variable'].attrib['name']
+                    if '.' in grpname:
+                        grpname = grpname.rsplit('.', 1)[-1]
+                    if 'description' in group.attrib:
+                        description = group.attrib['description']
+                    elif 'description' in leader_obj['variable'].attrib:
+                        description = leader_obj['variable'].attrib['description']
+                    else:
+                        description = grpname
+                family = SubElement(leader_obj['parent'], 'family', name=grpname, description=description, leadership="True")
+                leader_obj['parent'].remove(leader_obj['variable'])
+                family.append(leader_obj['variable'])
+            else:
+                # append in group
+                follower = next(iter(group))
+                leader_name = group.attrib['leader']
+                if '.' in leader_name:
+                    leader_path = leader_name.rsplit('.', 1)[0]
+                    follower_path = leader_path + '.' + follower.text
+                else:
+                    follower_path = follower.text
+                obj = paths[follower_path]
+                family = SubElement(obj['parent'], 'family', name=leader_name, leadership="True")
+                grpname = leader_name
+            for follower in group:
+                leader_name = group.attrib['leader']
+                if '.' in leader_name:
+                    leader_path = leader_name.rsplit('.', 1)[0]
+                    follower_path = leader_path + '.' + follower.text
+                else:
+                    follower_path = follower.text
+                follower_obj = paths[follower_path]
+                follower_obj['parent'].remove(follower_obj['variable'])
+                family.append(follower_obj['variable'])
+                if '.' in follower_path:
+                    new_path = follower_path.rsplit('.', 1)[0] + '.' + grpname + '.' + follower_path.rsplit('.', 1)[1]
+                    paths[new_path] = paths[follower_path]
+
+        # convert choice option
+        valid_enums = []
+        if constraints is not None:
+            for constraint in constraints:
+                if constraint.tag == 'check' and constraint.attrib['name'] == 'valid_enum':
+                    constraints.remove(constraint)
+                    valid_enums.append(constraint)
+        for valid_enum in valid_enums:
+            targets = []
+            for target in valid_enum:
+                if target.tag != 'target':
+                    continue
+                if target.text in paths:
+                    # not in paths if it's optional
+                    # but not check it
+                    targets.append(paths[target.text]['variable'])
+            params = []
+            function_param = None
+            for param in valid_enum:
+                if param.tag != 'param':
+                    continue
+                if 'type' in param.attrib and param.attrib['type'] == 'function':
+                    function_param = param.text
+                    continue
+                params.append(param)
+            first_choice = None
+            for target in targets:
+                if function_param is not None:
+                    function = SubElement(target, 'choice', type='function', name=function_param)
+                for param in params:
+                    if function_param is not None:
+                        function.append(param)
+                    else:
+                        choice = SubElement(target, 'choice')
+                        if first_choice is None and ('type' not in param.attrib or param.attrib['type'] != 'variable'):
+                            first_choice = choice
+                        choice.text = param.text
+                        if 'type' not in param.attrib and param.text is None:
+                            choice_type = 'nil'
+                        elif 'type' in param.attrib:
+                            choice_type = param.attrib['type']
+                        elif 'type' in target.attrib:
+                            choice_type = target.attrib['type']
+                        else:
+                            choice_type = 'string'
+                        choice.attrib['type'] = choice_type
+            has_value = False
+            for target in targets:
+                if 'remove_check' in target.attrib:
+                    target.attrib['remove_choice'] = target.attrib['remove_check']
+            for target in targets:
+                for value in target:
+                    if value.tag == 'value':
+                        has_value = True
+                        if 'type' in target.attrib:
+                            value.attrib['type'] = target.attrib['type']
+            if first_choice is not None and not has_value:
+                value = SubElement(target, 'value')
+                value.attrib['type'] = first_choice.attrib['type']
+                value.text = first_choice.text
+            for target in targets:
+                if 'remove_choice' not in target.attrib or target.attrib['remove_choice'] != 'True':
+                    target.attrib['type'] = 'choice'
+        return root
+
+    def _get_path_variables(self, variables, is_variable_namespace, path, dico=None):
+        if dico is None:
+            dico = {}
+        for variable in variables:
+            if not is_variable_namespace and path:
+                subpath = path + '.'
+            else:
+                subpath = ''
+            if variable.tag not in ['variable', 'family']:
+                continue
+            subpath += variable.attrib['name']
+            if variable.tag == 'family':
+                self._get_path_variables(variable, is_variable_namespace, subpath, dico)
+            elif variable.tag == 'variable':
+                dico[subpath] = {'variable': variable, 'parent': variables}
+        return dico
+
+    def update_rougail_0_9(self,
+                           root: 'Element',
+                           upgrade_help: dict,
+                           namespace: str,
+                           ) -> 'Element':
+        # rename root
+        root.tag = 'rougail'
+        variables_auto_valid_enum = {}
+        variables_help = {}
+        families_help = {}
+        variables = {}
+        families = {}
+        valid_enums = {}
+        current_service = upgrade_help.get('services', {}).get('default', 'unknown')
+        files = {current_service: {}}
+        ips = {current_service: []}
+        servicelists = {}
+        test_unknowns_variables = set()
+        autos = []
+        constraints_obj = None
+        for subelement in root:
+            if not isinstance(subelement.tag, str):
+                # XML comment
+                continue
+            if subelement.tag == 'family_action':
+                root.remove(subelement)
+                continue
+            for subsubelement in subelement:
+                if not isinstance(subsubelement.tag, str):
+                    # XML comment
+                    continue
+                for subsubsubelement in subsubelement:
+                    if not isinstance(subsubsubelement.tag, str):
+                        # XML comment
+                        continue
+                    if subsubsubelement.tag == 'variable':
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('remove', []):
+                            subsubelement.remove(subsubsubelement)
+                            continue
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('type', {}):
+                            subsubsubelement.attrib['type'] = upgrade_help['variables']['type'][subsubsubelement.attrib['name']]
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('hidden', {}).get('add', []):
+                            subsubsubelement.attrib['hidden'] = 'True'
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('hidden', {}).get('remove', []):
+                            self.remove(subsubsubelement, 'hidden', optional=True)
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mandatory', {}).get('remove', []):
+                            self.remove(subsubsubelement, 'mandatory')
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mandatory', {}).get('add', []):
+                            subsubsubelement.attrib['mandatory'] = 'True'
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('type', {}):
+                            subsubsubelement.attrib['type'] = upgrade_help.get('variables', {}).get('type', {})[subsubsubelement.attrib['name']]
+                        if namespace == 'configuration':
+                            path = subsubsubelement.attrib['name']
+                            npath = normalize_family(subsubsubelement.attrib['name'])
+                        else:
+                            path = namespace + '.' + subsubelement.attrib['name'] + '.' + subsubsubelement.attrib['name']
+                            npath = normalize_family(namespace) + '.' + normalize_family(subsubelement.attrib['name']) + '.' + normalize_family(subsubsubelement.attrib['name'])
+                        variables[path] = subsubsubelement
+                        variables[npath] = subsubsubelement
+                        type = subsubsubelement.attrib.get('type')
+                        if type in ['oui/non', 'yes/no', 'on/off']:
+                            variables_auto_valid_enum.setdefault(subsubsubelement.attrib['type'], []).append(path)
+                            del subsubsubelement.attrib['type']
+                        elif type == 'hostname_strict':
+                            subsubsubelement.attrib['type'] = 'hostname'
+                        elif type in ['domain', 'domain_strict']:
+                            subsubsubelement.attrib['type'] = 'domainname'
+                        elif type == 'string':
+                            del subsubsubelement.attrib['type']
+                        if self.test and subsubsubelement.attrib.get('auto_freeze') == 'True':
+                            del subsubsubelement.attrib['auto_freeze']
+                        #if self.test and subsubsubelement.attrib.get('redefine') == 'True':
+                        #    subsubsubelement.attrib['exists'] = 'False'
+                        #    del subsubsubelement.attrib['redefine']
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('redefine', {}).get('remove', {}):
+                            del subsubsubelement.attrib['redefine']
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('remove_check', []):
+                            subsubsubelement.attrib['remove_check'] = 'True'
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('mode', {}).get('modify', {}):
+                            subsubsubelement.attrib['mode'] = upgrade_help.get('variables', {}).get('mode', {}).get('modify', {})[subsubsubelement.attrib['name']]
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('type', {}).get('modify', {}):
+                            subsubsubelement.attrib['type'] = upgrade_help.get('variables', {}).get('type', {}).get('modify', {})[subsubsubelement.attrib['name']]
+                            type = subsubsubelement.attrib['type']
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('test', {}):
+                            subsubsubelement.attrib['test'] = upgrade_help.get('variables', {}).get('test', {})[subsubsubelement.attrib['name']]
+                        if subsubsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('remove_value', []):
+                            for value in subsubsubelement:
+                                subsubsubelement.remove(value)
+                        if subsubsubelement.attrib['name'] != normalize_family(subsubsubelement.attrib['name']):
+                            if "description" not in subsubsubelement.attrib:
+                                subsubsubelement.attrib['description'] = subsubsubelement.attrib['name']
+                            subsubsubelement.attrib['name'] = normalize_family(subsubsubelement.attrib['name'])
+                    elif subsubsubelement.tag == 'param':
+                        if subsubelement.tag == 'check' and subsubelement.attrib['target'] in upgrade_help.get('check', {}).get('remove', []):
+                            continue
+                        type = subsubsubelement.attrib.get('type')
+                        if type == 'eole':
+                            subsubsubelement.attrib['type'] = 'variable'
+                            type = 'variable'
+                        if type == 'python':
+                            subsubsubelement.attrib['type'] = 'function'
+                            if subsubsubelement.text.startswith('range('):
+                                func_ast = ast_parse(subsubsubelement.text)
+                                subsubsubelement.text = 'range'
+                                for arg in func_ast.body[0].value.args:
+                                    SubElement(subsubelement, 'param', type='number').text = str(arg.value)
+                            else:
+                                raise Exception(f'{subsubsubelement.text} is not a supported function')
+                            type = 'function'
+                        elif type in ('container', 'context'):
+                            raise UpgradeError(_(f'cannot convert param with type "{type}"'))
+                        if subsubelement.attrib['name'] == 'valid_entier' and not 'type' in subsubsubelement.attrib:
+                            subsubsubelement.attrib['type'] = 'number'
+                        if (not type or type == 'variable') and subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []):
+                            subsubelement.remove(subsubsubelement)
+                            continue
+                        if subsubelement.attrib['name'] == 'valid_enum' and not type:
+                            if subsubsubelement.attrib.get('name') == 'checkval':
+                                if subsubelement.attrib['target'] in upgrade_help.get('check', {}).get('valid_enums', {}).get('checkval', {}).get('remove', []):
+                                    subsubelement.remove(subsubsubelement)
+                                    continue
+                                raise UpgradeError(_('checkval in valid_enum is no more supported'))
+                            if subsubsubelement.text.startswith('['):
+                                for val in eval(subsubsubelement.text):
+                                    SubElement(subsubelement, 'param').text = str(val)
+                                subsubelement.remove(subsubsubelement)
+                        self.move(subsubsubelement, 'hidden', 'propertyerror', optional=True)
+                        if type == 'variable' and subsubsubelement.text in upgrade_help.get('variables', {}).get('rename', []):
+                            subsubsubelement.text = upgrade_help.get('variables', {}).get('rename', [])[subsubsubelement.text]
+                        up = upgrade_help.get('params', {}).get(subsubsubelement.text)
+                        if up:
+                            if 'text' in up:
+                                subsubsubelement.text = up['text']
+                            if 'type' in up:
+                                subsubsubelement.attrib['type'] = up['type']
+                    elif subsubsubelement.tag == 'target':
+                        type = subsubsubelement.attrib.get('type')
+                        if type in ['service_accesslist', 'service_restrictionlist', 'interfacelist', 'hostlist']:
+                            subsubelement.remove(subsubsubelement)
+                        elif (not type or type == 'variable') and subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []):
+                            subsubelement.remove(subsubsubelement)
+                        elif type == 'family' and subsubsubelement.text in upgrade_help.get('families', {}).get('remove', []):
+                            subsubelement.remove(subsubsubelement)
+                        elif type == 'actionlist':
+#                            for family in root.find('variables'):
+#                                family_list = SubElement(subsubelement, 'target')
+#                                family_list.attrib['type'] = 'family'
+#                                family_list.text = namespace + '.' + family.attrib['name']
+                            subsubelement.remove(subsubsubelement)
+                        if upgrade_help.get('targets', {}).get(subsubsubelement.text, {}).get('optional'):
+                            subsubsubelement.attrib['optional'] = upgrade_help.get('targets', {}).get(subsubsubelement.text, {}).get('optional')
+                        has_target = False
+                        for target in subsubelement:
+                            if target.tag == 'target':
+                                has_target = True
+                                break
+                        if not has_target:
+                            subelement.remove(subsubelement)
+                        continue
+                    elif subsubsubelement.tag == 'slave':
+                        if subsubsubelement.text in upgrade_help.get('variables', {}).get('remove', []):
+                            subsubelement.remove(subsubsubelement)
+                            continue
+                        subsubsubelement.tag = 'follower'
+                        subsubsubelement.text = normalize_family(subsubsubelement.text)
+                    if subelement.tag == 'containers':
+                        current_service = self.upgrade_container(subsubsubelement, current_service, files, ips, servicelists, upgrade_help)
+                        subsubelement.remove(subsubsubelement)
+                if subelement.tag == 'help':
+                    if subsubelement.tag == 'variable':
+                        if not subsubelement.attrib['name'] in upgrade_help.get('variables', {}).get('remove', []):
+                            variables_help[subsubelement.attrib['name']] = subsubelement.text
+                    elif subsubelement.tag == 'family':
+                        if subsubelement.attrib['name'] not in upgrade_help.get('families', {}).get('remove', []):
+                            families_help[subsubelement.attrib['name']] = subsubelement.text
+                    else:
+                        raise Exception(f'unknown help tag {subsubelement.tag}')
+                    subelement.remove(subsubelement)
+                    continue
+                if subsubelement.tag == 'auto':
+                    if subsubelement.attrib['target'] in upgrade_help.get('variables', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                        continue
+                    autos.append(subsubelement.attrib['target'])
+                    subsubelement.tag = 'fill'
+                if subsubelement.tag in ['fill', 'check']:
+                    if subsubelement.tag == 'check' and subsubelement.attrib['name'] == 'valid_enum':
+                        if subsubelement.attrib['target'] in upgrade_help.get('variables', {}).get('remove', []):
+                            subelement.remove(subsubelement)
+                            continue
+                        params = []
+                        for param in subsubelement:
+                            if param.tag == 'param':
+                                params.append(param.text)
+                        key = '~'.join(params)
+                        if key in valid_enums:
+                            SubElement(valid_enums[key], 'target').text = subsubelement.attrib['target']
+                            subelement.remove(subsubelement)
+                            continue
+                        valid_enums['~'.join(params)] = subsubelement
+                    if subsubelement.tag == 'fill' and subsubelement.attrib['target'] in upgrade_help.get('fills', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                        continue
+                    if subsubelement.tag == 'check' and subsubelement.attrib['target'] in upgrade_help.get('check', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                        continue
+                    if subsubelement.attrib['target'] in upgrade_help.get('variables', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                        continue
+                    if subsubelement.attrib['name'] == 'valid_networknetmask':
+                        subsubelement.attrib['name'] = 'valid_network_netmask'
+                    target = SubElement(subsubelement, 'target')
+                    target.text = subsubelement.attrib['target']
+                    if self.test:
+                        target.attrib['optional'] = 'True'
+                    del subsubelement.attrib['target']
+                elif subsubelement.tag == 'separators':
+                    subelement.remove(subsubelement)
+                elif subsubelement.tag == 'condition':
+                    if subsubelement.attrib['source'] in upgrade_help.get('variables', {}).get('remove', []):
+                        try:
+                            subelement.remove(subsubelement)
+                        except:
+                            pass
+                    else:
+                        if subsubelement.attrib['name'] == 'hidden_if_not_in':
+                            subsubelement.attrib['name'] = 'disabled_if_not_in'
+                        if subsubelement.attrib['name'] == 'hidden_if_in':
+                            subsubelement.attrib['name'] = 'disabled_if_in'
+                        if subsubelement.attrib['name'] == 'frozen_if_in':
+                            subsubelement.attrib['name'] = 'hidden_if_in'
+                        self.move(subsubelement, 'fallback', 'optional', optional='True')
+                    if subsubelement.attrib['source'] in upgrade_help.get('variables', {}).get('rename', []):
+                        subsubelement.attrib['source'] = upgrade_help.get('variables', {}).get('rename', [])[subsubelement.attrib['source']]
+                elif subsubelement.tag == 'group':
+                    if subsubelement.attrib['master'] in upgrade_help.get('groups', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                    else:
+                        self.move(subsubelement, 'master', 'leader')
+                        for follower in subsubelement:
+                            if '.' in subsubelement.attrib['leader']:
+                                path = subsubelement.attrib['leader'].rsplit('.', 1)[0] + '.' + follower.text
+                            else:
+                                path = follower.text
+                            self.remove(variables[path], 'multi', optional=True)
+                        if subsubelement.attrib['leader'] in upgrade_help.get('groups', {}).get('reorder', {}):
+                            for follower in subsubelement:
+                                subsubelement.remove(follower)
+                            for follower in upgrade_help.get('groups', {}).get('reorder', {})[subsubelement.attrib['leader']]:
+                                SubElement(subsubelement, 'follower').text = follower
+                if subelement.tag == 'files':
+                    current_service = self.upgrade_container(subsubelement, current_service, files, ips, servicelists, upgrade_help)
+                    subelement.remove(subsubelement)
+                if subsubelement.tag == 'family':
+                    self.remove(subsubelement, 'icon', optional=True)
+                    if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('mode', {}).get('remove', []) and 'mode' in subsubelement.attrib:
+                        del subsubelement.attrib['mode']
+                    if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('rename', {}):
+                        subsubelement.attrib['name'] = upgrade_help.get('families', {}).get('rename', {})[subsubelement.attrib['name']]
+                    if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('remove', []):
+                        subelement.remove(subsubelement)
+                        continue
+                    if subsubelement.attrib['name'] in upgrade_help.get('families', {}).get('hidden', {}).get('add', []):
+                        subsubelement.attrib['hidden'] = 'True'
+                    if subsubelement.attrib['name'] != normalize_family(subsubelement.attrib['name']):
+                        if "description" not in subsubelement.attrib:
+                            subsubelement.attrib['description'] = subsubelement.attrib['name']
+                        subsubelement.attrib['name'] = normalize_family(subsubelement.attrib['name'])
+                    families[subsubelement.attrib['name']] = subsubelement
+            # if empty, remove
+            if is_empty(subelement) or subelement.tag in ['containers', 'files', 'help']:
+                root.remove(subelement)
+                continue
+            # copy subelement
+            if subelement.tag == 'constraints':
+                constraints_obj = subelement
+        services = None
+        service_elt = {}
+        for variable, obj in upgrade_help.get('variables', {}).get('add', {}).items():
+            family = next(iter(families.values()))
+            variables[variable] = SubElement(family, 'variable', name=variable)
+            if 'value' in obj:
+                SubElement(variables[variable], 'value').text = obj['value']
+            for name in ['hidden', 'exists']:
+                if name in obj:
+                    variables[variable].attrib[name] = obj[name]
+        files_is_empty = True
+        for lst in files.values():
+            if len(lst):
+                files_is_empty = False
+                break
+        if not files_is_empty:
+            services = Element('services')
+            root.insert(0, services)
+            for service_name, lst in files.items():
+                service = self.create_service(services, service_name, service_elt, servicelists, upgrade_help)
+                for file_ in lst.values():
+                    self.move(file_, 'name_type', 'file_type', optional=True)
+                    if 'file_type' in file_.attrib and file_.attrib['file_type'] == 'SymLinkOption':
+                        file_.attrib['file_type'] = 'variable'
+                        if variables[file_.text].attrib['type'] == 'string':
+                            variables[file_.text].attrib['type'] = 'filename'
+                    self.remove(file_, 'rm', optional=True)
+                    self.remove(file_, 'mkdir', optional=True)
+                    service.append(file_)
+        ips_is_empty = True
+        for lst in ips.values():
+            if len(lst):
+                ips_is_empty = False
+                break
+        if not ips_is_empty:
+            if services is None:
+                services = Element('services')
+                root.insert(0, services)
+            for service_name, lst in ips.items():
+                service = self.create_service(services, service_name, service_elt, servicelists, upgrade_help)
+                for ip in lst:
+                    if ip.text in upgrade_help.get('ips', {}).get('remove', []):
+                        continue
+                    for type in ['ip_type', 'netmask_type']:
+                        if type in ip.attrib and ip.attrib[type] == 'SymLinkOption':
+                            ip.attrib[type] = 'variable'
+                    if 'netmask' in ip.attrib:
+                        if ip.attrib['netmask'] == '255.255.255.255':
+                            del ip.attrib['netmask']
+                            if ip.text in variables and 'type' not in variables[ip.text].attrib:
+                                variables[ip.text].attrib['type'] = 'ip'
+                        else:
+                            if ip.text in variables and 'type' not in variables[ip.text].attrib:
+                                variables[ip.text].attrib['type'] = 'network'
+                            if ip.attrib['netmask'] in variables and 'type' not in variables[ip.attrib['netmask']].attrib:
+                                variables[ip.attrib['netmask']].attrib['type'] = 'netmask'
+                    elif ip.text in variables and 'type' not in variables[ip.text].attrib:
+                        variables[ip.text].attrib['type'] = 'ip'
+                    self.remove(ip, 'interface')
+                    self.remove(ip, 'interface_type', optional=True)
+                    self.remove(ip, 'service_restrictionlist', optional=True)
+                    service.append(ip)
+        if ips_is_empty and files_is_empty:
+            for service_name in files:
+                if services is None:
+                    services = Element('services')
+                    root.insert(0, services)
+                if service_name != 'unknown':
+                    self.create_service(services, service_name, service_elt, servicelists, upgrade_help)
+        if constraints_obj is None:
+            constraints_obj = SubElement(root, 'constraints')
+        if variables_auto_valid_enum:
+            for type, variables_valid_enum in variables_auto_valid_enum.items():
+                valid_enum = SubElement(constraints_obj, 'check')
+                valid_enum.attrib['name'] = 'valid_enum'
+                for val in type.split('/'):
+                    param = SubElement(valid_enum, 'param')
+                    param.text = val
+                for variable in variables_valid_enum:
+                    target = SubElement(valid_enum, 'target')
+                    target.text = variable
+        for name, text in variables_help.items():
+            variables[name].attrib['help'] = text
+        for name, text in families_help.items():
+            if name in upgrade_help.get('families', {}).get('rename', {}):
+                name = upgrade_help.get('families', {}).get('rename', {})[name]
+            families[normalize_family(name)].attrib['help'] = text
+        for auto in autos:
+            if auto in variables:
+                variables[auto].attrib['hidden'] = 'True'
+            else:
+                # redefine value in first family
+                family = next(iter(families.values()))
+                variable = SubElement(family, 'variable', name=auto, redefine="True", hidden="True")
+                if self.test:
+                    del variable.attrib['redefine']
+        return root
+
+    @staticmethod
+    def move(elt, src, dst, optional=False):
+        if src == 'text':
+            value = elt.text
+            elt.text = None
+        else:
+            if optional and src not in elt.attrib:
+                return
+            value = elt.attrib[src]
+            del elt.attrib[src]
+        #
+        if dst == 'text':
+            elt.text = value
+        else:
+            elt.attrib[dst] = value
+
+    @staticmethod
+    def remove(elt, src, optional=False):
+        if optional and src not in elt.attrib:
+            return
+        del elt.attrib[src]
+
+    @staticmethod
+    def create_service(services, service_name, service_elt, servicelists, upgrade_help):
+        if service_name in service_elt:
+            return service_elt[service_name]
+        service = SubElement(services, 'service')
+        service.attrib['name'] = service_name
+        if service_name == 'unknown':
+            service.attrib['manage'] = 'False'
+        if service_name in upgrade_help.get('services', {}).get('unmanage', []):
+            service.attrib['manage'] = 'False'
+        service_elt[service_name] = service
+        if upgrade_help.get('servicelists', {}).get(service_name):
+            service.attrib['servicelist'] = upgrade_help.get('servicelists', {}).get(service_name)
+        elif service_name in servicelists:
+            service.attrib['servicelist'] = servicelists[service_name]
+        return service
+
+    def upgrade_container(self, elt, current_service, files, ips, servicelists, upgrade_help):
+        if elt.tag == 'file':
+            self.move(elt, 'name', 'text')
+            self.remove(elt, 'del_comment', optional=True)
+            elt.attrib['engine'] = 'creole_legacy'
+            if (not 'instance_mode' in elt.attrib or elt.attrib['instance_mode'] != 'when_container') and \
+                    elt.text not in upgrade_help.get('files', {}).get('remove', {}):
+                if elt.attrib.get('filelist') in upgrade_help.get('services', {}).get('filelist_service', {}):
+                    elt_service = upgrade_help.get('services', {}).get('filelist_service', {})[elt.attrib['filelist']]
+                    if elt_service in files:
+                        service = elt_service
+                    else:
+                        service = current_service
+                else:
+                    service = current_service
+                files[service][elt.text] = elt
+        elif elt.tag in ['host', 'disknod', 'fstab', 'interface', 'package', 'service_access']:
+            pass
+        elif elt.tag == 'service_restriction':
+            for restriction in elt:
+                if restriction.tag == 'ip' and restriction.text != '0.0.0.0':
+                    self.remove(restriction, 'ip_type', optional=True)
+                    self.remove(restriction, 'netmask_type', optional=True)
+                    if elt.attrib['service'] in upgrade_help.get('services', {}).get('rename', {}):
+                        elt_service = upgrade_help.get('services', {}).get('rename', {})[elt.attrib['service']]
+                    else:
+                        elt_service = elt.attrib['service']
+                    if elt_service in ips:
+                        service = elt_service
+                    else:
+                        service = current_service
+                    ips[service].append(restriction)
+        elif elt.tag == 'service':
+            new_name = elt.text
+            if current_service == 'unknown':
+                if new_name in files:
+                    raise Exception('hu?')
+                files[new_name] = files[current_service]
+                del files[current_service]
+                ips[new_name] = ips[current_service]
+                del ips[current_service]
+            elif new_name not in files:
+                files[new_name] = {}
+                ips[new_name] = []
+            current_service = new_name
+            if 'servicelist' in elt.attrib:
+                servicelists[current_service] = elt.attrib['servicelist']
+        else:
+            raise Exception(f'unknown containers tag {elt.tag}')
+        return current_service
+
+
+def is_empty(elt, not_check_attrib=False):
+    if not not_check_attrib and elt.attrib:
+        return False
+    empty = True
+    for file_ in elt:
+        empty = False
+    return empty
+
+
+def update_creole_1():
+    print('pfff')

src/rougail/utils.py

@@ -1,23 +1,66 @@
+"""Rougail's tools
+
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 """
-utilitaires créole
-"""
-import unicodedata
+from typing import List
+from unicodedata import normalize, combining
+import re
+
+from importlib.machinery import SourceFileLoader
+from importlib.util import spec_from_loader, module_from_spec
+
 from .i18n import _
+from .error import DictConsistencyError
+
+NAME_REGEXP = re.compile(r"^[a-z0-9_]*$")
 
 
-def normalize_family(family_name: str,
-                     check_name: bool=True,
-                     allow_dot: bool=False) -> str:
+def valid_variable_family_name(name: str,
+                               xmlfiles: List[str],
+                               ) -> None:
+    match = NAME_REGEXP.search(name)
+    if not match:
+        msg = _(f'invalid variable or family name "{name}" must only contains '
+                'lowercase ascii character, number or _')
+        raise DictConsistencyError(msg, 76, xmlfiles)
+
+
+def normalize_family(family_name: str) -> str:
     """replace space, accent, uppercase, ... by valid character
     """
-    family_name = family_name.replace('-', '_')
-    if not allow_dot:
-        family_name = family_name.replace('.', '_')
-    family_name = family_name.replace(' ', '_')
-    nfkd_form = unicodedata.normalize('NFKD', family_name)
-    family_name = ''.join([c for c in nfkd_form if not unicodedata.combining(c)])
-    family_name = family_name.lower()
-    if check_name and family_name == 'containers':
-        raise ValueError(_(f'"{family_name}" is a forbidden family name'))
-    return family_name
+    family_name = family_name.replace('-', '_').replace(' ', '_').replace('.', '_')
+    nfkd_form = normalize('NFKD', family_name)
+    family_name = ''.join([c for c in nfkd_form if not combining(c)])
+    return family_name.lower()
 
+
+def load_modules(eosfunc_file) -> List[str]:
+    """list all functions in eosfunc
+    """
+    loader = SourceFileLoader('eosfunc', eosfunc_file)
+    spec = spec_from_loader(loader.name, loader)
+    eosfunc = module_from_spec(spec)
+    loader.exec_module(eosfunc)
+    return eosfunc

src/rougail/xml_compare.py

@@ -1,161 +0,0 @@
-try:
-    import doctest
-    doctest.OutputChecker
-except (AttributeError, ImportError): # Python < 2.4
-    import util.doctest24 as doctest
-try:
-    import xml.etree.ElementTree as ET
-except ImportError:
-    import elementtree.ElementTree as ET
-from xml.parsers.expat import ExpatError as XMLParseError
-
-RealOutputChecker = doctest.OutputChecker
-
-
-def debug(*msg):
-    import sys
-    print >> sys.stderr, ' '.join(map(str, msg))
-
-
-class HTMLOutputChecker(RealOutputChecker):
-
-    def check_output(self, want, got, optionflags):
-        normal = RealOutputChecker.check_output(self, want, got, optionflags)
-        if normal or not got:
-            return normal
-        try:
-            want_xml = make_xml(want)
-        except XMLParseError:
-            pass
-        else:
-            try:
-                got_xml = make_xml(got)
-            except XMLParseError:
-                pass
-            else:
-                if xml_compare(want_xml, got_xml):
-                    return True
-        return False
-
-    def output_difference(self, example, got, optionflags):
-        actual = RealOutputChecker.output_difference(
-            self, example, got, optionflags)
-        want_xml = got_xml = None
-        try:
-            want_xml = make_xml(example.want)
-            want_norm = make_string(want_xml)
-        except XMLParseError as e:
-            if example.want.startswith('<'):
-                want_norm = '(bad XML: %s)' % e
-                #  '<xml>%s</xml>' % example.want
-            else:
-                return actual
-        try:
-            got_xml = make_xml(got)
-            got_norm = make_string(got_xml)
-        except XMLParseError as e:
-            if example.want.startswith('<'):
-                got_norm = '(bad XML: %s)' % e
-            else:
-                return actual
-        s = '%s\nXML Wanted: %s\nXML Got   : %s\n' % (
-            actual, want_norm, got_norm)
-        if got_xml and want_xml:
-            result = []
-            xml_compare(want_xml, got_xml, result.append)
-            s += 'Difference report:\n%s\n' % '\n'.join(result)
-        return s
-
-
-def xml_sort(children):
-    tcl1 = {}
-    #idx = 0
-
-    for child in children:
-        if 'name' in child.attrib:
-            key = child.attrib['name']
-        else:
-            key = child.tag
-        if key not in tcl1:
-            tcl1[key] = []
-        tcl1[key].append(child)
-    cl1_keys = list(tcl1.keys())
-    cl1_keys.sort()
-    cl1 = []
-    for key in cl1_keys:
-        cl1.extend(tcl1[key])
-    return cl1
-
-def xml_compare(x1, x2):
-    if x1.tag != x2.tag:
-        print ('Tags do not match: %s and %s' % (x1.tag, x2.tag))
-        return False
-    for name, value in x1.attrib.items():
-        if x2.attrib.get(name) != value:
-            print ('Attributes do not match: %s=%r, %s=%r'
-                     % (name, value, name, x2.attrib.get(name)))
-            return False
-    for name in x2.attrib:
-        if name not in x1.attrib:
-            print ('x2 has an attribute x1 is missing: %s'
-                     % name)
-            return False
-    if not text_compare(x1.text, x2.text):
-        print ('text: %r != %r' % (x1.text, x2.text))
-        return False
-    if not text_compare(x1.tail, x2.tail):
-        print ('tail: %r != %r' % (x1.tail, x2.tail))
-        return False
-
-    cl1 = xml_sort(x1.getchildren())
-    cl2 = xml_sort(x2.getchildren())
-
-    if len(cl1) != len(cl2):
-        cl1_tags = []
-        for c in cl1:
-            cl1_tags.append(c.tag)
-        cl2_tags = []
-        for c in cl2:
-            cl2_tags.append(c.tag)
-        print ('children length differs, %i != %i (%s != %s)'
-                 % (len(cl1), len(cl2), cl1_tags, cl2_tags))
-        return False
-    i = 0
-    for c1, c2 in zip(cl1, cl2):
-        i += 1
-        if not xml_compare(c1, c2):
-            if 'name' in c1.attrib:
-                name = c1.attrib['name']
-            else:
-                name = i
-            print ('in tag "%s" with name "%s"'
-                     % (c1.tag, name))
-            return False
-    return True
-
-
-def text_compare(t1, t2):
-    if not t1 and not t2:
-        return True
-    if t1 == '*' or t2 == '*':
-        return True
-    return (t1 or '').strip() == (t2 or '').strip()
-
-
-def make_xml(s):
-    return ET.XML('<xml>%s</xml>' % s)
-
-
-def make_string(xml):
-    if isinstance(xml, (str, unicode)):
-        xml = make_xml(xml)
-    s = ET.tostring(xml)
-    if s == '<xml />':
-        return ''
-    assert s.startswith('<xml>') and s.endswith('</xml>'), repr(s)
-    return s[5:-6]
-
-
-def install():
-    doctest.OutputChecker = HTMLOutputChecker
-

src/rougail/xmlreflector.py

@@ -1,90 +1,86 @@
-# coding: utf-8
-from os.path import join, isfile, basename, isdir
-from os import listdir
-#from io import BytesIO
+"""load XML file from directory
 
-from lxml.etree import DTD, parse, tostring  # , XMLParser
+Created by:
+EOLE (http://eole.orion.education.fr)
+Copyright (C) 2005-2018
+
+Forked by:
+Cadoles (http://www.cadoles.com)
+Copyright (C) 2019-2021
+
+distribued with GPL-2 or later license
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""
+from typing import List
+from os.path import join, isfile
+from os import listdir
+
+from lxml.etree import DTD, parse, XMLSyntaxError  # pylint: disable=E0611
 
 from .i18n import _
 from .error import DictConsistencyError
 
 
-class XMLReflector(object):
+class XMLReflector:
     """Helper class for loading the Creole XML file,
     parsing it, validating against the Creole DTD,
     writing the xml result on the disk
     """
-    def __init__(self):
-        self.dtd = None
-
-    def parse_dtd(self, dtdfilename):
+    def __init__(self,
+                 rougailconfig: 'RougailConfig',
+                 ) -> None:
         """Loads the Creole DTD
 
         :raises IOError: if the DTD is not found
 
         :param dtdfilename: the full filename of the Creole DTD
         """
+        dtdfilename = rougailconfig['dtdfilename']
         if not isfile(dtdfilename):
             raise IOError(_("no such DTD file: {}").format(dtdfilename))
         with open(dtdfilename, 'r') as dtdfd:
             self.dtd = DTD(dtdfd)
 
-    def parse_xmlfile(self, xmlfile):
-        """Parses and validates some Creole XML against the Creole DTD
-
-        :returns: the root element tree object
-        """
-        # document = parse(BytesIO(xmlfile), XMLParser(remove_blank_text=True))
-        document = parse(xmlfile)
-        if not self.dtd.validate(document):
-            raise DictConsistencyError(_("not a valid xml file: {}").format(xmlfile))
-        return document.getroot()
-
-    def load_xml_from_folders(self, xmlfolders):
+    def load_xml_from_folders(self,
+                              xmlfolders: List[str],
+                              ):
         """Loads all the XML files located in the xmlfolders' list
 
         :param xmlfolders: list of full folder's name
         """
-        documents = []
-        if not isinstance(xmlfolders, list):
-            xmlfolders = [xmlfolders]
+        filenames = {}
         for xmlfolder in xmlfolders:
-            if isinstance(xmlfolder, list) or isinstance(xmlfolder, tuple):
-                # directory group : collect files from each
-                # directory and sort them before loading
-                group_files = []
-                for idx, subdir in enumerate(xmlfolder):
-                    if isdir(subdir):
-                        for filename in listdir(subdir):
-                            group_files.append((filename, idx, subdir))
-                    else:
-                        group_files.append(basename(subdir), idx, dirname(subdir))
-                def sort_group(file1, file2):
-                    if file1[0] == file2[0]:
-                        # sort by initial xmlfolder order if same name
-                        return file1[1].__cmp__(file2[1])
-                    # sort by filename
-                    elif file1[0] > file2[0]:
-                        return 1
-                    else:
-                        return -1
-                group_files.sort(sort_group)
-                filenames = [join(f[2], f[0]) for f in group_files]
-            elif isdir(xmlfolder):
-                filenames = []
-                for filename in listdir(xmlfolder):
-                    filenames.append(join(xmlfolder, filename))
-                filenames.sort()
-            else:
-                filenames = [xmlfolder]
-            for xmlfile in filenames:
-                if xmlfile.endswith('.xml'):
-                    #xmlfile_path = join(xmlfolder, xmlfile)
-                    documents.append((xmlfile, self.parse_xmlfile(xmlfile)))
-        return documents
-
-    def save_xmlfile(self, xmlfilename, xml):  # pylint: disable=R0201
-        """Write a bunch of XML on the disk
-        """
-        with open(xmlfilename, 'w') as xmlfh:
-            xmlfh.write(tostring(xml, pretty_print=True, encoding="UTF-8", xml_declaration=True).decode('utf8'))
+            for filename in listdir(xmlfolder):
+                if not filename.endswith('.xml'):
+                    continue
+                if filename in filenames:
+                    raise DictConsistencyError(_(f'duplicate xml file name {filename}'), 78, [xmlfolder])
+                filenames[filename] = join(xmlfolder, filename)
+        if not filenames:
+            raise DictConsistencyError(_('there is no XML file'), 77, [xmlfolder])
+        file_names = list(filenames.keys())
+        file_names.sort()
+        for filename in file_names:
+            xmlfile = filenames[filename]
+            try:
+                document = parse(xmlfile)
+            except XMLSyntaxError as err:
+                raise DictConsistencyError(_(f'not a XML file: {err}'), 52, [xmlfile]) from err
+            if not self.dtd.validate(document):
+                dtd_error = self.dtd.error_log.filter_from_errors()[0]
+                msg = _(f'not a valid XML file: {dtd_error}')
+                raise DictConsistencyError(msg, 43, [xmlfile])
+            yield xmlfile, document.getroot()

tests/dictionaries/00empty/00_base.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail version="0.10">
+  <services>
+    <service name="tata">
+        </service>
+  </services>
+</rougail>

tests/dictionaries/00empty/makedict/after.json

@@ -0,0 +1,10 @@
+{
+    "services.tata_service.activate": {
+        "owner": "default",
+        "value": true
+    },
+    "services.tata_service.manage": {
+        "owner": "default",
+        "value": true
+    }
+}

tests/dictionaries/00empty/makedict/base.json

@@ -0,0 +1,4 @@
+{
+    "services.tata_service.activate": true,
+    "services.tata_service.manage": true
+}

tests/dictionaries/00empty/makedict/before.json

@@ -0,0 +1,10 @@
+{
+    "services.tata_service.activate": {
+        "owner": "default",
+        "value": true
+    },
+    "services.tata_service.manage": {
+        "owner": "default",
+        "value": true
+    }
+}

tests/dictionaries/00empty/tiramisu/base.py

@@ -0,0 +1,21 @@
+from importlib.machinery import SourceFileLoader as _SourceFileLoader
+from importlib.util import spec_from_loader as _spec_from_loader, module_from_spec as _module_from_spec
+class func:
+    pass
+_loader = _SourceFileLoader('func', 'tests/dictionaries/../eosfunc/test.py')
+_spec = _spec_from_loader(_loader.name, _loader)
+_func = _module_from_spec(_spec)
+_loader.exec_module(_func)
+for function in dir(_func):
+    if function.startswith('_'):
+        continue
+    setattr(func, function, getattr(_func, function))
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+option_3 = BoolOption(name="activate", doc="activate", default=True)
+option_4 = BoolOption(name="manage", doc="manage", default=True)
+option_2 = OptionDescription(name="tata_service", doc="tata.service", children=[option_3, option_4])
+option_1 = OptionDescription(name="services", doc="services", children=[option_2], properties=frozenset({"hidden"}))
+option_0 = OptionDescription(name="baseoption", doc="baseoption", children=[option_1])

tests/dictionaries/00load_autofreeze/00-base.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail version="0.10">
+  <variables>
+    <variable name="myvar" auto_freeze="True">
+      <value>no</value>
+    </variable>
+    <variable name="server_deployed" type="boolean">
+      <value>False</value>
+    </variable>
+  </variables>
+</rougail>

tests/dictionaries/00load_autofreeze/makedict/after.json

@@ -0,0 +1,10 @@
+{
+    "rougail.myvar": {
+        "owner": "forced",
+        "value": "no"
+    },
+    "rougail.server_deployed": {
+        "owner": "default",
+        "value": false
+    }
+}

tests/dictionaries/00load_autofreeze/makedict/base.json

@@ -0,0 +1,4 @@
+{
+    "rougail.myvar": "no",
+    "rougail.server_deployed": false
+}

tests/dictionaries/00load_autofreeze/makedict/before.json

@@ -0,0 +1,10 @@
+{
+    "rougail.myvar": {
+        "owner": "default",
+        "value": "no"
+    },
+    "rougail.server_deployed": {
+        "owner": "default",
+        "value": false
+    }
+}

tests/dictionaries/00load_autofreeze/tiramisu/base.py

@@ -0,0 +1,20 @@
+from importlib.machinery import SourceFileLoader as _SourceFileLoader
+from importlib.util import spec_from_loader as _spec_from_loader, module_from_spec as _module_from_spec
+class func:
+    pass
+_loader = _SourceFileLoader('func', 'tests/dictionaries/../eosfunc/test.py')
+_spec = _spec_from_loader(_loader.name, _loader)
+_func = _module_from_spec(_spec)
+_loader.exec_module(_func)
+for function in dir(_func):
+    if function.startswith('_'):
+        continue
+    setattr(func, function, getattr(_func, function))
+try:
+    from tiramisu3 import *
+except:
+    from tiramisu import *
+option_3 = BoolOption(name="server_deployed", doc="server_deployed", default=False, properties=frozenset({"mandatory", "normal"}))
+option_2 = StrOption(name="myvar", doc="myvar", default="no", properties=frozenset({"basic", "force_store_value", "mandatory", Calculation(func.calc_value, Params(ParamValue('frozen'), kwargs={'condition': ParamOption(option_3, todict=True, notraisepropertyerror=True), 'expected': ParamValue(True)}))}))
+option_1 = OptionDescription(name="rougail", doc="rougail", children=[option_2, option_3])
+option_0 = OptionDescription(name="baseoption", doc="baseoption", children=[option_1])

tests/dictionaries/00load_autofreezeexpert/00-base.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<rougail version="0.10">
+  <variables>
+    <variable name="my_var" auto_freeze="True" mode="expert">
+      <value>no</value>
+    </variable>
+    <variable name="server_deployed" type="boolean">
+      <value>False</value>
+    </variable>
+  </variables>
+</rougail>

tests/dictionaries/00load_autofreezeexpert/makedict/after.json

@@ -0,0 +1,10 @@
+{
+    "rougail.my_var": {
+        "owner": "forced",
+        "value": "no"
+    },
+    "rougail.server_deployed": {
+        "owner": "default",
+        "value": false
+    }
+}