Merge branch 'master' into dist/eole/2.6.2/master

README.md

@@ -1,32 +1,5 @@
 # eole-mariadb
 
-Paquet porté pour la 2.7.0. Les tests n'ont été réalisé que sur un Eolebase.
-
-La version utilisé est mariadb 10.3
-
-Ajout de ce paquet sur une 2.7.0 :
-
-## Add the Cadoles repository on all the nodes and the Leader
-
-  GenConfig  [Mode Expert] -> Dépôts Tiers
-
- * Libellé du dépôt : Cadoles dev
- * Déclaration du dépôt : deb [ arch=all ] https://vulcain.cadoles.com 2.7.0-dev main
- * Méthode de récupération de la clé publique du dépôt : URL de la clé
- * URL de la clé : https://vulcain.cadoles.com/cadoles.gpg
-
-## Add the MariaDB repository on all the nodes and the Leader
-
-GenConfig (Mode Expert) -> Dépôt tiers :
- * Libellé du dépôt : MariaDB
- * Déclaration du dépôt :  deb [ arch=amd64 ] http://mariadb.mirrors.ovh.net/MariaDB/repo/10.3/ubuntu bionic main
- * Méthode de récupération de la clé publique du dépôt : serveur de clés
- * URL du serveur de clés : hkp://keyserver.ubuntu.com:80
- * Empreinte de la clé : 0xF1656F24C74CD1D8
-
-#######################################################################
-
-
 Début des travaux pour l'utilisation de mariadb
 dans Eole avec des fonctionalitées avancées du type
 mise en clustter

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,18 @@
+Source: eole-mariadb
+Section: web
+Priority: optional
+Maintainer: Cadoles <eole@ac-dijon.fr>
+Build-Depends: debhelper (>= 9)
+Standards-Version: 3.9.3
+Homepage: https://forge.cadoles.com/Cadoles/eole-mariadb
+Vcs-Git: https://forge.cadoles.com/Cadoles/eole-mariadb.git
+Vcs-Browser: https://forge.cadoles.com/Cadoles/eole-mariadb
+
+Package: eole-mariadb
+Architecture: all
+Depends: ${misc:Depends}, mariadb-server
+Conflicts: eole-mysql
+Provides: eole-mysql
+Description: Dictionnaires et templates pour la configuration d'un serveur MariaDB, testée uniquement avec eolebase
+ .
+ Pour toute information complémentaire, veuillez vous rendre sur la forge Cadoles.

debian/copyright

@@ -0,0 +1,44 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: {PROJECT}
+Source: {URL}
+
+Files: *
+Copyright: YEAR {UPSTREAM} {AUTHOR} <{MAIL}>
+License: {UPSTREAM LICENSE}
+
+Files: debian/*
+Copyright: 2012 Équipe EOLE <eole@ac-dijon.fr>
+License: CeCILL-2
+
+License: {UPSTREAM LICENSE}
+ {TEXT OF THE LICENSE}
+
+License: CeCILL-2
+ This software is governed by the CeCILL-2 license under French law and
+ abiding by the rules of distribution of free software.  You can  use,
+ modify and or redistribute the software under the terms of the CeCILL-2
+ license as circulated by CEA, CNRS and INRIA at the following URL
+ "http://www.cecill.info";.
+ .
+ As a counterpart to the access to the source code and  rights to copy,
+ modify and redistribute granted by the license, users are provided only
+ with a limited warranty  and the software's author,  the holder of the
+ economic rights,  and the successive licensors  have only  limited
+ liability.
+ .
+ In this respect, the user's attention is drawn to the risks associated
+ with loading,  using,  modifying and/or developing or reproducing the
+ software by the user in light of its specific status of free software,
+ that may mean  that it is complicated to manipulate,  and  that  also
+ therefore means  that it is reserved for developers  and  experienced
+ professionals having in-depth computer knowledge. Users are therefore
+ encouraged to load and test the software's suitability as regards their
+ requirements in conditions enabling the security of their systems and/or
+ data to be ensured and,  more generally, to use and operate it in the
+ same conditions as regards security.
+ .
+ The fact that you are presently reading this means that you have had
+ knowledge of the CeCILL-2 license and that you accept its terms.
+ .
+ On Eole systems, the complete text of the CeCILL-2 License can be found
+ in '/usr/share/common-licenses/CeCILL-2-en'.

debian/gbp.conf

@@ -0,0 +1,3 @@
+# Set per distribution debian tag
+[DEFAULT]
+debian-tag = debian/eole/%(version)s

debian/rules

@@ -0,0 +1,8 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+	dh $@

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

dicos/25_bdd_server.xml

@@ -2,38 +2,39 @@
     <files>
         <file filelist='dbMariaDB' name='/etc/mysql/conf.d/mariadb.cnf' rm='True' mkdir='True'/>
         <file filelist='dbCluster' name='/etc/mysql/conf.d/galera.cnf'  rm='True' mkdir='True'/>
-        <file filelist='dbTunning' name='/etc/mysql/conf.d/tunning.cnf' rm='True' mkdir='True'/>
 
+        <file filelist='dbTunning' name='/etc/mysql/conf.d/tunning.cnf' rm='True' mkdir='True'/>
         <service servicelist='bdd'>mariadb</service>
         <service_access service='mariadb'>
-            <port protocol='tcp'>3306</port>
+            <port service_accesslist='mariadb' protocol='tcp'>3306</port>
-            <tcpwrapper>mariadb</tcpwrapper>
+            <tcpwrapper service_accesslist='mariadb'>mariadb</tcpwrapper>
         </service_access>
-        <service_restriction service='mariadb'>
-           <ip interface='ifMariaDBLimit' interface_type="SymLinkOption" netmask='maskMariaDBLimit' netmask_type='SymLinkOption' ip_type='SymLinkOption'>ipMariaDBLimit</ip>
-        </service_restriction>
-
         <service_access service='galera'>
-            <port service_accesslist='dbCluster' protocol='tcp'>3306</port>
             <port service_accesslist='dbCluster' protocol='tcp'>4444</port>
             <port service_accesslist='dbCluster' protocol='tcp'>4567</port>
             <port service_accesslist='dbCluster' protocol='tcp'>4568</port>
             <port service_accesslist='dbCluster' protocol='udp'>4567</port>
+            <tcpwrapper service_accesslist='mariadb'>mariadb</tcpwrapper>
         </service_access>
+
+        <service_restriction service='mariadb'>
+           <ip interface='ifMariaDBLimit' interface_type="SymLinkOption" netmask='maskMariaDBLimit' netmask_type='SymLinkOption' ip_type='SymLinkOption'>ipMariaDBLimit</ip>
+        </service_restriction>
+
         <service_restriction service='galera'>
-           <ip interface='ifDBCluster' interface_type="SymLinkOption" netmask='255.255.255.255' ip_type='SymLinkOption'>dbClusterMember</ip>
+           <ip interface='ifDBCluster' interface_type="SymLinkOption" netmask='255.255.255.255' ip_type='SymLinkOption'>dbClusterMemberIP</ip>
         </service_restriction>
     </files>
     <variables>
         <family name="Services">
            <variable name="dbEnable" type='oui/non' description='Activer le serveur de base de données MariaDB'>
-              <value>oui</value>
+              <value>non</value>
            </variable>
            <variable name="dbEnableBackup" type='oui/non' description='Activer la sauvegarde sur le serveur de base de données MariaDB'>
               <value>non</value>
            </variable>
         </family>
-        <family name="MariaDB" icon='mysql-alt'>
+        <family name="Database">
             <variable name='dbEnableCluster' type='oui/non' description="Activer la mise en grappe MariaDB Galera ?">
                 <value>non</value>
             </variable> 
@@ -50,10 +51,13 @@
             </variable>
             <variable name="accLimitTarget" type="string" description="Restriction d'accès à ces IP/noms d'hôtes (liste séparateur séparateur ',')" />
 
+            <variable name="dbClusterGcacheSizeMb" type="number" description="FIXME Gcachesize">
+                <value>128</value>
+            </variable>
         </family>
 
         <!-- Tunning -->
-        <family name="MariaDB Tunning" mode="expert" icon='mysql-alt'>
+        <family name="Database Tunning" mode="expert">
             <variable name="dbInnoDBBufferPoolPercentage" type="number" description="Pourcentage de mémoire à dédier à MariaDB">
                 <value>20</value>
             </variable>
@@ -110,15 +114,18 @@
             </variable>
         </family>
 
-        <family name="Grappe MariaDB" icon='mysql-alt'>
+        <family name="Database Cluster">
             <variable name='dbClusterPosition' type='string' description="Rôle dans la grappe" mandatory='True'/>
+            <variable name='ifDBCluster' type='string' description="Interface réseau dédiée à la grappe BDD"/>
+            <!-- FIXME : Fill this variable with nom_machine -->
+            <variable name='nodeName' type='string' description="Nom du noeud local"/>
+
+            <variable name='dbClusterMember' type='string' description="Membre de la grappe BDD" multi='True'/>
+            <variable name='dbClusterMemberIP' type='ip' description="Adresse IP"/>
+            <!-- Expert Variables -->
             <variable name='dbClusterName' type='string' description='Nom de la grappe'>
                <value>galera_cluster</value>
             </variable>
-            <variable name='ifDBCluster' type='string' description="Interface réseau dédiée à la grappe BDD" mandatory='True'/>
-            <variable name='dbClusterMember' type='domain' description="Nom de domaine des membres de la grappe" multi="True" mandatory='True'/>
-            <variable name="dbClusterMemberIndex" description="Index du serveur dans la liste des membres de la grappe" type="number" mandatory="True"/>
-            <!-- Expert Variables -->
             <variable name='dbBinLogFormat' type='string' mode='expert' description='Format du binlog'>
                <value>ROW</value>
             </variable>
@@ -128,7 +135,7 @@
             <variable name='dbSSTMethod' type='string' mode='expert' description='Méthode SST'>
                <value>rsync</value>
             </variable>
-            <variable name='dbClusterGcacheSizeMb' type='number' mode='expert' description="Write-set Cache Size (gcache.size)">
+            <variable name='dbClusterGcacheSize' type='number' mode='expert' description="Write-set Cache Size (gcache.size)">
                 <value>128</value>
             </variable>
             <!-- End -->
@@ -138,17 +145,14 @@
         </separators>
     </variables>
     <constraints>
-	<check name="valid_ipnetmask" target="maskMariaDBLimit" level="warning">
-            <param type='eole'>ipMariaDBLimit</param>
-        </check>	
-
-        <check name='valid_enum' target='dbBinLogFormat'>
-            <param>['ROW', 'STATEMENT', 'MIXED', 'NONE']</param>
-        </check>
         <check name='valid_enum' target='ifDBCluster'>
             <param>['eth0', 'eth1', 'eth2', 'eth3', 'eth4']</param>
         </check>
 
+        <group master='dbClusterMember'>
+            <slave>dbClusterMemberIP</slave>
+        </group>
+
         <check name='valid_enum' target='accLimits'>
             <param>['Default','Custom']</param>
         </check>
@@ -176,12 +180,13 @@
 
         <condition name='disabled_if_in' source="dbEnable">
             <param>non</param>
-            <target type='family'>MariaDB</target>
+            <target type='family'>Database</target>
-            <!--target type='family'>Grappe MariaDB</target-->
+            <target type='family'>Database Cluster</target>
-            <target type='family'>MariaDB Tunning</target>
+            <target type='family'>Database Tunning</target>
+            <target type='service_accesslist'>dbCluster</target>
             <target type='filelist'>dbTunning</target>
             <target type='filelist'>dbMariaDB</target>
-            <!--target type='filelist'>dbCluster</target-->
+            <target type='filelist'>dbCluster</target>
             <target type='servicelist'>bdd</target>
         </condition>
 
@@ -194,7 +199,7 @@
 
         <condition name='disabled_if_in' source="dbEnableCluster">
             <param>non</param>
-            <target type='family'>Grappe MariaDB</target>
+            <target type='family'>Database Cluster</target>
             <target type='service_accesslist'>dbCluster</target>
             <target type='filelist'>dbCluster</target>
         </condition>

postservice/00-bdd-cluster

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [[ "$1" == "instance" ]] && [[ "$(CreoleGet dbEnable)" == "oui" ]] && [[ "$(CreoleGet dbEnableCluster)" == "oui" ]] && [[ "$(CreoleGet dbClusterPosition)" == "Leader" ]]
-then
-    service mariadb stop
-    galera_new_cluster
-fi
-
-exit 0

postservice/25-bdd-admin

@@ -28,8 +28,6 @@ then
 
     previousUsers=($(awk -F ':' '{print $1}' ${readerfile} ))
 
-    index=0
-
     for user in ${accounts[@]}
     do
         PASS=$(awk -F ':' "/${user}:/ {print \$2}" ${readerfile})
@@ -56,6 +54,7 @@ then
             done
         fi
 
+        index=0
         for hst in ${hostsList[@]}
         do
             SQL="DROP USER IF EXISTS '${user}'@'${hst}';"
@@ -87,3 +86,4 @@ then
 fi
 
 exit 0
+

posttemplate/25-bdd-cluster

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+db=$(CreoleGet dbEnable non)
+cluster=$(CreoleGet dbEnableCluster non)
+role=$(CreoleGet dbClusterPosition 'Node')
+gstateFile="/var/lib/mysql/grastate.dat"
+
+if [[ ${db} == "oui" ]]
+then
+    if [[ ${cluster} == "oui" ]]
+    then
+		if [[ ${role} == "Leader"  ]]
+		then
+			service mariadb stop
+		    galera_new_cluster
+		fi
+    fi
+fi
+
+exit 0

schedule/scripts/mariadb

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-set -e
-
-DESC="Exportation des bases MariaDB"
-
-. /usr/share/eole/schedule/config.sh
-
-MYSQLSAVDIR=$SAVDIR/sql
-OPTION="--lock-tables"
-
-if [[ -d ${MYSQLSAVDIR} ]] 
-then 
-   rm -rf ${MYSQLSAVDIR}
-   mkdir -p ${MYSQLSAVDIR}
-fi
-
-mariabackup --defaults-file=/etc/mysql/debian.cnf --backup --target-dir=$MYSQLSAVDIR
-exit $?

schedule_config/config.sh

@@ -0,0 +1,7 @@
+# Configuration commune aux scripts schedule
+# Configuration de base modifiée pour copier dans le partage nfs plutôt que /home
+
+SAVDIR=/mnt/sauvegardes/
+# pour que l'affichage de [ ok ] soit ok
+export TERM='dumb'
+umask 0077

schedule_extra/01_mysql.xml

@@ -2,14 +2,14 @@
 
 <creole>
     <variables>
-        <family name="mariadb" description="Exportation des bases de données MariaDB">
+        <family name="mysql" description="Exportation des bases de données MySQL">
-            <variable name="description" type="string" hidden="True"><value>Exportation des bases MariaDB</value></variable>
+            <variable name="description" type="string" hidden="True"><value>Exportation des bases MySQL</value></variable>
             <variable name="day" type="schedule" description="Périodicité d'exécution"><value>daily</value></variable>
             <variable name="mode" type="schedulemod" hidden="True"><value>pre</value></variable>
         </family>
     </variables>
     <constraints>
-        <fill name='calc_multi_condition' target='schedule.mariadb.day'>
+        <fill name='calc_multi_condition' target='schedule.mysql.day'>
             <param>non</param>
 	    <param type='eole' name='condition_1'>dbEnableBackup</param>
             <param name='match'>none</param>

schedule_scripts/mysql

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+
+DESC="Exportation des bases MySQL"
+
+. /usr/share/eole/schedule/config.sh
+
+MYSQLSAVDIR=$SAVDIR/sql
+OPTION="--lock-tables"
+
+rm -f $MYSQLSAVDIR/*.sql
+mkdir -p $MYSQLSAVDIR
+
+CMD="mysql --defaults-file=/etc/mysql/mariadb.cnf -e 'show databases' | grep -v '^Database$'"
+DATABASES=$(CreoleRun "$CMD" mysql)
+for databasename in $DATABASES; do
+    case "$databasename" in
+        information_schema|performance_schema|bareos)
+            continue
+            ;;
+        *)
+            CMD="mysqldump --defaults-file=/etc/mysql/mariadb.cnf --databases $databasename --flush-privileges --create-options -Q -c $OPTION 2>/dev/null"
+            CreoleRun "$CMD" mysql > $MYSQLSAVDIR/$databasename.sql
+            ;;
+    esac
+done

scripts/extract_hydra

@@ -1,69 +0,0 @@
-#!/bin/bash
-
-DESC="Extraction des logs de connexion de la base Hydra"
-
-. /usr/share/eole/schedule/config.sh
-
-file_date=$(date '+%Y%m%d' -d@$(($(date +%s) -3600*24))) # Date pour le format des noms de fichier
-today=$(date '+%Y-%m-%d') # Date de fin de requête
-yesterday=$(date '+%Y-%m-%d' -d@$(($(date +%s) -3600*24))) # Date de début de requête
-exit_status=0
-
-echo "Begin date: $(date)"
-
-mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
-select concat ('requested_at: ', requested_at),
-       concat ('request_id: ', request_id),
-       concat ('client_id: ', client_id),
-       concat ('subject: ', subject)
-       into outfile 'access-${file_date}' character set utf8 fields terminated by '|'
-       from hydra_oauth2_access
-       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
-       order by requested_at ;
-EOF
-(( exit_status = exit_status || $? ))
-
-mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
-select concat ('requested_at: ', requested_at),
-       concat ('challenge: ', challenge),
-       concat ('error: ', error),
-       concat ('session_access_token: ', session_access_token),
-       concat ('session_id_token: ', session_id_token),
-       concat ('handled at: ', handled_at)
-       into outfile 'consent-request-handled-${file_date}' character set utf8 fields terminated by '|'
-       from hydra_oauth2_consent_request_handled
-       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
-       order by requested_at ;
-EOF
-(( exit_status = exit_status || $? ))
-
-mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
-select concat ('requested_at: ', requested_at),
-       concat ('challenge: ', challenge),
-       concat ('request_url: ', request_url),
-       concat ('client_id: ', client_id),
-       concat ('login_session_id: ', login_session_id)
-       into outfile 'authentication-request-${file_date}' character set utf8 fields terminated by '|'
-       from hydra_oauth2_authentication_request
-       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
-       order by requested_at ;
-EOF
-(( exit_status = exit_status || $? ))
-
-# Tags must be 32 char long max (otherwise, default fwd template truncates it)
-logger -f /var/lib/mysql/hydra/access-${file_date} -t mariadb_hydra-access-${file_date}
-logger -f /var/lib/mysql/hydra/consent-request-handled-${file_date} -t mariadb_hydra-consreq-h-${file_date}
-logger -f /var/lib/mysql/hydra/authentication-request-${file_date} -t mariadb_hydra-authreq-${file_date}
-
-# Remove log files older than 7 days (already saved on the NAS)
-find /var/lib/mysql/hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
-find /var/lib/mysql/hydra/ -name 'consent-request-handled-*' -type f -mtime +7 -exec rm -vf {} \;
-find /var/lib/mysql/hydra/ -name 'authentication-request-*' -type f -mtime +7 -exec rm -vf {} \;
-
-find /var/log/rsyslog/local/mariadb_hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
-find /var/log/rsyslog/local/mariadb_hydra/ -name 'consreq-h-*' -type f -mtime +7 -exec rm -vf {} \;
-find /var/log/rsyslog/local/mariadb_hydra/ -name 'authreq-*' -type f -mtime +7 -exec rm -vf {} \;
-
-echo "End date: $(date)"
-
-exit $exit_status

scripts/get_hydra_table_sizes

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-(
-mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
-SELECT
-  TABLE_NAME AS \`Table\`,
-  ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024) AS \`Size\`
-FROM
-  information_schema.TABLES
-WHERE
-  TABLE_SCHEMA = "hydra"
-ORDER BY
-  (DATA_LENGTH + INDEX_LENGTH)
-DESC;
-EOF
-) | while read table size; do
-  if [ "${table}" != Table ]; then
-    real_size=$(ls -lh /var/lib/mysql/hydra/${table}.ibd | cut -d' ' -f 5)
-  else
-    real_size="Taille réelle"
-  fi
-  echo -e ${table}\\t${size}M\\t${real_size}
-done
-
-echo $a

scripts/optimize_hydra

@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Optimize sur les tables d'Hydra
-
-echo "Begin date: $(date)"
-
-part=$1
-
-if [ -z "$part" ]; then
-    echo 'Bad empty part. Please give a number between 1 and 7'
-    exit 1
-fi
-
-case $part in
-    1)
-	tables='hydra_oauth2_access'
-	;;
-    2)
-	tables='hydra_oauth2_oidc'
-	;;
-    3)
-	tables='hydra_oauth2_code'
-	;;
-    4)
-	tables='hydra_oauth2_authentication_request'
-	;;
-    5)
-	tables='hydra_oauth2_consent_request'
-	;;
-    6)
-	tables='hydra_oauth2_logout_request, hydra_oauth2_consent_request_handled'
-	;;
-    7)
-	tables='hydra_oauth2_authentication_session, hydra_oauth2_authentication_request_handled, hydra_oauth2_pkce'
-	;;
-    *)
-	echo "Unexpected error. Part: $part"
-	exit 1
-esac
-
-mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
-OPTIMIZE TABLE $tables;
-EOF
-
-exit_val=$?
-
-echo "End date:   $(date)"
-
-exit $exit_val

tmpl/galera.cnf

@@ -8,7 +8,12 @@ wsrep_on=ON
 wsrep_provider=/usr/lib/galera/libgalera_smm.so
 # Galera Cluster Configuration
 wsrep_cluster_name="%%dbClusterName"
-wsrep_cluster_address="gcomm://%%custom_join(%%dbClusterMember, ',')"
+%set %%nodeIP = %%getVar('adresse_ip_' + %%ifDBCluster, '')
+wsrep_cluster_address="gcomm://%%nodeIP%slurp
+%for %%node in %%dbClusterMember
+,%%node.dbClusterMemberIP%slurp
+%end for
+"
 
 # Tunning
 wsrep_provider_options="gcache.size=%%dbClusterGcacheSizeMb"
@@ -17,5 +22,5 @@ wsrep_provider_options="gcache.size=%%dbClusterGcacheSizeMb"
 wsrep_sst_method=%%dbSSTMethod
 
 # Galera Node Configuration
-wsrep_node_address="%%getVar('adresse_ip_' + %%ifDBCluster)"
+wsrep_node_address="%%nodeIP"
-wsrep_node_name="%%dbClusterMember[%%dbClusterMemberIndex]
+wsrep_node_name="%%nodeName"