Suppression de la tâche schedule extract_hydra

Raison : les tâches post en 2.7.0 ne sont pas appelées correctement
si le serveur est client d'un serveur de sauvegarde (bug)
-> Retour à une tâche cron.

README.md

@@ -1,5 +1,32 @@
 # eole-mariadb
 
+Paquet porté pour la 2.7.0. Les tests n'ont été réalisé que sur un Eolebase.
+
+La version utilisé est mariadb 10.3
+
+Ajout de ce paquet sur une 2.7.0 :
+
+## Add the Cadoles repository on all the nodes and the Leader
+
+  GenConfig  [Mode Expert] -> Dépôts Tiers
+
+ * Libellé du dépôt : Cadoles dev
+ * Déclaration du dépôt : deb [ arch=all ] https://vulcain.cadoles.com 2.7.0-dev main
+ * Méthode de récupération de la clé publique du dépôt : URL de la clé
+ * URL de la clé : https://vulcain.cadoles.com/cadoles.gpg
+
+## Add the MariaDB repository on all the nodes and the Leader
+
+GenConfig (Mode Expert) -> Dépôt tiers :
+ * Libellé du dépôt : MariaDB
+ * Déclaration du dépôt :  deb [ arch=amd64 ] http://mariadb.mirrors.ovh.net/MariaDB/repo/10.3/ubuntu bionic main
+ * Méthode de récupération de la clé publique du dépôt : serveur de clés
+ * URL du serveur de clés : hkp://keyserver.ubuntu.com:80
+ * Empreinte de la clé : 0xF1656F24C74CD1D8
+
+#######################################################################
+
+
 Début des travaux pour l'utilisation de mariadb
 dans Eole avec des fonctionalitées avancées du type
 mise en clustter
@@ -81,6 +108,7 @@ GenConfig (Mode Normal) -> Database cluster :
   If you have only 2 nodes you need to setup an arbitrator, for this follow the tutorial in the eole-galera-arbitrator project page https://forge.cadoles.com/Cadoles/eole-galera-arbitrator
 
 
+
 # How-to for Eole 2.6.2 (From scratch)
 
 ## Sources

dicos/25_bdd_server.xml

@@ -2,36 +2,38 @@
     <files>
         <file filelist='dbMariaDB' name='/etc/mysql/conf.d/mariadb.cnf' rm='True' mkdir='True'/>
         <file filelist='dbCluster' name='/etc/mysql/conf.d/galera.cnf'  rm='True' mkdir='True'/>
-
         <file filelist='dbTunning' name='/etc/mysql/conf.d/tunning.cnf' rm='True' mkdir='True'/>
+
         <service servicelist='bdd'>mariadb</service>
         <service_access service='mariadb'>
-            <port service_accesslist='mariadb' protocol='tcp'>3306</port>
-            <tcpwrapper service_accesslist='mariadb'>mariadb</tcpwrapper>
+            <port protocol='tcp'>3306</port>
+            <tcpwrapper>mariadb</tcpwrapper>
         </service_access>
-        <service_access service='galera'>
-            <port service_accesslist='dbCluster' protocol='tcp'>4444</port>
-            <port service_accesslist='dbCluster' protocol='tcp'>4567</port>
-            <port service_accesslist='dbCluster' protocol='tcp'>4568</port>
-            <port service_accesslist='dbCluster' protocol='udp'>4567</port>
-            <tcpwrapper service_accesslist='mariadb'>mariadb</tcpwrapper>
-        </service_access>
-
         <service_restriction service='mariadb'>
            <ip interface='ifMariaDBLimit' interface_type="SymLinkOption" netmask='maskMariaDBLimit' netmask_type='SymLinkOption' ip_type='SymLinkOption'>ipMariaDBLimit</ip>
         </service_restriction>
 
+        <service_access service='galera'>
+            <port service_accesslist='dbCluster' protocol='tcp'>3306</port>
+            <port service_accesslist='dbCluster' protocol='tcp'>4444</port>
+            <port service_accesslist='dbCluster' protocol='tcp'>4567</port>
+            <port service_accesslist='dbCluster' protocol='tcp'>4568</port>
+            <port service_accesslist='dbCluster' protocol='udp'>4567</port>
+        </service_access>
         <service_restriction service='galera'>
-           <ip interface='ifDBCluster' interface_type="SymLinkOption" netmask='255.255.255.255' ip_type='SymLinkOption'>dbClusterMemberIP</ip>
+           <ip interface='ifDBCluster' interface_type="SymLinkOption" netmask='255.255.255.255' ip_type='SymLinkOption'>dbClusterMember</ip>
         </service_restriction>
     </files>
     <variables>
         <family name="Services">
            <variable name="dbEnable" type='oui/non' description='Activer le serveur de base de données MariaDB'>
+              <value>oui</value>
+           </variable>
+           <variable name="dbEnableBackup" type='oui/non' description='Activer la sauvegarde sur le serveur de base de données MariaDB'>
               <value>non</value>
            </variable>
         </family>
-        <family name="Database">
+        <family name="MariaDB" icon='mysql-alt'>
             <variable name='dbEnableCluster' type='oui/non' description="Activer la mise en grappe MariaDB Galera ?">
                 <value>non</value>
             </variable> 
@@ -48,16 +50,22 @@
             </variable>
             <variable name="accLimitTarget" type="string" description="Restriction d'accès à ces IP/noms d'hôtes (liste séparateur séparateur ',')" />
 
-            <variable name="dbClusterGcacheSizeMb" type="number" description="FIXME Gcachesize">
-                <value>128</value>
-            </variable>
         </family>
 
         <!-- Tunning -->
-        <family name="Database Tunning" mode="expert">
+        <family name="MariaDB Tunning" mode="expert" icon='mysql-alt'>
             <variable name="dbInnoDBBufferPoolPercentage" type="number" description="Pourcentage de mémoire à dédier à MariaDB">
                 <value>20</value>
             </variable>
+            <variable name="dbInnoDBLogBufferSize" type='number' description="Taille du tampon utilisé pour écrire les fichiers de transaction innodb sur le disque (en Mo)">
+                <value>8</value>
+            </variable>
+            <variable name='dbInnoDBLogFileSize' type='number' description="Taille des fichiers de transaction InnoDB">
+                <value>6</value>
+            </variable>
+            <variable name='dbInnoDBLockWaitTimeout' type='number' description="Nombre de millisecondes d'attente avant de fermer la connection en lock (innodb_lock_wait_timeout)">
+                <value>120</value>
+            </variable>
             <variable name="dbMaxConnections" type="number" description="Nombre de connection maximum pour le serveur de bases de données (max_connections)">
                 <value>100</value>
             </variable>
@@ -94,20 +102,23 @@
             <variable name="dbAutoOptimizeAndRepare" type="oui/non" description="Activer l'autoréparation et optimisation de toutes les bases de données">
                 <value>non</value>
             </variable>
+            <variable name="dbMaxAllowedPacket" type='number' description="Taille maximum d'un paquet (en Mo)">
+                <value>32</value>
+            </variable>
+            <variable name='dbInteractiveTimeout' type='number' description="Nombre de secondes d'attente avant de fermer la connection en mode interactif">
+                <value>28800</value>
+            </variable>
         </family>
 
-        <family name="Database Cluster">
+        <family name="Grappe MariaDB" icon='mysql-alt'>
             <variable name='dbClusterPosition' type='string' description="Rôle dans la grappe" mandatory='True'/>
-            <variable name='ifDBCluster' type='string' description="Interface réseau dédiée à la grappe BDD"/>
-            <!-- FIXME : Fill this variable with nom_machine -->
-            <variable name='nodeName' type='string' description="Nom du noeud local"/>
-
-            <variable name='dbClusterMember' type='string' description="Membre de la grappe BDD" multi='True'/>
-            <variable name='dbClusterMemberIP' type='ip' description="Adresse IP"/>
-            <!-- Expert Variables -->
             <variable name='dbClusterName' type='string' description='Nom de la grappe'>
                <value>galera_cluster</value>
             </variable>
+            <variable name='ifDBCluster' type='string' description="Interface réseau dédiée à la grappe BDD" mandatory='True'/>
+            <variable name='dbClusterMember' type='domain' description="Nom de domaine des membres de la grappe" multi="True" mandatory='True'/>
+            <variable name="dbClusterMemberIndex" description="Index du serveur dans la liste des membres de la grappe" type="number" mandatory="True"/>
+            <!-- Expert Variables -->
             <variable name='dbBinLogFormat' type='string' mode='expert' description='Format du binlog'>
                <value>ROW</value>
             </variable>
@@ -117,7 +128,7 @@
             <variable name='dbSSTMethod' type='string' mode='expert' description='Méthode SST'>
                <value>rsync</value>
             </variable>
-            <variable name='dbClusterGcacheSize' type='number' mode='expert' description="Write-set Cache Size (gcache.size)">
+            <variable name='dbClusterGcacheSizeMb' type='number' mode='expert' description="Write-set Cache Size (gcache.size)">
                 <value>128</value>
             </variable>
             <!-- End -->
@@ -127,14 +138,17 @@
         </separators>
     </variables>
     <constraints>
+	<check name="valid_ipnetmask" target="maskMariaDBLimit" level="warning">
+            <param type='eole'>ipMariaDBLimit</param>
+        </check>	
+
+        <check name='valid_enum' target='dbBinLogFormat'>
+            <param>['ROW', 'STATEMENT', 'MIXED', 'NONE']</param>
+        </check>
         <check name='valid_enum' target='ifDBCluster'>
             <param>['eth0', 'eth1', 'eth2', 'eth3', 'eth4']</param>
         </check>
 
-        <group master='dbClusterMember'>
-            <slave>dbClusterMemberIP</slave>
-        </group>
-
         <check name='valid_enum' target='accLimits'>
             <param>['Default','Custom']</param>
         </check>
@@ -162,13 +176,12 @@
 
         <condition name='disabled_if_in' source="dbEnable">
             <param>non</param>
-            <target type='family'>Database</target>
-            <target type='family'>Database Cluster</target>
-            <target type='family'>Database Tunning</target>
-            <target type='service_accesslist'>dbCluster</target>
+            <target type='family'>MariaDB</target>
+            <!--target type='family'>Grappe MariaDB</target-->
+            <target type='family'>MariaDB Tunning</target>
             <target type='filelist'>dbTunning</target>
             <target type='filelist'>dbMariaDB</target>
-            <target type='filelist'>dbCluster</target>
+            <!--target type='filelist'>dbCluster</target-->
             <target type='servicelist'>bdd</target>
         </condition>
 
@@ -181,7 +194,7 @@
 
         <condition name='disabled_if_in' source="dbEnableCluster">
             <param>non</param>
-            <target type='family'>Database Cluster</target>
+            <target type='family'>Grappe MariaDB</target>
             <target type='service_accesslist'>dbCluster</target>
             <target type='filelist'>dbCluster</target>
         </condition>

eole-mariadb.mk

@@ -1 +1,3 @@
 creolefuncs_DATA_DIR := $(DESTDIR)/usr/share/creole/funcs
+schedule_extra_DATA_DIR := $(DESTDIR)/usr/share/eole/creole/extra/schedule/
+schedule_config_DATA_DIR := $(DESTDIR)/usr/share/eole/schedule/

postservice/00-bdd-cluster

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+if [[ "$1" == "instance" ]] && [[ "$(CreoleGet dbEnable)" == "oui" ]] && [[ "$(CreoleGet dbEnableCluster)" == "oui" ]] && [[ "$(CreoleGet dbClusterPosition)" == "Leader" ]]
+then
+    service mariadb stop
+    galera_new_cluster
+fi
+
+exit 0

postservice/25-bdd-admin

@@ -25,10 +25,11 @@ then
     accounts=($(CreoleGet accName))
     limits="$(CreoleGet accLimitTarget '')"
     accLimits=(${limits// /})
-    index=0
 
     previousUsers=($(awk -F ':' '{print $1}' ${readerfile} ))
 
+    index=0
+
     for user in ${accounts[@]}
     do
         PASS=$(awk -F ':' "/${user}:/ {print \$2}" ${readerfile})
@@ -42,13 +43,19 @@ then
         else
             ipAddr=($(CreoleGet ipMariaDBLimit))
             ipMask=($(CreoleGet maskMariaDBLimit))
-            if [[ ${ipMask} == "255.255.255.255" ]]
+            ix=0
+            for ip in ${ipAddr[@]}
+            do
+                if [[ ${ipMask[${ix}]} == "255.255.255.255" ]]
                 then
-                hostsList+=("${ipAddr}")
+                    hostsList+=("${ip}")
                 else
-                hostsList+=("${ipAddr}/${ipMask}")
+                    hostsList+=("${ip}/${ipMask[${ix}]}")
                 fi
+                ix=$((ix+1))
+            done
         fi
+
         for hst in ${hostsList[@]}
         do
             SQL="DROP USER IF EXISTS '${user}'@'${hst}';"
@@ -80,4 +87,3 @@ then
 fi
 
 exit 0
-

posttemplate/25-bdd-cluster

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-db=$(CreoleGet dbEnable non)
-cluster=$(CreoleGet dbEnableCluster non)
-role=$(CreoleGet dbClusterPosition 'Node')
-gstateFile="/var/lib/mysql/grastate.dat"
-
-if [[ ${db} == "oui" ]]
-then
-    if [[ ${cluster} == "oui" ]]
-    then
-		if [[ ${role} == "Leader"  ]]
-		then
-			service mariadb stop
-		    galera_new_cluster
-		fi
-    fi
-fi
-
-exit 0

schedule/extra/01_mariadb.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<creole>
+    <variables>
+        <family name="mariadb" description="Exportation des bases de données MariaDB">
+            <variable name="description" type="string" hidden="True"><value>Exportation des bases MariaDB</value></variable>
+            <variable name="day" type="schedule" description="Périodicité d'exécution"><value>daily</value></variable>
+            <variable name="mode" type="schedulemod" hidden="True"><value>pre</value></variable>
+        </family>
+    </variables>
+    <constraints>
+        <fill name='calc_multi_condition' target='schedule.mariadb.day'>
+            <param>non</param>
+	    <param type='eole' name='condition_1'>dbEnableBackup</param>
+            <param name='match'>none</param>
+            <param name='mismatch'>daily</param>
+        </fill>
+    </constraints>
+</creole>

schedule/scripts/mariadb

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+DESC="Exportation des bases MariaDB"
+
+. /usr/share/eole/schedule/config.sh
+
+MYSQLSAVDIR=$SAVDIR/sql
+OPTION="--lock-tables"
+
+if [[ -d ${MYSQLSAVDIR} ]] 
+then 
+   rm -rf ${MYSQLSAVDIR}
+   mkdir -p ${MYSQLSAVDIR}
+fi
+
+mariabackup --defaults-file=/etc/mysql/debian.cnf --backup --target-dir=$MYSQLSAVDIR
+exit $?

scripts/extract_hydra

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+DESC="Extraction des logs de connexion de la base Hydra"
+
+. /usr/share/eole/schedule/config.sh
+
+file_date=$(date '+%Y%m%d' -d@$(($(date +%s) -3600*24))) # Date pour le format des noms de fichier
+today=$(date '+%Y-%m-%d') # Date de fin de requête
+yesterday=$(date '+%Y-%m-%d' -d@$(($(date +%s) -3600*24))) # Date de début de requête
+exit_status=0
+
+echo "Begin date: $(date)"
+
+mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
+select concat ('requested_at: ', requested_at),
+       concat ('request_id: ', request_id),
+       concat ('client_id: ', client_id),
+       concat ('subject: ', subject)
+       into outfile 'access-${file_date}' character set utf8 fields terminated by '|'
+       from hydra_oauth2_access
+       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
+       order by requested_at ;
+EOF
+(( exit_status = exit_status || $? ))
+
+mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
+select concat ('requested_at: ', requested_at),
+       concat ('challenge: ', challenge),
+       concat ('error: ', error),
+       concat ('session_access_token: ', session_access_token),
+       concat ('session_id_token: ', session_id_token),
+       concat ('handled at: ', handled_at)
+       into outfile 'consent-request-handled-${file_date}' character set utf8 fields terminated by '|'
+       from hydra_oauth2_consent_request_handled
+       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
+       order by requested_at ;
+EOF
+(( exit_status = exit_status || $? ))
+
+mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
+select concat ('requested_at: ', requested_at),
+       concat ('challenge: ', challenge),
+       concat ('request_url: ', request_url),
+       concat ('client_id: ', client_id),
+       concat ('login_session_id: ', login_session_id)
+       into outfile 'authentication-request-${file_date}' character set utf8 fields terminated by '|'
+       from hydra_oauth2_authentication_request
+       where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
+       order by requested_at ;
+EOF
+(( exit_status = exit_status || $? ))
+
+# Tags must be 32 char long max (otherwise, default fwd template truncates it)
+logger -f /var/lib/mysql/hydra/access-${file_date} -t mariadb_hydra-access-${file_date}
+logger -f /var/lib/mysql/hydra/consent-request-handled-${file_date} -t mariadb_hydra-consreq-h-${file_date}
+logger -f /var/lib/mysql/hydra/authentication-request-${file_date} -t mariadb_hydra-authreq-${file_date}
+
+# Remove log files older than 7 days (already saved on the NAS)
+find /var/lib/mysql/hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
+find /var/lib/mysql/hydra/ -name 'consent-request-handled-*' -type f -mtime +7 -exec rm -vf {} \;
+find /var/lib/mysql/hydra/ -name 'authentication-request-*' -type f -mtime +7 -exec rm -vf {} \;
+
+find /var/log/rsyslog/local/mariadb_hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
+find /var/log/rsyslog/local/mariadb_hydra/ -name 'consreq-h-*' -type f -mtime +7 -exec rm -vf {} \;
+find /var/log/rsyslog/local/mariadb_hydra/ -name 'authreq-*' -type f -mtime +7 -exec rm -vf {} \;
+
+echo "End date: $(date)"
+
+exit $exit_status

scripts/get_hydra_table_sizes

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+(
+mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
+SELECT
+  TABLE_NAME AS \`Table\`,
+  ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024) AS \`Size\`
+FROM
+  information_schema.TABLES
+WHERE
+  TABLE_SCHEMA = "hydra"
+ORDER BY
+  (DATA_LENGTH + INDEX_LENGTH)
+DESC;
+EOF
+) | while read table size; do
+  if [ "${table}" != Table ]; then
+    real_size=$(ls -lh /var/lib/mysql/hydra/${table}.ibd | cut -d' ' -f 5)
+  else
+    real_size="Taille réelle"
+  fi
+  echo -e ${table}\\t${size}M\\t${real_size}
+done
+
+echo $a

scripts/optimize_hydra

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Optimize sur les tables d'Hydra
+
+echo "Begin date: $(date)"
+
+part=$1
+
+if [ -z "$part" ]; then
+    echo 'Bad empty part. Please give a number between 1 and 7'
+    exit 1
+fi
+
+case $part in
+    1)
+	tables='hydra_oauth2_access'
+	;;
+    2)
+	tables='hydra_oauth2_oidc'
+	;;
+    3)
+	tables='hydra_oauth2_code'
+	;;
+    4)
+	tables='hydra_oauth2_authentication_request'
+	;;
+    5)
+	tables='hydra_oauth2_consent_request'
+	;;
+    6)
+	tables='hydra_oauth2_logout_request, hydra_oauth2_consent_request_handled'
+	;;
+    7)
+	tables='hydra_oauth2_authentication_session, hydra_oauth2_authentication_request_handled, hydra_oauth2_pkce'
+	;;
+    *)
+	echo "Unexpected error. Part: $part"
+	exit 1
+esac
+
+mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
+OPTIMIZE TABLE $tables;
+EOF
+
+exit_val=$?
+
+echo "End date:   $(date)"
+
+exit $exit_val

tmpl/galera.cnf

@@ -8,12 +8,7 @@ wsrep_on=ON
 wsrep_provider=/usr/lib/galera/libgalera_smm.so
 # Galera Cluster Configuration
 wsrep_cluster_name="%%dbClusterName"
-%set %%nodeIP = %%getVar('adresse_ip_' + %%ifDBCluster, '')
-wsrep_cluster_address="gcomm://%%nodeIP%slurp
-%for %%node in %%dbClusterMember
-,%%node.dbClusterMemberIP%slurp
-%end for
-"
+wsrep_cluster_address="gcomm://%%custom_join(%%dbClusterMember, ',')"
 
 # Tunning
 wsrep_provider_options="gcache.size=%%dbClusterGcacheSizeMb"
@@ -22,5 +17,5 @@ wsrep_provider_options="gcache.size=%%dbClusterGcacheSizeMb"
 wsrep_sst_method=%%dbSSTMethod
 
 # Galera Node Configuration
-wsrep_node_address="%%nodeIP"
-wsrep_node_name="%%nodeName"
+wsrep_node_address="%%getVar('adresse_ip_' + %%ifDBCluster)"
+wsrep_node_name="%%dbClusterMember[%%dbClusterMemberIndex]

tmpl/tunning.cnf

@@ -3,6 +3,10 @@ innodb_file_per_table=1
 skip-name-resolve
 
 innodb_buffer_pool_size=%%getMemoryPercentage(%%getVar('dbInnoDBBufferPoolPercentage', 10))
+innodb_log_buffer_size=%%{dbInnoDBLogBufferSize}M
+innodb_log_file_size=%%{dbInnoDBLogFileSize}M
+innodb_lock_wait_timeout=%%{dbInnoDBLockWaitTimeout}
+
 max_connections=%%dbMaxConnections
 thread_cache_size=%%dbThreadCacheSize
 
@@ -11,10 +15,17 @@ query_cache_type = 1
 query_cache_limit = %%getVar('dbQueryCacheLimit')
 query_cache_min_res_unit = %%getVar('dbQueryCacheMinRestUnit')
 query_cache_size = %%getVar('dbQueryCacheSize')
+%else
+query_cache_type = 0
+query_cache_size = 0
 %end if
 
 tmp_table_size = %%dbTmpTableSize
 max_heap_table_size = %%dbTmpTableSize
+max_allowed_packet = %%{dbMaxAllowedPacket}M
+
+wait_timeout = %%dbWaitTimeout
+interactive_timeout = %%{dbInteractiveTimeout}
 
 %if %%dbEnableSlowQueryLogs == "oui"
 slow-query-log = 1
@@ -22,5 +33,4 @@ slow-query-log-file = %%getVar('dbSlowQueryLogFile')
 long_query_time = %%getVar('dbSlowQueryLogTime')
 %end if
 
-wait_timeout = %%dbWaitTimeout