Compare commits

..

59 Commits

Author SHA1 Message Date
Laurent Gourvenec 1096687a37 Merge branch '2.7.0/develop' into dist/eole/2.7.0/develop 2023-09-27 15:59:33 +02:00
Laurent Gourvenec 834ee16f02 Suppression de la tâche schedule extract_hydra
Raison : les tâches post en 2.7.0 ne sont pas appelées correctement
si le serveur est client d'un serveur de sauvegarde (bug)
-> Retour à une tâche cron.
2023-09-27 15:59:00 +02:00
Laurent Gourvenec 2be6389797 Merge branch '2.7.0/develop' into dist/eole/2.7.0/develop 2023-09-21 10:45:48 +02:00
Laurent Gourvenec 85d1c09aff Corrections du dico schedule extract_hydra 2023-09-13 16:28:16 +02:00
Laurent Gourvenec 52a57da5a4 Merge branch '2.7.0/develop' into dist/eole/2.7.0/develop 2023-09-06 12:06:15 +02:00
Laurent Gourvenec eb80e522f7 Renommage du script get_tables_sizes.sh en get_hydra_table_sizes
Plus significatif et proche des noms des autres scripts.
2023-09-06 11:45:10 +02:00
vfebvre a14baa049e Merge pull request 'Ajout scripts pour voir et réduire la taille des tables hydra' (#1) from f/optimize_hydra into 2.7.0/develop
Reviewed-on: #1
2023-09-06 09:48:49 +02:00
Laurent Gourvenec 765b0c9102 Ajout d'un script schedule pour réduire la taille de la bdd hydra
Extraction des données de la veille puis suppression des données
2023-09-05 17:26:20 +02:00
Laurent Gourvenec 0ab65611b0 Ajout scripts pour voir et réduire la taille des tables hydra 2023-07-26 17:19:14 +02:00
Philippe Caseiro 52eadb6bf1 Fix schedule backup script 2020-11-19 10:15:32 +01:00
vincent cf632ec6e6 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-06-07 15:08:26 +02:00
Emmanuel Garette aec567045d extract backup with mariabackup 2019-06-07 14:49:03 +02:00
vincent 558482e860 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-06-06 14:41:28 +02:00
vincent 51811c96db Merge branch 'develop' into dist/eole/2.7.0/develop 2019-05-20 10:44:58 +02:00
vincent b9fb5ff6ad Merge branch 'develop' into dist/eole/2.7.0/develop 2019-05-16 10:29:55 +02:00
vincent b609ee9761 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-04-18 15:01:21 +02:00
vincent 3f214856a4 XMerge branch 'dist/eole/2.7.0/develop' of ssh://forge.cadoles.com:4242/Cadoles/eole-mariadb into dist/eole/2.7.0/develop 2019-04-18 14:56:37 +02:00
vincent cbc84bd841 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-04-18 14:53:36 +02:00
Teddy Cornaut 0c455115c8 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-03-13 15:20:06 +01:00
vincent a0684dab2f Merge branch 'dist/eole/2.7.0/develop' of ssh://forge.cadoles.com:4242/Cadoles/eole-mariadb into dist/eole/2.7.0/develop 2019-03-11 09:47:34 +01:00
vincent 997e33daf2 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-03-11 09:45:26 +01:00
vincent 79785b4b60 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-03-04 12:08:27 +01:00
vincent c3040e3762 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-03-04 10:29:23 +01:00
vincent 2ba4b56d51 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-03-01 15:36:10 +01:00
vincent 065308a921 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-02-19 17:26:31 +01:00
vincent 21b93a3462 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-02-19 17:13:59 +01:00
vincent c540b65e08 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-02-19 10:03:53 +01:00
vincent 5e9eae5c13 Merge branch 'develop' into dist/eole/2.7.0/develop 2019-02-15 11:07:15 +01:00
Philippe Caseiro b4cccc5e12 Merge branch 'master' into dist/eole/2.6.2/master 2018-09-03 14:02:59 +02:00
Philippe Caseiro 68eaa03db2 Merge branch 'master' into dist/eole/2.6.2/master 2018-06-21 15:59:22 +02:00
Philippe Caseiro 5c7bbbde79 Merge branch 'master' into dist/eole/2.6.2/master 2018-06-21 15:28:02 +02:00
Philippe Caseiro ed910e72ee Merge branch 'master' into dist/eole/2.6.2/master 2018-06-21 13:46:21 +02:00
Philippe Caseiro 142d73afad Merge branch 'master' into dist/eole/2.6.2/master 2018-06-21 11:04:20 +02:00
Philippe Caseiro 08d7ed927c Merge branch 'master' into dist/eole/2.6.2/master 2018-06-21 10:48:13 +02:00
Philippe Caseiro f375a121ae Merge branch 'master' into dist/eole/2.6.2/master 2018-05-28 16:14:08 +02:00
Philippe Caseiro 4709c28223 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-22 16:39:58 +02:00
Philippe Caseiro 98bfcc1416 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-17 10:17:21 +02:00
Philippe Caseiro ae696769ed Merge branch 'master' into dist/eole/2.6.2/master 2018-05-17 10:14:19 +02:00
Philippe Caseiro 534e04671a Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 18:32:49 +02:00
Philippe Caseiro 61383bb17a Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 17:07:35 +02:00
Philippe Caseiro 5aa0262331 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 17:01:35 +02:00
Philippe Caseiro b550833a25 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 15:54:59 +02:00
Philippe Caseiro 517625cf66 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 15:41:23 +02:00
Philippe Caseiro 7a9855f234 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 14:26:02 +02:00
Philippe Caseiro 764ae6320c Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 13:33:15 +02:00
Philippe Caseiro d9425736c6 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 12:27:57 +02:00
Philippe Caseiro cbb1319e48 Merge branch 'master' into dist/eole/2.6.2/master 2018-05-09 12:22:39 +02:00
Philippe Caseiro be88b407b4 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 17:39:50 +02:00
Philippe Caseiro c87a989b22 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 17:20:23 +02:00
Philippe Caseiro d1e813aaa8 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 17:14:18 +02:00
Philippe Caseiro eb6854d313 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 17:11:04 +02:00
Philippe Caseiro c723edd095 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 16:34:03 +02:00
Philippe Caseiro 37e5cbb3b2 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 16:13:06 +02:00
Philippe Caseiro 4d00207271 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 16:02:01 +02:00
Philippe Caseiro 5e98dafab2 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 14:10:43 +02:00
Philippe Caseiro 1fb5fd21b8 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 12:14:23 +02:00
Philippe Caseiro 8cf9168763 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 12:07:52 +02:00
Philippe Caseiro 0dfdb70080 Merge branch 'master' into dist/eole/2.6.2/master 2018-04-12 10:51:18 +02:00
Philippe Caseiro 6d4b75c679 Adding Debian packaging directory 2018-04-12 09:30:13 +02:00
13 changed files with 222 additions and 46 deletions

View File

@ -4,8 +4,8 @@
SOURCE=eole-mariadb SOURCE=eole-mariadb
VERSION=0.0.1 VERSION=0.0.1
EOLE_VERSION=2.7 EOLE_VERSION=2.6
EOLE_RELEASE=2.7.1 EOLE_RELEASE=2.6.2
PKGAPPS=non PKGAPPS=non
#FLASK_MODULE=<APPLICATION> #FLASK_MODULE=<APPLICATION>

View File

@ -1,31 +1,5 @@
# eole-mariadb # eole-mariadb
Paquet porté pour la 2.7.1. Les tests n'ont été réalisé que sur un Eolebase.
La version utilisé est mariadb 10.3
Ajout de ce paquet sur une 2.7.1 :
## Add the Cadoles repository on all the nodes and the Leader
GenConfig [Mode Expert] -> Dépôts Tiers
* Libellé du dépôt : Cadoles dev
* Déclaration du dépôt : deb [ arch=all ] https://vulcain.cadoles.com 2.7.1-dev main
* Méthode de récupération de la clé publique du dépôt : URL de la clé
* URL de la clé : https://vulcain.cadoles.com/cadoles.gpg
## Add the MariaDB repository on all the nodes and the Leader
GenConfig (Mode Expert) -> Dépôt tiers :
* Libellé du dépôt : MariaDB
* Déclaration du dépôt : deb [ arch=amd64 ] http://mariadb.mirrors.ovh.net/MariaDB/repo/10.3/ubuntu bionic main
* Méthode de récupération de la clé publique du dépôt : serveur de clés
* URL du serveur de clés : hkp://keyserver.ubuntu.com:80
* Empreinte de la clé : 0xF1656F24C74CD1D8
#######################################################################
Paquet porté pour la 2.7.0. Les tests n'ont été réalisé que sur un Eolebase. Paquet porté pour la 2.7.0. Les tests n'ont été réalisé que sur un Eolebase.
La version utilisé est mariadb 10.3 La version utilisé est mariadb 10.3

1
debian/compat vendored Normal file
View File

@ -0,0 +1 @@
9

18
debian/control vendored Normal file
View File

@ -0,0 +1,18 @@
Source: eole-mariadb
Section: web
Priority: optional
Maintainer: Cadoles <eole@ac-dijon.fr>
Build-Depends: debhelper (>= 9)
Standards-Version: 3.9.3
Homepage: https://forge.cadoles.com/Cadoles/eole-mariadb
Vcs-Git: https://forge.cadoles.com/Cadoles/eole-mariadb.git
Vcs-Browser: https://forge.cadoles.com/Cadoles/eole-mariadb
Package: eole-mariadb
Architecture: all
Depends: ${misc:Depends}, mariadb-server, mariadb-backup
Conflicts: eole-mysql
Provides: eole-mysql
Description: Dictionnaires et templates pour la configuration d'un serveur MariaDB, testée uniquement avec eolebase
.
Pour toute information complémentaire, veuillez vous rendre sur la forge Cadoles.

44
debian/copyright vendored Normal file
View File

@ -0,0 +1,44 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: {PROJECT}
Source: {URL}
Files: *
Copyright: YEAR {UPSTREAM} {AUTHOR} <{MAIL}>
License: {UPSTREAM LICENSE}
Files: debian/*
Copyright: 2012 Équipe EOLE <eole@ac-dijon.fr>
License: CeCILL-2
License: {UPSTREAM LICENSE}
{TEXT OF THE LICENSE}
License: CeCILL-2
This software is governed by the CeCILL-2 license under French law and
abiding by the rules of distribution of free software. You can use,
modify and or redistribute the software under the terms of the CeCILL-2
license as circulated by CEA, CNRS and INRIA at the following URL
"http://www.cecill.info";.
.
As a counterpart to the access to the source code and rights to copy,
modify and redistribute granted by the license, users are provided only
with a limited warranty and the software's author, the holder of the
economic rights, and the successive licensors have only limited
liability.
.
In this respect, the user's attention is drawn to the risks associated
with loading, using, modifying and/or developing or reproducing the
software by the user in light of its specific status of free software,
that may mean that it is complicated to manipulate, and that also
therefore means that it is reserved for developers and experienced
professionals having in-depth computer knowledge. Users are therefore
encouraged to load and test the software's suitability as regards their
requirements in conditions enabling the security of their systems and/or
data to be ensured and, more generally, to use and operate it in the
same conditions as regards security.
.
The fact that you are presently reading this means that you have had
knowledge of the CeCILL-2 license and that you accept its terms.
.
On Eole systems, the complete text of the CeCILL-2 License can be found
in '/usr/share/common-licenses/CeCILL-2-en'.

3
debian/gbp.conf vendored Normal file
View File

@ -0,0 +1,3 @@
# Set per distribution debian tag
[DEFAULT]
debian-tag = debian/eole/%(version)s

8
debian/rules vendored Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/make -f
# -*- makefile -*-
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1
%:
dh $@

BIN
debian/source/.format.un~ vendored Normal file

Binary file not shown.

1
debian/source/format vendored Normal file
View File

@ -0,0 +1 @@
3.0 (native)

View File

@ -49,13 +49,7 @@
<value>Default</value> <value>Default</value>
</variable> </variable>
<variable name="accLimitTarget" type="string" description="Restriction d'accès à ces IP/noms d'hôtes (liste séparateur séparateur ',')" /> <variable name="accLimitTarget" type="string" description="Restriction d'accès à ces IP/noms d'hôtes (liste séparateur séparateur ',')" />
<variable name="create_db" type='oui/non' description="Créer des bases de données">
<value>non</value>
</variable>
</family>
<family name="MariaDB Databases">
<variable name="db_name" type='string' description='Nom de la base de donnée' multi="True"/>
<variable name="db_admin_user" type='string' description="Nom de l'utilisateur gestionnaire de la base de données"/>
</family> </family>
<!-- Tunning --> <!-- Tunning -->
@ -141,11 +135,10 @@
</family> </family>
<separators> <separators>
<separator name='dbCreateFixAdmin'>Comptes d'administration supplémentaires</separator> <separator name='dbCreateFixAdmin'>Comptes d'administration supplémentaires</separator>
<separator name='create_db'>Ajouter des bases de données</separator>
</separators> </separators>
</variables> </variables>
<constraints> <constraints>
<check name="valid_ipnetmask" target="maskMariaDBLimit" level="warning"> <check name="valid_ipnetmask" target="maskMariaDBLimit" level="warning">
<param type='eole'>ipMariaDBLimit</param> <param type='eole'>ipMariaDBLimit</param>
</check> </check>
@ -174,10 +167,6 @@
<slave>ifMariaDBLimit</slave> <slave>ifMariaDBLimit</slave>
</group> </group>
<group master='db_name'>
<slave>db_admin_user</slave>
</group>
<fill name='calc_multi_condition' target='accLimitTarget'> <fill name='calc_multi_condition' target='accLimitTarget'>
<param>Default</param> <param>Default</param>
<param type='eole' name='condition_1'>accLimits</param> <param type='eole' name='condition_1'>accLimits</param>
@ -196,11 +185,6 @@
<target type='servicelist'>bdd</target> <target type='servicelist'>bdd</target>
</condition> </condition>
<condition name='disabled_if_in' source='create_db'>
<param>non</param>
<target type='family'>MariaDB Databases</target>
</condition>
<condition name='disabled_if_in' source="dbCreateFixAdmin"> <condition name='disabled_if_in' source="dbCreateFixAdmin">
<param>non</param> <param>non</param>
<target type='variable'>accName</target> <target type='variable'>accName</target>

69
scripts/extract_hydra Normal file
View File

@ -0,0 +1,69 @@
#!/bin/bash
DESC="Extraction des logs de connexion de la base Hydra"
. /usr/share/eole/schedule/config.sh
file_date=$(date '+%Y%m%d' -d@$(($(date +%s) -3600*24))) # Date pour le format des noms de fichier
today=$(date '+%Y-%m-%d') # Date de fin de requête
yesterday=$(date '+%Y-%m-%d' -d@$(($(date +%s) -3600*24))) # Date de début de requête
exit_status=0
echo "Begin date: $(date)"
mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
select concat ('requested_at: ', requested_at),
concat ('request_id: ', request_id),
concat ('client_id: ', client_id),
concat ('subject: ', subject)
into outfile 'access-${file_date}' character set utf8 fields terminated by '|'
from hydra_oauth2_access
where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
order by requested_at ;
EOF
(( exit_status = exit_status || $? ))
mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
select concat ('requested_at: ', requested_at),
concat ('challenge: ', challenge),
concat ('error: ', error),
concat ('session_access_token: ', session_access_token),
concat ('session_id_token: ', session_id_token),
concat ('handled at: ', handled_at)
into outfile 'consent-request-handled-${file_date}' character set utf8 fields terminated by '|'
from hydra_oauth2_consent_request_handled
where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
order by requested_at ;
EOF
(( exit_status = exit_status || $? ))
mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
select concat ('requested_at: ', requested_at),
concat ('challenge: ', challenge),
concat ('request_url: ', request_url),
concat ('client_id: ', client_id),
concat ('login_session_id: ', login_session_id)
into outfile 'authentication-request-${file_date}' character set utf8 fields terminated by '|'
from hydra_oauth2_authentication_request
where requested_at >= '${yesterday} 00:00:00' and requested_at < '${today} 00:00:00'
order by requested_at ;
EOF
(( exit_status = exit_status || $? ))
# Tags must be 32 char long max (otherwise, default fwd template truncates it)
logger -f /var/lib/mysql/hydra/access-${file_date} -t mariadb_hydra-access-${file_date}
logger -f /var/lib/mysql/hydra/consent-request-handled-${file_date} -t mariadb_hydra-consreq-h-${file_date}
logger -f /var/lib/mysql/hydra/authentication-request-${file_date} -t mariadb_hydra-authreq-${file_date}
# Remove log files older than 7 days (already saved on the NAS)
find /var/lib/mysql/hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
find /var/lib/mysql/hydra/ -name 'consent-request-handled-*' -type f -mtime +7 -exec rm -vf {} \;
find /var/lib/mysql/hydra/ -name 'authentication-request-*' -type f -mtime +7 -exec rm -vf {} \;
find /var/log/rsyslog/local/mariadb_hydra/ -name 'access-*' -type f -mtime +7 -exec rm -vf {} \;
find /var/log/rsyslog/local/mariadb_hydra/ -name 'consreq-h-*' -type f -mtime +7 -exec rm -vf {} \;
find /var/log/rsyslog/local/mariadb_hydra/ -name 'authreq-*' -type f -mtime +7 -exec rm -vf {} \;
echo "End date: $(date)"
exit $exit_status

25
scripts/get_hydra_table_sizes Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash
(
mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
SELECT
TABLE_NAME AS \`Table\`,
ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024) AS \`Size\`
FROM
information_schema.TABLES
WHERE
TABLE_SCHEMA = "hydra"
ORDER BY
(DATA_LENGTH + INDEX_LENGTH)
DESC;
EOF
) | while read table size; do
if [ "${table}" != Table ]; then
real_size=$(ls -lh /var/lib/mysql/hydra/${table}.ibd | cut -d' ' -f 5)
else
real_size="Taille réelle"
fi
echo -e ${table}\\t${size}M\\t${real_size}
done
echo $a

49
scripts/optimize_hydra Executable file
View File

@ -0,0 +1,49 @@
#!/bin/bash
# Optimize sur les tables d'Hydra
echo "Begin date: $(date)"
part=$1
if [ -z "$part" ]; then
echo 'Bad empty part. Please give a number between 1 and 7'
exit 1
fi
case $part in
1)
tables='hydra_oauth2_access'
;;
2)
tables='hydra_oauth2_oidc'
;;
3)
tables='hydra_oauth2_code'
;;
4)
tables='hydra_oauth2_authentication_request'
;;
5)
tables='hydra_oauth2_consent_request'
;;
6)
tables='hydra_oauth2_logout_request, hydra_oauth2_consent_request_handled'
;;
7)
tables='hydra_oauth2_authentication_session, hydra_oauth2_authentication_request_handled, hydra_oauth2_pkce'
;;
*)
echo "Unexpected error. Part: $part"
exit 1
esac
mysql --defaults-file=/etc/mysql/debian.cnf hydra <<EOF
OPTIMIZE TABLE $tables;
EOF
exit_val=$?
echo "End date: $(date)"
exit $exit_val