augmentation du seuil d'alerte température CPU

Makefile

@@ -1,11 +1,11 @@
 ################################
-# Makefile pour eole-prometheus
+# Makefile pour XXX-XXX
 ################################
 
 SOURCE=eole-prometheus
-VERSION=0.0.1
-EOLE_VERSION=2.6
-EOLE_RELEASE=2.6.2
+VERSION=0.1
+EOLE_VERSION=2.7
+EOLE_RELEASE=2.7.0
 PKGAPPS=non
 
 ################################

README.md

@@ -1,41 +1,33 @@
-## eole-prometheus 
+# eole-prometheus 
+
 Eolisation de la solution de surveillance Prometheus.
+
 Grafana est pris en charge dans l'eolisation et peut ou non être activé.
-L'exporter système (node-exporter) est dans la configuration par défaut (Prométheus se surveille lui même).
 
-### eole-prometheus :
+L'exporter système (node-exporter) est dans la configuration par défaut (Prometheus se surveille lui même).
 
+### Installation
 
-1. gen_config
+#### Installer `eole-prometheus`
 
+1. Ajouter le dépôt officiel de [Grafana](http://docs.grafana.org/installation/debian/#apt-repository). Dans l'interface `GenConfig`
+    ```
+    Mode expert > Dépot tiers > Ajouter un dépot
+            Dépôt officiel Grafana
+                Libellé du dépot = Dépôt officiel Grafana
+                Déclaration du dépôt = deb https://packages.grafana.com/oss/deb stable main
+                Méthode de récupération de la clé = URL de la clé
+                URL de la clé = https://packages.grafana.com/gpg.key
+    ```
+2. Ajouter le dépôt Cadoles. Dans l'interface `GenConfig`
+    ```
+    Mode expert > Dépot tiers > Ajouter un dépot
+            Cadoles pour environnement de Dev
+                Libellé du dépot = Cadoles
+                Déclaration du dépôt = deb https://vulcain.cadoles.com 2.7.0-dev main
+                Méthode de récupération de la clé = URL de la clé
+                URL de la clé = https://vulcain.cadoles.com/cadoles.gpg
     ```
-Mode expert > Dépot tiers > Ajouter un dépot
-        Cadoles pour environnement de Qualification
-            Libellé du dépot = Cadoles
-            Déclaration du dépôt = deb https://vulcain.cadoles.com 2.6.2-staging main
-            Méthode de récupération de la clé = URL de la clé
-            URL de la clé = https://vulcain.cadoles.com/cadoles.gpg
-```
-
-* Pour ajouter une sonde sur eole :
-
-1. gen_config
-
-   ```
- Mode expert > Dépot tiers > Ajouter un dépot
-        Cadoles pour environnement de Qualification
-            Libellé du dépot = Cadoles
-            Déclaration du dépôt = deb https://vulcain.cadoles.com xenial-staging main
-            Méthode de récupération de la clé = URL de la clé
-            URL de la clé = https://vulcain.cadoles.com/cadoles.gpg
-```
-
-* Pour ajouter une sonde sur ubuntu xenial:
-
-```
-echo "deb https://vulcain.cadoles.com xenial-staging main" > /etc/apt/sources.list.d/cadoles.list
-wget -O - https://vulcain.cadoles.com/cadoles.gpg|apt-key add -
-```
 
 Il faut ouvrir les ports en fonction des exporters. Tous les exporters n'utilisent pas le même port.
 
@@ -44,4 +36,3 @@ Le paquet eole-prometheus ouvre les ports sur le serveur où Prometheus sera ins
 * 9090 pour le serveur prometheus
 * 9100 pour la sonde node-exporter
 * 3000 pour le serveur Grafana
-

debian/compat

@@ -1 +0,0 @@
-7

debian/control

@@ -1,11 +0,0 @@
-Source: eole-prometheus
-Section: web
-Priority: optional
-Maintainer: Cadoles <contact@cadoles.com>
-Build-Depends: debhelper (>= 9)
-Standards-Version: 3.9.3
-
-Package: eole-prometheus
-Architecture: amd64
-Depends: ${misc:Depends}, prometheus, grafana, eole-node-exporter, prometheus-alertmanager, curl
-Description: Eolisation de Prometheus

debian/copyright

@@ -1,44 +0,0 @@
-Format: http://dep.debian.net/deps/dep5
-Upstream-Name: {PROJECT}
-Source: {URL}
-
-Files: *
-Copyright: YEAR {UPSTREAM} {AUTHOR} <{MAIL}>
-License: {UPSTREAM LICENSE}
-
-Files: debian/*
-Copyright: 2012 Équipe EOLE <eole@ac-dijon.fr>
-License: CeCILL-2
-
-License: {UPSTREAM LICENSE}
- {TEXT OF THE LICENSE}
-
-License: CeCILL-2
- This software is governed by the CeCILL-2 license under French law and
- abiding by the rules of distribution of free software.  You can  use,
- modify and or redistribute the software under the terms of the CeCILL-2
- license as circulated by CEA, CNRS and INRIA at the following URL
- "http://www.cecill.info";.
- .
- As a counterpart to the access to the source code and  rights to copy,
- modify and redistribute granted by the license, users are provided only
- with a limited warranty  and the software's author,  the holder of the
- economic rights,  and the successive licensors  have only  limited
- liability.
- .
- In this respect, the user's attention is drawn to the risks associated
- with loading,  using,  modifying and/or developing or reproducing the
- software by the user in light of its specific status of free software,
- that may mean  that it is complicated to manipulate,  and  that  also
- therefore means  that it is reserved for developers  and  experienced
- professionals having in-depth computer knowledge. Users are therefore
- encouraged to load and test the software's suitability as regards their
- requirements in conditions enabling the security of their systems and/or
- data to be ensured and,  more generally, to use and operate it in the
- same conditions as regards security.
- .
- The fact that you are presently reading this means that you have had
- knowledge of the CeCILL-2 license and that you accept its terms.
- .
- On Eole systems, the complete text of the CeCILL-2 License can be found
- in '/usr/share/common-licenses/CeCILL-2-en'.

debian/rules

@@ -1,8 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-# Uncomment this to turn on verbose mode.
-# export DH_VERBOSE=1
-
-%:
-	dh $@

debian/source/format

@@ -1 +0,0 @@
-3.0 (quilt)

dicos/70_prometheus.xml

@@ -1,18 +1,22 @@
 <?xml version="1.0" encoding="utf-8"?>
 <creole>
     <files>
-		<file filelist='prometheus'   name='/etc/default/prometheus' source='prometheus.defaults'  mkdir='True' rm='True'/>
+      <file filelist='prometheus'   name='/etc/default/prometheus' source='prometheus.defaults'  mkdir='True' rm='True'/>
         <file filelist='prometheus'   name='/etc/prometheus/prometheus.yml'          mkdir='True' rm='True'/>
-        <file filelist='alertmanager' name='/etc/prometheus/alertmanager.yml'        mkdir='True' rm='True'/>
-		<file filelist='alertmanager' name='/etc/prometheus/rules.d/alert-rules.yml' mkdir='True' rm='True'/>
+        <file filelist='prometheus-alertmanager' name='/etc/prometheus/alertmanager.yml'        mkdir='True' rm='True'/>
+        <file filelist='prometheus-alertmanager' name='/etc/prometheus/rules.d/alert-rules.yml' mkdir='True' rm='True'/>
+        <file filelist='prometheus-alertmanager' name='/etc/prometheus/rules.d/predict-rules.yml' mkdir='True' rm='True'/>
         <file filelist='grafana'      name='/etc/grafana/grafana.ini'                mkdir='True' rm='True'/>
+        <file filelist='grafana'      name='/etc/grafana/provisioning/dashboards/eole.yml' source='grafana-dashboards.yml'     mkdir='True' rm='True'/>
+        <file filelist='grafana'      name='/etc/grafana/provisioning/datasources/eole.yml' source='grafana-datasources.yml'     mkdir='True' rm='True'/>
+        <file filelist='grafana'      name='/var/lib/grafana/dashboards/eole.json' source='grafana-node-dashboard.json'     mkdir='True' rm='True'/>
 
         <service>prometheus</service>
-        <service>alertmanager</service>
+        <service>prometheus-alertmanager</service>
         <service>grafana-server</service>
         <service_access service='prometheus'>
             <port service_accesslist="prometheus">9090</port>
-            <port service_accesslist="alertmanager">9093</port>
+            <port service_accesslist="prometheus-alertmanager">9093</port>
         </service_access>
         <service_access service='grafana-server'>
             <port service_accesslist="grafana">3000</port>
@@ -58,7 +62,7 @@
 
             <!-- Job standard  -->
             <variable name='prTarg' type='string' description='Nom de la cible prometheus' multi='True'/>
-            <variable name='prTargIP' type='ip' description="Adresse IP de la cible prometheus"/>
+            <variable name='prTargIP' type='string' description="Adresse IP ou nom de domaine de la cible prometheus"/>
             <variable name='prTargSonde' type='string' description="Sonde a utiliser pour ce client">
                 <value>Node Exporter</value>
             </variable>
@@ -77,6 +81,9 @@
             <variable name='scrpScheme' type='string' description="Protocole à utiliser pour l'interrogation de la sonde">
                 <value>http</value>
             </variable>
+           <variable name='scrpMetricPath' type='string' description="Chemin d'accès de la ressource">
+                <value>/metrics</value>
+            </variable>
 
             <variable name='addPrOpenTarg' type='oui/non' description="Ajouter des cibles statiques pour les jobs personnalisé">
                 <value>non</value>
@@ -84,27 +91,28 @@
             <!-- Job libre  -->
             <variable name='prOpenTarg' type='string' description='Nom de la cible personnalisé prometheus' multi='True'/>
             <variable name='prOpenTargJob' type='string' description='Nom du job de rattachement de la cible'/>
-            <variable name='prOpenTargIP' type='ip' description="Adresse IP de la cible"/>
+            <variable name='prOpenTargIP' type='string' description="Adresse IP ou nom de domaine de la cible"/>
             <variable name='prOpenTargPort' type='number' description="Port d'écoute de la sonde"/>
         </family>
 
         <family name="grafana">
-                    <variable name='grafana_domain' type='string' description="Nom de Domaine ou IP pour accèder à l'interface Grafana" mandatory='True'>
-                        <value>localhost</value>
-                    </variable>
-                    <variable name='grafana_session_max_lifetime' type='string' description="Durée avant déconnexion de l'interface Grafana (en seconde)">
-                        <value>86400</value>
-                    </variable>
-                    <variable name='grafana_admin_passwd' type='string' description="Mot de passe admin pour la première connexion">
-                        <value>admin</value>
-                    </variable>
-                    <variable name='grafana_sign_up' type='string' description="Activer l'enregistrement automatique">
-                        <value>false</value>
-                    </variable>
-                    <variable name='grafana_auth_anonymous' type='string' description="Activer l'accès aux utilisateurs non enregistrés">
-                        <value>false</value>
-                    </variable>
-                </family>
+            <variable name='grafana_domain' type='string' description="Nom de Domaine ou IP pour accèder à l'interface Grafana" mandatory='True'>
+                <value>localhost</value>
+            </variable>
+            <variable name='grafana_session_max_lifetime' type='string' description="Durée avant déconnexion de l'interface Grafana (en seconde)">
+                <value>86400</value>
+            </variable>
+            <variable name='grafana_admin_passwd' type='string' description="Mot de passe admin pour la première connexion">
+                <value>admin</value>
+            </variable>
+            <variable name='grafana_sign_up' type='string' description="Activer l'enregistrement automatique">
+                <value>false</value>
+            </variable>
+            <variable name='grafana_auth_anonymous' type='string' description="Activer l'accès aux utilisateurs non enregistrés">
+                <value>false</value>
+            </variable>
+            <variable name='grafanaRootURL' type='string' description='Url publique de grafana (avec http:// ou https://)' mode='expert'/>
+        </family>
 
         <family name="alertes prometheus">
 			<variable name='alSMTPUseSys' type='oui/non' description="Utiliser la passerelle SMTP du système ?">
@@ -147,7 +155,7 @@
     <separators>
         <separator name='activer_grafana'>Services complèmentairse</separator>
         <separator name='prometheusJobName'>Configuration du serveur Prometheus</separator>
-		<separator name='job_name_node'>Configuration des jobs standards</separator>
+        <separator name='job_name_node'>Configuration des jobs standards</separator>
         <separator name='alSMTPHost'>Configuration SMTP pour l'envois des alertes</separator>
         <separator name='alReceiver'>Destinatires</separator>
         <separator name='alRoute'>Rêgles de distribution simples</separator>
@@ -166,7 +174,8 @@
             <slave>scrpInterval</slave>
             <slave>scrpTimeout</slave>
             <slave>honorLabels</slave>
-			<slave>scrpScheme</slave>
+	    <slave>scrpScheme</slave>
+            <slave>scrpMetricPath</slave>
         </group>
 
         <group master='alRoute'>
@@ -251,8 +260,8 @@
         <condition name='disabled_if_in' source='activerAlertmanager'>
             <param>non</param>
             <target type='family'>alertes prometheus</target>
-            <target type='filelist'>alertmanager</target>
-            <target type='service_accesslist'>alertmanager</target>
+            <target type='filelist'>prometheus-alertmanager</target>
+            <target type='service_accesslist'>prometheus-alertmanager</target>
         </condition>
 
         <condition name='disabled_if_in' source='addTargetPrometheus'>

postservice/88_grafana

@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-
-
-function importDataSource()
-{
-    local grURL="${1}/api/datasources"  # Grafana API URL
-    local dsName=${2} # Datasource Name
-    local dsURL="${3}" # Datasource URL
-    local dsType="prometheus" # Datasource type
-
-    cmd="curl"
-
-
-    data=$(cat <<__EOF__
-{"name":"${dsName}","type":"${dsType}","url":"${dsURL}","access":"direct"}
-__EOF__
-    )
-
-    echo -ne "\tCreating datasource for Prometheus "
-    res=$(${cmd} "${grURL}" -H "Content-Type: application-json" --data-binary "${data}" 2>&1 )
-    excode=${?}
-    case $res in
-        *"already exists"*)
-            echo " ... [Exists]"
-            ;;
-        *"Datasource added"*)
-            echo " ... [OK]"
-            ;;
-        *)
-            echo " ... Ooops ${res}"
-            ;;
-    esac
-}
-
-function importDashboardFromMarket()
-{
-    local grURL="${1}/api/dashboards/import" # API URL to create new Dashboard
-    local dsName="${2}"                  # Datasource name (to link dashboard to data)
-    local dhID="${3}"                    # Dashboard ID in the market
-    local dhRev="${4}"                   # Dashboard Revision to download
-    local tmpFile=$(mktemp)
-    local dhFile=$(mktemp)
-
-    # URL To download the json file for dashboard
-    local pubDashBoardURL="https://grafana.com/api/dashboards/${dhID}/revisions/${dhRev}/download"
-
-    local cmd="curl"
-
-    local dh=$(${cmd} --silent ${pubDashBoardURL} 2>&1)
-
-    cat <<_EOF_ > ${tmpFile}
-{
-  "inputs": [ { "name":"DS_LOCALHOST", "type":"datasource", "pluginId":"prometheus", "value":"${dsName}" } ],
-  "dashboard": { "title": "Surveillance Système", ${dh:1:-1} },
-  "folderId": 0,
-  "overwrite": true
-}
-_EOF_
-
-    res=$(${cmd} "${grURL}" -H "Content-Type: application-json" --data-binary "@${tmpFile}" 2>&1 )
-    excode=${?}
-    case $res in
-        *"\"imported\":true"*)
-            echo " ... [Overwrited]"
-            ;;
-        *"Datasource added"*)
-            echo " ... [OK]"
-            ;;
-        *)
-            echo " ... Ooops ${res}"
-            ;;
-    esac
-#    rm -rf ${tmpFile}
-}
-
-grafanaHost=$(CreoleGet srvGrafanaIP 127.0.0.1)
-grafanaPort=$(CreoleGet srvGrafanaPort 3000)
-grafanaUser="admin"
-grafanaPasswd=$(CreoleGet grafana_admin_passwd admin)
-grafanaURL="http://${grafanaUser}:${grafanaPasswd}@${grafanaHost}:${grafanaPort}"
-datasource_name=$(CreoleGet promDataSource "prometheus")
-promHost=$(CreoleGet adresse_ip_eth0)
-promPort='9090'
-datasourceURL="http://${promHost}:${promPort}"
-
-importDataSource ${grafanaURL} ${datasource_name} ${datasourceURL}
-echo
-echo -ne "\tImporting Node Exporter Full Dashboard "
-importDashboardFromMarket ${grafanaURL} ${datasource_name} 1860 11
-
-rm -rf ${dashBoardFile}

tmpl/alert-rules.yml

@@ -33,9 +33,18 @@ groups:
   # Heavy "/" use
   - alert: filesystem_threshold_exceeded
     expr: node_filesystem_avail{job="%%{job_name_node}",mountpoint="/"} / node_filesystem_size{job="%%{job_name_node}"}
-      * 100 < 90
+      * 100 < 20
     annotations:
       description: This device's filesystem usage has exceeded the threshold with
         a value of {{ $value }}.
       summary: Instance {{ $labels.instance }} filesystem usage is dangerously high
 
+  # Heavy CPU temperature
+  - alert: cpu_temp_threshold_exceeded
+    expr: avg(node_hwmon_temp_celsius{job="node"}) BY (instance)
+      > 70
+    annotations:
+      description: This device's cpu temperature has exceeded the threshold with a value
+        of {{ $value }}.
+      summary: Instance {{ $labels.instance }} CPU temperature is dangerously high
+

tmpl/alertmanager.yml

@@ -4,12 +4,13 @@ global:
   %if %%tls_smtp == "non"
   smtp_smarthost: '%%exim_relay_smtp:25'
   %elif %%tls_smtp == "port 25"
-  smtp_smarthost: '%%exim_relay_smtp:25'
-  smtp_require_tls: true
-  %else
-  smtp_smarthost: '%%exim_relay_smtp:465'
-  smtp_require_tls: true
+    smtp_smarthost: '%%exim_relay_smtp:25'
+    smtp_require_tls: true
+    %else
+    smtp_smarthost: '%%exim_relay_smtp:465'
+    smtp_require_tls: true
   %end if
+  smtp_from: '%%system_mail_from'
 %else
   smtp_smarthost: '%%alSMTPHost:%%alSMTPPort'
   smtp_from: '%%alFrom'
@@ -31,8 +32,8 @@ global:
   #hipchat_api_url: 'https://hipchat.foobar.org/'
 
 # The directory from which notification templates are read.
-templates: 
-- '/etc/alertmanager/template/*.tmpl'
+templates:
+  - '/etc/alertmanager/template/*.tmpl'
 
 # The root route on which each incoming alert enters.
 route:
@@ -54,12 +55,12 @@ route:
 
   # If an alert has successfully been sent, wait 'repeat_interval' to
   # resend them.
-  repeat_interval: 3h 
+  repeat_interval: 3h
 
   # A default receiver
   receiver: %%alDefaultReceiver
 
-  # All the above attributes are inherited by all child routes and can 
+  # All the above attributes are inherited by all child routes and can
   # overwritten on each.
 
   # The child route trees.
@@ -80,6 +81,7 @@ route:
     - match:
         %%{sroute.alSubRouteMatchSource}: %%alSubRouteMatchValue
       receiver: %%alSubRouteMatchReceiver
+      continue: true
       %end if
     %end for
   %end if
@@ -88,6 +90,7 @@ route:
   - match:
       %%{rt.alRouteMatchSource}: %%{rt.alRouteMatchValue}
     receiver: %%rt.alRouteMatchReceiver
+    continue: true
 
   %if not %%is_empty('alSubRoute')
     routes:
@@ -96,6 +99,7 @@ route:
     - match:
         %%{sroute.alSubRouteMatchSource}: %%{sroute.alSubRouteMatchValue}
       receiver: %%sroute.alSubRouteMatchReceiver
+      continue: true
       %end if
     %end for
   %end if
@@ -119,7 +123,7 @@ route:
 
 # Inhibition rules allow to mute a set of alerts given that another alert is
 # firing.
-# We use this to mute any warning-level notifications if the same alert is 
+# We use this to mute any warning-level notifications if the same alert is
 # already critical.
 inhibit_rules:
 - source_match:

tmpl/grafana-dashboards.yml

@@ -0,0 +1,10 @@
+# # config file version
+apiVersion: 1
+
+providers:
+- name: 'eole'
+  orgId: 1
+  folder: ''
+  type: file
+  options:
+    path: /var/lib/grafana/dashboards

tmpl/grafana-datasources.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+- name: Prometheus
+  type: prometheus
+  access: proxy
+  orgId: 1
+  url: http://%%adresse_ip_eth0:9090
+  isDefault: true
+  version: 1
+  editable: false

tmpl/grafana.ini

@@ -21,7 +21,7 @@
 ;plugins = /var/lib/grafana/plugins
 
 # folder that contains provisioning config files that grafana will apply on startup and while running.
-; provisioning = conf/provisioning
+provisioning = /etc/grafana/provisioning
 
 #################################### Server ####################################
 [server]
@@ -40,11 +40,13 @@ domain = %%grafana_domain
 
 # Redirect to correct domain if host header does not match domain
 # Prevents DNS rebinding attacks
-;enforce_domain = false
+enforce_domain = true
 
 # The full public facing url you use in browser, used for redirects and emails
 # If you use reverse proxy and sub path specify full url (with sub path)
-;root_url = http://localhost:3000
+%if %%is_empty('grafanaRootURL')
+root_url = %%grafanaRootURL
+%end if
 
 # Log web requests
 ;router_logging = false
@@ -299,18 +301,20 @@ enabled = %%grafana_auth_anonymous
 
 #################################### SMTP / Emailing ##########################
 [smtp]
-;enabled = false
-;host = localhost:25
+%if %%getVar('activer_exim_relay_smtp','non') == 'oui'
+enabled = true
+host = %%exim_relay_smtp:25
 ;user =
 # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
 ;password =
 ;cert_file =
 ;key_file =
-;skip_verify = false
-;from_address = admin@grafana.localhost
-;from_name = Grafana
+skip_verify = true
+from_address = %%system_mail_from
+from_name = Grafana
 # EHLO identity in SMTP dialog (defaults to instance_name)
 ;ehlo_identity = dashboard.example.com
+%end if
 
 [emails]
 ;welcome_email_on_sign_up = false

tmpl/predict-rules.yml

@@ -0,0 +1,6 @@
+groups:
+  - name: PredictRules
+    rules:
+      - alert: disk_full_within_6_hours
+        expr: predict_linear(node_filesystem_free{job="%%{job_name_node}",mountpoint="/"}[1h], 6 * 3600) < 0
+        for: 5m

tmpl/prometheus.defaults

@@ -1 +1 @@
-PROMETHEUS_OPTS='--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/data --storage.local.retention=%%{promStorageRetention}h"
+PROMETHEUS_OPTS='--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/data'

tmpl/prometheus.yml

@@ -22,7 +22,7 @@ scrape_configs:
 
   - job_name: '%%job_name_node'
     file_sd_configs:
-      - files: [ "%%job_file_config/*.yml" ]
+#      - files: [ "%%job_file_config/*.yml" ]
 %if %%getVar('addTargetPrometheus','non') == 'oui'
     static_configs:
         - targets: [ "%%adresse_ip_eth0:9100"%slurp
@@ -44,6 +44,7 @@ scrape_configs:
     scrape_interval: %%{job.scrpInterval}s
     scrape_timeout: %%{job.scrpTimeout}s
     scheme: %%job.scrpScheme
+    metrics_path: %%job.scrpMetricPath
   %set first = True
     static_configs:
       - targets: [ %slurp