From 638295626b78348ba0e324a09005840af45bcf89 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:28:55 +0200 Subject: [PATCH 01/13] Try 001 --- dicos/70_prometheus.xml | 21 +++++++++++++++++---- tmpl/prometheus.yml | 8 ++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index 384aa48..9c181e4 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -36,10 +36,6 @@ 10s - - non - - node @@ -47,6 +43,14 @@ /etc/prometheus/nodes + + + non + + + + + localhost @@ -66,6 +70,15 @@ + + prCliIP + prCliSonde + + + + ['Node Exporter'] + + non prometheus diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index a2034bc..7df2f07 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -22,11 +22,11 @@ scrape_configs: - files: [ "%%job_file_config/*.yml" ] static_configs: %if %%getVar('ajout_client_prometheus','non') == 'oui' - %if not %%is_empty(%%nouveau_node_exporter) - %for %%client_prometheus in %%nouveau_node_exporter + %for %%cliPr in %%getVar('prCli',[]) + %if %%cliPr.CliSonde == 'Node Exporter' - targets: ['%%client_prometheus:9100'] - %end for - %end if + %end if + %end for %end if #alerting: From f9be6604236fa649f399443cf142edae110b1e4c Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:30:13 +0200 Subject: [PATCH 02/13] Try 002 --- dicos/70_prometheus.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index 9c181e4..8a8bdad 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -92,7 +92,9 @@ non - nouveau_node_exporter + prCli + prCliIP + prCliSonde From 325acbf537e9925fcbf6663d42d665b3f9a488bb Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:33:36 +0200 Subject: [PATCH 03/13] Try 003 --- dicos/70_prometheus.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index 8a8bdad..7aa9549 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -49,7 +49,9 @@ - + + Node Exporter + From 38883c4fcac0245e4420e0d3440634bc131a9243 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:36:34 +0200 Subject: [PATCH 04/13] Try 004 --- tmpl/prometheus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index 7df2f07..e7607f8 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -23,8 +23,8 @@ scrape_configs: static_configs: %if %%getVar('ajout_client_prometheus','non') == 'oui' %for %%cliPr in %%getVar('prCli',[]) - %if %%cliPr.CliSonde == 'Node Exporter' - - targets: ['%%client_prometheus:9100'] + %if %%cliPr.prCliSonde == 'Node Exporter' + - targets: ['%%cliPr.prCliIP:9100'] %end if %end for %end if From d815f156e3a39c4dba088521fe026ff5610870e6 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:42:19 +0200 Subject: [PATCH 05/13] Try 005 --- dicos/70_prometheus.xml | 8 +++++++- tmpl/prometheus.yml | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index 7aa9549..05d092a 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -52,6 +52,7 @@ Node Exporter + @@ -78,7 +79,7 @@ - ['Node Exporter'] + ['Node Exporter','Port'] @@ -97,7 +98,12 @@ prCli prCliIP prCliSonde + prCliPort + + Port + prCliPort + diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index e7607f8..b2cab46 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -26,6 +26,9 @@ scrape_configs: %if %%cliPr.prCliSonde == 'Node Exporter' - targets: ['%%cliPr.prCliIP:9100'] %end if + %if %%cliPr.prCliSonde == 'Port' + - targets: ['%%cliPr.prCliIP:%%cliPr.prCliPort'] + %end if %end for %end if From 3eaa7feee16d88bbab23b08fe1501a45366c0cdf Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:47:01 +0200 Subject: [PATCH 06/13] Try 006 --- dicos/70_prometheus.xml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index 05d092a..b407c78 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -47,12 +47,20 @@ non + Node Exporter - + + + non + + + + + @@ -98,11 +106,12 @@ prCli prCliIP prCliSonde - prCliPort - - Port - prCliPort + + non + prOpenCli + prOpenCliIP + prOpenCliPort From 4bd29eaa043f94ae29ea2b95e316e6a71893cf9d Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:48:47 +0200 Subject: [PATCH 07/13] Try 006bis --- tmpl/prometheus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index b2cab46..8bf2f51 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -26,9 +26,9 @@ scrape_configs: %if %%cliPr.prCliSonde == 'Node Exporter' - targets: ['%%cliPr.prCliIP:9100'] %end if - %if %%cliPr.prCliSonde == 'Port' - - targets: ['%%cliPr.prCliIP:%%cliPr.prCliPort'] - %end if + %end for + %for %%cliOpen in %%getVar('prOpenCli',[]) + - targets: ['%%cliOpen.prOpenCliIP:%%cliOpen.prOpenCliPort'] %end for %end if From f61a15d609bfced690bd395ec75d569cf43ac1e6 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 10:51:53 +0200 Subject: [PATCH 08/13] Try 007 --- dicos/70_prometheus.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dicos/70_prometheus.xml b/dicos/70_prometheus.xml index b407c78..edc0de9 100644 --- a/dicos/70_prometheus.xml +++ b/dicos/70_prometheus.xml @@ -86,6 +86,11 @@ prCliSonde + + prOpenCliIP + prOpenCliPort + + ['Node Exporter','Port'] From 4a24601beaf1fc3fa5942fb9b80e6df030ce7ba6 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 11:16:18 +0200 Subject: [PATCH 09/13] Try 008 --- tmpl/prometheus.yml | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index 8bf2f51..e360435 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -8,30 +8,38 @@ rule_files: - "/etc/prometheus/rules.d/*.yml" scrape_configs: - - job_name: %%prometheusJobName + - job_name: %%prometheusJobName honor_labels: true static_configs: - - targets: ['%%nom_domaine_machine:9090'] + - targets: ['%%nom_domaine_machine:9090'%slurp %if %%getVar('activerSndNodeExpoter','non') == 'oui' - - targets: ['%%nom_domaine_machine:9100'] +, '%%nom_domaine_machine:9100'%slurp %end if +] - job_name: '%%job_name_node' file_sd_configs: - files: [ "%%job_file_config/*.yml" ] +%if %%is_empty('ajout_client_prometheus') or %%is_empty('addPrOpenCli') +# Nothing to configure +%else static_configs: -%if %%getVar('ajout_client_prometheus','non') == 'oui' - %for %%cliPr in %%getVar('prCli',[]) - %if %%cliPr.prCliSonde == 'Node Exporter' - - targets: ['%%cliPr.prCliIP:9100'] + - targets: [ "127.0.0.1:9100"%slurp + %if %%getVar('ajout_client_prometheus','non') == 'oui' + %for %%cliPr in %%getVar('prCli',[]) + %if %%cliPr.prCliSonde == 'Node Exporter' +, '%%cliPr.prCliIP:9100'%slurp + %end if + %end for %end if - %end for - %for %%cliOpen in %%getVar('prOpenCli',[]) - - targets: ['%%cliOpen.prOpenCliIP:%%cliOpen.prOpenCliPort'] - %end for + %if %%getVar('addPrOpenCli','non') == 'oui' + %for %%cliOpen in %%getVar('prOpenCli',[]) +, '%%cliOpen.prOpenCliIP:%%cliOpen.prOpenCliPort'%slurp + %end for + %end if +] %end if - #alerting: # alertmanagers: # - scheme: https From 3df296b3663c0ff5795fd6bf875c48760862f3db Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 13:59:45 +0200 Subject: [PATCH 10/13] Adding datasource --- postservice/88_grafana | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 postservice/88_grafana diff --git a/postservice/88_grafana b/postservice/88_grafana new file mode 100644 index 0000000..6c01a7c --- /dev/null +++ b/postservice/88_grafana @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import requests +import json + +from creole.client import CreoleClient + +creole = c = CreoleClient() + +grafanaHost = creole.get_creole('srvGrafanaIP','127.0.0.1') +grafanaPort = creole.get_creole('srvGrafanaPort', '3000') +grafanaURL = os.path.join('http://', '%s:%s' % (grafanaHost, grafanaPort)) +grafanaUser = "admin" +grafanaPasswd = creole.get_creole('grafana_admin_passwd') +datasource_name = creole.get_creole('promDataSource',"prometheus") +promHost = creole.get_creole('addresse_ip_eth0') +promPort = '9090' + +session = requests.Session() +login_post = session.post ( + os.path.join(grafanaURL, 'login'), + data=json.dumps({ + 'user': grafanaUser, + 'email': '', + 'password': grafanaPasswd + }), + headers={'content-type': 'application/json'} +) + +# Get list of datasources +datasources_get = session.get(os.path.join(grafanaURL, 'api', 'datasources')) +datasources = datasources_get.json() + +if datasources['message'] == "Unauthorized" : + exit(34) + +# Add new datasource +datasources_put = session.put( + os.path.join(grafanaURL, 'api', 'datasources'), + data=json.dumps({ + 'access': 'direct', + 'name': datasource_name, + 'type': 'prometheus', + 'url': 'http://%s:%u' % (promHost, promPort), + 'basicAuth': 'false', + 'withCredentials': 'false' + }), + headers={'content-type': 'application/json'} +) From a3def897fa97a4f1715f7e47c5d321624c8f113a Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 4 Jun 2018 17:01:42 +0200 Subject: [PATCH 11/13] Moving to bash --- postservice/88_grafana | 69 ++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 43 deletions(-) mode change 100644 => 100755 postservice/88_grafana diff --git a/postservice/88_grafana b/postservice/88_grafana old mode 100644 new mode 100755 index 6c01a7c..10592a1 --- a/postservice/88_grafana +++ b/postservice/88_grafana @@ -1,51 +1,34 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env bash -import os -import requests -import json +grafanaHost=$(CreoleGet srvGrafanaIP 192.168.30.57) +grafanaPort=$(CreoleGet srvGrafanaPort 3000) +grafanaUser="admin" +grafanaPasswd=$(CreoleGet grafana_admin_passwd admin) +grafanaURL="http://${grafanaUser}:${grafanaPasswd}@${grafanaHost}:${grafanaPort}" +datasource_name=$(CreoleGet promDataSource "prometheus") +promHost=$(CreoleGet adresse_ip_eth0) +promPort='9090' -from creole.client import CreoleClient -creole = c = CreoleClient() +cmd="curl" +url="${grafanaURL}/api/datasources" -grafanaHost = creole.get_creole('srvGrafanaIP','127.0.0.1') -grafanaPort = creole.get_creole('srvGrafanaPort', '3000') -grafanaURL = os.path.join('http://', '%s:%s' % (grafanaHost, grafanaPort)) -grafanaUser = "admin" -grafanaPasswd = creole.get_creole('grafana_admin_passwd') -datasource_name = creole.get_creole('promDataSource',"prometheus") -promHost = creole.get_creole('addresse_ip_eth0') -promPort = '9090' - -session = requests.Session() -login_post = session.post ( - os.path.join(grafanaURL, 'login'), - data=json.dumps({ - 'user': grafanaUser, - 'email': '', - 'password': grafanaPasswd - }), - headers={'content-type': 'application/json'} +data=$(cat <<__EOF__ +{"name":"${datasource_name}","type":"prometheus","url":"http://${promHost}:${promPort}","access":"direct"} +__EOF__ ) -# Get list of datasources -datasources_get = session.get(os.path.join(grafanaURL, 'api', 'datasources')) -datasources = datasources_get.json() +echo -ne "\tCreating datasource for Prometheus " +res=$(${cmd} "${url}" -H "Content-Type: application-json" --data-binary "${data}" 2>&1 ) +excode=${?} +case $res in + *"already exists"*) + echo " ... [Exists]" + ;; + *"Datasource added"*) + echo " ... [OK]" + ;; +esac -if datasources['message'] == "Unauthorized" : - exit(34) +echo -ne "\tImporting Node Exporter Full Dashboard " -# Add new datasource -datasources_put = session.put( - os.path.join(grafanaURL, 'api', 'datasources'), - data=json.dumps({ - 'access': 'direct', - 'name': datasource_name, - 'type': 'prometheus', - 'url': 'http://%s:%u' % (promHost, promPort), - 'basicAuth': 'false', - 'withCredentials': 'false' - }), - headers={'content-type': 'application/json'} -) From 2de724da5725c22e158b24da70c19d295f6034b0 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Tue, 5 Jun 2018 11:45:05 +0200 Subject: [PATCH 12/13] Moving to bash 2 --- init/prometheus.service | 12 ----- postservice/88_grafana | 99 ++++++++++++++++++++++++++++++++--------- 2 files changed, 78 insertions(+), 33 deletions(-) delete mode 100644 init/prometheus.service diff --git a/init/prometheus.service b/init/prometheus.service deleted file mode 100644 index b7f3eee..0000000 --- a/init/prometheus.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Prometheus Server -Documentation=https://prometheus.io/docs/introduction/overview/ -After=network-online.target - -[Service] -User=root -Restart=on-failure -ExecStart=/usr/bin/prometheus --config.file=/etc/prometheus/prometheus.yml - -[Install] -WantedBy=multi-user.target diff --git a/postservice/88_grafana b/postservice/88_grafana index 10592a1..3459bcd 100755 --- a/postservice/88_grafana +++ b/postservice/88_grafana @@ -1,5 +1,78 @@ #!/usr/bin/env bash + +function importDataSource() +{ + local grURL="${1}/api/datasources" # Grafana API URL + local dsName=${2} # Datasource Name + local dsURL="${3}" # Datasource URL + local dsType="prometheus" # Datasource type + + cmd="curl" + + + data=$(cat <<__EOF__ +{"name":"${dsName}","type":"${dsType}","url":"${dsURL}","access":"direct"} +__EOF__ + ) + + echo -ne "\tCreating datasource for Prometheus " + res=$(${cmd} "${grURL}" -H "Content-Type: application-json" --data-binary "${data}" 2>&1 ) + excode=${?} + case $res in + *"already exists"*) + echo " ... [Exists]" + ;; + *"Datasource added"*) + echo " ... [OK]" + ;; + *) + echo " ... Ooops ${res}" + ;; + esac +} + +function importDashboardFromMarket() +{ + local grURL="${1}/api/dashboards/import" # API URL to create new Dashboard + local dsName="${2}" # Datasource name (to link dashboard to data) + local dhID="${3}" # Dashboard ID in the market + local dhRev="${4}" # Dashboard Revision to download + local tmpFile=$(mktemp) + local dhFile=$(mktemp) + + # URL To download the json file for dashboard + local pubDashBoardURL="https://grafana.com/api/dashboards/${dhID}/revisions/${dhRev}/download" + + local cmd="curl" + + local dh=$(${cmd} --silent ${pubDashBoardURL} 2>&1) + + cat <<_EOF_ > ${tmpFile} +{ + "inputs": [ { "name":"DS_LOCALHOST", "type":"datasource", "pluginId":"prometheus", "value":"${dsName}" } ], + "dashboard": { "title": "Surveillance Système", ${dh:1:-1} }, + "folderId": 0, + "overwrite": true +} +_EOF_ + + res=$(${cmd} "${grURL}" -H "Content-Type: application-json" --data-binary "@${tmpFile}" 2>&1 ) + excode=${?} + case $res in + *"\"imported\":true"*) + echo " ... [Overwrited]" + ;; + *"Datasource added"*) + echo " ... [OK]" + ;; + *) + echo " ... Ooops ${res}" + ;; + esac +# rm -rf ${tmpFile} +} + grafanaHost=$(CreoleGet srvGrafanaIP 192.168.30.57) grafanaPort=$(CreoleGet srvGrafanaPort 3000) grafanaUser="admin" @@ -8,27 +81,11 @@ grafanaURL="http://${grafanaUser}:${grafanaPasswd}@${grafanaHost}:${grafanaPort} datasource_name=$(CreoleGet promDataSource "prometheus") promHost=$(CreoleGet adresse_ip_eth0) promPort='9090' +datasourceURL="http://${promHost}:${promPort}" - -cmd="curl" -url="${grafanaURL}/api/datasources" - -data=$(cat <<__EOF__ -{"name":"${datasource_name}","type":"prometheus","url":"http://${promHost}:${promPort}","access":"direct"} -__EOF__ -) - -echo -ne "\tCreating datasource for Prometheus " -res=$(${cmd} "${url}" -H "Content-Type: application-json" --data-binary "${data}" 2>&1 ) -excode=${?} -case $res in - *"already exists"*) - echo " ... [Exists]" - ;; - *"Datasource added"*) - echo " ... [OK]" - ;; -esac - +importDataSource ${grafanaURL} ${datasource_name} ${datasourceURL} +echo echo -ne "\tImporting Node Exporter Full Dashboard " +importDashboardFromMarket ${grafanaURL} ${datasource_name} 1860 11 +rm -rf ${dashBoardFile} From 598c1d180772fc08d67fedbc0bc1341072d38ed2 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Tue, 5 Jun 2018 14:27:32 +0200 Subject: [PATCH 13/13] Fix template --- tmpl/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tmpl/prometheus.yml b/tmpl/prometheus.yml index e360435..59363ed 100644 --- a/tmpl/prometheus.yml +++ b/tmpl/prometheus.yml @@ -25,7 +25,7 @@ scrape_configs: # Nothing to configure %else static_configs: - - targets: [ "127.0.0.1:9100"%slurp + - targets: [ "%%adresse_ip_eth0:9100"%slurp %if %%getVar('ajout_client_prometheus','non') == 'oui' %for %%cliPr in %%getVar('prCli',[]) %if %%cliPr.prCliSonde == 'Node Exporter'