From 50215e373bf1de9b0e63f629322f8e27bcb7190e Mon Sep 17 00:00:00 2001
From: Dalton Hubble <dghubble@gmail.com>
Date: Wed, 29 Dec 2021 11:51:11 -0800
Subject: [PATCH] Add Prometheus config for monitoring Kubernetes Ingress

* Allow Kubernetes Ingress resources to be probed via Blackbox
Exporter (if present) if annotated `prometheus.io/probe: "true"`
* Fix probes of Services via Blackbox Exporter. Require Blackbox
Exporter to be deployed in the same `monitoring` namespace, be
named `blackbox-exporter`, and use port 8080
---
 CHANGES.md                               |  2 +
 addons/prometheus/config.yaml            | 93 ++++++++++++++++--------
 addons/prometheus/rbac/cluster-role.yaml | 13 +++-
 3 files changed, 75 insertions(+), 33 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 2354fb65..8f3c9f0f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -14,6 +14,8 @@ Notable changes between versions.
 ### Addons
 
 * Configure Prometheus to allow a custom scrape query parameter ([#1095](https://github.com/poseidon/typhoon/pull/1095))
+* Configure Prometheus to probe Kubernetes Ingress via `blackbox-exporter` ([#1096](https://github.com/poseidon/typhoon/pull/1096))
+* Fix Prometheus Service probes to use `blackbox-exporter`, not `blackbox` ([#1096](https://github.com/poseidon/typhoon/pull/1096))
 
 ## v1.23.0
 
diff --git a/addons/prometheus/config.yaml b/addons/prometheus/config.yaml
index 4d98a359..1e47db85 100644
--- a/addons/prometheus/config.yaml
+++ b/addons/prometheus/config.yaml
@@ -220,38 +220,6 @@ data:
         action: drop
         regex: etcd_(debugging|disk|request|server).*
 
-    # Example scrape config for probing services via the Blackbox Exporter.
-    #
-    # The relabeling allows the actual service scrape endpoint to be configured
-    # via the following annotations:
-    #
-    # * `prometheus.io/probe`: Only probe services that have a value of `true`
-    - job_name: 'kubernetes-services'
-
-      metrics_path: /probe
-      params:
-        module: [http_2xx]
-
-      kubernetes_sd_configs:
-      - role: service
-
-      relabel_configs:
-      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
-        action: keep
-        regex: true
-      - source_labels: [__address__]
-        target_label: __param_target
-      - target_label: __address__
-        replacement: blackbox
-      - source_labels: [__param_target]
-        target_label: instance
-      - action: labelmap
-        regex: __meta_kubernetes_service_label_(.+)
-      - source_labels: [__meta_kubernetes_namespace]
-        target_label: namespace
-      - source_labels: [__meta_kubernetes_service_name]
-        target_label: job
-
     # Example scrape config for pods
     #
     # The relabeling allows the actual pod scrape endpoint to be configured via the
@@ -288,6 +256,67 @@ data:
         action: replace
         target_label: kubernetes_pod_name
 
+    # Example scrape config for probing Services via the Blackbox Exporter.
+    #
+    # Relabeling allows service scraping to be configured via annotations:
+    # * `prometheus.io/probe`: Only probe services that have a value of `true`
+    - job_name: 'kubernetes-services'
+
+      metrics_path: /probe
+      params:
+        module: [http_2xx]
+
+      kubernetes_sd_configs:
+      - role: service
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
+        action: keep
+        regex: true
+      - source_labels: [__address__]
+        target_label: __param_target
+      - target_label: __address__
+        replacement: blackbox-exporter:8080
+      - source_labels: [__param_target]
+        target_label: instance
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: job
+
+    # Example scrape config for probing Ingresses via a Blackbox Exporter.
+    #
+    # Relabeling allows service scraping to be configured via annotations:
+    # * `prometheus.io/probe`: Only probe ingresses that have a value of `true`
+    - job_name: 'kubernetes-ingresses'
+      metrics_path: /probe
+      params:
+        module: [http_2xx]
+
+      kubernetes_sd_configs:
+      - role: ingress
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_ingress_scheme, __address__, __meta_kubernetes_ingress_path]
+        regex: (.+);(.+);(.+)
+        replacement: ${1}://${2}${3}
+        target_label: __param_target
+      - target_label: __address__
+        replacement: blackbox-exporter:8080
+      - source_labels: [__param_target]
+        target_label: instance
+      - action: labelmap
+        regex: __meta_kubernetes_ingress_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: job
+
     # Rule files
     rule_files:
       - "/etc/prometheus/rules/*.rules"
diff --git a/addons/prometheus/rbac/cluster-role.yaml b/addons/prometheus/rbac/cluster-role.yaml
index 6f6cee0f..8c76b831 100644
--- a/addons/prometheus/rbac/cluster-role.yaml
+++ b/addons/prometheus/rbac/cluster-role.yaml
@@ -10,6 +10,17 @@ rules:
   - services
   - endpoints
   - pods
-  verbs: ["get", "list", "watch"]
+  verbs:
+  - get
+  - list
+  - watch
 - nonResourceURLs: ["/metrics"]
   verbs: ["get"]
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch