mirror of
https://github.com/puppetmaster/typhoon.git
synced 2024-12-26 05:19:32 +01:00
c4683c5bad
* Add 2 min wait before KubeNodeUnreachable to be less noisy on premeptible clusters * Add a BlackboxProbeFailure alert for any failing probes for services annotated `prometheus.io/probe: true`
1460 lines
78 KiB
YAML
1460 lines
78 KiB
YAML
apiVersion: v1
|
|
data:
|
|
etcd.yaml: |-
|
|
{
|
|
"groups": [
|
|
{
|
|
"name": "etcd",
|
|
"rules": [
|
|
{
|
|
"alert": "etcdMembersDown",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": members are down ({{ $value }})."
|
|
},
|
|
"expr": "max by (job) (\n sum by (job) (up{job=~\".*etcd.*\"} == bool 0)\nor\n count by (job,endpoint) (\n sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~\".*etcd.*\"}[3m])) > 0.01\n )\n)\n> 0\n",
|
|
"for": "3m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdInsufficientMembers",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": insufficient members ({{ $value }})."
|
|
},
|
|
"expr": "sum(up{job=~\".*etcd.*\"} == bool 1) by (job) < ((count(up{job=~\".*etcd.*\"}) by (job) + 1) / 2)\n",
|
|
"for": "3m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdNoLeader",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": member {{ $labels.instance }} has no leader."
|
|
},
|
|
"expr": "etcd_server_has_leader{job=~\".*etcd.*\"} == 0\n",
|
|
"for": "1m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighNumberOfLeaderChanges",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated."
|
|
},
|
|
"expr": "increase((max by (job) (etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\"}))[15m:1m]) >= 3\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdGRPCRequestsSlow",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_type=\"unary\"}[5m])) by (job, instance, grpc_service, grpc_method, le))\n> 0.15\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdMemberCommunicationSlow",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.15\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighNumberOfFailedProposals",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "rate(etcd_server_proposals_failed_total{job=~\".*etcd.*\"}[15m]) > 5\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighFsyncDurations",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": 99th percentile fync durations are {{ $value }}s on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.5\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighCommitDurations",
|
|
"annotations": {
|
|
"message": "etcd cluster \"{{ $labels.job }}\": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 0.25\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighNumberOfFailedHTTPRequests",
|
|
"annotations": {
|
|
"message": "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}"
|
|
},
|
|
"expr": "sum(rate(etcd_http_failed_total{job=~\".*etcd.*\", code!=\"404\"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~\".*etcd.*\"}[5m]))\nBY (method) > 0.01\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHighNumberOfFailedHTTPRequests",
|
|
"annotations": {
|
|
"message": "{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}."
|
|
},
|
|
"expr": "sum(rate(etcd_http_failed_total{job=~\".*etcd.*\", code!=\"404\"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~\".*etcd.*\"}[5m]))\nBY (method) > 0.05\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "etcdHTTPRequestsSlow",
|
|
"annotations": {
|
|
"message": "etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow."
|
|
},
|
|
"expr": "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))\n> 0.15\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
kube.yaml: |-
|
|
{
|
|
"groups": [
|
|
{
|
|
"name": "kube-apiserver-error",
|
|
"rules": [
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[5m]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate5m"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[30m]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate30m"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[1h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate1h"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[2h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate2h"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[6h]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate6h"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[1d]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate1d"
|
|
},
|
|
{
|
|
"expr": "sum by (status_class) (\n label_replace(\n rate(apiserver_request_total{job=\"apiserver\"}[3d]\n ), \"status_class\", \"${1}xx\", \"code\", \"([0-9])..\")\n)\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class:apiserver_request_total:rate3d"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate5m{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate5m{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate5m"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate30m{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate30m{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate30m"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate1h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate1h{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate1h"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate2h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate2h{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate2h"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate6h{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate6h{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate6h"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate1d{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate1d{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate1d"
|
|
},
|
|
{
|
|
"expr": "sum(status_class:apiserver_request_total:rate3d{job=\"apiserver\",status_class=\"5xx\"})\n/\nsum(status_class:apiserver_request_total:rate3d{job=\"apiserver\"})\n",
|
|
"labels": {
|
|
"job": "apiserver"
|
|
},
|
|
"record": "status_class_5xx:apiserver_request_total:ratio_rate3d"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kube-apiserver.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "sum(rate(apiserver_request_duration_seconds_sum{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n/\nsum(rate(apiserver_request_duration_seconds_count{subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod)\n",
|
|
"record": "cluster:apiserver_request_duration_seconds:mean5m"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.99"
|
|
},
|
|
"record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.9"
|
|
},
|
|
"record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|PROXY|CONNECT\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.5"
|
|
},
|
|
"record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "k8s.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])) by (namespace)\n",
|
|
"record": "namespace:container_cpu_usage_seconds_total:sum_rate"
|
|
},
|
|
{
|
|
"expr": "sum by (cluster, namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}[5m])\n) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (\n 1, max by(cluster, namespace, pod, node) (kube_pod_info)\n)\n",
|
|
"record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate"
|
|
},
|
|
{
|
|
"expr": "container_memory_working_set_bytes{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n",
|
|
"record": "node_namespace_pod_container:container_memory_working_set_bytes"
|
|
},
|
|
{
|
|
"expr": "container_memory_rss{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n",
|
|
"record": "node_namespace_pod_container:container_memory_rss"
|
|
},
|
|
{
|
|
"expr": "container_memory_cache{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n",
|
|
"record": "node_namespace_pod_container:container_memory_cache"
|
|
},
|
|
{
|
|
"expr": "container_memory_swap{job=\"kubernetes-cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info)\n)\n",
|
|
"record": "node_namespace_pod_container:container_memory_swap"
|
|
},
|
|
{
|
|
"expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", image!=\"\", container!=\"POD\"}) by (namespace)\n",
|
|
"record": "namespace:container_memory_usage_bytes:sum"
|
|
},
|
|
{
|
|
"expr": "sum by (namespace) (\n sum by (namespace, pod) (\n max by (namespace, pod, container) (\n kube_pod_container_resource_requests_memory_bytes{job=\"kube-state-metrics\"}\n ) * on(namespace, pod) group_left() max by (namespace, pod) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n",
|
|
"record": "namespace:kube_pod_container_resource_requests_memory_bytes:sum"
|
|
},
|
|
{
|
|
"expr": "sum by (namespace) (\n sum by (namespace, pod) (\n max by (namespace, pod, container) (\n kube_pod_container_resource_requests_cpu_cores{job=\"kube-state-metrics\"}\n ) * on(namespace, pod) group_left() max by (namespace, pod) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n",
|
|
"record": "namespace:kube_pod_container_resource_requests_cpu_cores:sum"
|
|
},
|
|
{
|
|
"expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"ReplicaSet\"},\n \"replicaset\", \"$1\", \"owner_name\", \"(.*)\"\n ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (\n 1, max by (replicaset, namespace, owner_name) (\n kube_replicaset_owner{job=\"kube-state-metrics\"}\n )\n ),\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n",
|
|
"labels": {
|
|
"workload_type": "deployment"
|
|
},
|
|
"record": "mixin_pod_workload"
|
|
},
|
|
{
|
|
"expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"DaemonSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n",
|
|
"labels": {
|
|
"workload_type": "daemonset"
|
|
},
|
|
"record": "mixin_pod_workload"
|
|
},
|
|
{
|
|
"expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"StatefulSet\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n",
|
|
"labels": {
|
|
"workload_type": "statefulset"
|
|
},
|
|
"record": "mixin_pod_workload"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kube-scheduler.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.99"
|
|
},
|
|
"record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.99"
|
|
},
|
|
"record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.99"
|
|
},
|
|
"record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.9"
|
|
},
|
|
"record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.9"
|
|
},
|
|
"record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.9"
|
|
},
|
|
"record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.5"
|
|
},
|
|
"record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.5"
|
|
},
|
|
"record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\"}[5m])) without(instance, pod))\n",
|
|
"labels": {
|
|
"quantile": "0.5"
|
|
},
|
|
"record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "node.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "sum(min(kube_pod_info) by (cluster, node))\n",
|
|
"record": ":kube_pod_info_node_count:"
|
|
},
|
|
{
|
|
"expr": "topk by(namespace, pod) (1,\n max by (node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n",
|
|
"record": "node_namespace_pod:kube_pod_info:"
|
|
},
|
|
{
|
|
"expr": "count by (cluster, node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n node_namespace_pod:kube_pod_info:\n))\n",
|
|
"record": "node:node_num_cpu:sum"
|
|
},
|
|
{
|
|
"expr": "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n) by (cluster)\n",
|
|
"record": ":node_memory_MemAvailable_bytes:sum"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubelet.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n",
|
|
"labels": {
|
|
"quantile": "0.99"
|
|
},
|
|
"record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n",
|
|
"labels": {
|
|
"quantile": "0.9"
|
|
},
|
|
"record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"})\n",
|
|
"labels": {
|
|
"quantile": "0.5"
|
|
},
|
|
"record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-apps",
|
|
"rules": [
|
|
{
|
|
"alert": "KubePodCrashLooping",
|
|
"annotations": {
|
|
"message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 5 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping"
|
|
},
|
|
"expr": "rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\"}[15m]) * 60 * 5 > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubePodNotReady",
|
|
"annotations": {
|
|
"message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready"
|
|
},
|
|
"expr": "sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!=\"Job\"})) > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeDeploymentGenerationMismatch",
|
|
"annotations": {
|
|
"message": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch"
|
|
},
|
|
"expr": "kube_deployment_status_observed_generation{job=\"kube-state-metrics\"}\n !=\nkube_deployment_metadata_generation{job=\"kube-state-metrics\"}\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeDeploymentReplicasMismatch",
|
|
"annotations": {
|
|
"message": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch"
|
|
},
|
|
"expr": "(\n kube_deployment_spec_replicas{job=\"kube-state-metrics\"}\n !=\n kube_deployment_status_replicas_available{job=\"kube-state-metrics\"}\n) and (\n changes(kube_deployment_status_replicas_updated{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeStatefulSetReplicasMismatch",
|
|
"annotations": {
|
|
"message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch"
|
|
},
|
|
"expr": "(\n kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\"}\n !=\n kube_statefulset_status_replicas{job=\"kube-state-metrics\"}\n) and (\n changes(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeStatefulSetGenerationMismatch",
|
|
"annotations": {
|
|
"message": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch"
|
|
},
|
|
"expr": "kube_statefulset_status_observed_generation{job=\"kube-state-metrics\"}\n !=\nkube_statefulset_metadata_generation{job=\"kube-state-metrics\"}\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeStatefulSetUpdateNotRolledOut",
|
|
"annotations": {
|
|
"message": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout"
|
|
},
|
|
"expr": "max without (revision) (\n kube_statefulset_status_current_revision{job=\"kube-state-metrics\"}\n unless\n kube_statefulset_status_update_revision{job=\"kube-state-metrics\"}\n)\n *\n(\n kube_statefulset_replicas{job=\"kube-state-metrics\"}\n !=\n kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\"}\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeDaemonSetRolloutStuck",
|
|
"annotations": {
|
|
"message": "Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck"
|
|
},
|
|
"expr": "kube_daemonset_status_number_ready{job=\"kube-state-metrics\"}\n /\nkube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"} < 1.00\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeContainerWaiting",
|
|
"annotations": {
|
|
"message": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting"
|
|
},
|
|
"expr": "sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job=\"kube-state-metrics\"}) > 0\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeDaemonSetNotScheduled",
|
|
"annotations": {
|
|
"message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled"
|
|
},
|
|
"expr": "kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n -\nkube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"} > 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeDaemonSetMisScheduled",
|
|
"annotations": {
|
|
"message": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled"
|
|
},
|
|
"expr": "kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"} > 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeCronJobRunning",
|
|
"annotations": {
|
|
"message": "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning"
|
|
},
|
|
"expr": "time() - kube_cronjob_next_schedule_time{job=\"kube-state-metrics\"} > 3600\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeJobCompletion",
|
|
"annotations": {
|
|
"message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than one hour to complete.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion"
|
|
},
|
|
"expr": "kube_job_spec_completions{job=\"kube-state-metrics\"} - kube_job_status_succeeded{job=\"kube-state-metrics\"} > 0\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeJobFailed",
|
|
"annotations": {
|
|
"message": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed"
|
|
},
|
|
"expr": "kube_job_failed{job=\"kube-state-metrics\"} > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeHpaReplicasMismatch",
|
|
"annotations": {
|
|
"message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch"
|
|
},
|
|
"expr": "(kube_hpa_status_desired_replicas{job=\"kube-state-metrics\"}\n !=\nkube_hpa_status_current_replicas{job=\"kube-state-metrics\"})\n and\nchanges(kube_hpa_status_current_replicas[15m]) == 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeHpaMaxedOut",
|
|
"annotations": {
|
|
"message": "HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout"
|
|
},
|
|
"expr": "kube_hpa_status_current_replicas{job=\"kube-state-metrics\"}\n ==\nkube_hpa_spec_max_replicas{job=\"kube-state-metrics\"}\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-resources",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeCPUOvercommit",
|
|
"annotations": {
|
|
"message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
|
|
},
|
|
"expr": "sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{})\n /\nsum(kube_node_status_allocatable_cpu_cores)\n >\n(count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores)\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeMemOvercommit",
|
|
"annotations": {
|
|
"message": "Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit"
|
|
},
|
|
"expr": "sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{})\n /\nsum(kube_node_status_allocatable_memory_bytes)\n >\n(count(kube_node_status_allocatable_memory_bytes)-1)\n /\ncount(kube_node_status_allocatable_memory_bytes)\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeCPUOvercommit",
|
|
"annotations": {
|
|
"message": "Cluster has overcommitted CPU resource requests for Namespaces.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
|
|
},
|
|
"expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable_cpu_cores)\n > 1.5\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeMemOvercommit",
|
|
"annotations": {
|
|
"message": "Cluster has overcommitted memory resource requests for Namespaces.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememovercommit"
|
|
},
|
|
"expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable_memory_bytes{job=\"node-exporter\"})\n > 1.5\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeQuotaExceeded",
|
|
"annotations": {
|
|
"message": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded"
|
|
},
|
|
"expr": "kube_resourcequota{job=\"kube-state-metrics\", type=\"used\"}\n / ignoring(instance, job, type)\n(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\"} > 0)\n > 0.90\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "CPUThrottlingHigh",
|
|
"annotations": {
|
|
"message": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh"
|
|
},
|
|
"expr": "sum(increase(container_cpu_cfs_throttled_periods_total{container!=\"\", }[5m])) by (container, pod, namespace)\n /\nsum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)\n > ( 100 / 100 )\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-storage",
|
|
"rules": [
|
|
{
|
|
"alert": "KubePersistentVolumeUsageCritical",
|
|
"annotations": {
|
|
"message": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeusagecritical"
|
|
},
|
|
"expr": "kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\nkubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n < 0.03\n",
|
|
"for": "1m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubePersistentVolumeFullInFourDays",
|
|
"annotations": {
|
|
"message": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefullinfourdays"
|
|
},
|
|
"expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}\n) < 0.15\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\"}[6h], 4 * 24 * 3600) < 0\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubePersistentVolumeErrors",
|
|
"annotations": {
|
|
"message": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors"
|
|
},
|
|
"expr": "kube_persistentvolume_status_phase{phase=~\"Failed|Pending\",job=\"kube-state-metrics\"} > 0\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-system",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeVersionMismatch",
|
|
"annotations": {
|
|
"message": "There are {{ $value }} different semantic versions of Kubernetes components running.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch"
|
|
},
|
|
"expr": "count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"},\"gitVersion\",\"$1\",\"gitVersion\",\"(v[0-9]*.[0-9]*.[0-9]*).*\"))) > 1\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeClientErrors",
|
|
"annotations": {
|
|
"message": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors"
|
|
},
|
|
"expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job))\n> 0.01\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kube-apiserver-error-alerts",
|
|
"rules": [
|
|
{
|
|
"alert": "ErrorBudgetBurn",
|
|
"annotations": {
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn"
|
|
},
|
|
"expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1h{job=\"apiserver\"} > (14.4*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate5m{job=\"apiserver\"} > (14.4*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (6*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate30m{job=\"apiserver\"} > (6*0.010000)\n)\n",
|
|
"labels": {
|
|
"job": "apiserver",
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "ErrorBudgetBurn",
|
|
"annotations": {
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-errorbudgetburn"
|
|
},
|
|
"expr": "(\n status_class_5xx:apiserver_request_total:ratio_rate1d{job=\"apiserver\"} > (3*0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate2h{job=\"apiserver\"} > (3*0.010000)\n)\nor\n(\n status_class_5xx:apiserver_request_total:ratio_rate3d{job=\"apiserver\"} > (0.010000)\n and\n status_class_5xx:apiserver_request_total:ratio_rate6h{job=\"apiserver\"} > (0.010000)\n)\n",
|
|
"labels": {
|
|
"job": "apiserver",
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-system-apiserver",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeAPILatencyHigh",
|
|
"annotations": {
|
|
"message": "The API server has an abnormal latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh"
|
|
},
|
|
"expr": "(\n cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"}\n >\n on (verb) group_left()\n (\n avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\n +\n 2*stddev by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\n )\n) > on (verb) group_left()\n1.2 * avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job=\"apiserver\"} >= 0)\nand on (verb,resource)\ncluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\"}\n>\n1\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPILatencyHigh",
|
|
"annotations": {
|
|
"message": "The API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh"
|
|
},
|
|
"expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job=\"apiserver\",quantile=\"0.99\"} > 4\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPIErrorsHigh",
|
|
"annotations": {
|
|
"message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
|
|
},
|
|
"expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.03\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPIErrorsHigh",
|
|
"annotations": {
|
|
"message": "API server is returning errors for {{ $value | humanizePercentage }} of requests.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
|
|
},
|
|
"expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m]))\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) > 0.01\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPIErrorsHigh",
|
|
"annotations": {
|
|
"message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
|
|
},
|
|
"expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.10\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPIErrorsHigh",
|
|
"annotations": {
|
|
"message": "API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh"
|
|
},
|
|
"expr": "sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (resource,subresource,verb)\n /\nsum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (resource,subresource,verb) > 0.05\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeClientCertificateExpiration",
|
|
"annotations": {
|
|
"message": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
|
|
},
|
|
"expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeClientCertificateExpiration",
|
|
"annotations": {
|
|
"message": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
|
|
},
|
|
"expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "AggregatedAPIErrors",
|
|
"annotations": {
|
|
"message": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors"
|
|
},
|
|
"expr": "sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2\n",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "AggregatedAPIDown",
|
|
"annotations": {
|
|
"message": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} is down. It has not been available at least for the past five minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown"
|
|
},
|
|
"expr": "sum by(name, namespace)(sum_over_time(aggregator_unavailable_apiservice[5m])) > 0\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeAPIDown",
|
|
"annotations": {
|
|
"message": "KubeAPI has disappeared from Prometheus target discovery.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown"
|
|
},
|
|
"expr": "absent(up{job=\"apiserver\"} == 1)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-system-kubelet",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeNodeNotReady",
|
|
"annotations": {
|
|
"message": "{{ $labels.node }} has been unready for more than 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready"
|
|
},
|
|
"expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeNodeUnreachable",
|
|
"annotations": {
|
|
"message": "{{ $labels.node }} is unreachable and some workloads may be rescheduled.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable"
|
|
},
|
|
"expr": "kube_node_spec_taint{job=\"kube-state-metrics\",key=\"node.kubernetes.io/unreachable\",effect=\"NoSchedule\"} == 1\n",
|
|
"for": "2m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeletTooManyPods",
|
|
"annotations": {
|
|
"message": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods"
|
|
},
|
|
"expr": "max(max(kubelet_running_pod_count{job=\"kubelet\"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\"}) by(node) / max(kube_node_status_capacity_pods{job=\"kube-state-metrics\"}) by(node) > 0.95\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeNodeReadinessFlapping",
|
|
"annotations": {
|
|
"message": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping"
|
|
},
|
|
"expr": "sum(changes(kube_node_status_condition{status=\"true\",condition=\"Ready\"}[15m])) by (node) > 2\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeletPlegDurationHigh",
|
|
"annotations": {
|
|
"message": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh"
|
|
},
|
|
"expr": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile=\"0.99\"} >= 10\n",
|
|
"for": "5m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeletPodStartUpLatencyHigh",
|
|
"annotations": {
|
|
"message": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh"
|
|
},
|
|
"expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name > 5\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "KubeletDown",
|
|
"annotations": {
|
|
"message": "Kubelet has disappeared from Prometheus target discovery.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown"
|
|
},
|
|
"expr": "absent(up{job=\"kubelet\"} == 1)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-system-scheduler",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeSchedulerDown",
|
|
"annotations": {
|
|
"message": "KubeScheduler has disappeared from Prometheus target discovery.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown"
|
|
},
|
|
"expr": "absent(up{job=\"kube-scheduler\"} == 1)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "kubernetes-system-controller-manager",
|
|
"rules": [
|
|
{
|
|
"alert": "KubeControllerManagerDown",
|
|
"annotations": {
|
|
"message": "KubeControllerManager has disappeared from Prometheus target discovery.",
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown"
|
|
},
|
|
"expr": "absent(up{job=\"kube-controller-manager\"} == 1)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
node-exporter.yaml: |-
|
|
{
|
|
"groups": [
|
|
{
|
|
"name": "node-exporter",
|
|
"rules": [
|
|
{
|
|
"alert": "NodeFilesystemSpaceFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up.",
|
|
"summary": "Filesystem is predicted to run out of space within the next 24 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemSpaceFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up fast.",
|
|
"summary": "Filesystem is predicted to run out of space within the next 4 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 20\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfSpace",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.",
|
|
"summary": "Filesystem has less than 5% space left."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfSpace",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.",
|
|
"summary": "Filesystem has less than 3% space left."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemFilesFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up.",
|
|
"summary": "Filesystem is predicted to run out of inodes within the next 24 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 40\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemFilesFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up fast.",
|
|
"summary": "Filesystem is predicted to run out of inodes within the next 4 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 20\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfFiles",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.",
|
|
"summary": "Filesystem has less than 5% inodes left."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfFiles",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.",
|
|
"summary": "Filesystem has less than 3% inodes left."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} / node_filesystem_files{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!~\"tmpfs|nsfs|vfat\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeNetworkReceiveErrs",
|
|
"annotations": {
|
|
"description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.",
|
|
"summary": "Network interface is reporting many receive errors."
|
|
},
|
|
"expr": "increase(node_network_receive_errs_total[2m]) > 10\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeNetworkTransmitErrs",
|
|
"annotations": {
|
|
"description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.",
|
|
"summary": "Network interface is reporting many transmit errors."
|
|
},
|
|
"expr": "increase(node_network_transmit_errs_total[2m]) > 10\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
prom.yaml: |-
|
|
{
|
|
"groups": [
|
|
{
|
|
"name": "prometheus",
|
|
"rules": [
|
|
{
|
|
"alert": "PrometheusBadConfig",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} has failed to reload its configuration.",
|
|
"summary": "Failed Prometheus configuration reload."
|
|
},
|
|
"expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\nmax_over_time(prometheus_config_last_reload_successful{job=\"prometheus\"}[5m]) == 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusNotificationQueueRunningFull",
|
|
"annotations": {
|
|
"description": "Alert notification queue of Prometheus {{$labels.instance}} is running full.",
|
|
"summary": "Prometheus alert notification queue predicted to run full in less than 30m."
|
|
},
|
|
"expr": "# Without min_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n predict_linear(prometheus_notifications_queue_length{job=\"prometheus\"}[5m], 60 * 30)\n>\n min_over_time(prometheus_notifications_queue_capacity{job=\"prometheus\"}[5m])\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusErrorSendingAlertsToSomeAlertmanagers",
|
|
"annotations": {
|
|
"description": "{{ printf \"%.1f\" $value }}% errors while sending alerts from Prometheus {{$labels.instance}} to Alertmanager {{$labels.alertmanager}}.",
|
|
"summary": "Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager."
|
|
},
|
|
"expr": "(\n rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m])\n/\n rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m])\n)\n* 100\n> 1\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusErrorSendingAlertsToAnyAlertmanager",
|
|
"annotations": {
|
|
"description": "{{ printf \"%.1f\" $value }}% minimum errors while sending alerts from Prometheus {{$labels.instance}} to any Alertmanager.",
|
|
"summary": "Prometheus encounters more than 3% errors sending alerts to any Alertmanager."
|
|
},
|
|
"expr": "min without(alertmanager) (\n rate(prometheus_notifications_errors_total{job=\"prometheus\"}[5m])\n/\n rate(prometheus_notifications_sent_total{job=\"prometheus\"}[5m])\n)\n* 100\n> 3\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusNotConnectedToAlertmanagers",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} is not connected to any Alertmanagers.",
|
|
"summary": "Prometheus is not connected to any Alertmanagers."
|
|
},
|
|
"expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\nmax_over_time(prometheus_notifications_alertmanagers_discovered{job=\"prometheus\"}[5m]) < 1\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusTSDBReloadsFailing",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} reload failures over the last 3h.",
|
|
"summary": "Prometheus has issues reloading blocks from disk."
|
|
},
|
|
"expr": "increase(prometheus_tsdb_reloads_failures_total{job=\"prometheus\"}[3h]) > 0\n",
|
|
"for": "4h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusTSDBCompactionsFailing",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} has detected {{$value | humanize}} compaction failures over the last 3h.",
|
|
"summary": "Prometheus has issues compacting blocks."
|
|
},
|
|
"expr": "increase(prometheus_tsdb_compactions_failed_total{job=\"prometheus\"}[3h]) > 0\n",
|
|
"for": "4h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusNotIngestingSamples",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} is not ingesting samples.",
|
|
"summary": "Prometheus is not ingesting samples."
|
|
},
|
|
"expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[5m]) <= 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusDuplicateTimestamps",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} is dropping {{ printf \"%.4g\" $value }} samples/s with different values but duplicated timestamp.",
|
|
"summary": "Prometheus is dropping samples with duplicate timestamps."
|
|
},
|
|
"expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\"}[5m]) > 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusOutOfOrderTimestamps",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} is dropping {{ printf \"%.4g\" $value }} samples/s with timestamps arriving out of order.",
|
|
"summary": "Prometheus drops samples with out-of-order timestamps."
|
|
},
|
|
"expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\"}[5m]) > 0\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusRemoteStorageFailures",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} failed to send {{ printf \"%.1f\" $value }}% of the samples to {{ if $labels.queue }}{{ $labels.queue }}{{ else }}{{ $labels.url }}{{ end }}.",
|
|
"summary": "Prometheus fails to send samples to remote storage."
|
|
},
|
|
"expr": "(\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n/\n (\n rate(prometheus_remote_storage_failed_samples_total{job=\"prometheus\"}[5m])\n +\n rate(prometheus_remote_storage_succeeded_samples_total{job=\"prometheus\"}[5m])\n )\n)\n* 100\n> 1\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusRemoteWriteBehind",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} remote write is {{ printf \"%.1f\" $value }}s behind for {{ if $labels.queue }}{{ $labels.queue }}{{ else }}{{ $labels.url }}{{ end }}.",
|
|
"summary": "Prometheus remote write is behind."
|
|
},
|
|
"expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job=\"prometheus\"}[5m])\n- on(job, instance) group_right\n max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job=\"prometheus\"}[5m])\n)\n> 120\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusRemoteWriteDesiredShards",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} remote write desired shards calculation wants to run {{ $value }} shards, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance=\"%s\",job=\"prometheus\"}` $labels.instance | query | first | value }}.",
|
|
"summary": "Prometheus remote write desired shards calculation wants to run more than configured max shards."
|
|
},
|
|
"expr": "# Without max_over_time, failed scrapes could create false negatives, see\n# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.\n(\n max_over_time(prometheus_remote_storage_shards_desired{job=\"prometheus\"}[5m])\n>\n max_over_time(prometheus_remote_storage_shards_max{job=\"prometheus\"}[5m])\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusRuleFailures",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} has failed to evaluate {{ printf \"%.0f\" $value }} rules in the last 5m.",
|
|
"summary": "Prometheus is failing rule evaluations."
|
|
},
|
|
"expr": "increase(prometheus_rule_evaluation_failures_total{job=\"prometheus\"}[5m]) > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "PrometheusMissingRuleEvaluations",
|
|
"annotations": {
|
|
"description": "Prometheus {{$labels.instance}} has missed {{ printf \"%.0f\" $value }} rule group evaluations in the last 5m.",
|
|
"summary": "Prometheus is missing rule evaluations due to slow rule group evaluation."
|
|
},
|
|
"expr": "increase(prometheus_rule_group_iterations_missed_total{job=\"prometheus\"}[5m]) > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
typhoon.yaml: |-
|
|
{
|
|
"groups": [
|
|
{
|
|
"name": "general.rules",
|
|
"rules": [
|
|
{
|
|
"alert": "TargetDown",
|
|
"annotations": {
|
|
"message": "{{ printf \"%.4g\" $value }}% of the {{ $labels.job }} targets are down."
|
|
},
|
|
"expr": "100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "BlackboxProbeFailure",
|
|
"annotations": {
|
|
"message": "Blackbox probe {{$labels.instance}} failed"
|
|
},
|
|
"expr": "probe_success == 0",
|
|
"for": "2m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "extra.rules",
|
|
"rules": [
|
|
{
|
|
"alert": "InactiveRAIDDisk",
|
|
"annotations": {
|
|
"message": "{{ $value }} RAID disk(s) on node {{ $labels.instance }} are inactive."
|
|
},
|
|
"expr": "node_md_disks{state=\"failed\"} > 0",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-rules
|
|
namespace: monitoring
|