CadolesKube
/
typhoon
mirror of https://github.com/puppetmaster/typhoon.git
7
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update Prometheus discovery of kube components 

* Kubernetes v1.22.0 disabled kube-controller-manager insecure
port, which was used internally for Prometheus metrics scraping
* Configure Prometheus to discover and scrape endpoints for
kube-scheduler and kube-controller-manager via the authenticated
https ports, via bearer token
* Change firewall ports to allow Prometheus (on worker nodes)
to scrape kube-scheduler and kube-controller-manager targets
that run on controller(s) with hostNetwork
* Disable the insecure port on kube-scheduler
This commit is contained in:
Dalton Hubble 2021-08-10 21:08:49 -07:00
parent 0c99b909a9
commit cbef202eec
22 changed files with 74 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.md
View File

@ -7,6 +7,8 @@ Notable changes between versions.
## v1.22.0
* Kubernetes [v1.22.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#v1220)
* Switch `kube-controller-manager` and `kube-scheduler` to use secure port only
* Update Prometheus config to discover endpoints and use a bearer token to scrape
### Fedora CoreOS

42
addons/prometheus/config.yaml
View File

@ -72,6 +72,48 @@ data:
regex: apiserver_request_duration_seconds_count;.+
action: drop
# Scrape config for kube-controller-manager endpoints.
#
# kube-controller-manager service endpoints can be discovered by using the
# `endpoints` role and relabelling to only keep only endpoints associated with
# kube-system/kube-controller-manager and the `https` port.
- job_name: 'kube-controller-manager'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: kube-system;kube-controller-manager;metrics
- replacement: kube-controller-manager
action: replace
target_label: job
# Scrape config for kube-scheduler endpoints.
#
# kube-scheduler service endpoints can be discovered by using the `endpoints`
# role and relabelling to only keep only endpoints associated with
# kube-system/kube-scheduler and the `https` port.
- job_name: 'kube-scheduler'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: kube-system;kube-scheduler;metrics
- replacement: kube-scheduler
action: replace
target_label: job
# Scrape config for node (i.e. kubelet) /metrics (e.g. 'kubelet_'). Explore
# metrics from a node by scraping kubelet (127.0.0.1:10250/metrics).
- job_name: 'kubelet'

8
addons/prometheus/discovery/kube-controller-manager.yaml
View File

@ -1,11 +1,9 @@
# Allow Prometheus to scrape service endpoints
# Allow Prometheus to discover service endpoints
apiVersion: v1
kind: Service
metadata:
name: kube-controller-manager
namespace: kube-system
annotations:
prometheus.io/scrape: 'true'
spec:
type: ClusterIP
clusterIP: None
@ -14,5 +12,5 @@ spec:
ports:
- name: metrics
protocol: TCP
port: 10252
targetPort: 10252
port: 10257
targetPort: 10257

8
addons/prometheus/discovery/kube-scheduler.yaml
View File

@ -1,11 +1,9 @@
# Allow Prometheus to scrape service endpoints
# Allow Prometheus to discover service endpoints
apiVersion: v1
kind: Service
metadata:
name: kube-scheduler
namespace: kube-system
annotations:
prometheus.io/scrape: 'true'
spec:
type: ClusterIP
clusterIP: None
@ -14,5 +12,5 @@ spec:
ports:
- name: metrics
protocol: TCP
port: 10251
targetPort: 10251
port: 10259
targetPort: 10259

2
aws/fedora-coreos/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

8
aws/fedora-coreos/kubernetes/security.tf
View File

@ -201,8 +201,8 @@ resource "aws_security_group_rule" "controller-scheduler-metrics" {
type = "ingress"
protocol = "tcp"
from_port = 10251
to_port = 10251
from_port = 10259
to_port = 10259
source_security_group_id = aws_security_group.worker.id
}
@ -212,8 +212,8 @@ resource "aws_security_group_rule" "controller-manager-metrics" {
type = "ingress"
protocol = "tcp"
from_port = 10252
to_port = 10252
from_port = 10257
to_port = 10257
source_security_group_id = aws_security_group.worker.id
}

2
aws/flatcar-linux/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

8
aws/flatcar-linux/kubernetes/security.tf
View File

@ -201,8 +201,8 @@ resource "aws_security_group_rule" "controller-scheduler-metrics" {
type = "ingress"
protocol = "tcp"
from_port = 10251
to_port = 10251
from_port = 10259
to_port = 10259
source_security_group_id = aws_security_group.worker.id
}
@ -212,8 +212,8 @@ resource "aws_security_group_rule" "controller-manager-metrics" {
type = "ingress"
protocol = "tcp"
from_port = 10252
to_port = 10252
from_port = 10257
to_port = 10257
source_security_group_id = aws_security_group.worker.id
}

2
azure/fedora-coreos/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
azure/fedora-coreos/kubernetes/security.tf
View File

@ -95,7 +95,7 @@ resource "azurerm_network_security_rule" "controller-kube-metrics" {
direction = "Inbound"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "10251-10252"
destination_port_range = "10257-10259"
source_address_prefix = azurerm_subnet.worker.address_prefix
destination_address_prefix = azurerm_subnet.controller.address_prefix
}

2
azure/flatcar-linux/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
azure/flatcar-linux/kubernetes/security.tf
View File

@ -95,7 +95,7 @@ resource "azurerm_network_security_rule" "controller-kube-metrics" {
direction = "Inbound"
protocol = "Tcp"
source_port_range = "*"
destination_port_range = "10251-10252"
destination_port_range = "10257-10259"
source_address_prefix = azurerm_subnet.worker.address_prefix
destination_address_prefix = azurerm_subnet.controller.address_prefix
}

2
bare-metal/fedora-coreos/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [var.k8s_domain_name]

2
bare-metal/flatcar-linux/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [var.k8s_domain_name]

2
digital-ocean/fedora-coreos/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
digital-ocean/fedora-coreos/kubernetes/network.tf
View File

@ -116,7 +116,7 @@ resource "digitalocean_firewall" "controllers" {
# kube-scheduler metrics, kube-controller-manager metrics
inbound_rule {
protocol = "tcp"
port_range = "10251-10252"
port_range = "10257-10259"
source_tags = [digitalocean_tag.workers.name]
}
}

2
digital-ocean/flatcar-linux/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
digital-ocean/flatcar-linux/kubernetes/network.tf
View File

@ -116,7 +116,7 @@ resource "digitalocean_firewall" "controllers" {
# kube-scheduler metrics, kube-controller-manager metrics
inbound_rule {
protocol = "tcp"
port_range = "10251-10252"
port_range = "10257-10259"
source_tags = [digitalocean_tag.workers.name]
}
}

2
google-cloud/fedora-coreos/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
google-cloud/fedora-coreos/kubernetes/network.tf
View File

@ -55,7 +55,7 @@ resource "google_compute_firewall" "internal-kube-metrics" {
allow {
protocol = "tcp"
ports = [10251, 10252]
ports = [10257, 10259]
}
source_tags = ["${var.cluster_name}-worker"]

2
google-cloud/flatcar-linux/kubernetes/bootstrap.tf
View File

@ -1,6 +1,6 @@
# Kubernetes assets (kubeconfig, manifests)
module "bootstrap" {
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
cluster_name = var.cluster_name
api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)]

2
google-cloud/flatcar-linux/kubernetes/network.tf
View File

@ -55,7 +55,7 @@ resource "google_compute_firewall" "internal-kube-metrics" {
allow {
protocol = "tcp"
ports = [10251, 10252]
ports = [10257, 10259]
}
source_tags = ["${var.cluster_name}-worker"]