Update Prometheus discovery of kube components

* Kubernetes v1.22.0 disabled kube-controller-manager insecure
port, which was used internally for Prometheus metrics scraping
* Configure Prometheus to discover and scrape endpoints for
kube-scheduler and kube-controller-manager via the authenticated
https ports, via bearer token
* Change firewall ports to allow Prometheus (on worker nodes)
to scrape kube-scheduler and kube-controller-manager targets
that run on controller(s) with hostNetwork
* Disable the insecure port on kube-scheduler

CHANGES.md

@@ -7,6 +7,8 @@ Notable changes between versions.
 ## v1.22.0
 
 * Kubernetes [v1.22.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#v1220)
+* Switch `kube-controller-manager` and `kube-scheduler` to use secure port only
+  * Update Prometheus config to discover endpoints and use a bearer token to scrape
 
 ### Fedora CoreOS
 

addons/prometheus/config.yaml

@@ -72,6 +72,48 @@ data:
         regex: apiserver_request_duration_seconds_count;.+
         action: drop
 
+    # Scrape config for kube-controller-manager endpoints.
+    #
+    # kube-controller-manager service endpoints can be discovered by using the
+    # `endpoints` role and relabelling to only keep only endpoints associated with
+    # kube-system/kube-controller-manager and the `https` port.
+    - job_name: 'kube-controller-manager'
+      kubernetes_sd_configs:
+      - role: endpoints
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: kube-system;kube-controller-manager;metrics
+      - replacement: kube-controller-manager
+        action: replace
+        target_label: job
+
+    # Scrape config for kube-scheduler endpoints.
+    #
+    # kube-scheduler service endpoints can be discovered by using the `endpoints`
+    # role and relabelling to only keep only endpoints associated with
+    # kube-system/kube-scheduler and the `https` port.
+    - job_name: 'kube-scheduler'
+      kubernetes_sd_configs:
+      - role: endpoints
+      scheme: https
+      tls_config:
+        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        insecure_skip_verify: true
+      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+        action: keep
+        regex: kube-system;kube-scheduler;metrics
+      - replacement: kube-scheduler
+        action: replace
+        target_label: job
+
     # Scrape config for node (i.e. kubelet) /metrics (e.g. 'kubelet_'). Explore
     # metrics from a node by scraping kubelet (127.0.0.1:10250/metrics).
     - job_name: 'kubelet'

addons/prometheus/discovery/kube-controller-manager.yaml

@@ -1,11 +1,9 @@
-# Allow Prometheus to scrape service endpoints
+# Allow Prometheus to discover service endpoints
 apiVersion: v1
 kind: Service
 metadata:
   name: kube-controller-manager
   namespace: kube-system
-  annotations:
-    prometheus.io/scrape: 'true'
 spec:
   type: ClusterIP
   clusterIP: None
@@ -14,5 +12,5 @@ spec:
   ports:
     - name: metrics
       protocol: TCP
-      port: 10252
+      port: 10257
-      targetPort: 10252
+      targetPort: 10257

addons/prometheus/discovery/kube-scheduler.yaml

@@ -1,11 +1,9 @@
-# Allow Prometheus to scrape service endpoints
+# Allow Prometheus to discover service endpoints
 apiVersion: v1
 kind: Service
 metadata:
   name: kube-scheduler
   namespace: kube-system
-  annotations:
-    prometheus.io/scrape: 'true'
 spec:
   type: ClusterIP
   clusterIP: None
@@ -14,5 +12,5 @@ spec:
   ports:
     - name: metrics
       protocol: TCP
-      port: 10251
+      port: 10259
-      targetPort: 10251
+      targetPort: 10259

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/fedora-coreos/kubernetes/security.tf

@@ -201,8 +201,8 @@ resource "aws_security_group_rule" "controller-scheduler-metrics" {
 
   type                     = "ingress"
   protocol                 = "tcp"
-  from_port                = 10251
+  from_port                = 10259
-  to_port                  = 10251
+  to_port                  = 10259
   source_security_group_id = aws_security_group.worker.id
 }
 
@@ -212,8 +212,8 @@ resource "aws_security_group_rule" "controller-manager-metrics" {
 
   type                     = "ingress"
   protocol                 = "tcp"
-  from_port                = 10252
+  from_port                = 10257
-  to_port                  = 10252
+  to_port                  = 10257
   source_security_group_id = aws_security_group.worker.id
 }
 

aws/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/flatcar-linux/kubernetes/security.tf

@@ -201,8 +201,8 @@ resource "aws_security_group_rule" "controller-scheduler-metrics" {
 
   type                     = "ingress"
   protocol                 = "tcp"
-  from_port                = 10251
+  from_port                = 10259
-  to_port                  = 10251
+  to_port                  = 10259
   source_security_group_id = aws_security_group.worker.id
 }
 
@@ -212,8 +212,8 @@ resource "aws_security_group_rule" "controller-manager-metrics" {
 
   type                     = "ingress"
   protocol                 = "tcp"
-  from_port                = 10252
+  from_port                = 10257
-  to_port                  = 10252
+  to_port                  = 10257
   source_security_group_id = aws_security_group.worker.id
 }
 

azure/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/fedora-coreos/kubernetes/security.tf

@@ -95,7 +95,7 @@ resource "azurerm_network_security_rule" "controller-kube-metrics" {
   direction                   = "Inbound"
   protocol                    = "Tcp"
   source_port_range           = "*"
-  destination_port_range      = "10251-10252"
+  destination_port_range      = "10257-10259"
   source_address_prefix       = azurerm_subnet.worker.address_prefix
   destination_address_prefix  = azurerm_subnet.controller.address_prefix
 }

azure/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/flatcar-linux/kubernetes/security.tf

@@ -95,7 +95,7 @@ resource "azurerm_network_security_rule" "controller-kube-metrics" {
   direction                   = "Inbound"
   protocol                    = "Tcp"
   source_port_range           = "*"
-  destination_port_range      = "10251-10252"
+  destination_port_range      = "10257-10259"
   source_address_prefix       = azurerm_subnet.worker.address_prefix
   destination_address_prefix  = azurerm_subnet.controller.address_prefix
 }

bare-metal/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

bare-metal/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

digital-ocean/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

digital-ocean/fedora-coreos/kubernetes/network.tf

@@ -116,7 +116,7 @@ resource "digitalocean_firewall" "controllers" {
   # kube-scheduler metrics, kube-controller-manager metrics
   inbound_rule {
     protocol    = "tcp"
-    port_range  = "10251-10252"
+    port_range  = "10257-10259"
     source_tags = [digitalocean_tag.workers.name]
   }
 }

digital-ocean/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

digital-ocean/flatcar-linux/kubernetes/network.tf

@@ -116,7 +116,7 @@ resource "digitalocean_firewall" "controllers" {
   # kube-scheduler metrics, kube-controller-manager metrics
   inbound_rule {
     protocol    = "tcp"
-    port_range  = "10251-10252"
+    port_range  = "10257-10259"
     source_tags = [digitalocean_tag.workers.name]
   }
 }

google-cloud/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

google-cloud/fedora-coreos/kubernetes/network.tf

@@ -55,7 +55,7 @@ resource "google_compute_firewall" "internal-kube-metrics" {
 
   allow {
     protocol = "tcp"
-    ports    = [10251, 10252]
+    ports    = [10257, 10259]
   }
 
   source_tags = ["${var.cluster_name}-worker"]

google-cloud/flatcar-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b766ff2346921a4f5587a45b948b5c79969357ae"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=b5f5d843ec9babcd2eeea98b8edcef972a5c178d"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

google-cloud/flatcar-linux/kubernetes/network.tf

@@ -55,7 +55,7 @@ resource "google_compute_firewall" "internal-kube-metrics" {
 
   allow {
     protocol = "tcp"
-    ports    = [10251, 10252]
+    ports    = [10257, 10259]
   }
 
   source_tags = ["${var.cluster_name}-worker"]