Return Prometheus deployment to be a worker workload

* Expose etcd metrics to workers so Prometheus can
run on a worker, rather than a controller
* Drop temporary firewall rules allowing Prometheus
to run on a controller and scrape targes
* Related to https://github.com/poseidon/typhoon/pull/175

addons/prometheus/deployment.yaml

@@ -15,12 +15,6 @@ spec:
         name: prometheus
         phase: prod
     spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
       serviceAccountName: prometheus
       containers:
       - name: prometheus

aws/container-linux/kubernetes/security.tf

@@ -51,6 +51,16 @@ resource "aws_security_group_rule" "controller-etcd" {
   self      = true
 }
 
+resource "aws_security_group_rule" "controller-etcd-metrics" {
+  security_group_id = "${aws_security_group.controller.id}"
+
+  type                     = "ingress"
+  protocol                 = "tcp"
+  from_port                = 2381
+  to_port                  = 2381
+  source_security_group_id = "${aws_security_group.worker.id}"
+}
+
 resource "aws_security_group_rule" "controller-flannel" {
   security_group_id = "${aws_security_group.controller.id}"
 
@@ -81,16 +91,6 @@ resource "aws_security_group_rule" "controller-node-exporter" {
   source_security_group_id = "${aws_security_group.worker.id}"
 }
 
-resource "aws_security_group_rule" "controller-node-exporter-self" {
-  security_group_id = "${aws_security_group.controller.id}"
-
-  type      = "ingress"
-  protocol  = "tcp"
-  from_port = 9100
-  to_port   = 9100
-  self      = true
-}
-
 resource "aws_security_group_rule" "controller-kubelet-self" {
   security_group_id = "${aws_security_group.controller.id}"
 
@@ -266,16 +266,6 @@ resource "aws_security_group_rule" "worker-flannel-self" {
 resource "aws_security_group_rule" "worker-node-exporter" {
   security_group_id = "${aws_security_group.worker.id}"
 
-  type                     = "ingress"
-  protocol                 = "tcp"
-  from_port                = 9100
-  to_port                  = 9100
-  source_security_group_id = "${aws_security_group.controller.id}"
-}
-
-resource "aws_security_group_rule" "worker-node-exporter-self" {
-  security_group_id = "${aws_security_group.worker.id}"
-
   type      = "ingress"
   protocol  = "tcp"
   from_port = 9100

google-cloud/container-linux/kubernetes/network.tf

@@ -56,6 +56,20 @@ resource "google_compute_firewall" "internal-etcd" {
   target_tags = ["${var.cluster_name}-controller"]
 }
 
+# Allow Prometheus to scrape etcd metrics
+resource "google_compute_firewall" "internal-etcd-metrics" {
+  name    = "${var.cluster_name}-internal-etcd-metrics"
+  network = "${google_compute_network.network.name}"
+
+  allow {
+    protocol = "tcp"
+    ports    = [2381]
+  }
+
+  source_tags = ["${var.cluster_name}-worker"]
+  target_tags = ["${var.cluster_name}-controller"]
+}
+
 # Calico BGP and IPIP
 # https://docs.projectcalico.org/v2.5/reference/public-cloud/gce
 resource "google_compute_firewall" "internal-calico" {
@@ -103,7 +117,7 @@ resource "google_compute_firewall" "internal-node-exporter" {
     ports    = [9100]
   }
 
-  source_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
+  source_tags = ["${var.cluster_name}-worker"]
   target_tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"]
 }