Add optional cluster_domain_suffix variable

* Allow kube-dns to respond to DNS queries with a custom
suffix, instead of the default 'cluster.local'
* Useful when multiple clusters exist on the same local
network and wish to query services on one another

aws/container-linux/kubernetes/bootkube.tf

@@ -2,12 +2,13 @@
 module "bootkube" {
   source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.9.1"
 
-  cluster_name = "${var.cluster_name}"
-  api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers = ["${aws_route53_record.etcds.*.fqdn}"]
-  asset_dir    = "${var.asset_dir}"
-  networking   = "${var.networking}"
-  network_mtu  = "${var.network_mtu}"
-  pod_cidr     = "${var.pod_cidr}"
-  service_cidr = "${var.service_cidr}"
+  cluster_name          = "${var.cluster_name}"
+  api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
+  etcd_servers          = ["${aws_route53_record.etcds.*.fqdn}"]
+  asset_dir             = "${var.asset_dir}"
+  networking            = "${var.networking}"
+  network_mtu           = "${var.network_mtu}"
+  pod_cidr              = "${var.pod_cidr}"
+  service_cidr          = "${var.service_cidr}"
+  cluster_domain_suffix = "${var.cluster_domain_suffix}"
 }

aws/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -73,7 +73,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --kubeconfig=/etc/kubernetes/kubeconfig \

aws/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -49,7 +49,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --kubeconfig=/etc/kubernetes/kubeconfig \

aws/container-linux/kubernetes/controllers.tf

@@ -54,6 +54,7 @@ data "template_file" "controller_config" {
 
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
     ssh_authorized_key      = "${var.ssh_authorized_key}"
+    cluster_domain_suffix   = "${var.cluster_domain_suffix}"
     kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
     kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
     kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"

aws/container-linux/kubernetes/variables.tf

@@ -94,3 +94,9 @@ EOD
   type    = "string"
   default = "10.3.0.0/16"
 }
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}

aws/container-linux/kubernetes/workers.tf

@@ -59,6 +59,7 @@ data "template_file" "worker_config" {
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
     k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
     ssh_authorized_key      = "${var.ssh_authorized_key}"
+    cluster_domain_suffix   = "${var.cluster_domain_suffix}"
     kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
     kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
     kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"

bare-metal/container-linux/kubernetes/bootkube.tf

@@ -2,12 +2,13 @@
 module "bootkube" {
   source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.9.1"
 
-  cluster_name = "${var.cluster_name}"
-  api_servers  = ["${var.k8s_domain_name}"]
-  etcd_servers = ["${var.controller_domains}"]
-  asset_dir    = "${var.asset_dir}"
-  networking   = "${var.networking}"
-  network_mtu  = "${var.network_mtu}"
-  pod_cidr     = "${var.pod_cidr}"
-  service_cidr = "${var.service_cidr}"
+  cluster_name          = "${var.cluster_name}"
+  api_servers           = ["${var.k8s_domain_name}"]
+  etcd_servers          = ["${var.controller_domains}"]
+  asset_dir             = "${var.asset_dir}"
+  networking            = "${var.networking}"
+  network_mtu           = "${var.network_mtu}"
+  pod_cidr              = "${var.pod_cidr}"
+  service_cidr          = "${var.service_cidr}"
+  cluster_domain_suffix = "${var.cluster_domain_suffix}"
 }

bare-metal/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -82,7 +82,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --hostname-override=${domain_name} \

bare-metal/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -58,7 +58,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --hostname-override=${domain_name} \

bare-metal/container-linux/kubernetes/profiles.tf

@@ -84,11 +84,12 @@ data "template_file" "controller-configs" {
   template = "${file("${path.module}/cl/controller.yaml.tmpl")}"
 
   vars {
-    domain_name          = "${element(var.controller_domains, count.index)}"
-    etcd_name            = "${element(var.controller_names, count.index)}"
-    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
-    k8s_dns_service_ip   = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key   = "${var.ssh_authorized_key}"
+    domain_name           = "${element(var.controller_domains, count.index)}"
+    etcd_name             = "${element(var.controller_names, count.index)}"
+    etcd_initial_cluster  = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}"
+    k8s_dns_service_ip    = "${module.bootkube.kube_dns_service_ip}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
+    ssh_authorized_key    = "${var.ssh_authorized_key}"
 
     # Terraform evaluates both sides regardless and element cannot be used on 0 length lists
     networkd_content = "${length(var.controller_networkds) == 0 ? "" : element(concat(var.controller_networkds, list("")), count.index)}"
@@ -108,9 +109,10 @@ data "template_file" "worker-configs" {
   template = "${file("${path.module}/cl/worker.yaml.tmpl")}"
 
   vars {
-    domain_name        = "${element(var.worker_domains, count.index)}"
-    k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}"
-    ssh_authorized_key = "${var.ssh_authorized_key}"
+    domain_name           = "${element(var.worker_domains, count.index)}"
+    k8s_dns_service_ip    = "${module.bootkube.kube_dns_service_ip}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
+    ssh_authorized_key    = "${var.ssh_authorized_key}"
 
     # Terraform evaluates both sides regardless and element cannot be used on 0 length lists
     networkd_content = "${length(var.worker_networkds) == 0 ? "" : element(concat(var.worker_networkds, list("")), count.index)}"

bare-metal/container-linux/kubernetes/variables.tf

@@ -92,6 +92,12 @@ EOD
 
 # optional
 
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}
+
 variable "cached_install" {
   type        = "string"
   default     = "false"

bare-metal/container-linux/pxe-worker/cl/bootkube-worker.yaml.tmpl

@@ -58,7 +58,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns={{.k8s_dns_service_ip}} \
-          --cluster_domain=cluster.local \
+          --cluster_domain={{.cluster_domain_suffix}} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --hostname-override={{.domain_name}} \

bare-metal/container-linux/pxe-worker/groups.tf

@@ -13,9 +13,10 @@ resource "matchbox_group" "workers" {
     etcd_endpoints = "${join(",", formatlist("%s:2379", var.controller_domains))}"
 
     # TODO
-    etcd_on_host        = "true"
-    k8s_etcd_service_ip = "10.3.0.15"
-    k8s_dns_service_ip  = "${var.kube_dns_service_ip}"
-    ssh_authorized_key  = "${var.ssh_authorized_key}"
+    etcd_on_host          = "true"
+    k8s_etcd_service_ip   = "10.3.0.15"
+    k8s_dns_service_ip    = "${var.kube_dns_service_ip}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
+    ssh_authorized_key    = "${var.ssh_authorized_key}"
   }
 }

bare-metal/container-linux/pxe-worker/variables.tf

@@ -64,3 +64,9 @@ variable "kernel_args" {
     "root=/dev/sda1",
   ]
 }
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}

digital-ocean/container-linux/kubernetes/bootkube.tf

@@ -2,12 +2,13 @@
 module "bootkube" {
   source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.9.1"
 
-  cluster_name = "${var.cluster_name}"
-  api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers = "${digitalocean_record.etcds.*.fqdn}"
-  asset_dir    = "${var.asset_dir}"
-  networking   = "${var.networking}"
-  network_mtu  = 1440
-  pod_cidr     = "${var.pod_cidr}"
-  service_cidr = "${var.service_cidr}"
+  cluster_name          = "${var.cluster_name}"
+  api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
+  etcd_servers          = "${digitalocean_record.etcds.*.fqdn}"
+  asset_dir             = "${var.asset_dir}"
+  networking            = "${var.networking}"
+  network_mtu           = 1440
+  pod_cidr              = "${var.pod_cidr}"
+  service_cidr          = "${var.service_cidr}"
+  cluster_domain_suffix = "${var.cluster_domain_suffix}"
 }

digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -84,7 +84,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \

digital-ocean/container-linux/kubernetes/cl/worker.yaml.tmpl

@@ -60,7 +60,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --hostname-override=$${COREOS_DIGITALOCEAN_IPV4_PRIVATE_0} \

digital-ocean/container-linux/kubernetes/controllers.tf

@@ -69,8 +69,9 @@ data "template_file" "controller_config" {
     etcd_domain = "${var.cluster_name}-etcd${count.index}.${var.dns_zone}"
 
     # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
-    etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
-    k8s_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
+    etcd_initial_cluster  = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
+    k8s_dns_service_ip    = "${cidrhost(var.service_cidr, 10)}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
   }
 }
 

digital-ocean/container-linux/kubernetes/variables.tf

@@ -76,3 +76,10 @@ EOD
   type    = "string"
   default = "10.3.0.0/16"
 }
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}
+

digital-ocean/container-linux/kubernetes/workers.tf

@@ -43,8 +43,9 @@ data "template_file" "worker_config" {
   template = "${file("${path.module}/cl/worker.yaml.tmpl")}"
 
   vars = {
-    k8s_dns_service_ip  = "${cidrhost(var.service_cidr, 10)}"
-    k8s_etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
+    k8s_dns_service_ip    = "${cidrhost(var.service_cidr, 10)}"
+    k8s_etcd_service_ip   = "${cidrhost(var.service_cidr, 15)}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
   }
 }
 

docs/aws.md

@@ -228,6 +228,7 @@ Reference the DNS zone id with `"${aws_route53_zone.zone-for-clusters.zone_id}"`
 | host_cidr | CIDR range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" |
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
+| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
 
 Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-types/).
 

docs/bare-metal.md

@@ -355,5 +355,6 @@ Learn about [version pinning](concepts.md#versioning), maintenance, and [addons]
 | network_mtu | CNI interface MTU (calico-only) | 1480 | - | 
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
+| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
 | kernel_args | Additional kernel args to provide at PXE boot | [] | "kvm-intel.nested=1" |
 

docs/digital-ocean.md

@@ -238,6 +238,7 @@ If you uploaded an SSH key to DigitalOcean (not required), find the fingerprint
 | networking | Choice of networking provider | "flannel" | "flannel" |
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
+| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
 
 !!! warning
     Do not choose a `controller_type` smaller than `2gb`. The `1gb` droplet is not sufficient for running a controller and bootstrapping will fail.

docs/google-cloud.md

@@ -230,6 +230,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" {
 | networking | Choice of networking provider | "calico" | "calico" or "flannel" |
 | pod_cidr | CIDR range to assign to Kubernetes pods | "10.2.0.0/16" | "10.22.0.0/16" |
 | service_cidr | CIDR range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" |
+| cluster_domain_suffix | FQDN suffix for Kubernetes services answered by kube-dns. | "cluster.local" | "k8s.example.com" |
 
 Check the list of valid [machine types](https://cloud.google.com/compute/docs/machine-types).
 

google-cloud/container-linux/kubernetes/bootkube.tf

@@ -2,12 +2,13 @@
 module "bootkube" {
   source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=v0.9.1"
 
-  cluster_name = "${var.cluster_name}"
-  api_servers  = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
-  etcd_servers = "${module.controllers.etcd_fqdns}"
-  asset_dir    = "${var.asset_dir}"
-  networking   = "${var.networking}"
-  network_mtu  = 1440
-  pod_cidr     = "${var.pod_cidr}"
-  service_cidr = "${var.service_cidr}"
+  cluster_name          = "${var.cluster_name}"
+  api_servers           = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
+  etcd_servers          = "${module.controllers.etcd_fqdns}"
+  asset_dir             = "${var.asset_dir}"
+  networking            = "${var.networking}"
+  network_mtu           = 1440
+  pod_cidr              = "${var.pod_cidr}"
+  service_cidr          = "${var.service_cidr}"
+  cluster_domain_suffix = "${var.cluster_domain_suffix}"
 }

google-cloud/container-linux/kubernetes/cluster.tf

@@ -15,6 +15,7 @@ module "controllers" {
   # configuration
   networking              = "${var.networking}"
   service_cidr            = "${var.service_cidr}"
+  cluster_domain_suffix   = "${var.cluster_domain_suffix}"
   kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
   kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
   kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"
@@ -36,6 +37,7 @@ module "workers" {
 
   # configuration
   service_cidr            = "${var.service_cidr}"
+  cluster_domain_suffix   = "${var.cluster_domain_suffix}"
   kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
   kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
   kubeconfig_kubelet_key  = "${module.bootkube.kubelet_key}"

google-cloud/container-linux/kubernetes/controllers/cl/controller.yaml.tmpl

@@ -74,7 +74,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --kubeconfig=/etc/kubernetes/kubeconfig \

google-cloud/container-linux/kubernetes/controllers/controllers.tf

@@ -66,6 +66,7 @@ data "template_file" "controller_config" {
     etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", null_resource.repeat.*.triggers.name, null_resource.repeat.*.triggers.domain))}"
 
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
+    cluster_domain_suffix   = "${var.cluster_domain_suffix}"
     ssh_authorized_key      = "${var.ssh_authorized_key}"
     kubeconfig_ca_cert      = "${var.kubeconfig_ca_cert}"
     kubeconfig_kubelet_cert = "${var.kubeconfig_kubelet_cert}"

google-cloud/container-linux/kubernetes/controllers/variables.tf

@@ -69,6 +69,12 @@ EOD
   default = "10.3.0.0/16"
 }
 
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}
+
 // kubeconfig
 
 variable "kubeconfig_ca_cert" {

google-cloud/container-linux/kubernetes/variables.tf

@@ -80,3 +80,9 @@ EOD
   type    = "string"
   default = "10.3.0.0/16"
 }
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}

google-cloud/container-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -50,7 +50,7 @@ systemd:
           --anonymous-auth=false \
           --client-ca-file=/etc/kubernetes/ca.crt \
           --cluster_dns=${k8s_dns_service_ip} \
-          --cluster_domain=cluster.local \
+          --cluster_domain=${cluster_domain_suffix} \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --exit-on-lock-contention \
           --kubeconfig=/etc/kubernetes/kubeconfig \

google-cloud/container-linux/kubernetes/workers/variables.tf

@@ -59,6 +59,12 @@ EOD
   default = "10.3.0.0/16"
 }
 
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns. Default is cluster.local (e.g. foo.default.svc.cluster.local) "
+  type        = "string"
+  default     = "cluster.local"
+}
+
 # kubeconfig
 
 variable "kubeconfig_ca_cert" {

google-cloud/container-linux/kubernetes/workers/workers.tf

@@ -24,6 +24,7 @@ data "template_file" "worker_config" {
   vars = {
     k8s_dns_service_ip      = "${cidrhost(var.service_cidr, 10)}"
     k8s_etcd_service_ip     = "${cidrhost(var.service_cidr, 15)}"
+    cluster_domain_suffix   = "${var.cluster_domain_suffix}"
     ssh_authorized_key      = "${var.ssh_authorized_key}"
     kubeconfig_ca_cert      = "${var.kubeconfig_ca_cert}"
     kubeconfig_kubelet_cert = "${var.kubeconfig_kubelet_cert}"