From ec46bc13ae09ed2549b62383bd4bc044304e5d70 Mon Sep 17 00:00:00 2001
From: Dalton Hubble <dghubble@gmail.com>
Date: Mon, 4 Sep 2017 21:14:05 -0700
Subject: [PATCH] Add support for Calico networking on GCE

* Calico on GCE with IP-in-IP encapsulation and MTU 1440
* Calico on DO with IP-in-IP encapsulation and MTU 1440
* Digital Ocean firewalls don't support IPIP protocol yet
---
 .../container-linux/kubernetes/bootkube.tf    |  4 +++-
 .../kubernetes/cl/controller.yaml.tmpl        |  1 +
 .../container-linux/kubernetes/variables.tf   |  6 ++++++
 .../controllers/cl/controller.yaml.tmpl       |  1 +
 .../container-linux/controllers/variables.tf  |  6 ++++++
 .../container-linux/kubernetes/bootkube.tf    |  4 +++-
 .../container-linux/kubernetes/cluster.tf     |  1 +
 .../container-linux/kubernetes/network.tf     | 20 +++++++++++++++++++
 .../container-linux/kubernetes/variables.tf   |  6 ++++++
 9 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf
index d53b2d70..a2e3a75d 100644
--- a/digital-ocean/container-linux/kubernetes/bootkube.tf
+++ b/digital-ocean/container-linux/kubernetes/bootkube.tf
@@ -1,11 +1,13 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.6.1"
+  source = "git::https://github.com/poseidon/bootkube-terraform.git?ref=5ffbfec46dc05721eaf9d15c3c9bbedefaead1bc"
 
   cluster_name                  = "${var.cluster_name}"
   api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
   etcd_servers                  = ["http://127.0.0.1:2379"]
   asset_dir                     = "${var.asset_dir}"
+  networking                    = "${var.networking}"
+  network_mtu                   = 1440
   pod_cidr                      = "${var.pod_cidr}"
   service_cidr                  = "${var.service_cidr}"
   experimental_self_hosted_etcd = "true"
diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl
index 8024e200..4caac8ff 100644
--- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl
+++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml.tmpl
@@ -124,6 +124,7 @@ storage:
           # Wrapper for bootkube start
           set -e
           # Move experimental manifests
+          [ -d /opt/bootkube/assets/manifests-* ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
diff --git a/digital-ocean/container-linux/kubernetes/variables.tf b/digital-ocean/container-linux/kubernetes/variables.tf
index 4260a0ad..d74331d4 100644
--- a/digital-ocean/container-linux/kubernetes/variables.tf
+++ b/digital-ocean/container-linux/kubernetes/variables.tf
@@ -55,6 +55,12 @@ variable "asset_dir" {
   type        = "string"
 }
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"
diff --git a/google-cloud/container-linux/controllers/cl/controller.yaml.tmpl b/google-cloud/container-linux/controllers/cl/controller.yaml.tmpl
index 1e04df43..b23b6168 100644
--- a/google-cloud/container-linux/controllers/cl/controller.yaml.tmpl
+++ b/google-cloud/container-linux/controllers/cl/controller.yaml.tmpl
@@ -120,6 +120,7 @@ storage:
           # Wrapper for bootkube start
           set -e
           # Move experimental manifests
+          [ -d /opt/bootkube/assets/manifests-* ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
           [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests
           [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests
           BOOTKUBE_ACI="$${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
diff --git a/google-cloud/container-linux/controllers/variables.tf b/google-cloud/container-linux/controllers/variables.tf
index 7d37d144..3244473c 100644
--- a/google-cloud/container-linux/controllers/variables.tf
+++ b/google-cloud/container-linux/controllers/variables.tf
@@ -59,6 +59,12 @@ variable "preemptible" {
 
 // configuration
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
 variable "service_cidr" {
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf
index d53b2d70..a2e3a75d 100644
--- a/google-cloud/container-linux/kubernetes/bootkube.tf
+++ b/google-cloud/container-linux/kubernetes/bootkube.tf
@@ -1,11 +1,13 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.6.1"
+  source = "git::https://github.com/poseidon/bootkube-terraform.git?ref=5ffbfec46dc05721eaf9d15c3c9bbedefaead1bc"
 
   cluster_name                  = "${var.cluster_name}"
   api_servers                   = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"]
   etcd_servers                  = ["http://127.0.0.1:2379"]
   asset_dir                     = "${var.asset_dir}"
+  networking                    = "${var.networking}"
+  network_mtu                   = 1440
   pod_cidr                      = "${var.pod_cidr}"
   service_cidr                  = "${var.service_cidr}"
   experimental_self_hosted_etcd = "true"
diff --git a/google-cloud/container-linux/kubernetes/cluster.tf b/google-cloud/container-linux/kubernetes/cluster.tf
index d0f4feaf..6eb6e476 100644
--- a/google-cloud/container-linux/kubernetes/cluster.tf
+++ b/google-cloud/container-linux/kubernetes/cluster.tf
@@ -14,6 +14,7 @@ module "controllers" {
   preemptible   = "${var.controller_preemptible}"
 
   # configuration
+  networking              = "${var.networking}"
   service_cidr            = "${var.service_cidr}"
   kubeconfig_ca_cert      = "${module.bootkube.ca_cert}"
   kubeconfig_kubelet_cert = "${module.bootkube.kubelet_cert}"
diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf
index 0f087230..264ff3a3 100644
--- a/google-cloud/container-linux/kubernetes/network.tf
+++ b/google-cloud/container-linux/kubernetes/network.tf
@@ -44,3 +44,23 @@ resource "google_compute_firewall" "allow-internal" {
 
   source_ranges = ["10.0.0.0/8"]
 }
+
+# Calico BGP and IPIP
+# https://docs.projectcalico.org/v2.5/reference/public-cloud/gce
+resource "google_compute_firewall" "allow-calico" {
+  count = "${var.networking == "calico" ? 1 : 0}"
+
+  name    = "${var.cluster_name}-allow-calico"
+  network = "${google_compute_network.network.name}"
+
+  allow {
+    protocol = "tcp"
+    ports    = ["179"]
+  }
+
+  allow {
+    protocol = "ipip"
+  }
+
+  source_ranges = ["10.0.0.0/8"]
+}
diff --git a/google-cloud/container-linux/kubernetes/variables.tf b/google-cloud/container-linux/kubernetes/variables.tf
index 25f16408..d36ea66e 100644
--- a/google-cloud/container-linux/kubernetes/variables.tf
+++ b/google-cloud/container-linux/kubernetes/variables.tf
@@ -65,6 +65,12 @@ variable "asset_dir" {
   type        = "string"
 }
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"