From f503dc525858311e64a012425bc28a4a5488a1e7 Mon Sep 17 00:00:00 2001
From: Tarun Pothulapati <tarunpothulapati@outlook.com>
Date: Mon, 8 Nov 2021 11:27:59 +0530
Subject: [PATCH 1/5] pin k8s versions with integration tests

Fixes #7112

This pins the default integration tests to run on the latest
k3d version (which is currently `v5.0.3` and runs k3s `v1.21.5-k3s2`) by
retrieving the latest k3d binary, while also pinning `helm-deep` to run on
`v4.4.4` which runs k3s `v1.20.6-k3s1` (our min k8s version). This way
we have integration tests running on both sides (i.e latest and minimum)

Changes include:

- Update `./bin/k3d` to always retrieve the latest k3d binary and
  use it.
- Add new `./bin/k3d-min` script which pins the minimum k3d version
  (`v1.20.6-k3s1`) and uses it.
- Update `_test-helpers.sh` to use the new `./bin/k3d-min` script based
  on the test. here, we also replace flags that are removed and replaced
  with newer ones with the latest k3d binary.

I've tried to find a GitHub action for K3s, but most of them seem to
be have a had a release while ago, and not goes well with our plan of
somehow getting the latest k8s.

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>
---
 bin/_test-helpers.sh | 32 ++++++++++++++++++++++++++++++--
 bin/k3d              |  6 ++----
 bin/k3d-min          | 33 +++++++++++++++++++++++++++++++++
 3 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100755 bin/k3d-min

diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh
index cee6b7b4ff37d..dae3e91ba5057 100644
--- a/bin/_test-helpers.sh
+++ b/bin/_test-helpers.sh
@@ -229,6 +229,18 @@ cleanup_cluster() {
   exit_on_err 'error removing existing Linkerd resources'
 }
 
+setup_min_cluster() {
+  local name=$1
+  export helm_path="$bindir"/helm
+
+  test_setup
+  if [ -z "$skip_cluster_create" ]; then
+    "$bindir"/k3d-min cluster create "$@"
+    image_load "$name"
+  fi
+  check_cluster
+}
+
 setup_cluster() {
   local name=$1
   export helm_path="$bindir"/helm
@@ -304,7 +316,8 @@ image_load() {
 
 start_test() {
   local name=$1
-  local config=(--no-hostip --k3s-server-arg '--disable=local-storage,metrics-server')
+  local config=(--k3s-arg '--disable=local-storage,metrics-server')
+  local min_config=(--no-hostip --k3s-server-arg '--disable=local-storage,metrics-server')
 
   case $name in
     cluster-domain)
@@ -317,17 +330,32 @@ start_test() {
       config=("${config[@]}" --network multicluster-test)
       ;;
     *)
-      config=("$name" "${config[@]}" --no-lb --k3s-server-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg '--disable=servicelb,traefik')
+      min_config=("$name" "${min_config[@]}" --no-lb --k3s-server-arg '--disable=servicelb,traefik')
       ;;
   esac
 
   if [ "$name" == "multicluster" ]; then
     start_multicluster_test "${config[@]}"
+  elif [ "$name" == "helm-deep" ]; then
+    start_min_single_test "${min_config[@]}"
   else
     start_single_test "${config[@]}"
   fi
 }
 
+start_min_single_test() {
+  name=$1
+  setup_min_cluster "$@"
+  if [ -n "$cleanup_docker" ]; then
+    rm -rf image-archives
+    docker system prune --force --all
+  fi
+  run_"$name"_test
+  exit_on_err "error calling 'run_${name}_test'"
+  finish "$name"
+}
+
 start_single_test() {
   name=$1
   setup_cluster "$@"
diff --git a/bin/k3d b/bin/k3d
index 94e9113f4f0e4..6546877b7a9ec 100755
--- a/bin/k3d
+++ b/bin/k3d
@@ -2,11 +2,9 @@
 
 set -eu
 
-k3dversion=v4.4.5
-
 bindir=$( cd "${0%/*}" && pwd )
 targetbin=$( cd "$bindir"/.. && pwd )/target/bin
-k3dbin=$targetbin/.k3d-$k3dversion
+k3dbin=$targetbin/.k3d
 
 if [ ! -f "$k3dbin" ]; then
   if [ "$(uname -s)" = Darwin ]; then
@@ -24,7 +22,7 @@ if [ ! -f "$k3dbin" ]; then
   fi
 
   mkdir -p "$targetbin"
-  curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/download/$k3dversion/k3d-$os-$arch
+  curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/latest/download/k3d-$os-$arch
   chmod +x "$k3dbin"
 fi
 
diff --git a/bin/k3d-min b/bin/k3d-min
new file mode 100755
index 0000000000000..d1515ebd90aef
--- /dev/null
+++ b/bin/k3d-min
@@ -0,0 +1,33 @@
+#!/usr/bin/env sh
+
+set -eu
+
+# v4.4.4 seems to be the last version which defaults to k8s v1.20.*
+k3dversion=v4.4.4
+
+bindir=$( cd "${0%/*}" && pwd )
+targetbin=$( cd "$bindir"/.. && pwd )/target/bin
+k3dbin=$targetbin/.k3d-$k3dversion
+
+if [ ! -f "$k3dbin" ]; then
+  if [ "$(uname -s)" = Darwin ]; then
+    os=darwin
+    arch=amd64
+  elif [ "$(uname -o)" = Msys ]; then
+    os=windows
+    arch=amd64
+  else
+    os=linux
+    case $(uname -m) in
+      x86_64) arch=amd64 ;;
+      arm) arch=arm64 ;;
+    esac
+  fi
+
+  mkdir -p "$targetbin"
+  curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/download/$k3dversion/k3d-$os-$arch
+  chmod +x "$k3dbin"
+fi
+
+"$k3dbin" "$@"
+

From fbc7979b086522e590326128708d5916dc84dacf Mon Sep 17 00:00:00 2001
From: Tarun Pothulapati <tarunpothulapati@outlook.com>
Date: Mon, 8 Nov 2021 15:22:43 +0530
Subject: [PATCH 2/5] use latest args

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>
---
 bin/_test-helpers.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh
index dae3e91ba5057..c79a901518ff9 100644
--- a/bin/_test-helpers.sh
+++ b/bin/_test-helpers.sh
@@ -321,10 +321,10 @@ start_test() {
 
   case $name in
     cluster-domain)
-      config=("$name" "${config[@]}" --no-lb --k3s-server-arg --cluster-domain=custom.domain --k3s-server-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg --cluster-domain=custom.domain --k3s-arg '--disable=servicelb,traefik')
       ;;
     cni-calico-deep)
-      config=("$name" "${config[@]}" --no-lb --k3s-server-arg --write-kubeconfig-mode=644 --k3s-server-arg --flannel-backend=none --k3s-server-arg --cluster-cidr=192.168.0.0/16 --k3s-server-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg --write-kubeconfig-mode=644 --k3s-arg --flannel-backend=none --k3s-arg --cluster-cidr=192.168.0.0/16 --k3s-arg '--disable=servicelb,traefik')
       ;;
     multicluster)
       config=("${config[@]}" --network multicluster-test)

From d854761b877e76ba5dbee5bbb1787f300de0b8f4 Mon Sep 17 00:00:00 2001
From: Tarun Pothulapati <tarunpothulapati@outlook.com>
Date: Mon, 8 Nov 2021 17:06:31 +0530
Subject: [PATCH 3/5] add node labels with k3d commands

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>
---
 bin/_test-helpers.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh
index c79a901518ff9..9f0096cf2d298 100644
--- a/bin/_test-helpers.sh
+++ b/bin/_test-helpers.sh
@@ -316,21 +316,21 @@ image_load() {
 
 start_test() {
   local name=$1
-  local config=(--k3s-arg '--disable=local-storage,metrics-server')
+  local config=(--k3s-arg '--disable=local-storage,metrics-server@server:0')
   local min_config=(--no-hostip --k3s-server-arg '--disable=local-storage,metrics-server')
 
   case $name in
     cluster-domain)
-      config=("$name" "${config[@]}" --no-lb --k3s-arg --cluster-domain=custom.domain --k3s-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg --cluster-domain=custom.domain --k3s-arg '--disable=servicelb,traefik@server:0')
       ;;
     cni-calico-deep)
-      config=("$name" "${config[@]}" --no-lb --k3s-arg --write-kubeconfig-mode=644 --k3s-arg --flannel-backend=none --k3s-arg --cluster-cidr=192.168.0.0/16 --k3s-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg --write-kubeconfig-mode=644 --k3s-arg --flannel-backend=none --k3s-arg --cluster-cidr=192.168.0.0/16 --k3s-arg '--disable=servicelb,traefik@server:0')
       ;;
     multicluster)
       config=("${config[@]}" --network multicluster-test)
       ;;
     *)
-      config=("$name" "${config[@]}" --no-lb --k3s-arg '--disable=servicelb,traefik')
+      config=("$name" "${config[@]}" --no-lb --k3s-arg '--disable=servicelb,traefik@server:0')
       min_config=("$name" "${min_config[@]}" --no-lb --k3s-server-arg '--disable=servicelb,traefik')
       ;;
   esac

From d91418e8f2cd1bbecdae05dd35804c98dae934b0 Mon Sep 17 00:00:00 2001
From: Tarun Pothulapati <tarunpothulapati@outlook.com>
Date: Wed, 10 Nov 2021 14:56:54 +0530
Subject: [PATCH 4/5] remove `k3d-min` and use the same latest binary

even to create a v1.20 cluster

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>
---
 bin/_test-helpers.sh | 24 ++++++------------------
 bin/k3d-min          | 33 ---------------------------------
 2 files changed, 6 insertions(+), 51 deletions(-)
 delete mode 100755 bin/k3d-min

diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh
index 9f0096cf2d298..e3be44c9e0b43 100644
--- a/bin/_test-helpers.sh
+++ b/bin/_test-helpers.sh
@@ -235,7 +235,7 @@ setup_min_cluster() {
 
   test_setup
   if [ -z "$skip_cluster_create" ]; then
-    "$bindir"/k3d-min cluster create "$@"
+    "$bindir"/k3d cluster create "$@"  --image +v1.20
     image_load "$name"
   fi
   check_cluster
@@ -317,7 +317,6 @@ image_load() {
 start_test() {
   local name=$1
   local config=(--k3s-arg '--disable=local-storage,metrics-server@server:0')
-  local min_config=(--no-hostip --k3s-server-arg '--disable=local-storage,metrics-server')
 
   case $name in
     cluster-domain)
@@ -331,34 +330,23 @@ start_test() {
       ;;
     *)
       config=("$name" "${config[@]}" --no-lb --k3s-arg '--disable=servicelb,traefik@server:0')
-      min_config=("$name" "${min_config[@]}" --no-lb --k3s-server-arg '--disable=servicelb,traefik')
       ;;
   esac
 
   if [ "$name" == "multicluster" ]; then
     start_multicluster_test "${config[@]}"
-  elif [ "$name" == "helm-deep" ]; then
-    start_min_single_test "${min_config[@]}"
   else
     start_single_test "${config[@]}"
   fi
 }
 
-start_min_single_test() {
-  name=$1
-  setup_min_cluster "$@"
-  if [ -n "$cleanup_docker" ]; then
-    rm -rf image-archives
-    docker system prune --force --all
-  fi
-  run_"$name"_test
-  exit_on_err "error calling 'run_${name}_test'"
-  finish "$name"
-}
-
 start_single_test() {
   name=$1
-  setup_cluster "$@"
+  if [ "$name" == "helm-deep" ]; then
+    setup_min_cluster "$@"
+  else
+    setup_cluster "$@"
+  fi
   if [ -n "$cleanup_docker" ]; then
     rm -rf image-archives
     docker system prune --force --all
diff --git a/bin/k3d-min b/bin/k3d-min
deleted file mode 100755
index d1515ebd90aef..0000000000000
--- a/bin/k3d-min
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-# v4.4.4 seems to be the last version which defaults to k8s v1.20.*
-k3dversion=v4.4.4
-
-bindir=$( cd "${0%/*}" && pwd )
-targetbin=$( cd "$bindir"/.. && pwd )/target/bin
-k3dbin=$targetbin/.k3d-$k3dversion
-
-if [ ! -f "$k3dbin" ]; then
-  if [ "$(uname -s)" = Darwin ]; then
-    os=darwin
-    arch=amd64
-  elif [ "$(uname -o)" = Msys ]; then
-    os=windows
-    arch=amd64
-  else
-    os=linux
-    case $(uname -m) in
-      x86_64) arch=amd64 ;;
-      arm) arch=arm64 ;;
-    esac
-  fi
-
-  mkdir -p "$targetbin"
-  curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/download/$k3dversion/k3d-$os-$arch
-  chmod +x "$k3dbin"
-fi
-
-"$k3dbin" "$@"
-

From 8282b8939850ca33754e07a1633aa9220dedc488 Mon Sep 17 00:00:00 2001
From: Tarun Pothulapati <tarunpothulapati@outlook.com>
Date: Wed, 10 Nov 2021 16:00:53 +0530
Subject: [PATCH 5/5] print k8s versions before running

Signed-off-by: Tarun Pothulapati <tarunpothulapati@outlook.com>
---
 bin/_test-helpers.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/_test-helpers.sh b/bin/_test-helpers.sh
index e3be44c9e0b43..b6271430997bf 100644
--- a/bin/_test-helpers.sh
+++ b/bin/_test-helpers.sh
@@ -215,6 +215,7 @@ check_linkerd_binary() {
 
 check_cluster() {
   check_if_k8s_reachable
+  kubectl version
   check_if_l5d_exists
 }