Kubeflow in a TKG Cluster within vSphere – Road to Kubeflow

#obtain storage class via "kubectl get sc"
storageclass=gold-tanzu-kubernetes-storage-policy
#obtain virtualmachineclass via "kubectl get virtualmachineclasses"
vmclass_cp=guaranteed-medium
vmclass_worker=guaranteed-xlarge
#obtain virtualmachineimage via "kubectl get virtualmachineimages"
vm_image=v1.17.8+vmware.1-tkg.1.5417466
#your namespace
namespace=kubeflow-ns
#your cluster name
clustername=kubeflow-tkg
#ip of workload management cluster ip
supervisor_ip=10.4.10.1
#utilized username for logging into the cluster
username_and_domain=administrator@vsphere.local

#export some tkg relevant variables
export CONTROL_PLANE_STORAGE_CLASS=$storageclass
export WORKER_STORAGE_CLASS=$storageclass
export DEFAULT_STORAGE_CLASS=$storageclass
export STORAGE_CLASSES=
export SERVICE_DOMAIN=cluster.local #this is necessary, do not change!
export CONTROL_PLANE_VM_CLASS=$vmclass_cp
export WORKER_VM_CLASS=$vmclass_worker
export SERVICE_CIDR=100.64.0.0/13 #use whatever you want to
export CLUSTER_CIDR=100.96.0.0/11 #use whatever you want to

#clean up old tkg settings
rm -f ~/.tkg/config.yaml

#setup tkg again (done in earlier posts)
kubectl vsphere login --server=$supervisor_ip \
--vsphere-username $username_and_domain

#proceed, when logged in
kubectl config use-context $supervisor_ip
tkg add mc $supervisor_ip
tkg set mc $supervisor_ip

#create TKG Cluster
tkg create cluster $clustername --plan=dev --namespace=$namespace \
--kubernetes-version=$vm_image --controlplane-machine-count=1 \
--worker-machine-count=7

supervisor_ip=10.4.10.1
namespace=kubeflow-ns
clustername=kubeflow-tkg
username_and_domain=administrator@vsphere.local

#log into tkg-cluster
kubectl-vsphere login \
--vsphere-username=$username_and_domain  \
--server=$supervisor_ip \
--tanzu-kubernetes-cluster-namespace=$namespace \
--tanzu-kubernetes-cluster-name=$clustername

#step 1 obtain pw of node on supervisor cluster
kubectl config use-context $supervisor_ip
pw=$(kubectl get secret ${clustername}-ssh-password -o jsonpath='{.data.ssh-passwordkey}' -n $namespace| base64 -d)

#step 2 obtain ips from control plane nodes within the private kubernetes network
kubectl config use-context $clustername
node_ips=$(kubectl get nodes -o jsonpath=\
"{range .items[*]}{.metadata.name}{'\t'}{.metadata.annotations.projectcalico\.org/IPv4IPIPTunnelAddr}{'\n'}" \
| grep control-plane | awk '{print $2}')

#step 4 create POD with ubuntu container in TKG Cluster
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: ubuntu
spec:
  containers:
  - name: ubuntu
    image: ubuntu:focal
    command:
    -  sh
    -  '-c'
    -  "while true; do echo working; sleep 100; done;"
EOF
kubectl wait --for=condition=Ready --timeout=300s pod/ubuntu

#this last step is rather ugly. because we ssh multiple levels and use
#variable substitution in different hosts, so we sometimes break up double quotes and use single quotes.
kubectl exec ubuntu -- bash -c "\
{
#install necessary tools
apt update
apt upgrade -y
apt install -y sshpass openssh-client
pw=$pw

for node in ${node_ips[@]}; do
  "'#use inner variable substitution
    sshpass -p $pw  ssh -o "StrictHostKeyChecking=no" vmware-system-user@$node \
    '\''apiServerFile=/etc/kubernetes/manifests/kube-apiserver.yaml; \
    sudo sed -i "s,- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key,- --tls-private-key-file=/etc/kubernetes/pki/apiserver.key\n\    - --service-account-issuer=kubernetes.default.svc\n\    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key," $apiServerFile '\''
  '"
done;
}"
kubectl delete po ubuntu

function download_and_unpack_kubeflow() {
  echo "downloading kfctl and exporting path"
  # Add kfctl to PATH, to make the kfctl binary easier to use.
  # Use only alphanumeric characters or - in the directory name.
  rm -rf /tmp/kube/kfctl_tmp
  rm -rf /tmp/kubeflowtmp/
  mkdir -p /tmp/kube/kfctl_tmp
  cd /tmp/kube/kfctl_tmp
  wget https://github.com/kubeflow/kfctl/releases/download/v1.1.0/kfctl_v1.1.0-0-g9a3621e_linux.tar.gz
  tar -xvf kfctl_*.tar.gz
  rm kfctl_v1.1.0-0-g9a3621e_linux.tar.gz
  export PATH=$PATH:"/tmp/kube/kfctl_tmp"
}

function create_local_kubeflow_kustomize_build() {
  echo "creating initial install suite for kubeflow"
  # Set the following kfctl configuration file:
  export CONFIG_URI="https://raw.githubusercontent.com/kubeflow/manifests/v1.1-branch/kfdef/kfctl_istio_dex.v1.1.0.yaml"
  # Set KF_NAME to the name of your Kubeflow deployment. You also use this
  # value as directory name when creating your configuration directory.
  # For example, your deployment name can be 'my-kubeflow' or 'kf-test'.
  export KF_NAME=kf-devenv

  # Set the path to the base directory where you want to store one or more
  # Kubeflow deployments. For example, /opt.
  # Then set the Kubeflow application directory for this deployment.
  export BASE_DIR="/tmp/kubeflowtmp"
  export KF_DIR=${BASE_DIR}/${KF_NAME}

  mkdir -p ${KF_DIR}
  cd ${KF_DIR}

  # Download the config file and change default settings.
  kfctl build -V -f ${CONFIG_URI}
  export CONFIG_FILE="${KF_DIR}/kfctl_istio_dex.v1.1.0.yaml"
}

download_and_unpack_kubeflow
create_local_kubeflow_kustomize_build

#---skipped---#
spec:
  applications:
  - kustomizeConfig:
      repoRef:
        name: manifests #where are my building blocks
        path: namespaces/base #which of the building blocks
    name: namespaces #where can I further customize the building block
#---skipped---#
  - kustomizeConfig:
      repoRef:
        name: manifests
        path: stacks/kubernetes
    name: kubeflow-apps
#---skipped---#

function create_psp_yaml_file() {
  patch_path=$1
  which_namespace=$2
  echo "creating pod security policy for $which_namespace in $patch_path "

if [ "$which_namespace" = "kubeflow" ]; then
    cat << EOF >"${patch_path}/${which_namespace}-rb.yaml"
apiVersion: v1
kind: Namespace
metadata:
  name: $namespace
  labels:
    control-plane: kubeflow
    istio-injection: enabled
---
EOF
else
  cat << EOF >"${patch_path}/${which_namespace}-rb.yaml"
apiVersion: v1
kind: Namespace
metadata:
  name: $namespace
---
EOF

fi
  #create rb yaml file for each namespace
  cat << EOF >>"${patch_path}/${which_namespace}-rb.yaml"
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rb-all-sa_ns-$which_namespace
  namespace: $which_namespace
roleRef:
  kind: ClusterRole
  name: psp:vmware-system-privileged
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: Group
  apiGroup: rbac.authorization.k8s.io
  name: system:serviceaccounts:$which_namespace
EOF
  return $?
}

function update_kustomization_file() {
  kustomization_file_path=$1
  cat << EOF >> "${kustomization_file_path}"/kustomization.yaml
- ${namespace}-rb.yaml
EOF
}

function create_patches() {
  certmanager_patch_patch="kustomize/application"
  #identified namespaces
  namespaces=(auth knative-serving istio-system kubeflow cert-manager)

  for namespace in "${namespaces[@]}"; do
    create_psp_yaml_file ${certmanager_patch_patch} "${namespace}"
    update_kustomization_file $certmanager_patch_patch
  done
}

create_patches

#KUBERNETES_MIN_VERSION
#is based on the utilized virtualmachineimage we've declared earlier for the tkg cluster
#but it differs significantly from the provided string, obtain
#the necessary version value via kubectl get nodes -o wide
#in the corresponding tkg cluster

function create_knative_patch_file() {
  kub_version=$(kubectl get nodes | awk '{print $5}' | tail -n 1)
  echo 'creating patchfile'
  cat << EOF > kustomize/knative/knative-deploy-patch.yaml
- op: add
  path: /spec/template/spec/containers/0/env/-
  value:
    name: KUBERNETES_MIN_VERSION
    value: $kub_version
EOF
  return $?
}

function add_patch_segment() {
  echo "adding $comp patch"
  comp=$1
  cat << EOF >>kustomize/knative/kustomization.yaml
- path: knative-deploy-patch.yaml
  target:
    group: apps
    version: v1
    kind: Deployment
    name: $comp
    namespace: knative-serving
EOF
  return $?
}

function patch_knative_deployment_errors() {
  if create_knative_patch_file ; then
    echo "created knative patch file "
  else
    echo "could not create knative patch file "
    return 1
  fi

  echo "creating patch header "
  cat << EOF >>kustomize/knative/kustomization.yaml
patchesJson6902:
EOF

  # shellcheck disable=SC2181
  if [ $? -eq 0 ] ; then

    echo "creating kustomize file with patches "
    #kubeflow has 6 components, which we need to patch
    list=(activator autoscaler autoscaler-hpa controller networking-istio webhook)
    for comp in "${list[@]}"; do
      if ! add_patch_segment "$comp"; then
        echo "Could not create patch component for $comp "
        return 1
      fi
    done
  else
    echo "Could not write to file kustomize/knative/kustomization.yaml should clean up created knative-patch-file "
    return 1
  fi
  return 0
}

patch_knative_deployment_errors

function apply_kubeflow_kustomize() {
  #we exported it already, but to make sure it's working..
  export KF_DIR=${BASE_DIR}/${KF_NAME}
  #config file is from earlier..
  export CONFIG_FILE="${KF_DIR}/kfctl_istio_dex.v1.1.0.yaml"
  echo "installing kubeflow"
  kfctl apply -V -f "${CONFIG_FILE}"
}

function wait_for_kubeflowinstall() {
  counter=0
  while true; do

    num_of_pods_not_running=$(kubectl get po -A -o wide | grep -v "Running" | grep -v -c "Evicted")
    if [ "$num_of_pods_not_running" -lt 3 ]; then
      echo "everything seems to be up and running, next steps can be started! "
      break
    fi

    if [ $counter -gt 100 ]; then
      echo "we waited $((10 * 100 / 60)) minutes, still not every pod is up and running.. please have a look "
        return 1;
      break
    fi

    echo "$((num_of_pods_not_running - 2)) pods are still not ready! "
    counter=$((counter + 1))
    sleep 10
  done
}

apply_kubeflow_kustomize
wait_for_kubeflowinstall

function patch_http_redirect_and_enable_https(){
  cat << EOF | kubectl apply -f -
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata: 
  name: kubeflow-gateway
  namespace: kubeflow
spec:
  selector:
    istio: ingressgateway
  servers:
  - hosts:
    - '*'
    port:
      name: http
      number: 80
      protocol: HTTP
    # Upgrade HTTP to HTTPS
    tls:
      httpsRedirect: true
  - hosts:
    - '*'
    port:
      name: https
      number: 443
      protocol: HTTPS
    tls:
      mode: SIMPLE
      privateKey: /etc/istio/ingressgateway-certs/tls.key
      serverCertificate: /etc/istio/ingressgateway-certs/tls.crt
EOF
  return $?
}

function expose_with_loadbalancer() {
  kubectl patch service -n istio-system istio-ingressgateway -p '{"spec": {"type": "LoadBalancer"}}'
}

function obtain_kubeflow_lb_ip() {
  kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'
}

function create_kubeflow_certificates() {
  if [ $# -ne 1 ]; then
    echo "ip should be provided.. canceling creating of certificate"
    return 1
  fi

  kubeflowIp=$1
  cat << EOF | kubectl apply -f -
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
  name: istio-ingressgateway-certs
  namespace: istio-system
spec:
  commonName: istio-ingressgateway.istio-system.svc
  ipAddresses:
  - $kubeflowIp
  isCA: true
  issuerRef:
    kind: ClusterIssuer
    name: kubeflow-self-signing-issuer
  secretName: istio-ingressgateway-certs
EOF

  return 0
}

function wait_for_kubeflow_to_be_ready() {
  lb_ip=$1
  for i in $(seq 50); do
    if curl -k "https://${lb_ip}/" -v; then
      echo "service is up and running "
      return 0
    else
      echo "service not yet ready waiting some seconds. "
      echo "This was try: $i of 50"
      sleep 10
    fi
  done
  echo "kubeflow is not yet ready.. please retry"
  return 1
}

function wait_for_kubeflow_lb_ip() {
  maxTries=50
  for i in $(seq $maxTries); do
    echo "waiting for ip of loadbalancer; try #${i} from $maxTries "
    ipOfKubeflowService=$(obtain_kubeflow_lb_ip)
    if echo "$ipOfKubeflowService" | grep '^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$'; then
      echo "found ip of loadbalancer! "
      echo "using $ipOfKubeflowService "
      break
    else
      if [ "$i" -ge $maxTries ]; then
        echo "Max tries exceeded.. Cancelling operation, check whether or not the IP is given to the service"
        return 1
      fi

      echo "no ip adress yet.. retrying in some seconds "
      sleep 5
    fi
  done
}

function postinstall() {
  patch_http_redirect_and_enable_https
  expose_with_loadbalancer
  if wait_for_kubeflow_lb_ip; then
    ipOfKubeflowService=$(obtain_kubeflow_lb_ip)
    create_kubeflow_certificates "$ipOfKubeflowService"
    wait_for_kubeflow_to_be_ready "$ipOfKubeflowService"
  fi
}

postinstall

#patch argo to use pns
kubectl get -n kubeflow configmaps workflow-controller-configmap -o yaml | sed "s/containerRuntimeExecutor: docker/containerRuntimeExecutor: pns/" | kubectl apply -f -

#and restart the deployment
kubectl rollout restart deployment/ml-pipeline -n kubeflow

kubectl get svc -n istio-system istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}'

#fix PSP errors for namespace admin
namespace=admin

  cat <<EOF | kubectl apply -f -
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: rb-all-sa_ns-$namespace
  namespace: $namespace
roleRef:
  kind: ClusterRole
  name: psp:vmware-system-privileged
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: Group
  apiGroup: rbac.authorization.k8s.io
  name: system:serviceaccounts:$namespace
EOF