Kubernetes Upgrades

# 1. Check which APIs in your cluster are deprecated or removed in the target version
# Use Pluto -- scans live cluster resources and Helm charts for deprecated APIs
pluto detect-all-in-cluster --target-versions k8s=v1.30.0
# Output: lists every resource using a deprecated API, its namespace, and the replacement

# 2. Scan all deployed Helm charts for deprecated APIs
helm list -A -o json | jq -r '.[] | "\(.name) \(.namespace)"' | \
  while read name ns; do
    helm get manifest $name -n $ns | pluto detect --target-versions k8s=v1.30.0 -f -
  done

# 3. Check the CHANGELOG for breaking changes in the target version
# https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.30.md

# 4. Verify your cluster add-ons support the target version
# (CoreDNS, kube-proxy, CNI plugin, metrics-server, Ingress controller)
kubectl get pods -A -o jsonpath='{range .items[*]}{.spec.containers[*].image}{"\n"}{end}' \
  | sort -u | grep -v pause
# Cross-check each image version against its compatibility matrix

# 5. Test the upgrade in a non-production cluster first
# -- staging should be one minor version ahead of production at all times

# Find all resources using deprecated API versions
kubectl get podsecuritypolicies.policy -A 2>/dev/null && \
  echo "WARNING: PodSecurityPolicy still in use -- removed in 1.25"

# For other deprecated APIs -- check what your manifests use
grep -r "apiVersion:" ./k8s-manifests/ | grep -E "extensions/v1beta1|networking.k8s.io/v1beta1"
# extensions/v1beta1 Ingress was removed in 1.22 -- must use networking.k8s.io/v1

# Common API migrations by version:
# 1.22: Ingress extensions/v1beta1 -> networking.k8s.io/v1
#       HorizontalPodAutoscaler autoscaling/v2beta1 -> autoscaling/v2
# 1.25: PodSecurityPolicy policy/v1beta1 -- REMOVED entirely (replaced by PSA)
# 1.26: HorizontalPodAutoscaler autoscaling/v2beta2 -> autoscaling/v2
# 1.29: FlowSchema/PriorityLevelConfiguration flowcontrol.apiserver.k8s.io/v1beta3 -> v1

# Use kubectl convert to migrate a manifest to the new API version
kubectl convert -f old-ingress.yaml --output-version networking.k8s.io/v1
# Then apply and remove the old manifest

# Step 1: Upgrade the EKS control plane
aws eks update-cluster-version \
  --name production \
  --kubernetes-version 1.30 \
  --region us-east-1
# EKS upgrades the API server, etcd, scheduler, controller-manager
# Control plane upgrade takes 10-20 minutes
# The cluster is available throughout -- EKS does a rolling control plane upgrade

# Watch the upgrade status
aws eks describe-cluster \
  --name production \
  --query "cluster.status"
# UPDATING -> ACTIVE (upgrade complete)

# Step 2: Update cluster add-ons to versions compatible with 1.30
aws eks update-addon --cluster-name production --addon-name kube-proxy \
  --addon-version v1.30.0-eksbuild.3
aws eks update-addon --cluster-name production --addon-name coredns \
  --addon-version v1.11.1-eksbuild.4
aws eks update-addon --cluster-name production --addon-name vpc-cni \
  --addon-version v1.18.0-eksbuild.1
# Check recommended versions:
aws eks describe-addon-versions --kubernetes-version 1.30 \
  --query 'addons[].{addon:addonName,version:addonVersions[0].addonVersion}'

# Step 3: Upgrade managed node groups (one at a time)
aws eks update-nodegroup-version \
  --cluster-name production \
  --nodegroup-name app-nodes \
  --kubernetes-version 1.30
# EKS cordons old nodes, drains Pods, terminates, launches new nodes at 1.30
# With proper PodDisruptionBudgets set, this is zero-downtime

# Step 4: Verify all nodes are at the new version
kubectl get nodes -o wide
# All nodes should show VERSION: v1.30.x

# On the CONTROL PLANE node:

# Step 1: Upgrade kubeadm
apt-get update && apt-get install -y kubeadm=1.30.0-1.1
kubeadm version   # verify: v1.30.0

# Step 2: Check what kubeadm will do
kubeadm upgrade plan
# Shows: current version, target version, add-on upgrades, API migrations

# Step 3: Apply the control plane upgrade
kubeadm upgrade apply v1.30.0
# Upgrades: kube-apiserver, kube-controller-manager, kube-scheduler, kube-proxy, CoreDNS, etcd
# Takes 3-5 minutes

# Step 4: Upgrade kubelet and kubectl on the control plane node
apt-get install -y kubelet=1.30.0-1.1 kubectl=1.30.0-1.1
systemctl daemon-reload && systemctl restart kubelet

# On each WORKER NODE (one at a time):

# Step 5: Cordon and drain the node
kubectl cordon worker-node-1        # Stop new Pods from scheduling here
kubectl drain worker-node-1 \
  --ignore-daemonsets \             # DaemonSet Pods stay (they're managed by the DaemonSet)
  --delete-emptydir-data            # Allow draining Pods with emptyDir volumes

# Step 6: Upgrade kubeadm, kubelet, kubectl on the worker
ssh worker-node-1
apt-get install -y kubeadm=1.30.0-1.1
kubeadm upgrade node                # Join config update only -- no control plane changes
apt-get install -y kubelet=1.30.0-1.1 kubectl=1.30.0-1.1
systemctl daemon-reload && systemctl restart kubelet

# Step 7: Uncordon -- return node to service
kubectl uncordon worker-node-1      # Pods can schedule here again

# Repeat Steps 5-7 for each remaining worker node

# Verify all nodes are at the new version
kubectl get nodes -o wide
# All STATUS: Ready, VERSION: v1.30.x

# Verify all system Pods are running
kubectl get pods -n kube-system
# All should be Running or Completed (Jobs)

# Verify workloads survived the upgrade
kubectl get deployments -A
kubectl get statefulsets -A
# All AVAILABLE replicas match DESIRED

# Run your Helm tests to validate application behaviour
helm test payment-api -n payments --logs
helm test ingress-nginx -n ingress-nginx --logs

# Check for any newly deprecated API usage introduced by the upgrade
pluto detect-all-in-cluster --target-versions k8s=v1.31.0
# Start planning ahead for the NEXT upgrade