Kubernetes Troubleshooting

kubectl describe pod payment-api-7d9f4-abc12 -n payments
# Look at the Events section at the bottom:

# "Insufficient cpu" or "Insufficient memory"
# --> No node has enough allocatable resources
# Fix: check node capacity, add nodes, reduce requests, or check reserved system resources
kubectl describe nodes | grep -A5 "Allocated resources"

# "didn't match Pod's node affinity/selector"
# --> nodeSelector or affinity rules exclude all available nodes
# Fix: check labels on nodes vs what the Pod requires
kubectl get nodes --show-labels | grep accelerator

# "had taint ... that the pod didn't tolerate"
# --> A node taint is blocking scheduling
# Fix: add a toleration to the Pod or remove the taint
kubectl describe node gpu-node-1 | grep Taints

# "persistentvolumeclaim not found" or "pod has unbound PersistentVolumeClaims"
# --> PVC is Pending (no PV available) or the PVC name is wrong
kubectl get pvc -n payments
kubectl describe pvc postgres-data-postgres-0 -n payments
# Check events for "no persistent volumes available" or StorageClass issues

# Get logs from the PREVIOUS (crashed) container -- not the current one
kubectl logs payment-api-7d9f4-abc12 -n payments --previous
# This shows what was logged before the last crash

# If the container crashes too fast to log anything -- check the exit code
kubectl describe pod payment-api-7d9f4-abc12 -n payments | grep -A3 "Last State"
# Last State:  Terminated
#   Reason:    OOMKilled      --> container exceeded memory limit
#   Exit Code: 137            --> 128 + 9 (SIGKILL)
#   Exit Code: 1              --> application error (check logs)
#   Exit Code: 2              --> misuse of shell builtin or missing config
#   Exit Code: 125            --> Docker/container runtime error

# Common causes and fixes:
# OOMKilled (137): increase memory limit or fix memory leak
kubectl set resources deployment/payment-api -n payments \
  --limits=memory=512Mi --requests=memory=256Mi

# Missing environment variable or secret:
# App logs: "Error: DATABASE_URL is required"
# Fix: check that the Secret and envFrom/valueFrom references are correct
kubectl get secret postgres-credentials -n payments
kubectl describe pod ... | grep -A10 "Environment"

# Failed readiness probe causing restart loop:
kubectl describe pod payment-api-7d9f4-abc12 -n payments | grep -A5 "Liveness\|Readiness"
# "Liveness probe failed" -- probe is killing healthy containers
# Fix: increase initialDelaySeconds or loosen the probe threshold

# Step 1: Test from inside the cluster using a debug Pod
kubectl run debug --image=nicolaka/netshoot --restart=Never -n payments -- sleep 3600
kubectl exec -it debug -n payments -- bash

# Step 2: Check DNS resolution
nslookup payment-api.payments.svc.cluster.local
# NXDOMAIN --> DNS is broken (CoreDNS issue or wrong service name)
# Returns IP but connection fails --> DNS ok, issue is at Service or Pod level

# Step 3: Check the Service has Endpoints
kubectl get endpoints payment-api -n payments
# NAME          ENDPOINTS         AGE
# payment-api               5m   --> No endpoints! Pods aren't matching the selector
# payment-api   192.168.2.15:8080 5m   --> Endpoints exist, issue is elsewhere

# Why no endpoints? The Service selector doesn't match any Pod labels
kubectl get service payment-api -n payments -o yaml | grep -A3 selector
# selector: app: payment-api
kubectl get pods -n payments --show-labels | grep payment-api
# Labels: app=payment-api-v2  ← label mismatch! Service selector doesn't match

# Step 4: If Endpoints exist, test direct Pod connectivity
kubectl exec -it debug -n payments -- curl -v http://192.168.2.15:8080/health
# Connection refused --> app isn't listening on that port
# 200 OK --> Pod is healthy, issue is with Service port mapping

# Step 5: Check Service port mapping
kubectl get service payment-api -n payments -o yaml
# spec.ports:
#   port: 80          ← ClusterIP listens on 80
#   targetPort: 8080  ← forwards to container port 8080
# If container is listening on 8081, this breaks the chain

# Step 6: Check Network Policy isn't blocking traffic
kubectl get networkpolicy -n payments
# If default-deny-all exists, ensure an ingress rule allows the traffic

# Identify what's wrong with the node
kubectl describe node ip-10-0-2-44.us-east-1.compute.internal

# Look at Conditions section:
# Type              Status  Message
# MemoryPressure    True    kubelet has memory pressure  --> node is low on memory
# DiskPressure      True    kubelet has disk pressure    --> node is low on disk
# PIDPressure       True    kubelet has PID pressure     --> too many processes
# Ready             False   kubelet stopped posting node status  --> kubelet crashed

# For kubelet issues -- SSH to the node and check:
sudo systemctl status kubelet
sudo journalctl -u kubelet -n 100 --no-pager
# Common: "failed to start ContainerManager" -> cgroup driver mismatch after runtime upgrade
# Common: certificate expired -> kubelet certs need rotation

# For disk pressure -- find what's consuming disk:
df -h /var/lib/kubelet    # kubelet data
df -h /var/log             # logs
crictl images | sort -k4 -h | tail -20    # Large container images
# Fix: remove unused images
crictl rmi --prune

# For memory pressure -- check what's consuming memory:
kubectl top pods -A --sort-by=memory | head -20
# Fix: evict the largest non-critical Pod or add a node

# Cordon the node to prevent new scheduling while you investigate
kubectl cordon ip-10-0-2-44.us-east-1.compute.internal
# If the node is unrecoverable:
kubectl drain ip-10-0-2-44.us-east-1.compute.internal \
  --ignore-daemonsets --delete-emptydir-data
kubectl delete node ip-10-0-2-44.us-east-1.compute.internal
# Cluster Autoscaler will provision a replacement