Jenkins on Kubernetes

# rbac.yaml — RBAC setup for Jenkins on Kubernetes
# Apply with: kubectl apply -f rbac.yaml

---
# ServiceAccount for the Jenkins master pod
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
  namespace: jenkins

---
# Role — permissions Jenkins needs to manage build agent pods
# Scoped to the 'jenkins' namespace only — not cluster-wide
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: jenkins-agent-manager
  namespace: jenkins
rules:
  # Pod management — create, list, watch, delete build agent pods
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["create", "delete", "get", "list", "patch", "update", "watch"]

  # Pod logs — Jenkins streams build output from agent pod logs
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get", "list", "watch"]

  # Pod exec — Jenkins may need to exec into pods for debug steps
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create", "get"]

  # Secrets — needed if build pods mount Jenkins credentials as Kubernetes secrets
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list", "watch"]

  # ConfigMaps — for passing build configuration to agent pods
  - apiGroups: [""]
    resources: ["configmaps"]
    verbs: ["create", "delete", "get", "list", "update"]

---
# RoleBinding — grant the jenkins ServiceAccount the Role above
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: jenkins-agent-manager-binding
  namespace: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins-agent-manager
subjects:
  - kind: ServiceAccount
    name: jenkins
    namespace: jenkins

---
# PodDisruptionBudget — prevent the Jenkins master from being evicted
# during cluster node drains or upgrades
# minAvailable: 1 means at least 1 Jenkins pod must be running at all times
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: jenkins-pdb
  namespace: jenkins
spec:
  minAvailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: jenkins

// JCasC kubernetes cloud config — production pod templates
// Each template gives Jenkinsfiles a specific label to request
jenkins:
  clouds:
    - kubernetes:
        name: "eks-build-cluster"
        serverUrl: ""               // empty = in-cluster config
        namespace: "jenkins"
        concurrencyLimit: 50        // max concurrent build pods

        podTemplates:

          # Template 1: Java / Maven builds
          - name: "java-builder"
            label: "java maven"
            serviceAccount: "jenkins"
            activeDeadlineSeconds: 3600   // 1 hour max — safety net
            nodeSelector: "role=build"    // dedicated build nodes only
            tolerations:
              - key: "dedicated"
                operator: "Equal"
                value: "build"
                effect: "NoSchedule"
            containers:
              - name: "jnlp"
                image: "jenkins/inbound-agent:latest-jdk21"
                resourceRequestCpu: "100m"
                resourceRequestMemory: "256Mi"
              - name: "maven"
                image: "maven:3.9-eclipse-temurin-21"
                command: "cat"
                ttyEnabled: true
                resourceRequestCpu: "1"
                resourceRequestMemory: "2Gi"
                resourceLimitCpu: "2"
                resourceLimitMemory: "4Gi"
            # Fast workspace for I/O intensive Java builds
            workspaceVolume:
              emptyDirWorkspaceVolume:
                memory: false

          # Template 2: Node.js / frontend builds
          - name: "node-builder"
            label: "node frontend"
            serviceAccount: "jenkins"
            activeDeadlineSeconds: 1800
            nodeSelector: "role=build"
            containers:
              - name: "jnlp"
                image: "jenkins/inbound-agent:latest"
                resourceRequestCpu: "100m"
                resourceRequestMemory: "256Mi"
              - name: "node"
                image: "node:20-alpine"
                command: "cat"
                ttyEnabled: true
                resourceRequestCpu: "500m"
                resourceRequestMemory: "1Gi"
                resourceLimitCpu: "1"
                resourceLimitMemory: "2Gi"

          # Template 3: Docker image builds — requires privileged DinD
          - name: "docker-builder"
            label: "docker"
            serviceAccount: "jenkins"
            activeDeadlineSeconds: 3600
            nodeSelector: "role=build"
            containers:
              - name: "jnlp"
                image: "jenkins/inbound-agent:latest"
                resourceRequestCpu: "100m"
                resourceRequestMemory: "256Mi"
              - name: "docker"
                image: "docker:24-dind"
                command: "cat"
                ttyEnabled: true
                privileged: true             // required for Docker-in-Docker
                resourceRequestCpu: "1"
                resourceRequestMemory: "1Gi"
                resourceLimitCpu: "4"
                resourceLimitMemory: "8Gi"   // large limit — image builds are memory-hungry
            # Large workspace PVC for Docker layer cache
            workspaceVolume:
              dynamicPVC:
                storageClassName: "gp3"
                requestsSize: "20Gi"
                accessModes: "ReadWriteOnce"

# Step 1: Find the failed pod name from the Jenkins console output
# It will look like: java-builder-abc123
# Then query Kubernetes directly

# Step 2: Check the pod status and events
kubectl describe pod java-builder-abc123 -n jenkins

# Key sections to read in the output:
# Events: — shows scheduling failures, image pull errors, OOMKilled
# Status: — shows which containers exited and why
# Conditions: — shows if the pod reached Ready state

# Step 3: Check container logs directly
# Useful when the build output in Jenkins is incomplete
kubectl logs java-builder-abc123 -c maven -n jenkins

# Step 4: Check node pressure if pods are Pending
kubectl describe node build-node-eu-west-1a | grep -A5 "Conditions:"

# Common failure patterns and their causes:
# ---
# OOMKilled: container exceeded its memory limit
#   Fix: increase resourceLimitMemory in the pod template
# ---
# ImagePullBackOff: cannot pull the container image
#   Fix: check image tag exists, check imagePullSecrets, check registry access
# ---
# Pending (Unschedulable): no node has enough resources
#   Fix: scale up the node group, reduce resource requests, or check taints
# ---
# activeDeadlineExceeded: pod ran longer than activeDeadlineSeconds
#   Fix: increase activeDeadlineSeconds or investigate the slow build step