Troubleshooting Jenkins

# Step 1: Read the full console output — scroll to the BOTTOM first
# Jenkins shows the error that caused the failure near the end
# The line just above "Finished: FAILURE" is almost always the root cause

# Step 2: Filter the console output for error keywords
# If the console is 5000 lines, grep saves time
grep -iE "error:|exception|failed|cannot|not found|permission denied" build.log

# Step 3: Check the exit code
# Non-zero exit codes from sh steps cause failures
# A sh step that fails with exit code 1 is a script error
# Exit code 126/127 = command not found
# Exit code 128+signal = process killed (OOM, timeout, SIGKILL)

# Step 4: Reproduce locally
# Run the failing shell command directly on the agent machine
# SSH to the agent and run: cd /var/jenkins-agent/workspace/job-name && ./the-failing-command.sh
# If it fails locally too: code/environment issue, not Jenkins
# If it passes locally: Jenkins environment issue (missing tool, wrong PATH, missing credential)

# Step 1: Check the Jenkins system log for disconnection events
sudo grep -i "agent-linux-01\|SlaveComputer\|disconnected\|offline" \
  /var/log/jenkins/jenkins.log | tail -30

# Step 2: Check the agent log in the Jenkins UI
# Manage Jenkins → Nodes → [agent name] → Log
# Look for: IOException, Connection refused, Authentication failed, timeout

# Step 3: Test SSH connectivity from the master to the agent directly
# This confirms whether the network path is the issue
ssh -v -i /var/lib/jenkins/.ssh/agent_rsa jenkins-agent@10.0.1.45

# Step 4: Check disk space on the agent — full disk causes silent failures
df -h /var/jenkins-agent

# Step 5: Check if the Jenkins agent process is still running on the agent machine
ps aux | grep jenkins

# Step 6: Check agent JVM memory — OOM on the agent kills the process
# Look in the agent process output or check for hs_err_pid*.log files
find /tmp -name "hs_err_pid*.log" -mtime -1 2>/dev/null

# Step 1: Confirm the OOM in jenkins.log
grep -i "OutOfMemoryError\|java.lang.OutOfMemory" /var/log/jenkins/jenkins.log

# Step 2: Check current heap usage from the Groovy script console
# Run at http://jenkins-master-01:8080/script
def rt = Runtime.getRuntime()
def mb = 1024 * 1024
println "Max heap:  ${rt.maxMemory() / mb} MB"
println "Used heap: ${(rt.totalMemory() - rt.freeMemory()) / mb} MB"
println "Free heap: ${rt.freeMemory() / mb} MB"

# Step 3: Check what's consuming heap — build retention is the most common cause
# Too many builds kept in memory
# Run in script console:
Jenkins.instance.getAllItems(hudson.model.Job.class).each { job ->
    def builds = job.builds.size()
    if (builds > 50) {
        println "${job.fullName}: ${builds} builds retained"
    }
}

# Step 4: Immediate relief — trigger garbage collection
# This is a temporary fix — address the root cause
System.gc()

# Step 5: Permanent fix — increase the JVM heap in /etc/default/jenkins
# JAVA_OPTS="-Xms4g -Xmx8g -XX:+UseG1GC ..."
# Then restart Jenkins
sudo systemctl restart jenkins

# Prevention — always add pipeline-level and stage-level timeouts
// In every Jenkinsfile options block:
options {
    timeout(time: 30, unit: 'MINUTES')  // entire pipeline
}

// For individual long-running steps:
stage('Deploy') {
    steps {
        timeout(time: 5, unit: 'MINUTES') {
            sh './deploy.sh'
        }
    }
}

// For waitUntil — always give it a deadline:
timeout(time: 2, unit: 'MINUTES') {
    waitUntil {
        def response = sh(script: 'curl -s -o /dev/null -w "%{http_code}" https://api/health',
                          returnStdout: true).trim()
        return response == '200'
    }
}

# If already stuck — abort from the CLI:
java -jar jenkins-cli.jar \
  -s http://jenkins-master-01:8080 \
  -auth admin:your-api-token \
  stop-build job-name BUILD_NUMBER

# 1. Is Jenkins running?
sudo systemctl status jenkins

# 2. What does the system log say? (last 50 SEVERE/WARNING entries)
sudo grep -E "SEVERE|WARNING" /var/log/jenkins/jenkins.log | tail -50

# 3. How is memory?  (run in the script console at /script)
def rt = Runtime.getRuntime(); def mb = 1024*1024
println "Heap: ${(rt.totalMemory()-rt.freeMemory())/mb}MB / ${rt.maxMemory()/mb}MB"

# 4. How are the agents?
java -jar jenkins-cli.jar -s http://localhost:8080 -auth admin:token \
  groovy = <<'EOF'
Jenkins.instance.nodes.each { node ->
    def c = node.toComputer()
    println "${node.name.padRight(30)} ${c?.isOnline() ? 'ONLINE' : 'OFFLINE'}"
}
EOF

# 5. Is the build queue growing?
java -jar jenkins-cli.jar -s http://localhost:8080 -auth admin:token \
  groovy = <<'EOF'
def q = Jenkins.instance.queue.items.size()
println "Queue depth: ${q}"
if (q > 5) println "WARNING: Build queue backing up"
EOF

# 6. Any recent plugin updates that coincide with the problem start time?
sudo grep -i "plugin\|update\|install" /var/log/jenkins/jenkins.log \
  | grep "$(date +%Y-%m-%d)" | tail -20