Docker Troubleshooting

# Step 1 — find the exited container:
docker ps -a
CONTAINER ID   NAME          STATUS                     IMAGE
a1b2c3d4e5f6   payment-api   Exited (1) 3 seconds ago   payment-api:v1.2.0
# STATUS: Exited (1) — exit code 1 means application error.
# Exited (137) = OOM kill.   Exited (143) = graceful SIGTERM.   Exited (0) = clean exit.

# Step 2 — read the logs from the exited container:
docker logs a1b2c3d4e5f6
Error: Cannot find module '/app/dist/server.js'
    at Function.Module._resolveFilename (node:internal/modules/cjs/loader:1039:15)
# The compiled output doesn't exist — the build step was skipped or failed.

# Step 3 — check the exit code for more context:
docker inspect a1b2c3d4e5f6 \
  --format 'ExitCode={{.State.ExitCode}} Error={{.State.Error}}'
ExitCode=1 Error=

# Step 4 — if logs are empty, run interactively to see what happens:
docker run -it --entrypoint sh payment-api:v1.2.0
# Drops into a shell inside the image — run the CMD manually to see the error.
/app # node dist/server.js
Error: Cannot find module '/app/dist/server.js'
# Confirmed — dist/ directory is missing from the image.

# Step 1 — confirm the port mapping is correct:
docker ps --format "table {{.Names}}\t{{.Ports}}"
NAME          PORTS
payment-api   0.0.0.0:3000->3000/tcp
# 0.0.0.0:3000 → listening on all interfaces on the host.
# If this shows nothing, the -p flag was missing from docker run.

docker port payment-api
3000/tcp -> 0.0.0.0:3000
# Confirms mapping: container port 3000 → host port 3000.

# Step 2 — confirm the application is reachable from INSIDE the container:
docker exec payment-api wget -qO- http://localhost:3000/health
{"status":"healthy"}
# If this works but external access fails, the problem is the port mapping or firewall.
# If this ALSO fails, the application itself is not listening — check its bind address.

# Step 3 — check what address the application is binding to:
docker exec payment-api ss -tlnp
State   Recv-Q  Send-Q  Local Address:Port
LISTEN  0       128     127.0.0.1:3000      ← WRONG — only accepts local connections
# Application is bound to 127.0.0.1 — only accepts connections from inside the container.
# Port mapping forwards traffic from the host to the container's network interface,
# not to 127.0.0.1. Traffic arrives but the application ignores it.

# Fix — bind to 0.0.0.0 in the application:
# Node.js:  app.listen(3000, '0.0.0.0')
# Python:   uvicorn main:app --host 0.0.0.0 --port 8000
# Go:       http.ListenAndServe(":8000", handler)  ← colon prefix = 0.0.0.0

# After fix — correct bind address:
docker exec payment-api ss -tlnp
State   Recv-Q  Send-Q  Local Address:Port
LISTEN  0       128     0.0.0.0:3000        ← CORRECT — accepts all connections

# Detect a crash loop:
docker ps
CONTAINER ID   NAME          STATUS
a1b2c3d4e5f6   payment-api   Restarting (1) 3 seconds ago

docker inspect payment-api --format '{{.RestartCount}}'
7
# Restarted 7 times — this is a crash loop.

# Read logs from the PREVIOUS run (the one that just crashed):
docker logs --previous payment-api
# shorthand: docker logs -p payment-api

Error: connect ECONNREFUSED 127.0.0.1:5432
    at TCPConnectWrap.afterConnect
# Application cannot connect to Postgres on 127.0.0.1:5432.
# Inside a container, 127.0.0.1 is the container itself — not the database container.
# The DB_HOST environment variable is wrong.

# Check the environment variable that was set:
docker inspect payment-api \
  --format '{{range .Config.Env}}{{println .}}{{end}}' | grep DB
DB_HOST=127.0.0.1
# Wrong — should be the Compose service name: DB_HOST=db

# Fix — correct the environment variable:
docker run -d \
  --name payment-api \
  -e DB_HOST=db \
  --network myapp_backend \
  payment-api:v1.2.0

docker ps
CONTAINER ID   NAME          STATUS
b2c3d4e5f6a7   payment-api   Up 12 seconds   ← stable

# Step 1 — list all networks:
docker network ls
NETWORK ID     NAME              DRIVER
a1b2c3d4e5f6   bridge            bridge   ← default — no DNS between containers
b2c3d4e5f6a7   myapp_backend     bridge   ← Compose-created — has DNS
c3d4e5f6a7b8   myapp_frontend    bridge

# Step 2 — check which containers are on which network:
docker network inspect myapp_backend \
  --format '{{range .Containers}}{{.Name}}: {{.IPv4Address}}{{"\n"}}{{end}}'
payment-api: 172.18.0.3/16
postgres-db: 172.18.0.2/16
# Both containers are on the same network — DNS should work.

# Step 3 — test DNS resolution from inside the container:
docker exec payment-api nslookup postgres-db
nslookup: can't resolve 'postgres-db'
# DNS fails even though both containers are on the same network.

# Step 4 — inspect the container's network config:
docker inspect payment-api \
  --format '{{json .NetworkSettings.Networks}}' | python3 -m json.tool
{
  "bridge": {           ← connected to the DEFAULT bridge, not myapp_backend
    "IPAddress": "172.17.0.2"
  }
}
# The container was started with docker run without --network myapp_backend.
# It joined the default bridge which has no DNS.

# Fix — connect the container to the correct network:
docker network connect myapp_backend payment-api

# Verify DNS now works:
docker exec payment-api nslookup postgres-db
Name:    postgres-db
Address: 172.18.0.2   ✓

# Diagnose the permission problem:

# Step 1 — check what user the container process runs as:
docker exec payment-api id
uid=1001(appuser) gid=1001(appgroup) groups=1001(appgroup)

# Step 2 — check the ownership of the mounted path inside the container:
docker exec payment-api ls -la /data
drwxr-xr-x  2 root root 4096 Jan 15 02:00 .
# Owned by root:root — appuser (uid 1001) cannot write here.

# Step 3 — check the host directory ownership:
ls -la ./data/
drwxr-xr-x 2 root root 4096 Jan 15 02:00 ./
# Created as root on the host — mounted into the container as root-owned.

# Fix Option A — chown the host directory to match the container user's UID:
sudo chown -R 1001:1001 ./data/
# The container's appuser (uid 1001) can now write to the mounted path.

# Fix Option B — chown inside the Dockerfile BEFORE switching USER:
RUN mkdir -p /data && chown -R appuser:appgroup /data
# Creates the directory with correct ownership before the container starts.
# Named volumes populated on first run will be owned by the creating user.

# Fix Option C — for named volumes, initialise ownership with a one-off container:
docker run --rm \
  -v myapp_data:/data \
  alpine chown -R 1001:1001 /data
# Runs alpine, mounts the named volume, fixes ownership, container exits.
# All subsequent containers mounting this volume see correct ownership.

# See how much disk Docker is using — broken down by type:
docker system df
TYPE            TOTAL   ACTIVE   SIZE      RECLAIMABLE
Images          23      4        14.7GB    11.2GB (76%)
Containers      12      3        2.1MB     1.8MB (86%)
Local Volumes   8       3        4.2GB     2.1GB (50%)
Build Cache     -       -        3.8GB     3.8GB

# Remove stopped containers, unused networks, dangling images, build cache:
docker system prune
WARNING! This will remove:
  - all stopped containers
  - all networks not used by at least one container
  - all dangling images
  - all dangling build cache
Are you sure you want to continue? [y/N] y
Total reclaimed space: 11.4GB

# More aggressive — also removes unused images (not just dangling):
docker system prune -a
# Use with care on a production server — this removes ALL images not currently
# used by a running container. The next deployment will need to pull them again.

# Targeted cleanup — more surgical:
docker container prune        # stopped containers only
docker image prune            # dangling images only (untagged)
docker image prune -a         # all unused images
docker volume prune           # volumes not attached to any container
docker builder prune          # build cache only

# DANGER — remove everything including running containers:
docker system prune -a --volumes --force
# Only run this if you intend to start completely from scratch on this host.