Image Pull Secrets

# Method 1: kubectl create secret (simplest)
kubectl create secret docker-registry ecr-pull-secret \
  --namespace=payments \
  --docker-server=123456789012.dkr.ecr.us-east-1.amazonaws.com \
  --docker-username=AWS \
  --docker-password=$(aws ecr get-login-password --region us-east-1) \
  --docker-email=unused@example.com
# --docker-server: the registry hostname
# --docker-username: for ECR this is always the literal string "AWS"
# --docker-password: ECR uses short-lived tokens (12h) from get-login-password
# --docker-email: required field but ignored by ECR

# Method 2: from a pre-existing Docker config file
# (useful if you already ran 'docker login' and have ~/.docker/config.json)
kubectl create secret generic ecr-pull-secret \
  --namespace=payments \
  --from-file=.dockerconfigjson=$HOME/.docker/config.json \
  --type=kubernetes.io/dockerconfigjson

# What the secret contains:
kubectl get secret ecr-pull-secret -n payments -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-api
  namespace: payments
spec:
  replicas: 3
  selector:
    matchLabels:
      app: payment-api
  template:
    metadata:
      labels:
        app: payment-api
    spec:
      # Method A: specify imagePullSecrets directly in the Pod spec
      imagePullSecrets:
        - name: ecr-pull-secret     # Must exist in the same namespace as the Pod
                                    # Can list multiple secrets for multiple registries
        - name: ghcr-pull-secret    # e.g. also pulling from GitHub Container Registry
      containers:
        - name: payment-api
          image: 123456789012.dkr.ecr.us-east-1.amazonaws.com/payment-api:3.0.0
          # Image path must match the registry hostname in the pull secret's 'auths' block

# Method B: attach the pull secret to a ServiceAccount
# Any Pod using this SA automatically gets the imagePullSecrets — no per-Pod spec needed
apiVersion: v1
kind: ServiceAccount
metadata:
  name: payment-api-sa
  namespace: payments
imagePullSecrets:
  - name: ecr-pull-secret           # SA-level pull secret — inherited by all Pods using this SA
                                    # This is the preferred pattern for team-wide registry access

apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore              # ClusterSecretStore: cluster-scoped, usable from any namespace
metadata:
  name: aws-secrets-manager-cluster   # No namespace — this is a cluster-wide resource
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: eso-sa
            namespace: external-secrets   # The ESO controller's SA namespace

---

# ExternalSecret in the payments namespace
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: ecr-pull-secret-sync
  namespace: payments
spec:
  refreshInterval: 10h                # ECR tokens last 12h — refresh at 10h to stay ahead
  secretStoreRef:
    name: aws-secrets-manager-cluster
    kind: ClusterSecretStore          # References the cluster-scoped store
  target:
    name: ecr-pull-secret
    template:
      type: kubernetes.io/dockerconfigjson    # Set the correct Secret type
      data:
        .dockerconfigjson: |
          {"auths":{"{{ .registry }}":{"username":"AWS","password":"{{ .token }}","auth":"{{ printf "AWS:%s" .token | b64enc }}"}}}
  data:
    - secretKey: registry
      remoteRef:
        key: production/ecr/credentials
        property: registry_url
    - secretKey: token
      remoteRef:
        key: production/ecr/credentials
        property: auth_token

# Option A: eksctl — attach managed policy when creating the node group
eksctl create nodegroup \
  --cluster=production \
  --name=app-nodes \
  --node-type=m5.large \
  --nodes=3 \
  --attach-policy-arn=arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
# AmazonEC2ContainerRegistryReadOnly: pull (GetAuthorizationToken + BatchGetImage + GetDownloadUrlForLayer)
# Does NOT grant push — read-only as the name implies

# Option B: add to an existing node group's IAM role via AWS CLI
aws iam attach-role-policy \
  --role-name eksctl-production-nodegroup-NodeInstanceRole \
  --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly

# Verify: test a pull from a node
kubectl run test-pull \
  --image=123456789012.dkr.ecr.us-east-1.amazonaws.com/payment-api:3.0.0 \
  --restart=Never \
  -n payments
kubectl get pod test-pull -n payments   # Should reach Running without imagePullSecrets
kubectl delete pod test-pull -n payments

# Step 1: See the exact error message
kubectl describe pod failing-pod -n payments
# Look for Events section at the bottom — the pull error appears here
# Common messages:
# "unauthorized: authentication required"  → pull secret missing or invalid
# "404 not found" or "manifest unknown"    → image tag doesn't exist
# "connection refused" / "no route to host" → registry unreachable (Network Policy?)
# "toomanyrequests"                        → Docker Hub rate limit (unauthenticated)

# Step 2: Verify the imagePullSecrets field exists on the Pod
kubectl get pod failing-pod -n payments -o jsonpath='{.spec.imagePullSecrets}'
# If empty: the pull secret wasn't specified or the SA doesn't have it

# Step 3: Verify the Secret exists and has the right type
kubectl get secret ecr-pull-secret -n payments
# NAME              TYPE                             DATA   AGE
# ecr-pull-secret   kubernetes.io/dockerconfigjson   1      2d
# Type MUST be kubernetes.io/dockerconfigjson — Opaque won't work as a pull secret

# Step 4: Decode and inspect the credentials
kubectl get secret ecr-pull-secret -n payments \
  -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d | jq .
# Check: does 'auths' contain the registry hostname from the image path?
# Mismatch between registry in the secret and registry in the image URL → auth fails

# Step 5: Test the credentials manually (on any machine with Docker)
TOKEN=$(kubectl get secret ecr-pull-secret -n payments \
  -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d | jq -r '.auths[].password')
docker login 123456789012.dkr.ecr.us-east-1.amazonaws.com -u AWS -p "$TOKEN"
docker pull 123456789012.dkr.ecr.us-east-1.amazonaws.com/payment-api:3.0.0