Secrets Management Best Practices

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: secret-reader-payment-api
  namespace: payments
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]                    # get only — NOT list
    resourceNames:
      - postgres-credentials          # Restrict to exactly these two Secrets
      - payment-gateway-key
      # Why omit 'list'?
      # list returns Secret metadata (names, annotations) even without 'get'
      # Knowing secret names is valuable recon for an attacker
      # Only grant list if code genuinely needs to iterate all secrets

---

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: payment-api-read-secrets
  namespace: payments
subjects:
  - kind: ServiceAccount
    name: payment-api-sa
    namespace: payments
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: secret-reader-payment-api

# /etc/kubernetes/encryption-config.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
      - configmaps          # Optional — encrypt ConfigMaps too
    providers:
      - aescbc:             # AES-CBC with a local key
          keys:
            - name: key1
              secret: c2VjcmV0MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0
              # 32-byte base64 key. Generate: head -c 32 /dev/urandom | base64
              # Protect this file — if it leaks, all Secrets are decryptable

      - kms:                # KMS provider — key never leaves the HSM
          name: aws-kms
          endpoint: unix:///tmp/kms-provider.sock
          cachesize: 1000
          timeout: 3s
          # API server sends ciphertext to KMS, gets plaintext back
          # The encryption key lives in AWS KMS protected by HSM

      - identity: {}        # Plaintext fallback — MUST be last
                            # Allows reading Secrets written before encryption
                            # Remove after running the re-encrypt command below

# Step 1: Add flag to kube-apiserver static Pod manifest:
# --encryption-provider-config=/etc/kubernetes/encryption-config.yaml
# (API server restarts automatically when the manifest changes)

# Step 2: Re-encrypt all pre-existing Secrets
kubectl get secrets --all-namespaces -o json | kubectl replace -f -
# Reads every Secret (triggers decryption if needed)
# Writes it back (triggers encryption with the new provider)

# Step 3: Verify — raw etcd query on the control plane node
ETCDCTL_API=3 etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  get /registry/secrets/default/my-secret | hexdump -C | head -2
# If prefix reads "k8s:enc:aescbc:v1" → encrypted ✓
# If you can read base64 values → not yet encrypted ✗

# Managed cluster shortcut (EKS):
eksctl utils enable-secrets-encryption \
  --cluster=my-cluster \
  --key-arn=arn:aws:kms:us-east-1:123456789012:key/abc-def

apiVersion: external-secrets.io/v1beta1
kind: SecretStore                      # How ESO connects to the backend
metadata:
  name: aws-secrets-manager
  namespace: payments
spec:
  provider:
    aws:
      service: SecretsManager
      region: us-east-1
      auth:
        jwt:
          serviceAccountRef:
            name: eso-sa               # ServiceAccount with IRSA annotation
                                       # No static AWS keys anywhere in the cluster

---

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret                   # What to fetch and where to put it
metadata:
  name: postgres-credentials-sync
  namespace: payments
spec:
  refreshInterval: 1h                  # Re-read from AWS Secrets Manager every hour
  secretStoreRef:
    name: aws-secrets-manager
    kind: SecretStore
  target:
    name: postgres-credentials         # Kubernetes Secret to create or update
    creationPolicy: Owner              # ESO owns the Secret — deletes it if this object is deleted
  data:
    - secretKey: POSTGRES_USER         # Key in the resulting Kubernetes Secret
      remoteRef:
        key: production/payments/postgres   # Path in AWS Secrets Manager
        property: username             # JSON field within the AWS secret value
                                       # AWS secret: {"username":"pay_user","password":"..."}
    - secretKey: POSTGRES_PASSWORD
      remoteRef:
        key: production/payments/postgres
        property: password
    - secretKey: POSTGRES_DB
      remoteRef:
        key: production/payments/postgres
        property: database

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-api
  namespace: payments
  annotations:
    # Reloader: watch these Secrets — rolling restart when either changes
    secret.reloader.stakater.com/reload: "postgres-credentials,payment-gateway-key"
    # When ESO updates either Secret after a rotation,
    # Reloader triggers a rolling restart — new Pods get fresh env var values
spec:
  replicas: 3
  selector:
    matchLabels:
      app: payment-api
  template:
    metadata:
      labels:
        app: payment-api
    spec:
      serviceAccountName: payment-api-sa
      containers:
        - name: payment-api
          image: registry.company.com/payment-api:3.0.0
          envFrom:
            - secretRef:
                name: postgres-credentials   # ESO keeps this current from AWS SM
            - secretRef:
                name: payment-gateway-key