Kubernetes Monitoring

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --version 57.0.3 \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.retentionSize=50GB \
  --set grafana.adminPassword=changeme \
  --set alertmanager.alertmanagerSpec.replicas=2

# Access Grafana locally
kubectl port-forward svc/kube-prometheus-stack-grafana 3000:80 -n monitoring
# Open http://localhost:3000  admin / changeme

# Python: expose metrics with prometheus_client
from prometheus_client import Counter, Histogram, start_http_server

payments_total = Counter(
    'payments_total',
    'Total payment attempts',
    ['status', 'currency']   # Labels -- group metrics by status and currency
)
payment_duration = Histogram(
    'payment_duration_seconds',
    'Payment processing duration',
    buckets=[0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0]
)

# In your payment handler:
with payment_duration.time():       # Automatically record duration
    result = process_payment(amount, currency)
    payments_total.labels(
        status='success' if result.ok else 'failed',
        currency=currency
    ).inc()

# ServiceMonitor: tells Prometheus where to scrape your application
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: payment-api
  namespace: payments
  labels:
    release: kube-prometheus-stack   # Must match Prometheus's serviceMonitorSelector
spec:
  selector:
    matchLabels:
      app: payment-api               # Selects the Service exposing /metrics
  endpoints:
    - port: metrics                  # Named port on the Service
      path: /metrics
      interval: 30s                  # Scrape every 30 seconds
  namespaceSelector:
    matchNames:
      - payments

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: payment-api-alerts
  namespace: payments
  labels:
    release: kube-prometheus-stack
spec:
  groups:
    - name: payment-api
      interval: 30s                         # Evaluate these rules every 30 seconds
      rules:
        - alert: PaymentErrorRateTooHigh
          expr: |
            (
              sum(rate(payments_total{status="failed"}[5m]))
              /
              sum(rate(payments_total[5m]))
            ) * 100 > 1
          for: 2m                           # Must be true for 2 minutes before firing
          labels:
            severity: critical
            team: payments
          annotations:
            summary: "Payment error rate above 1%"
            description: "Payment error rate is {{ $value | humanize }}% — SLO breach imminent."
            runbook_url: "https://wiki.company.com/runbooks/payment-errors"

        - alert: PodCrashLooping
          expr: |
            increase(kube_pod_container_status_restarts_total{namespace="payments"}[1h]) > 3
          for: 0m                           # Fire immediately -- crash loops need fast response
          labels:
            severity: warning
          annotations:
            summary: "Pod {{ $labels.pod }} is crash-looping"
            description: "{{ $labels.pod }} has restarted {{ $value }} times in the last hour."

        - alert: HighCPUThrottling
          expr: |
            sum by (pod, container) (
              rate(container_cpu_cfs_throttled_seconds_total{namespace="payments"}[5m])
            )
            /
            sum by (pod, container) (
              rate(container_cpu_cfs_periods_total{namespace="payments"}[5m])
            )
            > 0.25
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Container {{ $labels.container }} heavily throttled"
            description: "CPU throttling at {{ $value | humanizePercentage }} -- increase CPU limit."