Terraform with Kubernetes

terraform {
  required_providers {
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.23"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.11"
    }
  }
}

# ── Provider configuration for EKS ─────────────────────────────────────────

data "aws_eks_cluster" "main" {
  name = "my-eks-cluster"  # Name of the existing EKS cluster
}

data "aws_eks_cluster_auth" "main" {
  name = "my-eks-cluster"  # Generates a temporary token for authentication
}

provider "kubernetes" {
  host                   = data.aws_eks_cluster.main.endpoint   # Cluster API endpoint
  cluster_ca_certificate = base64decode(                         # Decode the CA cert
    data.aws_eks_cluster.main.certificate_authority[0].data
  )
  token = data.aws_eks_cluster_auth.main.token  # Temporary auth token from AWS
}

# ── Provider configuration for GKE ─────────────────────────────────────────

data "google_container_cluster" "main" {
  name     = "my-gke-cluster"
  location = "us-central1"
  project  = var.project_id
}

provider "kubernetes" {
  host                   = "https://${data.google_container_cluster.main.endpoint}"
  cluster_ca_certificate = base64decode(
    data.google_container_cluster.main.master_auth[0].cluster_ca_certificate
  )

  exec {
    api_version = "client.authentication.k8s.io/v1beta1"
    command     = "gke-gcloud-auth-plugin"  # GKE auth plugin — generates short-lived tokens
  }
}

# ── Provider configuration from local kubeconfig (local development) ─────────
provider "kubernetes" {
  config_path    = "~/.kube/config"      # Read from local kubeconfig file
  config_context = "my-cluster-context"  # Use a specific context from the kubeconfig
}

# Namespaces — one per team or application domain
resource "kubernetes_namespace" "app" {
  metadata {
    name = "app-${var.environment}"

    labels = {
      environment = var.environment
      managed_by  = "terraform"
    }

    annotations = {
      "owner" = "platform-team"
    }
  }
}

resource "kubernetes_namespace" "monitoring" {
  metadata {
    name = "monitoring"
    labels = {
      managed_by = "terraform"
    }
  }
}

# ServiceAccount — a Kubernetes identity for a workload
resource "kubernetes_service_account" "app" {
  metadata {
    name      = "app-service-account"
    namespace = kubernetes_namespace.app.metadata[0].name  # Namespace reference
    annotations = {
      # For EKS IRSA — links the Kubernetes SA to an AWS IAM role
      "eks.amazonaws.com/role-arn" = aws_iam_role.app_eks.arn
    }
  }
}

# ClusterRole — permissions that apply across all namespaces
resource "kubernetes_cluster_role" "pod_reader" {
  metadata {
    name = "pod-reader"
  }

  rule {
    api_groups = [""]             # "" = core API group
    resources  = ["pods", "pods/log"]  # Resources this role can access
    verbs      = ["get", "list", "watch"]  # Allowed operations
  }
}

# ClusterRoleBinding — binds the ClusterRole to a subject
resource "kubernetes_cluster_role_binding" "pod_reader_binding" {
  metadata {
    name = "pod-reader-binding"
  }

  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role.pod_reader.metadata[0].name
  }

  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account.app.metadata[0].name
    namespace = kubernetes_namespace.app.metadata[0].name
  }
}

# ConfigMap — inject infrastructure outputs as application configuration
resource "kubernetes_config_map" "app_config" {
  metadata {
    name      = "app-config"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  data = {
    # Values derived from infrastructure resources — this is the right use case
    DATABASE_HOST = aws_db_instance.main.address   # From RDS resource
    DATABASE_PORT = "5432"
    REDIS_HOST    = aws_elasticache_cluster.main.cache_nodes[0].address
    AWS_REGION    = var.region
  }
}

# Kubernetes Secret — for sensitive values from infrastructure
# WARNING: Kubernetes Secrets stored in Terraform state are base64-encoded plaintext
# Use the external-secrets operator or a CSI driver for production-grade secret management
resource "kubernetes_secret" "db_credentials" {
  metadata {
    name      = "db-credentials"
    namespace = kubernetes_namespace.app.metadata[0].name
  }

  # type defaults to Opaque — generic key-value secret
  data = {
    # Values from AWS Secrets Manager — fetched at Terraform apply time
    # base64 encoding happens automatically — provide the plaintext value
    username = local.db_credentials.username
    password = local.db_credentials.password
    # These values will be stored in Terraform state — ensure state is encrypted
  }
}

# Better pattern for secrets: External Secrets Operator
# The operator syncs secrets from AWS Secrets Manager into Kubernetes automatically
# Terraform provisions the ExternalSecret resource — not the actual secret value
resource "kubernetes_manifest" "external_secret" {
  manifest = {
    apiVersion = "external-secrets.io/v1beta1"
    kind       = "ExternalSecret"
    metadata = {
      name      = "app-db-credentials"
      namespace = kubernetes_namespace.app.metadata[0].name
    }
    spec = {
      refreshInterval = "1h"  # Sync from AWS Secrets Manager every hour
      secretStoreRef = {
        name = "aws-secrets-manager"
        kind = "ClusterSecretStore"
      }
      target = { name = "db-credentials" }  # Kubernetes secret name to create
      data = [{
        secretKey = "password"
        remoteRef = {
          key      = "prod/rds/master-credentials"  # AWS Secrets Manager secret name
          property = "password"
        }
      }]
    }
  }
}

# Helm provider — uses same cluster authentication as Kubernetes provider
provider "helm" {
  kubernetes {
    host                   = data.aws_eks_cluster.main.endpoint
    cluster_ca_certificate = base64decode(
      data.aws_eks_cluster.main.certificate_authority[0].data
    )
    token = data.aws_eks_cluster_auth.main.token
  }
}

# Install cert-manager — manages TLS certificates automatically
resource "helm_release" "cert_manager" {
  name             = "cert-manager"
  repository       = "https://charts.jetstack.io"
  chart            = "cert-manager"
  version          = "v1.13.2"            # Always pin the chart version
  namespace        = "cert-manager"
  create_namespace = true                 # Create namespace if it does not exist

  set {
    name  = "installCRDs"
    value = "true"  # Install CustomResourceDefinitions alongside the chart
  }
}

# Install ingress-nginx — the ingress controller
resource "helm_release" "ingress_nginx" {
  name             = "ingress-nginx"
  repository       = "https://kubernetes.github.io/ingress-nginx"
  chart            = "ingress-nginx"
  version          = "4.8.3"
  namespace        = "ingress-nginx"
  create_namespace = true

  # Pass values as a YAML string for complex configurations
  values = [
    yamlencode({
      controller = {
        replicaCount = var.environment == "prod" ? 3 : 1
        service = {
          annotations = {
            "service.beta.kubernetes.io/aws-load-balancer-type" = "nlb"  # EKS NLB
          }
        }
      }
    })
  ]

  # Wait for all pods to be ready before marking the release as complete
  wait    = true
  timeout = 300  # 5 minutes — give the load balancer time to provision
}

# Install Prometheus stack for monitoring
resource "helm_release" "prometheus" {
  name             = "prometheus"
  repository       = "https://prometheus-community.github.io/helm-charts"
  chart            = "kube-prometheus-stack"
  version          = "55.5.0"
  namespace        = kubernetes_namespace.monitoring.metadata[0].name

  set {
    name  = "grafana.adminPassword"
    value = local.db_credentials.grafana_password  # From Secrets Manager
  }

  set {
    name  = "prometheus.prometheusSpec.retention"
    value = "30d"  # Keep 30 days of metrics
  }
}