Drift Detection

# Running plan as a drift check — no changes intended, just look for drift
terraform plan -detailed-exitcode

# Exit codes:
# 0 = success, no changes (no drift)
# 1 = error
# 2 = success, changes detected (drift found OR actual configuration change needed)
# Use -detailed-exitcode in scripts to distinguish "clean" from "drifted"

# Example: security group drift detection
# Original configuration: sg allows port 443 from 0.0.0.0/0
# Someone added port 22 from 0.0.0.0/0 manually during an incident

# terraform plan output:
# ~ resource "aws_security_group" "web" {
#     ~ ingress = [
#         + {
#             + cidr_blocks = ["0.0.0.0/0"]
#             + from_port   = 22
#             + protocol    = "tcp"
#             + to_port     = 22
#           },
#         # (existing rule unchanged)
#       ]
# }
# Plan: 0 to add, 1 to change, 0 to destroy.
# The ~ symbol shows this change will REMOVE the manually added SSH rule
# Applying this plan fixes the drift — but the engineer needs to know it happened

# Apply -refresh-only: update state to match reality WITHOUT changing infrastructure
# Use this when you ACCEPT the manual change and want to record it in state
terraform apply -refresh-only

# This is the "accept the drift" response:
# After refresh-only, the state file reflects the manually added SSH rule
# The next terraform plan will show no drift for this change
# BUT you should also update your .tf files to match — otherwise the next
# full apply will revert the change again

# .github/workflows/drift-detection.yml
# Runs every 6 hours — detects drift and reports it without applying

name: Terraform Drift Detection

on:
  schedule:
    - cron: '0 */6 * * *'  # Every 6 hours
  workflow_dispatch:         # Allow manual trigger

permissions:
  id-token: write
  contents: read
  issues: write              # Create GitHub issues for drift findings

jobs:
  detect-drift:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Run drift detection on all critical root modules in parallel
        module:
          - { name: networking,    dir: infrastructure/foundation/networking }
          - { name: databases,     dir: infrastructure/platform/databases }
          - { name: payments,      dir: infrastructure/services/payments }
      fail-fast: false       # Check all modules even if one has drift

    steps:
      - uses: actions/checkout@v4

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.DRIFT_DETECTION_ROLE_ARN }}
          aws-region: us-east-1

      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "1.6.3"

      - name: Terraform Init
        working-directory: ${{ matrix.module.dir }}
        run: terraform init -no-color

      - name: Terraform Plan — Drift Check
        id: plan
        working-directory: ${{ matrix.module.dir }}
        # -detailed-exitcode: exit 2 means changes detected (drift)
        # We capture the output and exit code separately
        run: |
          terraform plan -detailed-exitcode -no-color 2>&1 | tee plan-output.txt
          echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT
        continue-on-error: true

      - name: Report Drift
        if: steps.plan.outputs.exit_code == '2'
        uses: actions/github-script@v7
        with:
          script: |
            const planOutput = require('fs').readFileSync('${{ matrix.module.dir }}/plan-output.txt', 'utf8');
            const title = `Drift detected: ${{ matrix.module.name }} (${new Date().toISOString().split('T')[0]})`;

            // Check if a drift issue already exists for this module
            const issues = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: ['terraform-drift', '${{ matrix.module.name }}'],
              state: 'open'
            });

            if (issues.data.length === 0) {
              // Create a new issue only if one doesn't already exist
              await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: title,
                body: `## Drift Detected in ${{ matrix.module.name }}\n\n` +
                      `**Detected at:** ${new Date().toISOString()}\n\n` +
                      `**Plan output:**\n\`\`\`\n${planOutput.slice(0, 3000)}\n\`\`\`\n\n` +
                      `**Action required:** Review the drift and choose a response strategy.`,
                labels: ['terraform-drift', '${{ matrix.module.name }}', 'needs-review']
              });
            }

      - name: Notify Slack on Drift
        if: steps.plan.outputs.exit_code == '2'
        run: |
          curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} \
            -H 'Content-type: application/json' \
            --data '{
              "text": ":warning: Drift detected in *${{ matrix.module.name }}*. Review the GitHub issue for details.",
              "channel": "#infrastructure-alerts"
            }'

      # Exit 0 = no drift, exit 2 = drift found, exit 1 = error
      # We only fail the job on error (exit 1) — drift is a notification, not a blocker
      - name: Fail on Error
        if: steps.plan.outputs.exit_code == '1'
        run: exit 1

# Strategy 1: REVERT — apply Terraform to restore desired state
# Use when: the drift was accidental, unintended, or a security violation
# How: run terraform apply (the standard plan shows the correction)

# Example: SSH port 22 opened from the internet during panic — revert it
terraform plan   # Shows that applying will remove the rogue rule
terraform apply  # Restores the security group to the desired state
# Git commit: "revert: remove emergency SSH rule from payments SG"

# ─────────────────────────────────────────────────────────────────────────────

# Strategy 2: CODIFY — update .tf files to match reality, then refresh state
# Use when: the manual change was intentional and should be permanent
# How: update the .tf file, then run terraform apply to reconcile

# Example: database instance type upgraded manually for a performance incident
# The manual change is valid — update the config to match
# In main.tf: instance_class = "db.r6g.xlarge"  (was db.r6g.large)
terraform plan   # Should show no changes if the config now matches reality
# OR if the state and reality are already aligned:
terraform apply -refresh-only  # Update state to match current real-world state
# Then commit the .tf change so the codified version is in Git

# ─────────────────────────────────────────────────────────────────────────────

# Strategy 3: ACCEPT with documentation — acknowledge the deviation
# Use when: the drift is intentional, permanent, and managed outside Terraform
# How: use ignore_changes lifecycle to tell Terraform to stop tracking the attribute

resource "aws_db_instance" "main" {
  instance_class = "db.r6g.large"  # Initial value

  lifecycle {
    ignore_changes = [
      instance_class,   # Instance class managed by Auto Scaling — ignore manual changes
      engine_version,   # Minor version upgrades managed by AWS automatically
    ]
  }
}

# After adding ignore_changes:
# terraform plan will no longer show instance_class changes as drift
# Document WHY in a comment — otherwise future engineers will be confused

# ─────────────────────────────────────────────────────────────────────────────

# Strategy 4: IMPORT — bring the drifted resource back under Terraform management
# Use when: a resource was manually created outside Terraform and should be managed
# How: terraform import then verify plan shows no changes

terraform import aws_security_group.payments_v2 sg-0abc123def456789
terraform plan  # Should show no changes if the import captured the current state
# Add the resource block to .tf files matching the imported configuration

# ignore_changes — tell Terraform to stop tracking specific attributes
# Use for attributes that are legitimately managed outside Terraform

resource "aws_instance" "app" {
  ami           = "ami-0c55b159cbfafe1f0"
  instance_type = "t3.medium"

  lifecycle {
    ignore_changes = [
      ami,            # AMI updates managed by AWS Systems Manager Patch Manager
      user_data,      # User data may be updated by bootstrap automation
      tags["LastUpdated"],  # Specific tag key updated by a config management tool
    ]
  }
}

resource "aws_rds_cluster" "main" {
  engine_version = "15.3"

  lifecycle {
    ignore_changes = [
      engine_version,       # AWS auto-applies minor patches: 15.3 → 15.4 etc.
      availability_zones,   # AWS may rebalance AZs — ignore automatic rebalancing
    ]
  }
}

# CAUTION: ignore_changes applies to creation as well as updates
# If the initial resource is created with a wrong value for an ignored attribute,
# Terraform will not correct it on subsequent applies
# Test with a fresh resource before adding ignore_changes to existing ones

# CAUTION: Do not use ignore_changes = all
# ignore_changes = all tells Terraform to ignore every attribute change
# After creation, Terraform will never detect any drift on this resource
# The resource is effectively unmanaged — defeats the purpose of Terraform entirely
# Only use specific attribute names in ignore_changes, never the wildcard all