0.0.0.0/0

cluster-admin

env:
  password: mypassword

169.254.169.254

resources:
  limits:
    cpu: "1"
    memory: "1Gi"

kubectl get pods -A
kubectl get events -A
kubectl rollout history deployment app

kubectl describe pod <pod>
kubectl logs <pod>

kubectl get svc
kubectl get endpoints

kubectl rollout undo deployment app

kubectl describe pod <pod>

kubectl get pvc

kubectl get nodes
kubectl describe node <node>

kubectl cordon <node>
kubectl drain <node>

kubectl get hpa

kubectl top nodes
kubectl top pods

kubectl describe sa

kubectl get pods -n kube-system

nslookup kubernetes.default

Traditional approach — Service Account Keys:
Developer creates GCP service account
        ↓
Downloads JSON key file
        ↓
Stores key in Kubernetes secret
        ↓
Mounts secret into pod
        ↓
App reads key from disk
        ↓
Problems:
├── Key never expires — stolen key = permanent access
├── Must manually rotate every 90 days
├── Key stored in etcd (even if encrypted)
├── Key might end up in git, logs, or error messages
├── No audit trail of which pod used the key
└── If cluster is compromised — all keys exposed

Workload Identity — No Keys:
Pod starts with Kubernetes Service Account
        ↓
GCP metadata server intercepts credential request
        ↓
Issues short-lived token (1 hour)
        ↓
App uses token transparently
        ↓
Benefits:
├── No key files anywhere
├── Token auto-rotates every hour
├── Cryptographic binding to specific K8s SA
├── Full audit trail per pod identity
├── Cluster compromise → no long-lived credentials
└── Revoke access instantly by removing IAM binding

┌─────────────────────────────────────────────────────────────┐
│                    WORKLOAD IDENTITY FLOW                   │
│                                                             │
│  ┌──────────┐    1. App calls GCP API                       │
│  │   App    │────────────────────────────────────────────▶ │
│  │  in Pod  │                                               │
│  └────┬─────┘                                               │
│       │ 2. GCP SDK calls metadata server                    │
│       │    (169.254.169.254)                                │
│       ▼                                                     │
│  ┌──────────────────────────┐                               │
│  │   GKE Metadata Server    │  3. Validates K8s SA token    │
│  │   (intercepted by GKE)   │─────────────────────────────▶│
│  └──────────────────────────┘                               │
│                                  4. Checks IAM binding      │
│                             ┌────────────────────────────┐  │
│                             │   GCP IAM                  │  │
│                             │   K8s SA → GCP SA binding  │  │
│                             └────────────────────────────┘  │
│                                  5. Issues short-lived toke │
│  ┌──────────┐ ◀───────────────────────────────────────────  │
│  │   App    │  6. Uses token for GCP API call               │
│  └──────────┘────────────────────────────────────────────▶  │
│                             GCP Service (Storage, SQL, etc) │
└─────────────────────────────────────────────────────────────┘

GKE Workload Identity:
  Kubernetes pods → GCP service accounts
  Works inside GKE clusters
  Uses GKE metadata server
Workload Identity Federation:
  External workloads → GCP service accounts
  Works outside GCP (GitHub Actions, AWS, Azure, on-prem)
  Uses OIDC / SAML tokens
  No GCP infrastructure needed

# Terraform — enable Workload Identity on GKE cluster
resource "google_container_cluster" "primary" {
  name     = "my-cluster"
  project  = var.project_id
  location = var.region
  # Enable Workload Identity
  workload_identity_config {
    workload_pool = "${var.project_id}.svc.id.goog"
    # Format: PROJECT_ID.svc.id.goog
  }
}
# Enable on node pools too
resource "google_container_node_pool" "nodes" {
  cluster = google_container_cluster.primary.name
  node_config {
    # Use GKE metadata server — intercepts credential requests
    workload_metadata_config {
      mode = "GKE_METADATA"    # critical — must be set
      # GKE_METADATA = use Workload Identity
      # EXPOSE_METADATA = expose node SA (less secure)
    }
  }
}

# OR via gcloud
gcloud container clusters update my-cluster \
  --region us-central1 \
  --workload-pool=myproject.svc.id.goog
gcloud container node-pools update default-pool \
  --cluster my-cluster \
  --region us-central1 \
  --workload-metadata=GKE_METADATA

# GCP Service Account — the identity pods will assume
resource "google_service_account" "app_sa" {
  account_id   = "my-app-sa"
  display_name = "My Application Service Account"
  project      = var.project_id
  description  = "Used by my-app pods via Workload Identity"
}
# Grant permissions to GCP SA
resource "google_project_iam_member" "app_sa_storage" {
  project = var.project_id
  role    = "roles/storage.objectViewer"
  member  = "serviceAccount:${google_service_account.app_sa.email}"
}
resource "google_project_iam_member" "app_sa_sql" {
  project = var.project_id
  role    = "roles/cloudsql.client"
  member  = "serviceAccount:${google_service_account.app_sa.email}"
}
resource "google_project_iam_member" "app_sa_secrets" {
  project = var.project_id
  role    = "roles/secretmanager.secretAccessor"
  member  = "serviceAccount:${google_service_account.app_sa.email}"
}

# Kubernetes Service Account
apiVersion: v1
kind: ServiceAccount
metadata:
  name: my-app-ksa             # Kubernetes SA name
  namespace: production
  annotations:
    # This annotation binds K8s SA to GCP SA
    iam.gke.io/gcp-service-account: my-app-sa@myproject.iam.gserviceaccount.com

# Or via Terraform Kubernetes provider
resource "kubernetes_service_account" "app" {
  metadata {
    name      = "my-app-ksa"
    namespace = "production"
    annotations = {
      # Links this K8s SA to the GCP SA
      "iam.gke.io/gcp-service-account" = google_service_account.app_sa.email
    }
  }
}

# Allow K8s SA to impersonate GCP SA
# This is the binding that makes Workload Identity work
resource "google_service_account_iam_member" "workload_identity_binding" {
  service_account_id = google_service_account.app_sa.name
  role               = "roles/iam.workloadIdentityUser"
  # Format: serviceAccount:PROJECT.svc.id.goog[NAMESPACE/KSA_NAME]
  member = "serviceAccount:${var.project_id}.svc.id.goog[production/my-app-ksa]"
  #                          └── workload pool ──┘  └─namespace─┘ └──KSA name──┘
}

# Or via gcloud
gcloud iam service-accounts add-iam-policy-binding \
  my-app-sa@myproject.iam.gserviceaccount.com \
  --role="roles/iam.workloadIdentityUser" \
  --member="serviceAccount:myproject.svc.id.goog[production/my-app-ksa]"

# Pod uses the Kubernetes Service Account
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
  namespace: production
spec:
  template:
    spec:
      # Reference the K8s SA
      serviceAccountName: my-app-ksa    # ← connects to Workload Identity
      containers:
      - name: app
        image: myapp:latest
        # No credential env vars needed
        # No key file mounts needed
        # GCP SDK auto-detects via metadata server
        env:
        - name: GOOGLE_CLOUD_PROJECT
          value: "myproject"

# Python — GCP SDK auto-discovers credentials
from google.cloud import storage
from google.cloud import secretmanager
# No credentials needed — auto-detected via Workload Identity
storage_client = storage.Client()
bucket = storage_client.bucket("my-bucket")
blob = bucket.blob("data/file.json")
data = blob.download_as_text()
print("Downloaded:", data)
# Same for Secret Manager
sm_client = secretmanager.SecretManagerServiceClient()
secret = sm_client.access_secret_version(
    request={
        "name": "projects/myproject/secrets/my-secret/versions/latest"
    }
)
print("Secret:", secret.payload.data.decode())

// Go — same pattern
import (
    "cloud.google.com/go/storage"
    "context"
)
func main() {
    ctx := context.Background()
    // No credentials — auto-discovered
    client, err := storage.NewClient(ctx)
    if err != nil {
        log.Fatal(err)
    }
    defer client.Close()
    // Use normally
    bucket := client.Bucket("my-bucket")
    obj := bucket.Object("data/file.json")
    // ...
}

// Node.js — same pattern
const { Storage } = require('@google-cloud/storage');
// Auto-discovers Workload Identity credentials
const storage = new Storage();
const bucket = storage.bucket('my-bucket');
const [files] = await bucket.getFiles();

External workload presents its native identity token
        ↓
GCP validates token against trusted provider (OIDC)
        ↓
GCP issues short-lived GCP token
        ↓
External workload uses GCP services
        ↓
No GCP service account keys needed

# Step 1 — Create Workload Identity Pool
gcloud iam workload-identity-pools create "github-pool" \
  --project="${PROJECT_ID}" \
  --location="global" \
  --display-name="GitHub Actions Pool"
# Step 2 — Create OIDC Provider
gcloud iam workload-identity-pools providers create-oidc "github-provider" \
  --project="${PROJECT_ID}" \
  --location="global" \
  --workload-identity-pool="github-pool" \
  --display-name="GitHub Actions Provider" \
  --attribute-mapping="google.subject=assertion.sub,attribute.repository=assertion.repository,attribute.actor=assertion.actor,attribute.ref=assertion.ref" \
  --attribute-condition="assertion.repository_owner=='myorg'" \
  --issuer-uri="https://token.actions.githubusercontent.com"

# Terraform version
resource "google_iam_workload_identity_pool" "github" {
  project                   = var.project_id
  workload_identity_pool_id = "github-pool"
  display_name              = "GitHub Actions Pool"
  description               = "Pool for GitHub Actions workflows"
}
resource "google_iam_workload_identity_pool_provider" "github" {
  project                            = var.project_id
  workload_identity_pool_id          = google_iam_workload_identity_pool.github.workload_identity_pool_id
  workload_identity_pool_provider_id = "github-provider"
  display_name                       = "GitHub Actions Provider"
  # Map GitHub token claims to Google attributes
  attribute_mapping = {
    "google.subject"       = "assertion.sub"
    "attribute.repository" = "assertion.repository"
    "attribute.ref"        = "assertion.ref"
    "attribute.actor"      = "assertion.actor"
    "attribute.workflow"   = "assertion.workflow"
  }
  # Only trust tokens from our org
  attribute_condition = "assertion.repository_owner == 'myorg'"
  oidc {
    issuer_uri = "https://token.actions.githubusercontent.com"
  }
}
# Allow specific repo/branch to use GCP SA
resource "google_service_account_iam_member" "github_wif" {
  service_account_id = google_service_account.deploy_sa.name
  role               = "roles/iam.workloadIdentityUser"
  # Only main branch of specific repo
  member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.github.name}/attribute.repository/myorg/myrepo"
}

# GitHub Actions workflow using WIF
name: Deploy
on:
  push:
    branches: [main]
jobs:
  deploy:
    runs-on: ubuntu-latest
    permissions:
      id-token: write      # required for OIDC
      contents: read
    steps:
    - uses: actions/checkout@v4
    - name: Authenticate to GCP
      uses: google-github-actions/auth@v2
      with:
        # No credentials in secrets
        workload_identity_provider: >-
          projects/123456789/locations/global/
          workloadIdentityPools/github-pool/
          providers/github-provider
        service_account: deploy-sa@myproject.iam.gserviceaccount.com
    - name: Deploy to GKE
      run: |
        gcloud container clusters get-credentials \
          my-cluster --region us-central1
        kubectl apply -f k8s/

# Allow AWS workloads to use GCP
resource "google_iam_workload_identity_pool_provider" "aws" {
  project                            = var.project_id
  workload_identity_pool_id          = google_iam_workload_identity_pool.external.workload_identity_pool_id
  workload_identity_pool_provider_id = "aws-provider"
  attribute_mapping = {
    "google.subject"     = "assertion.arn"
    "attribute.aws_role" = "assertion.arn.extract('assumed-role/{role}/')"
    "attribute.aws_ec2"  = "assertion.arn.extract('instance/{instance}')"
  }
  # Trust specific AWS account
  attribute_condition = "assertion.account == '123456789012'"
  aws {
    account_id = "123456789012"
  }
}
# Allow specific AWS role
resource "google_service_account_iam_member" "aws_wif" {
  service_account_id = google_service_account.cross_cloud_sa.name
  role               = "roles/iam.workloadIdentityUser"
  member             = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.external.name}/attribute.aws_role/my-ec2-role"
}

# Check WI is configured on cluster
gcloud container clusters describe my-cluster \
  --region us-central1 \
  --format="value(workloadIdentityConfig.workloadPool)"
# Should output: myproject.svc.id.goog
# Check node pool metadata mode
gcloud container node-pools describe default-pool \
  --cluster my-cluster \
  --region us-central1 \
  --format="value(config.workloadMetadataConfig.mode)"
# Should output: GKE_METADATA
# Check K8s SA annotation
kubectl describe serviceaccount my-app-ksa -n production
# Should show: iam.gke.io/gcp-service-account: my-app-sa@myproject.iam.gserviceaccount.com
# Check IAM binding exists
gcloud iam service-accounts get-iam-policy \
  my-app-sa@myproject.iam.gserviceaccount.com
# Test from inside a pod
kubectl run wi-test \
  --image=google/cloud-sdk:slim \
  --serviceaccount=my-app-ksa \
  --namespace=production \
  --rm -it \
  -- /bin/bash
# Inside pod — test identity
curl -H "Metadata-Flavor: Google" \
  http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email
# Should return: my-app-sa@myproject.iam.gserviceaccount.com
# Test actual GCP access
gcloud auth list
gsutil ls gs://my-bucket/

# Issue 1 — Pod gets node SA instead of app SA
# Symptom: metadata returns node-sa@myproject not app-sa@myproject
# Cause: node pool not in GKE_METADATA mode
gcloud container node-pools update default-pool \
  --cluster my-cluster \
  --region us-central1 \
  --workload-metadata=GKE_METADATA
# Issue 2 — Permission denied
# Symptom: 403 when calling GCP APIs
# Check 1: IAM binding exists
gcloud iam service-accounts get-iam-policy my-app-sa@myproject.iam.gserviceaccount.com
# Check 2: K8s SA annotation correct
kubectl get serviceaccount my-app-ksa -n production -o yaml
# Check 3: Pod using correct K8s SA
kubectl get pod my-app-pod -o jsonpath='{.spec.serviceAccountName}'
# Issue 3 — Wrong namespace or SA name in binding
# The member format must exactly match:
# serviceAccount:PROJECT.svc.id.goog[NAMESPACE/KSA_NAME]
# Common mistake: wrong namespace, wrong KSA name
# Issue 4 — WI not enabled on cluster
# Symptom: metadata returns compute SA not app SA
# Fix: enable workload identity on cluster
gcloud container clusters update my-cluster \
  --region us-central1 \
  --workload-pool=myproject.svc.id.goog
# Issue 5 — Application not using ADC
# Symptom: app uses hardcoded credentials or env vars
# Fix: remove GOOGLE_APPLICATION_CREDENTIALS env var
# Let GCP SDK use Application Default Credentials (ADC)

Approach           Attack surface    Rotation    Audit
──────────────     ──────────────    ────────    ─────
SA Key in secret   High — key        Manual      Per key
                   persists forever  90 days     (limited)
SA Key in env var  Very high — in    Manual      None
                   memory + logs
Workload Identity  Minimal — no      Automatic   Per pod
(GKE)              key exists        1 hour      (full)
Workload Identity  Minimal — no      Automatic   Per workflow
Federation         key stored        ~1 hour     (full)
(GitHub/AWS)

If asked "explain Workload Identity":
Simple version (30 seconds):
  "Instead of giving pods a service account key file
   that could be stolen, Workload Identity lets pods
   prove their identity cryptographically using their
   Kubernetes Service Account. GCP issues a short-lived
   token automatically — no keys, no rotation, no risk."
Technical version (2 minutes):
  "When a pod calls a GCP API, the GCP SDK requests
   credentials from the metadata server at 169.254.169.254.
   GKE intercepts this request, validates the pod's
   Kubernetes Service Account token against the IAM
   binding we configured, and if it matches, GCP issues
   a one-hour OAuth token. The app never sees a key file.
   The token auto-rotates. Even if someone gets into the
   pod, they get a token that expires in at most one hour."
What risk did it reduce:
  "We had 28 service account key files across our clusters.
   Any of those could be leaked via logs, error messages,
   or git commits. After implementing Workload Identity,
   we have zero key files. We also gained per-pod audit
   trails in Cloud Audit Logs — we can see exactly which
   pod made which GCP API call."

gcp-gke-hub-spoke/
├── providers.tf
├── variables.tf
├── main.tf
├── outputs.tf
├── terraform.tfvars
└── modules/
    ├── network/
    │   └── main.tf
    ├── nat/
    │   └── main.tf
    └── gke-private/
        └── main.tf

terraform {
  required_version = ">= 1.6.0"
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "~> 6.0"
    }
  }
}
provider "google" {
  project = var.hub_project_id
  region  = var.region
}

variable "hub_project_id" {}
variable "spoke_project_id" {}
variable "region" {
  default = "northamerica-northeast1"
}
variable "hub_vpc_name" {
  default = "hub-vpc"
}
variable "spoke_vpc_name" {
  default = "spoke-prod-vpc"
}
variable "gke_cluster_name" {
  default = "prod-private-gke"
}
variable "gke_subnet_cidr" {
  default = "10.10.0.0/20"
}
variable "pod_cidr" {
  default = "10.20.0.0/16"
}
variable "service_cidr" {
  default = "10.30.0.0/20"
}
variable "master_ipv4_cidr_block" {
  default = "172.16.0.0/28"
}

module "hub_network" {
  source     = "./modules/network"
  project_id = var.hub_project_id
  name       = var.hub_vpc_name
  region     = var.region
  subnets = {
    hub-transit = "10.0.0.0/24"
    hub-security = "10.0.1.0/24"
  }
}
module "spoke_network" {
  source     = "./modules/network"
  project_id = var.spoke_project_id
  name       = var.spoke_vpc_name
  region     = var.region
  subnets = {
    gke-nodes = var.gke_subnet_cidr
  }
  secondary_ranges = {
    gke-nodes = {
      pods     = var.pod_cidr
      services = var.service_cidr
    }
  }
}
module "spoke_nat" {
  source      = "./modules/nat"
  project_id  = var.spoke_project_id
  region      = var.region
  network     = module.spoke_network.network_self_link
  router_name = "spoke-prod-router"
  nat_name    = "spoke-prod-cloud-nat"
}
module "private_gke" {
  source     = "./modules/gke-private"
  project_id = var.spoke_project_id
  region     = var.region
  cluster_name = var.gke_cluster_name
  network      = module.spoke_network.network_self_link
  subnetwork   = module.spoke_network.subnet_self_links["gke-nodes"]
  pod_range_name     = "pods"
  service_range_name = "services"
  master_ipv4_cidr_block = var.master_ipv4_cidr_block
}

variable "project_id" {}
variable "name" {}
variable "region" {}
variable "subnets" {
  type = map(string)
}
variable "secondary_ranges" {
  type    = map(any)
  default = {}
}
resource "google_compute_network" "vpc" {
  project                 = var.project_id
  name                    = var.name
  auto_create_subnetworks = false
  routing_mode            = "GLOBAL"
}
resource "google_compute_subnetwork" "subnet" {
  for_each = var.subnets
  project       = var.project_id
  name          = each.key
  region        = var.region
  network       = google_compute_network.vpc.id
  ip_cidr_range = each.value
  private_ip_google_access = true
  dynamic "secondary_ip_range" {
    for_each = lookup(var.secondary_ranges, each.key, {})
    content {
      range_name    = secondary_ip_range.key
      ip_cidr_range = secondary_ip_range.value
    }
  }
  log_config {
    aggregation_interval = "INTERVAL_5_SEC"
    flow_sampling        = 0.5
    metadata             = "INCLUDE_ALL_METADATA"
  }
}
resource "google_compute_firewall" "deny_all_ingress" {
  project   = var.project_id
  name      = "${var.name}-deny-all-ingress"
  network   = google_compute_network.vpc.name
  direction = "INGRESS"
  priority  = 65534
  deny {
    protocol = "all"
  }
  source_ranges = ["0.0.0.0/0"]
}
resource "google_compute_firewall" "allow_internal" {
  project   = var.project_id
  name      = "${var.name}-allow-internal"
  network   = google_compute_network.vpc.name
  direction = "INGRESS"
  priority  = 1000
  allow {
    protocol = "tcp"
  }
  allow {
    protocol = "udp"
  }
  allow {
    protocol = "icmp"
  }
  source_ranges = values(var.subnets)
}
output "network_self_link" {
  value = google_compute_network.vpc.self_link
}
output "subnet_self_links" {
  value = {
    for k, v in google_compute_subnetwork.subnet : k => v.self_link
  }
}

variable "project_id" {}
variable "region" {}
variable "network" {}
variable "router_name" {}
variable "nat_name" {}
resource "google_compute_router" "router" {
  project = var.project_id
  name    = var.router_name
  region  = var.region
  network = var.network
}
resource "google_compute_router_nat" "nat" {
  project                            = var.project_id
  name                               = var.nat_name
  router                             = google_compute_router.router.name
  region                             = var.region
  nat_ip_allocate_option             = "AUTO_ONLY"
  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
  log_config {
    enable = true
    filter = "ERRORS_ONLY"
  }
}

variable "project_id" {}
variable "region" {}
variable "cluster_name" {}
variable "network" {}
variable "subnetwork" {}
variable "pod_range_name" {}
variable "service_range_name" {}
variable "master_ipv4_cidr_block" {}
resource "google_service_account" "gke_nodes" {
  project      = var.project_id
  account_id   = "${var.cluster_name}-nodes"
  display_name = "GKE node service account"
}
resource "google_project_iam_member" "gke_node_logging" {
  project = var.project_id
  role    = "roles/logging.logWriter"
  member  = "serviceAccount:${google_service_account.gke_nodes.email}"
}
resource "google_project_iam_member" "gke_node_monitoring" {
  project = var.project_id
  role    = "roles/monitoring.metricWriter"
  member  = "serviceAccount:${google_service_account.gke_nodes.email}"
}
resource "google_container_cluster" "cluster" {
  project  = var.project_id
  name     = var.cluster_name
  location = var.region
  network    = var.network
  subnetwork = var.subnetwork
  remove_default_node_pool = true
  initial_node_count       = 1
  release_channel {
    channel = "REGULAR"
  }
  workload_identity_config {
    workload_pool = "${var.project_id}.svc.id.goog"
  }
  private_cluster_config {
    enable_private_nodes    = true
    enable_private_endpoint = true
    master_ipv4_cidr_block  = var.master_ipv4_cidr_block
  }
  ip_allocation_policy {
    cluster_secondary_range_name  = var.pod_range_name
    services_secondary_range_name = var.service_range_name
  }
  master_authorized_networks_config {}
  network_policy {
    enabled  = true
    provider = "CALICO"
  }
  addons_config {
    network_policy_config {
      disabled = false
    }
    http_load_balancing {
      disabled = false
    }
  }
  logging_config {
    enable_components = [
      "SYSTEM_COMPONENTS",
      "WORKLOADS",
      "APISERVER",
      "CONTROLLER_MANAGER",
      "SCHEDULER"
    ]
  }
  monitoring_config {
    enable_components = [
      "SYSTEM_COMPONENTS",
      "APISERVER",
      "CONTROLLER_MANAGER",
      "SCHEDULER"
    ]
  }
  enable_shielded_nodes = true
  binary_authorization {
    evaluation_mode = "PROJECT_SINGLETON_POLICY_ENFORCE"
  }
}
resource "google_container_node_pool" "secure_pool" {
  project    = var.project_id
  name       = "secure-pool"
  location   = var.region
  cluster    = google_container_cluster.cluster.name
  node_count = 2
  node_config {
    machine_type    = "e2-standard-4"
    service_account = google_service_account.gke_nodes.email
    oauth_scopes    = ["https://www.googleapis.com/auth/cloud-platform"]
    shielded_instance_config {
      enable_secure_boot          = true
      enable_integrity_monitoring = true
    }
    workload_metadata_config {
      mode = "GKE_METADATA"
    }
    labels = {
      environment = "prod"
      security    = "restricted"
    }
    metadata = {
      disable-legacy-endpoints = "true"
    }
  }
  management {
    auto_repair  = true
    auto_upgrade = true
  }
  autoscaling {
    min_node_count = 2
    max_node_count = 6
  }
}
output "cluster_name" {
  value = google_container_cluster.cluster.name
}
output "endpoint" {
  value     = google_container_cluster.cluster.endpoint
  sensitive = true
}

hub_project_id   = "my-hub-project"
spoke_project_id = "my-spoke-prod-project"
region           = "northamerica-northeast1"
hub_vpc_name     = "hub-vpc"
spoke_vpc_name   = "spoke-prod-vpc"
gke_cluster_name = "prod-private-gke"

terraform init
terraform fmt -recursive
terraform validate
terraform plan
terraform apply

Option 1: Shared VPC
Hub/Host Project owns the VPC
Spoke/Service Projects consume subnets
Option 2: Network Connectivity Center
Hub connects spokes with routing control
Option 3: VPC Peering
Simple hub-spoke, but no transitive routing

Shared VPC + Private GKE + Cloud NAT + Cloud Armor + Artifact Registry
+ Binary Authorization + Secret Manager + Workload Identity
+ NetworkPolicy + centralized logging

User → HTTPS LB → Cloud Armor → Ingress → GKE Service → Pod

Pod → Pod (via CNI)

Pod → Node → VPC → Hub → Cloud NAT → Internet

Region A Hub + Spokes
Region B Hub + Spokes
Global Load Balancer

Interview tip — always structure answers as:
  1. WHY  — what problem does it solve
  2. WHAT — what you built
  3. HOW  — how it works
  4. RESULT — what it achieved
Never just list technologies.
Say: "I built X to solve Y, which resulted in Z"

What we built:
┌─────────────────────────────────────────────────────────────┐
│                                                              │
│   Developer opens PR                                         │
│         ↓                                                    │
│   GitHub Actions triggers automatically                      │
│         ↓                                                    │
│   ┌─────────────────────────────────────────────┐           │
│   │  Validate → Security Scan → Plan → Apply    │           │
│   └─────────────────────────────────────────────┘           │
│         ↓                                                    │
│   GKE cluster fully provisioned in 12 minutes               │
│         ↓                                                    │
│   Private cluster, Workload Identity,                        │
│   KMS encryption, auto-scaling — all by default             │
│                                                              │
└─────────────────────────────────────────────────────────────┘

WHY modules matter:
Without modules:              With modules:
─────────────────             ─────────────
1 giant main.tf               networking/  → VPC, subnets, NAT
2000 lines of code            security/    → IAM, KMS, SA
Hard to test                  gke/         → cluster, node pools
Hard to reuse
Copy-paste between envs       App team just does:
Security gets missed          module "gke" {
                                source = "./modules/gke"
                                env    = "production"
                              }
                              — all security built in

Public cluster (avoid):
Internet → anyone can reach K8s API
         → nodes have public IPs
         → brute force, CVE exploitation risk
Private cluster (what we built):
Internet
   │
   └──▶ Authorized Networks only (office VPN, bastion)
              │
              ▼
        K8s API (172.16.0.0/28 — internal only)
              │
              ▼
        Worker Nodes (no public IPs)
              │
              ▼
        Cloud NAT → outbound to internet
        (pull images, call APIs)

# What makes it private — explain these two lines
private_cluster_config {
  enable_private_nodes    = true    # nodes get no public IP
  enable_private_endpoint = false   # API accessible via authorized nets
  master_ipv4_cidr_block  = "172.16.0.0/28"
}
master_authorized_networks_config {
  cidr_blocks {
    cidr_block   = "10.0.0.0/8"      # internal traffic
    display_name = "internal"
  }
  cidr_blocks {
    cidr_block   = "203.0.113.0/24"  # office VPN
    display_name = "office-vpn"
  }
}

VPC-Native (alias IPs):
Subnet:   10.0.0.0/20      ← nodes live here
Pods:     10.4.0.0/14      ← pods get IPs from here
Services: 10.0.16.0/20     ← ClusterIP services
Why it matters:
├── Pods are first-class VPC citizens
├── Firewall rules apply directly to pods
├── No double-NAT overhead
├── Cloud SQL can whitelist pod IPs directly
└── VPC flow logs show pod-level traffic

WITHOUT Workload Identity (bad):
─────────────────────────────────
1. Create GCP service account key file (JSON)
2. Store key in K8s secret
3. Mount secret into pod
4. App reads key file from disk
5. Remember to rotate every 90 days
6. Key can be stolen from secret store
Risk: leaked key = compromised GCP account
WITH Workload Identity (what we built):
────────────────────────────────────────
1. K8s Service Account (KSA) created
2. KSA annotated with GCP SA email
3. Pod uses KSA
4. GCP metadata server issues short-lived token
5. No key files anywhere
6. Token auto-rotates every hour
Risk: nothing to steal

# How it works in Terraform
# GKE cluster has Workload Identity enabled
workload_identity_config {
  workload_pool = "${var.project_id}.svc.id.goog"
}
# Node pool uses GKE_METADATA mode
workload_metadata_config {
  mode = "GKE_METADATA"    # intercepts metadata requests
}
# GCP SA allows K8s SA to impersonate it
resource "google_service_account_iam_member" "wi_binding" {
  service_account_id = google_service_account.app.name
  role               = "roles/iam.workloadIdentityUser"
  member             = "serviceAccount:myproject.svc.id.goog[production/app-ksa]"
  # ↑ This K8s SA in this namespace gets this GCP identity
}

Without KMS:
etcd stores K8s secrets → base64 encoded only
Anyone with etcd access reads secrets in plain text
With KMS (what we built):
K8s Secret created
    ↓
K8s API Server encrypts it with KMS key
    ↓
Encrypted blob stored in etcd
    ↓
Even raw etcd access shows encrypted data
KMS key is:
├── Separate from cluster IAM
├── Rotated every 90 days automatically
├── Audited — every decrypt operation logged
└── prevent_destroy = true — can't accidentally delete

Node Pool Strategy:
┌──────────────────────────────────────────────────────┐
│  SYSTEM POOL                                          │
│  n2-standard-2, on-demand, taint: CriticalAddonsOnly  │
│  Runs: Prometheus, Ingress, Cert-manager, etc.        │
│  → Isolated from app workloads                        │
└──────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────┐
│  APPLICATION POOL                                     │
│  n2-standard-8, on-demand, min=3 max=50              │
│  Runs: Production APIs, databases, services           │
│  → Stable, always available, no eviction              │
└──────────────────────────────────────────────────────┘
┌──────────────────────────────────────────────────────┐
│  SPOT POOL                                            │
│  n2-standard-4, spot VMs, min=0 max=20               │
│  Runs: batch jobs, ML training, data processing       │
│  → 60-90% cheaper, can be preempted                   │
│  → Tainted — only tolerating pods land here           │
└──────────────────────────────────────────────────────┘
Cost impact:
Before: all on-demand → $8,000/month
After: mixed pools   → $4,800/month (-40%)

# Interviewer might ask: how does the taint work?
# Node taint — repels pods
taint {
  key    = "cloud.google.com/gke-spot"
  value  = "true"
  effect = "NO_SCHEDULE"           # pods won't land here unless...
}
# Pod toleration — allows landing on spot nodes
tolerations:
- key: "cloud.google.com/gke-spot"
  operator: Equal
  value: "true"
  effect: NoSchedule               # ...they explicitly tolerate it

Two-level autoscaling:
Traffic spike hits:
  ↓
HPA detects high CPU on pods
  ↓
HPA adds more pods
  ↓
No nodes available for new pods
  ↓
Cluster Autoscaler detects pending pods
  ↓
Cluster Autoscaler adds new node
  ↓
Pods schedule on new node
  ↓
Traffic returns to normal
  ↓
HPA removes excess pods
  ↓
Cluster Autoscaler removes empty node (after 10 min)
Result: zero manual intervention
        cost scales with actual usage

PR opened:
    ↓
┌─────────────────────────────────────────────────────┐
│  1. VALIDATE (30 seconds)                            │
│     terraform fmt -check                             │
│     terraform validate                               │
│     Fails immediately on syntax errors               │
└──────────────────────┬──────────────────────────────┘
                       ↓
┌─────────────────────────────────────────────────────┐
│  2. SECURITY SCAN (2 minutes)                        │
│     Checkov — 2000+ IaC checks                       │
│     tfsec — Terraform security scanner               │
│     Fails on CRITICAL/HIGH findings                  │
│     Blocks deploy if public storage, open ports, etc.│
└──────────────────────┬──────────────────────────────┘
                       ↓
┌─────────────────────────────────────────────────────┐
│  3. PLAN (3 minutes)                                 │
│     terraform plan for each environment              │
│     Posts plan output as PR comment                  │
│     Team reviews: what will change?                  │
└──────────────────────┬──────────────────────────────┘
                       ↓ (PR merged to main)
┌─────────────────────────────────────────────────────┐
│  4. APPLY                                            │
│     Dev → auto-apply                                 │
│     Staging → 1 approval required                    │
│     Production → 2 approvals + manual gate           │
└─────────────────────────────────────────────────────┘

Traditional (risky):
  Store GCP service account key in GitHub secret
  → Long-lived key, could be leaked
  → Must rotate manually
  → If leaked, attacker has permanent access
OIDC (what we use):
  GitHub job starts
    ↓
  GitHub OIDC provider issues signed JWT
    ↓
  JWT contains: repo, branch, workflow, actor
    ↓
  Terraform job sends JWT to GCP
    ↓
  GCP validates: trusted issuer? correct repo?
    ↓
  GCP issues 1-hour access token
    ↓
  Terraform uses token to create/update resources
    ↓
  Token expires when job ends
Zero credentials stored anywhere
Zero rotation needed
Zero attack surface if GitHub account compromised

Why remote state matters:
Local state (bad):
  Alice runs terraform apply → state on Alice's laptop
  Bob runs terraform apply → state on Bob's laptop
  → Two different views of reality
  → Duplicate resources, conflicts
Remote state in GCS (what we built):
  Alice runs terraform apply → reads/writes GCS
  Bob runs terraform apply → reads same GCS
  → Single source of truth
  → State locking prevents simultaneous applies
  → Versioned — roll back if corrupted
State bucket config:
  versioning: on          ← backup every state change
  encryption: CMEK        ← encrypted with our KMS key
  uniform access: on      ← no ACLs, just IAM
  public access: blocked  ← never public

upgrade_settings {
  strategy        = "SURGE"
  max_surge       = 1    # add 1 extra node during upgrade
  max_unavailable = 0    # never reduce below desired count
}

If interviewer asks "explain X in one sentence":
VPC:              "Private network that isolates our cluster
                   from the public internet"
Private cluster:  "Nodes have no public IPs — attackers can't
                   reach them directly even if they find them"
Workload Identity:"Pods prove their identity cryptographically
                   — no key files that can be stolen"
KMS encryption:   "Even if someone steals etcd, they can't read
                   our secrets without the encryption key"
Node pools:       "Different machine types for different jobs —
                   spot VMs for batch saves 40% on cost"
Cluster Autoscaler:"Automatically adds nodes when pods can't
                    schedule — removes them when idle"
OIDC auth:        "GitHub proves who it is to GCP without
                   storing any credentials"
Remote state:     "Single source of truth for what Terraform
                   thinks exists in the cloud"
Modules:          "Reusable, tested building blocks so every
                   team gets secure infra without knowing the details"
Release channel:  "Google auto-upgrades our cluster on a
                   schedule we control"

30-second version:
"I built a modular Terraform platform that provisions
 production-grade GKE clusters — private networking,
 Workload Identity, KMS encryption — all by default.
 Provisioning time went from 4 hours to 12 minutes,
 and config drift dropped from hundreds of incidents
 per month to near zero."
2-minute version:
Add: module structure, three node pools, CI/CD pipeline,
     OIDC auth, one specific technical challenge you solved
5-minute version:
Add: specific decisions you made and why, alternatives
     you considered, what you'd do differently, metrics

gke-terraform/
├── main.tf
├── variables.tf
├── outputs.tf
├── versions.tf
├── terraform.tfvars
├── modules/
│   ├── networking/
│   │   ├── main.tf
│   │   ├── variables.tf
│   │   └── outputs.tf
│   ├── gke/
│   │   ├── main.tf
│   │   ├── variables.tf
│   │   └── outputs.tf
│   └── security/
│       ├── main.tf
│       ├── variables.tf
│       └── outputs.tf
└── environments/
    ├── dev.tfvars
    ├── staging.tfvars
    └── production.tfvars

# versions.tf
terraform {
  required_version = ">= 1.6.0"
  required_providers {
    google = {
      source  = "hashicorp/google"
      version = "~> 5.0"
    }
    google-beta = {
      source  = "hashicorp/google-beta"
      version = "~> 5.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.0"
    }
  }
  # Remote state backend
  backend "gcs" {
    bucket = "mycompany-terraform-state"
    prefix = "gke/production"
  }
}
provider "google" {
  project = var.project_id
  region  = var.region
}
provider "google-beta" {
  project = var.project_id
  region  = var.region
}
# K8s provider — uses GKE cluster output
provider "kubernetes" {
  host  = "https://${module.gke.endpoint}"
  token = data.google_client_config.default.access_token
  cluster_ca_certificate = base64decode(
    module.gke.ca_certificate
  )
}
data "google_client_config" "default" {}

# variables.tf
variable "project_id" {
  description = "GCP Project ID"
  type        = string
}
variable "region" {
  description = "GCP region"
  type        = string
  default     = "us-central1"
}
variable "environment" {
  description = "Environment name"
  type        = string
  validation {
    condition     = contains(["dev", "staging", "production"], var.environment)
    error_message = "Must be dev, staging, or production"
  }
}
variable "cluster_name" {
  description = "GKE cluster name"
  type        = string
}
# Networking
variable "vpc_cidr" {
  description = "VPC CIDR range"
  type        = string
  default     = "10.0.0.0/16"
}
variable "subnet_cidr" {
  description = "Subnet CIDR"
  type        = string
  default     = "10.0.0.0/20"
}
variable "pods_cidr" {
  description = "Pod IP range"
  type        = string
  default     = "10.4.0.0/14"
}
variable "services_cidr" {
  description = "Services IP range"
  type        = string
  default     = "10.0.16.0/20"
}
variable "master_cidr" {
  description = "Control plane CIDR"
  type        = string
  default     = "172.16.0.0/28"
}
# Node pools
variable "node_pools" {
  description = "Node pool configurations"
  type = map(object({
    machine_type = string
    min_nodes    = number
    max_nodes    = number
    disk_size_gb = number
    disk_type    = string
    spot         = bool
    taints = list(object({
      key    = string
      value  = string
      effect = string
    }))
    labels = map(string)
  }))
}
# Security
variable "authorized_networks" {
  description = "Networks authorized to access K8s API"
  type = list(object({
    cidr_block   = string
    display_name = string
  }))
  default = []
}
variable "enable_private_endpoint" {
  description = "Disable public K8s API endpoint"
  type        = bool
  default     = false
}
# Labels
variable "labels" {
  description = "Labels applied to all resources"
  type        = map(string)
  default     = {}
}

# terraform.tfvars
project_id   = "mycompany-prod"
region       = "us-central1"
environment  = "production"
cluster_name = "prod-cluster"
vpc_cidr      = "10.0.0.0/16"
subnet_cidr   = "10.0.0.0/20"
pods_cidr     = "10.4.0.0/14"
services_cidr = "10.0.16.0/20"
master_cidr   = "172.16.0.0/28"
node_pools = {
  # System components pool
  system = {
    machine_type = "n2-standard-2"
    min_nodes    = 1
    max_nodes    = 3
    disk_size_gb = 50
    disk_type    = "pd-ssd"
    spot         = false
    taints = [{
      key    = "CriticalAddonsOnly"
      value  = "true"
      effect = "NO_SCHEDULE"
    }]
    labels = { pool = "system" }
  }
  # Application workloads
  application = {
    machine_type = "n2-standard-4"
    min_nodes    = 2
    max_nodes    = 20
    disk_size_gb = 100
    disk_type    = "pd-ssd"
    spot         = false
    taints       = []
    labels       = { pool = "application" }
  }
  # Spot pool for batch workloads
  spot = {
    machine_type = "n2-standard-4"
    min_nodes    = 0
    max_nodes    = 10
    disk_size_gb = 100
    disk_type    = "pd-ssd"
    spot         = true
    taints = [{
      key    = "cloud.google.com/gke-spot"
      value  = "true"
      effect = "NO_SCHEDULE"
    }]
    labels = { pool = "spot" }
  }
}
authorized_networks = [
  {
    cidr_block   = "10.0.0.0/8"
    display_name = "internal"
  },
  {
    cidr_block   = "203.0.113.0/24"
    display_name = "office-vpn"
  }
]
labels = {
  environment = "production"
  team        = "platform"
  managed_by  = "terraform"
  cost_center = "engineering"
}

# modules/networking/main.tf
# ── VPC ──────────────────────────────────────────────────────
resource "google_compute_network" "vpc" {
  name                    = "${var.cluster_name}-vpc"
  project                 = var.project_id
  auto_create_subnetworks = false
  routing_mode            = "GLOBAL"
  description = "VPC for GKE cluster ${var.cluster_name}"
}
# ── Subnet ───────────────────────────────────────────────────
resource "google_compute_subnetwork" "subnet" {
  name                     = "${var.cluster_name}-subnet"
  project                  = var.project_id
  region                   = var.region
  network                  = google_compute_network.vpc.id
  ip_cidr_range            = var.subnet_cidr
  private_ip_google_access = true    # reach GCP APIs privately
  # Secondary ranges for pods and services
  secondary_ip_range {
    range_name    = "pods"
    ip_cidr_range = var.pods_cidr
  }
  secondary_ip_range {
    range_name    = "services"
    ip_cidr_range = var.services_cidr
  }
  # VPC Flow Logs
  log_config {
    aggregation_interval = "INTERVAL_5_SEC"
    flow_sampling        = 0.5
    metadata             = "INCLUDE_ALL_METADATA"
  }
}
# ── Cloud Router ─────────────────────────────────────────────
resource "google_compute_router" "router" {
  name    = "${var.cluster_name}-router"
  project = var.project_id
  region  = var.region
  network = google_compute_network.vpc.id
  bgp {
    asn = 64514
  }
}
# ── Cloud NAT (outbound internet for private nodes) ──────────
resource "google_compute_router_nat" "nat" {
  name                               = "${var.cluster_name}-nat"
  project                            = var.project_id
  router                             = google_compute_router.router.name
  region                             = var.region
  nat_ip_allocate_option             = "AUTO_ONLY"
  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
  log_config {
    enable = true
    filter = "ERRORS_ONLY"
  }
}
# ── Firewall Rules ────────────────────────────────────────────
# Allow internal traffic within VPC
resource "google_compute_firewall" "allow_internal" {
  name    = "${var.cluster_name}-allow-internal"
  project = var.project_id
  network = google_compute_network.vpc.id
  allow {
    protocol = "tcp"
    ports    = ["0-65535"]
  }
  allow {
    protocol = "udp"
    ports    = ["0-65535"]
  }
  allow {
    protocol = "icmp"
  }
  source_ranges = [
    var.subnet_cidr,
    var.pods_cidr,
    var.services_cidr
  ]
  description = "Allow internal VPC traffic"
}
# Allow GCP health checks (required for load balancers)
resource "google_compute_firewall" "allow_health_checks" {
  name    = "${var.cluster_name}-allow-health-checks"
  project = var.project_id
  network = google_compute_network.vpc.id
  allow {
    protocol = "tcp"
    ports    = ["10256", "8080", "8443"]
  }
  source_ranges = [
    "35.191.0.0/16",    # GCP LB health check range
    "130.211.0.0/22"    # GCP LB health check range
  ]
  target_tags = ["gke-${var.cluster_name}"]
  description = "Allow GCP load balancer health checks"
}
# Deny all ingress by default
resource "google_compute_firewall" "deny_all_ingress" {
  name     = "${var.cluster_name}-deny-all-ingress"
  project  = var.project_id
  network  = google_compute_network.vpc.id
  priority = 65534
  deny {
    protocol = "all"
  }
  source_ranges = ["0.0.0.0/0"]
  description   = "Default deny all ingress"
}

# modules/networking/outputs.tf
output "network_id" {
  value = google_compute_network.vpc.id
}
output "network_name" {
  value = google_compute_network.vpc.name
}
output "subnet_id" {
  value = google_compute_subnetwork.subnet.id
}
output "subnet_name" {
  value = google_compute_subnetwork.subnet.name
}
output "pods_range_name" {
  value = "pods"
}
output "services_range_name" {
  value = "services"
}

# modules/security/main.tf
# ── Service Account for GKE Nodes ────────────────────────────
resource "google_service_account" "gke_nodes" {
  account_id   = "${var.cluster_name}-nodes"
  display_name = "GKE Node Service Account"
  project      = var.project_id
  description  = "Minimal SA for GKE nodes"
}
# Minimal permissions for nodes
resource "google_project_iam_member" "node_permissions" {
  for_each = toset([
    "roles/logging.logWriter",           # write logs
    "roles/monitoring.metricWriter",     # write metrics
    "roles/monitoring.viewer",           # read monitoring
    "roles/stackdriver.resourceMetadata.writer",
    "roles/artifactregistry.reader",     # pull images
  ])
  project = var.project_id
  role    = each.value
  member  = "serviceAccount:${google_service_account.gke_nodes.email}"
}
# ── KMS Key for etcd encryption ───────────────────────────────
resource "google_kms_key_ring" "gke" {
  name     = "${var.cluster_name}-keyring"
  project  = var.project_id
  location = var.region
}
resource "google_kms_crypto_key" "etcd" {
  name            = "${var.cluster_name}-etcd-key"
  key_ring        = google_kms_key_ring.gke.id
  rotation_period = "7776000s"    # 90 days
  lifecycle {
    prevent_destroy = true
  }
}
# Allow GKE to use KMS key
resource "google_kms_crypto_key_iam_member" "gke_kms" {
  crypto_key_id = google_kms_crypto_key.etcd.id
  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
  member        = "serviceAccount:service-${data.google_project.project.number}@container-engine-robot.iam.gserviceaccount.com"
}
data "google_project" "project" {
  project_id = var.project_id
}
# ── Workload Identity Service Accounts ────────────────────────
# Example: Workload Identity for app teams
resource "google_service_account" "workload_identity_sa" {
  for_each = var.workload_identity_bindings
  account_id   = each.key
  display_name = each.value.display_name
  project      = var.project_id
}
resource "google_project_iam_member" "workload_sa_permissions" {
  for_each = {
    for binding in flatten([
      for sa_name, sa_config in var.workload_identity_bindings : [
        for role in sa_config.roles : {
          key     = "${sa_name}-${role}"
          sa_name = sa_name
          role    = role
        }
      ]
    ]) : binding.key => binding
  }
  project = var.project_id
  role    = each.value.role
  member  = "serviceAccount:${google_service_account.workload_identity_sa[each.value.sa_name].email}"
}
resource "google_service_account_iam_member" "workload_identity_binding" {
  for_each = var.workload_identity_bindings
  service_account_id = google_service_account.workload_identity_sa[each.key].name
  role               = "roles/iam.workloadIdentityUser"
  member             = "serviceAccount:${var.project_id}.svc.id.goog[${each.value.namespace}/${each.value.ksa_name}]"
}

# modules/security/outputs.tf
output "node_sa_email" {
  value = google_service_account.gke_nodes.email
}
output "kms_key_id" {
  value = google_kms_crypto_key.etcd.id
}
output "workload_sa_emails" {
  value = {
    for name, sa in google_service_account.workload_identity_sa :
    name => sa.email
  }
}

# modules/gke/main.tf
locals {
  cluster_name = var.cluster_name
  labels = merge(var.labels, {
    managed_by  = "terraform"
    environment = var.environment
  })
}
# ── GKE Cluster ──────────────────────────────────────────────
resource "google_container_cluster" "primary" {
  provider = google-beta
  name     = local.cluster_name
  project  = var.project_id
  location = var.region            # regional for HA
  # Remove default node pool
  remove_default_node_pool = true
  initial_node_count       = 1
  # Networking
  network    = var.network_id
  subnetwork = var.subnet_id
  # VPC-native — required for private clusters
  ip_allocation_policy {
    cluster_secondary_range_name  = var.pods_range_name
    services_secondary_range_name = var.services_range_name
  }
  # ── Private Cluster ─────────────────────────────────────
  private_cluster_config {
    enable_private_nodes    = true
    enable_private_endpoint = var.enable_private_endpoint
    master_ipv4_cidr_block  = var.master_cidr
    master_global_access_config {
      enabled = true             # access from any region
    }
  }
  # ── Authorized Networks ──────────────────────────────────
  master_authorized_networks_config {
    dynamic "cidr_blocks" {
      for_each = var.authorized_networks
      content {
        cidr_block   = cidr_blocks.value.cidr_block
        display_name = cidr_blocks.value.display_name
      }
    }
    gcp_public_cidrs_access_enabled = false
  }
  # ── Workload Identity ────────────────────────────────────
  workload_identity_config {
    workload_pool = "${var.project_id}.svc.id.goog"
  }
  # ── Security ─────────────────────────────────────────────
  binary_authorization {
    evaluation_mode = var.environment == "production" ? "PROJECT_SINGLETON_POLICY_ENFORCE" : "DISABLED"
  }
  # etcd encryption using KMS
  database_encryption {
    state    = "ENCRYPTED"
    key_name = var.kms_key_id
  }
  # ── Addons ───────────────────────────────────────────────
  addons_config {
    # HTTP load balancing (required for Ingress)
    http_load_balancing {
      disabled = false
    }
    # Horizontal Pod Autoscaling
    horizontal_pod_autoscaling {
      disabled = false
    }
    # CSI driver for persistent disks
    gce_persistent_disk_csi_driver_config {
      enabled = true
    }
    # GCS FUSE driver
    gcs_fuse_csi_driver_config {
      enabled = true
    }
    # DNS config
    dns_cache_config {
      enabled = true
    }
  }
  # ── Network Policy ───────────────────────────────────────
  network_policy {
    enabled  = true
    provider = "CALICO"
  }
  datapath_provider = "ADVANCED_DATAPATH"   # eBPF via Cilium
  # ── Logging and Monitoring ───────────────────────────────
  logging_config {
    enable_components = [
      "SYSTEM_COMPONENTS",
      "WORKLOADS",
      "APISERVER",
      "SCHEDULER",
      "CONTROLLER_MANAGER"
    ]
  }
  monitoring_config {
    enable_components = [
      "SYSTEM_COMPONENTS",
      "WORKLOADS",
      "APISERVER",
      "SCHEDULER",
      "CONTROLLER_MANAGER",
      "STORAGE",
      "HPA",
      "POD",
      "DAEMONSET",
      "DEPLOYMENT",
      "STATEFULSET"
    ]
    managed_prometheus {
      enabled = true
    }
    advanced_datapath_observability_config {
      enable_metrics = true
      enable_relay   = true
    }
  }
  # ── Release Channel ──────────────────────────────────────
  release_channel {
    channel = var.release_channel
  }
  # ── Maintenance Window ───────────────────────────────────
  maintenance_policy {
    recurring_window {
      start_time = "2024-01-01T02:00:00Z"
      end_time   = "2024-01-01T06:00:00Z"
      recurrence = "FREQ=WEEKLY;BYDAY=SA,SU"
    }
    maintenance_exclusion {
      exclusion_name = "black-friday"
      start_time     = "2024-11-25T00:00:00Z"
      end_time       = "2024-12-02T00:00:00Z"
      exclusion_options {
        scope = "NO_UPGRADES"
      }
    }
  }
  # ── Cluster Autoscaling ──────────────────────────────────
  cluster_autoscaling {
    enabled             = true
    autoscaling_profile = "OPTIMIZE_UTILIZATION"
    resource_limits {
      resource_type = "cpu"
      minimum       = 4
      maximum       = 200
    }
    resource_limits {
      resource_type = "memory"
      minimum       = 16
      maximum       = 800
    }
    auto_provisioning_defaults {
      service_account = var.node_sa_email
      oauth_scopes    = ["https://www.googleapis.com/auth/cloud-platform"]
      management {
        auto_repair  = true
        auto_upgrade = true
      }
      shielded_instance_config {
        enable_secure_boot          = true
        enable_integrity_monitoring = true
      }
      disk_size = 100
      disk_type = "pd-ssd"
    }
  }
  # ── Node Pool Defaults ───────────────────────────────────
  node_pool_defaults {
    node_config_defaults {
      logging_variant = "MAX_THROUGHPUT"
    }
  }
  # ── Security posture ─────────────────────────────────────
  security_posture_config {
    mode               = "BASIC"
    vulnerability_mode = "VULNERABILITY_BASIC"
  }
  resource_labels = local.labels
  lifecycle {
    ignore_changes = [
      initial_node_count,
      node_pool,
    ]
    prevent_destroy = var.environment == "production" ? true : false
  }
  depends_on = [
    var.node_sa_email
  ]
}
# ── Node Pools ────────────────────────────────────────────────
resource "google_container_node_pool" "pools" {
  for_each = var.node_pools
  name     = each.key
  project  = var.project_id
  cluster  = google_container_cluster.primary.name
  location = var.region
  # Autoscaling
  autoscaling {
    min_node_count  = each.value.min_nodes
    max_node_count  = each.value.max_nodes
    location_policy = "BALANCED"    # spread across zones
  }
  # Auto-repair and upgrade
  management {
    auto_repair  = true
    auto_upgrade = true
  }
  # Surge upgrade — zero downtime
  upgrade_settings {
    strategy        = "SURGE"
    max_surge       = 1
    max_unavailable = 0
  }
  node_config {
    machine_type    = each.value.machine_type
    disk_size_gb    = each.value.disk_size_gb
    disk_type       = each.value.disk_type
    spot            = each.value.spot
    image_type      = "COS_CONTAINERD"
    service_account = var.node_sa_email
    oauth_scopes    = ["https://www.googleapis.com/auth/cloud-platform"]
    # Workload Identity
    workload_metadata_config {
      mode = "GKE_METADATA"
    }
    # Shielded nodes
    shielded_instance_config {
      enable_secure_boot          = true
      enable_integrity_monitoring = true
    }
    # Taints
    dynamic "taint" {
      for_each = each.value.taints
      content {
        key    = taint.value.key
        value  = taint.value.value
        effect = taint.value.effect
      }
    }
    # Labels
    labels = merge(local.labels, each.value.labels)
    # Network tags
    tags = [
      "gke-${local.cluster_name}",
      "gke-${local.cluster_name}-${each.key}"
    ]
    # Metadata
    metadata = {
      disable-legacy-endpoints = "true"
    }
    # Resource labels for cost tracking
    resource_labels = merge(local.labels, {
      pool = each.key
      spot = tostring(each.value.spot)
    })
  }
  lifecycle {
    ignore_changes = [
      node_config[0].resource_labels,
    ]
  }
}

# modules/gke/outputs.tf
output "cluster_name" {
  description = "GKE cluster name"
  value       = google_container_cluster.primary.name
}
output "endpoint" {
  description = "Cluster endpoint"
  value       = google_container_cluster.primary.endpoint
  sensitive   = true
}
output "ca_certificate" {
  description = "Cluster CA certificate"
  value       = google_container_cluster.primary.master_auth[0].cluster_ca_certificate
  sensitive   = true
}
output "cluster_id" {
  description = "Cluster resource ID"
  value       = google_container_cluster.primary.id
}
output "workload_identity_pool" {
  description = "Workload Identity pool"
  value       = "${var.project_id}.svc.id.goog"
}
output "node_pool_names" {
  description = "Node pool names"
  value       = [for np in google_container_node_pool.pools : np.name]
}
output "get_credentials_command" {
  description = "Command to get cluster credentials"
  value       = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --region ${var.region} --project ${var.project_id}"
}

# main.tf
# ── Networking ────────────────────────────────────────────────
module "networking" {
  source = "./modules/networking"
  project_id   = var.project_id
  region       = var.region
  cluster_name = var.cluster_name
  subnet_cidr  = var.subnet_cidr
  pods_cidr    = var.pods_cidr
  services_cidr = var.services_cidr
  labels       = var.labels
}
# ── Security ──────────────────────────────────────────────────
module "security" {
  source = "./modules/security"
  project_id   = var.project_id
  region       = var.region
  cluster_name = var.cluster_name
  environment  = var.environment
  workload_identity_bindings = {
    "app-sa" = {
      display_name = "Application Service Account"
      namespace    = "production"
      ksa_name     = "app-ksa"
      roles = [
        "roles/storage.objectViewer",
        "roles/secretmanager.secretAccessor"
      ]
    }
  }
}
# ── GKE Cluster ───────────────────────────────────────────────
module "gke" {
  source = "./modules/gke"
  project_id   = var.project_id
  region       = var.region
  cluster_name = var.cluster_name
  environment  = var.environment
  # Networking
  network_id          = module.networking.network_id
  subnet_id           = module.networking.subnet_id
  pods_range_name     = module.networking.pods_range_name
  services_range_name = module.networking.services_range_name
  master_cidr         = var.master_cidr
  # Security
  kms_key_id              = module.security.kms_key_id
  node_sa_email           = module.security.node_sa_email
  authorized_networks     = var.authorized_networks
  enable_private_endpoint = var.enable_private_endpoint
  # Node pools
  node_pools      = var.node_pools
  release_channel = "REGULAR"
  labels = var.labels
  depends_on = [
    module.networking,
    module.security
  ]
}

# outputs.tf
output "cluster_name" {
  description = "GKE cluster name"
  value       = module.gke.cluster_name
}
output "cluster_endpoint" {
  description = "Cluster API endpoint"
  value       = module.gke.endpoint
  sensitive   = true
}
output "get_credentials" {
  description = "Command to configure kubectl"
  value       = module.gke.get_credentials_command
}
output "workload_identity_pool" {
  description = "Workload Identity pool"
  value       = module.gke.workload_identity_pool
}
output "node_sa_email" {
  description = "Node service account email"
  value       = module.security.node_sa_email
}

# environments/dev.tfvars
project_id   = "mycompany-dev"
environment  = "dev"
cluster_name = "dev-cluster"
region       = "us-central1"
subnet_cidr   = "10.10.0.0/20"
pods_cidr     = "10.10.16.0/20"
services_cidr = "10.10.32.0/20"
master_cidr   = "172.16.0.32/28"
node_pools = {
  application = {
    machine_type = "n2-standard-2"    # smaller for dev
    min_nodes    = 0                  # scale to zero
    max_nodes    = 5
    disk_size_gb = 50
    disk_type    = "pd-standard"      # cheaper disk
    spot         = true               # spot VMs in dev
    taints       = []
    labels       = { pool = "application", env = "dev" }
  }
}
authorized_networks = [
  {
    cidr_block   = "0.0.0.0/0"
    display_name = "all-for-dev"
  }
]
enable_private_endpoint = false
labels = {
  environment = "dev"
  team        = "platform"
  managed_by  = "terraform"
}

# environments/production.tfvars
project_id   = "mycompany-prod"
environment  = "production"
cluster_name = "prod-cluster"
region       = "us-central1"
subnet_cidr   = "10.0.0.0/20"
pods_cidr     = "10.4.0.0/14"
services_cidr = "10.0.16.0/20"
master_cidr   = "172.16.0.0/28"
node_pools = {
  system = {
    machine_type = "n2-standard-2"
    min_nodes    = 1
    max_nodes    = 3
    disk_size_gb = 50
    disk_type    = "pd-ssd"
    spot         = false
    taints = [{
      key    = "CriticalAddonsOnly"
      value  = "true"
      effect = "NO_SCHEDULE"
    }]
    labels = { pool = "system" }
  }
  application = {
    machine_type = "n2-standard-8"
    min_nodes    = 3
    max_nodes    = 50
    disk_size_gb = 100
    disk_type    = "pd-ssd"
    spot         = false
    taints       = []
    labels       = { pool = "application" }
  }
  spot = {
    machine_type = "n2-standard-4"
    min_nodes    = 0
    max_nodes    = 20
    disk_size_gb = 100
    disk_type    = "pd-ssd"
    spot         = true
    taints = [{
      key    = "cloud.google.com/gke-spot"
      value  = "true"
      effect = "NO_SCHEDULE"
    }]
    labels = { pool = "spot" }
  }
}
authorized_networks = [
  {
    cidr_block   = "10.0.0.0/8"
    display_name = "internal"
  },
  {
    cidr_block   = "203.0.113.0/24"
    display_name = "office-vpn"
  }
]
enable_private_endpoint = false
labels = {
  environment = "production"
  team        = "platform"
  managed_by  = "terraform"
  cost_center = "engineering"
  criticality = "high"
}

# .github/workflows/terraform-gke.yml
name: Deploy GKE
on:
  push:
    branches: [main]
    paths:
    - 'gke-terraform/**'
  pull_request:
    branches: [main]
    paths:
    - 'gke-terraform/**'
  workflow_dispatch:
    inputs:
      environment:
        description: Target environment
        required: true
        type: choice
        options: [dev, staging, production]
      action:
        description: Terraform action
        required: true
        type: choice
        options: [plan, apply, destroy]
env:
  TF_VERSION: "1.6.0"
  WORKING_DIR: gke-terraform
jobs:
  # ── Validate ────────────────────────────────────────────────
  validate:
    name: Validate
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: hashicorp/setup-terraform@v3
      with:
        terraform_version: ${{ env.TF_VERSION }}
    - name: Format check
      run: terraform fmt -check -recursive
      working-directory: ${{ env.WORKING_DIR }}
    - name: Validate
      run: |
        terraform init -backend=false
        terraform validate
      working-directory: ${{ env.WORKING_DIR }}
  # ── Security scan ───────────────────────────────────────────
  security:
    name: Security Scan
    runs-on: ubuntu-latest
    needs: validate
    steps:
    - uses: actions/checkout@v4
    - name: Checkov scan
      uses: bridgecrewio/checkov-action@master
      with:
        directory: ${{ env.WORKING_DIR }}
        framework: terraform
        soft_fail: false
    - name: tfsec scan
      uses: aquasecurity/tfsec-action@v1.0.0
      with:
        working_directory: ${{ env.WORKING_DIR }}
  # ── Plan ────────────────────────────────────────────────────
  plan:
    name: Plan ${{ matrix.environment }}
    runs-on: ubuntu-latest
    needs: [validate, security]
    strategy:
      matrix:
        environment: [dev, staging, production]
    permissions:
      contents: read
      id-token: write
      pull-requests: write
    steps:
    - uses: actions/checkout@v4
    - id: auth
      uses: google-github-actions/auth@v2
      with:
        workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
        service_account: terraform-sa@${{ secrets[format('{0}_PROJECT_ID', upper(matrix.environment))] }}.iam.gserviceaccount.com
    - uses: hashicorp/setup-terraform@v3
      with:
        terraform_version: ${{ env.TF_VERSION }}
    - name: Terraform Init
      run: |
        terraform init \
          -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" \
          -backend-config="prefix=gke/${{ matrix.environment }}"
      working-directory: ${{ env.WORKING_DIR }}
    - name: Terraform Plan
      id: plan
      run: |
        terraform plan \
          -var-file="environments/${{ matrix.environment }}.tfvars" \
          -out=tfplan-${{ matrix.environment }} \
          -no-color \
          2>&1 | tee plan-output.txt
        echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT
      working-directory: ${{ env.WORKING_DIR }}
    - name: Upload plan
      uses: actions/upload-artifact@v4
      with:
        name: tfplan-${{ matrix.environment }}
        path: ${{ env.WORKING_DIR }}/tfplan-${{ matrix.environment }}
        retention-days: 1
    - name: Post plan to PR
      uses: actions/github-script@v7
      if: github.event_name == 'pull_request'
      with:
        script: |
          const fs = require('fs');
          const plan = fs.readFileSync(
            '${{ env.WORKING_DIR }}/plan-output.txt',
            'utf8'
          );
          const maxLen = 65000;
          const truncated = plan.length > maxLen
            ? plan.substring(0, maxLen) + '\n... truncated'
            : plan;
          github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: `## Terraform Plan — \`${{ matrix.environment }}\`
            \`\`\`
            ${truncated}
            \`\`\``
          });
  # ── Apply Dev ───────────────────────────────────────────────
  apply-dev:
    name: Apply Dev
    runs-on: ubuntu-latest
    needs: plan
    if: github.ref == 'refs/heads/main'
    environment: dev
    permissions:
      contents: read
      id-token: write
    steps:
    - uses: actions/checkout@v4
    - id: auth
      uses: google-github-actions/auth@v2
      with:
        workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
        service_account: ${{ secrets.DEV_TF_SA }}
    - uses: hashicorp/setup-terraform@v3
      with:
        terraform_version: ${{ env.TF_VERSION }}
    - name: Download plan
      uses: actions/download-artifact@v4
      with:
        name: tfplan-dev
        path: ${{ env.WORKING_DIR }}
    - name: Terraform Init
      run: |
        terraform init \
          -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" \
          -backend-config="prefix=gke/dev"
      working-directory: ${{ env.WORKING_DIR }}
    - name: Terraform Apply
      run: terraform apply -auto-approve tfplan-dev
      working-directory: ${{ env.WORKING_DIR }}
    - name: Configure kubectl
      run: |
        ${{ steps.apply.outputs.get_credentials }}
    - name: Verify cluster
      run: |
        kubectl get nodes
        kubectl get pods -A
  # ── Apply Production (requires approval) ────────────────────
  apply-production:
    name: Apply Production
    runs-on: ubuntu-latest
    needs: apply-dev
    environment: production          # approval gate
    permissions:
      contents: read
      id-token: write
    steps:
    - uses: actions/checkout@v4
    - id: auth
      uses: google-github-actions/auth@v2
      with:
        workload_identity_provider: ${{ secrets.WIF_PROVIDER }}
        service_account: ${{ secrets.PROD_TF_SA }}
    - uses: hashicorp/setup-terraform@v3
      with:
        terraform_version: ${{ env.TF_VERSION }}
    - name: Download plan
      uses: actions/download-artifact@v4
      with:
        name: tfplan-production
        path: ${{ env.WORKING_DIR }}
    - name: Terraform Init
      run: |
        terraform init \
          -backend-config="bucket=${{ secrets.TF_STATE_BUCKET }}" \
          -backend-config="prefix=gke/production"
      working-directory: ${{ env.WORKING_DIR }}
    - name: Terraform Apply
      run: terraform apply -auto-approve tfplan-production
      working-directory: ${{ env.WORKING_DIR }}
    - name: Notify deployment
      uses: slackapi/slack-github-action@v1.26.0
      with:
        payload: |
          {
            "text": "✅ GKE Production cluster deployed successfully\nCluster: prod-cluster\nBy: ${{ github.actor }}"
          }
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

# ── Initial setup ────────────────────────────────────────────
# Create state bucket
gsutil mb -p mycompany-prod \
  -l us-central1 \
  gs://mycompany-terraform-state
# Enable versioning on state bucket
gsutil versioning set on \
  gs://mycompany-terraform-state
# Enable required GCP APIs
gcloud services enable \
  container.googleapis.com \
  compute.googleapis.com \
  iam.googleapis.com \
  cloudkms.googleapis.com \
  artifactregistry.googleapis.com \
  monitoring.googleapis.com \
  logging.googleapis.com \
  --project mycompany-prod
# ── Deploy dev ───────────────────────────────────────────────
cd gke-terraform
terraform init \
  -backend-config="bucket=mycompany-terraform-state" \
  -backend-config="prefix=gke/dev"
terraform plan \
  -var-file="environments/dev.tfvars" \
  -out=tfplan-dev
terraform apply tfplan-dev
# ── Get cluster credentials ──────────────────────────────────
gcloud container clusters get-credentials dev-cluster \
  --region us-central1 \
  --project mycompany-dev
# ── Verify cluster ───────────────────────────────────────────
kubectl get nodes
# NAME                                  STATUS   ROLES    AGE
# gke-dev-cluster-application-xxx       Ready    <none>   2m
# gke-dev-cluster-application-yyy       Ready    <none>   2m
kubectl get pods -A
# All system pods should be Running
# Check node pool details
kubectl get nodes -L cloud.google.com/gke-nodepool,topology.kubernetes.io/zone
# Verify Workload Identity
kubectl create serviceaccount test-ksa -n default
kubectl annotate serviceaccount test-ksa \
  iam.gke.io/gcp-service-account=app-sa@mycompany-dev.iam.gserviceaccount.com
# ── Verify private cluster ───────────────────────────────────
kubectl get nodes -o wide
# EXTERNAL-IP should show <none> for private nodes
# ── Check cluster security ───────────────────────────────────
gcloud container clusters describe dev-cluster \
  --region us-central1 \
  --format="yaml(masterAuth,networkConfig,privateClusterConfig)"
# ── Destroy dev when done ────────────────────────────────────
terraform destroy \
  -var-file="environments/dev.tfvars" \
  -auto-approve

# List state resources
terraform state list
# Show specific resource
terraform state show module.gke.google_container_cluster.primary
# Move resource (refactoring)
terraform state mv \
  module.gke.google_container_node_pool.pools[\"app\"] \
  module.gke.google_container_node_pool.pools[\"application\"]
# Import existing cluster
terraform import \
  module.gke.google_container_cluster.primary \
  projects/mycompany-prod/locations/us-central1/clusters/prod-cluster
# Remove from state (without destroying)
terraform state rm \
  module.gke.google_container_node_pool.pools[\"old-pool\"]
# Backup state
gsutil cp \
  gs://mycompany-terraform-state/gke/production/default.tfstate \
  ./backup-$(date +%Y%m%d).tfstate

# Issue 1 — API not enabled
# Error: googleapi: Error 403: ... has not been used in project
gcloud services enable container.googleapis.com
# Issue 2 — Insufficient permissions
# Error: Error creating Cluster: googleapi: Error 403
gcloud projects add-iam-policy-binding mycompany-prod \
  --member="serviceAccount:terraform-sa@mycompany-prod.iam.gserviceaccount.com" \
  --role="roles/container.admin"
# Issue 3 — KMS key permission
# Error: PERMISSION_DENIED: The caller does not have permission
gcloud kms keys add-iam-policy-binding etcd-key \
  --keyring=prod-cluster-keyring \
  --location=us-central1 \
  --member="serviceAccount:service-123@container-engine-robot.iam.gserviceaccount.com" \
  --role="roles/cloudkms.cryptoKeyEncrypterDecrypter"
# Issue 4 — Node pool update requires replacement
# Use lifecycle ignore_changes or blue/green node pool strategy
terraform plan -target=module.gke.google_container_node_pool.pools
# Issue 5 — Cluster stuck deleting
# Remove protection and retry
terraform state rm module.gke.google_container_cluster.primary
gcloud container clusters delete prod-cluster \
  --region us-central1 \
  --project mycompany-prod