Automate Velero AKS Backups with Terraform and Ansible

Automating Velero AKS Backup with Ansible & Terraform


Option 1: Terraform

Project Structure

velero-terraform/
├── main.tf
├── variables.tf
├── outputs.tf
├── providers.tf
└── modules/
├── storage/
│ ├── main.tf
│ └── variables.tf
├── identity/
│ ├── main.tf
│ └── variables.tf
└── velero/
├── main.tf
└── variables.tf

providers.tf

terraform {
required_providers {
azurerm = {
source = "hashicorp/azurerm"
version = "~> 3.0"
}
helm = {
source = "hashicorp/helm"
version = "~> 2.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
version = "~> 2.0"
}
}
}
provider "azurerm" {
features {}
}
provider "helm" {
kubernetes {
host = azurerm_kubernetes_cluster.aks.kube_config.0.host
client_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
client_key = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
}
}
provider "kubernetes" {
host = azurerm_kubernetes_cluster.aks.kube_config.0.host
client_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
client_key = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
}

variables.tf

variable "resource_group_name" {
description = "Resource group name"
type = string
default = "myResourceGroup"
}
variable "location" {
description = "Azure region"
type = string
default = "eastus"
}
variable "aks_cluster_name" {
description = "AKS cluster name"
type = string
default = "myAKSCluster"
}
variable "storage_account_name" {
description = "Storage account for Velero backups"
type = string
default = "velerobackupstorage"
}
variable "blob_container_name" {
description = "Blob container for Velero backups"
type = string
default = "velero-backups"
}
variable "velero_namespace" {
description = "Kubernetes namespace for Velero"
type = string
default = "velero"
}
variable "backup_retention_hours" {
description = "Backup TTL in hours"
type = number
default = 720 # 30 days
}
variable "backup_schedule" {
description = "Cron schedule for backups"
type = string
default = "0 2 * * *" # Daily at 2am
}

main.tf

# Resource Group
resource "azurerm_resource_group" "rg" {
name = var.resource_group_name
location = var.location
}
# ── STORAGE ──────────────────────────────────────────────
resource "azurerm_storage_account" "velero" {
name = var.storage_account_name
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
account_tier = "Standard"
account_replication_type = "LRS"
blob_properties {
delete_retention_policy {
days = 30
}
}
tags = {
purpose = "velero-backup"
}
}
resource "azurerm_storage_container" "velero" {
name = var.blob_container_name
storage_account_name = azurerm_storage_account.velero.name
container_access_type = "private"
}
# ── IDENTITY ─────────────────────────────────────────────
resource "azurerm_user_assigned_identity" "velero" {
name = "velero-identity"
resource_group_name = azurerm_resource_group.rg.name
location = azurerm_resource_group.rg.location
}
# Assign Contributor role to storage account
resource "azurerm_role_assignment" "velero_storage" {
scope = azurerm_storage_account.velero.id
role_definition_name = "Storage Blob Data Contributor"
principal_id = azurerm_user_assigned_identity.velero.principal_id
}
# Assign Contributor role to resource group (for disk snapshots)
resource "azurerm_role_assignment" "velero_rg" {
scope = azurerm_resource_group.rg.id
role_definition_name = "Contributor"
principal_id = azurerm_user_assigned_identity.velero.principal_id
}
# ── AKS CLUSTER ──────────────────────────────────────────
resource "azurerm_kubernetes_cluster" "aks" {
name = var.aks_cluster_name
location = azurerm_resource_group.rg.location
resource_group_name = azurerm_resource_group.rg.name
dns_prefix = var.aks_cluster_name
default_node_pool {
name = "default"
node_count = 2
vm_size = "Standard_D2_v2"
}
identity {
type = "SystemAssigned"
}
tags = {
environment = "production"
}
}
# ── VELERO NAMESPACE ─────────────────────────────────────
resource "kubernetes_namespace" "velero" {
metadata {
name = var.velero_namespace
}
depends_on = [azurerm_kubernetes_cluster.aks]
}
# ── VELERO CREDENTIALS SECRET ────────────────────────────
resource "kubernetes_secret" "velero_credentials" {
metadata {
name = "velero-credentials"
namespace = kubernetes_namespace.velero.metadata[0].name
}
data = {
"cloud" = <<EOF
AZURE_SUBSCRIPTION_ID=${data.azurerm_subscription.current.subscription_id}
AZURE_TENANT_ID=${data.azurerm_subscription.current.tenant_id}
AZURE_CLIENT_ID=${azurerm_user_assigned_identity.velero.client_id}
AZURE_RESOURCE_GROUP=${azurerm_resource_group.rg.name}
AZURE_CLOUD_NAME=AzurePublicCloud
EOF
}
}
# Current subscription data
data "azurerm_subscription" "current" {}
# ── VELERO HELM RELEASE ──────────────────────────────────
resource "helm_release" "velero" {
name = "velero"
repository = "https://vmware-tanzu.github.io/helm-charts"
chart = "velero"
namespace = kubernetes_namespace.velero.metadata[0].name
version = "5.0.0"
values = [
yamlencode({
configuration = {
provider = "azure"
backupStorageLocation = {
name = "default"
provider = "velero.io/azure"
bucket = azurerm_storage_container.velero.name
config = {
resourceGroup = azurerm_resource_group.rg.name
storageAccount = azurerm_storage_account.velero.name
subscriptionId = data.azurerm_subscription.current.subscription_id
}
}
volumeSnapshotLocation = {
name = "default"
provider = "velero.io/azure"
config = {
resourceGroup = azurerm_resource_group.rg.name
subscriptionId = data.azurerm_subscription.current.subscription_id
}
}
}
credentials = {
existingSecret = kubernetes_secret.velero_credentials.metadata[0].name
}
initContainers = [
{
name = "velero-plugin-for-azure"
image = "velero/velero-plugin-for-microsoft-azure:v1.8.0"
imagePullPolicy = "IfNotPresent"
volumeMounts = [
{
mountPath = "/target"
name = "plugins"
}
]
}
]
schedules = {
daily-backup = {
schedule = var.backup_schedule
template = {
ttl = "${var.backup_retention_hours}h0m0s"
includeClusterResources = true
excludedNamespaces = ["kube-system", "velero"]
}
}
}
})
]
depends_on = [
kubernetes_secret.velero_credentials,
azurerm_role_assignment.velero_storage,
azurerm_role_assignment.velero_rg
]
}

outputs.tf

output "storage_account_name" {
value = azurerm_storage_account.velero.name
}
output "blob_container_name" {
value = azurerm_storage_container.velero.name
}
output "velero_identity_client_id" {
value = azurerm_user_assigned_identity.velero.client_id
}
output "aks_cluster_name" {
value = azurerm_kubernetes_cluster.aks.name
}
output "kube_config" {
value = azurerm_kubernetes_cluster.aks.kube_config_raw
sensitive = true
}

Deploy with Terraform

# Initialize
terraform init
# Preview changes
terraform plan -out=tfplan
# Apply
terraform apply tfplan
# Get kubeconfig
terraform output -raw kube_config > ~/.kube/config
# Verify Velero
kubectl get pods -n velero
velero backup-location get


Option 2: Ansible

Project Structure

velero-ansible/
├── inventory/
│ └── hosts.yml
├── group_vars/
│ └── all.yml
├── roles/
│ ├── azure_storage/
│ │ └── tasks/
│ │ └── main.yml
│ ├── azure_identity/
│ │ └── tasks/
│ │ └── main.yml
│ └── velero/
│ ├── tasks/
│ │ └── main.yml
│ └── templates/
│ ├── credentials.j2
│ └── backup-schedule.yml.j2
└── site.yml

group_vars/all.yml

# Azure Settings
resource_group: "myResourceGroup"
location: "eastus"
aks_cluster_name: "myAKSCluster"
# Storage Settings
storage_account_name: "velerobackupstorage"
blob_container_name: "velero-backups"
# Velero Settings
velero_namespace: "velero"
velero_version: "v1.12.0"
velero_azure_plugin_version: "v1.8.0"
velero_chart_version: "5.0.0"
# Backup Settings
backup_schedule: "0 2 * * *"
backup_ttl: "720h"
backup_name: "daily-backup"
# Namespaces to exclude
excluded_namespaces:
- kube-system
- velero

roles/azure_storage/tasks/main.yml

---
- name: Create Resource Group
azure.azcollection.azure_rm_resourcegroup:
name: "{{ resource_group }}"
location: "{{ location }}"
state: present
- name: Create Storage Account
azure.azcollection.azure_rm_storageaccount:
resource_group: "{{ resource_group }}"
name: "{{ storage_account_name }}"
type: Standard_LRS
kind: StorageV2
state: present
register: storage_account_result
- name: Create Blob Container
azure.azcollection.azure_rm_storageblob:
resource_group: "{{ resource_group }}"
storage_account_name: "{{ storage_account_name }}"
container: "{{ blob_container_name }}"
state: present
- name: Get Storage Account Keys
azure.azcollection.azure_rm_storageaccount_info:
resource_group: "{{ resource_group }}"
name: "{{ storage_account_name }}"
register: storage_info
- name: Set Storage Key Fact
set_fact:
storage_account_key: "{{ storage_info.storageaccounts[0].primary_endpoints.key }}"

roles/azure_identity/tasks/main.yml

---
- name: Get Azure Subscription Info
azure.azcollection.azure_rm_subscription_info:
register: subscription_info
- name: Set Subscription Facts
set_fact:
subscription_id: "{{ subscription_info.subscriptions[0].subscription_id }}"
tenant_id: "{{ subscription_info.subscriptions[0].tenant_id }}"
- name: Create Service Principal for Velero
azure.azcollection.azure_rm_adserviceprincipal:
app_id: "velero-sp"
state: present
register: sp_result
- name: Assign Contributor Role to Service Principal
azure.azcollection.azure_rm_roleassignment:
scope: "/subscriptions/{{ subscription_id }}/resourceGroups/{{ resource_group }}"
assignee_object_id: "{{ sp_result.object_id }}"
role_definition_name: Contributor
state: present
- name: Assign Storage Blob Contributor Role
azure.azcollection.azure_rm_roleassignment:
scope: "/subscriptions/{{ subscription_id }}/resourceGroups/{{ resource_group }}/providers/Microsoft.Storage/storageAccounts/{{ storage_account_name }}"
assignee_object_id: "{{ sp_result.object_id }}"
role_definition_name: "Storage Blob Data Contributor"
state: present

roles/velero/templates/credentials.j2

AZURE_SUBSCRIPTION_ID={{ subscription_id }}
AZURE_TENANT_ID={{ tenant_id }}
AZURE_CLIENT_ID={{ client_id }}
AZURE_CLIENT_SECRET={{ client_secret }}
AZURE_RESOURCE_GROUP={{ resource_group }}
AZURE_CLOUD_NAME=AzurePublicCloud

roles/velero/templates/backup-schedule.yml.j2

apiVersion: velero.io/v1
kind: Schedule
metadata:
name: {{ backup_name }}
namespace: {{ velero_namespace }}
spec:
schedule: "{{ backup_schedule }}"
template:
ttl: "{{ backup_ttl }}"
includeClusterResources: true
excludedNamespaces:
{% for ns in excluded_namespaces %}
- {{ ns }}
{% endfor %}

roles/velero/tasks/main.yml

---
- name: Create Velero Namespace
kubernetes.core.k8s:
name: "{{ velero_namespace }}"
api_version: v1
kind: Namespace
state: present
- name: Create Velero Credentials File
template:
src: credentials.j2
dest: /tmp/credentials-velero
mode: '0600'
- name: Create Kubernetes Secret for Velero Credentials
kubernetes.core.k8s:
state: present
definition:
apiVersion: v1
kind: Secret
metadata:
name: velero-credentials
namespace: "{{ velero_namespace }}"
stringData:
cloud: |
AZURE_SUBSCRIPTION_ID={{ subscription_id }}
AZURE_TENANT_ID={{ tenant_id }}
AZURE_CLIENT_ID={{ client_id }}
AZURE_CLIENT_SECRET={{ client_secret }}
AZURE_RESOURCE_GROUP={{ resource_group }}
AZURE_CLOUD_NAME=AzurePublicCloud
- name: Add Velero Helm Repository
kubernetes.core.helm_repository:
name: vmware-tanzu
repo_url: "https://vmware-tanzu.github.io/helm-charts"
state: present
- name: Install Velero via Helm
kubernetes.core.helm:
name: velero
chart_ref: vmware-tanzu/velero
chart_version: "{{ velero_chart_version }}"
namespace: "{{ velero_namespace }}"
state: present
values:
configuration:
provider: azure
backupStorageLocation:
name: default
provider: velero.io/azure
bucket: "{{ blob_container_name }}"
config:
resourceGroup: "{{ resource_group }}"
storageAccount: "{{ storage_account_name }}"
subscriptionId: "{{ subscription_id }}"
volumeSnapshotLocation:
name: default
provider: velero.io/azure
config:
resourceGroup: "{{ resource_group }}"
subscriptionId: "{{ subscription_id }}"
credentials:
existingSecret: velero-credentials
initContainers:
- name: velero-plugin-for-azure
image: "velero/velero-plugin-for-microsoft-azure:{{ velero_azure_plugin_version }}"
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /target
name: plugins
- name: Wait for Velero Pod to be Ready
kubernetes.core.k8s_info:
kind: Pod
namespace: "{{ velero_namespace }}"
label_selectors:
- app.kubernetes.io/name=velero
register: velero_pod
until: velero_pod.resources[0].status.phase == "Running"
retries: 10
delay: 15
- name: Apply Backup Schedule
kubernetes.core.k8s:
state: present
template: backup-schedule.yml.j2
- name: Verify Backup Location
command: velero backup-location get
register: backup_location_status
changed_when: false
- name: Display Backup Location Status
debug:
msg: "{{ backup_location_status.stdout }}"
- name: Clean Up Credentials File
file:
path: /tmp/credentials-velero
state: absent

site.yml (Main Playbook)

---
- name: Setup Velero Backup for AKS
hosts: localhost
connection: local
gather_facts: false
pre_tasks:
- name: Verify required tools are installed
command: "{{ item }} --version"
loop:
- az
- kubectl
- helm
register: tool_check
changed_when: false
- name: Verify AKS context
command: kubectl cluster-info
register: cluster_info
changed_when: false
- name: Display cluster info
debug:
msg: "{{ cluster_info.stdout_lines[0] }}"
roles:
- azure_storage
- azure_identity
- velero
post_tasks:
- name: Trigger initial manual backup
command: >
velero backup create initial-backup
--include-cluster-resources=true
--wait
register: initial_backup
changed_when: true
- name: Display backup result
debug:
msg: "{{ initial_backup.stdout }}"

Run the Ansible Playbook

# Install required collections
ansible-galaxy collection install azure.azcollection
ansible-galaxy collection install kubernetes.core
# Install Python dependencies
pip install ansible[azure] kubernetes
# Run the playbook
ansible-playbook site.yml -v
# Run specific role only
ansible-playbook site.yml --tags "velero" -v
# Dry run
ansible-playbook site.yml --check -v

Comparison: Terraform vs Ansible

FeatureTerraformAnsible
Best ForInfrastructure provisioningConfiguration & app deployment
State ManagementYes (tfstate file)No native state
IdempotencyBuilt-inTask-level
Azure ResourcesExcellentGood
Kubernetes ResourcesGoodExcellent
Learning CurveMediumLow
RollbackVia stateManual
Recommended UseCreate AKS + StorageInstall & configure Velero

Best Practice: Combine Both

Terraform → Creates Azure infrastructure (AKS, Storage, Identity)
Ansible → Installs and configures Velero on the cluster
Velero → Runs scheduled backups automatically

This gives you the best of both worlds — Terraform for infrastructure and Ansible for application configuration.

How to Backup AKS with Velero: A Step-by-Step Guide

Backing Up AKS with Velero

Prerequisites

  • AKS cluster running
  • Azure CLI installed
  • kubectl configured
  • Helm installed

Step 1: Create Azure Storage Account & Blob Container

# Set variables
RESOURCE_GROUP="myResourceGroup"
LOCATION="eastus"
STORAGE_ACCOUNT="velerobackupstorage"
BLOB_CONTAINER="velero-backups"
AKS_CLUSTER="myAKSCluster"
# Create resource group (if not exists)
az group create --name $RESOURCE_GROUP --location $LOCATION
# Create storage account
az storage account create \
--name $STORAGE_ACCOUNT \
--resource-group $RESOURCE_GROUP \
--location $LOCATION \
--sku Standard_LRS \
--encryption-services blob
# Create blob container
az storage container create \
--name $BLOB_CONTAINER \
--account-name $STORAGE_ACCOUNT

Step 2: Create Service Principal for Velero

# Create service principal
AZURE_SUBSCRIPTION_ID=$(az account show --query id -o tsv)
AZURE_TENANT_ID=$(az account show --query tenantId -o tsv)
# Create SP and assign contributor role
AZURE_CLIENT_SECRET=$(az ad sp create-for-rbac \
--name velero-sp \
--role Contributor \
--scopes /subscriptions/$AZURE_SUBSCRIPTION_ID \
--query password \
-o tsv)
AZURE_CLIENT_ID=$(az ad sp list \
--display-name velero-sp \
--query '[0].appId' \
-o tsv)
echo "Client ID: $AZURE_CLIENT_ID"
echo "Client Secret: $AZURE_CLIENT_SECRET"

Step 3: Create Velero Credentials File

# Create credentials file
cat << EOF > ./credentials-velero
AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}
AZURE_TENANT_ID=${AZURE_TENANT_ID}
AZURE_CLIENT_ID=${AZURE_CLIENT_ID}
AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET}
AZURE_RESOURCE_GROUP=${RESOURCE_GROUP}
AZURE_CLOUD_NAME=AzurePublicCloud
EOF

Step 4: Install Velero on AKS

# Add Velero Helm repo
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
helm repo update
# Get storage account access key
STORAGE_ACCOUNT_KEY=$(az storage account keys list \
--account-name $STORAGE_ACCOUNT \
--resource-group $RESOURCE_GROUP \
--query '[0].value' \
-o tsv)
# Install Velero with Azure plugin
velero install \
--provider azure \
--plugins velero/velero-plugin-for-microsoft-azure:v1.8.0 \
--bucket $BLOB_CONTAINER \
--secret-file ./credentials-velero \
--backup-location-config \
resourceGroup=$RESOURCE_GROUP,\
storageAccount=$STORAGE_ACCOUNT,\
storageAccountKeyEnvVar=AZURE_STORAGE_ACCOUNT_ACCESS_KEY \
--snapshot-location-config \
resourceGroup=$RESOURCE_GROUP,\
apiTimeout=5m \
--use-volume-snapshots=true \
--wait

Step 5: Verify Velero Installation

# Check Velero pods are running
kubectl get pods -n velero
# Check backup storage location is available
velero backup-location get
# Expected output:
# NAME PROVIDER BUCKET/PREFIX PHASE LAST VALIDATED
# default azure velero-backups Available ...

Step 6: Create Backups

Manual Backup (Full Cluster)

# Backup entire cluster
velero backup create full-cluster-backup \
--include-cluster-resources=true \
--wait
# Check backup status
velero backup describe full-cluster-backup
velero backup logs full-cluster-backup

Backup Specific Namespace

# Backup a single namespace
velero backup create my-app-backup \
--include-namespaces my-app-namespace \
--wait
# Backup multiple namespaces
velero backup create multi-ns-backup \
--include-namespaces namespace1,namespace2 \
--wait

Backup with Labels

# Backup resources matching a label
velero backup create label-backup \
--selector app=my-application \
--wait

Exclude Specific Resources

# Exclude secrets and specific namespaces
velero backup create selective-backup \
--exclude-namespaces kube-system,velero \
--exclude-resources secrets \
--wait

Step 7: Schedule Automated Backups

# Daily backup at 2am UTC (keep 30 days)
velero schedule create daily-backup \
--schedule="0 2 * * *" \
--ttl 720h \
--include-cluster-resources=true
# Weekly backup every Sunday at midnight (keep 90 days)
velero schedule create weekly-backup \
--schedule="0 0 * * 0" \
--ttl 2160h
# Namespace-specific daily backup
velero schedule create daily-app-backup \
--schedule="0 2 * * *" \
--include-namespaces production \
--ttl 168h
# List all schedules
velero schedule get

Step 8: Restore from Backup

Full Restore

# List available backups
velero backup get
# Restore full cluster backup
velero restore create \
--from-backup full-cluster-backup \
--wait
# Check restore status
velero restore get
velero restore describe <restore-name>

Restore Specific Namespace

# Restore a specific namespace
velero restore create \
--from-backup my-app-backup \
--include-namespaces my-app-namespace \
--wait

Restore to Different Namespace

# Restore namespace-A into namespace-B
velero restore create \
--from-backup my-app-backup \
--namespace-mappings old-namespace:new-namespace \
--wait

Restore Specific Resources Only

# Restore only deployments and services
velero restore create \
--from-backup full-cluster-backup \
--include-resources deployments,services \
--wait

Step 9: Backup Hooks (Pre/Post Actions)

Useful for database consistency before backup:

# deployment-with-backup-hooks.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql
annotations:
pre.hook.backup.velero.io/command: '["/bin/bash", "-c", "mysqldump -u root -p$MYSQL_ROOT_PASSWORD mydb > /backup/dump.sql"]'
pre.hook.backup.velero.io/timeout: "60s"
post.hook.backup.velero.io/command: '["/bin/bash", "-c", "rm -f /backup/dump.sql"]'
kubectl apply -f deployment-with-backup-hooks.yaml

Step 10: Monitor & Troubleshoot

# View backup details
velero backup describe my-backup --details
# View backup logs
velero backup logs my-backup
# View restore logs
velero restore logs my-restore
# Check Velero server logs
kubectl logs deployment/velero -n velero
# List all backups with status
velero backup get
# Delete old backup manually
velero backup delete old-backup-name

Summary Cheat Sheet

TaskCommand
Full backupvelero backup create <name> --include-cluster-resources=true
Namespace backupvelero backup create <name> --include-namespaces <ns>
Schedule dailyvelero schedule create <name> --schedule="0 2 * * *"
List backupsvelero backup get
Restore backupvelero restore create --from-backup <name>
Check statusvelero backup describe <name>
View logsvelero backup logs <name>

Best Practices

  • Test restores regularly – don’t wait for disaster to find out your backup is broken
  • Use TTL to automatically expire old backups and save storage costs
  • Backup before upgrades – always take a snapshot before AKS version upgrades
  • Store credentials securely – use Azure Key Vault instead of plain credentials files
  • Use hooks for databases to ensure data consistency
  • Cross-region storage – store backups in a different Azure region for disaster recovery
  • Combine with GitOps – use Velero for stateful data + Git for manifests

Top Methods for Backing Up AKS Effectively

There are several ways to back up AKS:

1. Velero (Most Popular Open-Source Tool)

  • Backs up Kubernetes resources (deployments, configmaps, secrets, etc.)
  • Backs up persistent volumes using snapshots
  • Stores backups in Azure Blob Storage
  • Supports scheduled backups and restores to same or different cluster

2. Azure Backup for AKS (Native Solution)

  • Microsoft’s built-in backup service for AKS
  • Backs up workloads and persistent volumes
  • Managed via Azure Portal, CLI, or ARM templates
  • Supports backup policies and restore points
  • Integrates with Azure Backup Vault

3. Persistent Volume Snapshots

  • Use Azure Disk Snapshots for stateful data
  • Can be automated via Azure Snapshot Policies
  • Best for database workloads (SQL, MongoDB, etc.)

4. GitOps / Infrastructure as Code

  • Store all manifests in Git (Helm charts, YAML files)
  • Tools like Flux or ArgoCD redeploy state from Git
  • Not a traditional backup but enables fast cluster recreation

5. etcd Backup

  • Backs up the cluster state directly
  • More relevant for self-managed Kubernetes; in AKS, Microsoft manages etcd

What You Should Typically Back Up

  • Kubernetes manifests (deployments, services, configmaps)
  • Secrets (or reference them from Key Vault)
  • Persistent volume data
  • Namespace configurations
  • RBAC roles and bindings

Best Practice Recommendation

ApproachBest For
Azure Backup for AKSSimplicity, native Azure integration
Velero + Blob StorageFlexibility, multi-cloud, open-source
GitOpsStateless workloads, fast redeployment
Disk SnapshotsStateful apps with heavy data

A solid strategy usually combines Azure Backup or Velero for cluster state + Disk Snapshots for persistent data + GitOps for fast recovery.

Key Components of Azure Kubernetes Service Explained

Azure Kubernetes Service (AKS) has these main components:

Control Plane (managed by Azure)

  • API Server – entry point for all Kubernetes commands and REST requests
  • etcd – distributed key-value store that holds the cluster state and configuration
  • Scheduler – assigns pods to nodes based on resource availability
  • Controller Manager – maintains desired cluster state (handles node, pod, and endpoint controllers)

Data Plane (managed by you)

  • Nodes – VMs that run your workloads; grouped into node pools
  • Kubelet – agent on each node that ensures containers are running as instructed
  • Kube-proxy – handles network routing and load balancing across pods
  • Container Runtime – runs containers (AKS uses containerd by default)

Networking

  • Virtual Network (VNet) – connects nodes and pods; supports kubenet and Azure CNI
  • Load Balancer – exposes services externally
  • DNS – CoreDNS for internal service discovery
  • Ingress Controller – manages external HTTP/S routing into the cluster

Storage

  • Persistent Volumes (PV) – backed by Azure Disks or Azure Files
  • Storage Classes – define dynamic provisioning policies

Identity & Security

  • Azure Active Directory (Entra ID) integration – for RBAC and authentication
  • Managed Identity – allows AKS to interact with other Azure services securely
  • Azure Key Vault – secrets management via the Secrets Store CSI driver

Monitoring & Management

  • Azure Monitor / Container Insights – metrics, logs, and diagnostics
  • Cluster Autoscaler – automatically scales nodes based on demand
  • Horizontal Pod Autoscaler (HPA) – scales pods based on CPU/memory usage

Top AKS Troubleshooting Steps for Common Issues

AKS Troubleshooting Decision Tree

AKS ISSUE
├── 1. Is the app unreachable?
│ │
│ ├── External access issue
│ │ ├── Check DNS
│ │ ├── Check Public/Internal Load Balancer
│ │ ├── Check Ingress Controller
│ │ ├── Check Service
│ │ └── Check Pods
│ │
│ └── Internal access issue
│ ├── Check Service name
│ ├── Check CoreDNS
│ ├── Check Network Policy
│ └── Check Pod-to-Pod connectivity
├── 2. Are pods not running?
│ │
│ ├── Pending
│ │ ├── Not enough CPU/memory
│ │ ├── Taints/tolerations issue
│ │ ├── Node selector/affinity issue
│ │ ├── Cluster autoscaler maxed out
│ │ └── Subnet IP exhaustion
│ │
│ ├── ImagePullBackOff
│ │ ├── Wrong image name/tag
│ │ ├── ACR permission missing
│ │ └── Network/DNS issue to registry
│ │
│ └── CrashLoopBackOff
│ ├── App bug
│ ├── Missing secret/config map
│ ├── Bad env variable
│ └── Probe misconfigured
├── 3. Are nodes unhealthy?
│ │
│ ├── Node NotReady
│ │ ├── VMSS health
│ │ ├── Kubelet issue
│ │ ├── Disk pressure
│ │ ├── Memory pressure
│ │ └── Network issue
│ │
│ └── Node pool issue
│ ├── Upgrade failed
│ ├── Scale operation failed
│ └── Quota/capacity issue
├── 4. Is it a networking issue?
│ │
│ ├── DNS failure
│ │ ├── CoreDNS
│ │ ├── Private DNS zone
│ │ └── Custom DNS forwarders
│ │
│ ├── Routing failure
│ │ ├── UDR
│ │ ├── Azure Firewall
│ │ ├── NSG
│ │ └── Route table association
│ │
│ └── Private Endpoint failure
│ ├── DNS resolves to private IP?
│ ├── VNet peering working?
│ ├── NSG allows traffic?
│ └── Private endpoint approved?
├── 5. Is it identity/security?
│ │
│ ├── Azure resource access failing
│ │ ├── Managed identity assigned?
│ │ ├── RBAC role correct?
│ │ └── Workload Identity configured?
│ │
│ └── Kubernetes access failing
│ ├── Azure AD login?
│ ├── Kubernetes RBAC?
│ └── Namespace permissions?
└── 6. Is it platform-wide?
├── Multiple services affected
│ ├── Check Azure Service Health
│ ├── Check regional outage
│ └── Check dependency outage
└── Only one app affected
├── Check recent deployment
├── Rollback if needed
└── Compare config/secrets

Memorize This Shortcut

DNS → LB/Ingress → Service → Pod → Node → Network → Identity → Azure

Interview Line

“I troubleshoot AKS layer by layer: first access path, then Kubernetes objects, then node health, then Azure networking, identity, and platform dependencies.”

Securing Azure Kubernetes Service: A 2026 Guide

Securing Azure Kubernetes Service (AKS) involves a Shared Responsibility Model: Microsoft manages the control plane (API server, etcd), while you are responsible for securing the worker nodes, networking, and the workloads themselves.

As of 2026, here are the non-negotiable security best practices for AKS.


1. Identity and Access Management (IAM)

Authentication is your first line of defense.

  • Microsoft Entra ID (Azure AD) Integration: Never use local Kubernetes accounts. Integrate AKS with Entra ID to manage cluster access via existing corporate identities and groups.
  • Azure RBAC for Kubernetes: Use Azure Role-Based Access Control to provide granular permissions (e.g., Azure Kubernetes Service RBAC Reader).
  • Workload Identity: Avoid using “Secret” objects for Azure credentials. Use Entra Workload ID, which allows pods to authenticate to Azure services (like Key Vault or Storage) using managed identities instead of long-lived passwords.

2. Network Security

Don’t let your cluster be a “sitting duck” on the public internet.

  • Private Clusters: Deploy AKS as a Private Cluster. This ensures that the API server is only accessible via a private IP within your Virtual Network (VNet), completely removing it from the public internet.
  • Authorized IP Ranges: If you must have a public API server, strictly limit access to specific CIDR ranges (e.g., your office or VPN IP).
  • Default-Deny Network Policies: By default, all pods in Kubernetes can talk to each other. Implement Azure Network Policies or Calico to enforce a “Default Deny” posture, only allowing explicitly permitted traffic.

3. Host and Node Security

The underlying VMs (nodes) are often the weakest link.

  • Azure Linux (CBL-Mariner): Use Azure Linux as your Node OS. It is a lightweight, security-hardened distribution maintained by Microsoft specifically for AKS.Critical Update: Support for Azure Linux 2.0 ended in late 2025. Ensure you are on Azure Linux 3.0 or higher to receive security patches.
  • Automatic Upgrades: Enable the Auto-upgrade channel (e.g., stable or node-image) to ensure your nodes automatically receive OS security patches and Kubernetes version updates.
  • Disable Public IPs for Nodes: Ensure worker nodes do not have public IP addresses; use an Azure Load Balancer or NAT Gateway for egress.

4. Workload and Secret Management

How you run your code determines your “blast radius” during an attack.

  • Secrets Store CSI Driver: Do not store secrets in standard Kubernetes YAML files. Use the Azure Key Vault Provider for Secrets Store CSI Driver to mount secrets directly from Key Vault into your pods as volumes.
  • Pod Security Standards: Use Azure Policy for Kubernetes to enforce the “Restricted” pod security standard. This prevents:
    • Containers running as root.
    • Privileged escalation.
    • Writing to the root filesystem (use read-only filesystems instead).
  • Resource Limits: Always define CPU and Memory limits. This prevents a single compromised or “runaway” container from causing a Denial of Service (DoS) for the entire node.

5. Continuous Monitoring and Defense

You cannot protect what you cannot see.

  • Microsoft Defender for Containers: Enable this for real-time threat detection. It scans images for vulnerabilities in the registry and monitors running containers for suspicious behavior (e.g., unexpected shell execution).
  • Image Scanning in CI/CD: Use tools like Trivy or Microsoft Defender to scan images before they are pushed to the Azure Container Registry (ACR).
  • Audit Logging: Stream Kubernetes Audit Logs to a Log Analytics Workspace. This provides a paper trail of who did what in your cluster.

AKS Security Checklist (Quick Reference)

CategoryHigh Priority Task
AccessDisable local accounts; use Entra ID integration.
API ServerUse a Private Cluster or Authorized IP ranges.
OSMigrate to Azure Linux 3.0; enable auto-patches.
SecretsUse Azure Key Vault via CSI Driver (no YAML secrets).
TrafficImplement “Default Deny” Network Policies.
PolicyApply Azure Policy to block non-compliant deployments.

To get you started, I’ve provided a Network Policy to secure traffic and an Azure Policy to enforce security rules at the cluster level.


1. Default Deny Network Policy

By default, Kubernetes allows all pods to talk to each other. This is a major security risk. The following policy creates a “Default Deny” posture for a specific namespace, meaning you must explicitly “whitelist” any traffic you want to allow.

YAML

kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
name: default-deny-all
namespace: production # Apply to your specific namespace
spec:
podSelector: {} # Selects all pods in the namespace
policyTypes:
- Ingress
- Egress

2. Azure Policy for Kubernetes (Enforcement)

While Network Policies handle traffic, Azure Policy ensures that developers don’t deploy “insecure” containers in the first place. You can apply the “Kubernetes cluster pod security restricted standards for Linux-based workloads” initiative directly from the Azure Portal.

Once assigned, it will automatically block any deployment that:

  • Tries to run as the Root user.
  • Attempts to mount the host path (which could allow a container to escape to the node).
  • Uses Privileged mode.

How to apply it via CLI:

Bash

# Get the ID of the 'Restricted' policy initiative
policyId=$(az policy set-definition list --query "[?displayName=='Kubernetes cluster pod security restricted standards for Linux-based workloads'].id" -o tsv)
# Assign it to your Resource Group or Subscription
az policy assignment create --name 'aks-restricted-policy' --policy-set-definition $policyId --scope /subscriptions/<your-subscription-id>/resourceGroups/<your-rg-name>

3. Securing the Flow

To visualize how these layers interact, imagine the lifecycle of a request:

  1. Identity: The user authenticates via Entra ID.
  2. Admission: Azure Policy checks if the YAML is “safe.”
  3. Network: Network Policies ensure the pod only talks to its specific database.
  4. Secrets: The pod pulls credentials from Azure Key Vault at runtime.

Pro-Tip: The “Audit” Mode

If you are worried that applying these policies will break the current app, you can set the Azure Policy effect to Audit instead of Deny. This allows you to see a list of non-compliant pods in the Azure Portal without actually taking them offline. Once you’ve fixed the configurations, you can flip the switch to Deny.

To pass the “Restricted” policy you just applied, your pod deployments can no longer be “lazy.” You must explicitly tell Kubernetes that the container is not allowed to do anything dangerous.

Below is a compliant deployment.yaml for a microservice.

The “Secure” Pod Template

YAML

apiVersion: apps/v1
kind: Deployment
metadata:
name: secure-microservice
namespace: production
spec:
replicas: 2
selector:
matchLabels:
app: web-api
template:
metadata:
labels:
app: web-api
spec:
# 1. Don't run as Root!
securityContext:
runAsNonRoot: true
runAsUser: 1000
fsGroup: 2000
containers:
- name: api-container
image: myacr.azurecr.io/web-api:v1.2
# 2. Tighten the container's capabilities
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop:
- ALL
# 3. Resource limits prevent DoS attacks
resources:
limits:
cpu: "500m"
memory: "256Mi"
requests:
cpu: "100m"
memory: "128Mi"
ports:
- containerPort: 8080

Why this works (The “Why” for your client)

If you are proposing this to your client, explain the benefits of these specific lines:

  • runAsNonRoot: true: If a hacker exploits a bug in the app code, they only get “user” access. They can’t install software, change kernel settings, or break out of the container to the host server.
  • readOnlyRootFilesystem: true: This is a massive security win. It prevents malware from being downloaded and saved into the container. If the app needs to write temporary data, you should mount an emptyDir volume specifically for that.
  • capabilities: drop: - ALL: Linux containers usually come with “capabilities” they don’t need (like changing the system clock). Dropping all of them follows the Principle of Least Privilege.

How to handle “Storage” in a Read-Only Filesystem

Since the filesystem is now read-only, your app might crash if it tries to write to /tmp. You can solve this by adding a small “scratch space” in memory:

YAML

        volumeMounts:
        - name: tmp-volume
          mountPath: /tmp
      volumes:
      - name: tmp-volume
        emptyDir: {} # This creates a temporary writable folder in RAM

Proposing this as a “Service”

You can frame this to your company as a “Hardening Sprint.” > “I want to audit our existing 15 microservices to ensure they meet the 2026 AKS Restricted Security Standards. This will reduce our attack surface by 80% and ensure we pass our next compliance audit.”

High-Level Design for Azure Banking Chatbots

Here’s a practical banking chatbot HLD on Azure with the three things you asked for: components, data flow, and security controls.

1. Scope and goals

This design is for a banking chatbot that can answer grounded questions, retrieve approved internal knowledge, perform tightly controlled actions through backend APIs, and escalate sensitive cases to a human. On Azure, the common enterprise pattern is an application/orchestration layer in front of Azure OpenAI and Azure AI Search, with private networking and identity controls around the whole path. (Microsoft Learn)

2. High-level architecture

Customers / Employees
Web / Mobile / Contact Center UI
API Gateway / WAF
Chat Orchestrator (App Service or AKS)
├─ Auth / session / rate limits
├─ Prompt assembly
├─ Policy & compliance checks
├─ PII redaction
├─ Tool calling / workflow engine
├─ Confidence scoring
└─ Human handoff
+----------------------+----------------------+----------------------+
| | | |
v v v
Azure OpenAI Azure AI Search Banking APIs / Systems
(chat + (hybrid RAG) (accounts, cards, CRM,
embeddings) ticketing, fraud, etc.)
\ | /
\ | /
\ v /
\ Enterprise data /
\ (Blob, SharePoint, SQL) /
+-----------------------------+
Supporting:
- Microsoft Entra ID
- Azure Key Vault
- Azure Monitor / App Insights / Log Analytics
- Microsoft Sentinel
- Private Endpoints / VNet Integration

Microsoft’s current baseline enterprise chat architecture uses a secured app layer in front of model and retrieval services, and Azure AI Search is the recommended grounding layer for RAG with hybrid retrieval options. (Microsoft Learn)

3. Core components

A. Channels

The chatbot can be exposed through mobile banking, web banking, employee portal, or contact-center console. The UI should not call the model directly; it should go through a backend orchestrator so the bank can enforce policy, logging, and authorization centrally. That backend-first pattern is part of Microsoft’s baseline architecture. (Microsoft Learn)

B. API gateway / edge

Put a gateway and WAF in front of the chatbot for TLS termination, request filtering, DDoS protection, and traffic governance. This is consistent with Microsoft’s baseline Azure web/chat reference designs, which assume a secured edge in front of the application layer. (Microsoft Learn)

C. Chat orchestrator

This is the main control layer. It manages:

  • authentication and session state
  • prompt templates
  • retrieval requests
  • business-rule checks
  • tool calling to banking APIs
  • confidence scoring
  • citations/disclosures
  • escalation to humans

Microsoft’s enterprise chat reference architecture explicitly separates this orchestration layer from the model and data stores. (Microsoft Learn)

D. Azure OpenAI

Use Azure OpenAI for:

  • chat generation
  • embeddings for retrieval

Azure documents content filtering and abuse monitoring for Azure OpenAI / Azure Direct Models, which makes it suitable for enterprise guardrail layers, though that does not replace your own banking-specific controls. (Microsoft Learn)

E. Azure AI Search

Use Azure AI Search as the RAG layer for policies, product docs, SOPs, FAQs, forms, and knowledge articles. Azure AI Search supports hybrid retrieval with keyword, vector, and semantic ranking, plus chunking/enrichment patterns for PDFs and images. It also supports document-level security trimming patterns. (Microsoft Learn)

F. Enterprise data sources

Typical sources are:

  • Blob Storage
  • SharePoint
  • SQL / Cosmos-style operational data stores
  • document repositories
  • internal policy systems

These sources should feed an ingestion pipeline that extracts, chunks, enriches, and indexes content into Azure AI Search. Microsoft’s RAG guidance calls out chunking, OCR, document extraction, and enrichment as core parts of the pattern. (Microsoft Learn)

G. Banking systems / tools

The orchestrator should call approved backend APIs, not let the model talk directly to core banking systems. Examples:

  • account summary API
  • card freeze/unfreeze API
  • loan status API
  • CRM/ticketing API
  • fraud escalation workflow

This is an architectural recommendation rather than a Microsoft product rule, but it follows the same control-layer pattern in Microsoft’s baseline chat architecture. (Microsoft Learn)

4. End-to-end data flow

Flow 1: Knowledge question

Example: “What is the fee for an international wire?”

  1. User sends a message from mobile/web.
  2. Gateway forwards it to the orchestrator.
  3. Orchestrator authenticates the user and applies policy checks.
  4. Orchestrator sends a retrieval query to Azure AI Search.
  5. Azure AI Search returns grounded chunks and metadata.
  6. Orchestrator builds the prompt with citations and instructions.
  7. Azure OpenAI generates the answer.
  8. Orchestrator adds disclosure text and returns the response.

This is a standard RAG pattern: retrieve first, then generate with grounded context. Azure AI Search documentation explicitly describes this model. (Microsoft Learn)

Flow 2: Action request

Example: “Freeze my debit card.”

  1. User sends the request.
  2. Orchestrator authenticates and checks entitlements.
  3. Orchestrator classifies this as an action, not just Q&A.
  4. Orchestrator optionally uses the model to interpret intent.
  5. Orchestrator calls the bank’s card-management API.
  6. Backend system performs the action.
  7. Orchestrator returns a confirmed result or escalates if needed.

The key design principle is that the model can help interpret intent, but the backend system remains the source of truth and enforcement. This is an architectural best practice built on the app-layer separation Microsoft recommends. (Microsoft Learn)

Flow 3: Sensitive or low-confidence case

Example: fraud complaint, legal complaint, hardship, uncertain answer.

  1. Orchestrator detects a sensitive topic or low confidence.
  2. It blocks automated completion or limits the response.
  3. It routes the case to a human banker/contact-center agent.
  4. Logs and case metadata are stored for audit.

Human handoff is not a single Azure feature, but it is a recommended enterprise control pattern for regulated use cases where accuracy and accountability matter. Azure’s baseline architecture supports orchestrator-driven workflow and escalation patterns. (Microsoft Learn)

5. Security controls

Identity and access

Use Microsoft Entra ID for workforce identities and your customer identity layer for retail users. For retrieval, use identity-aware filtering and document-level access trimming so users only retrieve content they are allowed to see. Microsoft documents Entra-based auth and Azure AI Search security trimming for this purpose. (Microsoft Learn)

Private networking

Use private endpoints and VNet integration for Azure OpenAI, Azure AI Search, Key Vault, and the application tier where possible. Microsoft’s baseline chat architecture emphasizes private connectivity, and Key Vault supports Private Link integration. (Microsoft Learn)

Secrets and keys

Store secrets, certificates, and encryption keys in Azure Key Vault. Key Vault is designed for secure storage of secrets, keys, and certificates and supports logging and integration with Azure Monitor. (Microsoft Learn)

Managed identities

Prefer managed identities between Azure services instead of hard-coded secrets. Microsoft documents managed-identity-based authentication for Key Vault and uses passwordless patterns in Azure application architectures. (Microsoft Learn)

Content safety

Use Azure OpenAI’s built-in content filtering and abuse monitoring, but treat those as baseline controls rather than your only compliance layer. Banking-specific policies still belong in the orchestrator. Azure documents both abuse monitoring and configurable harm categories/severity concepts. (Microsoft Learn)

Data protection

Minimize prompt data, redact unnecessary PII before model calls, and keep regulated records in approved systems of record. Azure publishes data privacy/security information for Azure Direct Models, including Azure OpenAI. (Microsoft Learn)

Monitoring and audit

Send logs, traces, and security events to Azure Monitor / Application Insights / Log Analytics, and use Microsoft Sentinel for SIEM/SOC workflows. Key Vault also supports exporting logs to Azure Monitor. (Microsoft Learn)

6. Non-functional requirements

Availability

Deploy the app tier with redundancy and design for zone/region resilience where needed. Microsoft’s baseline chat architecture is explicitly aimed at secure, highly available, zone-redundant enterprise chat applications. (Microsoft Learn)

Scalability

Scale the stateless app/orchestrator tier horizontally, keep chat history in a dedicated store, and scale search/model capacity independently. This separation follows the Azure reference pattern where the app, model, and retrieval tiers are distinct. (Microsoft Learn)

Auditability

Every model call, retrieval event, tool call, and escalation path should be logged with correlation IDs. This is a design recommendation built on Azure’s monitoring stack and the needs of regulated environments. (Microsoft Learn)

7. Recommended deployment split

For a bank, split into two bots:

Customer bot

  • narrow scope
  • strict action permissions
  • early human escalation
  • only approved public/customer-facing knowledge

Employee copilot

  • broader internal knowledge
  • document-level access trimming
  • workflow tools for CRM and case systems
  • stronger audit controls

This split is an architectural recommendation because customer-facing and employee-facing risk profiles are usually different, while Azure’s identity and retrieval controls support both models. (Microsoft Learn)

8. HLD summary table

LayerMain componentsPurposeKey controls
ChannelsMobile, web, contact center, employee portalUser interactionAuth, session controls
EdgeAPI gateway, WAFSecure entry pointTLS, DDoS, request filtering
App tierOrchestrator on App Service/AKSPrompting, policy, tool calling, handoffRate limits, PII redaction, audit
AI tierAzure OpenAIResponse generation, embeddingsContent filtering, abuse monitoring
Retrieval tierAzure AI SearchGrounding and citationsHybrid search, ACL/security trimming
Data tierBlob, SharePoint, SQL, docsKnowledge sourcesAccess control, ingestion governance
Systems tierCore banking APIs, CRM, fraud, cardsTrusted actions and transactionsAPI auth, least privilege
Security/opsEntra ID, Key Vault, Monitor, SentinelIdentity, secrets, monitoringPrivate endpoints, logging, SIEM

The Azure-specific parts of this table are grounded in Microsoft’s official architecture, retrieval, identity, and Key Vault guidance. (Microsoft Learn)

9. Best one-line design

Use a secure orchestrator in front of Azure OpenAI and Azure AI Search, ground policy answers with RAG, route actions through approved banking APIs, and enforce identity, private networking, secrets management, logging, and human escalation throughout. (Microsoft Learn)

Step-by-Step Guide to Install OADP on OpenShift

Here’s a practical step-by-step OADP install for OpenShift, using AWS S3 as the backup location. This is the most common pattern and maps to Red Hat’s current OADP flow: install the OADP Operator, create the default credentials secret, then create a DataProtectionApplication (DPA). OADP is the supported OpenShift path for application backup/restore, and for PV snapshots your provider must support native snapshots or CSI snapshots. (Red Hat Documentation)

1. Prereqs

You need:

  • cluster-admin access
  • an S3 bucket
  • AWS credentials with access to the bucket
  • snapshot support if you want PV snapshots
  • oc logged into the cluster. OADP also requires a default credentials secret during installation. (Red Hat Documentation)

2. Create the OADP namespace

oc create namespace openshift-adp

Red Hat’s OADP examples use openshift-adp as the namespace. (Red Hat Documentation)

3. Install the OADP Operator

In the OpenShift web console:

  • go to Operators → OperatorHub
  • search for OADP
  • open OpenShift API for Data Protection
  • click Install
  • install it into openshift-adp

Wait for the operator pod to be running:

oc get pods -n openshift-adp

The Red Hat flow is to install the OADP Operator first, then configure credentials and the DPA. (Red Hat Documentation)

4. Create the AWS credentials file

Create a local file named credentials-velero:

cat <<'EOF' > credentials-velero
[default]

aws_access_key_id=YOUR_AWS_ACCESS_KEY_ID

aws_secret_access_key=YOUR_AWS_SECRET_ACCESS_KEY

EOF

Red Hat documents this credentials-velero pattern for AWS-backed OADP installs. (Red Hat Documentation)

5. Create the default OADP secret

Create the required secret in openshift-adp:

oc create secret generic cloud-credentials \
-n openshift-adp \
--from-file cloud=./credentials-velero

For AWS, the default secret name is cloud-credentials. Red Hat notes that the DPA install expects a default secret; otherwise installation fails. (Red Hat Documentation)

6. Create the DataProtectionApplication

Apply a DPA like this:

apiVersion: oadp.openshift.io/v1alpha1
kind: DataProtectionApplication
metadata:
name: dpa
namespace: openshift-adp
spec:
backupLocations:
- velero:
provider: aws
default: true
objectStorage:
bucket: YOUR_S3_BUCKET
prefix: ocp-backups
config:
region: us-east-1
snapshotLocations:
- velero:
provider: aws
config:
region: us-east-1
configuration:
velero:
defaultPlugins:
- openshift
- aws
- csi

Apply it:

oc apply -f dpa.yaml

The DPA is the main OADP custom resource that wires backup storage and snapshot locations, and current OpenShift docs describe these OADP objects as the supported app backup path. (Red Hat Documentation)

7. Wait for OADP to become ready

Check the DPA and pods:

oc get dpa -n openshift-adp
oc get pods -n openshift-adp

You want the DPA to move to a ready state before creating backups. Red Hat’s backup flow requires the DataProtectionApplication to be Ready before backup CRs are used. (Red Hat Documentation)

8. Create your first backup

Once OADP is ready, back up a namespace:

apiVersion: velero.io/v1
kind: Backup
metadata:
name: app-backup
namespace: openshift-adp
spec:
includedNamespaces:
- my-app
snapshotVolumes: true
ttl: 720h

Apply it:

oc apply -f backup.yaml

OADP uses Velero backup CRs for application backup and supports filtering by namespace, labels, or resource type. (Red Hat Documentation)

9. Check backup status

oc get backup -n openshift-adp
oc describe backup app-backup -n openshift-adp

This confirms whether the backup finished and whether volume snapshots were taken.

10. Optional: schedule automatic backups

apiVersion: velero.io/v1
kind: Schedule
metadata:
name: daily-backup
namespace: openshift-adp
spec:
schedule: "0 2 * * *"
template:
includedNamespaces:
- my-app
snapshotVolumes: true
ttl: 720h

Apply it:

oc apply -f schedule.yaml

OADP supports scheduled Velero backups through Schedule objects. (Red Hat Documentation)

11. Common mistakes

  • No default cloud-credentials secret
  • wrong bucket region
  • no snapshot support for your storage class
  • assuming OADP backs up etcd; it does not
  • installing into a namespace with an overly long name can cause secret-labeling issues in some OADP cases. (Red Hat Documentation)

12. Minimal install checklist

oc create namespace openshift-adp
# install OADP Operator from OperatorHub
oc create secret generic cloud-credentials -n openshift-adp --from-file cloud=./credentials-velero
oc apply -f dpa.yaml
oc get dpa -n openshift-adp
oc apply -f backup.yaml
oc get backup -n openshift-adp

Understanding the Shared Responsibility Model in ROSA/ARO

When moving from on-premise to a Managed Service like ROSA (Red Hat OpenShift on AWS) or ARO (Azure Red Hat OpenShift), the interview shifts. The technical “heavy lifting” of managing master nodes and etcd is now handled by Red Hat and AWS/Microsoft (the SRE team).

Your role as an administrator moves from “Keeping the lights on” to “Governance, Cost Optimization, and Integration.”


1. The Shared Responsibility Model

This is the #1 question for managed services.

Q: Who is responsible for what in a ROSA/ARO environment?

  • The Provider (Red Hat/Cloud Provider): Manages the Control Plane (Masters), etcd health, patching of the underlying OS, and the core OpenShift Operators.
  • The Customer (You): Manages Worker nodes (scaling), Application lifecycle, RBAC, Network Policies, and Quotas.

Interview Tip: Mention that you no longer have cluster-admin in the traditional sense on ARO; you have a customer-admin role. You cannot SSH into master nodes or modify the etcd configuration directly.


2. Day 1: Provisioning & Connectivity

Q1: How does networking differ in ROSA/ARO compared to on-prem?

Answer: In a managed service, OpenShift is integrated into the Cloud’s Virtual Private Cloud (VPC/VNet).

  • Private vs. Public Clusters: You must decide if the API and Ingress are “Public” (accessible over the internet) or “Private” (only accessible via VPN/DirectConnect/ExpressRoute).
  • VPC Peering/Transit Gateway: You are responsible for connecting the OpenShift VPC to the rest of your cloud infrastructure (e.g., to reach a managed RDS database or Azure SQL).

Q2: What is the “Assisted Installer” vs. “Cloud CLI”?

Answer: For ROSA, you use the rosa CLI. For ARO, you use the az aro command. These tools abstract the CloudFormation or ARM templates required to spin up the infrastructure.


3. Day 2: Managed Operations

Q3: How do you handle cluster upgrades in ROSA/ARO?

Answer: You don’t just “hit update” and pray.

  • In ROSA, you can schedule upgrade windows via the OpenShift Cluster Manager (OCM).
  • The Red Hat SRE team monitors the upgrade. If it fails, they are the ones paged, not you. However, you must ensure your applications have correct Pod Disruption Budgets (PDBs) so the rolling update doesn’t take down your service.

Q4: How do you scale the cluster in the cloud?

Answer: You use MachineAutoscalers.

  • Unlike on-prem, where you are limited by physical hardware, in ROSA/ARO you define a MachineAutoscaler that monitors the cluster’s resource requests. If a pod can’t be scheduled due to lack of CPU, the autoscaler automatically provisions a new EC2/Azure VM and joins it to the cluster.

4. Cost & Security

Q5: How do you control costs in a managed OpenShift environment?

Answer: Since you pay for every worker node, I implement:

  1. Cluster Autoscaling: Scaling down to minimum nodes at night.
  2. Resource Quotas: Preventing developers from requesting 16GB of RAM for a “Hello World” app.
  3. Spot Instances: Using AWS Spot or Azure Priority instances for non-production workloads to save up to 70% on compute costs.

Q6: How do you handle Authentication?

Answer: You typically don’t use local users. You integrate OpenShift with Azure AD (Entra ID) or AWS IAM/OIDC.

  • Question: “How do pods access cloud resources (like S3 or Azure Vault)?”
  • Answer: STS (Security Token Service) or Managed Identities. This allows pods to assume a cloud role without needing to store static “Access Keys” inside a Secret.

5. Summary Comparison: On-Prem vs. Managed

FeatureOn-Prem (Bare Metal/VMware)Managed (ROSA/ARO)
Control PlaneYou manage (3 VMs/Servers)Managed by SRE (Hidden/Bundled)
UpdatesManual / High RiskScheduled / Automated
Load BalancerMetalLB / F5 / HAProxyAWS NLB/ALB or Azure LB
StorageODF / vSphere CSIEBS/EFS or Azure Disk/Files
Failure ResponseYou get paged at 3 AMRed Hat/Cloud SRE handles infra

The “Pro” Managed Question:

“If the cluster is managed by Red Hat, why do they still need an Administrator like you?”

Winning Answer: “Because while Red Hat manages the platform, I manage the consumption. I ensure the networking between our VPCs is secure, I manage the RBAC and onboarding for our developers, I optimize costs so we aren’t over-provisioning cloud resources, and I implement the CI/CD patterns that allow our apps to run reliably on that platform.”

Kong – full mini project folder

Here’s a full mini project folder for Kong that you can copy as-is.

It uses Kong Gateway in DB-less mode, so all config lives in one declarative kong.yml file. That mode is a good fit for CI/CD and Git-managed config, but the Admin API is effectively read-only for config changes in this setup. (Kong Docs)

Folder structure

kong-mini-project/
├── app/
│ ├── package.json
│ └── server.js
├── kong/
│ └── kong.yml
├── .dockerignore
├── Dockerfile
└── compose.yml

1) app/package.json

{
"name": "kong-mini-project",
"version": "1.0.0",
"description": "Node app behind Kong Gateway",
"main": "server.js",
"scripts": {
"start": "node server.js"
},
"license": "MIT"
}

2) app/server.js

const http = require("http");
const PORT = process.env.PORT || 3000;
const server = http.createServer((req, res) => {
if (req.url === "/healthz") {
res.writeHead(200, { "Content-Type": "application/json" });
return res.end(JSON.stringify({ ok: true }));
}
const body = {
ok: true,
message: "Hello from app behind Kong",
method: req.method,
url: req.url,
host: req.headers.host,
time: new Date().toISOString()
};
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify(body, null, 2));
});
server.listen(PORT, () => {
console.log(`Server listening on ${PORT}`);
});

3) Dockerfile

FROM node:20-alpine
WORKDIR /app
COPY app/package.json ./
RUN npm install --omit=dev
COPY app/server.js ./
ENV PORT=3000
EXPOSE 3000
CMD ["npm", "start"]

4) .dockerignore

node_modules
npm-debug.log
.git
.github

5) kong/kong.yml

This is the heart of the project. It defines:

  • one upstream Service
  • one public Route
  • a key-auth plugin
  • a rate-limiting plugin
  • one Consumer with an API key

Kong’s declarative config format supports entities like Services, Routes, Consumers, and Plugins in DB-less mode. The Key Auth plugin can require API keys, and the Rate Limiting plugin can throttle requests by time window such as per minute. When authentication is present, rate limiting uses the authenticated Consumer identity. (Kong Docs)

_format_version: "3.0"
services:
- name: app-service
url: http://app:3000
routes:
- name: app-route
paths:
- /api
protocols:
- http
- https
plugins:
- name: key-auth
service: app-service
config:
key_names:
- apikey
- name: rate-limiting
service: app-service
config:
minute: 5
policy: local
consumers:
- username: demo-client
keyauth_credentials:
- key: super-secret-demo-key

A note on policy: local: that works well for a single local node, but Kong notes that plugins needing shared database state do not fully function in DB-less mode, so this is best for learning or single-node setups rather than clustered distributed quotas. (Kong Docs)

6) compose.yml

Kong’s Docker docs support running Kong with Docker Compose, and the read-only Docker Compose guide for DB-less mode uses KONG_DATABASE=off plus KONG_DECLARATIVE_CONFIG pointing to the config file. (Kong Docs)

services:
kong:
image: kong:3.10
environment:
KONG_DATABASE: "off"
KONG_DECLARATIVE_CONFIG: /kong/declarative/kong.yml
KONG_PROXY_ACCESS_LOG: /dev/stdout
KONG_ADMIN_ACCESS_LOG: /dev/stdout
KONG_PROXY_ERROR_LOG: /dev/stderr
KONG_ADMIN_ERROR_LOG: /dev/stderr
KONG_ADMIN_LISTEN: 0.0.0.0:8001
ports:
- "8000:8000" # public proxy
- "8001:8001" # admin api (read-only for config in DB-less mode)
volumes:
- ./kong/kong.yml:/kong/declarative/kong.yml:ro
app:
build:
context: .
dockerfile: Dockerfile

7) Run it

docker compose up -d --build

Then test it.

Without an API key, access should fail because the route is protected by the Key Auth plugin. (Kong Docs)

curl -i http://localhost:8000/api

With the API key in a header, it should succeed. Kong’s Key Auth plugin supports reading keys from headers, query parameters, or request body, depending on config. (Kong Docs)

curl -i \
-H "apikey: super-secret-demo-key" \
http://localhost:8000/api

You can also use a query string:

curl -i "http://localhost:8000/api?apikey=super-secret-demo-key"

8) Test rate limiting

The plugin is set to 5 requests per minute, so the sixth quick request should return 429. Kong’s rate-limiting plugin supports time windows including seconds, minutes, hours, days, months, and years. (Kong Docs)

for i in {1..6}; do
curl -s -o /dev/null -w "%{http_code}\n" \
-H "apikey: super-secret-demo-key" \
http://localhost:8000/api
done

9) Useful checks

See running containers:

docker compose ps

Follow Kong logs:

docker compose logs -f kong

Follow app logs:

docker compose logs -f app

Read the service list from the Admin API:

curl http://localhost:8001/services

In DB-less mode, that Admin API is useful for inspection, but Kong’s docs say you cannot use it for normal write-based configuration management because the declarative file is the source of truth. (Kong Docs)

10) What makes this different from Traefik

With Traefik, the main workflow was “discover containers and route traffic to them.” With Kong, the model is “define Services and Routes, then attach policy plugins like auth and rate limiting.” Kong’s docs emphasize entities such as Services, Routes, Consumers, Upstreams, and Plugins as the core gateway model. (Kong Docs)

So in practice:

  • Traefik is great for app routing and reverse proxying.
  • Kong is better when you want API-specific control like identity, quotas, and policy.

11) Resume line

Built a containerized API behind Kong Gateway in DB-less mode using declarative configuration, API key authentication, and per-consumer rate limiting.

12) Best next upgrade

The strongest next step is to add JWT auth or request transformation, because those show off Kong as an API gateway rather than just a reverse proxy. Kong’s plugin ecosystem is one of its main strengths. (Kong Docs)