What are the disaster-recovery patterns and their RTO/RPO/cost trade-offs, and why must a DR plan be tested, not just documented?

package com.bank.dr;

import java.time.Duration;

public class RecoveryObjectives {
    private final String service;
    private final Duration rto;
    private final Duration rpo;

    public RecoveryObjectives(String service, Duration rto, Duration rpo) {
        this.service = service;
        this.rto = rto;
        this.rpo = rpo;
    }

    public static void main(String[] args) {
        // Core payments: near-zero downtime, near-zero data loss
        var payments = new RecoveryObjectives("payments-ledger",
            Duration.ofMinutes(5), Duration.ofSeconds(0));

        // Monthly reports: can tolerate a day of downtime
        var reports = new RecoveryObjectives("monthly-reports",
            Duration.ofHours(24), Duration.ofHours(1));

        System.out.println(payments.service + " RTO: " + payments.rto);
        System.out.println(reports.service + " RTO: " + reports.rto);
    }
}

# Nightly backup to recovery region
aws rds create-db-snapshot --db-instance-identifier bank-accounts-db \
  --db-snapshot-identifier bank-accounts-$(date +%Y%m%d)

# Copy snapshot to recovery region
aws rds copy-db-snapshot \
  --source-db-snapshot-identifier arn:aws:rds:ap-south-1:123:snapshot:bank-accounts-20240315 \
  --target-db-snapshot-identifier bank-accounts-20240315-dr \
  --source-region ap-south-1 \
  --region ap-southeast-1

# Terraform for recovery -- only applied during a disaster
resource "aws_db_instance" "bank_accounts_dr" {
  snapshot_identifier = "bank-accounts-20240315-dr"
  instance_class      = "db.r6g.xlarge"
  # RTO: hours (provision + restore + DNS cutover)
  # RPO: up to 24 hours (nightly backup frequency)
}

# Pilot Light: database always replicating, compute off
# Primary region (ap-south-1):
resource "aws_db_instance" "bank_accounts" {
  instance_class = "db.r6g.2xlarge"
  # Full production database
}

# Recovery region (ap-southeast-1):
resource "aws_db_instance" "bank_accounts_replica" {
  replicate_source_db = aws_db_instance.bank_accounts.arn
  instance_class      = "db.r6g.xlarge"  # Can be smaller
  # Continuously replicating -- RPO near zero
}
# Compute: AMIs ready, auto-scaling groups at 0 instances
# On failover: promote replica, scale ASG to desired count

# Warm Standby: everything running, just smaller
# Primary: 5 replicas, full database
# Standby: 2 replicas, read replica, smaller instances

# Failover steps:
# 1. Promote read replica to primary (seconds)
# 2. Scale standby replicas from 2 to 5 (minutes)
# 3. Update DNS to point to standby region
# Total RTO: minutes

# Common failures found ONLY during testing:

1. Backup restore script references old bucket name  -- FAILS
2. DNS cutover requires a permission nobody has       -- BLOCKED
3. Read replica secretly stopped replicating 3 weeks ago -- DATA LOSS
4. Runbook says "call Rajesh" but Rajesh left the company -- BLOCKED
5. Recovery region has no pre-provisioned VPC/subnets -- SLOW

# None of these show up in the document. They only surface during execution.

# Game-day checklist:
# 1. Announce the test window
# 2. Execute the failover exactly as documented
# 3. Measure actual RTO and RPO against targets
# 4. Record every gap and surprise
# 5. Fix all gaps before the next game-day
# 6. Repeat quarterly

package com.bank.dr;

import java.time.Duration;

public class DrPatternAdvisor {

    public String recommend(Duration rto, Duration rpo, String criticality) {
        if (rto.toMinutes() <= 5 && rpo.toSeconds() == 0) {
            return "Multi-Site Active/Active -- near-zero RTO/RPO, highest cost";
        }
        if (rto.toMinutes() <= 30) {
            return "Warm Standby -- minutes of RTO, high cost";
        }
        if (rpo.toMinutes() <= 15) {
            return "Pilot Light -- moderate RTO, low RPO, moderate cost";
        }
        return "Backup and Restore -- hours of RTO, lowest cost";
    }

    public static void main(String[] args) {
        var advisor = new DrPatternAdvisor();
        System.out.println("Payments: " + advisor.recommend(
            Duration.ofMinutes(5), Duration.ofSeconds(0), "tier-1"));
        System.out.println("Statements: " + advisor.recommend(
            Duration.ofMinutes(30), Duration.ofMinutes(5), "tier-2"));
        System.out.println("Interest batch: " + advisor.recommend(
            Duration.ofHours(24), Duration.ofHours(1), "tier-3"));
    }
}

Payments: Multi-Site Active/Active -- near-zero RTO/RPO, highest cost
Statements: Warm Standby -- minutes of RTO, high cost
Interest batch: Backup and Restore -- hours of RTO, lowest cost

# WRONG: same DR pattern for every workload
"Everything gets active-active."
# Result: enormous cost for services that could tolerate hours of downtime

# CORRECT: match the pattern to the business criticality
Tier 1 (payments): Active/Active -- near-zero downtime justified
Tier 2 (statements): Warm Standby -- minutes acceptable
Tier 3 (batch): Backup and Restore -- hours acceptable

# WRONG: writing a DR plan and never testing it
"The document says RTO is 10 minutes."
# Actual RTO during first real incident: 4 hours (because of untested steps)

# CORRECT: quarterly game-days that measure actual RTO/RPO
Documented RTO: 10 minutes
Game-day measured RTO: 40 minutes (DNS step was manual)
Fix: automate DNS cutover
Next game-day measured RTO: 8 minutes (target met)