Cloud Disaster Recovery: Backup, Failover, and Business Continuity

# Automated S3 cross-region backup script
#!/bin/bash
SOURCE_BUCKET="dodatech-prod-us-east-1"
DEST_BUCKET="dodatech-dr-us-west-2"
DATE=$(date +%Y-%m-%d)

# Sync data to DR region (incremental)
aws s3 sync s3://$SOURCE_BUCKET s3://$DEST_BUCKET/$DATE \
  --source-region us-east-1 \
  --region us-west-2

# RDS snapshot copy to DR region
aws rds copy-db-snapshot \
  --source-db-snapshot-identifier prod-db-snapshot-$DATE \
  --target-db-snapshot-identifier prod-db-snapshot-$DATE-dr \
  --source-region us-east-1 \
  --region us-west-2

echo "Backup complete: $(date)"

# pilot-light-dr.tf
# Minimal DR infrastructure — scale up on failover
resource "aws_instance" "pilot_db" {
  provider = aws.west
  ami           = data.aws_ami.ubuntu.id
  instance_type = "db.r5.large"  # Replica of prod database
}

resource "aws_instance" "pilot_app" {
  provider = aws.west
  ami           = data.aws_ami.ubuntu.id
  instance_type = "t3.micro"  # Minimal — scale on failover
  user_data     = <<-EOF
    #!/bin/bash
    # Pre-warm with read replica of production database
    # On failover: scale up and switch to primary
  EOF
}

# warm-standby.tf
resource "aws_autoscaling_group" "app_dr" {
  provider = aws.west
  name     = "app-warm-standby"
  min_size = 2       # Running and ready
  max_size = 20
  desired_capacity = 2  # Scaled-down, ready to burst
}

resource "aws_rds_cluster_instance" "db_dr" {
  provider           = aws.west
  count              = 1  # Single reader in DR, scale on failover
  cluster_identifier = aws_rds_cluster.prod_dr.id
  instance_class     = "db.r5.large"
  engine             = "aurora-postgresql"
}

# Route 53 — active-active DNS routing
resource "aws_route53_record" "app" {
  zone_id = data.aws_route53_zone.main.zone_id
  name    = "app.dodatech.com"
  type    = "A"

  alias {
    name                   = aws_lb.app_us_east.dns_name
    zone_id                = aws_lb.app_us_east.zone_id
    evaluate_target_health = true
  }

  failover_routing_policy {
    type = "PRIMARY"
  }

  set_identifier = "us-east-1"
}

resource "aws_route53_record" "app_dr" {
  zone_id = data.aws_route53_zone.main.zone_id
  name    = "app.dodatech.com"
  type    = "A"

  alias {
    name                   = aws_lb.app_us_west.dns_name
    zone_id                = aws_lb.app_us_west.zone_id
    evaluate_target_health = true
  }

  failover_routing_policy {
    type = "SECONDARY"
  }

  set_identifier = "us-west-2"
}

# RDS cross-region read replica
aws rds create-db-instance-read-replica \
  --db-instance-identifier prod-db-dr \
  --source-db-instance-identifier prod-db \
  --source-region us-east-1 \
  --region us-west-2 \
  --db-instance-class db.r5.large

# Verify replication lag
aws rds describe-db-instances \
  --db-instance-identifier prod-db-dr \
  --region us-west-2 \
  --query 'DBInstances[0].ReadReplicaSourceDBInstanceIdentifier'

# Output:
# {
#     "ReplicaLag": "0.25"
# }

# dr_test_runner.py
import subprocess
import json
import time

class DRTest:
    def __init__(self, primary_region, dr_region):
        self.primary = primary_region
        self.dr = dr_region
        self.steps = []

    def run_step(self, name, command):
        print(f"  → {name}...", end=" ", flush=True)
        result = subprocess.run(command, shell=True, capture_output=True, text=True)
        success = result.returncode == 0
        self.steps.append({"name": name, "success": success, "output": result.stdout[:200]})
        print("PASS" if success else "FAIL")
        return success

    def simulate_failure(self):
        print("\n=== DR Failover Test ===")
        self.run_step("Block primary region traffic",
                      "aws ec2 revoke-security-group-ingress ...")
        self.run_step("Verify health check fails",
                      "aws route53 get-health-check-status --health-check-id ...")
        self.run_step("Promote DR database to primary",
                      "aws rds promote-read-replica --db-instance-identifier prod-db-dr")
        self.run_step("Verify DR application health",
                      "curl -f https://dr.app.dodatech.com/health")
        self.run_step("Verify DNS failover",
                      "dig +short app.dodatech.com @8.8.8.8")

test = DRTest("us-east-1", "us-west-2")
test.simulate_failure()
print(f"\n{'='*40}\n{'All tests passed!' if all(s['success'] for s in test.steps) else 'Some tests failed!'}")

# DR workspace
terraform workspace new dr-us-west-2
terraform workspace select dr-us-west-2

# Apply DR infrastructure
terraform apply -var-file=dr.tfvars

# Output:
# Apply complete! Resources: 12 added, 0 changed, 0 destroyed.
# DR infrastructure ready in us-west-2

# dr_dashboard.py
class DRDashboard:
    def __init__(self):
        self.regions = []

    def add_service(self, name, primary, dr, replication_lag_s=0):
        self.regions.append({
            "name": name, "primary": primary, "dr": dr,
            "lag": replication_lag_s, "healthy": True,
        })

    def report(self):
        print("=== DR Status Dashboard ===")
        for svc in self.regions:
            status = "✅" if svc["healthy"] else "❌"
            print(f"  {status} {svc['name']:<20} Primary: {svc['primary']:<12} "
                  f"DR: {svc['dr']:<12} Lag: {svc['lag']}s")
        print(f"\n  Overall: {'ALL HEALTHY' if all(s['healthy'] for s in self.regions) else 'ISSUES DETECTED'}")

dash = DRDashboard()
dash.add_service("API", "us-east-1", "us-west-2", 0.5)
dash.add_service("Database", "us-east-1", "us-west-2", 0.3)
dash.add_service("Storage", "us-east-1", "us-west-2", 0)
dash.report()