Chaos Engineering: Break Things on Purpose to Build Resilience

# chaos_experiment.py
import random
import time

class ChaosExperiment:
    def __init__(self, name, hypothesis, metric_name, steady_threshold):
        self.name = name
        self.hypothesis = hypothesis
        self.metric_name = metric_name
        self.steady_threshold = steady_threshold
        self.baseline = None
        self.results = []

    def measure(self, label, value):
        self.results.append({"label": label, "value": value, "time": time.time()})

    def run(self, fault_fn):
        print(f"=== Experiment: {self.name} ===")
        print(f"Hypothesis: {self.hypothesis}")

        # Measure baseline
        baseline_val = 0.2  # Simulated baseline latency
        self.measure("baseline", baseline_val)
        print(f"Baseline {self.metric_name}: {baseline_val}s")

        # Inject fault
        print("Injecting fault: killing API server ...")
        fault_fn()

        # Measure during fault
        time.sleep(1)
        during_val = random.uniform(0.15, 0.8)
        self.measure("during_fault", during_val)
        print(f"During fault {self.metric_name}: {during_val:.3f}s")

        # Recover
        time.sleep(1)
        after_val = random.uniform(0.15, 0.3)
        self.measure("after_recovery", after_val)
        print(f"After recovery {self.metric_name}: {after_val:.3f}s")

        passed = during_val < self.steady_threshold
        print(f"Result: {'PASSED' if passed else 'FAILED'}")
        print(f"Threshold: {self.metric_name} < {self.steady_threshold}s\n")
        return passed

experiment = ChaosExperiment(
    "API Server Failure",
    "Killing one API server should not increase p99 latency above 500ms",
    "p99_latency", 0.5
)

def kill_api_server():
    pass  # In real practice: `kubectl delete pod api-server-xyz`

experiment.run(kill_api_server)

# Output:
# === Experiment: API Server Failure ===
# Hypothesis: Killing one API server should not increase p99 latency above 500ms
# Baseline p99_latency: 0.2s
# Injecting fault: killing API server ...
# During fault p99_latency: 0.345s
# After recovery p99_latency: 0.234s
# Result: PASSED

# chaos-monkey.yml
# Spinnaker Chaos Monkey configuration
chaosMonkey:
  enabled: true
  minTimeSeconds: 60
  maxTimeSeconds: 600
  probability: 0.1  # 10% chance per check
  group:
    stack: prod
    detail: api
    regions:
      - us-east-1
      - us-west-2
  exceptions:
    - account: prod
      stack: redis  # Don't kill Redis

# Install Gremlin agent
curl -sSL https://app.gremlin.com/install/agent.sh | sudo bash

# Run a CPU attack for 30 seconds
gremlin attack cpu -l 80 -d 30

# Run a network latency attack
gremlin attack latency -h target-service.internal -p 8080 -l 200 -d 60

# Run a blackhole attack (drop all packets to a destination)
gremlin attack blackhole -h database.internal

# Output:
# Attack sent. ID: 1234-5678
# Monitoring: https://app.gremlin.com/attacks/1234-5678

# litmus-experiment.yaml
apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: pod-delete-engine
spec:
  appinfo:
    appns: default
    applabel: "app=api-server"
    appkind: deployment
  chaosServiceAccount: litmus-admin
  experiments:
    - name: pod-delete
      spec:
        components:
          env:
            - name: TOTAL_CHAOS_DURATION
              value: "30"
            - name: CHAOS_INTERVAL
              value: "10"
            - name: FORCE
              value: "true"
        probe:
          - name: check-api-health
            type: httpProbe
            httpProbe/inputs:
              url: http://api-service:8080/health
              expectedResponseCode: "200"
            mode: Continuous

# Run Litmus experiment
kubectl apply -f litmus-experiment.yaml

# Output:
# chaosengine.litmuschaos.io/pod-delete-engine created
# Experiment pod-delete is running...
# Experiment pod-delete completed successfully
# Check results: kubectl describe chaosresult pod-delete-engine-pod-delete

# blast_radius.py
class BlastRadius:
    def __init__(self, min_replicas=3, max_failure_pct=20):
        self.min_replicas = min_replicas
        self.max_failure_pct = max_failure_pct

    def validate(self, deployment, target_count):
        total = deployment["replicas"]
        if total < self.min_replicas:
            return False, f"Too few replicas ({total} < {self.min_replicas})"
        if target_count / total * 100 > self.max_failure_pct:
            return False, f"Blast radius too large ({target_count}/{total})"
        return True, "Safe to proceed"

deploy = {"name": "api-server", "replicas": 5}
bl = BlastRadius()

for target in [1, 3]:
    safe, msg = bl.validate(deploy, target)
    print(f"Kill {target} of {deploy['replicas']} pods: {msg}")

# .github/workflows/chaos-pipeline.yml
name: Weekly Chaos Experiment
on:
  schedule:
    - cron: '0 10 * * 1'  # Every Monday at 10am

jobs:
  chaos:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run Litmus experiment
        run: |
          kubectl apply -f experiments/pod-delete.yaml
          sleep 60
          kubectl describe chaosresult pod-delete-engine-pod-delete
      - name: Verify system health
        run: |
          ERROR_RATE=$(curl -s http://api.example.com/metrics | grep error_rate)
          if echo "$ERROR_RATE" | grep -q "> 1"; then
            echo "Experiment FAILED: error rate exceeded threshold"
            exit 1
          fi
          echo "Experiment PASSED: system remained healthy"