SLIs, SLOs, and SLAs: Measuring Reliability

# Availability SLI: percentage of non-5xx responses
avg(
  sum(rate(http_requests_total{status!~"5.."}[5m]))
  / sum(rate(http_requests_total[5m]))
) * 100

# Latency SLI: p99 response time in seconds
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
)

# Error rate SLI: percentage of 5xx responses
avg(
  sum(rate(http_requests_total{status=~"5.."}[5m]))
  / sum(rate(http_requests_total[5m]))
) * 100

# Output: Single values representing current reliability
# Availability: 99.87%
# p99 Latency: 0.345s
# Error Rate: 0.13%

# sli_calculator.py
from datetime import datetime, timedelta
import random

class SLICalculator:
    def __init__(self, window_minutes=5):
        self.window = timedelta(minutes=window_minutes)
        self.requests = []

    def record(self, success, latency_ms):
        self.requests.append({
            "timestamp": datetime.now(),
            "success": success,
            "latency": latency_ms,
        })

    def availability(self):
        recent = [r for r in self.requests
                  if datetime.now() - r["timestamp"] < self.window]
        if not recent:
            return 100.0
        successes = sum(1 for r in recent if r["success"])
        return successes / len(recent) * 100

    def p99_latency(self):
        recent = sorted(
            [r["latency"] for r in self.requests
             if datetime.now() - r["timestamp"] < self.window]
        )
        if not recent:
            return 0
        idx = int(len(recent) * 0.99)
        return recent[min(idx, len(recent) - 1)]

sli = SLICalculator()
for i in range(1000):
    success = random.random() > 0.005  # 0.5% failure rate
    latency = random.gauss(150, 50)  # mean 150ms, stddev 50ms
    sli.record(success, latency)

print(f"Availability SLI: {sli.availability():.2f}%")
print(f"p99 Latency SLI: {sli.p99_latency():.0f}ms")

# Output:
# Availability SLI: 99.50%
# p99 Latency SLI: 298ms

# error_budget_dashboard.py
class ErrorBudgetDashboard:
    def __init__(self, sli_name, slo_pct, window_days):
        self.sli_name = sli_name
        self.slo = slo_pct
        self.window = window_days
        self.budget = 100 - slo_pct  # Total budget
        self.consumed = 0

    def record_period(self, sli_value_pct):
        error_rate = 100 - sli_value_pct
        self.consumed += error_rate
        remaining = max(0, self.budget - self.consumed)
        print(f"Period SLI: {sli_value_pct:.2f}%")
        print(f"  Error this period: {error_rate:.2f}%")
        print(f"  Budget remaining: {remaining:.2f}% (consumed {self.consumed:.2f}% of {self.budget:.2f}%)")
        return remaining

dash = ErrorBudgetDashboard("availability", 99.9, 30)
dash.record_period(99.95)  # 0.05% error
dash.record_period(99.80)  # 0.20% error
dash.record_period(99.50)  # 0.50% error - budget critical

groups:
  - name: slo_alerts
    rules:
      # Page: burning budget fast (> 14x allowed burn rate)
      - alert: SLOHighBurnRate
        expr: |
          (1 - (sum(rate(http_requests_total{status!~"5.."}[1h]))
                / sum(rate(http_requests_total[1h]))))
          * 100 > 14 * (100 - 99.9)
        for: 5m
        labels: { severity: critical }
        annotations:
          summary: "Error budget burning > 14x allowed rate"

      # Ticket: burning budget moderately (> 3x allowed burn rate)
      - alert: SLOMediumBurnRate
        expr: |
          (1 - (sum(rate(http_requests_total{status!~"5.."}[6h]))
                / sum(rate(http_requests_total[6h]))))
          * 100 > 3 * (100 - 99.9)
        for: 30m
        labels: { severity: warning }
        annotations:
          summary: "Error budget burning > 3x allowed rate"

# slo_report.py
def generate_slo_report(services):
    print("=== SLO Compliance Report ===")
    print(f"{'Service':<20} {'SLI':<15} {'SLO':<10} {'Actual':<10} {'Status':<10}")
    print("-" * 65)
    for s in services:
        compliant = s["actual"] >= s["slo"]
        status = "PASS" if compliant else "FAIL"
        print(f"{s['name']:<20} {s['sli']:<15} {s['slo']:<10} "
              f"{s['actual']:<10} {status:<10}")

services = [
    {"name": "api-gateway", "sli": "Availability", "slo": 99.95, "actual": 99.97},
    {"name": "api-gateway", "sli": "p99 Latency", "slo": 500, "actual": 320},
    {"name": "user-db", "sli": "Availability", "slo": 99.99, "actual": 99.98},
    {"name": "cdn", "sli": "Availability", "slo": 99.9, "actual": 99.95},
]
generate_slo_report(services)

# slo_dashboard_generator.py
import json

def generate_grafana_panel(title, expr, slo_target, unit="percent"):
    return {
        "title": title,
        "type": "graph",
        "targets": [{"expr": expr, "legendFormat": "{{ instance }}"}],
        "thresholds": [
            {"value": slo_target - 0.1, "color": "green", "op": "gt"},
            {"value": slo_target, "color": "yellow", "op": "gt"},
        ],
    }

dashboard = {
    "title": "SLO Overview",
    "panels": [
        generate_grafana_panel("Availability",
            'avg(sum(rate(http_requests_total{status!~"5.."}[5m])) '
            '/ sum(rate(http_requests_total[5m]))) * 100', 99.9),
        generate_grafana_panel("p99 Latency",
            'histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))',
            0.5, "seconds"),
    ]
}
print(json.dumps(dashboard, indent=2))