5.4.2 SLO 驱动的告警体系 · 错误预算 + OpenTelemetry 标准化

# Prometheus Recording Rule: error_budget_tracking.yaml
# 错误预算跟踪规则 —— 月度 + 周度双窗口

groups:
  - name: error_budget_tracking
    interval: 1m
    rules:
      # SLI: 请求成功率(过去 28 天窗口)
      - record: sli:request_success_rate:5m
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[5m]))
          /
          sum(rate(http_requests_total{job="api"}[5m]))

      # 当前月度错误率(过去 30 天)
      - record: slo:error_rate_current:30d
        expr: |
          1 - (
            sum(rate(http_requests_total{job="api",status!~"5.."}[30d]))
            /
            sum(rate(http_requests_total{job="api"}[30d]))
          )

      # 月度预算剩余比例 (0~1)
      # 公式: 1 - (当前错误率 / 允许错误率)
      - record: slo:error_budget_remaining:30d
        expr: |
          1 - (
            slo:error_rate_current:30d
            /
            (1 - 0.999)   # 允许错误率 0.1%
          )

      # 周度预算消耗速度(过去 7 天)
      - record: slo:error_budget_burn_rate_weekly:7d
        expr: |
          (
            1 - (
              sum(rate(http_requests_total{job="api",status!~"5.."}[7d]))
              /
              sum(rate(http_requests_total{job="api"}[7d]))
            )
          )
          /
          (1 - 0.999)    # 归一化到 30 天等价值

# alertmanager_freeze_release.yaml
# 预算耗尽 → 冻结发布

groups:
  - name: slo_budget_freeze
    rules:
      - alert: SLO_BudgetExhausted_FreezeRelease
        expr: slo:error_budget_remaining:30d <= 0
        for: 10m
        labels:
          severity: critical
          slo_violation: "true"
          action: freeze_release
          pager: pageduty-oncall
        annotations:
          summary: "月度 Error Budget 已耗尽,自动冻结发布"
          description: |
            当前 SLO 错误率已超过月度预算(0.1%)。
            剩余预算: {{ $value | humanizePercentage }}
            建议措施:
              1. 停止所有非紧急发布
              2. 启动可靠性改进专项 (reliability sprint)
              3. 复盘本月所有故障,识别根因
          runbook_url: "https://wiki.internal/runbooks/slo-budget-freeze"

# burn_rate_calculation.yaml
# Burn Rate 计算 + 多档告警阈值

groups:
  - name: burn_rate_alerts
    interval: 30s
    rules:
      # 基础 SLI 计算(过去 1 小时窗口)
      - record: sli:http_request_success:1h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[1h]))
          /
          sum(rate(http_requests_total{job="api"}[1h]))

      # 当前错误率
      - record: error_rate:1h
        expr: 1 - sli:http_request_success:1h

      # Burn Rate 1h 窗口(BR = error_rate / budget)
      - record: slo:burn_rate:1h
        expr: error_rate:1h / (1 - 0.999)

      # Burn Rate 5min 窗口(快速检测)
      - record: slo:burn_rate:5m
        expr: |
          (
            1 - (
              sum(rate(http_requests_total{job="api",status!~"5.."}[5m]))
              /
              sum(rate(http_requests_total{job="api"}[5m]))
            )
          )
          /
          (1 - 0.999)

      # === 告警规则 ===

      # 快速烧:1h 窗口 Burn Rate > 14.4x(2 天耗尽预算)
      - alert: SLO_FastBurn_1h
        expr: slo:burn_rate:1h > 14.4
        for: 2m
        labels:
          severity: critical
          window: 1h
          burn_rate: "14.4x"
        annotations:
          summary: "SLO 快速烧:1h 窗口 Burn Rate {{ $value }}x"
          description: "若不处理,2 天内耗尽月度 Error Budget"

      # 慢速烧:6h 窗口 Burn Rate > 6x(5 天耗尽预算)
      - alert: SLO_SlowBurn_6h
        expr: slo:burn_rate:6h > 6
        for: 5m
        labels:
          severity: warning
          window: 6h
          burn_rate: "6x"
        annotations:
          summary: "SLO 慢速烧:6h 窗口 Burn Rate {{ $value }}x"

# google_mwm_4_buckets.yaml
# Google SRE Workbook 推荐 4 档告警模板

groups:
  - name: slo_multi_window_multi_burn_rate
    interval: 30s
    rules:

      # ===== 通用 SLI 计算函数 =====
      - record: slo:sli_5m
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[5m]))
          /
          sum(rate(http_requests_total{job="api"}[5m]))
      - record: slo:sli_30m
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[30m]))
          /
          sum(rate(http_requests_total{job="api"}[30m]))
      - record: slo:sli_2h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[2h]))
          /
          sum(rate(http_requests_total{job="api"}[2h]))
      - record: slo:sli_6h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[6h]))
          /
          sum(rate(http_requests_total{job="api"}[6h]))
      - record: slo:sli_1h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[1h]))
          /
          sum(rate(http_requests_total{job="api"}[1h]))
      - record: slo:sli_24h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[24h]))
          /
          sum(rate(http_requests_total{job="api"}[24h]))
      - record: slo:sli_72h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[72h]))
          /
          sum(rate(http_requests_total{job="api"}[72h]))

      # ===== 档位 1: Page 级(最快响应) =====
      # 长窗口 1h BR=14.4x + 短窗口 5min BR=14.4x
      - alert: SLO_HighBurn_Page_1h_5m
        expr: (1 - slo:sli_1h) > (14.4 * (1 - 0.999)) and (1 - slo:sli_5m) > (14.4 * (1 - 0.999))
        for: 2m
        labels:
          severity: critical
          slo: "true"
          paging: "page"
          burn_rate: "14.4x"
          window_combo: "1h+5m"
        annotations:
          summary: "Page 级告警:BR 14.4x,1h+5m 双窗口触发"
          description: "若不处理,2 天内耗尽预算。请立即响应。"

      # ===== 档位 2: Page 级(中等速度) =====
      # 长窗口 6h BR=6x + 短窗口 30min BR=6x
      - alert: SLO_HighBurn_Page_6h_30m
        expr: (1 - slo:sli_6h) > (6 * (1 - 0.999)) and (1 - slo:sli_30m) > (6 * (1 - 0.999))
        for: 5m
        labels:
          severity: critical
          slo: "true"
          paging: "page"
          burn_rate: "6x"
          window_combo: "6h+30m"
        annotations:
          summary: "Page 级告警:BR 6x,6h+30m 双窗口触发"
          description: "若不处理,5 天内耗尽预算。请立即响应。"

      # ===== 档位 3: Ticket 级(慢速烧) =====
      # 长窗口 24h BR=3x + 短窗口 2h BR=3x
      - alert: SLO_MediumBurn_Ticket_24h_2h
        expr: (1 - slo:sli_24h) > (3 * (1 - 0.999)) and (1 - slo:sli_2h) > (3 * (1 - 0.999))
        for: 30m
        labels:
          severity: warning
          slo: "true"
          paging: "ticket"
          burn_rate: "3x"
          window_combo: "24h+2h"
        annotations:
          summary: "Ticket 级告警:BR 3x,24h+2h 双窗口触发"
          description: "若不处理,10 天内耗尽预算。请当天响应。"

      # ===== 档位 4: Ticket 级(最慢速烧) =====
      # 长窗口 72h BR=1x + 短窗口 6h BR=1x
      - alert: SLO_SlowBurn_Ticket_72h_6h
        expr: (1 - slo:sli_72h) > (1 * (1 - 0.999)) and (1 - slo:sli_6h) > (1 * (1 - 0.999))
        for: 6h
        labels:
          severity: info
          slo: "true"
          paging: "ticket"
          burn_rate: "1x"
          window_combo: "72h+6h"
        annotations:
          summary: "Ticket 级告警:BR 1x,72h+6h 双窗口触发"
          description: "若不处理,30 天耗尽预算。请本周内响应。"

# otel_alerting_rule.yaml
# OpenTelemetry 标准化告警规则

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: otel-slo-alerts
  namespace: observability
  labels:
    app.kubernetes.io/part-of: opentelemetry
    slo.tier: critical
spec:
  groups:
    - name: otel_slo_alerts
      interval: 30s
      rules:

        # 基于 OTel Metric: HTTP 服务 SLI
        - alert: OTel_HighErrorRate_TraceBased
          expr: |
            (
              sum(rate(otel_http_server_requests_total{code=~"5.."}[5m]))
              /
              sum(rate(otel_http_server_requests_total[5m]))
            )
            > 0.01
          for: 5m
          labels:
            severity: critical
            source: opentelemetry
            signal: metric
            service: "{{ $labels.service_name }}"
          annotations:
            summary: "OTel 检测到服务错误率超 1% [{{ $labels.service_name }}]"
            trace_query: |
              { service.name="{{ $labels.service_name }}"
                status=error }

        # 基于 OTel Trace Span 错误
        - alert: OTel_TraceSpanErrors
          expr: |
            sum(rate(otel_traces_spanmetrics_errors_total[5m])) by (service_name)
            > 0.05
          for: 5m
          labels:
            severity: warning
            source: opentelemetry
            signal: trace
          annotations:
            summary: "OTel Trace Span 错误率超 5% [{{ $labels.service_name }}]"

        # 基于 OTel Log 模式匹配
        - alert: OTel_LogErrorBurst
          expr: |
            sum(rate(otel_logs_severity_number_total{severity="ERROR"}[5m])) by (service_name)
            > 10
          for: 2m
          labels:
            severity: critical
            source: opentelemetry
            signal: log
          annotations:
            summary: "OTel 检测到 ERROR 日志暴增 [{{ $labels.service_name }}]"

# 错误的写法(删除)
- alert: CPU_High
  expr: cpu_usage > 80
  for: 5m

# 正确的写法
- alert: SLO_BurnRate_High
  expr: slo:burn_rate:1h > 14.4 and slo:burn_rate:5m > 14.4
  for: 2m

# alertmanager_route.yaml
route:
  receiver: 'default-slack'
  group_by: ['alertname', 'service']
  routes:
    - matchers:
        - severity = "critical"
      receiver: 'pagerduty-critical'
      group_wait: 10s
      repeat_interval: 5m
    - matchers:
        - severity = "warning"
      receiver: 'slack-warnings'
      group_wait: 1m
      repeat_interval: 1h

# alertmanager_grouping.yaml
route:
  receiver: 'default'
  group_by: ['service', 'alertname']   # 关键:按服务+告警名聚合
  group_wait: 30s                      # 等待 30s 收集同组告警
  group_interval: 5m                   # 同组告警 5 分钟内不重发
  repeat_interval: 4h
  routes:
    - matchers: [severity = "critical"]
      receiver: 'pagerduty'

inhibit_rules:
  # 父告警触发时,抑制子告警
  - source_matchers: [alertname = "ServiceDown"]
    target_matchers: [severity = "warning"]
    equal: ['service']

# slo_budget_freeze_release.yaml
# Error Budget 耗尽 → 自动冻结发布(集成 CI/CD)

apiVersion: v1
kind: ConfigMap
metadata:
  name: release-freeze-policy
data:
  policy.yaml: |
    rules:
      - name: error_budget_exhausted
        condition: slo:error_budget_remaining:30d <= 0
        action:
          type: block_release
          channels: [slack-sre, email-product]
          message: "SLO 月度预算耗尽,所有非紧急发布自动冻结"
          ci_gate: "block_non_emergency_deploy"

# 14d_window_alert_template.yaml
# Google SRE Workbook 推荐 14 天窗口 4 档告警模板
groups:
  - name: slo_14d_4buckets
    interval: 30s
    rules:

      # === SLI 基础 ===
      - record: slo:sli:5m
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[5m]))
          /
          sum(rate(http_requests_total{job="api"}[5m]))
      - record: slo:sli:30m
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[30m]))
          /
          sum(rate(http_requests_total{job="api"}[30m]))
      - record: slo:sli:2h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[2h]))
          /
          sum(rate(http_requests_total{job="api"}[2h]))
      - record: slo:sli:6h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[6h]))
          /
          sum(rate(http_requests_total{job="api"}[6h]))
      - record: slo:sli:1h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[1h]))
          /
          sum(rate(http_requests_total{job="api"}[1h]))
      - record: slo:sli:24h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[24h]))
          /
          sum(rate(http_requests_total{job="api"}[24h]))
      - record: slo:sli:72h
        expr: |
          sum(rate(http_requests_total{job="api",status!~"5.."}[72h]))
          /
          sum(rate(http_requests_total{job="api"}[72h]))

      # === 4 档告警(基于 14 天窗口,假设 SLO=99.9%) ===
      # 档位 1: BR=14.4x → 2 天耗尽 → Page
      - alert: PageBurn_14.4x_1h_5m
        expr: (1 - slo:sli:1h) > (14.4 * 0.001) and (1 - slo:sli:5m) > (14.4 * 0.001)
        for: 2m
        labels: {severity: critical, paging: page}
      # 档位 2: BR=6x → 5 天耗尽 → Page
      - alert: PageBurn_6x_6h_30m
        expr: (1 - slo:sli:6h) > (6 * 0.001) and (1 - slo:sli:30m) > (6 * 0.001)
        for: 5m
        labels: {severity: critical, paging: page}
      # 档位 3: BR=3x → 10 天耗尽 → Ticket
      - alert: TicketBurn_3x_24h_2h
        expr: (1 - slo:sli:24h) > (3 * 0.001) and (1 - slo:sli:2h) > (3 * 0.001)
        for: 30m
        labels: {severity: warning, paging: ticket}
      # 档位 4: BR=1x → 30 天耗尽 → Ticket
      - alert: TicketBurn_1x_72h_6h
        expr: (1 - slo:sli:72h) > (1 * 0.001) and (1 - slo:sli:6h) > (1 * 0.001)
        for: 6h
        labels: {severity: info, paging: ticket}