Monitoring & Alerting: Metrics to Action Flow

package main

import (
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

// Define metrics
var (
    // Counter - only goes up
    httpRequestsTotal = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests",
        },
        []string{"method", "endpoint", "status"},
    )

    // Histogram - for request durations
    httpRequestDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "HTTP request duration in seconds",
            Buckets: prometheus.DefBuckets, // 0.005, 0.01, 0.025, 0.05, ...
        },
        []string{"method", "endpoint"},
    )

    // Gauge - current value (can go up or down)
    activeConnections = prometheus.NewGauge(
        prometheus.GaugeOpts{
            Name: "active_connections",
            Help: "Number of active connections",
        },
    )
)

func init() {
    // Register metrics with Prometheus
    prometheus.MustRegister(httpRequestsTotal)
    prometheus.MustRegister(httpRequestDuration)
    prometheus.MustRegister(activeConnections)
}

func trackMetrics(method, endpoint string, statusCode int, duration time.Duration) {
    // Increment request counter
    httpRequestsTotal.WithLabelValues(
        method,
        endpoint,
        fmt.Sprintf("%d", statusCode),
    ).Inc()

    // Record request duration
    httpRequestDuration.WithLabelValues(
        method,
        endpoint,
    ).Observe(duration.Seconds())
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
    start := time.Now()

    // Increment active connections
    activeConnections.Inc()
    defer activeConnections.Dec()

    // Your application logic here
    processRequest(w, r)

    // Track metrics
    duration := time.Since(start)
    trackMetrics(r.Method, r.URL.Path, http.StatusOK, duration)
}

func main() {
    // Expose /metrics endpoint for Prometheus
    http.Handle("/metrics", promhttp.Handler())

    // Application endpoints
    http.HandleFunc("/api/users", handleRequest)

    http.ListenAndServe(":8080", nil)
}

# prometheus-rules.yaml
groups:
- name: application_alerts
  interval: 60s  # Evaluate every 60 seconds

  rules:
  # High Error Rate Alert
  - alert: HighErrorRate
    expr: |
      (
        rate(http_requests_total{status=~"5.."}[5m])
        /
        rate(http_requests_total[5m])
      ) > 0.05      
    for: 5m  # Must be true for 5 minutes before firing
    labels:
      severity: critical
      team: backend
    annotations:
      summary: "High error rate on {{ $labels.instance }}"
      description: "Error rate is {{ $value | humanizePercentage }} (threshold: 5%)"
      dashboard: "https://grafana.example.com/d/app"

  # High Latency Alert
  - alert: HighLatency
    expr: |
      histogram_quantile(0.95,
        rate(http_request_duration_seconds_bucket[5m])
      ) > 0.5      
    for: 10m
    labels:
      severity: warning
      team: backend
    annotations:
      summary: "High latency on {{ $labels.instance }}"
      description: "P95 latency is {{ $value }}s (threshold: 0.5s)"

  # Service Down Alert
  - alert: ServiceDown
    expr: up{job="myapp"} == 0
    for: 1m
    labels:
      severity: critical
      team: sre
    annotations:
      summary: "Service {{ $labels.instance }} is down"
      description: "Cannot scrape metrics from {{ $labels.instance }}"

  # Memory Usage Alert
  - alert: HighMemoryUsage
    expr: |
      (
        container_memory_usage_bytes{pod=~"myapp-.*"}
        /
        container_spec_memory_limit_bytes{pod=~"myapp-.*"}
      ) > 0.90      
    for: 5m
    labels:
      severity: warning
      team: platform
    annotations:
      summary: "High memory usage on {{ $labels.pod }}"
      description: "Memory usage is {{ $value | humanizePercentage }} of limit"

# alertmanager.yaml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/XXX'
  pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'

# Inhibition rules - suppress alerts when higher priority alert is firing
inhibit_rules:
  # If node is down, don't alert on pods on that node
  - source_match:
      alertname: 'NodeDown'
    target_match:
      alertname: 'PodDown'
    equal: ['node']

  # If entire cluster is down, don't alert on individual services
  - source_match:
      severity: 'critical'
      alertname: 'ClusterDown'
    target_match_re:
      severity: 'warning|info'
    equal: ['cluster']

# Route tree - how to send alerts
route:
  receiver: 'default-email'
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s        # Wait 30s to collect more alerts
  group_interval: 5m     # Send updates every 5m for grouped alerts
  repeat_interval: 4h    # Resend if still firing after 4h

  routes:
  # Critical alerts to PagerDuty
  - match:
      severity: critical
    receiver: 'pagerduty-critical'
    group_wait: 10s      # Page quickly for critical
    continue: true       # Also send to Slack

  - match:
      severity: critical
    receiver: 'slack-critical'

  # Warning alerts to Slack
  - match:
      severity: warning
    receiver: 'slack-warnings'
    group_wait: 1m

  # Team-specific routing
  - match:
      team: backend
    receiver: 'backend-team'

  - match:
      team: frontend
    receiver: 'frontend-team'

# Receivers - where to send alerts
receivers:
- name: 'default-email'
  email_configs:
  - to: '[email protected]'
    headers:
      Subject: '{{ .GroupLabels.alertname }}: {{ .Status | toUpper }}'

- name: 'pagerduty-critical'
  pagerduty_configs:
  - service_key: 'your-pagerduty-key'
    description: '{{ .GroupLabels.alertname }}: {{ .CommonAnnotations.summary }}'
    severity: 'critical'

- name: 'slack-critical'
  slack_configs:
  - channel: '#alerts-critical'
    title: '🚨 CRITICAL: {{ .GroupLabels.alertname }}'
    text: |
      {{ range .Alerts }}
      *Alert:* {{ .Annotations.summary }}
      *Description:* {{ .Annotations.description }}
      *Severity:* {{ .Labels.severity }}
      *Dashboard:* {{ .Annotations.dashboard }}
      {{ end }}      
    color: 'danger'
    send_resolved: true

- name: 'slack-warnings'
  slack_configs:
  - channel: '#alerts-warning'
    title: '⚠️  WARNING: {{ .GroupLabels.alertname }}'
    text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
    color: 'warning'

- name: 'backend-team'
  slack_configs:
  - channel: '#backend-alerts'

□ Is this symptom-based or user-impacting?
□ Does it require immediate action?
□ Can the on-call engineer fix it?
□ Is the threshold appropriate?
□ Is the 'for' duration set correctly?
□ Does it include dashboard/runbook links?
□ Is the severity level correct?
□ Will this wake someone at 3 AM? Should it?
□ Have we tested it?
□ Is there a plan to reduce these over time?