monitoring-stack/alertmanager/alertmanager.yml

# Alertmanager Configuration
# Production-ready template with multi-channel routing
# Docs: https://prometheus.io/docs/alerting/latest/configuration/

global:
  # Default SMTP settings (configure as needed)
  # smtp_smarthost: 'smtp.example.com:587'
  # smtp_from: 'alertmanager@example.com'
  # smtp_auth_username: 'alertmanager'
  # smtp_auth_password: '{{ .SMTP_PASSWORD }}'
  # smtp_require_tls: true

  # Slack API URL (set via environment or secrets)
  # slack_api_url: '{{ .SLACK_WEBHOOK_URL }}'

  # PagerDuty routing key
  # pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'

  # Default resolve timeout
  resolve_timeout: 5m

# Inhibition rules - suppress lower severity alerts when higher severity fires
inhibit_rules:
  # If critical fires, suppress warning for same alertname + instance
  - source_matchers:
      - severity = critical
    target_matchers:
      - severity = warning
    equal: ['alertname', 'instance']

  # If cluster-wide alert fires, suppress instance-level alerts
  - source_matchers:
      - severity = critical
      - scope = cluster
    target_matchers:
      - severity =~ warning|critical
      - scope != cluster
    equal: ['alertname']

# Route tree
route:
  # Default receiver
  receiver: 'default-receiver'

  # Group alerts by these labels
  group_by: ['alertname', 'severity', 'instance']

  # Wait before sending initial notification
  group_wait: 30s

  # Wait before sending updated notifications
  group_interval: 5m

  # Minimum wait before resending
  repeat_interval: 4h

  # Child routes (evaluated in order, first match wins)
  routes:
    # Critical alerts -> immediate paging
    - matchers:
        - severity = critical
      receiver: 'pagerduty-critical'
      group_wait: 10s
      repeat_interval: 1h
      continue: true  # Also send to Slack

    # Warning alerts -> Slack
    - matchers:
        - severity = warning
      receiver: 'slack-warnings'
      group_wait: 1m
      repeat_interval: 4h

    # Info alerts -> low priority channel
    - matchers:
        - severity = info
      receiver: 'slack-info'
      group_wait: 5m
      repeat_interval: 24h

    # Watchdog/Deadman switch -> dedicated receiver
    - matchers:
        - alertname = Watchdog
      receiver: 'null'
      repeat_interval: 5m

# Receivers
receivers:
  # Null receiver (discard alerts)
  - name: 'null'

  # Default fallback
  - name: 'default-receiver'
    # Uncomment and configure your preferred channel:
    # slack_configs:
    #   - channel: '#alerts'
    #     send_resolved: true
    #     title: '{{ template "slack.default.title" . }}'
    #     text: '{{ template "slack.default.text" . }}'

  # Critical alerts -> PagerDuty
  - name: 'pagerduty-critical'
    # pagerduty_configs:
    #   - routing_key: '{{ .PAGERDUTY_ROUTING_KEY }}'
    #     severity: critical
    #     description: '{{ .CommonAnnotations.summary }}'
    #     details:
    #       firing: '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
    #       resolved: '{{ template "pagerduty.default.instances" .Alerts.Resolved }}'

  # Warnings -> Slack
  - name: 'slack-warnings'
    # slack_configs:
    #   - channel: '#alerts-warnings'
    #     send_resolved: true
    #     color: '{{ if eq .Status "firing" }}warning{{ else }}good{{ end }}'
    #     title: '{{ template "slack.default.title" . }}'
    #     text: '{{ template "slack.default.text" . }}'
    #     actions:
    #       - type: button
    #         text: 'Runbook :book:'
    #         url: '{{ (index .Alerts 0).Annotations.runbook_url }}'
    #       - type: button
    #         text: 'Silence :mute:'
    #         url: '{{ template "slack.default.silence_url" . }}'

  # Info -> Slack (low priority)
  - name: 'slack-info'
    # slack_configs:
    #   - channel: '#alerts-info'
    #     send_resolved: false
    #     title: 'ℹ️ {{ .CommonLabels.alertname }}'
    #     text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  # Email receiver (optional)
  - name: 'email-oncall'
    # email_configs:
    #   - to: 'oncall@example.com'
    #     send_resolved: true
    #     headers:
    #       Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'

  # Webhook receiver (for custom integrations)
  - name: 'webhook-custom'
    # webhook_configs:
    #   - url: 'http://alerthandler.internal/webhook'
    #     send_resolved: true
    #     http_config:
    #       bearer_token: '{{ .WEBHOOK_TOKEN }}'

# Templates
# templates:
#   - '/etc/alertmanager/templates/*.tmpl'