# Alertmanager Configuration # Production-ready template with multi-channel routing # Docs: https://prometheus.io/docs/alerting/latest/configuration/ global: # Default SMTP settings (configure as needed) # smtp_smarthost: 'smtp.example.com:587' # smtp_from: 'alertmanager@example.com' # smtp_auth_username: 'alertmanager' # smtp_auth_password: '{{ .SMTP_PASSWORD }}' # smtp_require_tls: true # Slack API URL (set via environment or secrets) # slack_api_url: '{{ .SLACK_WEBHOOK_URL }}' # PagerDuty routing key # pagerduty_url: 'https://events.pagerduty.com/v2/enqueue' # Default resolve timeout resolve_timeout: 5m # Inhibition rules - suppress lower severity alerts when higher severity fires inhibit_rules: # If critical fires, suppress warning for same alertname + instance - source_matchers: - severity = critical target_matchers: - severity = warning equal: ['alertname', 'instance'] # If cluster-wide alert fires, suppress instance-level alerts - source_matchers: - severity = critical - scope = cluster target_matchers: - severity =~ warning|critical - scope != cluster equal: ['alertname'] # Route tree route: # Default receiver receiver: 'default-receiver' # Group alerts by these labels group_by: ['alertname', 'severity', 'instance'] # Wait before sending initial notification group_wait: 30s # Wait before sending updated notifications group_interval: 5m # Minimum wait before resending repeat_interval: 4h # Child routes (evaluated in order, first match wins) routes: # Critical alerts -> immediate paging - matchers: - severity = critical receiver: 'pagerduty-critical' group_wait: 10s repeat_interval: 1h continue: true # Also send to Slack # Warning alerts -> Slack - matchers: - severity = warning receiver: 'slack-warnings' group_wait: 1m repeat_interval: 4h # Info alerts -> low priority channel - matchers: - severity = info receiver: 'slack-info' group_wait: 5m repeat_interval: 24h # Watchdog/Deadman switch -> dedicated receiver - matchers: - alertname = Watchdog receiver: 'null' repeat_interval: 5m # Receivers receivers: # Null receiver (discard alerts) - name: 'null' # Default fallback - name: 'default-receiver' # Uncomment and configure your preferred channel: # slack_configs: # - channel: '#alerts' # send_resolved: true # title: '{{ template "slack.default.title" . }}' # text: '{{ template "slack.default.text" . }}' # Critical alerts -> PagerDuty - name: 'pagerduty-critical' # pagerduty_configs: # - routing_key: '{{ .PAGERDUTY_ROUTING_KEY }}' # severity: critical # description: '{{ .CommonAnnotations.summary }}' # details: # firing: '{{ template "pagerduty.default.instances" .Alerts.Firing }}' # resolved: '{{ template "pagerduty.default.instances" .Alerts.Resolved }}' # Warnings -> Slack - name: 'slack-warnings' # slack_configs: # - channel: '#alerts-warnings' # send_resolved: true # color: '{{ if eq .Status "firing" }}warning{{ else }}good{{ end }}' # title: '{{ template "slack.default.title" . }}' # text: '{{ template "slack.default.text" . }}' # actions: # - type: button # text: 'Runbook :book:' # url: '{{ (index .Alerts 0).Annotations.runbook_url }}' # - type: button # text: 'Silence :mute:' # url: '{{ template "slack.default.silence_url" . }}' # Info -> Slack (low priority) - name: 'slack-info' # slack_configs: # - channel: '#alerts-info' # send_resolved: false # title: 'ℹ️ {{ .CommonLabels.alertname }}' # text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' # Email receiver (optional) - name: 'email-oncall' # email_configs: # - to: 'oncall@example.com' # send_resolved: true # headers: # Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}' # Webhook receiver (for custom integrations) - name: 'webhook-custom' # webhook_configs: # - url: 'http://alerthandler.internal/webhook' # send_resolved: true # http_config: # bearer_token: '{{ .WEBHOOK_TOKEN }}' # Templates # templates: # - '/etc/alertmanager/templates/*.tmpl'