Uptime Monitoring - Alertmanager ile Kesinti Bildirimi Otomasyonu

version: "3.8"
services:
  prometheus:
    image: prom/prometheus:v2.50.0
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alert-rules.yml:/etc/prometheus/alert-rules.yml
    ports:
      - "9090:9090"
    networks:
      - monitoring

  blackbox-exporter:
    image: prom/blackbox-exporter:v0.25.0
    volumes:
      - ./blackbox.yml:/etc/blackbox_exporter/config.yml
    ports:
      - "9115:9115"
    networks:
      - monitoring

  alertmanager:
    image: prom/alertmanager:v0.27.0
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
    ports:
      - "9093:9093"
    networks:
      - monitoring

networks:
  monitoring:
    driver: bridge

modules:
  http_2xx:
    prober: http
    timeout: 10s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 201, 301, 302]
      follow_redirects: true
      preferred_ip_protocol: ip4
      tls_config:
        insecure_skip_verify: false

  http_post_api:
    prober: http
    timeout: 10s
    http:
      method: POST
      headers:
        Content-Type: application/json
      body: '{"health": "check"}'
      valid_status_codes: [200]

  tcp_connect:
    prober: tcp
    timeout: 5s

  dns_check:
    prober: dns
    timeout: 5s
    dns:
      query_name: hosted.cloud
      query_type: A
      transport_protocol: udp

  ssl_expiry:
    prober: http
    timeout: 10s
    http:
      valid_status_codes: [200, 301, 302]
      tls_config:
        insecure_skip_verify: false

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - alert-rules.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  - job_name: "blackbox-http"
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
          - https://hosted.cloud
          - https://api.hosted.cloud/health
          - https://admin.hosted.cloud
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  - job_name: "blackbox-tcp"
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
          - db-server:5432
          - redis-server:6379
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

groups:
  - name: uptime-alerts
    rules:
      # Endpoint tamamen erisilemez
      - alert: EndpointDown
        expr: probe_success == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Endpoint {{ $labels.instance }} down"
          description: "{{ $labels.instance }} 2 dakikadir erisilemez."

      # Yavas yanit suresi (2 saniyeden fazla)
      - alert: HighResponseTime
        expr: probe_http_duration_seconds{phase="transfer"} > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Slow response: {{ $labels.instance }}"
          description: "{{ $labels.instance }} yanit suresi 5 dakikadir 2 saniyenin uzerinde."

      # SSL sertifikasi 14 gun icinde sona erecek
      - alert: SSLCertExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 14
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "SSL cert expiring: {{ $labels.instance }}"
          description: "{{ $labels.instance }} SSL sertifikasi {{ $value | humanize }} gun icinde sona erecek."

      # HTTP durum kodu 5xx
      - alert: HTTPStatusError
        expr: probe_http_status_code >= 500
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "HTTP 5xx: {{ $labels.instance }}"
          description: "{{ $labels.instance }} HTTP {{ $value }} durum kodu donuyor."

global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/T.../B.../xxx'

route:
  receiver: slack-default
  group_by: [alertname, instance]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    # Critical alert'ler PagerDuty'ye
    - match:
        severity: critical
      receiver: pagerduty-critical
      repeat_interval: 1h
      continue: true

    # Tum alert'ler Slack'e de gitsin
    - match_re:
        severity: critical|warning
      receiver: slack-alerts

receivers:
  - name: slack-default
    slack_configs:
      - channel: '#monitoring'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  - name: slack-alerts
    slack_configs:
      - channel: '#alerts-critical'
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'
        title: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}\n{{ end }}'

  - name: pagerduty-critical
    pagerduty_configs:
      - service_key: 'your-pagerduty-integration-key'
        severity: critical

inhibit_rules:
  # EndpointDown varsa HighResponseTime'i bastir
  - source_match:
      alertname: EndpointDown
    target_match:
      alertname: HighResponseTime
    equal: [instance]