##############################################################
# GMLM Platform — Prometheus Alert Rules
# Alerts fire when something crosses a threshold and
# sends notifications via Alertmanager → Slack/email.
##############################################################

groups:

  # ── Application Alerts ─────────────────────────────────────
  - name: gmlm_application
    interval: 30s
    rules:

      - alert: HighResponseTime
        expr: |
          histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
          team:     platform
        annotations:
          summary:     "High P95 response time on {{ $labels.instance }}"
          description: "P95 response time is {{ $value | humanizeDuration }} (threshold: 2s)"
          runbook:     "https://docs.gmlm.internal/runbooks/high-response-time"

      - alert: CriticalResponseTime
        expr: |
          histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) > 5
        for: 3m
        labels:
          severity: critical
          team:     platform
        annotations:
          summary:     "Critical P99 response time on {{ $labels.instance }}"
          description: "P99 response time is {{ $value | humanizeDuration }}"

      - alert: HighErrorRate
        expr: |
          rate(http_requests_total{status=~"5.."}[5m]) /
          rate(http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
          team:     platform
        annotations:
          summary:     "High HTTP 5xx error rate: {{ $value | humanizePercentage }}"
          description: "5xx error rate exceeds 5% for 5 minutes"

      - alert: ApplicationDown
        expr: up{job="gmlm_app"} == 0
        for: 1m
        labels:
          severity: critical
          team:     platform
          page:     "true"  # Triggers PagerDuty escalation
        annotations:
          summary:     "GMLM application is down"
          description: "Application health check failing on {{ $labels.instance }}"

  # ── Queue Alerts ────────────────────────────────────────────
  - name: gmlm_queues
    rules:

      - alert: QueueBacklogHigh
        expr: gmlm_queue_pending_jobs{queue="high"} > 5000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High priority queue backlog: {{ $value }} jobs"
          description: "The 'high' queue has {{ $value }} pending jobs for >10 minutes"

      - alert: FailedJobsAccumulating
        expr: increase(gmlm_failed_jobs_total[1h]) > 50
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: "{{ $value }} jobs failed in the last hour"

      - alert: HorizonInactive
        expr: gmlm_horizon_status != 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Horizon queue supervisor is not running"

  # ── Database Alerts ─────────────────────────────────────────
  - name: gmlm_database
    rules:

      - alert: MySQLDown
        expr: up{job="mysql"} == 0
        for: 1m
        labels:
          severity: critical
          page: "true"
        annotations:
          summary: "MySQL database is unreachable"

      - alert: MySQLHighConnections
        expr: mysql_global_status_threads_connected / mysql_global_variables_max_connections > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MySQL connection pool {{ $value | humanizePercentage }} full"

      - alert: MySQLSlowQueryRate
        expr: rate(mysql_global_status_slow_queries[5m]) > 5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "{{ $value }} slow queries/second in MySQL"

      - alert: MySQLReplicationLag
        expr: mysql_slave_status_seconds_behind_master > 30
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "MySQL replica lag: {{ $value }}s behind primary"

  # ── Redis Alerts ────────────────────────────────────────────
  - name: gmlm_redis
    rules:

      - alert: RedisDown
        expr: up{job="redis"} == 0
        for: 1m
        labels:
          severity: critical
          page: "true"
        annotations:
          summary: "Redis is unreachable"

      - alert: RedisMemoryHigh
        expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Redis memory {{ $value | humanizePercentage }} full (evictions may occur)"

      - alert: RedisCacheHitRateLow
        expr: |
          rate(redis_keyspace_hits_total[5m]) /
          (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 0.7
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Redis cache hit rate dropped to {{ $value | humanizePercentage }}"

  # ── Infrastructure Alerts ───────────────────────────────────
  - name: gmlm_infrastructure
    rules:

      - alert: DiskSpaceLow
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.15
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disk space is {{ $value | humanizePercentage }} available on {{ $labels.instance }}"

      - alert: DiskSpaceCritical
        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) < 0.05
        for: 1m
        labels:
          severity: critical
          page: "true"
        annotations:
          summary: "CRITICAL: Disk space below 5% on {{ $labels.instance }}"

      - alert: HighCPULoad
        expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CPU usage {{ $value | humanize }}% on {{ $labels.instance }}"

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Memory usage {{ $value | humanizePercentage }} on {{ $labels.instance }}"

      - alert: SSLCertificateExpiring
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 14
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "SSL certificate expires in {{ $value | humanizeDuration }}"

      - alert: BackupNotRunning
        expr: time() - gmlm_last_backup_timestamp > 86400 * 2
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "No backup detected in the last 48 hours"
