# Prometheus alert rules for capacity (neenv patterns/capacity): the disk that will be full in a week, and the drive
# that is failing. Facts: node_exporter's filesystem collector, and Debian's collectors (smartmon.prom, nvme.prom).
groups:
  - name: capacity
    rules:
      # A trend, not a level: three days of history projected a week ahead. Only on a filesystem already past 70%,
      # so a log burst on an empty disk does not page.
      - alert: DiskWillFillIn7Days
        expr: >-
          predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"}[3d], 7 * 86400) < 0
          and on (instance, mountpoint)
          (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"}
            / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs|ramfs"}) < 0.3
        for: 6h
        labels:
          severity: warning
          owner: platform
        annotations:
          summary: "{{ $labels.instance }} {{ $labels.mountpoint }}: full within a week at the last three days' rate"
          description: >-
            {{ $labels.mountpoint }} on {{ $labels.instance }} is past 70% and its free space, projected from the last
            three days, reaches zero within seven. Find what grows (`du -xh --max-depth=2 {{ $labels.mountpoint }}`).

      # SMART's own verdict, only where the drive offers SMART: a virtual disk reports none (healthy reads 0 there), and
      # smartmon.sh reports NVMe drives as not available too (no 'SMART support is:' line), so NVMe is the nvme rules'.
      # `type` is in the match: behind a RAID controller one /dev/sdX is several drives (megaraid,N).
      - alert: DriveSmartUnhealthy
        expr: smartmon_device_smart_healthy == 0 and on (instance, disk, type) smartmon_device_smart_available == 1
        for: 15m
        labels:
          severity: critical
          owner: platform
        annotations:
          summary: "{{ $labels.instance }} {{ $labels.disk }}: SMART says the drive is failing"
          description: >-
            The drive's own health check fails. Copy what matters off it, and read `smartctl -a {{ $labels.disk }}`.

      - alert: NvmeCriticalWarning
        expr: nvme_critical_warning > 0
        for: 15m
        labels:
          severity: critical
          owner: platform
        annotations:
          summary: "{{ $labels.instance }} {{ $labels.device }}: the NVMe drive raised a critical warning ({{ $value }})"
          description: >-
            A bit field: spare space low, temperature, reliability degraded, read-only, or volatile backup failed.
            Read `nvme smart-log /dev/{{ $labels.device }}`.

      # Wear: the drive's own estimate of its rated endurance used. 80% is time to buy its replacement.
      - alert: NvmeWearHigh
        expr: nvme_percentage_used_ratio > 0.8
        for: 1h
        labels:
          severity: warning
          owner: platform
        annotations:
          summary: "{{ $labels.instance }} {{ $labels.device }}: {{ $value | humanizePercentage }} of its rated endurance used"
          description: >-
            Plan its replacement; at 100% the drive is past its warranted writes, not dead, but no longer promised.

      # The drive facts stopped: a timer that never started (the smartmon timer starts only if /dev/sd* exists at
      # boot, so an NVMe-only machine never ran it: measured on one machine: 160 days stale) leaves old numbers served.
      - alert: DriveFactsStale
        expr: time() - max by (instance, file) (node_textfile_mtime_seconds{file=~".*/(smartmon|nvme)\\.prom"}) > 3 * 3600
        for: 30m
        labels:
          severity: warning
          owner: platform
        annotations:
          summary: "{{ $labels.instance }}: {{ $labels.file }} is {{ $value | humanizeDuration }} old"
          description: >-
            Its timer writes every 15 minutes and has not, so the drive health served is old. Check
            `systemctl status prometheus-node-exporter-smartmon.timer` (or -nvme) on {{ $labels.instance }}.

      # A facts file that never existed cannot go stale: a machine that answers its scrape and has no smartmon.prom
      # at all (a fresh rebuild whose timer never ran) is caught here.
      - alert: DriveFactsMissing
        expr: up{job=~"nodes|dev_nodes"} == 1 unless on (instance) node_textfile_mtime_seconds{file=~".*/smartmon\\.prom"}
        for: 2h
        labels:
          severity: warning
          owner: platform
        annotations:
          summary: "{{ $labels.instance }}: no drive facts at all"
          description: >-
            node_exporter on {{ $labels.instance }} serves no smartmon.prom: the timer never ran, or smartmontools is
            not installed. Check `systemctl status prometheus-node-exporter-smartmon.timer`.