From 25ff150b3d2f3deadb0598bdfe9ad28fd83e0f67 Mon Sep 17 00:00:00 2001 From: Simon Bruder Date: Wed, 2 Aug 2023 23:22:19 +0200 Subject: [PATCH] prometheus: Add disk full alert --- machines/renge/services/prometheus.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/machines/renge/services/prometheus.nix b/machines/renge/services/prometheus.nix index b054194..8f91fa4 100644 --- a/machines/renge/services/prometheus.nix +++ b/machines/renge/services/prometheus.nix @@ -169,6 +169,12 @@ in expr = "sum by (instance) (qBittorrent_torrent_connected_leechs) == 0"; description = "qBittorrent instance {{ $labels.instance }} has no peers. There might be a network connectivity problem"; } + # <40% is to account for /boot being full (which causes ugly errors on rebuild) + { + name = "DiskFull"; + expr = ''node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} / node_filesystem_size_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 0.4 and node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 4*1024^3''; + description = "Device {{ $labels.device }} on {{ $labels.instance }}:{{ $labels.mountpoint }} has less than 4GiB free space while being used over 40%"; + } ]; }; })