nixos-config/machines/fuuko/services/prometheus.nix

150 lines
4.6 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{ config, lib, pkgs, ... }:
let
cfg = config.services.prometheus;
mkStaticTargets = targets: lib.singleton { inherit targets; };
mkStaticTarget = target: mkStaticTargets (lib.singleton target);
in
{
systemd.mounts = lib.singleton {
after = [ "data.mount" ];
before = [ "prometheus.service" ];
wantedBy = [ "local-fs.target" ];
what = "/data/prometheus";
where = config.systemd.services.prometheus.serviceConfig.WorkingDirectory;
type = "none";
options = "bind";
};
services.prometheus = {
enable = true;
listenAddress = "127.0.0.1";
globalConfig = {
scrape_interval = "15s";
evaluation_interval = "15s";
};
alertmanagers = [
{ static_configs = mkStaticTarget "${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}"; }
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
configuration = {
global.resolve_timeout = "2m";
route = {
receiver = "matrix";
group_by = [ "alertname" ];
group_wait = "3m";
};
receivers = [
{
name = "matrix";
webhook_configs = [
# FIXME: Add correct URL
{ url = "http://matrix-alertmanager:3000/alerts"; }
];
}
];
};
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = mkStaticTarget "localhost:${toString cfg.port}";
}
{
job_name = "node";
static_configs = mkStaticTargets [
"fuuko.vpn.sbruder.de:9100"
"issei.vpn.sbruder.de:9100"
"nunotaba.vpn.sbruder.de:9100"
"sayuri.vpn.sbruder.de:9100"
"vueko.vpn.sbruder.de:9100"
];
}
{
job_name = "fritzbox";
static_configs = mkStaticTarget "127.0.0.1:9133";
}
];
rules =
let
mkAlert = { name, expr, for ? "1m", description ? null }: {
alert = name;
inherit expr for;
annotations = lib.optionalAttrs (description != null) { inherit description; };
};
in
[
(lib.generators.toYAML { } {
groups = lib.singleton {
name = "alert.rules";
rules = map mkAlert [
{
name = "InstanceDown";
expr = ''up{instance!~"(nunotaba|sayuri).vpn.sbruder.de:.*"} == 0'';
description = "Instance {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes.";
}
{
name = "SystemdUnitFailed";
expr = ''node_systemd_unit_state{state="failed"} == 1'';
description = "Systemd unit {{ $labels.name }} on {{ $labels.instance }} has state failed.";
}
{
name = "NodeHighLoad";
expr = ''sum by (instance) (node_load15) / count by (instance) (node_cpu_seconds_total{mode="system"}) > 2'';
for = "15m";
description = "This node is having a per-core load 2 for the last 15 minutes.";
}
{
name = "NodeHighMemory";
expr = ''(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9'';
for = "2m";
description = "This node is using more than 90% of available RAM.";
}
{
name = "TP440ACPIBroken";
expr = ''node_hwmon_temp_celsius{chip="thermal_thermal_zone0",instance="nunotaba.vpn.sbruder.de:9100",job="node",sensor="temp1"} == 48'';
for = "10m";
description = "Thinkpad T440s ACPI temperature is broken. Its reported temperature is 48°C for the last 10 minutes. That doesnt seem right. Try suspending";
}
];
};
})
];
exporters = {
fritzbox = {
enable = true;
gatewayAddress = "192.168.100.1";
listenAddress = "127.0.0.1";
};
};
};
# get rid of “could not call action: authorization required” every scrape
systemd.services.prometheus-fritzbox-exporter.serviceConfig.StandardOutput = "null";
krops.secrets.prometheus-htpasswd = {
group = "nginx";
};
services.nginx.virtualHosts."prometheus.sbruder.de" = {
enableACME = true;
forceSSL = true;
basicAuthFile = config.krops.secrets.prometheus-htpasswd.path;
locations = {
"/".proxyPass = "http://${cfg.listenAddress}:${toString cfg.port}";
"/alertmanager/".proxyPass = "http://${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}/";
};
};
}