2024-01-06 01:19:35 +01:00
|
|
|
|
# SPDX-FileCopyrightText: 2021-2024 Simon Bruder <simon@sbruder.de>
|
|
|
|
|
#
|
|
|
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
|
|
2021-02-24 20:55:07 +01:00
|
|
|
|
{ config, lib, pkgs, ... }:
|
|
|
|
|
let
|
|
|
|
|
cfg = config.services.prometheus;
|
|
|
|
|
|
|
|
|
|
mkStaticTargets = targets: lib.singleton { inherit targets; };
|
|
|
|
|
mkStaticTarget = target: mkStaticTargets (lib.singleton target);
|
2024-08-20 12:15:39 +02:00
|
|
|
|
|
|
|
|
|
relabelVpnConfig = {
|
|
|
|
|
target_label = "instance";
|
|
|
|
|
source_labels = lib.singleton "__address__";
|
|
|
|
|
regex = "(.*)\\.vpn\\.sbruder\\.de:[0-9]*";
|
|
|
|
|
};
|
2021-02-24 20:55:07 +01:00
|
|
|
|
in
|
|
|
|
|
{
|
|
|
|
|
services.prometheus = {
|
|
|
|
|
enable = true;
|
|
|
|
|
listenAddress = "127.0.0.1";
|
2021-04-02 16:44:17 +02:00
|
|
|
|
webExternalUrl = "https://prometheus.sbruder.de";
|
2021-02-24 20:55:07 +01:00
|
|
|
|
globalConfig = {
|
|
|
|
|
scrape_interval = "15s";
|
|
|
|
|
evaluation_interval = "15s";
|
|
|
|
|
};
|
2021-03-28 11:04:25 +02:00
|
|
|
|
extraFlags = [
|
|
|
|
|
"--storage.tsdb.retention.time=90d"
|
2021-03-28 11:04:48 +02:00
|
|
|
|
"--web.enable-admin-api"
|
2021-03-28 11:04:25 +02:00
|
|
|
|
];
|
2021-02-24 20:55:07 +01:00
|
|
|
|
|
|
|
|
|
alertmanagers = [
|
2021-04-02 16:44:17 +02:00
|
|
|
|
{
|
|
|
|
|
static_configs = mkStaticTarget "${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
|
|
|
|
|
path_prefix = "/alertmanager/";
|
|
|
|
|
}
|
2021-02-24 20:55:07 +01:00
|
|
|
|
];
|
|
|
|
|
alertmanager = {
|
|
|
|
|
enable = true;
|
|
|
|
|
listenAddress = "127.0.0.1";
|
2021-04-02 16:44:17 +02:00
|
|
|
|
webExternalUrl = "https://prometheus.sbruder.de/alertmanager";
|
2021-02-24 20:55:07 +01:00
|
|
|
|
configuration = {
|
|
|
|
|
global.resolve_timeout = "2m";
|
|
|
|
|
|
|
|
|
|
route = {
|
|
|
|
|
receiver = "matrix";
|
|
|
|
|
group_by = [ "alertname" ];
|
|
|
|
|
group_wait = "3m";
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
receivers = [
|
|
|
|
|
{
|
|
|
|
|
name = "matrix";
|
2021-04-02 17:46:07 +02:00
|
|
|
|
webhook_configs = lib.singleton {
|
|
|
|
|
url = (lib.elemAt
|
|
|
|
|
(lib.filter
|
|
|
|
|
({ ID, ... }: ID == "alertmanager_service")
|
|
|
|
|
config.services.go-neb.config.services)
|
|
|
|
|
0).Config.webhook_url;
|
|
|
|
|
};
|
2021-02-24 20:55:07 +01:00
|
|
|
|
}
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
scrapeConfigs = [
|
|
|
|
|
{
|
|
|
|
|
job_name = "prometheus";
|
|
|
|
|
static_configs = mkStaticTarget "localhost:${toString cfg.port}";
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
job_name = "node";
|
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"fuuko.vpn.sbruder.de:9100"
|
2021-09-30 07:32:03 +02:00
|
|
|
|
"mayushii.vpn.sbruder.de:9100"
|
2022-06-09 17:38:24 +02:00
|
|
|
|
"nunotaba.vpn.sbruder.de:9100"
|
2022-03-23 15:03:08 +01:00
|
|
|
|
"renge.vpn.sbruder.de:9100"
|
2022-12-30 19:52:58 +01:00
|
|
|
|
"hitagi.vpn.sbruder.de:9100"
|
2021-02-24 20:55:07 +01:00
|
|
|
|
"vueko.vpn.sbruder.de:9100"
|
2023-05-06 10:54:03 +02:00
|
|
|
|
"okarin.vpn.sbruder.de:9100"
|
2023-07-01 12:37:12 +02:00
|
|
|
|
"shinobu.vpn.sbruder.de:9100"
|
2023-10-04 15:15:54 +02:00
|
|
|
|
"nazuna.vpn.sbruder.de:9100"
|
2024-01-02 23:26:46 +01:00
|
|
|
|
"yuzuru.vpn.sbruder.de:9100"
|
2024-05-11 21:14:17 +02:00
|
|
|
|
"koyomi.vpn.sbruder.de:9100"
|
2021-02-24 20:55:07 +01:00
|
|
|
|
];
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2021-02-24 20:55:07 +01:00
|
|
|
|
}
|
2024-05-25 16:13:03 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "smartctl";
|
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"fuuko.vpn.sbruder.de:9633"
|
|
|
|
|
"mayushii.vpn.sbruder.de:9633"
|
|
|
|
|
"nunotaba.vpn.sbruder.de:9633"
|
|
|
|
|
"hitagi.vpn.sbruder.de:9633"
|
|
|
|
|
"shinobu.vpn.sbruder.de:9633"
|
|
|
|
|
"koyomi.vpn.sbruder.de:9633"
|
|
|
|
|
];
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2024-05-25 16:13:03 +02:00
|
|
|
|
}
|
2021-02-19 18:35:44 +01:00
|
|
|
|
{
|
2022-03-18 23:25:05 +01:00
|
|
|
|
job_name = "qbittorrent";
|
2022-03-19 10:17:25 +01:00
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"fuuko.vpn.sbruder.de:9561"
|
2023-10-04 17:02:16 +02:00
|
|
|
|
"nazuna.vpn.sbruder.de:9561"
|
2022-03-19 10:17:25 +01:00
|
|
|
|
];
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2021-02-19 18:35:44 +01:00
|
|
|
|
}
|
2021-03-18 13:01:59 +01:00
|
|
|
|
(
|
|
|
|
|
let
|
2022-05-31 14:43:51 +02:00
|
|
|
|
listenerCfg = (lib.elemAt config.services.matrix-synapse.settings.listeners 0);
|
2021-03-18 13:01:59 +01:00
|
|
|
|
in
|
|
|
|
|
{
|
|
|
|
|
job_name = "synapse";
|
2022-05-31 14:43:51 +02:00
|
|
|
|
static_configs = mkStaticTarget "${lib.elemAt listenerCfg.bind_addresses 0}:${toString listenerCfg.port}";
|
2021-03-18 13:01:59 +01:00
|
|
|
|
metrics_path = "/_synapse/metrics";
|
|
|
|
|
relabel_configs = lib.singleton {
|
|
|
|
|
target_label = "instance";
|
|
|
|
|
replacement = "matrix.sbruder.de";
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
)
|
2023-04-02 01:14:01 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "dnsmasq";
|
2023-07-01 12:37:12 +02:00
|
|
|
|
static_configs = mkStaticTarget "shinobu.vpn.sbruder.de:${toString config.services.prometheus.exporters.dnsmasq.port}";
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2023-04-02 01:14:01 +02:00
|
|
|
|
}
|
2021-08-28 13:53:38 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "hcloud";
|
|
|
|
|
static_configs = mkStaticTarget config.services.hcloud_exporter.listenAddress;
|
|
|
|
|
}
|
2022-11-03 16:40:05 +01:00
|
|
|
|
{
|
|
|
|
|
job_name = "co2";
|
2023-07-01 13:14:32 +02:00
|
|
|
|
static_configs = mkStaticTarget "shinobu.vpn.sbruder.de:9672";
|
2022-11-03 16:40:05 +01:00
|
|
|
|
}
|
2023-04-30 13:09:43 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "rspamd";
|
|
|
|
|
static_configs = mkStaticTarget "vueko.vpn.sbruder.de";
|
|
|
|
|
metrics_path = "/rspamd/metrics";
|
|
|
|
|
relabel_configs = lib.singleton {
|
|
|
|
|
target_label = "instance";
|
|
|
|
|
replacement = "vueko.sbruder.de";
|
|
|
|
|
};
|
|
|
|
|
}
|
2023-10-26 01:18:17 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "knot";
|
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"vueko.vpn.sbruder.de:9433"
|
2024-02-24 13:22:17 +01:00
|
|
|
|
"renge.vpn.sbruder.de:9433"
|
|
|
|
|
"okarin.vpn.sbruder.de:9433"
|
|
|
|
|
"yuzuru.vpn.sbruder.de:9433"
|
2023-10-26 01:18:17 +02:00
|
|
|
|
];
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2023-10-26 01:18:17 +02:00
|
|
|
|
}
|
2023-11-04 13:57:59 +01:00
|
|
|
|
{
|
|
|
|
|
job_name = "snmp";
|
|
|
|
|
metrics_path = "/snmp";
|
|
|
|
|
params = {
|
|
|
|
|
module = [ "if_mib" ];
|
|
|
|
|
};
|
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"karibik.management.shinonome-lab.de"
|
|
|
|
|
];
|
|
|
|
|
relabel_configs = [
|
|
|
|
|
{
|
|
|
|
|
source_labels = lib.singleton "__address__";
|
|
|
|
|
target_label = "__param_target";
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
source_labels = lib.singleton "__param_target";
|
|
|
|
|
target_label = "instance";
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
target_label = "__address__";
|
|
|
|
|
replacement = "shinobu.vpn.sbruder.de:9116";
|
|
|
|
|
}
|
|
|
|
|
];
|
|
|
|
|
}
|
2024-05-18 15:24:41 +02:00
|
|
|
|
{
|
|
|
|
|
job_name = "haproxy";
|
|
|
|
|
static_configs = mkStaticTargets [
|
|
|
|
|
"koyomi.vpn.sbruder.de:8404"
|
|
|
|
|
];
|
2024-08-20 12:15:39 +02:00
|
|
|
|
relabel_configs = lib.singleton relabelVpnConfig;
|
2024-05-18 15:24:41 +02:00
|
|
|
|
}
|
2021-02-24 20:55:07 +01:00
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
rules =
|
|
|
|
|
let
|
|
|
|
|
mkAlert = { name, expr, for ? "1m", description ? null }: {
|
|
|
|
|
alert = name;
|
|
|
|
|
inherit expr for;
|
|
|
|
|
annotations = lib.optionalAttrs (description != null) { inherit description; };
|
|
|
|
|
};
|
|
|
|
|
in
|
|
|
|
|
[
|
|
|
|
|
(lib.generators.toYAML { } {
|
|
|
|
|
groups = lib.singleton {
|
|
|
|
|
name = "alert.rules";
|
|
|
|
|
rules = map mkAlert [
|
|
|
|
|
{
|
|
|
|
|
name = "InstanceDown";
|
2023-12-25 23:37:18 +01:00
|
|
|
|
expr = ''up{instance!~"(nunotaba|hitagi|mayushii|fuuko)"} == 0'';
|
2021-02-24 20:55:07 +01:00
|
|
|
|
description = "Instance {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes.";
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
name = "SystemdUnitFailed";
|
|
|
|
|
expr = ''node_systemd_unit_state{state="failed"} == 1'';
|
|
|
|
|
description = "Systemd unit {{ $labels.name }} on {{ $labels.instance }} has state failed.";
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
name = "NodeHighLoad";
|
|
|
|
|
expr = ''sum by (instance) (node_load15) / count by (instance) (node_cpu_seconds_total{mode="system"}) > 2'';
|
|
|
|
|
for = "15m";
|
2021-04-04 14:34:44 +02:00
|
|
|
|
description = "Node {{ $labels.instance }} is having a per-core load ≥ 2 for the last 15 minutes.";
|
2021-02-24 20:55:07 +01:00
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
name = "NodeHighMemory";
|
|
|
|
|
expr = ''(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9'';
|
|
|
|
|
for = "2m";
|
2021-04-04 14:34:44 +02:00
|
|
|
|
description = "Node {{ $labels.instance }} is using more than 90 % of available RAM.";
|
2021-02-24 20:55:07 +01:00
|
|
|
|
}
|
2022-06-09 17:38:24 +02:00
|
|
|
|
{
|
|
|
|
|
name = "TP440ACPIBroken";
|
|
|
|
|
expr = ''node_hwmon_temp_celsius{chip="thermal_thermal_zone0",instance="nunotaba.vpn.sbruder.de:9100",job="node",sensor="temp1"} == 48'';
|
|
|
|
|
for = "10m";
|
|
|
|
|
description = "Thinkpad T440’s ACPI temperature is broken. Its reported temperature is 48 °C for the last 10 minutes. That doesn’t seem right. Try suspending";
|
|
|
|
|
}
|
2021-02-19 18:35:44 +01:00
|
|
|
|
{
|
|
|
|
|
name = "TorrentNoPeers";
|
2022-03-18 23:25:05 +01:00
|
|
|
|
expr = "sum by (instance) (qBittorrent_torrent_connected_leechs) == 0";
|
|
|
|
|
description = "qBittorrent instance {{ $labels.instance }} has no peers. There might be a network connectivity problem";
|
2021-02-19 18:35:44 +01:00
|
|
|
|
}
|
2023-08-02 23:22:19 +02:00
|
|
|
|
# <40% is to account for /boot being full (which causes ugly errors on rebuild)
|
|
|
|
|
{
|
|
|
|
|
name = "DiskFull";
|
|
|
|
|
expr = ''node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} / node_filesystem_size_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 0.4 and node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 4*1024^3'';
|
|
|
|
|
description = "Device {{ $labels.device }} on {{ $labels.instance }}:{{ $labels.mountpoint }} has less than 4GiB free space while being used over 40%";
|
|
|
|
|
}
|
2021-02-24 20:55:07 +01:00
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
];
|
|
|
|
|
};
|
|
|
|
|
|
2021-08-28 13:53:38 +02:00
|
|
|
|
# exporters that are not part of nixpkgs’ prometheus infrastructure
|
|
|
|
|
services.hcloud_exporter = {
|
|
|
|
|
enable = true;
|
|
|
|
|
listenAddress = "127.0.0.1:9501";
|
|
|
|
|
environmentFile = config.sops.secrets.hcloud_exporter-environment.path;
|
|
|
|
|
};
|
|
|
|
|
sops.secrets.hcloud_exporter-environment.sopsFile = ../secrets.yaml;
|
|
|
|
|
|
2021-03-01 15:27:18 +01:00
|
|
|
|
sops.secrets.prometheus-htpasswd = {
|
|
|
|
|
owner = "nginx";
|
|
|
|
|
sopsFile = ../secrets.yaml;
|
2021-02-24 20:55:07 +01:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
services.nginx.virtualHosts."prometheus.sbruder.de" = {
|
|
|
|
|
enableACME = true;
|
|
|
|
|
forceSSL = true;
|
|
|
|
|
|
2021-03-01 15:27:18 +01:00
|
|
|
|
basicAuthFile = config.sops.secrets.prometheus-htpasswd.path;
|
2021-02-24 20:55:07 +01:00
|
|
|
|
|
|
|
|
|
locations = {
|
|
|
|
|
"/".proxyPass = "http://${cfg.listenAddress}:${toString cfg.port}";
|
|
|
|
|
|
2021-04-02 16:44:17 +02:00
|
|
|
|
"/alertmanager/".proxyPass = "http://${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
|
2021-02-24 20:55:07 +01:00
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
}
|