nixos-config/machines/renge/services/prometheus.nix

271 lines
9.1 KiB
Nix
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# SPDX-FileCopyrightText: 2021-2024 Simon Bruder <simon@sbruder.de>
#
# SPDX-License-Identifier: AGPL-3.0-or-later
{ config, lib, pkgs, ... }:
let
cfg = config.services.prometheus;
mkStaticTargets = targets: lib.singleton { inherit targets; };
mkStaticTarget = target: mkStaticTargets (lib.singleton target);
relabelVpnConfig = {
target_label = "instance";
source_labels = lib.singleton "__address__";
regex = "(.*)\\.vpn\\.sbruder\\.de:[0-9]*";
};
in
{
services.prometheus = {
enable = true;
listenAddress = "127.0.0.1";
webExternalUrl = "https://prometheus.sbruder.de";
globalConfig = {
scrape_interval = "15s";
evaluation_interval = "15s";
};
extraFlags = [
"--storage.tsdb.retention.time=90d"
"--web.enable-admin-api"
];
alertmanagers = [
{
static_configs = mkStaticTarget "${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
path_prefix = "/alertmanager/";
}
];
alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
webExternalUrl = "https://prometheus.sbruder.de/alertmanager";
configuration = {
global.resolve_timeout = "2m";
route = {
receiver = "matrix";
group_by = [ "alertname" ];
group_wait = "3m";
};
receivers = [
{
name = "matrix";
webhook_configs = lib.singleton {
url = (lib.elemAt
(lib.filter
({ ID, ... }: ID == "alertmanager_service")
config.services.go-neb.config.services)
0).Config.webhook_url;
};
}
];
};
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = mkStaticTarget "localhost:${toString cfg.port}";
}
{
job_name = "node";
static_configs = mkStaticTargets [
"fuuko.vpn.sbruder.de:9100"
"mayushii.vpn.sbruder.de:9100"
"nunotaba.vpn.sbruder.de:9100"
"renge.vpn.sbruder.de:9100"
"hitagi.vpn.sbruder.de:9100"
"vueko.vpn.sbruder.de:9100"
"okarin.vpn.sbruder.de:9100"
"shinobu.vpn.sbruder.de:9100"
"nazuna.vpn.sbruder.de:9100"
"yuzuru.vpn.sbruder.de:9100"
"koyomi.vpn.sbruder.de:9100"
];
relabel_configs = lib.singleton relabelVpnConfig;
}
{
job_name = "smartctl";
static_configs = mkStaticTargets [
"fuuko.vpn.sbruder.de:9633"
"mayushii.vpn.sbruder.de:9633"
"nunotaba.vpn.sbruder.de:9633"
"hitagi.vpn.sbruder.de:9633"
"shinobu.vpn.sbruder.de:9633"
"koyomi.vpn.sbruder.de:9633"
];
relabel_configs = lib.singleton relabelVpnConfig;
}
{
job_name = "qbittorrent";
static_configs = mkStaticTargets [
"fuuko.vpn.sbruder.de:9561"
"nazuna.vpn.sbruder.de:9561"
];
relabel_configs = lib.singleton relabelVpnConfig;
}
(
let
listenerCfg = (lib.elemAt config.services.matrix-synapse.settings.listeners 0);
in
{
job_name = "synapse";
static_configs = mkStaticTarget "${lib.elemAt listenerCfg.bind_addresses 0}:${toString listenerCfg.port}";
metrics_path = "/_synapse/metrics";
relabel_configs = lib.singleton {
target_label = "instance";
replacement = "matrix.sbruder.de";
};
}
)
{
job_name = "dnsmasq";
static_configs = mkStaticTarget "shinobu.vpn.sbruder.de:${toString config.services.prometheus.exporters.dnsmasq.port}";
relabel_configs = lib.singleton relabelVpnConfig;
}
{
job_name = "hcloud";
static_configs = mkStaticTarget config.services.hcloud_exporter.listenAddress;
}
{
job_name = "co2";
static_configs = mkStaticTarget "shinobu.vpn.sbruder.de:9672";
}
{
job_name = "rspamd";
static_configs = mkStaticTarget "vueko.vpn.sbruder.de";
metrics_path = "/rspamd/metrics";
relabel_configs = lib.singleton {
target_label = "instance";
replacement = "vueko.sbruder.de";
};
}
{
job_name = "knot";
static_configs = mkStaticTargets [
"vueko.vpn.sbruder.de:9433"
"renge.vpn.sbruder.de:9433"
"okarin.vpn.sbruder.de:9433"
"yuzuru.vpn.sbruder.de:9433"
];
relabel_configs = lib.singleton relabelVpnConfig;
}
{
job_name = "snmp";
metrics_path = "/snmp";
params = {
module = [ "if_mib" ];
};
static_configs = mkStaticTargets [
"karibik.management.shinonome-lab.de"
];
relabel_configs = [
{
source_labels = lib.singleton "__address__";
target_label = "__param_target";
}
{
source_labels = lib.singleton "__param_target";
target_label = "instance";
}
{
target_label = "__address__";
replacement = "shinobu.vpn.sbruder.de:9116";
}
];
}
{
job_name = "haproxy";
static_configs = mkStaticTargets [
"koyomi.vpn.sbruder.de:8404"
];
relabel_configs = lib.singleton relabelVpnConfig;
}
];
rules =
let
mkAlert = { name, expr, for ? "1m", description ? null }: {
alert = name;
inherit expr for;
annotations = lib.optionalAttrs (description != null) { inherit description; };
};
in
[
(lib.generators.toYAML { } {
groups = lib.singleton {
name = "alert.rules";
rules = map mkAlert [
{
name = "InstanceDown";
expr = ''up{instance!~"(nunotaba|hitagi|mayushii|fuuko)"} == 0'';
description = "Instance {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes.";
}
{
name = "SystemdUnitFailed";
expr = ''node_systemd_unit_state{state="failed"} == 1'';
description = "Systemd unit {{ $labels.name }} on {{ $labels.instance }} has state failed.";
}
{
name = "NodeHighLoad";
expr = ''sum by (instance) (node_load15) / count by (instance) (node_cpu_seconds_total{mode="system"}) > 2'';
for = "15m";
description = "Node {{ $labels.instance }} is having a per-core load 2 for the last 15 minutes.";
}
{
name = "NodeHighMemory";
expr = ''(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9'';
for = "2m";
description = "Node {{ $labels.instance }} is using more than 90% of available RAM.";
}
{
name = "TP440ACPIBroken";
expr = ''node_hwmon_temp_celsius{chip="thermal_thermal_zone0",instance="nunotaba.vpn.sbruder.de:9100",job="node",sensor="temp1"} == 48'';
for = "10m";
description = "Thinkpad T440s ACPI temperature is broken. Its reported temperature is 48°C for the last 10 minutes. That doesnt seem right. Try suspending";
}
{
name = "TorrentNoPeers";
expr = "sum by (instance) (qBittorrent_torrent_connected_leechs) == 0";
description = "qBittorrent instance {{ $labels.instance }} has no peers. There might be a network connectivity problem";
}
# <40% is to account for /boot being full (which causes ugly errors on rebuild)
{
name = "DiskFull";
expr = ''node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} / node_filesystem_size_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 0.4 and node_filesystem_free_bytes{fstype!~"ramfs|tmpfs", mountpoint!~"/nix/store"} < 4*1024^3'';
description = "Device {{ $labels.device }} on {{ $labels.instance }}:{{ $labels.mountpoint }} has less than 4GiB free space while being used over 40%";
}
];
};
})
];
};
# exporters that are not part of nixpkgs prometheus infrastructure
services.hcloud_exporter = {
enable = true;
listenAddress = "127.0.0.1:9501";
environmentFile = config.sops.secrets.hcloud_exporter-environment.path;
};
sops.secrets.hcloud_exporter-environment.sopsFile = ../secrets.yaml;
sops.secrets.prometheus-htpasswd = {
owner = "nginx";
sopsFile = ../secrets.yaml;
};
services.nginx.virtualHosts."prometheus.sbruder.de" = {
enableACME = true;
forceSSL = true;
basicAuthFile = config.sops.secrets.prometheus-htpasswd.path;
locations = {
"/".proxyPass = "http://${cfg.listenAddress}:${toString cfg.port}";
"/alertmanager/".proxyPass = "http://${cfg.alertmanager.listenAddress}:${toString cfg.alertmanager.port}";
};
};
}