nixos-config/systems/LoutreOS/monitoring.nix

{ config, lib, pkgs, ... }:

let
  domaine = "nyanlout.re";
in
{
  services = {
    smartd = {
      enable = true;
      defaults.monitored = "-a -o on -s (S/../.././02|L/../15/./02)";
      notifications.mail = {
        enable = true;
        recipient = "paul@nyanlout.re";
      };
    };

    influxdb = {
      enable = true;
      dataDir = "/var/db/influxdb";
    };

    telegraf = {
      enable = true;
      extraConfig = {
        inputs = {
          zfs = { poolMetrics = true; };
          net = { interfaces = [ "eno1" "eno2" "eno3" "eno4" ]; };
          netstat = {};
          cpu = { totalcpu = true; };
          kernel = {};
          mem = {};
          processes = {};
          system = {};
          disk = {};
          cgroup = [
            {
              paths = [
                "/sys/fs/cgroup/system.slice/*"
              ];
              files = ["memory.current" "cpu.stat"];
            }
          ];
          ipmi_sensor = { path = "${pkgs.ipmitool}/bin/ipmitool"; };
          smart = {
            path = "${pkgs.writeShellScriptBin "smartctl" "/run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl $@"}/bin/smartctl";
          };
          exec= [
            {
              commands = [
                "${pkgs.python3}/bin/python ${pkgs.writeText "zpool.py" ''
                  import json
                  from subprocess import check_output

                  columns = ["NAME", "SIZE", "ALLOC", "FREE", "CKPOINT", "EXPANDSZ", "FRAG", "CAP", "DEDUP", "HEALTH", "ALTROOT"]
                  health = {'ONLINE':0, 'DEGRADED':11, 'OFFLINE':21, 'UNAVAIL':22, 'FAULTED':23, 'REMOVED':24}

                  stdout = check_output(["${pkgs.zfs}/bin/zpool", "list", "-Hp"],encoding='UTF-8').split('\n')
                  parsed_stdout = list(map(lambda x: dict(zip(columns,x.split('\t'))), stdout))[:-1]

                  for pool in parsed_stdout:
                    for item in pool:
                      if item in ["SIZE", "ALLOC", "FREE", "FRAG", "CAP"]:
                        pool[item] = int(pool[item])
                      if item in ["DEDUP"]:
                        pool[item] = float(pool[item])
                      if item == "HEALTH":
                        pool[item] = health[pool[item]]

                  print(json.dumps(parsed_stdout))
                ''}"
              ];
              tag_keys = [ "NAME" ];
              data_format = "json";
              name_suffix = "_python_zpool";
            }
          ];
        };
        outputs = {
          influxdb = { database = "telegraf"; urls = [ "http://localhost:8086" ]; };
        };
      };
    };

    udev.extraRules = ''
      KERNEL=="ipmi*", MODE="660", OWNER="telegraf"
    '';

    grafana = {
      enable = true;
      dataDir = "/var/lib/grafana";
      settings = {
        server = {
          http_addr = "127.0.0.1";
          root_url = "https://grafana.${domaine}";
        };
        smtp = {
          enabled = true;
          from_address = "grafana@${domaine}";
          skip_verify = true;
        };
        auth = {
          disable_signout_menu = true;
        };
        "auth.basic" = {
          enabled = false;
        };
        "auth.proxy" = {
          enabled = true;
          header_name = "X-WEBAUTH-USER";
        };
      };
    };

    zfs.zed.settings = {
      ZED_EMAIL_ADDR = [ "paul@nyanlout.re" ];
      ZED_NOTIFY_VERBOSE = true;
    };
  };

  systemd.services.influxdb.serviceConfig = {
    TimeoutStartSec = "10min";
  };

  security.sudo.extraRules = [
    { commands = [ { command = "${pkgs.smartmontools}/bin/smartctl"; options = [ "NOPASSWD" ]; } ]; users = [ "telegraf" ]; }
  ];
}