From 227c0c4a8860a1d0297d8f83e7db081eaa469c7b Mon Sep 17 00:00:00 2001
From: Dustin Frisch <fooker@lab.sh>
Date: Thu, 9 Nov 2023 12:36:53 +0100
Subject: [PATCH] Basic monitoring

---
 machines/manager/default.nix    |  1 +
 machines/manager/docs.nix       | 16 ++++------
 machines/manager/monitoring.nix | 53 +++++++++++++++++++++++++++++++++
 shared/default.nix              |  5 ++++
 4 files changed, 65 insertions(+), 10 deletions(-)
 create mode 100644 machines/manager/monitoring.nix

diff --git a/machines/manager/default.nix b/machines/manager/default.nix
index fed885b..e8af505 100644
--- a/machines/manager/default.nix
+++ b/machines/manager/default.nix
@@ -22,6 +22,7 @@ with lib;
     ./mpi.nix
     ./slurm.nix
     ./docs.nix
+    ./monitoring.nix
   ];
 
   deployment = {
diff --git a/machines/manager/docs.nix b/machines/manager/docs.nix
index 1733528..29b2191 100644
--- a/machines/manager/docs.nix
+++ b/machines/manager/docs.nix
@@ -7,17 +7,13 @@ with lib;
     ../../docs/module.nix
   ];
 
-  services.nginx = {
-    virtualHosts = {
-      "${config.networking.domain}" = {
-        default = true;
+  services.nginx.virtualHosts."${config.networking.domain}" = {
+    default = true;
 
-        serverAliases = [
-          "doku.${config.networking.domain}"
-        ];
+    serverAliases = [
+      "doku.${config.networking.domain}"
+    ];
 
-        locations."/".root = config.system.build.docs;
-      };
-    };
+    locations."/".root = config.system.build.docs;
   };
 }
diff --git a/machines/manager/monitoring.nix b/machines/manager/monitoring.nix
new file mode 100644
index 0000000..9878ab4
--- /dev/null
+++ b/machines/manager/monitoring.nix
@@ -0,0 +1,53 @@
+{ pkgs, lib, config, nodes, ... }:
+
+with lib;
+
+{
+  services.grafana = {
+    enable = true;
+
+    settings.server = {
+      addr = "127.0.0.1";
+      inherit (config.networking) domain;
+      root_url = "%(protocol)s://%(domain)s:%(http_port)s/grafana/";
+      serve_from_sub_path = true;
+    };
+
+    # provision = {
+    #   enable = true;
+
+    #   datasources.path = ./grafana/datasources.yaml
+    # };
+  };
+
+  services.prometheus = {
+    enable = true;
+
+    webExternalUrl = "http://${config.networking.domain}/prometheus";
+
+    scrapeConfigs = mapAttrsToList
+      (name: node: {
+        job_name = name;
+        static_configs = [
+          {
+            targets = [
+              "${name}:${toString node.config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      })
+      nodes;
+  };
+
+  services.nginx.virtualHosts.${config.networking.domain} = {
+    locations."/grafana" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+      proxyWebsockets = true;
+    };
+
+    locations."/prometheus" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
+      proxyWebsockets = true;
+    };
+  };
+}
diff --git a/shared/default.nix b/shared/default.nix
index 814fc17..4fa1929 100644
--- a/shared/default.nix
+++ b/shared/default.nix
@@ -53,5 +53,10 @@
 
   services.haveged.enable = true;
 
+  services.prometheus.exporters = {
+    node.enable = true;
+    ipmi.enable = true;
+  };
+
   system.stateVersion = "22.11";
 }