Basic monitoring

machines/manager/default.nix

@@ -22,6 +22,7 @@ with lib;
     ./mpi.nix
     ./slurm.nix
     ./docs.nix
+    ./monitoring.nix
   ];
 
   deployment = {

machines/manager/docs.nix

@@ -7,9 +7,7 @@ with lib;
     ../../docs/module.nix
   ];
 
-  services.nginx = {
-    virtualHosts = {
-      "${config.networking.domain}" = {
+  services.nginx.virtualHosts."${config.networking.domain}" = {
     default = true;
 
     serverAliases = [
@@ -18,6 +16,4 @@ with lib;
 
     locations."/".root = config.system.build.docs;
   };
-    };
-  };
 }

machines/manager/monitoring.nix

@@ -0,0 +1,53 @@
+{ pkgs, lib, config, nodes, ... }:
+
+with lib;
+
+{
+  services.grafana = {
+    enable = true;
+
+    settings.server = {
+      addr = "127.0.0.1";
+      inherit (config.networking) domain;
+      root_url = "%(protocol)s://%(domain)s:%(http_port)s/grafana/";
+      serve_from_sub_path = true;
+    };
+
+    # provision = {
+    #   enable = true;
+
+    #   datasources.path = ./grafana/datasources.yaml
+    # };
+  };
+
+  services.prometheus = {
+    enable = true;
+
+    webExternalUrl = "http://${config.networking.domain}/prometheus";
+
+    scrapeConfigs = mapAttrsToList
+      (name: node: {
+        job_name = name;
+        static_configs = [
+          {
+            targets = [
+              "${name}:${toString node.config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      })
+      nodes;
+  };
+
+  services.nginx.virtualHosts.${config.networking.domain} = {
+    locations."/grafana" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
+      proxyWebsockets = true;
+    };
+
+    locations."/prometheus" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
+      proxyWebsockets = true;
+    };
+  };
+}

shared/default.nix

@@ -53,5 +53,10 @@
 
   services.haveged.enable = true;
 
+  services.prometheus.exporters = {
+    node.enable = true;
+    ipmi.enable = true;
+  };
+
   system.stateVersion = "22.11";
 }