From 4240539bca94d3d396975c34b4021b3c420d7924 Mon Sep 17 00:00:00 2001
From: Dustin Frisch <fooker@lab.sh>
Date: Fri, 7 Jul 2023 15:02:19 +0200
Subject: [PATCH] Basic slurm

---
 machines/manager/default.nix |  1 +
 machines/manager/slurm.nix   |  5 +++++
 machines/node/default.nix    |  1 +
 machines/node/slurm.nix      |  7 +++++++
 shared/default.nix           |  1 +
 shared/secrets.yaml          |  6 ++++--
 shared/slurm.nix             | 35 +++++++++++++++++++++++++++++++++++
 7 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 machines/manager/slurm.nix
 create mode 100644 machines/node/slurm.nix
 create mode 100644 shared/slurm.nix

diff --git a/machines/manager/default.nix b/machines/manager/default.nix
index ebb3b8c..64515bb 100644
--- a/machines/manager/default.nix
+++ b/machines/manager/default.nix
@@ -18,6 +18,7 @@ with lib;
     ./cache.nix
     ./rdma.nix
     ./mpi.nix
+    ./slurm.nix
   ];
 
   deployment = {
diff --git a/machines/manager/slurm.nix b/machines/manager/slurm.nix
new file mode 100644
index 0000000..793767c
--- /dev/null
+++ b/machines/manager/slurm.nix
@@ -0,0 +1,5 @@
+{
+  services.slurm = {
+    server.enable = true;
+  };
+}
diff --git a/machines/node/default.nix b/machines/node/default.nix
index af424fd..6402137 100644
--- a/machines/node/default.nix
+++ b/machines/node/default.nix
@@ -9,6 +9,7 @@ with lib;
     ./disk.nix
     ./network.nix
     ./users.nix
+    ./slurm.nix
   ];
 
   deployment = {
diff --git a/machines/node/slurm.nix b/machines/node/slurm.nix
new file mode 100644
index 0000000..026e714
--- /dev/null
+++ b/machines/node/slurm.nix
@@ -0,0 +1,7 @@
+{ config, ... }:
+
+{
+  services.slurm = {
+    client.enable = true;
+  };
+}
diff --git a/shared/default.nix b/shared/default.nix
index c27b856..a66bd60 100644
--- a/shared/default.nix
+++ b/shared/default.nix
@@ -7,6 +7,7 @@
     ./ssh.nix
     ./rdma.nix
     ./ssl.nix
+    ./slurm.nix
 
     "${modulesPath}/profiles/headless.nix"
     "${modulesPath}/profiles/all-hardware.nix"
diff --git a/shared/secrets.yaml b/shared/secrets.yaml
index b27e24a..c996ff0 100644
--- a/shared/secrets.yaml
+++ b/shared/secrets.yaml
@@ -3,6 +3,8 @@ ldap:
         password: ENC[AES256_GCM,data:IFPwehOGSYore+HEv7MyymCKaOKn5XEH,iv:JTrZucSL/MohMgUdWqalpgjCCh7ueXd3cgNB0FuJo/U=,tag:o/1nvTrfojYsXYeuvxKfNg==,type:str]
 beegfs:
     connection: ENC[AES256_GCM,data:YTHMg76+5Azb+ex5ArUHt4xP+YYWr9Ph,iv:TEf8i+yezPsaW12Lg5jRnhds9uW9WhV6duZPdxeW9co=,tag:bPGsl7ofwE1Jh+FTyHJqzQ==,type:str]
+munge:
+    password: ENC[AES256_GCM,data:3cVMO9Q5G/UyM8r2x/XG0sHJalt4cvTZpw1o9tdUVeJ5sm6kWdhzCYu2mTnHn10A8gPtnP16Cibj+hwgT3Ib/g==,iv:dOeQyOmWwDB6irEyqNmPGHhZdgRhLQsQt48lKAX9jh4=,tag:OBZzzfCz04JOa7GmCbGCTw==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -81,8 +83,8 @@ sops:
             c01zMTA2dnZlTGxGd0oxYklXMHdWTUkKYjSQ7Y16AXfiLaOwsyV8LFjUtbUJ744A
             uxlImBcQnbiPkJY4DRxmtrBrTSzfX1pdepNH8DR0ZpjpI/6bibHEag==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-06-27T09:58:35Z"
-    mac: ENC[AES256_GCM,data:pPgwJnUdwQegqaCXdh7lweQq2Kos6szvo/mfBul+2TruUSSRXlGwKmNVLM2BuodMNZpTan2vCyvVlXvN4zBfW6nVWPzlBrCTbgtyBNodB+k3OJsfgUElQ32T9KccsMVuUsfKDzjhlFnV3NA9A7DVnrYz+jf1NcNSsz4yOjHudzA=,iv:ciFHyXhIcNFlB9fhzcAX8LICIsGPWDe29fxtjmJ0G+s=,tag:oldhGvm8vfPnuhpIXIpVWw==,type:str]
+    lastmodified: "2023-07-07T10:50:20Z"
+    mac: ENC[AES256_GCM,data:84PcC2J2peF6ZyEWH4o8gFw1yopC2o7DN5fg2I1+eUQVRmm8WqJbMkIF2taQeJndliEvsPBg6XXvbtJqdTs2L8o4EkkEwK4whbIosFyuVBuI3NRjjc1qswyYHudZa8CAtXPrVXqtD0q5QOtHwlUdGAyoBCpT8x2ZFaeye+JDuec=,iv:GhvwtEQMZlojwi0KoKUAQeuL53a0EFw1h+ysI9jeMuU=,tag:YJ8iYskhY8r3nDJIYxMusA==,type:str]
     pgp:
         - created_at: "2023-07-05T13:59:05Z"
           enc: |
diff --git a/shared/slurm.nix b/shared/slurm.nix
new file mode 100644
index 0000000..1f4a32a
--- /dev/null
+++ b/shared/slurm.nix
@@ -0,0 +1,35 @@
+{ pkgs, lib, config, nodes, ... }:
+
+with lib;
+
+{
+  services.slurm = {
+    clusterName = "mogli";
+
+    controlMachine = "manager";
+
+    nodeName = map
+      (node: "${node.config.networking.hostName} CPUs=64")
+      (filter
+        (node: elem "node" node.config.deployment.tags)
+        (attrValues nodes));
+
+    partitionName = [
+      "all Nodes=ALL AllowGroups=cluster Default=YES MaxTime=INFINITE State=UP"
+    ];
+
+    extraConfig = ''
+      MailProg=${pkgs.coreutils}/bin/false
+    '';
+  };
+
+  services.munge = {
+    enable = true;
+    password = config.sops.secrets."munge/password".path;
+  };
+
+  sops.secrets."munge/password" = {
+    sopsFile = ./secrets.yaml;
+    owner = "munge";
+  };
+}