From 5ad108a0ab58797415e1cfdf62bd28bc36cb7e33 Mon Sep 17 00:00:00 2001
From: Dustin Frisch <fooker@lab.sh>
Date: Thu, 24 Aug 2023 15:25:37 +0200
Subject: [PATCH] FHS user env

---
 machines/manager/secrets/ldap-sync.conf |  6 +--
 shared/default.nix                      |  4 +-
 shared/slurm.nix                        | 16 ++++++--
 shared/userenv.nix                      | 51 +++++++++++++++++++++++++
 4 files changed, 68 insertions(+), 9 deletions(-)
 create mode 100644 shared/userenv.nix

diff --git a/machines/manager/secrets/ldap-sync.conf b/machines/manager/secrets/ldap-sync.conf
index 724f5d2..05232e8 100644
--- a/machines/manager/secrets/ldap-sync.conf
+++ b/machines/manager/secrets/ldap-sync.conf
@@ -1,5 +1,5 @@
 {
-	"data": "ENC[AES256_GCM,data:j4Gf2o5bInxx7kIK6zAbe6BzWKpSTDHhNXL7SigHb1RlSKpRj6JJ/M56h9omr+Aa6mNAacUiiiqkOjyT2n7w3+1OdQVcR+ZGTvRHc9TJGvQ483SxuiW5mjSYAxJ3fhJONU6KHYEauMIT/MSTgYO9NInUG9p05cZgIw5QGBFqEpHj/DiQ3EL2elvmu4YbaC4J3I2DrxZdh5i60yrWq2ny4UdpqqK8XEbfN7SY+8VDis73JCGEULJVhsYodlSx4ZMBXS0rbce7SZjJ/3O4EaCUJJOjcMcn5o7sOtXGUUAMpNnVvUuoz6SArkwGgfheL+HTf1z7rJzNpZJ2nvMFn8M0fAfH2okpX14rPvGOaYusV+VtOPwAMd4axrR0BHuDkV5L1tTyiQHcjrt6775kGuo3O+q+je3Ok5SZLu6Ysgkt3Y0T9QLYcKBc0UX+d0K3n6A8esUIBIj4KWc1iNjeE73eMb9lyp+6hFiv4TxHnsssgDMvam5KyH3Ltd5ZfqtH51CSSbi9QcvXi3ynODBVxoI1pvIIbPgaah6yGSdNH3QjZ7TtSa5yXkncwmtYhZa2DoNeTav40Uko0xdYqTOuHsOpBoH0fw9CVmoxsTjcvvhIXjLeXv4N9RelKkkFKuDK3QZojdUzoKXPt2HLjonH5Pqgk0xVSqiMHFbIQJ0uPirWEHJsJKCQfjyxkzYm24EfeAbSYLLGGMzebWB4rZw/acDA/+RdD2c+aL+L1EvRz3Tmqw4jwqwhJKeYIsuyLRG64MjmEnIlkIhfR1gSebPuEseBrsFg+0mOO8xS9aoFKJ1jujzYHxcQfyE+gW2qXIFcmBXHknM1wJI03fmyxyUIU19pPK17d1D+C1WPFvsNavXhQAXJRqUqGlUjGhqFM63w7Ij7vZHpcDaHZ69w6rPjuN/wuXHczNbdyWc0W6jp5sWYLZi+xRpYbKprfg9YL6o+zzb7D1yoxIUOty6XiVrlHb/lJlJ0t4CJbbtXNsXVNfojJ32otYM2t/X16+EdWLaFpqy2g8819luVrL+bDENm3bmtq9ARewBMGnGC4hCNaRRRnN9o9AX3sXyIgpOYotFpB/Vpl5CVQ6c1ODjNQJQgcFI=,iv:f1ZwZgu9UyzGnxE3qKPl4K6tlnqvk9jPLAYVXP7W+jI=,tag:iAXKNN/EFh4Z5HjDQogNPQ==,type:str]",
+	"data": "ENC[AES256_GCM,data:P/MCZdzK9TmH9UYNj0+FrA71cRTGylDNxbvY2z21RaCm5m3adh5laQWcGQMPA7dv7TLkniIOb5FOW/7bPwRlmnul/h1sjML6DnpnxDeKAt8jJaGtNJ6G7VkClObGxfaDvBJ2j8EEzjPIdIAISc7QTSOc91gN0U7TF3xZUkDwlxvsL1IK0oqvcSOf8HPB5TxmToe6tns8Ctaugl2h8N0yo4NTK4kFtE1BO8XT4n17KLJAsaIkbg5Rox3AgPOBIdOhbajpWWgpJEbIa38gO66V4ep93Iy7yrJHVt2jJ/bH31wQouxn/FFitLnxqe4naJjrw6XP2auLgEPfM95B7rsRrDH/HYW1hXsJklVc349qZm593eI3SnkDcUow1RxE6eIqXqU1mdBWIiE7RCyxdSgXwtQ2AZu3n+5m+V5wGHTBOuJPbOxX91LmXcX0pyzxrYigZNNUDCJTY02uQuyr8x8XFrYOvr3UuAfCFrWNQTcKYo7QlwgkPvKekYRMklkQ6f6pPwoyrNotSboSz0tThwVKP7c0XUlT4Yi9sEe50b4C6CAz1iw4UI/Z2UFz022H/gV4qx2BAIDKyFaFxSjEB7zZ4B42PIRLx/D7MFx1ie436TCqicYowQFg/ZGVZLvcE+iChAtI1Kl6h45eYI59XCqgfYce5KAAiwHdCkHqlkR66JPQc3Bu4xOsas8e441mQ1HGuHOmEwNbkXum/S4gDT+YFQkjrASuzFsxAVOFsj1gtXhoWscP47c+YszVH7iCCt/iXNZFFaITjWAk8mSnVLkshbeqEYpE57UsVDlv3gjPbwXARTAIDr1//puMGOhUM5JvLcxExq8q67A9+DSr+eKVZLCFxqjxyWvwf/Zx5WwiH1LuWMdMHPXDENoE49zlZHfiia6p3RWTvgPtP7ag/J0fyUKhF/SHmFExeAB1gcnrWaMKCUxfBpwM6rs58oAEgk+Sd4YXPgHjM6QEA4I9OJ2g8yoSIUVu8mkLFZhpId46tRPE5Wup5zd4dvKUCe51rD511pT5GtpYsRex+7VQLbVYDm0hgEVrKPnvgANWnHDdI7sW/sfNfC3k9MtMhIDpWZxO4fB/CMkfbJxt8s+pxI3ukzYMfpOo0Kj/Olmj2sotePwmcD8ARSBmlaeKojcay6jFpE1zVJw2iaJNENk=,iv:Mkq9T1fe9r18E4cumOxoG/u8MZ6Jv1Ee7P5ynJ2uTLo=,tag:/VHv16mSinntACDZALgA/g==,type:str]",
 	"sops": {
 		"kms": null,
 		"gcp_kms": null,
@@ -15,8 +15,8 @@
 				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHOUsrS2tyTUVEUEZaN3pR\nR0drZ3JDdUtMRHhJaGtONWtwK2Ftc1JsUFRJCjZtYXFubmlpTWtHNVpRU1ZhdThl\nUFVXUERDazdvSGtDOXc1VFNqeTRKRGcKLS0tIDhTdWhWU0dCUUYrZkdSRkVxbGFE\nYkg2Nk42VnUwZFhZVXdsWHFKYnUrMVEK0Aj6aON/QIFT2fsv2D9Ajvu+f6mHT4Q3\nm5uo99snnGEl3VIcvhC2yKGEtw3XOVpCfk5xHYLV2nlSs4WCc2DrkA==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2023-06-29T15:34:22Z",
-		"mac": "ENC[AES256_GCM,data:T4RlkuFsOJflLOkuvfRnhtnAp5iytfSPEla+Tf4v2zvdo1Gvh3wBmCItBdxhL8mGAl7JZCtJ5InGEccxsjBi+rgNrw9iQwYJMk4hLi6NrUYRCObhzk06JyMW3XM5N4yOQZBUEg/KWUuFR9oQhIP5A0pPdYqctalTg2GKTyusERo=,iv:dErVyHcD9A3elIZcOa0S5kryC6jmYeW4xxvfjHHviZ4=,tag:OupqMXrY147GxxEow7Hkjw==,type:str]",
+		"lastmodified": "2023-08-24T13:12:16Z",
+		"mac": "ENC[AES256_GCM,data:1zIxWiMGyS25X4C+Her3aJS2c3NdCunX0GyOTdGScavzbgv10ndjrkGPk/uuvTGY2DquGUxdQFu4Pot8zsAM3c5mCnl1ebTWLNQaS8futgllbGJG+EnOr7gUzg518GhmuKm87AN/HdcU0rTNPn3vDx3bTzo7znecSdi0v/6tOPI=,iv:rnyg7+YWN9VrJzli8hdNnQshl/qC80n5Ya3gYB8Yshs=,tag:u5aE64fUhuLuvWXk0ZF4vA==,type:str]",
 		"pgp": [
 			{
 				"created_at": "2023-06-26T09:22:36Z",
diff --git a/shared/default.nix b/shared/default.nix
index a66bd60..ddc692c 100644
--- a/shared/default.nix
+++ b/shared/default.nix
@@ -4,6 +4,7 @@
   imports = [
     ./network.nix
     ./users.nix
+    ./userenv.nix
     ./ssh.nix
     ./rdma.nix
     ./ssl.nix
@@ -29,18 +30,15 @@
   console.keyMap = "de";
 
   environment.systemPackages = with pkgs; [
-    openmpi
     vim
     wget
     curl
     tmux
     fd
     ripgrep
-    zlib
     htop
     iotop
     iftop
-    git
   ];
 
   nix.nixPath = [
diff --git a/shared/slurm.nix b/shared/slurm.nix
index 1f4a32a..3cab19a 100644
--- a/shared/slurm.nix
+++ b/shared/slurm.nix
@@ -9,13 +9,14 @@ with lib;
     controlMachine = "manager";
 
     nodeName = map
-      (node: "${node.config.networking.hostName} CPUs=64")
-      (filter
+      (node: "${node.config.networking.hostName} CPUs=32")
+      (filter # Filter all nodes that have a tag "node" being a compute node
         (node: elem "node" node.config.deployment.tags)
         (attrValues nodes));
 
     partitionName = [
-      "all Nodes=ALL AllowGroups=cluster Default=YES MaxTime=INFINITE State=UP"
+      "all Nodes=ALL AllowGroups=cluster Default=YES MaxTime=6:00:00 State=UP"
+      "vip Nodes=ALL AllowGroups=vip Default=NO MaxTime=INFINITE State=UP"
     ];
 
     extraConfig = ''
@@ -32,4 +33,13 @@ with lib;
     sopsFile = ./secrets.yaml;
     owner = "munge";
   };
+
+  # Have a VIP group for users with unlimited queues
+  users.groups."vip" = {
+    members = [
+      "fdai2856"
+      "fdai0159"
+      "fdai0231"
+    ];
+  };
 }
diff --git a/shared/userenv.nix b/shared/userenv.nix
new file mode 100644
index 0000000..c697a64
--- /dev/null
+++ b/shared/userenv.nix
@@ -0,0 +1,51 @@
+{ pkgs, lib, ... }:
+
+with lib;
+
+let
+  userenv = pkgs.buildFHSUserEnv {
+    name = "userenv";
+    targetPkgs = pkgs: with pkgs; [
+      git
+      postgresql
+      postgresql.lib
+      papi
+      gcc
+      gdb
+      gnumake
+      valgrind
+      autoconf
+      automake
+      perl
+      zlib
+      python3
+      libtool
+      flex
+      pandoc
+      hwloc
+      munge
+      numactl
+      libbfd
+      libiberty
+      pkgconfig
+      ucx
+      openmpi
+      vim
+      nano
+      wget
+      curl
+      tmux
+      fd
+      ripgrep
+      bash
+    ];
+  };
+in
+{
+  environment.systemPackages = [ userenv ];
+
+  # Install userenv to a well-known path
+  system.systemBuilderCommands = ''
+    ln -s ${userenv}/bin/userenv $out/userenv
+  '';
+}