net: dhcp restruct

machines/manager/dhcp.nix

@@ -2,6 +2,15 @@
 
 with lib;
 
+let
+  mkReservations = net: concatLists (mapAttrsToList
+    (_: node: optional (hasAttr net node.config.hpc.dhcp.reservations) {
+      "hw-address" = node.config.hpc.dhcp.reservations.${net}.hwAddress;
+      "ip-address" = node.config.hpc.dhcp.reservations.${net}.ipAddress;
+    })
+    nodes);
+
+in
 {
   services.kea = {
     dhcp4 = {
@@ -29,7 +38,7 @@ with lib;
             "option-data" = [
               {
                 "name" = "routers";
-                "data" = "10.32.46.1";
+                "data" = config.networking.defaultGateway.address;
               }
               {
                 "name" = "domain-name-servers";
@@ -51,14 +60,7 @@ with lib;
               }
             ];
 
-            "reservations" = concatLists (mapAttrsToList
-              (_: node: optional
-                (node.config.networking.interfaces."mngt".macAddress != null)
-                {
-                  "hw-address" = node.config.networking.interfaces."mngt".macAddress;
-                  "ip-address" = "10.32.46.${toString (100 + node.options._module.args.value.nodeId)}";
-                })
-              nodes);
+            "reservations" = mkReservations "mngt";
           }
 
           {
@@ -66,10 +68,6 @@ with lib;
             "interface" = "data";
 
             "option-data" = [
-              {
-                "name" = "routers";
-                "data" = "10.32.47.1";
-              }
               {
                 "name" = "domain-name-servers";
                 "data" = "10.0.0.53,10.1.1.10";
@@ -90,14 +88,7 @@ with lib;
               }
             ];
 
-            "reservations" = concatLists (mapAttrsToList
-              (_: node: optional
-                (node.config.networking.interfaces."data".macAddress != null)
-                {
-                  "hw-address" = node.config.networking.interfaces."data".macAddress;
-                  "ip-address" = "10.32.47.${toString (100 + node.options._module.args.value.nodeId)}";
-                })
-              nodes);
+            "reservations" = mkReservations "data";
           }
         ];
       };

machines/manager/mpi.nix

@@ -7,7 +7,7 @@ with lib;
     text = concatMapStringsSep "\n"
       (node: "${node.config.networking.hostName} max_slots=32")
       (filter
-        (node: elem "node" node.config.deployment.tags)
+        (node: node.config.hpc.node.enable)
         (attrValues nodes));
   };
 }

machines/manager/netinstall/default.nix

@@ -7,15 +7,10 @@
 with lib;
 
 let
-  targets = [
-    "node-00"
-    "node-01"
-    "node-02"
-    "node-03"
-    "node-04"
-    "node-05"
-    "node-06"
-  ];
+  targets = attrNames
+    (filterAttrs
+      (_: node: node.config.hpc.netinstall.enable)
+      nodes);
 
   installer = pkgs.nixos [
     ./installer.nix
@@ -60,7 +55,7 @@ let
     });
 
   api = pkgs.linkFarm "pixiecore-api" (listToAttrs (map
-    (name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.networking.interfaces."data".macAddress}" (apiEntry name))
+    (name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.hpc.dhcp.reservations."data".hwAddress}" (apiEntry name))
     targets));
 in
 {

machines/manager/network.nix

@@ -22,6 +22,8 @@
     }];
   };
 
+  # This is not our real management interface but the hosts interface to the
+  # manangement network
   networking.interfaces."mngt" = {
     ipv4.addresses = [{
       address = "10.32.46.253";
@@ -33,4 +35,11 @@
     address = "10.32.47.1";
     interface = "data";
   };
+
+  hpc.dhcp.reservations = {
+    "mngt" = {
+      hwAddress = "e4:1f:13:28:c7:b9";
+      ipAddress = "10.32.46.10";
+    };
+  };
 }

machines/node/default.nix

@@ -3,10 +3,6 @@
 
 with lib;
 
-let
-  nodeName = "node-${fixedWidthNumber 2 id}";
-
-in
 {
   imports = [
     ./hardware.nix
@@ -14,22 +10,19 @@ in
     ./network.nix
     ./users.nix
     ./slurm.nix
-    (import ../nodes.nix).${nodeName}
   ];
 
   deployment = {
-    targetHost = "10.32.47.${fixedWidthNumber 3 (100 + id)}";
+    targetHost = "10.32.47.${toString (100 + id)}";
     targetUser = "root";
-
-    tags = [ "node" ];
   };
 
-  _module.args = {
-    nodeId = id;
-    inherit nodeName;
+  hpc.node = {
+    enable = true;
+    inherit id;
   };
 
-  networking.hostName = nodeName;
+  networking.hostName = config.hpc.node.name;
 
   networking.timeServers = [
     "manager.${config.networking.domain}"

machines/node/network.nix

@@ -1,11 +1,15 @@
-{ lib, nodeId, ... }:
+{ lib, config, ... }:
 
 with lib;
 
+let
+  node = (import ../nodes.nix).${config.hpc.node.name};
+
+in
 {
   networking.interfaces."enp2s0f0" = {
     ipv4.addresses = [{
-      address = "10.32.47.${fixedWidthNumber 3 (100 + nodeId)}";
+      address = "10.32.47.${toString (100 + config.hpc.node.id)}";
       prefixLength = 24;
     }];
   };
@@ -14,4 +18,15 @@ with lib;
     address = "10.32.47.1";
     interface = "enp2s0f0";
   };
+
+  hpc.dhcp.reservations = {
+    "mngt" = {
+      hwAddress = node.mngt;
+      ipAddress = "10.32.46.${toString (100 + config.hpc.node.id)}";
+    };
+    "data" = {
+      hwAddress = node.data;
+      ipAddress = "10.32.47.${toString (100 + config.hpc.node.id)}";
+    };
+  };
 }

machines/nodes.nix

@@ -1,30 +1,30 @@
 {
   "node-00" = {
-    networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:C9";
-    networking.interfaces."data".macAddress = "50:46:5d:da:0b:d6";
+    mngt = "50:46:5D:DA:0C:C9";
+    data = "50:46:5d:da:0b:d6";
   };
   "node-01" = {
-    networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:07";
-    networking.interfaces."data".macAddress = "50:46:5d:da:0c:56";
+    mngt = "50:46:5D:DA:0C:07";
+    data = "50:46:5d:da:0c:56";
   };
   "node-02" = {
-    networking.interfaces."mngt".macAddress = "10:BF:48:19:B0:04";
-    networking.interfaces."data".macAddress = "10:bf:48:1f:a6:8f";
+    mngt = "10:BF:48:19:B0:04";
+    data = "10:bf:48:1f:a6:8f";
   };
   "node-03" = {
-    networking.interfaces."mngt".macAddress = "10:BF:48:19:A4:FE";
-    networking.interfaces."data".macAddress = "10:bf:48:1b:57:47";
+    mngt = "10:BF:48:19:A4:FE";
+    data = "10:bf:48:1b:57:47";
   };
   "node-04" = {
-    networking.interfaces."mngt".macAddress = "10:BF:48:19:A2:E2";
-    networking.interfaces."data".macAddress = "10:bf:48:19:a2:4d";
+    mngt = "10:BF:48:19:A2:E2";
+    data = "10:bf:48:19:a2:4d";
   };
   "node-05" = {
-    networking.interfaces."mngt".macAddress = "10:BF:48:15:00:F5";
-    networking.interfaces."data".macAddress = "10:bf:48:1b:56:df";
+    mngt = "10:BF:48:15:00:F5";
+    data = "10:bf:48:1b:56:df";
   };
   "node-06" = {
-    networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:09";
-    networking.interfaces."data".macAddress = "50:46:5d:da:0c:52";
+    mngt = "50:46:5D:DA:0C:09";
+    data = "50:46:5d:da:0c:52";
   };
 }

modules/default.nix

@@ -1,6 +1,9 @@
 {
   imports = [
+    ./node.nix
     ./hostFile.nix
+    ./dhcp.nix
+    ./netinstall.nix
     ./beegfs.nix
   ];
 }

modules/dhcp.nix

@@ -0,0 +1,29 @@
+{ lib, config, ... }:
+
+with lib;
+
+{
+  options.hpc.dhcp = {
+    reservations = mkOption {
+      description = ''
+        DHCP reservations for this host.
+      '';
+      type = types.attrsOf (types.submodule {
+        options = {
+          hwAddress = mkOption {
+            description = ''
+              MAC address of the interface in this network.
+            '';
+            type = types.str;
+          };
+          ipAddress = mkOption {
+            description = ''
+              IP address of the host.
+            '';
+          };
+        };
+      });
+      default = { };
+    };
+  };
+}

modules/netinstall.nix

@@ -0,0 +1,20 @@
+{ config, lib, ... }:
+
+with lib;
+
+{
+  options.hpc.netinstall = {
+    enable = mkEnableOption "NetInstall";
+  };
+
+  config = mkIf config.hpc.netinstall.enable {
+    deployment.tags = [ "netinstall" ];
+
+    assertions = [
+      {
+        assertion = elem "data" config.hpc.dhcp.reservations;
+        message = "NetInstall needs DHCP reservation in data network";
+      }
+    ];
+  };
+}

modules/node.nix

@@ -0,0 +1,30 @@
+{ lib, config, ... }:
+
+with lib;
+
+{
+  options.hpc.node = {
+    enable = mkEnableOption "Compute Node";
+
+    id = mkOption {
+      description = ''
+        ID of the compute node.
+      '';
+      type = types.ints.unsigned;
+    };
+
+    name = mkOption {
+      description = ''
+        Name of the node.
+      '';
+      type = types.str;
+      readOnly = true;
+    };
+  };
+
+  config = mkIf config.hpc.node.enable {
+    hpc.node.name = "node-${fixedWidthNumber 2 config.hpc.node.id}";
+
+    deployment.tags = [ "node" ];
+  };
+}

shared/slurm.nix

@@ -11,7 +11,7 @@ with lib;
     nodeName = map
       (node: "${node.config.networking.hostName} CPUs=32")
       (filter # Filter all nodes that have a tag "node" being a compute node
-        (node: elem "node" node.config.deployment.tags)
+        (node: node.config.hpc.node.enable)
         (attrValues nodes));
 
     partitionName = [