diff --git a/machines/manager/dhcp.nix b/machines/manager/dhcp.nix index 82f0a0d..0e33524 100644 --- a/machines/manager/dhcp.nix +++ b/machines/manager/dhcp.nix @@ -2,6 +2,15 @@ with lib; +let + mkReservations = net: concatLists (mapAttrsToList + (_: node: optional (hasAttr net node.config.hpc.dhcp.reservations) { + "hw-address" = node.config.hpc.dhcp.reservations.${net}.hwAddress; + "ip-address" = node.config.hpc.dhcp.reservations.${net}.ipAddress; + }) + nodes); + +in { services.kea = { dhcp4 = { @@ -29,7 +38,7 @@ with lib; "option-data" = [ { "name" = "routers"; - "data" = "10.32.46.1"; + "data" = config.networking.defaultGateway.address; } { "name" = "domain-name-servers"; @@ -51,14 +60,7 @@ with lib; } ]; - "reservations" = concatLists (mapAttrsToList - (_: node: optional - (node.config.networking.interfaces."mngt".macAddress != null) - { - "hw-address" = node.config.networking.interfaces."mngt".macAddress; - "ip-address" = "10.32.46.${toString (100 + node.options._module.args.value.nodeId)}"; - }) - nodes); + "reservations" = mkReservations "mngt"; } { @@ -66,10 +68,6 @@ with lib; "interface" = "data"; "option-data" = [ - { - "name" = "routers"; - "data" = "10.32.47.1"; - } { "name" = "domain-name-servers"; "data" = "10.0.0.53,10.1.1.10"; @@ -90,14 +88,7 @@ with lib; } ]; - "reservations" = concatLists (mapAttrsToList - (_: node: optional - (node.config.networking.interfaces."data".macAddress != null) - { - "hw-address" = node.config.networking.interfaces."data".macAddress; - "ip-address" = "10.32.47.${toString (100 + node.options._module.args.value.nodeId)}"; - }) - nodes); + "reservations" = mkReservations "data"; } ]; }; diff --git a/machines/manager/mpi.nix b/machines/manager/mpi.nix index dc37270..a3918be 100644 --- a/machines/manager/mpi.nix +++ b/machines/manager/mpi.nix @@ -7,7 +7,7 @@ with lib; text = concatMapStringsSep "\n" (node: "${node.config.networking.hostName} max_slots=32") (filter - (node: elem "node" node.config.deployment.tags) + (node: node.config.hpc.node.enable) (attrValues nodes)); }; } diff --git a/machines/manager/netinstall/default.nix b/machines/manager/netinstall/default.nix index f895a02..63a9f5f 100644 --- a/machines/manager/netinstall/default.nix +++ b/machines/manager/netinstall/default.nix @@ -7,15 +7,10 @@ with lib; let - targets = [ - "node-00" - "node-01" - "node-02" - "node-03" - "node-04" - "node-05" - "node-06" - ]; + targets = attrNames + (filterAttrs + (_: node: node.config.hpc.netinstall.enable) + nodes); installer = pkgs.nixos [ ./installer.nix @@ -60,7 +55,7 @@ let }); api = pkgs.linkFarm "pixiecore-api" (listToAttrs (map - (name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.networking.interfaces."data".macAddress}" (apiEntry name)) + (name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.hpc.dhcp.reservations."data".hwAddress}" (apiEntry name)) targets)); in { diff --git a/machines/manager/network.nix b/machines/manager/network.nix index 6dbd151..86ae232 100644 --- a/machines/manager/network.nix +++ b/machines/manager/network.nix @@ -22,6 +22,8 @@ }]; }; + # This is not our real management interface but the hosts interface to the + # manangement network networking.interfaces."mngt" = { ipv4.addresses = [{ address = "10.32.46.253"; @@ -33,4 +35,11 @@ address = "10.32.47.1"; interface = "data"; }; + + hpc.dhcp.reservations = { + "mngt" = { + hwAddress = "e4:1f:13:28:c7:b9"; + ipAddress = "10.32.46.10"; + }; + }; } diff --git a/machines/node/default.nix b/machines/node/default.nix index ebfb39b..b011373 100644 --- a/machines/node/default.nix +++ b/machines/node/default.nix @@ -3,10 +3,6 @@ with lib; -let - nodeName = "node-${fixedWidthNumber 2 id}"; - -in { imports = [ ./hardware.nix @@ -14,22 +10,19 @@ in ./network.nix ./users.nix ./slurm.nix - (import ../nodes.nix).${nodeName} ]; deployment = { - targetHost = "10.32.47.${fixedWidthNumber 3 (100 + id)}"; + targetHost = "10.32.47.${toString (100 + id)}"; targetUser = "root"; - - tags = [ "node" ]; }; - _module.args = { - nodeId = id; - inherit nodeName; + hpc.node = { + enable = true; + inherit id; }; - networking.hostName = nodeName; + networking.hostName = config.hpc.node.name; networking.timeServers = [ "manager.${config.networking.domain}" diff --git a/machines/node/network.nix b/machines/node/network.nix index 93a3d2a..eba165f 100644 --- a/machines/node/network.nix +++ b/machines/node/network.nix @@ -1,11 +1,15 @@ -{ lib, nodeId, ... }: +{ lib, config, ... }: with lib; +let + node = (import ../nodes.nix).${config.hpc.node.name}; + +in { networking.interfaces."enp2s0f0" = { ipv4.addresses = [{ - address = "10.32.47.${fixedWidthNumber 3 (100 + nodeId)}"; + address = "10.32.47.${toString (100 + config.hpc.node.id)}"; prefixLength = 24; }]; }; @@ -14,4 +18,15 @@ with lib; address = "10.32.47.1"; interface = "enp2s0f0"; }; + + hpc.dhcp.reservations = { + "mngt" = { + hwAddress = node.mngt; + ipAddress = "10.32.46.${toString (100 + config.hpc.node.id)}"; + }; + "data" = { + hwAddress = node.data; + ipAddress = "10.32.47.${toString (100 + config.hpc.node.id)}"; + }; + }; } diff --git a/machines/nodes.nix b/machines/nodes.nix index f651194..69604e9 100644 --- a/machines/nodes.nix +++ b/machines/nodes.nix @@ -1,30 +1,30 @@ { "node-00" = { - networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:C9"; - networking.interfaces."data".macAddress = "50:46:5d:da:0b:d6"; + mngt = "50:46:5D:DA:0C:C9"; + data = "50:46:5d:da:0b:d6"; }; "node-01" = { - networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:07"; - networking.interfaces."data".macAddress = "50:46:5d:da:0c:56"; + mngt = "50:46:5D:DA:0C:07"; + data = "50:46:5d:da:0c:56"; }; "node-02" = { - networking.interfaces."mngt".macAddress = "10:BF:48:19:B0:04"; - networking.interfaces."data".macAddress = "10:bf:48:1f:a6:8f"; + mngt = "10:BF:48:19:B0:04"; + data = "10:bf:48:1f:a6:8f"; }; "node-03" = { - networking.interfaces."mngt".macAddress = "10:BF:48:19:A4:FE"; - networking.interfaces."data".macAddress = "10:bf:48:1b:57:47"; + mngt = "10:BF:48:19:A4:FE"; + data = "10:bf:48:1b:57:47"; }; "node-04" = { - networking.interfaces."mngt".macAddress = "10:BF:48:19:A2:E2"; - networking.interfaces."data".macAddress = "10:bf:48:19:a2:4d"; + mngt = "10:BF:48:19:A2:E2"; + data = "10:bf:48:19:a2:4d"; }; "node-05" = { - networking.interfaces."mngt".macAddress = "10:BF:48:15:00:F5"; - networking.interfaces."data".macAddress = "10:bf:48:1b:56:df"; + mngt = "10:BF:48:15:00:F5"; + data = "10:bf:48:1b:56:df"; }; "node-06" = { - networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:09"; - networking.interfaces."data".macAddress = "50:46:5d:da:0c:52"; + mngt = "50:46:5D:DA:0C:09"; + data = "50:46:5d:da:0c:52"; }; } diff --git a/modules/default.nix b/modules/default.nix index f5d01be..2321ec4 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -1,6 +1,9 @@ { imports = [ + ./node.nix ./hostFile.nix + ./dhcp.nix + ./netinstall.nix ./beegfs.nix ]; } diff --git a/modules/dhcp.nix b/modules/dhcp.nix new file mode 100644 index 0000000..e6efa1f --- /dev/null +++ b/modules/dhcp.nix @@ -0,0 +1,29 @@ +{ lib, config, ... }: + +with lib; + +{ + options.hpc.dhcp = { + reservations = mkOption { + description = '' + DHCP reservations for this host. + ''; + type = types.attrsOf (types.submodule { + options = { + hwAddress = mkOption { + description = '' + MAC address of the interface in this network. + ''; + type = types.str; + }; + ipAddress = mkOption { + description = '' + IP address of the host. + ''; + }; + }; + }); + default = { }; + }; + }; +} diff --git a/modules/netinstall.nix b/modules/netinstall.nix new file mode 100644 index 0000000..5e582e7 --- /dev/null +++ b/modules/netinstall.nix @@ -0,0 +1,20 @@ +{ config, lib, ... }: + +with lib; + +{ + options.hpc.netinstall = { + enable = mkEnableOption "NetInstall"; + }; + + config = mkIf config.hpc.netinstall.enable { + deployment.tags = [ "netinstall" ]; + + assertions = [ + { + assertion = elem "data" config.hpc.dhcp.reservations; + message = "NetInstall needs DHCP reservation in data network"; + } + ]; + }; +} diff --git a/modules/node.nix b/modules/node.nix new file mode 100644 index 0000000..4543eb9 --- /dev/null +++ b/modules/node.nix @@ -0,0 +1,30 @@ +{ lib, config, ... }: + +with lib; + +{ + options.hpc.node = { + enable = mkEnableOption "Compute Node"; + + id = mkOption { + description = '' + ID of the compute node. + ''; + type = types.ints.unsigned; + }; + + name = mkOption { + description = '' + Name of the node. + ''; + type = types.str; + readOnly = true; + }; + }; + + config = mkIf config.hpc.node.enable { + hpc.node.name = "node-${fixedWidthNumber 2 config.hpc.node.id}"; + + deployment.tags = [ "node" ]; + }; +} diff --git a/shared/slurm.nix b/shared/slurm.nix index d4e051c..60f49a6 100644 --- a/shared/slurm.nix +++ b/shared/slurm.nix @@ -11,7 +11,7 @@ with lib; nodeName = map (node: "${node.config.networking.hostName} CPUs=32") (filter # Filter all nodes that have a tag "node" being a compute node - (node: elem "node" node.config.deployment.tags) + (node: node.config.hpc.node.enable) (attrValues nodes)); partitionName = [