Browse Source

net: dhcp restruct

main
Dustin Frisch 1 year ago
parent
commit
88d09e6a48
No known key found for this signature in database GPG Key ID: B4C3BF012D9B26BE
  1. 33
      machines/manager/dhcp.nix
  2. 2
      machines/manager/mpi.nix
  3. 15
      machines/manager/netinstall/default.nix
  4. 9
      machines/manager/network.nix
  5. 17
      machines/node/default.nix
  6. 19
      machines/node/network.nix
  7. 28
      machines/nodes.nix
  8. 3
      modules/default.nix
  9. 29
      modules/dhcp.nix
  10. 20
      modules/netinstall.nix
  11. 30
      modules/node.nix
  12. 2
      shared/slurm.nix

33
machines/manager/dhcp.nix

@ -2,6 +2,15 @@
with lib;
let
mkReservations = net: concatLists (mapAttrsToList
(_: node: optional (hasAttr net node.config.hpc.dhcp.reservations) {
"hw-address" = node.config.hpc.dhcp.reservations.${net}.hwAddress;
"ip-address" = node.config.hpc.dhcp.reservations.${net}.ipAddress;
})
nodes);
in
{
services.kea = {
dhcp4 = {
@ -29,7 +38,7 @@ with lib;
"option-data" = [
{
"name" = "routers";
"data" = "10.32.46.1";
"data" = config.networking.defaultGateway.address;
}
{
"name" = "domain-name-servers";
@ -51,14 +60,7 @@ with lib;
}
];
"reservations" = concatLists (mapAttrsToList
(_: node: optional
(node.config.networking.interfaces."mngt".macAddress != null)
{
"hw-address" = node.config.networking.interfaces."mngt".macAddress;
"ip-address" = "10.32.46.${toString (100 + node.options._module.args.value.nodeId)}";
})
nodes);
"reservations" = mkReservations "mngt";
}
{
@ -66,10 +68,6 @@ with lib;
"interface" = "data";
"option-data" = [
{
"name" = "routers";
"data" = "10.32.47.1";
}
{
"name" = "domain-name-servers";
"data" = "10.0.0.53,10.1.1.10";
@ -90,14 +88,7 @@ with lib;
}
];
"reservations" = concatLists (mapAttrsToList
(_: node: optional
(node.config.networking.interfaces."data".macAddress != null)
{
"hw-address" = node.config.networking.interfaces."data".macAddress;
"ip-address" = "10.32.47.${toString (100 + node.options._module.args.value.nodeId)}";
})
nodes);
"reservations" = mkReservations "data";
}
];
};

2
machines/manager/mpi.nix

@ -7,7 +7,7 @@ with lib;
text = concatMapStringsSep "\n"
(node: "${node.config.networking.hostName} max_slots=32")
(filter
(node: elem "node" node.config.deployment.tags)
(node: node.config.hpc.node.enable)
(attrValues nodes));
};
}

15
machines/manager/netinstall/default.nix

@ -7,15 +7,10 @@
with lib;
let
targets = [
"node-00"
"node-01"
"node-02"
"node-03"
"node-04"
"node-05"
"node-06"
];
targets = attrNames
(filterAttrs
(_: node: node.config.hpc.netinstall.enable)
nodes);
installer = pkgs.nixos [
./installer.nix
@ -60,7 +55,7 @@ let
});
api = pkgs.linkFarm "pixiecore-api" (listToAttrs (map
(name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.networking.interfaces."data".macAddress}" (apiEntry name))
(name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.hpc.dhcp.reservations."data".hwAddress}" (apiEntry name))
targets));
in
{

9
machines/manager/network.nix

@ -22,6 +22,8 @@
}];
};
# This is not our real management interface but the hosts interface to the
# manangement network
networking.interfaces."mngt" = {
ipv4.addresses = [{
address = "10.32.46.253";
@ -33,4 +35,11 @@
address = "10.32.47.1";
interface = "data";
};
hpc.dhcp.reservations = {
"mngt" = {
hwAddress = "e4:1f:13:28:c7:b9";
ipAddress = "10.32.46.10";
};
};
}

17
machines/node/default.nix

@ -3,10 +3,6 @@
with lib;
let
nodeName = "node-${fixedWidthNumber 2 id}";
in
{
imports = [
./hardware.nix
@ -14,22 +10,19 @@ in
./network.nix
./users.nix
./slurm.nix
(import ../nodes.nix).${nodeName}
];
deployment = {
targetHost = "10.32.47.${fixedWidthNumber 3 (100 + id)}";
targetHost = "10.32.47.${toString (100 + id)}";
targetUser = "root";
tags = [ "node" ];
};
_module.args = {
nodeId = id;
inherit nodeName;
hpc.node = {
enable = true;
inherit id;
};
networking.hostName = nodeName;
networking.hostName = config.hpc.node.name;
networking.timeServers = [
"manager.${config.networking.domain}"

19
machines/node/network.nix

@ -1,11 +1,15 @@
{ lib, nodeId, ... }:
{ lib, config, ... }:
with lib;
let
node = (import ../nodes.nix).${config.hpc.node.name};
in
{
networking.interfaces."enp2s0f0" = {
ipv4.addresses = [{
address = "10.32.47.${fixedWidthNumber 3 (100 + nodeId)}";
address = "10.32.47.${toString (100 + config.hpc.node.id)}";
prefixLength = 24;
}];
};
@ -14,4 +18,15 @@ with lib;
address = "10.32.47.1";
interface = "enp2s0f0";
};
hpc.dhcp.reservations = {
"mngt" = {
hwAddress = node.mngt;
ipAddress = "10.32.46.${toString (100 + config.hpc.node.id)}";
};
"data" = {
hwAddress = node.data;
ipAddress = "10.32.47.${toString (100 + config.hpc.node.id)}";
};
};
}

28
machines/nodes.nix

@ -1,30 +1,30 @@
{
"node-00" = {
networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:C9";
networking.interfaces."data".macAddress = "50:46:5d:da:0b:d6";
mngt = "50:46:5D:DA:0C:C9";
data = "50:46:5d:da:0b:d6";
};
"node-01" = {
networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:07";
networking.interfaces."data".macAddress = "50:46:5d:da:0c:56";
mngt = "50:46:5D:DA:0C:07";
data = "50:46:5d:da:0c:56";
};
"node-02" = {
networking.interfaces."mngt".macAddress = "10:BF:48:19:B0:04";
networking.interfaces."data".macAddress = "10:bf:48:1f:a6:8f";
mngt = "10:BF:48:19:B0:04";
data = "10:bf:48:1f:a6:8f";
};
"node-03" = {
networking.interfaces."mngt".macAddress = "10:BF:48:19:A4:FE";
networking.interfaces."data".macAddress = "10:bf:48:1b:57:47";
mngt = "10:BF:48:19:A4:FE";
data = "10:bf:48:1b:57:47";
};
"node-04" = {
networking.interfaces."mngt".macAddress = "10:BF:48:19:A2:E2";
networking.interfaces."data".macAddress = "10:bf:48:19:a2:4d";
mngt = "10:BF:48:19:A2:E2";
data = "10:bf:48:19:a2:4d";
};
"node-05" = {
networking.interfaces."mngt".macAddress = "10:BF:48:15:00:F5";
networking.interfaces."data".macAddress = "10:bf:48:1b:56:df";
mngt = "10:BF:48:15:00:F5";
data = "10:bf:48:1b:56:df";
};
"node-06" = {
networking.interfaces."mngt".macAddress = "50:46:5D:DA:0C:09";
networking.interfaces."data".macAddress = "50:46:5d:da:0c:52";
mngt = "50:46:5D:DA:0C:09";
data = "50:46:5d:da:0c:52";
};
}

3
modules/default.nix

@ -1,6 +1,9 @@
{
imports = [
./node.nix
./hostFile.nix
./dhcp.nix
./netinstall.nix
./beegfs.nix
];
}

29
modules/dhcp.nix

@ -0,0 +1,29 @@
{ lib, config, ... }:
with lib;
{
options.hpc.dhcp = {
reservations = mkOption {
description = ''
DHCP reservations for this host.
'';
type = types.attrsOf (types.submodule {
options = {
hwAddress = mkOption {
description = ''
MAC address of the interface in this network.
'';
type = types.str;
};
ipAddress = mkOption {
description = ''
IP address of the host.
'';
};
};
});
default = { };
};
};
}

20
modules/netinstall.nix

@ -0,0 +1,20 @@
{ config, lib, ... }:
with lib;
{
options.hpc.netinstall = {
enable = mkEnableOption "NetInstall";
};
config = mkIf config.hpc.netinstall.enable {
deployment.tags = [ "netinstall" ];
assertions = [
{
assertion = elem "data" config.hpc.dhcp.reservations;
message = "NetInstall needs DHCP reservation in data network";
}
];
};
}

30
modules/node.nix

@ -0,0 +1,30 @@
{ lib, config, ... }:
with lib;
{
options.hpc.node = {
enable = mkEnableOption "Compute Node";
id = mkOption {
description = ''
ID of the compute node.
'';
type = types.ints.unsigned;
};
name = mkOption {
description = ''
Name of the node.
'';
type = types.str;
readOnly = true;
};
};
config = mkIf config.hpc.node.enable {
hpc.node.name = "node-${fixedWidthNumber 2 config.hpc.node.id}";
deployment.tags = [ "node" ];
};
}

2
shared/slurm.nix

@ -11,7 +11,7 @@ with lib;
nodeName = map
(node: "${node.config.networking.hostName} CPUs=32")
(filter # Filter all nodes that have a tag "node" being a compute node
(node: elem "node" node.config.deployment.tags)
(node: node.config.hpc.node.enable)
(attrValues nodes));
partitionName = [

Loading…
Cancel
Save