NixOS configuration for HPC cluster
https://docs.hpc.informatik.hs-fulda.de/
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
138 lines
3.5 KiB
138 lines
3.5 KiB
# TFTP boot with shared image
|
|
# Requests store path to install from master
|
|
# Runs disko and nixos-install
|
|
|
|
{ pkgs, lib, config, nodes, ... }@args:
|
|
|
|
with lib;
|
|
|
|
let
|
|
targets = attrNames
|
|
(filterAttrs
|
|
(_: node: node.config.hpc.netinstall.enable)
|
|
nodes);
|
|
|
|
installer = pkgs.nixos [
|
|
./installer.nix
|
|
{
|
|
_module.args = {
|
|
nodes = getAttrs [ "manager" ] nodes;
|
|
};
|
|
}
|
|
];
|
|
|
|
apiEntry = name:
|
|
let
|
|
node = nodes.${name}.config.system.build;
|
|
boot = installer.config.system.build;
|
|
|
|
install = pkgs.writeScript "install-${name}" ''
|
|
#!/usr/bin/env bash
|
|
|
|
set -xeuo pipefail
|
|
|
|
"${node.diskoScript}"
|
|
|
|
"${node.nixos-install}/bin/nixos-install" \
|
|
--root /mnt \
|
|
--system "${node.toplevel}" \
|
|
--no-channel-copy \
|
|
--no-root-password \
|
|
--verbose
|
|
|
|
${pkgs.ipmitool}/bin/ipmitool chassis bootdev disk
|
|
|
|
reboot
|
|
'';
|
|
in
|
|
pkgs.writeText "pixieboot-api-${name}" (builtins.toJSON {
|
|
kernel = "file://${boot.kernel}/bzImage";
|
|
initrd = [ "file://${boot.netbootRamdisk}/initrd" ];
|
|
cmdline = concatStringsSep " " [
|
|
"init=${boot.toplevel}/init"
|
|
"loglevel=4"
|
|
"nixos.install=${install}"
|
|
"console=tty0"
|
|
"console=ttyS1,57600n8"
|
|
];
|
|
message = "NixOS Automatic Installer for ${name}";
|
|
});
|
|
|
|
api = pkgs.linkFarm "pixiecore-api" (listToAttrs (map
|
|
(name: nameValuePair "pixiecore/v1/boot/${nodes."${name}".config.hpc.dhcp.reservations."data".hwAddress}" (apiEntry name))
|
|
targets));
|
|
|
|
ipxe-with-serial = pkgs.ipxe.override {
|
|
additionalOptions = [ "CONSOLE_SERIAL" ];
|
|
embedScript = "${pkgs.pixiecore.src}/pixiecore/boot.ipxe";
|
|
};
|
|
|
|
trigger-script = pkgs.writeScriptBin "auto-install" ''
|
|
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
case "$1" in
|
|
${concatMapStringsSep "\n" (node:
|
|
'' "${node}")
|
|
MNGT_IP="${nodes.${node}.config.hpc.dhcp.reservations."mngt".ipAddress}"
|
|
DATA_IP="${nodes.${node}.config.hpc.dhcp.reservations."data".ipAddress}"
|
|
;;
|
|
'')
|
|
targets}
|
|
*)
|
|
echo "No such node" >&2
|
|
exit 255
|
|
;;
|
|
esac
|
|
|
|
echo "Switch boot device to PXE"
|
|
${pkgs.ipmitool}/bin/ipmitool -I lanplus -H "$MNGT_IP" -U admin -P admin chassis bootdev pxe
|
|
|
|
sleep 1s
|
|
|
|
echo -n "Resetting node "
|
|
${pkgs.ipmitool}/bin/ipmitool -I lanplus -H "$MNGT_IP" -U admin -P admin chassis power reset
|
|
|
|
while ! ping -c 1 "$DATA_IP" -n > /dev/null; do
|
|
echo -n "."
|
|
done
|
|
echo " done"
|
|
|
|
echo "Reset boot device to disk"
|
|
${pkgs.ipmitool}/bin/ipmitool -I lanplus -H "$MNGT_IP" -U admin -P admin chassis bootdev disk
|
|
'';
|
|
|
|
in
|
|
{
|
|
services.pixiecore = {
|
|
enable = true;
|
|
mode = "api";
|
|
dhcpNoBind = true;
|
|
debug = true;
|
|
openFirewall = true;
|
|
port = 5080;
|
|
statusPort = 6080;
|
|
apiServer = "http://boot.${config.networking.domain}/pixiecore";
|
|
extraArguments = [
|
|
"--ipxe-bios"
|
|
"${ipxe-with-serial}/undionly.kpxe"
|
|
];
|
|
};
|
|
|
|
services.nginx = {
|
|
virtualHosts = {
|
|
"boot.${config.networking.domain}" = {
|
|
locations."/".proxyPass = "http://localhost:${toString config.services.pixiecore.port}";
|
|
locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}";
|
|
locations."/pixiecore".root = api;
|
|
};
|
|
};
|
|
};
|
|
|
|
users.users."root".packages = [ trigger-script ];
|
|
|
|
hpc.hostFile.aliases = [
|
|
"boot.${config.networking.domain}"
|
|
];
|
|
}
|