More nodes, more bugs

gathered/node-04/ssh_host_ed25519_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL4iEVnSANFNZOJQF77MfCLv+gyXY5Lj+JTxz4Htm5IU root@node-04

gathered/node-05/ssh_host_ed25519_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmFs81JIz4rfku3g6ht6TBkAlOhG3fiaNpk5sUHbLp1 root@node-05

gathered/node-06/ssh_host_ed25519_key.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPEl28pmWLFdL8lpVHuOiOSnAkumnzZpBTwS/rYFtuc root@node-06

machines.nix

@@ -3,7 +3,7 @@
 with lib;
 
 let
-  nrNodes = 4;
+  nrNodes = 7;
 
   mkMachine = { name, type, opts ? { } }: rec {
     inherit name type opts;

machines/manager/mpi.nix

@@ -5,7 +5,7 @@ with lib;
 {
   environment.etc."mpi/hosts" = {
     text = concatMapStringsSep "\n"
-      (node: "${node.config.networking.hostName}")
+      (node: "${node.config.networking.hostName} max_slots=64")
       (filter
         (node: elem "node" node.config.deployment.tags)
         (attrValues nodes));

machines/manager/netinstall/default.nix

@@ -10,34 +10,31 @@ let
   targets = {
     "50:46:5d:da:0b:d6" = "node-00";
     "50:46:5d:da:0c:56" = "node-01";
-    "50:46:5d:da:0c:52" = "node-02";
-    "10:bf:48:1f:a6:8f" = "node-03";
-    # "10:bf:48:1b:57:47" = "node-04";
-    # "10:bf:48:19:a2:4d" = "node-05";
-    # "10:bf:48:1b:56:df" = "node-06";
+    "10:bf:48:1f:a6:8f" = "node-02";
+    "10:bf:48:1b:57:47" = "node-03";
+    "10:bf:48:19:a2:4d" = "node-04";
+    "10:bf:48:1b:56:df" = "node-05";
+    "50:46:5d:da:0c:52" = "node-06";
   };
 
   installer = pkgs.nixos [
     ./installer.nix
     {
       _module.args = {
-        managerConfig = config;
+        nodes = getAttrs [ "manager" ] nodes;
       };
     }
   ];
 
-  commands = pkgs.symlinkJoin {
-    name = "pxeboot";
-    paths = mapAttrsToList
-      (mac: name:
+  apiEntry = name:
     let
       node = nodes.${name}.config.system.build;
       boot = installer.config.system.build;
 
-          install = pkgs.writers.writeBash "install-${name}" ''
-            set -o errexit
-            set -o nounset
-            set -o pipefail
+      install = pkgs.writeScript "install-${name}" ''
+        #!/usr/bin/env bash
+
+        set -xeuo pipefail
 
         "${node.diskoScript}"
 
@@ -50,24 +47,43 @@ let
 
         reboot
       '';
-
     in
-        pkgs.writers.writeBashBin "pxe-install-${name}" ''
-          exec ${pkgs.pixiecore}/bin/pixiecore \
-            boot "${boot.kernel}/bzImage" "${boot.netbootRamdisk}/initrd" \
-            --cmdline "init=${boot.toplevel}/init loglevel=4 nixos.install=${install}" \
-            --debug \
-            --dhcp-no-bind \
-            --port 64172 \
-            --status-port 64172 \
-            "$@"
-        '')
-      targets;
-  };
+    pkgs.writeText "pixieboot-api-${name}" (builtins.toJSON {
+      kernel = "file://${boot.kernel}/bzImage";
+      initrd = [ "file://${boot.netbootRamdisk}/initrd" ];
+      cmdline = concatStringsSep " " [
+        "init=${boot.toplevel}/init"
+        "loglevel=4"
+        "nixos.install=${install}"
+      ];
+      message = "NixOS Automatic Installer for ${name}";
+    });
 
+  api = pkgs.linkFarm "pixiecore-api" (mapAttrs'
+    (mac: name: nameValuePair "pixiecore/v1/boot/${mac}" (apiEntry name))
+    targets);
 in
 {
-  environment.systemPackages = [ commands ];
+  services.pixiecore = {
+    enable = true;
+    mode = "api";
+    dhcpNoBind = true;
+    debug = true;
+    openFirewall = true;
+    port = 5080;
+    statusPort = 6080;
+    apiServer = "http://boot.${config.networking.domain}/pixiecore";
+  };
+
+  services.nginx = {
+    virtualHosts = {
+      "boot.${config.networking.domain}" = {
+        locations."/".proxyPass = "http://localhost:${toString config.services.pixiecore.port}";
+        locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}";
+        locations."/pixiecore".root = api;
+      };
+    };
+  };
 
   services.dhcpd4 = {
     enable = true;
@@ -86,8 +102,7 @@ in
     '';
   };
 
-  networking.firewall = {
-    allowedTCPPorts = [ 4011 64172 ];
-    allowedUDPPorts = [ 67 69 ];
-  };
+  hpc.hostFile.aliases = [
+    "boot.${config.networking.domain}"
+  ];
 }

machines/manager/netinstall/installer.nix

@@ -1,8 +1,10 @@
-{ pkgs, lib, config, modulesPath, managerConfig, ... }:
+{ pkgs, lib, config, modulesPath, nodes, ... }:
 
 with lib;
 
 let
+  manager = nodes."manager".config;
+
   auto-install = pkgs.writers.writeBash "auto-install" ''
     set -o errexit
     set -o nounset
@@ -31,15 +33,17 @@ in
 {
   imports = [
     "${modulesPath}/installer/netboot/netboot-minimal.nix"
+    ../../../modules
   ];
 
   config = {
     services.getty.autologinUser = lib.mkForce "root";
 
     networking.hostName = "installer";
-    networking.hosts = mkForce managerConfig.networking.hosts;
 
-    users.users."root" = mkForce managerConfig.users.users."root";
+    users.users."root" = {
+      openssh.authorizedKeys.keys = manager.users.users."root".openssh.authorizedKeys.keys;
+    };
 
     systemd.services."auto-install" = {
       description = "Automated NixOS installer";
@@ -72,10 +76,10 @@ in
 
     nix.settings = {
       substituters = [
-        "http://cache.${managerConfig.networking.domain}"
+        "http://cache.${manager.networking.domain}"
       ];
       trusted-public-keys = [
-        (fileContents managerConfig.gather.parts."cache/key".path)
+        (fileContents manager.gather.parts."cache/key".path)
       ];
     };
 

machines/manager/network.nix

@@ -10,4 +10,6 @@
     address = "10.32.47.1";
     interface = "enp11s0f0";
   };
+
+  networking.firewall.extraInputRules = "ip saddr 10.32.47.0/24 accept";
 }

shared/network.nix

@@ -15,9 +15,4 @@
   networking.useDHCP = false;
 
   networking.nftables.enable = true;
-
-  services.openssh = {
-    enable = true;
-    settings.PermitRootLogin = "without-password";
-  };
 }

shared/secrets.yaml

@@ -12,64 +12,91 @@ sops:
         - recipient: age1ys5pskgkjsgqfy2lr0afcnl2edry8jmryhymkwtked2se74e9g4s23gunn
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpUUM1bnFZaTgwZ2dLK201
-            WXRQc2VFanJrd29mekpoNkxFcks4MnU4clFzCmdMa0xnemlVdlBhWE9NMW02NlZS
-            elZLa2VqSlBlQ2RrWkxGMXM3TWV4c2sKLS0tIFBTTG9rWEd2bmliZitjT0NtUExk
-            bWh6NDZiQlo0UHJyR2UrK3hhNzdBRVEKQYIQoTini1ptuCev4jNuZI9KikPOyn+k
-            z5oV9bQWMQ9Lr+oPYeT03ttMcKwtYy5MXJURe1JnVf3ARWWlKSk6LA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQd1NpZWtLR09ibjdibHNB
+            T3dxZ0U4RWxQT01zbVRiMHU2LzhsK3VvL0JzClVmZ1ZrTTI2MmxJOGlzSU1pVVpE
+            aHdQVDF4OE5xNWhxa2MzZXJjcGYrOGMKLS0tIG9OMGVOTnUxc0hISGg2QnhXOEta
+            RDJDNEw4R1o4Rm4rRzBrajI4bEJyK1kKNaZ1UPH0o3LHTFjqWsGoGPCB+2jtGxnF
+            n2OvPt5Rp24QThFwcrdAj9L6TGSo5CSKtUwJR5OWvMY2bFf0ZzyvMw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1ll2utvgdlmg2mrdh7xcxw93cdlghrlfxjj4fqmaxamem6vztsecsmghfek
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkUVlxdFBIdVFsS1NvNlhH
-            Wm40NkxCZlI0aVBaZFNQUmtCN2lsUHhZL1RVClFRdTE2VXpzZUVlYk1ibnUwRkp2
-            cUs1SXdQVU1sbDh5b2xLc2J2NDJzcUkKLS0tIHcyMlRnWEhsMzN1MGg3dVlCL1dt
-            TmlKSGJPRjNwV0NYakxhc25oZDhxUmcKrrdIq7F9/swLlXMiZDxwjPO0htl8rLX4
-            vU4BBE2/sT9w+jl3N0Z4jYJ6sH2DRWeUHim10jrolR3mUVXQFWTelQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDOGQ3VFQvNFlIdUxaTllt
+            VmpPc1VtRmtramFJUy8wOGU3cTk3OGF5dlNjClh2cVI4ZVhyM1k4NGZjc3hsc1c5
+            MzVQNXVydzVoWDdKZG5BcHhyQVY1NmsKLS0tIDNzYi9KUVpkcmJoNWsyUWJ6T3RI
+            dWRBVWRaYmp6Wmt3Y2pNWXQyV0lYNGsKY4Pn/jxHZV3rv5ImSxYORPStxyJxQRvq
+            r1mZyps568LLoEagvooanxoAYRWifFl7BZrEPNutkNHhSmyyZ+Bbdw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1ujldj2gprjmqjcn376mtj8chskyk40gvst3m765td8za9qcd2fksuyz2h3
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyODJxcEFMK1krQ016NDg4
-            bW5OVlZ6NmlkeHJjcGRhQk9QSnlFK3VVUWdzCitaVEd4ZXk3Q3pwbVlJL1RCVm5x
-            QW91V0ZlV25KRy9ob3ZINDJvcGQ3dUkKLS0tIGk0Qnc0Qi9ZREp4TnlOUGJkSHBY
-            K3BzaG9sc1lUZFZSZkljMWNWdDFnQTgKdBHdbcnbCUZvb9w35mKfvTvpDWgNJ10q
-            QTPRnIBYgGSvqb0SVuYZJZFRVB/V23Rt9g3pCyolo6WH7ZITivxRTg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTL1dKUEl1MDFxc1g4bU41
+            U2NZQldualg5WEozRnV6NHROVmZUMm80RjE0CkVPamx5TjRZcXo2ZVVoa2JJUVlh
+            b210M2Z5NzJMZW53RVNZZ0RhUTdzNU0KLS0tIHBKVm5zK3FldEwzSTN3c2FCR2NU
+            bCs2R3dKQmdvMU42ZEJGRWJhWm9JZ0kK5dt9W69nnsiyHb0KmzdtDraid9AnXl+o
+            Np+JU91sRkWr1yJekCNk6MpF8neGY6hZN1UufP0TovrqCshMwcaluQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1e8f629xakqvc6gl25j36d46vl4tqnntjfqv2re54savhtc9ysqrsj3tu09
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2L0lnT0NrU2FUbGtzWDhZ
-            MlRHWGJWQnNodytoT0xpelo2SzIwamRjSzBJClN0THFMbjNFcTNuaUJ4UldzQ2Jt
-            VEZFalZpczdmT09lOUp6djAxalBSUGcKLS0tIFk2Mi83SDJSc3J3TzN6cUQrajRv
-            eDZoRkxZNjBadWRzVTFsWlBmeGNFemsKKDq7jApNgZUQLyjPI6a9Wq4Txnv+atFa
-            ESwVNDJNGtIwVTUp1B28VUJZWtJOoqgW1rq1FQ7MPXBS5pt9tmxK9Q==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwQXdsam12NDFndjZzK0VF
+            eXg2TFVUM1Z1N0VWQ0FabkNrZVdKTDF2aXp3CnZZZG5ES0JyU0VLYlM2dWR6enI3
+            OER5eitEOEhUQlhnY0lsV0lwK2pWVk0KLS0tIGJROHV6aFU0M2tuZE1qL2tiOGZq
+            SGxWd2NGNndpQWNWbENMWm5VZ216M2sKDH/ZbOVccBsiBErNkHgCy1y9KdjLd/DI
+            ob9RD7pUf7XMF1sH7weAsQCdAd/M77B17qNp3BJJZYDftKvRPnx/0w==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1lqtfrgk2nfdgqm3tculqlph8r8nthrv7frzk7p8vxurwgwudedqs5s04d3
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxOEZKNjJnNEhkenhGdEla
-            UUtncHoyL1R2Y1YxZW9ZL2VWNXpidFoxSFJrCklvVzdYeUVyK0RRZ0E5b2U4c3pV
-            eEFZaGZQcmlHT0t2czlKZ2poMTkra0EKLS0tIEI0OGR1ay9hS2lDMksxaFBmZXRP
-            QkNZNEc1ZjdpeWJXRWtuQUhGaHlTbjAKKDP35Haj2ZIeECvm5AjJDZGEbJNiYGuJ
-            5tnDJtB3mMrBrDosYd2kffdbKCl3yp/CsWm6H6y5dXJJ51MFcdJddw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXS3Q3Y09Wc0VLNlNXTUZE
+            eG9VZThsVU1BbGZKZUo3ZmUyckF0NFZvRTE0CnZZVkVDSUQ0OVExOFdwL0RpMjVL
+            LzArUXl1M1NWcDQvRTQ1NFR1a3FCSWMKLS0tIGtUOGRwcjd2bVdBQmtVVW5paHEr
+            anY4ODJXZWEyeGNtNXRFb1NFVGhDMW8KprFkCdMO5HeWO3GzqsLR6a8t3W9ilUzs
+            NTJfB/Eub7My/Pw0FpA4n3WU28MQBPJe65+ikFOlTkrurszqFdmKOQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1hw95hm8056j7xu2dj96g95zqrnd6end664ws93ekqzv2xj0re3rs6yz0mf
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByNzEvZFRmMXI4d0Vpck5p
+            SE1uSXpsdWdWRDdaclk2eE5EQnkyYXBSVWs4CldNR1NRSlZjT1pjczQ1MXllUmFp
+            cW9VRUNzWHpDbWxEV0oxLzBFYUMzSFUKLS0tIHV1UjZaN1ZOY3pxVk9zblZZWmsz
+            KzVvSkdoS2tlQ1RSMWxvMkNveFEvU3cKi53Z4YfIP6mxTh/rR8sl16SBqJofXCc+
+            oOu7oBCnQQql7Zlk/ZRg2aLPHPtDh2wo14oL6lgJS8YqjG8k8lGPrg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1upmemuj08e27ug0us2stzl7ksmxynqcs4q0dweuhn59w0kfd4vsqr25ylh
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoMFZ1ZFdKUWFuRW5LR2d0
+            SjQ0VjNHVGNMNWRhbG9sNFFzRWk5QU45Umg4Cm1CVG05TmV1VTczS0x2a3JHZEJr
+            K0pJTWFXREpFTnRuczIyUjhhMGNjOVEKLS0tIGFrSjlYMXBzMnpvSUdGYWtJMjIv
+            aXpBSmlmOVNaZXR0VEVuaE9CemZTd0EKDz/F6Cl2ERWY8LPQGqT130AYNP4pMdoN
+            vVRWwo5PqXyZY+vBR4FkzpnaDOSDWHNMT0LDh+JzB0TkA/dI7+zOwg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1qwjtghh747xx7ssfyq48g9rucnwnfa3eslyk7futw2rqeakeuayqzlwj6g
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAraW1jdGRkMDlQWXVyQm9S
+            VW5KcUE2U3UzWFV0VVNEV1lRRG11SzMwSVJ3CmN6STlIODVRSG1ieWY0OTcvaDk1
+            RFJtMlNlY1lIWEExcGpWZlZGVmYzbkEKLS0tIDExeUMxM1gzZlROdHU5YjIyUFdY
+            c01zMTA2dnZlTGxGd0oxYklXMHdWTUkKYjSQ7Y16AXfiLaOwsyV8LFjUtbUJ744A
+            uxlImBcQnbiPkJY4DRxmtrBrTSzfX1pdepNH8DR0ZpjpI/6bibHEag==
             -----END AGE ENCRYPTED FILE-----
     lastmodified: "2023-06-27T09:58:35Z"
     mac: ENC[AES256_GCM,data:pPgwJnUdwQegqaCXdh7lweQq2Kos6szvo/mfBul+2TruUSSRXlGwKmNVLM2BuodMNZpTan2vCyvVlXvN4zBfW6nVWPzlBrCTbgtyBNodB+k3OJsfgUElQ32T9KccsMVuUsfKDzjhlFnV3NA9A7DVnrYz+jf1NcNSsz4yOjHudzA=,iv:ciFHyXhIcNFlB9fhzcAX8LICIsGPWDe29fxtjmJ0G+s=,tag:oldhGvm8vfPnuhpIXIpVWw==,type:str]
     pgp:
-        - created_at: "2023-07-03T09:59:45Z"
+        - created_at: "2023-07-05T13:59:05Z"
           enc: |
             -----BEGIN PGP MESSAGE-----
 
-            hQEMA5ntoryXZPD4AQf+JS+pr3qsdGsiIUYvY6K5H877NUtq1/SKAZxZnYwXnWZ5
-            UKrGXfavvh/OL9T/19BcK/eo6GSzrj/yvUHr6cDkJWgZezMaGPMMAR1r/2x2Wg9b
-            a7oewcOHmwN0yegVOZQilkbIfTebTGgGb4PnqT5w7bwrGVBjKjXNBUekObLR4O45
-            jjvNNhGAoOcemRVms4ErdQ0aRPt+8yNVcmOF9gIrNcvMvkO3IfvN1JTzwkoFEm9j
-            WYgPpvByesBWmbL76Xd95QGfJxOaIHDXgV+sSs8DA8xS81H0j0q2A+krVGpdu0nW
-            6tej5P9OlI6atQkmf66eca8di4ztwj1uGZjc9ocYEdJeAYm3Fg20OeYrKuN3ApfX
-            wWSkWm9b89GuptH5dcqlQsSSf6gpJ/9vfuKuurQiv7k2B5Ge4GD/bRcfTIQyz/mx
-            sqCxGbnVe4T/bhTAK8Ah60ZsuTRx463EXTpykFwoyA==
-            =agMc
+            hQEMA5ntoryXZPD4AQf+P0suHHym/OGK4CzSYSAI7cZZUzcF1skOvm+IX5nldLMx
+            TcF7t4x4dfwZmwKs34eVWI37VrULbeoLoFMPH98+VRj10OlOWxqoyl2xpz4cjGfa
+            LHnugGrFc5O0mKfeEvK5+2bmukzCKZ0ug2I1pifyLItYxMZl9/udX1aDTS/1qN+s
+            yYV0mZgOb/SV6v1i79CVFsfms0L8jouElx25CWS4cH12scuRURMse9dIuEkOn5kA
+            hxkIn5sa4ZuE2OqjrVWKZtiQ0P4kpISdbnsBdvMiultnL/kbNM2s67cwW5GPaGT4
+            Rksg/i6jZKjNbyNEP+0K+YugT99LzlILtwLnJ7nMVtJeAUpZDQyrPid49r91i78w
+            i90hVxPWCZdul0Ao6051Ga8vx2z+OTm/wtq4+ZQat2J9p+lKjsECgWFCpLiH1Ljx
+            Q5JfM0DzYAifr1ny/CgUAPzwpqkjhe8E2njBsdbXBw==
+            =DcfT
             -----END PGP MESSAGE-----
           fp: 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
     unencrypted_suffix: _unencrypted

shared/ssh.nix

@@ -3,6 +3,11 @@
 with lib;
 
 {
+  services.openssh = {
+    enable = true;
+    settings.PermitRootLogin = "without-password";
+  };
+
   programs.ssh = {
     # Add know-host entries for all machines in the cluster
     knownHosts = mapAttrs