installer ground work

gathered/manager/cache-pub-key.pem

@@ -0,0 +1 @@
+cache.hpc.informatik.hs-fulda.de:/CF5K1gnvbCQHyMFRd7fpEPgfKZeSS6zPAkgkY/u4NY=

machines/manager/cache.nix

@@ -1,19 +1,12 @@
-{ config, ... }:
+{ pkgs, config, ... }:
 
 {
   services.nix-serve = {
     enable = true;
-    secretKeyFile = config.sops.secrets."cache/privateKey".path;
-  };
-
-  sops.secrets."cache/privateKey" = {
-    format = "binary";
-    sopsFile = ../../secrets/cache-priv-key.pem;
+    secretKeyFile = "/var/lib/cache/priv-key.pem";
   };
 
   services.nginx = {
-    enable = true;
-    recommendedProxySettings = true;
     virtualHosts = {
       "cache.${config.networking.domain}" = {
         locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
@@ -24,4 +17,19 @@
   hpc.hostFile.aliases = [
     "cache.${config.networking.domain}"
   ];
+
+  system.activationScripts."generate-binary-cache-key" = ''
+    if ! [ -f "/var/lib/cache/priv-key.pem" ]; then
+      mkdir -pv /var/lib/cache
+      ${pkgs.nix}/bin/nix-store --generate-binary-cache-key \
+         "cache.${config.networking.domain}" \
+         /var/lib/cache/priv-key.pem \
+         /var/lib/cache/pub-key.pem
+     fi
+  '';
+
+  gather.parts."cache/key" = {
+    name = "cache-pub-key.pem";
+    file = "/var/lib/cache/pub-key.pem";
+  };
 }

machines/manager/default.nix

@@ -11,10 +11,11 @@ with lib;
     ./nfs.nix
     ./ldap.nix
     ./users.nix
+    ./nginx.nix
     #./beegfs.nix
     ./ntp.nix
     #./gateway.nix
-    #./netinstall.nix
+    ./netinstall
     ./cache.nix
     ./rdma.nix
     ./mpi.nix

machines/manager/netinstall.nix

@@ -1,33 +0,0 @@
-# TFTP boot with shared image
-# Requests store path to install from master
-# Runs disko and nixos-install
-
-{ pkgs, lib, config, ... }:
-
-{
-  services.pixiecore = {
-    enable = true;
-    mode = "api";
-    dhcpNoBind = true;
-    debug = true;
-    statusPort = 6080;
-    apiServer = "http://boot.${config.networking.domain}/pixiecore";
-  };
-
-  services.nginx = {
-    enable = true;
-    recommendedProxySettings = true;
-    virtualHosts = {
-      "boot.${config.networking.domain}" = {
-        locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}";
-        locations."/pixiecore" = {
-          root = "/srv/pixieboot";
-        };
-      };
-    };
-  };
-
-  hpc.hostFile = [
-    "boot.${config.networking.domain}"
-  ];
-}

machines/manager/netinstall/default.nix

@@ -0,0 +1,69 @@
+# TFTP boot with shared image
+# Requests store path to install from master
+# Runs disko and nixos-install
+
+{ pkgs, lib, config, nodes, ... }@args:
+
+with lib;
+
+let
+  targets = {
+    "50:46:5d:da:0b:d6" = "node-00";
+  };
+
+  installer = pkgs.nixos [
+    ./installer.nix
+    {
+      _module.args = {
+        manangerConfig = config;
+      };
+    }
+  ];
+
+  api = pkgs.linkFarm "pixiecore-api" (mapAttrs'
+    (mac: name: nameValuePair
+      "v1/boot/${mac}"
+      (pkgs.writeText "pixieboot-api-${name}" (
+        let
+          boot = installer.config.system.build;
+          node = nodes.${name}.config.system.build;
+        in
+        builtins.toJSON {
+          kernel = "file://${boot.kernel}/bzImage";
+          initrd = "file://${boot.netbootRamdisk}/initrd";
+          cmdline = concatStringsSep "\n" [
+            "init=${boot.toplevel}/init"
+            "loglevel=4"
+            "nixos.install=${node.toplevel}"
+          ];
+          message = "NixOS Automatic Installer for ${name}";
+        }
+      )))
+    targets);
+in
+{
+  services.pixiecore = {
+    enable = true;
+    mode = "api";
+    dhcpNoBind = true;
+    debug = true;
+    openFirewall = true;
+    port = 5080;
+    statusPort = 6080;
+    apiServer = "http://boot.${config.networking.domain}/pixiecore";
+  };
+
+  services.nginx = {
+    virtualHosts = {
+      "boot.${config.networking.domain}" = {
+        locations."/".proxyPass = "http://localhost:${toString config.services.pixiecore.port}";
+        locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}";
+        locations."/pixiecore".root = api;
+      };
+    };
+  };
+
+  hpc.hostFile.aliases = [
+    "boot.${config.networking.domain}"
+  ];
+}

machines/manager/netinstall/installer.nix

@@ -0,0 +1,64 @@
+{ pkgs, lib, config, modulesPath, manangerConfig, ... }:
+
+with lib;
+
+let
+  auto-install = pkgs.writeShellScript "nixos-install" ''
+    if [[ "$(cat /proc/cmdline)" =~ nixos\.install=([^ ]+) ]]; then
+      INSTALL="''${BASH_REMATCH[1]}"
+    else
+      echo "No install derivation found" >&2
+      exit 1
+    fi
+  '';
+in
+{
+  imports = [
+    "${modulesPath}/installer/netboot/netboot-minimal.nix"
+  ];
+
+  config = {
+    services.getty.autologinUser = lib.mkForce "root";
+
+    networking.hostName = "installer";
+    networking.hosts = mkForce manangerConfig.networking.hosts;
+
+    users.users."root" = mkForce manangerConfig.users.users."root";
+
+    systemd.services."auto-install" = {
+      description = "Automated NixOS installer";
+
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+
+      wantedBy = [ "multi-user.target" ];
+
+      unitConfig = {
+        AssertKernelCommandLine = "nixos.install";
+
+        FailureAction = "reboot-force";
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+
+        ExecStart = auto-install;
+
+        StandardInput = "tty-force";
+        TTYPath = "/dev/tty1";
+        TTYVTDisallocate = false;
+      };
+    };
+
+    nix.settings = {
+      substituters = [
+        "http://cache.${manangerConfig.networking.domain}"
+      ];
+      trusted-public-keys = [
+        (fileContents manangerConfig.gather.parts."cache/key".path)
+      ];
+    };
+
+    system.stateVersion = config.system.nixos.release;
+  };
+}

machines/manager/network.nix

@@ -10,8 +10,4 @@
     address = "10.32.30.1";
     interface = "enp11s0f0";
   };
-
-  networking.firewall.trustedInterfaces = [
-    "enp11s0f0"
-  ];
 }

machines/manager/nginx.nix

@@ -0,0 +1,8 @@
+{
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+  };
+
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+}

machines/node/default.nix

@@ -1,5 +1,5 @@
 { id, ... }:
-{ lib, config, ... }:
+{ lib, config, nodes, ... }:
 
 with lib;
 
@@ -25,7 +25,7 @@ with lib;
       "http://cache.${config.networking.domain}"
     ];
     trusted-public-keys = [
-      "cache.${config.networking.domain}:dc2abEGJAQfaZiBXhjvjPU0jx/wosQwAOQoz48/G6cA="
+      (fileContents nodes."manager".config.gather.parts."cache/key".path)
     ];
   };
 }

machines/node/hardware.nix

@@ -8,7 +8,6 @@ with lib;
   ];
 
   boot.loader.grub.enable = true;
-  boot.loader.grub.version = 2;
   boot.loader.grub.device = "/dev/sda";
 
   boot.initrd.systemd.enable = true;

secrets/cache-priv-key.pem

@@ -1,30 +0,0 @@
-{
-	"data": "ENC[AES256_GCM,data:zFVNY6fYkVEvHcZ/IaWvcmIkf+NwZ9p45XEy7/sxpSvr62F80pzxAiC99IX+1+XLH83zk5dqm1vMUuX9NdNAxB0Mousyp1YdkF0Zqi5/il9B/p7R24AIfgeQCa46qo5MbYVWRgs6R1rp9Y573+6/SbPtDqoChvE1Kic=,iv:uQa4O9WnyFZ+kPvp/ozXilCTyUJcLvwlVWF7rmTi9w8=,tag:2MuFj4/Mn9LECE7cToQwVQ==,type:str]",
-	"sops": {
-		"kms": null,
-		"gcp_kms": null,
-		"azure_kv": null,
-		"hc_vault": null,
-		"age": [
-			{
-				"recipient": "age1ys5pskgkjsgqfy2lr0afcnl2edry8jmryhymkwtked2se74e9g4s23gunn",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwcnAzWkhKbGdjQ1g4WjVw\na3YrRjkzczVuNjYrQjJ5Yk9qeFFOS0dEdlVVCncwdU9JdVB2ekJSdXhNQmtJd0dH\neEIyK25pdVhpTzIzTUdvYlJGaDBvQVEKLS0tIDNsQ2J0ampueWZuQkNnQ2tFWEwv\nOVdyYzI2emh5SktqQUljbUhuajR3NTQKfG5O3ToSgBzR+/LHLyq7IUkLNRFeI6zh\n9u2pkCMncrUHAqpHJUfhnd39pke4Hg8op2DPLq9y7vj0s3DJ2HyJWQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			},
-			{
-				"recipient": "age1q3tqh4w7yeae4xs0cxevtp5tn4gm8xthc39fsht2kv9rq7xm4q3qxqt9sh",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZE1ndVZUenBCa2dva3dh\nT1hLMEFJSVNzejVtdXZnSStrWGtRd3IxZHlzClkxMmN5Q3FtL1pUcklkZXB6alZr\nNjJ6RzRwdFBDaFY5K2I0dlI4WWF3SVEKLS0tIE1zMGZkZWNTTjJEcnFNcWxlb0E4\nbmd3ejZ4S0V5alh1ZFZRd2IrckpybUkKY9KS0r71NIye4Bf8Ekqi90e4/7I8hg/V\nOA1bfKGo+sb9nD5HTBKEc+ssTVN21xWd9z6GksVjU9l2M5VoLwTkhQ==\n-----END AGE ENCRYPTED FILE-----\n"
-			}
-		],
-		"lastmodified": "2023-05-26T08:59:34Z",
-		"mac": "ENC[AES256_GCM,data:8h8NREXye3DDL7DpvT7sVr1lyaAfEgDwOoaDMuCzzRyHFWPSELQHnjLjEjmexoRrrsE/U608/h62PU7m9EDSYuWlJsvuNBZ+HezR/Ve8oFrZ5ZE3HIoEt2aeM2enSEHGP+aYFL4jEZJJDn9xoW3chFu3JLTSez0NOAhuejghjnU=,iv:Dfxlfa/mwKswYL077oPV+rylKk5y67qKPz+6UFCje9c=,tag:lmM0U8H5FlVRMO51mqTZgg==,type:str]",
-		"pgp": [
-			{
-				"created_at": "2023-06-01T13:41:29Z",
-				"enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA5ntoryXZPD4AQf/STkH8HafCMan6au+LKbb5DriplyRLLPLzDcCvNn/VD5H\nYQU8rn/iJajpvbKxgBYo8c3bgz9hz+qfM1aSF57ezHkuiDHd0DDlnEHXGDfEsy5b\nnxPxXA432d412sfbjC69cqBba9mGYV88URplVm40RqyqZr+drnF6bsu3r5gY1sJT\nwG5ZYyyhXTO02ePYuAlS5J0yihHzA3rtWR7VEL5zwJVRo3D1fhMA0ZEnjCc9j14E\nT9yrOQZ1fPhiAJcvbWWxGWwDa50DpVGVBRwZ+N8mWbRN+Py4/OsjEe8f8s2h2IEp\nGKGirTIcc6hRhoOBRTNBmNeuTDbI04r+ai8XZBYxNNJeARvh1kh+5lx7gln92R7r\nDcgWchi/PioCHvDr9lfusuhio6rbAfS7LZ5fVREyHqRomQJEfFuq9Vder6cBYT+0\nd2/TG3Qc02Q0Q1yKXT3Fm+O9g8tXTWPyuZNt70npRA==\n=q6EO\n-----END PGP MESSAGE-----\n",
-				"fp": "3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE"
-			}
-		],
-		"unencrypted_suffix": "_unencrypted",
-		"version": "3.7.3"
-	}
-}

shared/default.nix

@@ -38,6 +38,10 @@
     htop
     iotop
     iftop
+    git
+    stdenv
+    gnumake
+    autoconf
   ];
 
   # Performance FTW!

shared/network.nix

@@ -14,8 +14,10 @@
 
   networking.useDHCP = false;
 
+  networking.nftables.enable = true;
+
   services.openssh = {
     enable = true;
-    permitRootLogin = "without-password";
+    settings.PermitRootLogin = "without-password";
   };
 }

shared/rdma.nix

@@ -8,4 +8,6 @@ with lib;
   environment.systemPackages = [
     pkgs.rdma-core
   ];
+
+  networking.firewall.extraInputRules = "meta iifname \"ib*\" accept";
 }