From f5465b23ee588905f1a6f19b75fa73d27b59c2c1 Mon Sep 17 00:00:00 2001 From: Dustin Frisch Date: Thu, 15 Jun 2023 13:39:23 +0200 Subject: [PATCH] installer ground work --- gathered/manager/cache-pub-key.pem | 1 + machines/manager/cache.nix | 26 ++++++--- machines/manager/default.nix | 3 +- machines/manager/netinstall.nix | 33 ----------- machines/manager/netinstall/default.nix | 69 +++++++++++++++++++++++ machines/manager/netinstall/installer.nix | 64 +++++++++++++++++++++ machines/manager/network.nix | 4 -- machines/manager/nginx.nix | 8 +++ machines/node/default.nix | 4 +- machines/node/hardware.nix | 1 - secrets/cache-priv-key.pem | 30 ---------- shared/default.nix | 4 ++ shared/network.nix | 4 +- shared/rdma.nix | 2 + 14 files changed, 172 insertions(+), 81 deletions(-) create mode 100644 gathered/manager/cache-pub-key.pem delete mode 100644 machines/manager/netinstall.nix create mode 100644 machines/manager/netinstall/default.nix create mode 100644 machines/manager/netinstall/installer.nix create mode 100644 machines/manager/nginx.nix delete mode 100644 secrets/cache-priv-key.pem diff --git a/gathered/manager/cache-pub-key.pem b/gathered/manager/cache-pub-key.pem new file mode 100644 index 0000000..e2f9370 --- /dev/null +++ b/gathered/manager/cache-pub-key.pem @@ -0,0 +1 @@ +cache.hpc.informatik.hs-fulda.de:/CF5K1gnvbCQHyMFRd7fpEPgfKZeSS6zPAkgkY/u4NY= \ No newline at end of file diff --git a/machines/manager/cache.nix b/machines/manager/cache.nix index 940c699..cad1ef7 100644 --- a/machines/manager/cache.nix +++ b/machines/manager/cache.nix @@ -1,19 +1,12 @@ -{ config, ... }: +{ pkgs, config, ... }: { services.nix-serve = { enable = true; - secretKeyFile = config.sops.secrets."cache/privateKey".path; - }; - - sops.secrets."cache/privateKey" = { - format = "binary"; - sopsFile = ../../secrets/cache-priv-key.pem; + secretKeyFile = "/var/lib/cache/priv-key.pem"; }; services.nginx = { - enable = true; - recommendedProxySettings = true; virtualHosts = { "cache.${config.networking.domain}" = { locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}"; @@ -24,4 +17,19 @@ hpc.hostFile.aliases = [ "cache.${config.networking.domain}" ]; + + system.activationScripts."generate-binary-cache-key" = '' + if ! [ -f "/var/lib/cache/priv-key.pem" ]; then + mkdir -pv /var/lib/cache + ${pkgs.nix}/bin/nix-store --generate-binary-cache-key \ + "cache.${config.networking.domain}" \ + /var/lib/cache/priv-key.pem \ + /var/lib/cache/pub-key.pem + fi + ''; + + gather.parts."cache/key" = { + name = "cache-pub-key.pem"; + file = "/var/lib/cache/pub-key.pem"; + }; } diff --git a/machines/manager/default.nix b/machines/manager/default.nix index 720f773..f50e049 100644 --- a/machines/manager/default.nix +++ b/machines/manager/default.nix @@ -11,10 +11,11 @@ with lib; ./nfs.nix ./ldap.nix ./users.nix + ./nginx.nix #./beegfs.nix ./ntp.nix #./gateway.nix - #./netinstall.nix + ./netinstall ./cache.nix ./rdma.nix ./mpi.nix diff --git a/machines/manager/netinstall.nix b/machines/manager/netinstall.nix deleted file mode 100644 index e085432..0000000 --- a/machines/manager/netinstall.nix +++ /dev/null @@ -1,33 +0,0 @@ -# TFTP boot with shared image -# Requests store path to install from master -# Runs disko and nixos-install - -{ pkgs, lib, config, ... }: - -{ - services.pixiecore = { - enable = true; - mode = "api"; - dhcpNoBind = true; - debug = true; - statusPort = 6080; - apiServer = "http://boot.${config.networking.domain}/pixiecore"; - }; - - services.nginx = { - enable = true; - recommendedProxySettings = true; - virtualHosts = { - "boot.${config.networking.domain}" = { - locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}"; - locations."/pixiecore" = { - root = "/srv/pixieboot"; - }; - }; - }; - }; - - hpc.hostFile = [ - "boot.${config.networking.domain}" - ]; -} diff --git a/machines/manager/netinstall/default.nix b/machines/manager/netinstall/default.nix new file mode 100644 index 0000000..d764248 --- /dev/null +++ b/machines/manager/netinstall/default.nix @@ -0,0 +1,69 @@ +# TFTP boot with shared image +# Requests store path to install from master +# Runs disko and nixos-install + +{ pkgs, lib, config, nodes, ... }@args: + +with lib; + +let + targets = { + "50:46:5d:da:0b:d6" = "node-00"; + }; + + installer = pkgs.nixos [ + ./installer.nix + { + _module.args = { + manangerConfig = config; + }; + } + ]; + + api = pkgs.linkFarm "pixiecore-api" (mapAttrs' + (mac: name: nameValuePair + "v1/boot/${mac}" + (pkgs.writeText "pixieboot-api-${name}" ( + let + boot = installer.config.system.build; + node = nodes.${name}.config.system.build; + in + builtins.toJSON { + kernel = "file://${boot.kernel}/bzImage"; + initrd = "file://${boot.netbootRamdisk}/initrd"; + cmdline = concatStringsSep "\n" [ + "init=${boot.toplevel}/init" + "loglevel=4" + "nixos.install=${node.toplevel}" + ]; + message = "NixOS Automatic Installer for ${name}"; + } + ))) + targets); +in +{ + services.pixiecore = { + enable = true; + mode = "api"; + dhcpNoBind = true; + debug = true; + openFirewall = true; + port = 5080; + statusPort = 6080; + apiServer = "http://boot.${config.networking.domain}/pixiecore"; + }; + + services.nginx = { + virtualHosts = { + "boot.${config.networking.domain}" = { + locations."/".proxyPass = "http://localhost:${toString config.services.pixiecore.port}"; + locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}"; + locations."/pixiecore".root = api; + }; + }; + }; + + hpc.hostFile.aliases = [ + "boot.${config.networking.domain}" + ]; +} diff --git a/machines/manager/netinstall/installer.nix b/machines/manager/netinstall/installer.nix new file mode 100644 index 0000000..8afe46a --- /dev/null +++ b/machines/manager/netinstall/installer.nix @@ -0,0 +1,64 @@ +{ pkgs, lib, config, modulesPath, manangerConfig, ... }: + +with lib; + +let + auto-install = pkgs.writeShellScript "nixos-install" '' + if [[ "$(cat /proc/cmdline)" =~ nixos\.install=([^ ]+) ]]; then + INSTALL="''${BASH_REMATCH[1]}" + else + echo "No install derivation found" >&2 + exit 1 + fi + ''; +in +{ + imports = [ + "${modulesPath}/installer/netboot/netboot-minimal.nix" + ]; + + config = { + services.getty.autologinUser = lib.mkForce "root"; + + networking.hostName = "installer"; + networking.hosts = mkForce manangerConfig.networking.hosts; + + users.users."root" = mkForce manangerConfig.users.users."root"; + + systemd.services."auto-install" = { + description = "Automated NixOS installer"; + + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + wantedBy = [ "multi-user.target" ]; + + unitConfig = { + AssertKernelCommandLine = "nixos.install"; + + FailureAction = "reboot-force"; + }; + + serviceConfig = { + Type = "oneshot"; + + ExecStart = auto-install; + + StandardInput = "tty-force"; + TTYPath = "/dev/tty1"; + TTYVTDisallocate = false; + }; + }; + + nix.settings = { + substituters = [ + "http://cache.${manangerConfig.networking.domain}" + ]; + trusted-public-keys = [ + (fileContents manangerConfig.gather.parts."cache/key".path) + ]; + }; + + system.stateVersion = config.system.nixos.release; + }; +} diff --git a/machines/manager/network.nix b/machines/manager/network.nix index b7e6920..78005cd 100644 --- a/machines/manager/network.nix +++ b/machines/manager/network.nix @@ -10,8 +10,4 @@ address = "10.32.30.1"; interface = "enp11s0f0"; }; - - networking.firewall.trustedInterfaces = [ - "enp11s0f0" - ]; } diff --git a/machines/manager/nginx.nix b/machines/manager/nginx.nix new file mode 100644 index 0000000..3b151a7 --- /dev/null +++ b/machines/manager/nginx.nix @@ -0,0 +1,8 @@ +{ + services.nginx = { + enable = true; + recommendedProxySettings = true; + }; + + networking.firewall.allowedTCPPorts = [ 80 443 ]; +} diff --git a/machines/node/default.nix b/machines/node/default.nix index 75edc85..ba9f63d 100644 --- a/machines/node/default.nix +++ b/machines/node/default.nix @@ -1,5 +1,5 @@ { id, ... }: -{ lib, config, ... }: +{ lib, config, nodes, ... }: with lib; @@ -25,7 +25,7 @@ with lib; "http://cache.${config.networking.domain}" ]; trusted-public-keys = [ - "cache.${config.networking.domain}:dc2abEGJAQfaZiBXhjvjPU0jx/wosQwAOQoz48/G6cA=" + (fileContents nodes."manager".config.gather.parts."cache/key".path) ]; }; } diff --git a/machines/node/hardware.nix b/machines/node/hardware.nix index f07b410..d42e157 100644 --- a/machines/node/hardware.nix +++ b/machines/node/hardware.nix @@ -8,7 +8,6 @@ with lib; ]; boot.loader.grub.enable = true; - boot.loader.grub.version = 2; boot.loader.grub.device = "/dev/sda"; boot.initrd.systemd.enable = true; diff --git a/secrets/cache-priv-key.pem b/secrets/cache-priv-key.pem deleted file mode 100644 index dab33f0..0000000 --- a/secrets/cache-priv-key.pem +++ /dev/null @@ -1,30 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:zFVNY6fYkVEvHcZ/IaWvcmIkf+NwZ9p45XEy7/sxpSvr62F80pzxAiC99IX+1+XLH83zk5dqm1vMUuX9NdNAxB0Mousyp1YdkF0Zqi5/il9B/p7R24AIfgeQCa46qo5MbYVWRgs6R1rp9Y573+6/SbPtDqoChvE1Kic=,iv:uQa4O9WnyFZ+kPvp/ozXilCTyUJcLvwlVWF7rmTi9w8=,tag:2MuFj4/Mn9LECE7cToQwVQ==,type:str]", - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": [ - { - "recipient": "age1ys5pskgkjsgqfy2lr0afcnl2edry8jmryhymkwtked2se74e9g4s23gunn", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwcnAzWkhKbGdjQ1g4WjVw\na3YrRjkzczVuNjYrQjJ5Yk9qeFFOS0dEdlVVCncwdU9JdVB2ekJSdXhNQmtJd0dH\neEIyK25pdVhpTzIzTUdvYlJGaDBvQVEKLS0tIDNsQ2J0ampueWZuQkNnQ2tFWEwv\nOVdyYzI2emh5SktqQUljbUhuajR3NTQKfG5O3ToSgBzR+/LHLyq7IUkLNRFeI6zh\n9u2pkCMncrUHAqpHJUfhnd39pke4Hg8op2DPLq9y7vj0s3DJ2HyJWQ==\n-----END AGE ENCRYPTED FILE-----\n" - }, - { - "recipient": "age1q3tqh4w7yeae4xs0cxevtp5tn4gm8xthc39fsht2kv9rq7xm4q3qxqt9sh", - "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCZE1ndVZUenBCa2dva3dh\nT1hLMEFJSVNzejVtdXZnSStrWGtRd3IxZHlzClkxMmN5Q3FtL1pUcklkZXB6alZr\nNjJ6RzRwdFBDaFY5K2I0dlI4WWF3SVEKLS0tIE1zMGZkZWNTTjJEcnFNcWxlb0E4\nbmd3ejZ4S0V5alh1ZFZRd2IrckpybUkKY9KS0r71NIye4Bf8Ekqi90e4/7I8hg/V\nOA1bfKGo+sb9nD5HTBKEc+ssTVN21xWd9z6GksVjU9l2M5VoLwTkhQ==\n-----END AGE ENCRYPTED FILE-----\n" - } - ], - "lastmodified": "2023-05-26T08:59:34Z", - "mac": "ENC[AES256_GCM,data:8h8NREXye3DDL7DpvT7sVr1lyaAfEgDwOoaDMuCzzRyHFWPSELQHnjLjEjmexoRrrsE/U608/h62PU7m9EDSYuWlJsvuNBZ+HezR/Ve8oFrZ5ZE3HIoEt2aeM2enSEHGP+aYFL4jEZJJDn9xoW3chFu3JLTSez0NOAhuejghjnU=,iv:Dfxlfa/mwKswYL077oPV+rylKk5y67qKPz+6UFCje9c=,tag:lmM0U8H5FlVRMO51mqTZgg==,type:str]", - "pgp": [ - { - "created_at": "2023-06-01T13:41:29Z", - "enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA5ntoryXZPD4AQf/STkH8HafCMan6au+LKbb5DriplyRLLPLzDcCvNn/VD5H\nYQU8rn/iJajpvbKxgBYo8c3bgz9hz+qfM1aSF57ezHkuiDHd0DDlnEHXGDfEsy5b\nnxPxXA432d412sfbjC69cqBba9mGYV88URplVm40RqyqZr+drnF6bsu3r5gY1sJT\nwG5ZYyyhXTO02ePYuAlS5J0yihHzA3rtWR7VEL5zwJVRo3D1fhMA0ZEnjCc9j14E\nT9yrOQZ1fPhiAJcvbWWxGWwDa50DpVGVBRwZ+N8mWbRN+Py4/OsjEe8f8s2h2IEp\nGKGirTIcc6hRhoOBRTNBmNeuTDbI04r+ai8XZBYxNNJeARvh1kh+5lx7gln92R7r\nDcgWchi/PioCHvDr9lfusuhio6rbAfS7LZ5fVREyHqRomQJEfFuq9Vder6cBYT+0\nd2/TG3Qc02Q0Q1yKXT3Fm+O9g8tXTWPyuZNt70npRA==\n=q6EO\n-----END PGP MESSAGE-----\n", - "fp": "3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE" - } - ], - "unencrypted_suffix": "_unencrypted", - "version": "3.7.3" - } -} \ No newline at end of file diff --git a/shared/default.nix b/shared/default.nix index f7a9e13..67246b8 100644 --- a/shared/default.nix +++ b/shared/default.nix @@ -38,6 +38,10 @@ htop iotop iftop + git + stdenv + gnumake + autoconf ]; # Performance FTW! diff --git a/shared/network.nix b/shared/network.nix index f36a3e1..a9fb894 100644 --- a/shared/network.nix +++ b/shared/network.nix @@ -14,8 +14,10 @@ networking.useDHCP = false; + networking.nftables.enable = true; + services.openssh = { enable = true; - permitRootLogin = "without-password"; + settings.PermitRootLogin = "without-password"; }; } diff --git a/shared/rdma.nix b/shared/rdma.nix index 4767508..c5e84c0 100644 --- a/shared/rdma.nix +++ b/shared/rdma.nix @@ -8,4 +8,6 @@ with lib; environment.systemPackages = [ pkgs.rdma-core ]; + + networking.firewall.extraInputRules = "meta iifname \"ib*\" accept"; }