Browse Source

More nodes, more bugs

main
Dustin Frisch 1 year ago
parent
commit
e9a577791a
No known key found for this signature in database GPG Key ID: B4C3BF012D9B26BE
  1. 1
      gathered/node-04/ssh_host_ed25519_key.pub
  2. 1
      gathered/node-05/ssh_host_ed25519_key.pub
  3. 1
      gathered/node-06/ssh_host_ed25519_key.pub
  4. 2
      machines.nix
  5. 2
      machines/manager/mpi.nix
  6. 115
      machines/manager/netinstall/default.nix
  7. 14
      machines/manager/netinstall/installer.nix
  8. 2
      machines/manager/network.nix
  9. 5
      shared/network.nix
  10. 97
      shared/secrets.yaml
  11. 5
      shared/ssh.nix

1
gathered/node-04/ssh_host_ed25519_key.pub

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL4iEVnSANFNZOJQF77MfCLv+gyXY5Lj+JTxz4Htm5IU root@node-04

1
gathered/node-05/ssh_host_ed25519_key.pub

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPmFs81JIz4rfku3g6ht6TBkAlOhG3fiaNpk5sUHbLp1 root@node-05

1
gathered/node-06/ssh_host_ed25519_key.pub

@ -0,0 +1 @@
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPEl28pmWLFdL8lpVHuOiOSnAkumnzZpBTwS/rYFtuc root@node-06

2
machines.nix

@ -3,7 +3,7 @@
with lib;
let
nrNodes = 4;
nrNodes = 7;
mkMachine = { name, type, opts ? { } }: rec {
inherit name type opts;

2
machines/manager/mpi.nix

@ -5,7 +5,7 @@ with lib;
{
environment.etc."mpi/hosts" = {
text = concatMapStringsSep "\n"
(node: "${node.config.networking.hostName}")
(node: "${node.config.networking.hostName} max_slots=64")
(filter
(node: elem "node" node.config.deployment.tags)
(attrValues nodes));

115
machines/manager/netinstall/default.nix

@ -10,64 +10,80 @@ let
targets = {
"50:46:5d:da:0b:d6" = "node-00";
"50:46:5d:da:0c:56" = "node-01";
"50:46:5d:da:0c:52" = "node-02";
"10:bf:48:1f:a6:8f" = "node-03";
# "10:bf:48:1b:57:47" = "node-04";
# "10:bf:48:19:a2:4d" = "node-05";
# "10:bf:48:1b:56:df" = "node-06";
"10:bf:48:1f:a6:8f" = "node-02";
"10:bf:48:1b:57:47" = "node-03";
"10:bf:48:19:a2:4d" = "node-04";
"10:bf:48:1b:56:df" = "node-05";
"50:46:5d:da:0c:52" = "node-06";
};
installer = pkgs.nixos [
./installer.nix
{
_module.args = {
managerConfig = config;
nodes = getAttrs [ "manager" ] nodes;
};
}
];
commands = pkgs.symlinkJoin {
name = "pxeboot";
paths = mapAttrsToList
(mac: name:
let
node = nodes.${name}.config.system.build;
boot = installer.config.system.build;
install = pkgs.writers.writeBash "install-${name}" ''
set -o errexit
set -o nounset
set -o pipefail
"${node.diskoScript}"
"${node.nixos-install}/bin/nixos-install" \
--root /mnt \
--system "${node.toplevel}" \
--no-channel-copy \
--no-root-password \
--verbose
reboot
'';
in
pkgs.writers.writeBashBin "pxe-install-${name}" ''
exec ${pkgs.pixiecore}/bin/pixiecore \
boot "${boot.kernel}/bzImage" "${boot.netbootRamdisk}/initrd" \
--cmdline "init=${boot.toplevel}/init loglevel=4 nixos.install=${install}" \
--debug \
--dhcp-no-bind \
--port 64172 \
--status-port 64172 \
"$@"
'')
targets;
};
apiEntry = name:
let
node = nodes.${name}.config.system.build;
boot = installer.config.system.build;
install = pkgs.writeScript "install-${name}" ''
#!/usr/bin/env bash
set -xeuo pipefail
"${node.diskoScript}"
"${node.nixos-install}/bin/nixos-install" \
--root /mnt \
--system "${node.toplevel}" \
--no-channel-copy \
--no-root-password \
--verbose
reboot
'';
in
pkgs.writeText "pixieboot-api-${name}" (builtins.toJSON {
kernel = "file://${boot.kernel}/bzImage";
initrd = [ "file://${boot.netbootRamdisk}/initrd" ];
cmdline = concatStringsSep " " [
"init=${boot.toplevel}/init"
"loglevel=4"
"nixos.install=${install}"
];
message = "NixOS Automatic Installer for ${name}";
});
api = pkgs.linkFarm "pixiecore-api" (mapAttrs'
(mac: name: nameValuePair "pixiecore/v1/boot/${mac}" (apiEntry name))
targets);
in
{
environment.systemPackages = [ commands ];
services.pixiecore = {
enable = true;
mode = "api";
dhcpNoBind = true;
debug = true;
openFirewall = true;
port = 5080;
statusPort = 6080;
apiServer = "http://boot.${config.networking.domain}/pixiecore";
};
services.nginx = {
virtualHosts = {
"boot.${config.networking.domain}" = {
locations."/".proxyPass = "http://localhost:${toString config.services.pixiecore.port}";
locations."/status".proxyPass = "http://localhost:${toString config.services.pixiecore.statusPort}";
locations."/pixiecore".root = api;
};
};
};
services.dhcpd4 = {
enable = true;
@ -86,8 +102,7 @@ in
'';
};
networking.firewall = {
allowedTCPPorts = [ 4011 64172 ];
allowedUDPPorts = [ 67 69 ];
};
hpc.hostFile.aliases = [
"boot.${config.networking.domain}"
];
}

14
machines/manager/netinstall/installer.nix

@ -1,8 +1,10 @@
{ pkgs, lib, config, modulesPath, managerConfig, ... }:
{ pkgs, lib, config, modulesPath, nodes, ... }:
with lib;
let
manager = nodes."manager".config;
auto-install = pkgs.writers.writeBash "auto-install" ''
set -o errexit
set -o nounset
@ -31,15 +33,17 @@ in
{
imports = [
"${modulesPath}/installer/netboot/netboot-minimal.nix"
../../../modules
];
config = {
services.getty.autologinUser = lib.mkForce "root";
networking.hostName = "installer";
networking.hosts = mkForce managerConfig.networking.hosts;
users.users."root" = mkForce managerConfig.users.users."root";
users.users."root" = {
openssh.authorizedKeys.keys = manager.users.users."root".openssh.authorizedKeys.keys;
};
systemd.services."auto-install" = {
description = "Automated NixOS installer";
@ -72,10 +76,10 @@ in
nix.settings = {
substituters = [
"http://cache.${managerConfig.networking.domain}"
"http://cache.${manager.networking.domain}"
];
trusted-public-keys = [
(fileContents managerConfig.gather.parts."cache/key".path)
(fileContents manager.gather.parts."cache/key".path)
];
};

2
machines/manager/network.nix

@ -10,4 +10,6 @@
address = "10.32.47.1";
interface = "enp11s0f0";
};
networking.firewall.extraInputRules = "ip saddr 10.32.47.0/24 accept";
}

5
shared/network.nix

@ -15,9 +15,4 @@
networking.useDHCP = false;
networking.nftables.enable = true;
services.openssh = {
enable = true;
settings.PermitRootLogin = "without-password";
};
}

97
shared/secrets.yaml

@ -12,64 +12,91 @@ sops:
- recipient: age1ys5pskgkjsgqfy2lr0afcnl2edry8jmryhymkwtked2se74e9g4s23gunn
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpUUM1bnFZaTgwZ2dLK201
WXRQc2VFanJrd29mekpoNkxFcks4MnU4clFzCmdMa0xnemlVdlBhWE9NMW02NlZS
elZLa2VqSlBlQ2RrWkxGMXM3TWV4c2sKLS0tIFBTTG9rWEd2bmliZitjT0NtUExk
bWh6NDZiQlo0UHJyR2UrK3hhNzdBRVEKQYIQoTini1ptuCev4jNuZI9KikPOyn+k
z5oV9bQWMQ9Lr+oPYeT03ttMcKwtYy5MXJURe1JnVf3ARWWlKSk6LA==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQd1NpZWtLR09ibjdibHNB
T3dxZ0U4RWxQT01zbVRiMHU2LzhsK3VvL0JzClVmZ1ZrTTI2MmxJOGlzSU1pVVpE
aHdQVDF4OE5xNWhxa2MzZXJjcGYrOGMKLS0tIG9OMGVOTnUxc0hISGg2QnhXOEta
RDJDNEw4R1o4Rm4rRzBrajI4bEJyK1kKNaZ1UPH0o3LHTFjqWsGoGPCB+2jtGxnF
n2OvPt5Rp24QThFwcrdAj9L6TGSo5CSKtUwJR5OWvMY2bFf0ZzyvMw==
-----END AGE ENCRYPTED FILE-----
- recipient: age1ll2utvgdlmg2mrdh7xcxw93cdlghrlfxjj4fqmaxamem6vztsecsmghfek
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkUVlxdFBIdVFsS1NvNlhH
Wm40NkxCZlI0aVBaZFNQUmtCN2lsUHhZL1RVClFRdTE2VXpzZUVlYk1ibnUwRkp2
cUs1SXdQVU1sbDh5b2xLc2J2NDJzcUkKLS0tIHcyMlRnWEhsMzN1MGg3dVlCL1dt
TmlKSGJPRjNwV0NYakxhc25oZDhxUmcKrrdIq7F9/swLlXMiZDxwjPO0htl8rLX4
vU4BBE2/sT9w+jl3N0Z4jYJ6sH2DRWeUHim10jrolR3mUVXQFWTelQ==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDOGQ3VFQvNFlIdUxaTllt
VmpPc1VtRmtramFJUy8wOGU3cTk3OGF5dlNjClh2cVI4ZVhyM1k4NGZjc3hsc1c5
MzVQNXVydzVoWDdKZG5BcHhyQVY1NmsKLS0tIDNzYi9KUVpkcmJoNWsyUWJ6T3RI
dWRBVWRaYmp6Wmt3Y2pNWXQyV0lYNGsKY4Pn/jxHZV3rv5ImSxYORPStxyJxQRvq
r1mZyps568LLoEagvooanxoAYRWifFl7BZrEPNutkNHhSmyyZ+Bbdw==
-----END AGE ENCRYPTED FILE-----
- recipient: age1ujldj2gprjmqjcn376mtj8chskyk40gvst3m765td8za9qcd2fksuyz2h3
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyODJxcEFMK1krQ016NDg4
bW5OVlZ6NmlkeHJjcGRhQk9QSnlFK3VVUWdzCitaVEd4ZXk3Q3pwbVlJL1RCVm5x
QW91V0ZlV25KRy9ob3ZINDJvcGQ3dUkKLS0tIGk0Qnc0Qi9ZREp4TnlOUGJkSHBY
K3BzaG9sc1lUZFZSZkljMWNWdDFnQTgKdBHdbcnbCUZvb9w35mKfvTvpDWgNJ10q
QTPRnIBYgGSvqb0SVuYZJZFRVB/V23Rt9g3pCyolo6WH7ZITivxRTg==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTL1dKUEl1MDFxc1g4bU41
U2NZQldualg5WEozRnV6NHROVmZUMm80RjE0CkVPamx5TjRZcXo2ZVVoa2JJUVlh
b210M2Z5NzJMZW53RVNZZ0RhUTdzNU0KLS0tIHBKVm5zK3FldEwzSTN3c2FCR2NU
bCs2R3dKQmdvMU42ZEJGRWJhWm9JZ0kK5dt9W69nnsiyHb0KmzdtDraid9AnXl+o
Np+JU91sRkWr1yJekCNk6MpF8neGY6hZN1UufP0TovrqCshMwcaluQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1e8f629xakqvc6gl25j36d46vl4tqnntjfqv2re54savhtc9ysqrsj3tu09
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA2L0lnT0NrU2FUbGtzWDhZ
MlRHWGJWQnNodytoT0xpelo2SzIwamRjSzBJClN0THFMbjNFcTNuaUJ4UldzQ2Jt
VEZFalZpczdmT09lOUp6djAxalBSUGcKLS0tIFk2Mi83SDJSc3J3TzN6cUQrajRv
eDZoRkxZNjBadWRzVTFsWlBmeGNFemsKKDq7jApNgZUQLyjPI6a9Wq4Txnv+atFa
ESwVNDJNGtIwVTUp1B28VUJZWtJOoqgW1rq1FQ7MPXBS5pt9tmxK9Q==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwQXdsam12NDFndjZzK0VF
eXg2TFVUM1Z1N0VWQ0FabkNrZVdKTDF2aXp3CnZZZG5ES0JyU0VLYlM2dWR6enI3
OER5eitEOEhUQlhnY0lsV0lwK2pWVk0KLS0tIGJROHV6aFU0M2tuZE1qL2tiOGZq
SGxWd2NGNndpQWNWbENMWm5VZ216M2sKDH/ZbOVccBsiBErNkHgCy1y9KdjLd/DI
ob9RD7pUf7XMF1sH7weAsQCdAd/M77B17qNp3BJJZYDftKvRPnx/0w==
-----END AGE ENCRYPTED FILE-----
- recipient: age1lqtfrgk2nfdgqm3tculqlph8r8nthrv7frzk7p8vxurwgwudedqs5s04d3
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxOEZKNjJnNEhkenhGdEla
UUtncHoyL1R2Y1YxZW9ZL2VWNXpidFoxSFJrCklvVzdYeUVyK0RRZ0E5b2U4c3pV
eEFZaGZQcmlHT0t2czlKZ2poMTkra0EKLS0tIEI0OGR1ay9hS2lDMksxaFBmZXRP
QkNZNEc1ZjdpeWJXRWtuQUhGaHlTbjAKKDP35Haj2ZIeECvm5AjJDZGEbJNiYGuJ
5tnDJtB3mMrBrDosYd2kffdbKCl3yp/CsWm6H6y5dXJJ51MFcdJddw==
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXS3Q3Y09Wc0VLNlNXTUZE
eG9VZThsVU1BbGZKZUo3ZmUyckF0NFZvRTE0CnZZVkVDSUQ0OVExOFdwL0RpMjVL
LzArUXl1M1NWcDQvRTQ1NFR1a3FCSWMKLS0tIGtUOGRwcjd2bVdBQmtVVW5paHEr
anY4ODJXZWEyeGNtNXRFb1NFVGhDMW8KprFkCdMO5HeWO3GzqsLR6a8t3W9ilUzs
NTJfB/Eub7My/Pw0FpA4n3WU28MQBPJe65+ikFOlTkrurszqFdmKOQ==
-----END AGE ENCRYPTED FILE-----
- recipient: age1hw95hm8056j7xu2dj96g95zqrnd6end664ws93ekqzv2xj0re3rs6yz0mf
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByNzEvZFRmMXI4d0Vpck5p
SE1uSXpsdWdWRDdaclk2eE5EQnkyYXBSVWs4CldNR1NRSlZjT1pjczQ1MXllUmFp
cW9VRUNzWHpDbWxEV0oxLzBFYUMzSFUKLS0tIHV1UjZaN1ZOY3pxVk9zblZZWmsz
KzVvSkdoS2tlQ1RSMWxvMkNveFEvU3cKi53Z4YfIP6mxTh/rR8sl16SBqJofXCc+
oOu7oBCnQQql7Zlk/ZRg2aLPHPtDh2wo14oL6lgJS8YqjG8k8lGPrg==
-----END AGE ENCRYPTED FILE-----
- recipient: age1upmemuj08e27ug0us2stzl7ksmxynqcs4q0dweuhn59w0kfd4vsqr25ylh
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoMFZ1ZFdKUWFuRW5LR2d0
SjQ0VjNHVGNMNWRhbG9sNFFzRWk5QU45Umg4Cm1CVG05TmV1VTczS0x2a3JHZEJr
K0pJTWFXREpFTnRuczIyUjhhMGNjOVEKLS0tIGFrSjlYMXBzMnpvSUdGYWtJMjIv
aXpBSmlmOVNaZXR0VEVuaE9CemZTd0EKDz/F6Cl2ERWY8LPQGqT130AYNP4pMdoN
vVRWwo5PqXyZY+vBR4FkzpnaDOSDWHNMT0LDh+JzB0TkA/dI7+zOwg==
-----END AGE ENCRYPTED FILE-----
- recipient: age1qwjtghh747xx7ssfyq48g9rucnwnfa3eslyk7futw2rqeakeuayqzlwj6g
enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAraW1jdGRkMDlQWXVyQm9S
VW5KcUE2U3UzWFV0VVNEV1lRRG11SzMwSVJ3CmN6STlIODVRSG1ieWY0OTcvaDk1
RFJtMlNlY1lIWEExcGpWZlZGVmYzbkEKLS0tIDExeUMxM1gzZlROdHU5YjIyUFdY
c01zMTA2dnZlTGxGd0oxYklXMHdWTUkKYjSQ7Y16AXfiLaOwsyV8LFjUtbUJ744A
uxlImBcQnbiPkJY4DRxmtrBrTSzfX1pdepNH8DR0ZpjpI/6bibHEag==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2023-06-27T09:58:35Z"
mac: ENC[AES256_GCM,data:pPgwJnUdwQegqaCXdh7lweQq2Kos6szvo/mfBul+2TruUSSRXlGwKmNVLM2BuodMNZpTan2vCyvVlXvN4zBfW6nVWPzlBrCTbgtyBNodB+k3OJsfgUElQ32T9KccsMVuUsfKDzjhlFnV3NA9A7DVnrYz+jf1NcNSsz4yOjHudzA=,iv:ciFHyXhIcNFlB9fhzcAX8LICIsGPWDe29fxtjmJ0G+s=,tag:oldhGvm8vfPnuhpIXIpVWw==,type:str]
pgp:
- created_at: "2023-07-03T09:59:45Z"
- created_at: "2023-07-05T13:59:05Z"
enc: |
-----BEGIN PGP MESSAGE-----
hQEMA5ntoryXZPD4AQf+JS+pr3qsdGsiIUYvY6K5H877NUtq1/SKAZxZnYwXnWZ5
UKrGXfavvh/OL9T/19BcK/eo6GSzrj/yvUHr6cDkJWgZezMaGPMMAR1r/2x2Wg9b
a7oewcOHmwN0yegVOZQilkbIfTebTGgGb4PnqT5w7bwrGVBjKjXNBUekObLR4O45
jjvNNhGAoOcemRVms4ErdQ0aRPt+8yNVcmOF9gIrNcvMvkO3IfvN1JTzwkoFEm9j
WYgPpvByesBWmbL76Xd95QGfJxOaIHDXgV+sSs8DA8xS81H0j0q2A+krVGpdu0nW
6tej5P9OlI6atQkmf66eca8di4ztwj1uGZjc9ocYEdJeAYm3Fg20OeYrKuN3ApfX
wWSkWm9b89GuptH5dcqlQsSSf6gpJ/9vfuKuurQiv7k2B5Ge4GD/bRcfTIQyz/mx
sqCxGbnVe4T/bhTAK8Ah60ZsuTRx463EXTpykFwoyA==
=agMc
hQEMA5ntoryXZPD4AQf+P0suHHym/OGK4CzSYSAI7cZZUzcF1skOvm+IX5nldLMx
TcF7t4x4dfwZmwKs34eVWI37VrULbeoLoFMPH98+VRj10OlOWxqoyl2xpz4cjGfa
LHnugGrFc5O0mKfeEvK5+2bmukzCKZ0ug2I1pifyLItYxMZl9/udX1aDTS/1qN+s
yYV0mZgOb/SV6v1i79CVFsfms0L8jouElx25CWS4cH12scuRURMse9dIuEkOn5kA
hxkIn5sa4ZuE2OqjrVWKZtiQ0P4kpISdbnsBdvMiultnL/kbNM2s67cwW5GPaGT4
Rksg/i6jZKjNbyNEP+0K+YugT99LzlILtwLnJ7nMVtJeAUpZDQyrPid49r91i78w
i90hVxPWCZdul0Ao6051Ga8vx2z+OTm/wtq4+ZQat2J9p+lKjsECgWFCpLiH1Ljx
Q5JfM0DzYAifr1ny/CgUAPzwpqkjhe8E2njBsdbXBw==
=DcfT
-----END PGP MESSAGE-----
fp: 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
unencrypted_suffix: _unencrypted

5
shared/ssh.nix

@ -3,6 +3,11 @@
with lib;
{
services.openssh = {
enable = true;
settings.PermitRootLogin = "without-password";
};
programs.ssh = {
# Add know-host entries for all machines in the cluster
knownHosts = mapAttrs

Loading…
Cancel
Save