Browse Source

Inital import

main
Dustin Frisch 12 months ago
commit
ce8862bf37
No known key found for this signature in database GPG Key ID: B4C3BF012D9B26BE
  1. 4
      .envrc
  2. 5
      .sops.yaml
  3. 453
      flake.lock
  4. 95
      flake.nix
  5. 38
      machines.nix
  6. 5
      machines/manager/autoinstall.nix
  7. 29
      machines/manager/beegfs.nix
  8. 23
      machines/manager/cache.nix
  9. 35
      machines/manager/default.nix
  10. 40
      machines/manager/disk.nix
  11. 4
      machines/manager/gateway.nix
  12. 44
      machines/manager/ldap.nix
  13. 8
      machines/manager/nfs.nix
  14. 5
      machines/manager/ntp.nix
  15. 24
      machines/node/default.nix
  16. 39
      machines/node/disk.nix
  17. 22
      machines/node/users.nix
  18. 227
      modules/beegfs.nix
  19. 5
      modules/default.nix
  20. 185
      packages/beegfs/001-build.patch
  21. 77
      packages/beegfs/default.nix
  22. 45
      packages/beegfs/module.nix
  23. 31
      secrets.yaml
  24. 21
      secrets/cache-priv-key.pem
  25. 35
      shared/default.nix
  26. 10
      shared/network.nix
  27. 11
      shared/root.nix

4
.envrc

@ -0,0 +1,4 @@
use flake
watch_file "flake.nix"
watch_file "flake.lock"

5
.sops.yaml

@ -0,0 +1,5 @@
creation_rules:
- key_groups:
- pgp:
- 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
path_regex: ^(secrets\.yaml|secrets/.+)$

453
flake.lock

@ -0,0 +1,453 @@
{
"nodes": {
"colmena": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"stable": "stable"
},
"locked": {
"lastModified": 1684127527,
"narHash": "sha256-tAzgb2jgmRaX9HETry38h2OvBf9YkHEH1fFvIJQV9A0=",
"owner": "zhaofengli",
"repo": "colmena",
"rev": "caf33af7d854c8d9b88a8f3dae7adb1c24c1407b",
"type": "github"
},
"original": {
"owner": "zhaofengli",
"repo": "colmena",
"type": "github"
}
},
"disko": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1684783210,
"narHash": "sha256-hxRbwwBTu1G1u1EdI9nEo/n4HIsQIfNi+2BQ1nEoj/o=",
"owner": "nix-community",
"repo": "disko",
"rev": "f0b9f374bb42fdcd57baa7d4448ac5d4788226bd",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "disko",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1650374568,
"narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "b4a34015c698c7793d592d66adbab377907a2be8",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_3": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_4": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_5": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_6": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"pre-commit-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1660459072,
"narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixago": {
"inputs": {
"flake-utils": [
"utils"
],
"nixago-exts": "nixago-exts",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1683210100,
"narHash": "sha256-bhGDOlkWtlhVECpoOog4fWiFJmLCpVEg09a40aTjCbw=",
"owner": "nix-community",
"repo": "nixago",
"rev": "1da60ad9412135f9ed7a004669fdcf3d378ec630",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago",
"type": "github"
}
},
"nixago-exts": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixago": "nixago_2",
"nixpkgs": [
"nixago",
"nixpkgs"
]
},
"locked": {
"lastModified": 1676070308,
"narHash": "sha256-QaJ65oc2l8iwQIGWUJ0EKjCeSuuCM/LqR8RauxZUUkc=",
"owner": "nix-community",
"repo": "nixago-extensions",
"rev": "e5380cb0456f4ea3c86cf94e3039eb856bf07d0b",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago-extensions",
"type": "github"
}
},
"nixago-exts_2": {
"inputs": {
"flake-utils": "flake-utils_4",
"nixago": "nixago_3",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixago",
"nixpkgs"
]
},
"locked": {
"lastModified": 1655508669,
"narHash": "sha256-BDDdo5dZQMmwNH/GNacy33nPBnCpSIydWFPZs0kkj/g=",
"owner": "nix-community",
"repo": "nixago-extensions",
"rev": "3022a932ce109258482ecc6568c163e8d0b426aa",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago-extensions",
"type": "github"
}
},
"nixago_2": {
"inputs": {
"flake-utils": "flake-utils_3",
"nixago-exts": "nixago-exts_2",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixpkgs"
]
},
"locked": {
"lastModified": 1676070010,
"narHash": "sha256-iYzJIWptE1EUD8VINAg66AAMUajizg8JUYN3oBmb8no=",
"owner": "nix-community",
"repo": "nixago",
"rev": "d480ba6c0c16e2c5c0bd2122852d6a0c9ad1ed0e",
"type": "github"
},
"original": {
"owner": "nix-community",
"ref": "rename-config-data",
"repo": "nixago",
"type": "github"
}
},
"nixago_3": {
"inputs": {
"flake-utils": "flake-utils_5",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixago",
"nixago-exts",
"nixpkgs"
]
},
"locked": {
"lastModified": 1655405483,
"narHash": "sha256-Crd49aZWNrpczlRTOwWGfwBMsTUoG9vlHDKQC7cx264=",
"owner": "nix-community",
"repo": "nixago",
"rev": "e6a9566c18063db5b120e69e048d3627414e327d",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1684858140,
"narHash": "sha256-dQStox5GYrVlVNMvxxXs3xX9bXG7J7ttSjqUcVm8EaA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a17f99dfcb9643200b3884ca195c69ae41d7f059",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1678872516,
"narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-utils": "flake-utils_6",
"gitignore": "gitignore",
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1684842236,
"narHash": "sha256-rYWsIXHvNhVQ15RQlBUv67W3YnM+Pd+DuXGMvCBq2IE=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "61e567d6497bc9556f391faebe5e410e6623217f",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"colmena": "colmena",
"disko": "disko",
"nixago": "nixago",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": "pre-commit-hooks",
"sops": "sops",
"utils": "utils"
}
},
"sops": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1684637723,
"narHash": "sha256-0vAxL7MVMhGbTkAyvzLvleELHjVsaS43p+PR1h9gzNQ=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "4ccdfb573f323a108a44c13bb7730e42baf962a9",
"type": "github"
},
"original": {
"owner": "Mic92",
"repo": "sops-nix",
"type": "github"
}
},
"stable": {
"locked": {
"lastModified": 1669735802,
"narHash": "sha256-qtG/o/i5ZWZLmXw108N2aPiVsxOcidpHJYNkT45ry9Q=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "731cc710aeebecbf45a258e977e8b68350549522",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

95
flake.nix

@ -0,0 +1,95 @@
{
inputs = {
nixpkgs = {
type = "github";
owner = "NixOS";
repo = "nixpkgs";
ref = "nixos-22.11";
};
colmena = {
type = "github";
owner = "zhaofengli";
repo = "colmena";
inputs.nixpkgs.follows = "nixpkgs";
};
nixago = {
type = "github";
owner = "nix-community";
repo = "nixago";
inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-utils.follows = "utils";
};
pre-commit-hooks = {
type = "github";
owner = "cachix";
repo = "pre-commit-hooks.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
sops = {
type = "github";
owner = "Mic92";
repo = "sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
inputs.nixpkgs-stable.follows = "nixpkgs";
};
utils = {
type = "github";
owner = "numtide";
repo = "flake-utils";
};
disko = {
type = "github";
owner = "nix-community";
repo = "disko";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { nixpkgs, utils, ... }@inputs: {
colmena = import ./machines.nix inputs;
devShell = utils.lib.eachSystemMap utils.lib.allSystems (system:
let
pkgs = nixpkgs.legacyPackages.${system};
pre-commit-hooks = inputs.pre-commit-hooks.lib.${system}.run {
src = ./.;
hooks = {
nixpkgs-fmt.enable = true;
statix.enable = true;
shellcheck.enable = true;
};
};
in
pkgs.mkShell {
buildInputs = [
inputs.colmena.defaultPackage.${system}
] ++ (with pkgs; [
bash
gitAndTools.git
gnutar
gzip
nixUnstable
openssh
sops
age
]);
shellHook = ''
${pre-commit-hooks.shellHook}
'';
});
};
}

38
machines.nix

@ -0,0 +1,38 @@
{ nixpkgs, disko, sops, ... }@inputs:
let
deploymentPkgs = import nixpkgs {
localSystem.system = "x86_64-linux";
};
in with deploymentPkgs.lib; let
mkMachine = type: opts: { lib, ... }:
let
machine = import ./machines/${type} opts;
in {
imports = [
./shared
./modules
machine
sops.nixosModules.sops
disko.nixosModules.disko
];
};
machines = {
manager = mkMachine "manager" {};
} // (listToAttrs (genList (i: nameValuePair
"node-${fixedWidthNumber 2 i}"
(mkMachine "node" { id = i; })
) 16));
in {
meta = {
nixpkgs = deploymentPkgs;
specialArgs = {
inherit inputs;
};
};
} // machines

5
machines/manager/autoinstall.nix

@ -0,0 +1,5 @@
# TFTP boot with shared image
# Requests store path to install from master
# Runs disko and nixos-install
{}

29
machines/manager/beegfs.nix

@ -0,0 +1,29 @@
{ pkgs, config, lib, ... }:
with lib;
let
connAuth = toString (pkgs.writeText "beegfs-conn-auth" "asdasdasdasd");
# connAuth = ${config.sops.secrets."beegfs/connection".path}
in
{
nixpkgs.config.allowUnfree = true;
hpc.beegfs = {
connAuthFile = connAuth;
mgmtdHost = "manager.hpc.informatik.hs-fulda.de";
client = {
enable = true;
mountPoint = "/projects";
};
mgmtd.enable = true;
meta.enable = true;
storage.enable = true;
};
sops.secrets."beegfs/connection" = {};
}

23
machines/manager/cache.nix

@ -0,0 +1,23 @@
{ config, ... }:
{
services.nix-serve = {
enable = true;
secretKeyFile = config.sops.secrets."cache/privateKey".path;
};
sops.secrets."cache/privateKey" = {
format = "binary";
sopsFile = ../../secrets/cache-priv-key.pem;
};
services.nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"cache.hpc.informatik.hs-fulda.de" = {
locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
};
};
};
}

35
machines/manager/default.nix

@ -0,0 +1,35 @@
{ ... }:
{ lib, config, ... }:
with lib;
{
imports = [
./disk.nix
./nfs.nix
#./ldap.nix
#./beegfs.nix
#./ntp.nix
#./gateway.nix
#./autoinstall.nix
#./cache.nix
];
users.users."root".password = "asdasd123";
virtualisation.useDefaultFilesystems = false;
virtualisation.fileSystems."/" = {
device = config.virtualisation.bootDevice;
fsType = "btrfs";
autoFormat = true;
};
deployment = {
targetHost = "10.32.30.240";
targetUser = "root";
tags = [ "manager" ];
};
networking.hostName = "manager";
}

40
machines/manager/disk.nix

@ -0,0 +1,40 @@
{ ... }:
{
disko.devices = {
disk.sda = {
device = "/dev/nvme0n1";
type = "disk";
content = {
type = "table";
format = "gpt";
partitions = [
{
name = "root";
start = "100MiB";
end = "100%";
part-type = "primary";
bootable = true;
content = {
type = "filesystem";
format = "ext4";
#format = "btrfs";
mountpoint = "/";
};
}
{
name = "ESP";
start = "1MiB";
end = "100MiB";
bootable = true;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
}
];
};
};
};
}

4
machines/manager/gateway.nix

@ -0,0 +1,4 @@
# DHCP server for nodes
# NAT gateway for nodes
{}

44
machines/manager/ldap.nix

@ -0,0 +1,44 @@
{ config, ... }:
{
services.openldap = {
enable = true;
settings = {
children = {
"cn=schema".includes = [
"${config.services.openldap.package}/etc/schema/core.ldif"
"${config.services.openldap.package}/etc/schema/cosine.ldif"
"${config.services.openldap.package}/etc/schema/inetorgperson.ldif"
"${config.services.openldap.package}/etc/schema/nis.ldif"
];
"olcDatabase={1}mdb" = {
attrs = {
objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
olcDatabase = "{1}mdb";
olcDbDirectory = "/var/lib/openldap/db";
olcSuffix = "dc=sc,dc=informatik,dc=hs-fulda,dc=de";
olcRootDN = "cn=root,dc=sc,dc=informatik,dc=hs-fulda,dc=de";
olcRootPW.path = config.sops.secrets."ldap/root/password".path;
olcAccess = [
# Custom access rules for userPassword attributes
''{0}to attrs=userPassword
by self write
by anonymous auth
by * none''
# Allow read on anything else
''{1}to *
by * read''
];
};
};
};
};
};
sops.secrets."ldap/root/password" = { };
}

8
machines/manager/nfs.nix

@ -0,0 +1,8 @@
{
services.nfs.server = {
enable = true;
exports = ''
/home node*.hpc.informatik.hs-fulda.de(rw)
'';
};
}

5
machines/manager/ntp.nix

@ -0,0 +1,5 @@
{
services.chrony = {
enable = true;
};
}

24
machines/node/default.nix

@ -0,0 +1,24 @@
{ id, ... }:
{ lib, ... }:
with lib;
{
imports = [
./disk.nix
];
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "node-${fixedWidthNumber 2 id}";
nix.settings = {
substituters = [
"http://cache.hpc.informatik.hs-fulda.de"
];
trusted-public-keys = [
"cache.hpc.informatik.hs-fulda.de:dc2abEGJAQfaZiBXhjvjPU0jx/wosQwAOQoz48/G6cA="
];
};
}

39
machines/node/disk.nix

@ -0,0 +1,39 @@
{ ... }:
{
disko.devices = {
disk.sda = {
device = "/dev/sda";
type = "disk";
content = {
type = "table";
format = "gpt";
partitions = [
{
name = "root";
start = "100MiB";
end = "100%";
part-type = "primary";
bootable = true;
content = {
type = "filesystem";
format = "btrfs";
mountpoint = "/";
};
}
{
name = "ESP";
start = "1MiB";
end = "100MiB";
bootable = true;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
}
];
};
};
};
}

22
machines/node/users.nix

@ -0,0 +1,22 @@
{
users.ldap = {
enable = true;
server = "ldap://manager.hpc.informatik.hs-fulda.de/";
base = "dc=hpc,dc=informatik,dc=hs-fulda,dc=de";
daemon.enable = true;
};
filesystem."home" = {
mountPoint = "/home";
device = "manager.hpc.informatik.hs-fulda.de:/home";
fsType = "nfs";
options = [
"nfsvers=4.2"
"noauto"
"x-systemd.automount"
"x-systemd.idle-timeout=600"
];
};
}

227
modules/beegfs.nix

@ -0,0 +1,227 @@
{ pkgs, config, lib, ... }:
with lib;
{
options.hpc.beegfs = {
package = mkOption {
description = ''
BeeGFS package.
'';
type = types.package;
default = config.boot.kernelPackages.callPackage ../packages/beegfs/default.nix { };
};
mgmtdHost = mkOption {
description = ''
Hostname of the management host.
'';
type = types.str;
};
connAuthFile = mkOption {
description = ''
File containing shared secret authentication.
'';
type = types.str;
};
client = {
enable = mkEnableOption "BeeGFS client";
mountPoint = mkOption {
description = ''
Mount point under which the BeeGFS filesystem is mounted.
'';
type = types.nullOr types.str;
default = null;
};
};
mgmtd = {
enable = mkEnableOption "BeeGFS management server daemon";
};
meta = {
enable = mkEnableOption "BeeGFS meta-data server daemon";
};
storage = {
enable = mkEnableOption "BeeGFS storage server daemon";
};
};
config = mkMerge [
(mkIf config.hpc.beegfs.client.enable {
boot.kernelModules = [ "beegfs" ];
boot.extraModulePackages = [ config.hpc.beegfs.package.module ];
environment.etc."beegfs-client" = {
enable = true;
target = "beegfs/client.conf";
text = ''
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
'';
};
systemd.mounts = mkIf (config.hpc.beegfs.client.mountPoint != null) [ {
where = config.hpc.beegfs.client.mountPoint;
what = "beegfs_nodev";
type = "beegfs";
options = "cfgFile=/etc/beegfs/client.conf,_netdev";
requires = [ "beegfs-helperd.service" ];
after = [ "beegfs-helperd.service" ];
} ];
systemd.services."beegfs-helperd" = let
cfgFile = pkgs.writeText "beegfs-helperd.conf" ''
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-helperd \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-helperd.pid";
TimeoutStopSec = "300";
};
};
environment.systemPackages = [ (pkgs.runCommandLocal "beegfs-utils" {
nativeBuildInputs = [ pkgs.makeWrapper ];
} ''
mkdir -p $out/bin
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-check-servers \
$out/bin/beegfs-check-servers \
--add-flags "-c /etc/beegfs/client.conf" \
--prefix PATH : ${lib.makeBinPath [ config.hpc.beegfs.package ]}
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-ctl \
$out/bin/beegfs-ctl \
--add-flags "--cfgFile=/etc/beegfs/client.conf"
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-ctl \
$out/bin/beegfs-df \
--add-flags "--cfgFile=/etc/beegfs/client.conf" \
--add-flags --listtargets \
--add-flags --hidenodeid \
--add-flags --pools \
--add-flags --spaceinfo
makeWrapper ${config.hpc.beegfs.package}/bin/fsck.beegfs \
$out/bin/beegfs-fsck \
--add-flags "--cfgFile=/etc/beegfs/client.conf"
'') ];
})
(mkIf config.hpc.beegfs.mgmtd.enable (let
cfgFile = pkgs.writeText "beegfs-mgmtd.conf" ''
storeMgmtdDirectory = /var/lib/beegs/mgmtd
storeAllowFirstRunInit = false
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-mgmtd" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/mgmtd; then
${config.hpc.beegfs.package}/bin/beegfs-setup-mgmtd -C -p /var/lib/beegs/mgmtd
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-mgmtd \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-mgmtd.pid";
TimeoutStopSec = "300";
};
};
}))
(mkIf config.hpc.beegfs.meta.enable (let
cfgFile = pkgs.writeText "beegfs-meta.conf" ''
storeMetaDirectory = /var/lib/beegs/meta
storeAllowFirstRunInit = false
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-meta" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "beegfs-mgmt.service" "beegfs-storage.service" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/meta; then
${config.hpc.beegfs.package}/bin/beegfs-setup-meta -C -p /var/lib/beegs/meta
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-meta \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-meta.pid";
TimeoutStopSec = "300";
};
};
}))
(mkIf config.hpc.beegfs.storage.enable (let
cfgFile = pkgs.writeText "beegfs-storage.conf" ''
storeStorageDirectory = /var/lib/beegs/storage
storeAllowFirstRunInit = false
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-storage" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "beegfs-mgmt.service" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/storage; then
${config.hpc.beegfs.package}/bin/beegfs-setup-storage -C -p /var/lib/beegs/storage
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-storage \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-storage.pid";
TimeoutStopSec = "300";
};
};
}))
];
}

5
modules/default.nix

@ -0,0 +1,5 @@
{
imports = [
./beegfs.nix
];
}

185
packages/beegfs/001-build.patch

@ -0,0 +1,185 @@
diff -r -u a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/CMakeLists.txt 2023-05-27 21:27:08.991487355 +0200
@@ -85,10 +85,8 @@
add_subdirectory("thirdparty/source/gtest")
endif()
-set(CMAKE_INSTALL_PREFIX "/")
-
-add_subdirectory("beeond")
-add_subdirectory("beeond_thirdparty_gpl")
+# add_subdirectory("beeond")
+# add_subdirectory("beeond_thirdparty_gpl")
# add_subdirectory("client_devel")
# add_subdirectory("client_module")
add_subdirectory("common")
diff -r -u a/common/CMakeLists.txt b/common/CMakeLists.txt
--- a/common/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/common/CMakeLists.txt 2023-05-29 10:57:53.502540650 +0200
@@ -572,6 +572,5 @@
install(
TARGETS beegfs_ib
- DESTINATION "usr/lib"
COMPONENT "libbeegfs-ib"
)
diff -r -u a/ctl/CMakeLists.txt b/ctl/CMakeLists.txt
--- a/ctl/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/ctl/CMakeLists.txt 2023-05-29 10:57:13.297068443 +0200
@@ -149,6 +149,5 @@
install(
TARGETS beegfs-ctl
- DESTINATION "usr/bin"
COMPONENT "utils"
)
diff -r -u a/event_listener/CMakeLists.txt b/event_listener/CMakeLists.txt
--- a/event_listener/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/event_listener/CMakeLists.txt 2023-05-29 10:57:16.262103203 +0200
@@ -10,13 +10,12 @@
install(
TARGETS beegfs-event-listener
- DESTINATION "usr/sbin"
COMPONENT "event-listener"
)
install(
FILES "include/beegfs/beegfs_file_event_log.hpp"
- DESTINATION "usr/include/beegfs"
+ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/beegfs"
COMPONENT "event-listener"
)
Only in v7-7.3.3: foo
diff -r -u a/fsck/CMakeLists.txt b/fsck/CMakeLists.txt
--- a/fsck/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/fsck/CMakeLists.txt 2023-05-29 10:57:18.938134583 +0200
@@ -148,6 +148,5 @@
install(
TARGETS fsck.beegfs
- DESTINATION "sbin"
COMPONENT "utils"
)
diff -r -u a/helperd/CMakeLists.txt b/helperd/CMakeLists.txt
--- a/helperd/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/helperd/CMakeLists.txt 2023-05-29 10:57:21.185160939 +0200
@@ -59,6 +59,5 @@
install(
TARGETS beegfs-helperd
- DESTINATION "usr/sbin"
COMPONENT "helperd"
)
diff -r -u a/meta/CMakeLists.txt b/meta/CMakeLists.txt
--- a/meta/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/meta/CMakeLists.txt 2023-05-29 11:00:36.501472258 +0200
@@ -326,13 +326,12 @@
install(
TARGETS beegfs-meta
- DESTINATION "usr/sbin"
COMPONENT "meta"
)
install(
PROGRAMS "build/dist/sbin/beegfs-setup-meta"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "meta"
)
@@ -347,10 +346,3 @@
DESTINATION "etc/beegfs"
COMPONENT "meta"
)
-
-install(
- PROGRAMS "build/beegfs-meta.sh"
- RENAME "beegfs-meta"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "meta"
-)
diff -r -u a/mgmtd/CMakeLists.txt b/mgmtd/CMakeLists.txt
--- a/mgmtd/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/mgmtd/CMakeLists.txt 2023-05-29 11:00:44.428566811 +0200
@@ -161,13 +161,12 @@
install(
TARGETS beegfs-mgmtd
- DESTINATION "usr/sbin"
COMPONENT "mgmtd"
)
install(
PROGRAMS "build/dist/sbin/beegfs-setup-mgmtd"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "mgmtd"
)
@@ -182,10 +181,3 @@
DESTINATION "etc/beegfs"
COMPONENT "mgmtd"
)
-
-install(
- PROGRAMS "build/beegfs-mgmtd.sh"
- RENAME "beegfs-mgmtd"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "mgmtd"
-)
diff -r -u a/storage/CMakeLists.txt b/storage/CMakeLists.txt
--- a/storage/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/storage/CMakeLists.txt 2023-05-29 11:00:53.385673711 +0200
@@ -188,7 +188,6 @@
install(
TARGETS beegfs-storage
- DESTINATION "usr/sbin"
COMPONENT "storage"
)
@@ -200,7 +199,7 @@
install(
PROGRAMS "build/dist/sbin/beegfs-setup-storage"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "storage"
)
@@ -209,10 +208,3 @@
DESTINATION "etc/beegfs"
COMPONENT "storage"
)
-
-install(
- PROGRAMS "build/beegfs-storage.sh"
- RENAME "beegfs-storage"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "storage"
-)
diff -r -u a/upgrade/beegfs_mirror_md/CMakeLists.txt b/upgrade/beegfs_mirror_md/CMakeLists.txt
--- a/upgrade/beegfs_mirror_md/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/upgrade/beegfs_mirror_md/CMakeLists.txt 2023-05-29 10:57:40.926392744 +0200
@@ -15,6 +15,5 @@
install(
TARGETS "beegfs-mirror-md"
- DESTINATION "usr/sbin"
COMPONENT "beegfs-mirror-md"
)
diff -r -u a/utils/CMakeLists.txt b/utils/CMakeLists.txt
--- a/utils/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/utils/CMakeLists.txt 2023-05-29 11:01:05.123813896 +0200
@@ -6,6 +6,6 @@
install(
PROGRAMS "scripts/beegfs-check-servers" "scripts/beegfs-df" "scripts/beegfs-net"
- DESTINATION "usr/bin"
+ TYPE BIN
COMPONENT "utils"
)

77
packages/beegfs/default.nix

@ -0,0 +1,77 @@
{ stdenv
, fetchurl
, pkgconfig
, util-linux
, which
, libuuid
, attr
, xfsprogs
, rdma-core
, zlib
, openssl
, openssh
, curl
, cmake
, callPackage
, kernel ? null
, ... } :
stdenv.mkDerivation (final: rec {
pname = "beegfs";
version = "7.3.3";
src = fetchurl {
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.gz";
sha256 = "sha256-XfZY6ge4KWNJn9UE41b7ds2YCMz9FNXFqZd51qCatig=";
};
nativeBuildInputs = [
pkgconfig
which
cmake
];
buildInputs = [
util-linux
libuuid
attr
xfsprogs
zlib
openssl
rdma-core
openssh
curl
];
patches = [
./001-build.patch
];
dontFixCmake = true;
cmakeFlags = [
"-DBEEGFS_VERSION=${version}"
"-DBEEGFS_SKIP_TESTS=true"
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=${placeholder "out"}"
"-DCMAKE_INSTALL_LIBDIR=lib"
];
hardeningDisable = [ "format" ];
passthru.module = callPackage ./module.nix {
inherit kernel;
beegfs = final;
};
meta = with stdenv.lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = {
fullName = "BeeGFS_EULA";
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
free = false;
};
};
})

45
packages/beegfs/module.nix

@ -0,0 +1,45 @@
{ beegfs
, kmod
, kernel
, ... } :
kernel.stdenv.mkDerivation rec {
name = "beegfs-module-${beegfs.version}-${kernel.version}";
inherit (beegfs) src;
nativeBuildInputs = [
kmod
];
buildInputs = kernel.moduleBuildDependencies;
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
sourceRoot = "v7-${beegfs.version}/client_module/build";
makeFlags = [
"KERNELRELEASE=${kernel.modDirVersion}"
"KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/"
"BEEGFS_VERSION=${beegfs.version}-nixos1"
];
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
'';
installPhase = ''
mkdir -p $out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
install -t $out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs beegfs.ko
'';
enableParallelBuilding = true;
meta = with kernel.stdenv.lib; {
description = beegfs.meta.description + " (kernel module)";
inherit (beegfs.meta) homepage license platforms;
};
}

31
secrets.yaml

@ -0,0 +1,31 @@
ldap:
root:
password: ENC[AES256_GCM,data:bYuw+9ywfRDNVt0nrLDmWE8+f8aHQvGd,iv:JHU3MxmNdxI2a62Dcky8xhHhjhcxyjM0Z0xLEnLxJwU=,tag:3VW0zTlRFxLDI8WxGu1lew==,type:str]
beegfs:
connection: ENC[AES256_GCM,data:YTHMg76+5Azb+ex5ArUHt4xP+YYWr9Ph,iv:TEf8i+yezPsaW12Lg5jRnhds9uW9WhV6duZPdxeW9co=,tag:bPGsl7ofwE1Jh+FTyHJqzQ==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age: []
lastmodified: "2023-05-29T12:51:30Z"
mac: ENC[AES256_GCM,data:02jKHbEZGs3QiNzXEQxcB8v/i5UVB/pCciz4hSI220+GEYPgQK6qR1cZJaMAyrHKjzJLhNZq3Gfgsj4zfA+FMg/d12vp2QNTMRrVD/hSh67NgloZ/iTmJC//S8OJfiHEPdGKkq7zXCVajnkGMT/0yLNWAKISAwL451ohgMzMQYw=,iv:8hqKXUolNA7WatnnYwwUN2EgOyZjTISG2bfToENYc7c=,tag:5y43RQJgZbPK8g3Cw8CBzQ==,type:str]
pgp:
- created_at: "2023-05-24T20:04:49Z"
enc: |
-----BEGIN PGP MESSAGE-----
hQEMA5ntoryXZPD4AQf6A69nF8BRpYRdz3ea8acqryKoMe5p2A44drykDQR0NO9r
I6j0Hg9AksgC+rGRIQtCuj18gYybDFXgYLCE8MYfgh2NSyqeGzq2+kPDqAXRong/
Wrg1+KRlDbvIqH7IZ5BS40TGdphh/U8BIUcO8N4tgP60G6C7z9FqfjiA5YByqau3
7uAtKg3kR6lL13Cf0AUnMrQ8AOZ+6p+BwdTcXeUW2bScw8ScbEQsw/MtoiEN9Een
jvPhqTczdcZLIgTV+DvmimwYmH8xwFiMNFBrt4uzsBMv9N4pb0EzL8TcKIOuE8iw
YserGEi/sMx5QzqYmS2yPvNxwcXsZi28SQrHOs4Lv9JeAdpqwrqJjAaV3pN0OgSy
31XV/oDL8GJ3SfNqUZEULB06gkemRZscehMOi0tN+UX1gd7fJGsqsDK6geuqpShP
IpfMLriGoQb6Zy4fwEq9N5+AfWXfSZ9Kb8ab8ksvuA==
=Wm8a
-----END PGP MESSAGE-----
fp: 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
unencrypted_suffix: _unencrypted
version: 3.7.3

21
secrets/cache-priv-key.pem

@ -0,0 +1,21 @@
{
"data": "ENC[AES256_GCM,data:zFVNY6fYkVEvHcZ/IaWvcmIkf+NwZ9p45XEy7/sxpSvr62F80pzxAiC99IX+1+XLH83zk5dqm1vMUuX9NdNAxB0Mousyp1YdkF0Zqi5/il9B/p7R24AIfgeQCa46qo5MbYVWRgs6R1rp9Y573+6/SbPtDqoChvE1Kic=,iv:uQa4O9WnyFZ+kPvp/ozXilCTyUJcLvwlVWF7rmTi9w8=,tag:2MuFj4/Mn9LECE7cToQwVQ==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": null,
"lastmodified": "2023-05-26T08:59:34Z",
"mac": "ENC[AES256_GCM,data:8h8NREXye3DDL7DpvT7sVr1lyaAfEgDwOoaDMuCzzRyHFWPSELQHnjLjEjmexoRrrsE/U608/h62PU7m9EDSYuWlJsvuNBZ+HezR/Ve8oFrZ5ZE3HIoEt2aeM2enSEHGP+aYFL4jEZJJDn9xoW3chFu3JLTSez0NOAhuejghjnU=,iv:Dfxlfa/mwKswYL077oPV+rylKk5y67qKPz+6UFCje9c=,tag:lmM0U8H5FlVRMO51mqTZgg==,type:str]",
"pgp": [
{
"created_at": "2023-05-26T08:54:32Z",
"enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA5ntoryXZPD4AQf/WmHVgATZ4cl/zI+aRMYGrWyAHAWJ/gtXzTbY2oGHdonw\nx4+5XBsFg28JcJXlI9Aq643e8+/2BPie4tawyrNfWBcaovHbFzEvc4EK0wPbx0Ax\nYW2P237lKyCfOhC4uzeghlr/IpX+SGZGvSDmg6R99/sXZ8pnPFG6PwPp2rdE1JMJ\nZRupMzZfSgJWgZXQIxJhiymHh1ddAMGuLhDzRSj7eVZiN8kl39Y0wEKzmCqCSvIl\n5nn3EmGsB2sSNo8W6C91WQRyxRBP48wWUSZ0P7lHXQEqJW9ioLGq+1qLaL2ZVA3h\nr++vjXf+v9yIsOSGVJAehVV4rXF1pJJJSDMewG6bJdJeAQOiR8+mLvdtwIQOfEFM\nQVvNJ6RfRKSYIrRxBqBJx4vDKTUtktmcBRZJazB7s+TWkhAtrFHyZXCcO9L9Uz7/\nePJ8xD8z6SDZTUa7Y2mJx416mVZwvz7yEWThIBrGGA==\n=Xn2g\n-----END PGP MESSAGE-----\n",
"fp": "3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE"
}
],
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}

35
shared/default.nix

@ -0,0 +1,35 @@
{ pkgs, modulesPath, ... }:
{
imports = [
./network.nix
./root.nix
#"${modulesPath}/profiles/headless.nix"
"${modulesPath}/profiles/all-hardware.nix"
];
sops = {
defaultSopsFile = ../secrets.yaml;
defaultSopsFormat = "yaml";
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
};
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.initrd.systemd.enable = true;
time.timeZone = "Europe/Berlin";
console.keyMap = "de";
environment.systemPackages = with pkgs; [
vim
wget
curl
tmux
];
system.stateVersion = "22.11";
}

10
shared/network.nix

@ -0,0 +1,10 @@
{
networking.domain = "hpc.informatik.hs-fulda.de";
networking.useDHCP = true;
services.openssh = {
enable = true;
permitRootLogin = "without-password";
};
}

11
shared/root.nix

@ -0,0 +1,11 @@
{
users.mutableUsers = false;
users.users."root" = {
hashedPassword = "$y$j9T$tz8ojZ2gVOQ5AUp6GMhoj.$mAeE0eTGGsKNGddC7ebk/zFr5IMDyIpOpMP/6o.GI6D";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK2nkarN0+uSuP5sGwDCb9KRu+FCjO/+da4VypGanPUZ fooker@k-2so"
];
};
}
Loading…
Cancel
Save