hpc
/
nixcfg
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Inital import

main
Dustin Frisch 2 years ago
commit
ce8862bf37
No known key found for this signature in database GPG Key ID: B4C3BF012D9B26BE
27 changed files with 1520 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      .envrc
  2. 5
      .sops.yaml
  3. 453
      flake.lock
  4. 95
      flake.nix
  5. 38
      machines.nix
  6. 5
      machines/manager/autoinstall.nix
  7. 29
      machines/manager/beegfs.nix
  8. 23
      machines/manager/cache.nix
  9. 35
      machines/manager/default.nix
  10. 40
      machines/manager/disk.nix
  11. 4
      machines/manager/gateway.nix
  12. 44
      machines/manager/ldap.nix
  13. 8
      machines/manager/nfs.nix
  14. 5
      machines/manager/ntp.nix
  15. 24
      machines/node/default.nix
  16. 39
      machines/node/disk.nix
  17. 22
      machines/node/users.nix
  18. 227
      modules/beegfs.nix
  19. 5
      modules/default.nix
  20. 185
      packages/beegfs/001-build.patch
  21. 77
      packages/beegfs/default.nix
  22. 45
      packages/beegfs/module.nix
  23. 31
      secrets.yaml
  24. 21
      secrets/cache-priv-key.pem
  25. 35
      shared/default.nix
  26. 10
      shared/network.nix
  27. 11
      shared/root.nix

4
.envrc
View File

@ -0,0 +1,4 @@
use flake
watch_file "flake.nix"
watch_file "flake.lock"

5
.sops.yaml
View File

@ -0,0 +1,5 @@
creation_rules:
- key_groups:
- pgp:
- 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
path_regex: ^(secrets\.yaml|secrets/.+)$

453
flake.lock
View File

@ -0,0 +1,453 @@
{
"nodes": {
"colmena": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"stable": "stable"
},
"locked": {
"lastModified": 1684127527,
"narHash": "sha256-tAzgb2jgmRaX9HETry38h2OvBf9YkHEH1fFvIJQV9A0=",
"owner": "zhaofengli",
"repo": "colmena",
"rev": "caf33af7d854c8d9b88a8f3dae7adb1c24c1407b",
"type": "github"
},
"original": {
"owner": "zhaofengli",
"repo": "colmena",
"type": "github"
}
},
"disko": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1684783210,
"narHash": "sha256-hxRbwwBTu1G1u1EdI9nEo/n4HIsQIfNi+2BQ1nEoj/o=",
"owner": "nix-community",
"repo": "disko",
"rev": "f0b9f374bb42fdcd57baa7d4448ac5d4788226bd",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "disko",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1650374568,
"narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "b4a34015c698c7793d592d66adbab377907a2be8",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_3": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_4": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_5": {
"locked": {
"lastModified": 1653893745,
"narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_6": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"pre-commit-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1660459072,
"narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixago": {
"inputs": {
"flake-utils": [
"utils"
],
"nixago-exts": "nixago-exts",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1683210100,
"narHash": "sha256-bhGDOlkWtlhVECpoOog4fWiFJmLCpVEg09a40aTjCbw=",
"owner": "nix-community",
"repo": "nixago",
"rev": "1da60ad9412135f9ed7a004669fdcf3d378ec630",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago",
"type": "github"
}
},
"nixago-exts": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixago": "nixago_2",
"nixpkgs": [
"nixago",
"nixpkgs"
]
},
"locked": {
"lastModified": 1676070308,
"narHash": "sha256-QaJ65oc2l8iwQIGWUJ0EKjCeSuuCM/LqR8RauxZUUkc=",
"owner": "nix-community",
"repo": "nixago-extensions",
"rev": "e5380cb0456f4ea3c86cf94e3039eb856bf07d0b",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago-extensions",
"type": "github"
}
},
"nixago-exts_2": {
"inputs": {
"flake-utils": "flake-utils_4",
"nixago": "nixago_3",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixago",
"nixpkgs"
]
},
"locked": {
"lastModified": 1655508669,
"narHash": "sha256-BDDdo5dZQMmwNH/GNacy33nPBnCpSIydWFPZs0kkj/g=",
"owner": "nix-community",
"repo": "nixago-extensions",
"rev": "3022a932ce109258482ecc6568c163e8d0b426aa",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago-extensions",
"type": "github"
}
},
"nixago_2": {
"inputs": {
"flake-utils": "flake-utils_3",
"nixago-exts": "nixago-exts_2",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixpkgs"
]
},
"locked": {
"lastModified": 1676070010,
"narHash": "sha256-iYzJIWptE1EUD8VINAg66AAMUajizg8JUYN3oBmb8no=",
"owner": "nix-community",
"repo": "nixago",
"rev": "d480ba6c0c16e2c5c0bd2122852d6a0c9ad1ed0e",
"type": "github"
},
"original": {
"owner": "nix-community",
"ref": "rename-config-data",
"repo": "nixago",
"type": "github"
}
},
"nixago_3": {
"inputs": {
"flake-utils": "flake-utils_5",
"nixpkgs": [
"nixago",
"nixago-exts",
"nixago",
"nixago-exts",
"nixpkgs"
]
},
"locked": {
"lastModified": 1655405483,
"narHash": "sha256-Crd49aZWNrpczlRTOwWGfwBMsTUoG9vlHDKQC7cx264=",
"owner": "nix-community",
"repo": "nixago",
"rev": "e6a9566c18063db5b120e69e048d3627414e327d",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixago",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1684858140,
"narHash": "sha256-dQStox5GYrVlVNMvxxXs3xX9bXG7J7ttSjqUcVm8EaA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "a17f99dfcb9643200b3884ca195c69ae41d7f059",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1678872516,
"narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-utils": "flake-utils_6",
"gitignore": "gitignore",
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1684842236,
"narHash": "sha256-rYWsIXHvNhVQ15RQlBUv67W3YnM+Pd+DuXGMvCBq2IE=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "61e567d6497bc9556f391faebe5e410e6623217f",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"colmena": "colmena",
"disko": "disko",
"nixago": "nixago",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": "pre-commit-hooks",
"sops": "sops",
"utils": "utils"
}
},
"sops": {
"inputs": {
"nixpkgs": [
"nixpkgs"
],
"nixpkgs-stable": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1684637723,
"narHash": "sha256-0vAxL7MVMhGbTkAyvzLvleELHjVsaS43p+PR1h9gzNQ=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "4ccdfb573f323a108a44c13bb7730e42baf962a9",
"type": "github"
},
"original": {
"owner": "Mic92",
"repo": "sops-nix",
"type": "github"
}
},
"stable": {
"locked": {
"lastModified": 1669735802,
"narHash": "sha256-qtG/o/i5ZWZLmXw108N2aPiVsxOcidpHJYNkT45ry9Q=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "731cc710aeebecbf45a258e977e8b68350549522",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

95
flake.nix
View File

@ -0,0 +1,95 @@
{
inputs = {
nixpkgs = {
type = "github";
owner = "NixOS";
repo = "nixpkgs";
ref = "nixos-22.11";
};
colmena = {
type = "github";
owner = "zhaofengli";
repo = "colmena";
inputs.nixpkgs.follows = "nixpkgs";
};
nixago = {
type = "github";
owner = "nix-community";
repo = "nixago";
inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-utils.follows = "utils";
};
pre-commit-hooks = {
type = "github";
owner = "cachix";
repo = "pre-commit-hooks.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
sops = {
type = "github";
owner = "Mic92";
repo = "sops-nix";
inputs.nixpkgs.follows = "nixpkgs";
inputs.nixpkgs-stable.follows = "nixpkgs";
};
utils = {
type = "github";
owner = "numtide";
repo = "flake-utils";
};
disko = {
type = "github";
owner = "nix-community";
repo = "disko";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { nixpkgs, utils, ... }@inputs: {
colmena = import ./machines.nix inputs;
devShell = utils.lib.eachSystemMap utils.lib.allSystems (system:
let
pkgs = nixpkgs.legacyPackages.${system};
pre-commit-hooks = inputs.pre-commit-hooks.lib.${system}.run {
src = ./.;
hooks = {
nixpkgs-fmt.enable = true;
statix.enable = true;
shellcheck.enable = true;
};
};
in
pkgs.mkShell {
buildInputs = [
inputs.colmena.defaultPackage.${system}
] ++ (with pkgs; [
bash
gitAndTools.git
gnutar
gzip
nixUnstable
openssh
sops
age
]);
shellHook = ''
${pre-commit-hooks.shellHook}
'';
});
};
}

38
machines.nix
View File

@ -0,0 +1,38 @@
{ nixpkgs, disko, sops, ... }@inputs:
let
deploymentPkgs = import nixpkgs {
localSystem.system = "x86_64-linux";
};
in with deploymentPkgs.lib; let
mkMachine = type: opts: { lib, ... }:
let
machine = import ./machines/${type} opts;
in {
imports = [
./shared
./modules
machine
sops.nixosModules.sops
disko.nixosModules.disko
];
};
machines = {
manager = mkMachine "manager" {};
} // (listToAttrs (genList (i: nameValuePair
"node-${fixedWidthNumber 2 i}"
(mkMachine "node" { id = i; })
) 16));
in {
meta = {
nixpkgs = deploymentPkgs;
specialArgs = {
inherit inputs;
};
};
} // machines

5
machines/manager/autoinstall.nix
View File

@ -0,0 +1,5 @@
# TFTP boot with shared image
# Requests store path to install from master
# Runs disko and nixos-install
{}

29
machines/manager/beegfs.nix
View File

@ -0,0 +1,29 @@
{ pkgs, config, lib, ... }:
with lib;
let
connAuth = toString (pkgs.writeText "beegfs-conn-auth" "asdasdasdasd");
# connAuth = ${config.sops.secrets."beegfs/connection".path}
in
{
nixpkgs.config.allowUnfree = true;
hpc.beegfs = {
connAuthFile = connAuth;
mgmtdHost = "manager.hpc.informatik.hs-fulda.de";
client = {
enable = true;
mountPoint = "/projects";
};
mgmtd.enable = true;
meta.enable = true;
storage.enable = true;
};
sops.secrets."beegfs/connection" = {};
}

23
machines/manager/cache.nix
View File

@ -0,0 +1,23 @@
{ config, ... }:
{
services.nix-serve = {
enable = true;
secretKeyFile = config.sops.secrets."cache/privateKey".path;
};
sops.secrets."cache/privateKey" = {
format = "binary";
sopsFile = ../../secrets/cache-priv-key.pem;
};
services.nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"cache.hpc.informatik.hs-fulda.de" = {
locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
};
};
};
}

35
machines/manager/default.nix
View File

@ -0,0 +1,35 @@
{ ... }:
{ lib, config, ... }:
with lib;
{
imports = [
./disk.nix
./nfs.nix
#./ldap.nix
#./beegfs.nix
#./ntp.nix
#./gateway.nix
#./autoinstall.nix
#./cache.nix
];
users.users."root".password = "asdasd123";
virtualisation.useDefaultFilesystems = false;
virtualisation.fileSystems."/" = {
device = config.virtualisation.bootDevice;
fsType = "btrfs";
autoFormat = true;
};
deployment = {
targetHost = "10.32.30.240";
targetUser = "root";
tags = [ "manager" ];
};
networking.hostName = "manager";
}

40
machines/manager/disk.nix
View File

@ -0,0 +1,40 @@
{ ... }:
{
disko.devices = {
disk.sda = {
device = "/dev/nvme0n1";
type = "disk";
content = {
type = "table";
format = "gpt";
partitions = [
{
name = "root";
start = "100MiB";
end = "100%";
part-type = "primary";
bootable = true;
content = {
type = "filesystem";
format = "ext4";
#format = "btrfs";
mountpoint = "/";
};
}
{
name = "ESP";
start = "1MiB";
end = "100MiB";
bootable = true;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
}
];
};
};
};
}

4
machines/manager/gateway.nix
View File

@ -0,0 +1,4 @@
# DHCP server for nodes
# NAT gateway for nodes
{}

44
machines/manager/ldap.nix
View File

@ -0,0 +1,44 @@
{ config, ... }:
{
services.openldap = {
enable = true;
settings = {
children = {
"cn=schema".includes = [
"${config.services.openldap.package}/etc/schema/core.ldif"
"${config.services.openldap.package}/etc/schema/cosine.ldif"
"${config.services.openldap.package}/etc/schema/inetorgperson.ldif"
"${config.services.openldap.package}/etc/schema/nis.ldif"
];
"olcDatabase={1}mdb" = {
attrs = {
objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
olcDatabase = "{1}mdb";
olcDbDirectory = "/var/lib/openldap/db";
olcSuffix = "dc=sc,dc=informatik,dc=hs-fulda,dc=de";
olcRootDN = "cn=root,dc=sc,dc=informatik,dc=hs-fulda,dc=de";
olcRootPW.path = config.sops.secrets."ldap/root/password".path;
olcAccess = [
# Custom access rules for userPassword attributes
''{0}to attrs=userPassword
by self write
by anonymous auth
by * none''
# Allow read on anything else
''{1}to *
by * read''
];
};
};
};
};
};
sops.secrets."ldap/root/password" = { };
}

8
machines/manager/nfs.nix
View File

@ -0,0 +1,8 @@
{
services.nfs.server = {
enable = true;
exports = ''
/home node*.hpc.informatik.hs-fulda.de(rw)
'';
};
}

5
machines/manager/ntp.nix
View File

@ -0,0 +1,5 @@
{
services.chrony = {
enable = true;
};
}

24
machines/node/default.nix
View File

@ -0,0 +1,24 @@
{ id, ... }:
{ lib, ... }:
with lib;
{
imports = [
./disk.nix
];
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
networking.hostName = "node-${fixedWidthNumber 2 id}";
nix.settings = {
substituters = [
"http://cache.hpc.informatik.hs-fulda.de"
];
trusted-public-keys = [
"cache.hpc.informatik.hs-fulda.de:dc2abEGJAQfaZiBXhjvjPU0jx/wosQwAOQoz48/G6cA="
];
};
}

39
machines/node/disk.nix
View File

@ -0,0 +1,39 @@
{ ... }:
{
disko.devices = {
disk.sda = {
device = "/dev/sda";
type = "disk";
content = {
type = "table";
format = "gpt";
partitions = [
{
name = "root";
start = "100MiB";
end = "100%";
part-type = "primary";
bootable = true;
content = {
type = "filesystem";
format = "btrfs";
mountpoint = "/";
};
}
{
name = "ESP";
start = "1MiB";
end = "100MiB";
bootable = true;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
}
];
};
};
};
}

22
machines/node/users.nix
View File

@ -0,0 +1,22 @@
{
users.ldap = {
enable = true;
server = "ldap://manager.hpc.informatik.hs-fulda.de/";
base = "dc=hpc,dc=informatik,dc=hs-fulda,dc=de";
daemon.enable = true;
};
filesystem."home" = {
mountPoint = "/home";
device = "manager.hpc.informatik.hs-fulda.de:/home";
fsType = "nfs";
options = [
"nfsvers=4.2"
"noauto"
"x-systemd.automount"
"x-systemd.idle-timeout=600"
];
};
}

227
modules/beegfs.nix
View File

@ -0,0 +1,227 @@
{ pkgs, config, lib, ... }:
with lib;
{
options.hpc.beegfs = {
package = mkOption {
description = ''
BeeGFS package.
'';
type = types.package;
default = config.boot.kernelPackages.callPackage ../packages/beegfs/default.nix { };
};
mgmtdHost = mkOption {
description = ''
Hostname of the management host.
'';
type = types.str;
};
connAuthFile = mkOption {
description = ''
File containing shared secret authentication.
'';
type = types.str;
};
client = {
enable = mkEnableOption "BeeGFS client";
mountPoint = mkOption {
description = ''
Mount point under which the BeeGFS filesystem is mounted.
'';
type = types.nullOr types.str;
default = null;
};
};
mgmtd = {
enable = mkEnableOption "BeeGFS management server daemon";
};
meta = {
enable = mkEnableOption "BeeGFS meta-data server daemon";
};
storage = {
enable = mkEnableOption "BeeGFS storage server daemon";
};
};
config = mkMerge [
(mkIf config.hpc.beegfs.client.enable {
boot.kernelModules = [ "beegfs" ];
boot.extraModulePackages = [ config.hpc.beegfs.package.module ];
environment.etc."beegfs-client" = {
enable = true;
target = "beegfs/client.conf";
text = ''
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
'';
};
systemd.mounts = mkIf (config.hpc.beegfs.client.mountPoint != null) [ {
where = config.hpc.beegfs.client.mountPoint;
what = "beegfs_nodev";
type = "beegfs";
options = "cfgFile=/etc/beegfs/client.conf,_netdev";
requires = [ "beegfs-helperd.service" ];
after = [ "beegfs-helperd.service" ];
} ];
systemd.services."beegfs-helperd" = let
cfgFile = pkgs.writeText "beegfs-helperd.conf" ''
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" ];
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-helperd \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-helperd.pid";
TimeoutStopSec = "300";
};
};
environment.systemPackages = [ (pkgs.runCommandLocal "beegfs-utils" {
nativeBuildInputs = [ pkgs.makeWrapper ];
} ''
mkdir -p $out/bin
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-check-servers \
$out/bin/beegfs-check-servers \
--add-flags "-c /etc/beegfs/client.conf" \
--prefix PATH : ${lib.makeBinPath [ config.hpc.beegfs.package ]}
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-ctl \
$out/bin/beegfs-ctl \
--add-flags "--cfgFile=/etc/beegfs/client.conf"
makeWrapper ${config.hpc.beegfs.package}/bin/beegfs-ctl \
$out/bin/beegfs-df \
--add-flags "--cfgFile=/etc/beegfs/client.conf" \
--add-flags --listtargets \
--add-flags --hidenodeid \
--add-flags --pools \
--add-flags --spaceinfo
makeWrapper ${config.hpc.beegfs.package}/bin/fsck.beegfs \
$out/bin/beegfs-fsck \
--add-flags "--cfgFile=/etc/beegfs/client.conf"
'') ];
})
(mkIf config.hpc.beegfs.mgmtd.enable (let
cfgFile = pkgs.writeText "beegfs-mgmtd.conf" ''
storeMgmtdDirectory = /var/lib/beegs/mgmtd
storeAllowFirstRunInit = false
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-mgmtd" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/mgmtd; then
${config.hpc.beegfs.package}/bin/beegfs-setup-mgmtd -C -p /var/lib/beegs/mgmtd
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-mgmtd \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-mgmtd.pid";
TimeoutStopSec = "300";
};
};
}))
(mkIf config.hpc.beegfs.meta.enable (let
cfgFile = pkgs.writeText "beegfs-meta.conf" ''
storeMetaDirectory = /var/lib/beegs/meta
storeAllowFirstRunInit = false
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-meta" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "beegfs-mgmt.service" "beegfs-storage.service" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/meta; then
${config.hpc.beegfs.package}/bin/beegfs-setup-meta -C -p /var/lib/beegs/meta
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-meta \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-meta.pid";
TimeoutStopSec = "300";
};
};
}))
(mkIf config.hpc.beegfs.storage.enable (let
cfgFile = pkgs.writeText "beegfs-storage.conf" ''
storeStorageDirectory = /var/lib/beegs/storage
storeAllowFirstRunInit = false
sysMgmtdHost = ${config.hpc.beegfs.mgmtdHost}
connAuthFile = ${config.hpc.beegfs.connAuthFile}
logType = syslog
'';
in {
systemd.services."beegfs-storage" = {
wantedBy = [ "multi-user.target" ];
requires = [ "network-online.target" ];
after = [ "network-online.target" "beegfs-mgmt.service" "rdma.service" "opensmd.service" "opensm.service" ];
preStart = ''
if ! test -e /var/lib/beegs/storage; then
${config.hpc.beegfs.package}/bin/beegfs-setup-storage -C -p /var/lib/beegs/storage
fi
'';
serviceConfig = rec {
ExecStart = ''
${config.hpc.beegfs.package}/bin/beegfs-storage \
cfgFile=${cfgFile} \
pidFile=${PIDFile} \
runDaemonized=false
'';
PIDFile = "/run/beegfs-storage.pid";
TimeoutStopSec = "300";
};
};
}))
];
}

5
modules/default.nix
View File

@ -0,0 +1,5 @@
{
imports = [
./beegfs.nix
];
}

185
packages/beegfs/001-build.patch
View File

@ -0,0 +1,185 @@
diff -r -u a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/CMakeLists.txt 2023-05-27 21:27:08.991487355 +0200
@@ -85,10 +85,8 @@
add_subdirectory("thirdparty/source/gtest")
endif()
-set(CMAKE_INSTALL_PREFIX "/")
-
-add_subdirectory("beeond")
-add_subdirectory("beeond_thirdparty_gpl")
+# add_subdirectory("beeond")
+# add_subdirectory("beeond_thirdparty_gpl")
# add_subdirectory("client_devel")
# add_subdirectory("client_module")
add_subdirectory("common")
diff -r -u a/common/CMakeLists.txt b/common/CMakeLists.txt
--- a/common/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/common/CMakeLists.txt 2023-05-29 10:57:53.502540650 +0200
@@ -572,6 +572,5 @@
install(
TARGETS beegfs_ib
- DESTINATION "usr/lib"
COMPONENT "libbeegfs-ib"
)
diff -r -u a/ctl/CMakeLists.txt b/ctl/CMakeLists.txt
--- a/ctl/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/ctl/CMakeLists.txt 2023-05-29 10:57:13.297068443 +0200
@@ -149,6 +149,5 @@
install(
TARGETS beegfs-ctl
- DESTINATION "usr/bin"
COMPONENT "utils"
)
diff -r -u a/event_listener/CMakeLists.txt b/event_listener/CMakeLists.txt
--- a/event_listener/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/event_listener/CMakeLists.txt 2023-05-29 10:57:16.262103203 +0200
@@ -10,13 +10,12 @@
install(
TARGETS beegfs-event-listener
- DESTINATION "usr/sbin"
COMPONENT "event-listener"
)
install(
FILES "include/beegfs/beegfs_file_event_log.hpp"
- DESTINATION "usr/include/beegfs"
+ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/beegfs"
COMPONENT "event-listener"
)
Only in v7-7.3.3: foo
diff -r -u a/fsck/CMakeLists.txt b/fsck/CMakeLists.txt
--- a/fsck/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/fsck/CMakeLists.txt 2023-05-29 10:57:18.938134583 +0200
@@ -148,6 +148,5 @@
install(
TARGETS fsck.beegfs
- DESTINATION "sbin"
COMPONENT "utils"
)
diff -r -u a/helperd/CMakeLists.txt b/helperd/CMakeLists.txt
--- a/helperd/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/helperd/CMakeLists.txt 2023-05-29 10:57:21.185160939 +0200
@@ -59,6 +59,5 @@
install(
TARGETS beegfs-helperd
- DESTINATION "usr/sbin"
COMPONENT "helperd"
)
diff -r -u a/meta/CMakeLists.txt b/meta/CMakeLists.txt
--- a/meta/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/meta/CMakeLists.txt 2023-05-29 11:00:36.501472258 +0200
@@ -326,13 +326,12 @@
install(
TARGETS beegfs-meta
- DESTINATION "usr/sbin"
COMPONENT "meta"
)
install(
PROGRAMS "build/dist/sbin/beegfs-setup-meta"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "meta"
)
@@ -347,10 +346,3 @@
DESTINATION "etc/beegfs"
COMPONENT "meta"
)
-
-install(
- PROGRAMS "build/beegfs-meta.sh"
- RENAME "beegfs-meta"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "meta"
-)
diff -r -u a/mgmtd/CMakeLists.txt b/mgmtd/CMakeLists.txt
--- a/mgmtd/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/mgmtd/CMakeLists.txt 2023-05-29 11:00:44.428566811 +0200
@@ -161,13 +161,12 @@
install(
TARGETS beegfs-mgmtd
- DESTINATION "usr/sbin"
COMPONENT "mgmtd"
)
install(
PROGRAMS "build/dist/sbin/beegfs-setup-mgmtd"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "mgmtd"
)
@@ -182,10 +181,3 @@
DESTINATION "etc/beegfs"
COMPONENT "mgmtd"
)
-
-install(
- PROGRAMS "build/beegfs-mgmtd.sh"
- RENAME "beegfs-mgmtd"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "mgmtd"
-)
diff -r -u a/storage/CMakeLists.txt b/storage/CMakeLists.txt
--- a/storage/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/storage/CMakeLists.txt 2023-05-29 11:00:53.385673711 +0200
@@ -188,7 +188,6 @@
install(
TARGETS beegfs-storage
- DESTINATION "usr/sbin"
COMPONENT "storage"
)
@@ -200,7 +199,7 @@
install(
PROGRAMS "build/dist/sbin/beegfs-setup-storage"
- DESTINATION "usr/sbin"
+ TYPE BIN
COMPONENT "storage"
)
@@ -209,10 +208,3 @@
DESTINATION "etc/beegfs"
COMPONENT "storage"
)
-
-install(
- PROGRAMS "build/beegfs-storage.sh"
- RENAME "beegfs-storage"
- DESTINATION "opt/beegfs/sbin"
- COMPONENT "storage"
-)
diff -r -u a/upgrade/beegfs_mirror_md/CMakeLists.txt b/upgrade/beegfs_mirror_md/CMakeLists.txt
--- a/upgrade/beegfs_mirror_md/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/upgrade/beegfs_mirror_md/CMakeLists.txt 2023-05-29 10:57:40.926392744 +0200
@@ -15,6 +15,5 @@
install(
TARGETS "beegfs-mirror-md"
- DESTINATION "usr/sbin"
COMPONENT "beegfs-mirror-md"
)
diff -r -u a/utils/CMakeLists.txt b/utils/CMakeLists.txt
--- a/utils/CMakeLists.txt 2023-03-15 12:09:15.000000000 +0100
+++ b/utils/CMakeLists.txt 2023-05-29 11:01:05.123813896 +0200
@@ -6,6 +6,6 @@
install(
PROGRAMS "scripts/beegfs-check-servers" "scripts/beegfs-df" "scripts/beegfs-net"
- DESTINATION "usr/bin"
+ TYPE BIN
COMPONENT "utils"
)

77
packages/beegfs/default.nix
View File

@ -0,0 +1,77 @@
{ stdenv
, fetchurl
, pkgconfig
, util-linux
, which
, libuuid
, attr
, xfsprogs
, rdma-core
, zlib
, openssl
, openssh
, curl
, cmake
, callPackage
, kernel ? null
, ... } :
stdenv.mkDerivation (final: rec {
pname = "beegfs";
version = "7.3.3";
src = fetchurl {
url = "https://git.beegfs.io/pub/v7/-/archive/${version}/v7-${version}.tar.gz";
sha256 = "sha256-XfZY6ge4KWNJn9UE41b7ds2YCMz9FNXFqZd51qCatig=";
};
nativeBuildInputs = [
pkgconfig
which
cmake
];
buildInputs = [
util-linux
libuuid
attr
xfsprogs
zlib
openssl
rdma-core
openssh
curl
];
patches = [
./001-build.patch
];
dontFixCmake = true;
cmakeFlags = [
"-DBEEGFS_VERSION=${version}"
"-DBEEGFS_SKIP_TESTS=true"
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=${placeholder "out"}"
"-DCMAKE_INSTALL_LIBDIR=lib"
];
hardeningDisable = [ "format" ];
passthru.module = callPackage ./module.nix {
inherit kernel;
beegfs = final;
};
meta = with stdenv.lib; {
description = "High performance distributed filesystem with RDMA support";
homepage = "https://www.beegfs.io";
platforms = [ "i686-linux" "x86_64-linux" ];
license = {
fullName = "BeeGFS_EULA";
url = "https://www.beegfs.io/docs/BeeGFS_EULA.txt";
free = false;
};
};
})

45
packages/beegfs/module.nix
View File

@ -0,0 +1,45 @@
{ beegfs
, kmod
, kernel
, ... } :
kernel.stdenv.mkDerivation rec {
name = "beegfs-module-${beegfs.version}-${kernel.version}";
inherit (beegfs) src;
nativeBuildInputs = [
kmod
];
buildInputs = kernel.moduleBuildDependencies;
hardeningDisable = [ "fortify" "pic" "stackprotector" ];
sourceRoot = "v7-${beegfs.version}/client_module/build";
makeFlags = [
"KERNELRELEASE=${kernel.modDirVersion}"
"KDIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build/"
"BEEGFS_VERSION=${beegfs.version}-nixos1"
];
postPatch = ''
patchShebangs ./
find -type f -name Makefile -exec sed -i "s:/bin/true:true:" \{} \;
find -type f -name "*.mk" -exec sed -i "s:/bin/true:true:" \{} \;
'';
installPhase = ''
mkdir -p $out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs
install -t $out/lib/modules/${kernel.modDirVersion}/extras/fs/beegfs beegfs.ko
'';
enableParallelBuilding = true;
meta = with kernel.stdenv.lib; {
description = beegfs.meta.description + " (kernel module)";
inherit (beegfs.meta) homepage license platforms;
};
}

31
secrets.yaml
View File

@ -0,0 +1,31 @@
ldap:
root:
password: ENC[AES256_GCM,data:bYuw+9ywfRDNVt0nrLDmWE8+f8aHQvGd,iv:JHU3MxmNdxI2a62Dcky8xhHhjhcxyjM0Z0xLEnLxJwU=,tag:3VW0zTlRFxLDI8WxGu1lew==,type:str]
beegfs:
connection: ENC[AES256_GCM,data:YTHMg76+5Azb+ex5ArUHt4xP+YYWr9Ph,iv:TEf8i+yezPsaW12Lg5jRnhds9uW9WhV6duZPdxeW9co=,tag:bPGsl7ofwE1Jh+FTyHJqzQ==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age: []
lastmodified: "2023-05-29T12:51:30Z"
mac: ENC[AES256_GCM,data:02jKHbEZGs3QiNzXEQxcB8v/i5UVB/pCciz4hSI220+GEYPgQK6qR1cZJaMAyrHKjzJLhNZq3Gfgsj4zfA+FMg/d12vp2QNTMRrVD/hSh67NgloZ/iTmJC//S8OJfiHEPdGKkq7zXCVajnkGMT/0yLNWAKISAwL451ohgMzMQYw=,iv:8hqKXUolNA7WatnnYwwUN2EgOyZjTISG2bfToENYc7c=,tag:5y43RQJgZbPK8g3Cw8CBzQ==,type:str]
pgp:
- created_at: "2023-05-24T20:04:49Z"
enc: |
-----BEGIN PGP MESSAGE-----
hQEMA5ntoryXZPD4AQf6A69nF8BRpYRdz3ea8acqryKoMe5p2A44drykDQR0NO9r
I6j0Hg9AksgC+rGRIQtCuj18gYybDFXgYLCE8MYfgh2NSyqeGzq2+kPDqAXRong/
Wrg1+KRlDbvIqH7IZ5BS40TGdphh/U8BIUcO8N4tgP60G6C7z9FqfjiA5YByqau3
7uAtKg3kR6lL13Cf0AUnMrQ8AOZ+6p+BwdTcXeUW2bScw8ScbEQsw/MtoiEN9Een
jvPhqTczdcZLIgTV+DvmimwYmH8xwFiMNFBrt4uzsBMv9N4pb0EzL8TcKIOuE8iw
YserGEi/sMx5QzqYmS2yPvNxwcXsZi28SQrHOs4Lv9JeAdpqwrqJjAaV3pN0OgSy
31XV/oDL8GJ3SfNqUZEULB06gkemRZscehMOi0tN+UX1gd7fJGsqsDK6geuqpShP
IpfMLriGoQb6Zy4fwEq9N5+AfWXfSZ9Kb8ab8ksvuA==
=Wm8a
-----END PGP MESSAGE-----
fp: 3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE
unencrypted_suffix: _unencrypted
version: 3.7.3

21
secrets/cache-priv-key.pem
View File

@ -0,0 +1,21 @@
{
"data": "ENC[AES256_GCM,data:zFVNY6fYkVEvHcZ/IaWvcmIkf+NwZ9p45XEy7/sxpSvr62F80pzxAiC99IX+1+XLH83zk5dqm1vMUuX9NdNAxB0Mousyp1YdkF0Zqi5/il9B/p7R24AIfgeQCa46qo5MbYVWRgs6R1rp9Y573+6/SbPtDqoChvE1Kic=,iv:uQa4O9WnyFZ+kPvp/ozXilCTyUJcLvwlVWF7rmTi9w8=,tag:2MuFj4/Mn9LECE7cToQwVQ==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": null,
"lastmodified": "2023-05-26T08:59:34Z",
"mac": "ENC[AES256_GCM,data:8h8NREXye3DDL7DpvT7sVr1lyaAfEgDwOoaDMuCzzRyHFWPSELQHnjLjEjmexoRrrsE/U608/h62PU7m9EDSYuWlJsvuNBZ+HezR/Ve8oFrZ5ZE3HIoEt2aeM2enSEHGP+aYFL4jEZJJDn9xoW3chFu3JLTSez0NOAhuejghjnU=,iv:Dfxlfa/mwKswYL077oPV+rylKk5y67qKPz+6UFCje9c=,tag:lmM0U8H5FlVRMO51mqTZgg==,type:str]",
"pgp": [
{
"created_at": "2023-05-26T08:54:32Z",
"enc": "-----BEGIN PGP MESSAGE-----\n\nhQEMA5ntoryXZPD4AQf/WmHVgATZ4cl/zI+aRMYGrWyAHAWJ/gtXzTbY2oGHdonw\nx4+5XBsFg28JcJXlI9Aq643e8+/2BPie4tawyrNfWBcaovHbFzEvc4EK0wPbx0Ax\nYW2P237lKyCfOhC4uzeghlr/IpX+SGZGvSDmg6R99/sXZ8pnPFG6PwPp2rdE1JMJ\nZRupMzZfSgJWgZXQIxJhiymHh1ddAMGuLhDzRSj7eVZiN8kl39Y0wEKzmCqCSvIl\n5nn3EmGsB2sSNo8W6C91WQRyxRBP48wWUSZ0P7lHXQEqJW9ioLGq+1qLaL2ZVA3h\nr++vjXf+v9yIsOSGVJAehVV4rXF1pJJJSDMewG6bJdJeAQOiR8+mLvdtwIQOfEFM\nQVvNJ6RfRKSYIrRxBqBJx4vDKTUtktmcBRZJazB7s+TWkhAtrFHyZXCcO9L9Uz7/\nePJ8xD8z6SDZTUa7Y2mJx416mVZwvz7yEWThIBrGGA==\n=Xn2g\n-----END PGP MESSAGE-----\n",
"fp": "3237CA7A1744B4DCE96B409FB4C3BF012D9B26BE"
}
],
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}

35
shared/default.nix
View File

@ -0,0 +1,35 @@
{ pkgs, modulesPath, ... }:
{
imports = [
./network.nix
./root.nix
#"${modulesPath}/profiles/headless.nix"
"${modulesPath}/profiles/all-hardware.nix"
];
sops = {
defaultSopsFile = ../secrets.yaml;
defaultSopsFormat = "yaml";
age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
};
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
boot.initrd.systemd.enable = true;
time.timeZone = "Europe/Berlin";
console.keyMap = "de";
environment.systemPackages = with pkgs; [
vim
wget
curl
tmux
];
system.stateVersion = "22.11";
}

10
shared/network.nix
View File

@ -0,0 +1,10 @@
{
networking.domain = "hpc.informatik.hs-fulda.de";
networking.useDHCP = true;
services.openssh = {
enable = true;
permitRootLogin = "without-password";
};
}

11
shared/root.nix
View File

@ -0,0 +1,11 @@
{
users.mutableUsers = false;
users.users."root" = {
hashedPassword = "$y$j9T$tz8ojZ2gVOQ5AUp6GMhoj.$mAeE0eTGGsKNGddC7ebk/zFr5IMDyIpOpMP/6o.GI6D";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIK2nkarN0+uSuP5sGwDCb9KRu+FCjO/+da4VypGanPUZ fooker@k-2so"
];
};
}
Write Preview
Loading…
Cancel
Save