From a06cf370cea65d9d8365610241daff16f245134d Mon Sep 17 00:00:00 2001 From: Dustin Frisch Date: Thu, 25 Apr 2024 22:09:36 +0200 Subject: [PATCH] update ucx and openmpi --- packages/openmpi/default.nix | 139 +++++++++++++++++++++++++++++++++++ shared/packages.nix | 13 ++++ 2 files changed, 152 insertions(+) create mode 100644 packages/openmpi/default.nix diff --git a/packages/openmpi/default.nix b/packages/openmpi/default.nix new file mode 100644 index 0000000..945a24a --- /dev/null +++ b/packages/openmpi/default.nix @@ -0,0 +1,139 @@ +{ lib +, stdenv +, fetchurl +, removeReferencesTo +, gfortran +, perl +, libnl +, rdma-core +, zlib +, numactl +, libevent +, hwloc +, targetPackages +, symlinkJoin +, libpsm2 +, libfabric +, pmix +, ucx +, ucc +, makeWrapper +, config + # Enable CUDA support +, cudaSupport ? config.cudaSupport +, cudaPackages + + # Enable the Sun Grid Engine bindings +, enableSGE ? false + + # Pass PATH/LD_LIBRARY_PATH to point to current mpirun by default +, enablePrefix ? false + + # Enable libfabric support (necessary for Omnipath networks) on x86_64 linux +, fabricSupport ? stdenv.isLinux && stdenv.isx86_64 + + # Enable Fortran support +, fortranSupport ? true + +, ripgrep +}: + +stdenv.mkDerivation rec { + pname = "openmpi"; + version = "5.0.3"; + + src = with lib.versions; fetchurl { + url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2"; + hash = "sha256-mQWC8gazqzLpOKoxu/B8Y5No5EBdyhlvq+fw927tqQs="; + }; + + postPatch = '' + patchShebangs ./ + ''; + + preConfigure = '' + # Ensure build is reproducible according to manual + # https://docs.open-mpi.org/en/v5.0.x/release-notes/general.html#general-notes + export USER=nixbld + export HOSTNAME=localhost + export SOURCE_DATE_EPOCH=0 + ''; + + #outputs = [ "out" "man" "dev" ]; + + buildInputs = [ zlib libevent hwloc ] + ++ lib.optionals stdenv.isLinux [ libnl numactl pmix ucx ucc ] + ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ] + ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core + ++ lib.optionals fabricSupport [ libpsm2 libfabric ]; + + nativeBuildInputs = [ perl removeReferencesTo makeWrapper ] + ++ lib.optionals cudaSupport [ cudaPackages.cuda_nvcc ] + ++ lib.optionals fortranSupport [ gfortran ]; + + configureFlags = lib.optional (!cudaSupport) "--disable-mca-dso" + ++ lib.optional (!fortranSupport) "--disable-mpi-fortran" + ++ lib.optionals stdenv.isLinux [ + "--with-libnl=${lib.getDev libnl}" + "--with-pmix=${lib.getDev pmix}" + "--with-pmix-libdir=${pmix}/lib" + ] ++ lib.optional enableSGE "--with-sge" + ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default" + # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build + # https://github.com/openucx/ucx + # https://www.open-mpi.org/faq/?category=buildcuda + ++ lib.optionals cudaSupport [ "--with-cuda=${cudaPackages.cuda_cudart}" "--enable-dlopen" ] + ++ lib.optionals fabricSupport [ + "--with-psm2=${lib.getDev libpsm2}" + "--with-libfabric=${lib.getDev libfabric}" + "--with-libfabric-libdir=${lib.getLib libfabric}/lib" + ]; + + enableParallelBuilding = true; + + postFixup = '' + remove-references-to -t $out $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary}) + remove-references-to -t $out $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary}) + + # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared. + wrapProgram $out/bin/opal_wrapper \ + --set OPAL_INCLUDEDIR $out/include \ + --set OPAL_PKGDATADIR $out/share/openmpi + + # default compilers should be indentical to the + # compilers at build time + + echo "$out/share/openmpi/mpicc-wrapper-data.txt" + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $out/share/openmpi/mpicc-wrapper-data.txt + + echo "$out/share/openmpi/shmemcc-wrapper-data.txt" + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $out/share/openmpi/shmemcc-wrapper-data.txt + '' + lib.optionalString fortranSupport '' + + echo "$out/share/openmpi/mpifort-wrapper-data.txt" + sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ + $out/share/openmpi/mpifort-wrapper-data.txt + + echo "$out/share/openmpi/shmemfort-wrapper-data.txt" + sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ + $out/share/openmpi/shmemfort-wrapper-data.txt + ''; + + doCheck = true; + + passthru = { + inherit cudaSupport; + inherit (cudaPackages) cudatoolkit; # For backward compatibility only + }; + + meta = with lib; { + homepage = "https://www.open-mpi.org/"; + description = "Open source MPI-3 implementation"; + longDescription = "The Open MPI Project is an open source MPI-3 implementation that is developed and maintained by a consortium of academic, research, and industry partners. Open MPI is therefore able to combine the expertise, technologies, and resources from all across the High Performance Computing community in order to build the best MPI library available. Open MPI offers advantages for system and software vendors, application developers and computer science researchers."; + maintainers = with maintainers; [ markuskowa ]; + license = licenses.bsd3; + platforms = platforms.unix; + }; +} diff --git a/shared/packages.nix b/shared/packages.nix index 2bb256b..843038a 100644 --- a/shared/packages.nix +++ b/shared/packages.nix @@ -11,6 +11,19 @@ with lib; otf2 = self.callPackage ../packages/otf2 { }; scalasca = self.callPackage ../packages/scalasca { }; scorep = self.callPackage ../packages/scorep { }; + + openmpi = self.callPackage ../packages/openmpi { }; + + ucx = super.ucx.overrideAttrs (final: prev: rec { + version = "1.16.0"; + + src = self.fetchFromGitHub { + owner = "openucx"; + repo = "ucx"; + rev = "v${version}"; + hash = "sha256-dihWwGlQclfa2ke+1V5c6coqfFjjuMyI8QRzNdx33zQ="; + }; + }); }) ]; }