Build Charm4Py with UCX and SlurmPMI on SDSC Expanse

[ZwFink]

Experimental setup (SDSC Expanse)

1. Install Miniconda, install packages

    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
    chmod +x ./Miniconda3-latest-Linux-x86_64.sh 
    ./Miniconda3-latest-Linux-x86_64.sh 
    conda create --name charm4py
    conda activate charm4py
    conda install --yes numpy scipy pandas greenlet cython

2. Install UCX

    git clone https://github.com/openucx/ucx.git
    # Version 1.10.1
    git checkout 6a5856ef4f72c8139951e7ed1d0a4cb75a2e82ec
    mkdir ~/.local
    cd ucx/
    module load gcc/10.2.0
    # This was required because UCX wasn't finding the NUMA libraries. 
    # This command should work directly on Expanse (from [[https://github.com/openucx/ucx/issues/4774#issuecomment-586646345][here]])
    export NUMA_HOME=/cm/shared/apps/spack/cpu/opt/spack/linux-centos8-zen/gcc-8.3.1/numactl-2.0.12-uvjxkgifpcwra25lv6tzxa5gof5ayfkq
    CFLAGS="-I$NUMA_HOME/include"
    LDFLAGS="-L$NUMA_HOME/lib -Wl,-rpath,$NUMA_HOME/lib"
    export CFLAGS LDFLAGS
    ./autogen.sh
    mkdir build
    cd build
    ../contrib/configure-release --prefix=$HOME/.local/ucx
    make -j && make install

3. Install Charm++

    cd ~
    mkdir charms
    cd charms
    git clone https://github.com/UIUC-PPL/charm.git
    cd charm
    git checkout charm4py_expanse
    # The paths containing UCX libraries, Slurm, and PMI should be on your LD_LIBRARY_PATH.
    # You will need to add the UCX libraries yourself, but on expanse Slurm and PMI should be there
    # to begin with.
    ./build charm4py ucx-linux-x86_64 slurmpmi --with-production --basedir=$HOME/.local/ucx -j --force

4. Install Charm4Py

 cd ~/charms
 git clone https://github.com/UIUC-PPL/charm4py.git
 cd charm4py
 git checkout expanse
 ln -s $HOME/charms charm_src
 python3 -m pip install --user -e .

[ZwFink]

OpenMPI/mpi4py

OpenMPI

git clone https://github.com/open-mpi/ompi.git
cd ompi/
git checkout a8dd8708d8b6d1346328d7f4612d63b307c25653
git submodule update --init --recursive
./autogen.pl
mkdir build; cd build
../configure --prefix=$HOME/.local/ompi --enable-mpirun-prefix-by-default --with-ucx=$HOME/.local/ucx --without-lsf --without-psm --without-libfabric --without-verbs --without-psm2 --without-alps --without-sge --with-slurm --without-tm --without-loadleveler --disable-debug --disable-memchecker --disable-oshmem --disable-java --disable-mpi-java --disable-man-pages --with-pmi=/cm/shared/apps/slurm/current/
make -j && make install
# These lines should also go into your bashrc
export OPAL_PREFIX=$HOME/.local/ompi/
export OPAL_LIBDIR=$OPAL_PREFIX/lib
export PATH=$OPAL_PREFIX/bin:$PATH
srun --nodes=1 -A TG-ASC050039N --ntasks-per-node=10 --time=15:00 --partition=compute --exclusive --pty bash -i

mpi4py

Diff applied to setup.cfg

diff --git a/mpi.cfg b/mpi.cfg
index a704440..542db7e 100644
--- a/mpi.cfg
+++ b/mpi.cfg
@@ -53,7 +53,7 @@ mpicxx               = %(mpi_dir)s/bin/mpicxx
# Open MPI example
# ----------------
[openmpi]
-mpi_dir              = /home/devel/mpi/openmpi/5.0.0
+mpi_dir              = /home/zanef2/.local/ompi
mpicc                = %(mpi_dir)s/bin/mpicc
mpicxx               = %(mpi_dir)s/bin/mpicxx
#include_dirs         = %(mpi_dir)s/include

Installation

git clone https://github.com/mpi4py/mpi4py.git
cd mpi4py/
python3 -m pip install --user --install-option="--mpi=openmpi" .