Build MPICH 3.x with ch3:nemesis

Author: dalcinl

bootstrap.sh

@@ -96,15 +96,11 @@ else
     check Release "$release" || (echo "error: RELEASE!=$release" && exit 1)
 fi
 
-if test "$(uname)" = "Linux"; then
-    case "$mpiname" in
-        mpich)   MODSOURCE="$SOURCE"/modules   ;;
-        openmpi) MODSOURCE="$SOURCE"/3rd-party ;;
-    esac
+boostrap-ofi() {
     ofigithub="https://github.com/ofiwg/libfabric"
     ofiurlbase="$ofigithub/releases/download/v$ofiversion"
     ofitarball="libfabric-$ofiversion.tar.bz2"
-    ofidestdir="$MODSOURCE"/"${ofitarball%%.tar.*}"
+    ofidestdir="$1"/"${ofitarball%%.tar.*}"
     if test ! -d "$ofidestdir"; then
         if test ! -f "$ofitarball"; then
             echo downloading "$ofiurlbase"/"$ofitarball"...
@@ -119,10 +115,13 @@ if test "$(uname)" = "Linux"; then
     else
         echo reusing directory "$ofidestdir"...
     fi
+}
+
+boostrap-ucx() {
     ucxgithub="https://github.com/openucx/ucx"
     ucxurlbase="$ucxgithub/releases/download/v$ucxversion"
     ucxtarball="ucx-$ucxversion.tar.gz"
-    ucxdestdir="$MODSOURCE"/"${ucxtarball%%.tar.*}"
+    ucxdestdir="$1"/"${ucxtarball%%.tar.*}"
     if test ! -d "$ucxdestdir"; then
         if test ! -f "$ucxtarball"; then
             echo downloading "$ucxurlbase"/"$ucxtarball"...
@@ -141,6 +140,17 @@ if test "$(uname)" = "Linux"; then
     else
         echo reusing directory "$ucxdestdir"...
     fi
+}
+
+if test "$(uname)" = "Linux"; then
+    if test "$mpiname" = "mpich" && test "${version%%.*}" -ge 4; then
+        boostrap-ofi "$SOURCE"/modules
+        boostrap-ucx "$SOURCE"/modules
+    fi
+    if test "$mpiname" = "openmpi"; then
+        boostrap-ofi "$SOURCE"/3rd-party
+        boostrap-ucx "$SOURCE"/3rd-party
+    fi
 fi
 
 if test "$mpiname" = "mpich"; then

cibw-build-mpi.sh

@@ -21,7 +21,7 @@ esac
 if test "$(uname)" = Linux; then
     # shellcheck disable=SC1091
     PLATFORM_ID="$(source /etc/os-release && echo "${PLATFORM_ID:-}")"
-    if test "${CIBUILDWHEEL}" = 1; then
+    if test "${CIBUILDWHEEL-}" = 1; then
         yum remove -y libatomic
         if test "$PLATFORM_ID" = "platform:el8"; then
             yum install -y rdma-core-devel
@@ -119,15 +119,22 @@ if test -d "$MODSOURCE"/libfabric-*; then
 fi
 
 if test "$mpiname" = "mpich"; then
-    case $(uname) in
-        Linux)  netmod=ucx,ofi ;;
-        Darwin) netmod=ofi ;;
-    esac
+    if test "${version%%.*}" -lt 4; then
+        case $(uname) in
+            Linux)  device="ch3:nemesis" ;;
+            Darwin) device="ch3:nemesis" ;;
+        esac
+    else
+        case $(uname) in
+            Linux)  device="ch4:ucx,ofi" ;;
+            Darwin) device="ch4:ofi" ;;
+        esac
+    fi
     options=(
         CC=cc
         CXX=c++
         --prefix="$PREFIX"
-        --with-device="ch4:$netmod"
+        --with-device="$device"
         --with-pm=hydra:gforker
         --with-ucx=embedded
         --with-libfabric=embedded
@@ -144,10 +151,10 @@ if test "$mpiname" = "mpich"; then
         --disable-libxml2
         --disable-dependency-tracking
     )
-    if [[ "$netmod" != *ucx* ]]; then
+    if [[ "$device" != *ucx* ]]; then
         options=("${options[@]/--with-ucx=embedded}")
     fi
-    if [[ "$netmod" != *ofi* ]]; then
+    if [[ "$device" != *ofi* ]]; then
         options=("${options[@]/--with-libfabric=embedded}")
     fi
     if test -d "$SOURCE"/modules/ucx-*; then

cibw-check-mpi.sh

@@ -61,7 +61,7 @@ if test "$mpiname" = "mpich"; then
     RUN mpichversion
 fi
 
-if test "$mpiname" = "mpich"; then
+if test "$mpiname" = "mpich" && test "${version%%.*}" -ge 4; then
     RUN command -v mpivars
     RUN mpivars -nodesc | grep 'Category .* has'
 fi
@@ -92,13 +92,12 @@ RUN mpiexec -help
 RUN mpiexec -n 3 ./helloworld-c
 RUN mpiexec -n 3 ./helloworld-cxx
 
-if test "$mpiname-$(uname)" = "mpich-Linux"; then
+if test "$mpiname-$(uname)" = "mpich-Linux" && test "${version%%.*}" -ge 4; then
     export MPICH_CH4_UCX_CAPABILITY_DEBUG=1
     export MPICH_CH4_OFI_CAPABILITY_DEBUG=1
     for netmod in ucx ofi; do
         printf "testing %s ... " "$netmod"
         export MPICH_CH4_NETMOD="$netmod"
-        test "${version%%.*}" -ge 4 || netmod=""
         mpiexec -n 1 ./helloworld-c | grep -i "$netmod" > /dev/null
         for nonlocal in 0 1; do
             export MPICH_NOLOCAL=$nonlocal