mpi4py: run the spawn and dynamic process tests

Split the mpi4py Github Action into 4 parts:

1. build: do everything to build, configure, and install Open MPI and
   mpi4py
2. run: run all the mpi4py tests with its defaults.  As of March 2024,
   this disables the spawn and dynamic tests, which means that the
   entire block of tests should pass.
3. run_spawn: run all the mpi4py tests, including the spawn tests, but
   only if the "mpi4py" label is set on the PR.  As of March 2024, we
   know some of these tests are failing.
4. run_dynamic: run all the mpi4py tests, including the dynamic tests,
   but only if the "mpi4py" label is set on the PR.  As of March 2024,
   we know some of these tests are failing.

The spawn and dynamic failures are different, so we split them up and
run them separately.

Note that for steps 2, 3, and 4, we use a GitHub reusable workflow so
that we don't have to duplicate the code -- we just set a different
environment variable for each, which triggers different behavior.

Signed-off-by: Jeff Squyres <jeff@squyres.com>

.github/workflows/mpi4py-tests.yaml

@@ -0,0 +1,69 @@
+# This is a reusable workflow that is invoked from mpi4py.yaml.
+#
+# It expects a single parameter: env_name.  This string sets an
+# environment variable of that name to value "1".
+
+name: mpi4py-tests
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        required: true
+        type: string
+
+jobs:
+  mpi4py-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      ${{ inputs.env_name}}: 1
+    steps:
+    - name: Use Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3
+        architecture: x64
+    - name: Get artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: /
+        name: build-artifacts
+    - name: Restore executable permissions
+      run: chmod a+x /opt/openmpi/bin/*
+    - name: Add Open MPI to PATH
+      run: echo /opt/openmpi/bin >> $GITHUB_PATH
+    - name: Install the mpi4py wheel
+      run: python -m pip install mpi4py --no-index --find-links=.
+
+    #----------------------------------------------
+
+    - name: Test mpi4py (singleton)
+      run:  python test/main.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=1)
+      run:  mpiexec -n 1 python test/main.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=2)
+      run:  mpiexec -n 2 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=3)
+      run:  mpiexec -n 3 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=4)
+      run:  mpiexec -n 4 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+    - name: Test mpi4py (np=5)
+      run:  mpiexec -n 5 python test/main.py -v -f
+      if:   ${{ true }}
+      timeout-minutes: 10
+
+    - name: Test mpi4py.run
+      run:  python demo/test-run/test_run.py -v
+      if:   ${{ true }}
+      timeout-minutes: 10

.github/workflows/ompi_mpi4py.yaml

@@ -1,14 +1,12 @@
 name: mpi4py
 
-on: [pull_request]
+on: [ pull_request ]
 
 jobs:
-  mpi4py:
+  build:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-
+    timeout-minutes: 30
     steps:
-
     - name: Configure hostname
       run:  echo 127.0.0.1 `hostname` | sudo tee -a /etc/hosts > /dev/null
       if:   ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
@@ -28,6 +26,11 @@ jobs:
       run:  ./autogen.pl
       working-directory: mpi-build
 
+    # Install into a separate directory (/opt/openmpi) so that we can
+    # bundle up that tree into an artifact to share with other jobs in
+    # this github action.  Specifically don't use /usr/local, because
+    # there's a bunch of other stuff already installed in /usr/local,
+    # and we don't need to include that in our artifact.
     - name: Configure Open MPI
       run:  ./configure
               --disable-dependency-tracking
@@ -36,7 +39,8 @@ jobs:
               --disable-sphinx
               --disable-mpi-fortran
               --disable-oshmem
-              LDFLAGS=-Wl,-rpath,/usr/local/lib
+              --prefix=/opt/openmpi
+              LDFLAGS=-Wl,-rpath,/opt/openmpi/lib
       working-directory: mpi-build
 
     - name: Build MPI
@@ -47,6 +51,9 @@ jobs:
       run:  sudo make install
       working-directory: mpi-build
 
+    - name: Add Open MPI to PATH
+      run: echo /opt/openmpi/bin >> $GITHUB_PATH
+
     - name: Tweak MPI
       run:  |
         # Tweak MPI
@@ -84,37 +91,65 @@ jobs:
       with:
         repository: "mpi4py/mpi4py"
 
-    - name: Install mpi4py
-      run:  python -m pip install .
+    - name: Build mpi4py wheel
+      run:  python -m pip wheel .
       env:
         CFLAGS: "-O0"
 
-    - name: Test mpi4py (singleton)
-      run:  python test/main.py -v
-      if:   ${{ true }}
-      timeout-minutes: 10
-    - name: Test mpi4py (np=1)
-      run:  mpiexec -n 1 python test/main.py -v
-      if:   ${{ true }}
-      timeout-minutes: 10
-    - name: Test mpi4py (np=2)
-      run:  mpiexec -n 2 python test/main.py -v -f
-      if:   ${{ true }}
-      timeout-minutes: 10
-    - name: Test mpi4py (np=3)
-      run:  mpiexec -n 3 python test/main.py -v -f
-      if:   ${{ true }}
-      timeout-minutes: 10
-    - name: Test mpi4py (np=4)
-      run:  mpiexec -n 4 python test/main.py -v -f
-      if:   ${{ true }}
-      timeout-minutes: 10
-    - name: Test mpi4py (np=5)
-      run:  mpiexec -n 5 python test/main.py -v -f
-      if:   ${{ true }}
-      timeout-minutes: 10
-
-    - name: Test mpi4py.run
-      run:  python demo/test-run/test_run.py -v
-      if:   ${{ true }}
-      timeout-minutes: 10
+    - name: Save the artifacts for other jobs
+      uses: actions/upload-artifact@v4
+      with:
+        path: |
+          /opt/openmpi
+          ~/.openmpi
+          ~/.prte
+          test
+          demo
+          mpi4py-*.whl
+        retention-days: 2
+        name: build-artifacts
+
+  #==============================================
+
+  run_defaults:
+    # This whole set of tests run with mpi4py's defaults.  As of March
+    # 2024, this means disabling the spawn and dynamic tests.  We want
+    # this job of tests to pass.
+    needs: [ build ]
+    timeout-minutes: 30
+    uses: ./.github/workflows/mpi4py-tests.yaml
+    with:
+      # This parameter is required, so send a meaningless
+      # environment variable name that will not affect the tests at
+      # all (i.e., the tests will be run with default values).
+      env_var: MAKE_TODAY_AN_OMPI_DAY
+
+  #==============================================
+
+  run_spawn:
+    # This whole set of tests runs explicitly with setting "enable the
+    # spawn tests".  As of March 2024, we know that Open MPI is
+    # failing these tests.
+    needs: [ build ]
+    timeout-minutes: 30
+    # Only run if the label "mpi4py" is set on this PR.
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'mpi4py-spawn') }}
+    uses: ./.github/workflows/mpi4py-tests.yaml
+    with:
+      # Enable the spawn tests
+      env_var: MPI4PY_TEST_SPAWN
+
+  #==============================================
+
+  run_dynamic:
+    # This whole set of tests runs explicitly with setting "enable the
+    # spawn tests".  As of March 2024, we know that Open MPI is
+    # failing these tests.
+    needs: [ build ]
+    timeout-minutes: 30
+    # Only run if the label "mpi4py" is set on this PR.
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'mpi4py-spawn') }}
+    uses: ./.github/workflows/mpi4py-tests.yaml
+    with:
+      # Enable the dynamic process tests
+      env_var: MPI4PY_TEST_DYNPROC