Requirements=(other.GlueCEUniqueID=="voms-proxy-init -voms gridit");

In this way we are connected to the test virtual organization that we use for testing. The specific requirements that one has to specify to run on one of the participating nodes are list below.

The basic test we first execute is a test the we are able to allocate two whole nodes and we are able to compile and run a very basic MPI program.

Up to now we successfully run this test on the PARMA and NAPOLI nodes. On the PISA nodes we are in queue. On the PERUGIA node we are able to compile but mpi-start fails with the error

....omissis...
+ /usr/lib64/openmpi/1.4-gcc/bin/mpiexec -wdir /home/gridit009/home_crcex_576377136/CREAM576377136 \
    -x X509_USER_PROXY -x OMP_NUM_THREADS --prefix -4_PATH -machinefile /tmp/tmp.qNoqU31285 \
    -npernode 1 ./GETINFO
bash: -4_PATH/bin/orted: No such file or directory
....omissis...

requirements=(other.GlueCEImplementationVersion=="1.13.4") &&
  Member("MPI-START",other.GlueHostApplicationSoftwareRunTimeEnvironment);

Requirements=(other.GlueCEUniqueID=="emi-ce.pr.infn.it:8443/cream-pbs-parallel");

Requirements=(other.GlueCEUniqueID=="gridce4.pi.infn.it:8443/cream-lsf-gridmpi");

Requirements=(other.GlueCEUniqueID=="emi-ce01.scope.unina.it:8443/cream-pbs-hpc");

Requirements=(other.GlueCEUniqueID=="cex.grid.unipg.it:8443/cream-pbs-grid");

MPI_START errr: mpi-start [DEBUG ]: using user supplied mpiexec: '/usr/lib64/openmpi/1.4-gcc/bin/mpiexec' /etc/mpi-start/openmpi.mpi: line 39: opal_info: command not found /etc/mpi-start/openmpi.mpi: line 40: opal_info: command not found /etc/mpi-start/openmpi.mpi: line 41: opal_info: command not found

executed with message

declare -x PBS_NP="2"
declare -x PBS_NUM_NODES="1"
declare -x PBS_NUM_PPN="2"
There are [1] MPI Process on this job !
--Process rank [0] with nthreads=1 Executing on node: nx02.grid.unipg.it

The main JDL to be used are made like this

JobType = "Normal";
Executable    = "TestMPI.bash";
Arguments     = "-vv -d MPI_USE_AFFINITY=1";
StdOutput     = "std.out";
StdError      = "std.err";
InputSandbox  = {"TestMPI.bash"}; 
OutputSandbox = {"std.out", "std.err"};
MyProxyServer = "myproxy.cnaf.infn.it";
WholeNodes = True;
HostNumber = 2;
Requirements=(other.GlueCEUniqueID=="emi-ce.pr.infn.it:8443/cream-pbs-parallel");

The behavior is controlled by the configuration file of mpi-start was rule are defined here

#!/bin/bash
cat > pre_run_hook.sh <<EOF
pre_run_hook () {
    echo "******************************************"
    echo "** pre run hook called "
    echo "******************************************"
    env|grep MPI_
    echo "******************************************"
    echo "** compile Programm "
    echo "******************************************"
    echo "COMMANDS: \$MPI_MPICC \$MPI_MPICC_OPTS -fopenmp mpi_getinfo.c -o GETINFO"
    \$MPI_MPICC \$MPI_MPICC_OPTS -fopenmp mpi_getinfo.c -o GETINFO
    ls -lth 
    echo "******************************************"
    echo "** END of compilation "
    echo "******************************************"
    return 0
}
EOF
cat > mpi_getinfo.c <<EOF
/*********************************/
/* mpi_getinfo.c                 */
/*********************************/
#include <omp.h>
#include <stdio.h>
#include "mpi.h"
int main(int argc, char **argv)
{
   int        rank, np, tot_threads;
   char       name[MPI_MAX_PROCESSOR_NAME];
   int        resultlen;
   int        i;
   int        nthreads;
   MPI_Status status;
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &np);
   MPI_Get_processor_name(name, &resultlen);
   #pragma omp parallel
   nthreads=omp_get_num_threads();

   if(rank == 0)
   {
     printf("There are [%d] MPI Process on this job !\n",np);
      printf("-- MPI process [%2d of %d] with %d nthreads is executing on node: %s\n", rank,np, nthreads,name);
      tot_threads = nthreads;
      for(i = 1; i < np; i++)
      {
         MPI_Recv(&nthreads, 1, MPI_INT, i, 0, MPI_COMM_WORLD, &status);
         MPI_Recv(&resultlen, 1, MPI_INT, i, 0, MPI_COMM_WORLD, &status);
         MPI_Recv(name, resultlen+1, MPI_CHAR, i, 0, MPI_COMM_WORLD, &status);
         printf("-- MPI process [%2d of %d] with %d nthreads is executing on node: %s\n",i,np, nthreads,name);
         tot_threads += nthreads;
      }
      printf("The total number of threads for this run is %d !\n",tot_threads);
   }
   else
   {
      MPI_Send(&nthreads, 1, MPI_INT, 0, 0, MPI_COMM_WORLD);
      MPI_Send(&resultlen, 1, MPI_INT, 0, 0, MPI_COMM_WORLD);
      MPI_Send(name, resultlen+1, MPI_CHAR, 0, 0, MPI_COMM_WORLD);
   }
  MPI_Finalize();
  return 0;
}
EOF

echo "****************************************"
echo "* Print the execution ENVINRONMENT     *"
echo "****************************************"
export
echo "****************************************"
echo " MPI start -- no opt with compiler PREHOOK  "
echo "****************************************"
mpi-start -t openmpi -pre ./pre_run_hook.sh  -- ./GETINFO 
echo "****************************************"
echo " MPI start PSOCKET "
echo "****************************************"
mpi-start -t openmpi -psocket $@ -- ./GETINFO 
echo "****************************************"
echo " MPI start PCORE  "
echo "****************************************"
mpi-start -t openmpi -pcore   $@ -- ./GETINFO 
echo "****************************************"
echo " MPI start PNODE  "
echo "****************************************"
mpi-start -t openmpi -pnode   $@  -- ./GETINFO