#!/bin/bash
#
# Copyright (c) 2006-2007 High Performance Computing Center Stuttgart,
#                         University of Stuttgart.  All rights reserved.
# Copyright (c) 2009-2010 Instituto de Fisica de Cantabria, 
#                         CSIC-UC. All rights reserved.
#
#
# Return values of this script:
# 0   - Success

if test "x$I2G_MPI_START_FULL_TRACE" = "x1"  ; then
    set -x
fi

# initialize internal variables
MPI_START_MACHINEFILE=""
MPI_START_HOSTFILE=""
MPI_START_HOST_SLOTS_FILE=""
MPI_START_NHOSTS=0
MPI_START_NSLOTS=0
MPI_START_NSLOTS_PER_HOST=0
MPI_START_READY=-1 
MPI_START_MPI_PREFIX=""
MPI_START_MPI_MODULE=""
MPI_START_CLEANUP_FILES=""
MPI_START_SCHEDULER=""
MPI_START_NP=1
MPI_START_NPHOST=''
unset MPI_START_TEMP_DIR 
# mpiexec params
MPI_GLOBAL_PARAMS=""
MPI_LOCAL_PARAMS=""
declare -a MPI_START_JOB_ENV
MPI_START_DUMMY_SCHEDULER=${MPI_START_DUMMY_SCHEDULER:-1}
I2G_MPI_START_KEEP_FILES=${I2G_MPI_START_KEEP_FILES:-0}
MPI_START_DO_NOT_USE_WRAPPER=${MPI_START_DO_NOT_USE_WRAPPER:-1}

#======================================================================
# Cleans up temporary files upon finishing execution 
#======================================================================
clean_up() {
    if test "x$I2G_MPI_START_KEEP_FILES" = "x1"; then
        exit
    fi
    debug_msg "Cleaning up mpi-start temporary files."
    for file in $MPI_START_CLEANUP_FILES; do
        [ -f "$file" ] && rm -f $file
        [ -d "$file" ] && rm -rf $file
    done
    exit
}

trap clean_up EXIT

#======================================================================
# Displays a warning message
#======================================================================
warn_msg() {
    #if test "x$I2G_MPI_START_VERBOSE" = "x1" ; then 
    echo "mpi-start [WARNING]:" "$@" 1>&2
    #fi
}

#======================================================================
# Displays a warning message
#======================================================================
error_msg() {
    echo "mpi-start [ERROR  ]:" "$@" 1>&2
}

#======================================================================
# Display a debug message in the case that I2G_MPI_START_DEBUG
# is 1.
#======================================================================
debug_msg() {
    if test "x$I2G_MPI_START_VERBOSE" = "x1" ; then 
        if test "x$I2G_MPI_START_DEBUG" = "x1"  ; then
            echo "mpi-start [DEBUG  ]: $@" 1>&2
        fi
    fi
}

#======================================================================
# Display a debug message in the case that I2G_MPI_START_DEBUG
# is 1.
#======================================================================
info_msg() {
    if test "x$I2G_MPI_START_VERBOSE" = "x1" ; then 
        echo "mpi-start [INFO   ]: $@" 1>&2
    fi
}

#======================================================================
# Dump environment.
#======================================================================
dump_env() {
    if test "x$I2G_MPI_START_TRACE" = "x1"  ; then
        set +x
    fi
    if test "x$I2G_MPI_START_VERBOSE" = "x1" ; then 
        for i in `env`; do
            echo "mpi-start [DUMPENV]:" $i 1>&2
        done
    fi
}

#======================================================================
# Find the correct parameter for the mktemp program.
#======================================================================
mpi_start_find_mktemp() {
    MPI_START_MKTEMP=mktemp
    local tempfile=`$MPI_START_MKTEMP 2> /dev/null`
    if test $? -ne 0 -o -z "$tempfile"  ; then
        # BSD style mktemp
        MPI_START_MKTEMP="mktemp -t MPI_START"
        tempfile=`$MPI_START_MKTEMP 2> /dev/null`
        if test $? -ne 0 -o -z "$tempfile" ; then
            error_msg "Could not find a proper mktemp utility"
            dump_env
            exit 1
        fi
    fi
    rm -f $tempfile
}

#======================================================================
# mpi-start mktemp 
# Creates a new temporary file / directory
# Name of file is returned in MPI_START_TEMP_FILE
#======================================================================
mpi_start_mktemp() {
    if test "x$MPI_START_TEMP_DIR" = "x" ; then
        MPI_START_TEMP_FILE=`$MPI_START_MKTEMP $*`
    else
        MPI_START_TEMP_FILE=`TMPDIR=$MPI_START_TEMP_DIR $MPI_START_MKTEMP $*`
    fi
    st=$?
    if test $st -ne 0 ; then
        error_msg "Unable to create temp file!"
    fi
    MPI_START_CLEANUP_FILES="$MPI_START_TEMP_FILE $MPI_START_CLEANUP_FILES"
    return $st
}

#======================================================================
# Look for the appropriate mpirun/mpiexec according to the loaded
# MPI type
#======================================================================
mpi_start_search_mpiexec() {
    export MPI_MPIEXEC_DEFINED=0
    export MPI_MPIRUN_DEFINED=0
    export MPI_SPECIFIC_PARAMS=""
    export MPI_SPECIFIC_MPIEXEC_PARAMS=""
    export MPI_SPECIFIC_MPIRUN_PARAMS=""
    export MPI_MPIEXEC=""
    export MPI_MPIRUN=""

    if test "x${I2G_MPI_TYPE}" = "x" ; then
        debug_msg "no mpi type selected, no mpiexec/mpirun search!"
        return 0
    fi

    MPI_TYPE=`echo $I2G_MPI_TYPE | tr "[:lower:]" "[:upper:]" | tr "-" "_"`

    VALUE=`eval echo \\$MPI_${MPI_TYPE}_MPIEXEC`
    if test ! -z "$VALUE" ; then
        MPI_MPIEXEC_DEFINED=1
        MPI_MPIEXEC=$VALUE
        debug_msg "using user supplied mpiexec: '$MPI_MPIEXEC'"
    else
        VALUE=`eval echo \\$MPI_${MPI_TYPE}_MPIRUN`
        if test ! -z "$VALUE" ; then
            MPI_MPIRUN_DEFINED=1
            MPI_MPIRUN=$VALUE
            debug_msg "using user supplied mpirun: '$MPI_MPIRUN'"
        else
            # define both 
            MPI_MPIEXEC=`which mpiexec 2> /dev/null`
            if test $? -eq 0 ; then
                debug_msg "found system default mpiexec: '$MPI_MPIEXEC'"
            fi
            MPI_MPIRUN=`which mpirun 2> /dev/null`
            if test $? -eq 0 ; then
                debug_msg "found system default mpirun: '$MPI_MPIRUN'"
            fi
        fi
    fi
    if test "x$MPI_MPIEXEC" != "x"; then
        MPI_SPECIFIC_MPIEXEC_PARAMS=`eval echo \\$MPI_${MPI_TYPE}_MPIEXEC_PARAMS`
    elif test "x$MPI_MPIRUN" != "x"; then
        MPI_SPECIFIC_MPIRUN_PARAMS=`eval echo \\$MPI_${MPI_TYPE}_MPIRUN_PARAMS`
    else
        debug_msg "no mpiexec/mpirun found!"
    fi
}

#======================================================================
# Activate a MPI by the given information. If there are module
# informations available then the modules system will be used
# otherwise the PATH and LD_LIBRARY_PATH is updated manually.
#
# $1 the MPI prefix path (mandatory)
# $2 the MPI module string (optional)
#======================================================================
mpi_start_activate_mpi() {
    if test "x$2" = "x" ; then
        debug_msg "activate MPI via manually update"
        if test -d "$1" ; then
            export PATH=$1/bin:$PATH
            export LD_LIBRARY_PATH=$1/lib:$LD_LIBRARY_PATH
        fi
    else
        shift
        debug_msg "activate MPI via modules : $*"
        for mod in $* ; do
            debug_msg  "+  module load  $mod"
            module load $mod
        done
    fi
    mpi_start_search_mpiexec
}

#======================================================================
# Create the wrapper that will contain the mpirun call
# It is executed as a child of mpistart in order to allow different
# env variables
#======================================================================
mpi_start_create_wrapper() {
    # Issue #63, avoid /tmp
    BASEDIR=$HOME/.mpi_start_tmp
    mkdir -p $BASEDIR
    if test $? -ne 0 ; then
        warn_msg "Unable to create temp dir for wrapper at $BASEDIR, using default TMPDIR"
    else
        MPI_START_TEMP_DIR=`TMPDIR=$BASEDIR $MPI_START_MKTEMP -d`
        if test $? -ne 0 ; then
            warn_msg "Unable to create temp dir for wrapper at $BASEDIR, using default TMPDIR"
        fi
        MPI_START_CLEANUP_FILES="$MPI_START_TEMP_DIR $MPI_START_CLEANUP_FILES"
    fi
    mpi_start_mktemp
    export MPI_START_MPI_WRAPPER=$MPI_START_TEMP_FILE
    unset MPI_START_TEMP_DIR
    cat > $MPI_START_MPI_WRAPPER << EOF
#!/bin/sh
test "x\$I2G_MPI_START_TRACE" = "x1" && set -x || true
EOF
    # check that the wrapper is executable
    chmod +x $MPI_START_MPI_WRAPPER 2> /dev/null
    $MPI_START_MPI_WRAPPER 2> /dev/null
    if test $? -ne 0 ; then
        warn_msg "Unable to execute wrapper, falling back to no wrapper" 
        return 1
    fi
    local vars=${#MPI_START_JOB_ENV[*]}
    local index=0

    while [ $index -lt $vars ]; do
        echo "export ${MPI_START_JOB_ENV[$index]}" >> $MPI_START_MPI_WRAPPER
        index=$(($index+1))
    done
    chmod +x $MPI_START_MPI_WRAPPER
    return 0
}

#======================================================================
# Export a variable for the MPI job. It will be defined only in the
# context of the job (executed in a different process)
#
# $1 the name of the variable
# rest of parameters: variable value 
#======================================================================
mpi_start_export_variable() {
    local var_name
    if test $1 = *=* ; then
        var_name=`echo $1 | cut -f1 -d"="`
        eval $1
    else
        var_name=$1
    fi
    export MPI_START_ENV_VARIABLES="$MPI_START_ENV_VARIABLES $var_name"
    local var_value=$1
    shift
    if test "x$*" != "x"; then
        MPI_START_JOB_ENV[${#MPI_START_JOB_ENV[*]}]="${var_value}="'"'${*}'"'
        eval $var_value="${*}"
    else
        MPI_START_JOB_ENV[${#MPI_START_JOB_ENV[*]}]="$var_value"
    fi
    export $var_name
}

#======================================================================
# Execute the MPI command line in the wrapper
#
# $* command line to execute 
#======================================================================
mpi_start_execute_no_wrapper() {
    if test "x${I2G_MPI_APPLICATION_STDIN}" = "x" ; then
        # no input
        if test "x${I2G_MPI_APPLICATION_STDOUT}" = "x" ; then
            # no output
            if test "x${I2G_MPI_APPLICATION_STDERR}" = "x" ; then
                # no error
                eval $@
            else
                # only error
                eval $@ 2> ${I2G_MPI_APPLICATION_STDERR}
            fi
        else
            if test "x${I2G_MPI_APPLICATION_STDERR}" = "x" ; then
                # only output
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT}
            elif test ${I2G_MPI_APPLICATION_STDOUT} = ${I2G_MPI_APPLICATION_STDERR} ; then
                # same output and error
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT} 2>&1
            else
                # different output and error
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT} 2> ${I2G_MPI_APPLICATION_STDERR} 
            fi
        fi
    else
        if test "x${I2G_MPI_APPLICATION_STDOUT}" = "x" ; then
            # no output
            if test "x${I2G_MPI_APPLICATION_STDERR}" = "x" ; then
                # no error
                eval $@ < $I2G_MPI_APPLICATION_STDIN
            else
                # only error
                eval $@ 2> ${I2G_MPI_APPLICATION_STDERR} < $I2G_MPI_APPLICATION_STDIN
            fi
        else
            if test "x${I2G_MPI_APPLICATION_STDERR}" = "x" ; then
                # only output
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT} < $I2G_MPI_APPLICATION_STDIN
            elif test ${I2G_MPI_APPLICATION_STDOUT} = ${I2G_MPI_APPLICATION_STDERR} ; then
                # same output and error
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT} 2>&1 < $I2G_MPI_APPLICATION_STDIN
            else
                # different output and error
                eval $@ > ${I2G_MPI_APPLICATION_STDOUT} 2> ${I2G_MPI_APPLICATION_STDERR} < $I2G_MPI_APPLICATION_STDIN
            fi
        fi
    fi
    return $?
}

mpi_start_execute_wrapper() {
    if test "x${MPI_START_DO_NOT_USE_WRAPPER}" = "x1" ; then
        mpi_start_execute_no_wrapper $@
    else
        mpi_start_create_wrapper
        if test $? -ne 0 ; then
            mpi_start_execute_no_wrapper $@
            return $?
        fi
        local output=""
        local error=""
        local input=""
        if test "x${I2G_MPI_APPLICATION_STDOUT}" != "x" ; then
            output="> ${I2G_MPI_APPLICATION_STDOUT}"
        fi
        if test "x${I2G_MPI_APPLICATION_STDERR}" != "x" ; then
            if test "x${I2G_MPI_APPLICATION_STDOUT}" = "x${I2G_MPI_APPLICATION_STDERR}" ; then
                error="2>&1"
            else
                error="2> ${I2G_MPI_APPLICATION_STDERR}"
            fi
        fi
        if test "x${I2G_MPI_APPLICATION_STDIN}" != "x" ; then
            input="< ${I2G_MPI_APPLICATION_STDIN}"
        fi 
        echo "$@ $output $error $input" >> $MPI_START_MPI_WRAPPER
        echo "exit \$?" >> $MPI_START_MPI_WRAPPER
        $MPI_START_MPI_WRAPPER
        err=$?
        return $err
    fi
}

#======================================================================
# Create dummy scheduler environment with just the current host 
#======================================================================
mpi_start_dummy_scheduler() {
    info_msg "no scheduler found, using dummy environment with localhost"
    mpi_start_mktemp
    export MPI_START_MACHINEFILE=$MPI_START_TEMP_FILE
    mpi_start_mktemp
    export MPI_START_HOSTFILE=$MPI_START_TEMP_FILE
    mpi_start_mktemp
    export MPI_START_HOST_SLOTS_FILE=$MPI_START_TEMP_FILE
    local myhost=`hostname`
    local slots=1
    if test "x${MPI_DUMMY_SCH_SLOTS}" != "x" ; then
        slots=$MPI_DUMMY_SCH_SLOTS
    fi
    local n=$slots
    while [ $n -ne 0 ]; do
        echo $myhost >> $MPI_START_MACHINEFILE
        n=`expr $n - 1`
    done
    echo $myhost > $MPI_START_HOSTFILE
    echo "$myhost $slots"> $MPI_START_HOST_SLOTS_FILE
    export MPI_START_NSLOTS=$slots
    export MPI_START_NHOSTS=1
    export MPI_START_NSLOTS_PER_HOST=$slots
    SCHEDULER_NAME="mpi-start-dummy"
    export MPI_START_SCHEDULER=$SCHEDULER_NAME
}

#======================================================================
# Gets a file or list of files for plugins matching a certain pattern
# $1 -> name of file (or pattern)
# $2 -> if 1 then do not try to return for ALL plugin dirs
#======================================================================
mpi_start_get_plugin() {
    PATTERN=$1
    shift
    if test "x$1" = "x1" ; then
        BREAK_SEARCH=1
    else
        BREAK_SEARCH=0
    fi
    MPI_START_PLUGIN_FILES=""
    for dir in $MPI_START_ETC_LIST ; do
        local files=`ls $dir/$PATTERN 2> /dev/null`
        if test "x${files}" != "x" ; then
            MPI_START_PLUGIN_FILES="$MPI_START_PLUGIN_FILES $files"
            if test $BREAK_SEARCH -eq 1 ; then
                set -- foo $MPI_START_PLUGIN_FILES
                shift
                # make sure only one file goes back
                MPI_START_PLUGIN_FILES=$1
                break
            fi
        fi
    done
}


#======================================================================
# Checks that mpi-start are correct and coherent
#======================================================================
mpi_start_check_options() {
    # debug me 
    if test "x$I2G_MPI_START_DEBUG" = "x1" ; then 
        debug_msg "dump configuration"
        for var in I2G_MPI_APPLICATION I2G_MPI_APPLICATION_ARGS I2G_MPI_TYPE \
                   I2G_MPI_VERSION I2G_MPI_PRE_RUN_HOOK I2G_MPI_POST_RUN_HOOK \
                   I2G_MPI_PRECOMMAND I2G_MPI_FLAVOUR I2G_MPI_JOB_NUMBER \
                   I2G_MPI_STARTUP_INFO I2G_MPI_RELAY \
                   I2G_MPI_PER_NODE I2G_MPI_PER_CORE I2G_MPI_PER_SOCKET \
                   I2G_MPI_SINGLE_PROCESS I2G_MPI_SINGLE_CORE I2G_MPI_SINGLE_SOCKET \
                   I2G_MPI_APPLICATION_STDIN I2G_MPI_APPLICATION_STDOUT I2G_MPI_APPLICATION_STDERR \
                   I2G_MPI_NP I2G_MPI_CONFIG; 
        do
            eval value="\${$var}"
            debug_msg "=> ${var}=${value}"
        done
    fi

    # trace me
    if test "x$I2G_MPI_START_TRACE" = "x1"  ; then
        debug_msg "enable full trace debugging"
        set -x
    fi

    MPI_START_UNAME=`uname -s | tr "[:upper:]" "[:lower:]"`

    if test -r "${I2G_MPI_CONFIG}" ; then 
        . ${I2G_MPI_CONFIG}
        if test $? -ne 0 ; then
            error_msg "Failed to load configuration file $I2G_MPI_CONFIG"
            dump_env
            exit 2
        fi
    fi
    
    if test "x${I2G_MPI_SINGLE_PROCESS}" = "x1" ; then
        if test "x${I2G_MPI_PER_NODE}" != "x" -a "x${I2G_MPI_PER_NODE}" != "x1" ; then
            warn_msg "Process per node option ($I2G_MPI_PER_NODE) overriding single process option!"
        else
            export I2G_MPI_PER_NODE=1
        fi
    fi

    if test "x${I2G_MPI_SINGLE_CORE}" = "x1" ; then
        if test "x${I2G_MPI_PER_CORE}" != "x" -a "x${I2G_MPI_PER_CORE}" != "x1" ; then
            warn_msg "Process per core option ($I2G_MPI_PER_CORE) overriding single process option!"
        else
            export I2G_MPI_PER_CORE=1
        fi
    fi

    if test "x${I2G_MPI_SINGLE_SOCKET}" = "x1" ; then
        if test "x${I2G_MPI_PER_SOCKET}" != "x" -a "x${I2G_MPI_PER_SOCKET}" != "x1" ; then
            warn_msg "Process per socket option ($I2G_MPI_PER_SOCKET) overriding single process option!"
        else
            export I2G_MPI_PER_SOCKET=1
        fi
    fi


    if test "x${I2G_MPI_PER_NODE}" != "x" ; then
        if test "x${I2G_MPI_PER_SOCKET}${I2G_MPI_PER_CORE}" != "x" ; then
            warn_msg "Node option used together with core or socket! only node specification will be considered"
            unset I2G_MPI_PER_CORE
            unset I2G_MPI_PER_SOCKET
        fi
    elif test "x${I2G_MPI_PER_CORE}" != "x" -a  "x${I2G_MPI_PER_SOCKET}" != "x" ; then
        warn_msg "Core option used together with socket! only socket specification will be considered"
        unset I2G_MPI_PER_CORE
    fi

    if test "x${I2G_MPI_PER_NODE}${I2G_MPI_PER_SOCKET}${I2G_MPI_PER_CORE}" != "x" \
            -a "x${I2G_MPI_NP}" != "x" ; then
        warn_msg "Total number of processes (np) option overriden by node/socket/core option!"
        unset I2G_MPI_NP
    fi

    # check for correct mktemp
    mpi_start_find_mktemp

    # set global internal variables 
    MPI_START_PREFIX=`dirname "$I2G_MPI_START"`

    CONFIG_DIRS="~/.mpi-start"
    
    if test -d $MPI_START_PREFIX/../etc/mpi-start ; then
        CONFIG_DIRS="$CONFIG_DIRS $MPI_START_PREFIX/../etc/mpi-start"
    else
        CONFIG_DIRS="$CONFIG_DIRS /etc/mpi-start"
    fi

    export MPI_START_ETC_LIST="$MPI_START_ETC"
    for dir in $CONFIG_DIRS; do
        if test -d $dir ; then
            MPI_START_ETC_LIST="$MPI_START_ETC_LIST $dir"
        fi
    done
    if test "x${MPI_START_ETC_LIST}" = "x" ; then
        error_msg "unable to find mpi-start modules"
        dump_env
    fi

    # source the local mpi-start variables if any
    mpi_start_get_plugin "mpi-config.local" 1
    if test "x${MPI_START_PLUGIN_FILES}" != "x" ; then
        . ${MPI_START_PLUGIN_FILES}
    fi
}


#======================================================================
# Scheduler detector function 
#======================================================================
mpi_start_scheduler_detector() {
    # check for scheduling system and set environment variables
    info_msg "search for scheduler"
    mpi_start_get_plugin "*.scheduler"
    for i in $MPI_START_PLUGIN_FILES ; do
        # source the function definitions
        unset scheduler
        unset scheduler_available
        unset scheduler_get_machinefile
        debug_msg "source $i"
        . $i
        if test $? -ne 0  ; then 
            error_msg "failed to source : $i"
            continue
        fi

        local scheduler=`basename $i .scheduler`

        # check if support for this kind of schedulers is supported
        debug_msg "checking for scheduler support : $scheduler"
        scheduler_available
        local result=$?

        if test "x$result" = "x0" ; then 
            info_msg "activate support for $scheduler"

            # support for this scheduler is found.
            # So lets setup the internal environment.
            scheduler_get_machinefile
            result=$?

            if test $result -ne 0 ; then
                error_msg "cannot create machine file"
                dump_env
                exit 2
            fi

            # mark MPI_START as ready to go 
            MPI_START_READY=0
            export MPI_START_SCHEDULER=$SCHEDULER_NAME
            break
        fi
    done

    # check if we have a scheduler 
    if test $MPI_START_READY -ne 0  ; then 
        if test "x${MPI_START_DUMMY_SCHEDULER}" = "x1" ; then
            mpi_start_dummy_scheduler
        else
            error_msg "cannot find scheduler"
            dump_env
            exit 3 
        fi
    fi
}

#======================================================================
# Detects number of sockets/cores in linux
#======================================================================
mpi_start_linux_core_detector() {
    MPI_START_SOCKETS=`cat /proc/cpuinfo | grep "physical id" | sort -u | wc -l`
    if test $? -ne 0 ; then
        info_msg "Unable to detect number of cpus, assuming 1"
        MPI_START_SOCKETS=1
    fi
    MPI_START_COREPERSOCKET=`cat /proc/cpuinfo | grep "cpu cores" | sort -u | cut -f2 -d":" | tr -d " "`
    if test $? -ne 0 -o "x$MPI_START_COREPERSOCKET" = "x" ; then
        info_msg "Unable to detect number of cores per cpu, assuming 1"
        MPI_START_COREPERSOCKET=1
    fi
}

#======================================================================
# Detects number of sockets/cores in darwin (MacOS) 
#======================================================================
mpi_start_darwin_core_detector() {
    MPI_START_COREPERSOCKET=`sysctl -n machdep.cpu.cores_per_package`
    if echo $MPI_START_COREPERSOCKET | grep "^machdep:" > /dev/null ; then
        info_msg "Unable to detect number of cores per cpu, assuming 1"
        MPI_START_COREPERSOCKET=1
    fi
    local totalcpus=`sysctl -n hw.ncpu`
    if test $? -ne 0 ; then
        info_msg "Unable to detect total number of cpus, assuming 1"
        MPI_START_SOCKETS=1
    else
        MPI_START_SOCKETS=`expr $totalcpus / $MPI_START_COREPERSOCKET`
    fi  
}

#======================================================================
# Detects number of sockets/cores 
#======================================================================
mpi_start_core_detector() {
    debug_msg "Detection of core/cpu topology."
    if test "x${MPI_START_SOCKETS}" != "x" -a "x${MPI_START_COREPERSOCKET}" != "x" ; then
        info_msg "User defined values for number of sockets ($MPI_START_SOCKETS) and cores per socket ($MPI_START_COREPERSOCKET)"
        return 0
    fi

    MPI_START_SOCKETS=1
    MPI_START_COREPERSOCKET=1
    if test "x${MPI_START_UNAME}" = "xlinux" ; then
        mpi_start_linux_core_detector
    elif test "x${MPI_START_UNAME}" = "xdarwin" ; then
        mpi_start_darwin_core_detector
    else
        info_msg "Physical layout of CPU not implemented for your OS ($MPI_START_UNAME)"
    fi
    info_msg "Detected $MPI_START_SOCKETS CPU socket(s) and $MPI_START_COREPERSOCKET core(s) per CPU"
    # fix wrong detections
    if test "x${MPI_START_SOCKETS}" = "x" -o $MPI_START_SOCKETS -eq 0 ; then
        MPI_START_SOCKETS=1
    fi
    if test "x${MPI_START_COREPERSOCKET}" = "x" -o $MPI_START_COREPERSOCKET -eq 0 ; then
        MPI_START_COREPERSOCKET=1
    fi
    return 0
}

#======================================================================
# Sets up the number of processes to start 
#======================================================================
mpi_start_np_setup() {
    debug_msg "dump hosts:"
    for i in `cat "$MPI_START_HOSTFILE" 2> /dev/null` ; do
        debug_msg "=> $i"
    done

    # setup the np count 
    if test "x${I2G_MPI_PER_NODE}" != "x" ; then
        export MPI_START_NPHOST=$I2G_MPI_PER_NODE
        export MPI_START_NP=`expr $MPI_START_NHOSTS '*' $I2G_MPI_PER_NODE`
    elif test "x${I2G_MPI_PER_SOCKET}${I2G_MPI_PER_CORE}" != "x" ; then
        info_msg "Assuming all hosts have same CPU cores configuration and whole hosts are allocated for the job"
        if test "x${I2G_MPI_PER_SOCKET}" != "x" ; then
            export MPI_START_NPHOST=`expr $MPI_START_SOCKETS '*' $I2G_MPI_PER_SOCKET`
            export MPI_START_NP=`expr $MPI_START_NHOSTS '*' $MPI_START_SOCKETS '*' $I2G_MPI_PER_SOCKET`
        else
            export MPI_START_NPHOST=`expr $MPI_START_SOCKETS '*' $MPI_START_COREPERSOCKET \
                                         '*' $I2G_MPI_PER_CORE`
            export MPI_START_NP=`expr $MPI_START_NHOSTS '*' $MPI_START_SOCKETS \
                                 '*' $MPI_START_COREPERSOCKET '*' $I2G_MPI_PER_CORE`
        fi
    elif test "x${I2G_MPI_NP}" != "x" ; then
        export MPI_START_NP=$I2G_MPI_NP
    else
        export MPI_START_NP=$MPI_START_NSLOTS
    fi

    debug_msg "starting with $MPI_START_NP processes."
}

#======================================================================
# Load execution environment  
#======================================================================
mpi_start_load_execenv() {
    debug_msg "Loading execution environment"

    # check if we should should use default MPI falvour
    if test "x$I2G_MPI_TYPE" = "x"  ; then 
        debug_msg " check for site default MPI flavour"
        if test "x$MPI_DEFAULT_FLAVOUR" != "x"  ; then
            debug_msg "   using default MPI flavour : $MPI_DEFAULT_FLAVOUR"
            export I2G_MPI_TYPE=$MPI_DEFAULT_FLAVOUR
        else
            debug_msg " no MPI flavour specified, using generic."
            export I2G_MPI_TYPE="generic"
        fi
    else
        debug_msg " using user requested MPI flavour"
    fi 

    # load the mpi plugin
    mpi_start_get_plugin "$I2G_MPI_TYPE.mpi" 1
    MPI_PLUGIN_FILE=$MPI_START_PLUGIN_FILES
    if test ! -e "$MPI_PLUGIN_FILE"  ; then 
        error_msg "failed to find requested MPI type : $I2G_MPI_TYPE"
        dump_env 
        exit 2
    fi

    MPI_TYPE=`echo $I2G_MPI_TYPE | tr "[:lower:]" "[:upper:]" | tr "-" "_"`
    local US=_
    local MPI_PREFIX="MPI_"
    local MPI_PATH_SUFFIX="_PATH"
    local MPI_MODULE_SUFFIX="_MODULES"
    local MPI_VERSION_SUFFIX="_VERSION"

    MPI_VERSION=
    if test "x$I2G_MPI_VERSION" != "x" ; then
        debug_msg " user requested MPI version : $I2G_MPI_VERSION"
        MPI_VERSION=$I2G_MPI_VERSION
    else
        MPI_VERSION=`eval echo \\$${MPI_PREFIX}${MPI_TYPE}${MPI_VERSION_SUFFIX}`
    fi

    if test "x{$MPI_VERSION}" != "x" ; then
        debug_msg " will check for specific MPI version : $MPI_VERSION"
        MPI_VERSION=`echo $MPI_VERSION | sed -e s/\\\\./__/g | sed -e s/-/_/g` 
    fi

    # FIXME: why there is I2G_<flavour>_PREFIX and MPI_<flavour>_PATH ?
    #        will keep both to keep compatibility, but one of them should
    #        be removed
    #        I2G_<flavour>_PREFIX takes precedence over the others
    #        then MPI_<flavour>_<version>_PATH and last
    #        MPI_<flavour>_PATH
    I2G_PREFIX=`eval echo \\$I2G_${MPI_TYPE}_PREFIX`
    if test "x$I2G_PREFIX" = "x" ; then
        # first check the one with version included
        VALUE=`eval echo \\$${MPI_PREFIX}${MPI_TYPE}${US}${MPI_VERSION}${MPI_PATH_SUFFIX}`
        if test "x$VALUE" != "x" ; then
            debug_msg " found MPI path for version $MPI_VERSION"
            MPI_START_MPI_PREFIX=$VALUE
        else
            # try default without version
            VALUE=`eval echo \\$${MPI_PREFIX}${MPI_TYPE}${MPI_PATH_SUFFIX}`
            if test "x$VALUE" != "x"  ; then 
                debug_msg " found default MPI in: $VALUE"
                MPI_START_MPI_PREFIX=$VALUE
            else
                #debug_msg " coulnd't find EGEE environment"
                MPI_START_MPI_PREFIX=
            fi
        fi
    else
        debug_msg "use user provided prefix : $I2G_PREFIX"
        MPI_START_MPI_PREFIX=$I2G_PREFIX
    fi
    export MPI_START_MPI_PREFIX

    # same thing for modules
    VALUE=`eval echo \\$${MPI_PREFIX}${MPI_TYPE}${US}${MPI_VERSION}${MPI_MODULE_SUFFIX}`
    if test "x$VALUE" != "x" ; then
        debug_msg " found MPI modules for version $MPI_VERSION"
        MPI_START_MPI_MODULE="$VALUE"
    else
        # try default without version
        VALUE=`eval echo \\$${MPI_PREFIX}${MPI_TYPE}${MPI_MODULE_SUFFIX}`
        if test "x$VALUE" != "x"  ; then 
            debug_msg " found default MPI in: $VALUE"
            MPI_START_MPI_MODULE="$VALUE"
        else
            MPI_START_MPI_MODULE=
        fi
    fi
    export MPI_START_MPI_MODULE

    
    # source the MPI specific configuration file
    info_msg "activate support for $I2G_MPI_TYPE"
    debug_msg "source : $MPI_PLUGIN_FILE"
    . $MPI_PLUGIN_FILE
    if test $? -ne 0 ; then
        error_msg "Error loading the MPI plugin: $MPI_PLUGIN_FILE"
        dump_env
        exit 1
    fi
}

#======================================================================
# The main function that binds everything together
#======================================================================
main() {
    #
    # Output general information
    #
    info_msg "***********************************************************" 
    info_msg " UID     = " `whoami` 
    info_msg " HOST    = " `hostname` 
    info_msg " DATE    = " `date`
    info_msg " VERSION =  1.5.0"  
    info_msg "***********************************************************" 

    debug_msg "Command line arguments: $*" 

    mpi_start_check_options
    
    mpi_start_scheduler_detector

    # core/cpu detection
    mpi_start_core_detector

    mpi_start_np_setup

    mpi_start_load_execenv

    # export X509_USER_PROXY if it's there (grid related)
    if test "x$X509_USER_PROXY" != "x" ; then
        mpi_start_export_variable X509_USER_PROXY
    fi

    # call the MPI specific startup functions
    info_msg "call backend MPI implementation"
    mpi_start
    local result=$?

    exit $result
}


#======================================================================
#  Print command line usage.
#======================================================================
print_usage() {
    echo "mpi-start [-h] [-t mpi_type] [-v] [-vv] [-vvv]" 1>&2
    echo "    [-pre hook] [-post hook] [-pcmd cmd]" 1>&2
    echo "    [-np n]" 1>&2
    echo "    [-pnode|-npnode n]" 1>&2
    echo "    [-psocket|-npsocket n]" 1>&2
    echo "    [-pcore|-npcore n]" 1>&2
    echo "    [-i file] [-o file] [-e file]" 1>&2
    echo "    [-x VAR[=VALUE]] [-d VAR=VALUE] [--]" 1>&2
    echo "    application [...]" 1>&2
    echo  1>&2
    echo "Parallel job starter" 1>&2
    echo  1>&2
    echo "optional arguments:" 1>&2
    echo "  -h             show this help message and exit" 1>&2
    echo "  -V             show mpi-start version" 1>&2
    echo "  -t type        use the mpi type (sets I2G_MPI_TYPE)" 1>&2
    echo "  -v             verbose" 1>&2
    echo "  -vv            debug" 1>&2
    echo "  -vvv           full trace" 1>&2
    echo "  -pre hook      use specified pre run hook script" 1>&2
    echo "  -post hook     use specified post run hook script" 1>&2
    echo "  -pcmd cmd      use specified pre command" 1>&2
    echo "  -npnode n      set number of processes per node" 1>&2
    echo "  -pnode         start only one process per node (equivalent to -npnode 1)" 1>&2
    echo "  -npcore n      set number of processes per core" 1>&2
    echo "  -pcore         start only one process per core (equivalent to -npcore 1)" 1>&2
    echo "  -npsocket n    set number of processes per cpu socket" 1>&2
    echo "  -psocket       start only one process per cpu socket (equivalent to -npsocket 1)" 1>&2
    echo "  -np n          set total number of processes" 1>&2
    echo "  -c file        use file for sourcing mpi-start variables" 1>&2
    echo "  -i file        use file for standard input" 1>&2
    echo "  -o file        use file for standard output" 1>&2
    echo "  -e file        use file for standard error" 1>&2
    echo "  -x VAR[=VALUE] export the environment variable VAR," 1>&2
    echo "                   optionally define value" 1>&2
    echo "  -d VAR=VALUE   define mpi-start variable VAR with specified VALUE" 1>&2
    echo "  --             separator for application and arguments" 1>&2
}

#======================================================================
#  Print version.
#======================================================================
print_version() {
    echo "mpi-start v1.5.0" 
}

if test "$I2G_MPI_START_ENABLE_TESTING" != "TEST" ; then
    #======================================================================
    #  Parse the command line arguments and call the main function 
    #======================================================================
    export I2G_MPI_START=$0
    arguments="$*"
    while [ "x$1" != "x" ] ; do
        case "$1" in 
            -h)
                print_usage
                exit 0
                ;;
            -V)
                print_version
                exit 0
                ;;
            -t)
                shift
                export I2G_MPI_TYPE=$1
                shift
                ;;
            -v)
                export I2G_MPI_START_VERBOSE=1
                shift
                ;;
            -vv)
                export I2G_MPI_START_VERBOSE=1
                export I2G_MPI_START_DEBUG=1
                shift
                ;;
            -vvv)
                export I2G_MPI_START_VERBOSE=1
                export I2G_MPI_START_DEBUG=1
                export I2G_MPI_START_TRACE=1
                shift
                ;;
            -pre)
                shift
                export I2G_MPI_PRE_RUN_HOOK=$1
                shift
                ;;
            -post)
                shift
                export I2G_MPI_POST_RUN_HOOK=$1
                shift
                ;;
            -pcmd)
                shift
                export I2G_MPI_PRECOMMAND=$1
                shift
                ;;
            -pnode)
                export I2G_MPI_SINGLE_PROCESS=1
                shift
                ;;
            -pcore)
                export I2G_MPI_SINGLE_CORE=1
                shift
                ;;
            -psocket)
                export I2G_MPI_SINGLE_SOCKET=1
                shift
                ;;
            -npnode)
                shift
                export I2G_MPI_PER_NODE=$1
                shift
                ;;
            -npcore)
                shift
                export I2G_MPI_PER_CORE=$1
                shift
                ;;
            -npsocket)
                shift
                export I2G_MPI_PER_SOCKET=$1
                shift
                ;;
            -np)
                shift
                export I2G_MPI_NP=$1
                shift
                ;;
            -c)
                shift
                export I2G_MPI_CONFIG=$1
                shift
                ;;
            -i)
                shift
                export I2G_MPI_APPLICATION_STDIN=$1
                shift
                ;;
            -o)
                shift
                export I2G_MPI_APPLICATION_STDOUT=$1
                shift
                ;;
            -e)
                shift
                export I2G_MPI_APPLICATION_STDERR=$1
                shift
                ;;
            -x)
                shift
                mpi_start_export_variable $1 
                shift
                ;;
            -d)
                shift
                var_name=`echo $1 | cut -f1 -d"="`
                if test "x$var_name" = "x$1" ; then
                    eval $var_name="`echo \\$$var_name`" 2> /dev/null
                elif test "x$var_name" != "x" ; then
                    var_value=`echo $1 | cut -f2- -d"="`
                    eval $var_name=\"$var_value\" 2> /dev/null
                else
                    error_msg "Wrongly formatted argument ($1) to -d option."
                    exit 1
                fi
                if test $? -ne 0 ; then
                    error_msg "Wrongly formatted argument ($1) to -d option."
                    exit 1
                fi
                shift
                ;;
            --)
                shift
                break
                ;;
            *)
                if test "${1:0:1}" = "-" ; then
                    echo "Invalid option $1" 1>&2
                    echo 1>&2
                    print_usage
                    exit 1
                fi
                break
                ;;
        esac
    done
    if test "x$1" != "x" ; then
        export I2G_MPI_APPLICATION=$1
        shift
    fi
    if test "x$*" != "x" ; then
        export I2G_MPI_APPLICATION_ARGS="$*"
    fi
    main "$arguments"
fi


