#!/bin/bash
#PBS -A STF006
#PBS -N genarray
#PBS -j oe
#PBS -q debug
#PBS -l walltime=0:20:00,size=48
## On the Cray, you need at least 3 nodes for 3 separate application runs

cd $PBS_O_WORKDIR

if [ `hostname | cut -c 1-4` == "sith" ]; then
    source /etc/profile.d/modules.sh
    module unload PE-pgi PE-intel PE-gnu
    module load PE-gnu
    module load szip
    module load dataspaces/1.4.0
    module load mxml/2.7
    module load flexpath/adios-1.7
    #DATASPACES_DIR=/ccs/proj/e2e/dataspaces/sith/pgi
fi

SERVER=$DATASPACES_DIR/bin/dataspaces_server

WRITEPROC=32
STAGINGPROC=2
STAGINGPROC_PER_NODE=2
READPROC=8
let "PROCALL=WRITEPROC+READPROC"

# clean-up previous run
rm -f log.* core* conf dataspaces.conf 
rm staged.bp genarray.bp

# Prepare config file for DataSpaces
echo "## Config file for DataSpaces
ndim = 3
dims = 512,512,512
max_versions = 10
#max_readers =" $READPROC " 
#lock_type = 2
" > dataspaces.conf

# Run DataSpaces
echo "-- Start DataSpaces server "$SERVER" on $STAGINGPROC PEs, -s$STAGINGPROC -c$PROCALL"
mpirun -np $STAGINGPROC -npernode $STAGINGPROC_PER_NODE $SERVER -s$STAGINGPROC -c$PROCALL &> log.server &

## Give some time for the servers to load and startup
sleep 1s
while [ ! -f conf ]; do
    echo "-- File conf is not yet available from server. Sleep more"
    sleep 1s
done
sleep 3s  # wait server to fill up the conf file

## Export the main server config to the environment
while read line; do
    export set "${line}"
done < conf

echo "-- DataSpaces IDs: P2TNID = $P2TNID   P2TPID = $P2TPID"

# Start GENARRAY
echo "-- Start GENARRAY on $WRITEPROC PEs"
mpirun -np $WRITEPROC ./genarray_varying genarray.bp $WRITEPROC 1 1 16 32 64 5 10 >& log.genarray &

# Start STAGE_WRITE
echo "-- Start STAGE_WRITE on $READPROC PEs"
mpirun -np $READPROC ./stage_write genarray.bp staged.bp DATASPACES "enable_collective_read_meta;verbose=4" MPI "" $READPROC 1 1 >& log.stage_write &

echo "-- Wait until all applications exit. Run ./check.sh to see status"
wait
rm -f conf

