#!/bin/ksh
#set -x
######################################################################
###
host=`uname -n`
user=`whoami`
############### User's section #######################################
##
## - Define jobnumber
##  To get a proper result, the jobs have to be executed once each, one 
##  after the other (i.e. 0, 1, 2, 3, 4 in sequence)
jobnum=0   # 0, 1, 2, 3, 4

arch=Linux       # architecture: Linux/SX/ibm_power4/crayx1/crayxd1/crayxt/SX8/SX9
chan=MPI1        # Communication technique MPI1/MPI2
gridswr=0        # 1 to make toy models to write their grid data files
                 #   with prism_write_grid/corner/mask/area 
comp_para=0      # 1 if oasis was compiled with -Duse_oasis_para; 0 otherwise
run_para=0       # 1 to run oasis3 in parallel mode (with 2 oasis3 processes);
                 #   0 otherwise
cmcc_comp_para=1 # 1 if oasis was compiled with -Duse_oasis_cmcc_para; 0 otherwise
cmcc_run_para=1  # 1 to run oasis3 in cmcc parallel mode
nproc_oa3=2    # number of processes to run oasis3
                 # for run_para=1, automatically nproc_oa3=2 (see below) 

if  [ $cmcc_run_para == 0 ] ; then
if  [ $run_para == 1 ] ; then
   nproc_oa3=2
else
   nproc_oa3=1
fi
fi

## - Define number of processes to run atmoa3 and cheoa3
##   (oceoa3 always runs on one process)
nproc_atmche=1 # must be 1 or 3

## - Define number of atmoa3 and cheoa3 processes involved in coupling
##   (oceoa3 always runs on one process)
ncpl_atmche=1 # must be 1 or 3

## - Define paths
cd ../../../../../..
prismdir=`pwd`

if  [ $arch == SX ]; then
  rundir=/import/ds9b/hpf/b/b340041/prism/toyoasis3/work
elif  [ $arch == SX8 ] || [ $arch == SX9 ]; then
  rundir=/work/$user/work_toyoasis3
elif [ $arch == Linux ]; then
  rundir=/space/valcke/toyoasis3/cmccpara_2111_210110
else
  rundir=$prismdir/src/mod/oasis3/examples/toyoasis3/work
fi

## - Define MPIRUN, only used for arch=Linux
MPIRUN=/usr/local/pgi/linux86/9.0/mpi/mpich/bin/mpirun

############### End of user's section ################################
##
inputdir=$prismdir/src/mod/oasis3/examples/toyoasis3/input
datadir=$prismdir/src/mod/oasis3/examples/toyoasis3/data
bindir=/space/valcke/$arch/bin
##
## - Perform some checks
if  [ $cmcc_run_para == 1 ] && [ $run_para == 1 ] ; then
   echo 'You have to choose between ipsl parallel and cmcc parallel mode'
   exit
fi

if  [ $cmcc_run_para == 1 ] && [ $cmcc_comp_para == 0 ] ; then
   echo 'It is mandatory to compile oasis3 with CPP key use_oasis_cmcc_para'
   echo 'to run oasis3 in cmcc parallel mode'
   exit
fi

if  [ $run_para == 1 ] && [ $comp_para == 0 ] ; then
   echo 'It is mandatory to compile oasis3 with CPP key use_oasis_para'
   echo 'to run oasis3 in ipsl parallel mode'
   exit
fi

if  [ $run_para == 1 ] && [ $gridswr == 1 ] ; then
   echo 'To run oasis3 in ISPL parallel mode, grid data files '
   echo 'grids.nc, masks.nc and areas.nc must be created'
   echo 'by the user before the run and available in rundir.'
   exit
fi

if  [ $cmcc_run_para == 1 ] && [ $gridswr == 1 ] ; then
   echo 'To run oasis3 in CMCC parallel mode, grid data files '
   echo 'grids.nc, masks.nc and areas.nc must be created'
   echo 'by the user before the run and available in rundir.'
   exit
fi

if [ $run_para == 1 ] && [ $chan == MPI2 ]; then
   echo 'It is not possible to run oasis3 in parallel mode using'
   echo 'MPI2 spawning procedure'
fi

## - Define names of executables

oa3_exe1=oasis1
if  [ $run_para == 1 ] ; then
   oa3_exe2=oasis2
fi
atm_exe=atmoa3
che_exe=cheoa3
oce_exe=oceoa3



echo ''
echo '*****************************************************************'
echo '***    toyoasis3 '$jobnum
echo ''
echo 'rundir       :' $rundir
echo 'inputdir     :' $inputdir
echo 'datadir      :' $datadir
echo 'bindir       :' $bindir
echo 'architecture :' $arch
echo 'host         : '$host
echo 'chan         : '$chan
echo 'run          : '$jobnum
if [ $run_para == 1 ]; then
echo 'oasis3 runs on ' $nproc_oa3 'processes with IPSL parallelisation mode'
elif [ $cmcc_run_para == 1 ]; then
echo 'oasis3 runs on ' $nproc_oa3 'processes with CMCC parallelisation mode'
else
echo 'oasis3 runs not parallel on 1 process'
fi
echo 'atmoa3 runs on '$nproc_atmche 'process(es)'
echo 'oceoa3 runs on 1 process'
echo 'cheoa3 runs on '$nproc_atmche 'process(es)'
echo ''
######################################################################
###
### 1. Copy and modify necessary files depending on job number

# find date

if [ $jobnum == 0 ]; then
    date=00010101
elif  [ $jobnum == 1 ]; then
    date=00010107
elif  [ $jobnum == 2 ]; then
    date=00010113
elif  [ $jobnum == 3 ]; then
    date=00010119
elif  [ $jobnum == 4 ]; then
    date=00010125
else
    echo 'EXIT : run can only be start or restart'
    exit 
fi

echo 'start date   :' $date
echo ''

if [ $jobnum == 0 ]; then
  #
  # First run : start
  #
  rm -fr $rundir
  mkdir -p $rundir

  cp -f $bindir/oasis3.$chan.x $rundir/$oa3_exe1
  if  [ $run_para == 1 ] ;   then
      cp -f $bindir/oasis3.$chan.x $rundir/$oa3_exe2
  fi
  cp -f $bindir/oceoa3.$chan.x $rundir/$oce_exe
  cp -f $bindir/atmoa3.$chan.x $rundir/$atm_exe
  cp -f $bindir/cheoa3.$chan.x $rundir/$che_exe

  cp -f $inputdir/cf_name_table.txt $rundir/.
  cp -f $datadir/fld*.nc $rundir/.
  cp -f $datadir/SOALBEDO.nc $rundir/.
  cp -f $datadir/bt42orca $rundir/.

  if [ $gridswr == 1 ]; then
      cp -f $datadir/toy*.nc $rundir/.
  else
      cp -f $datadir/grids.nc $datadir/areas.nc $datadir/masks.nc $rundir/.
  fi

  chmod +x $rundir/$oa3_exe1 $rundir/$oce_exe $rundir/$atm_exe $rundir/$che_exe
  if  [ $run_para == 1 ] ;   then
      chmod +x $rundir/$oa3_exe2
  fi

  cd $rundir

else
  #
  # Restart run 
  # Initial run jobnum=0 must have been done to perform the restart run
  #
  rm -f $rundir/*.prt*
  rm -f $rundir/anaisout
fi

cd $rundir

### Copy and update of namcouple with date, MPI1/MPI2 and number of procesuss

if [ $run_para == 1 ] ;   then
	  cp -f $inputdir/namcouple_0      $rundir/namcouple
	  cp -f $inputdir/namcouple_1      $rundir/namcouple_1
else
	  cp -f $inputdir/namcouple      $rundir/namcouple
fi
    
ed $rundir/namcouple <<EOF
g/mpixx/s/mpixx/$chan/
g/ini_date/s/ini_date/$date/
g/nproc_atm/s/nproc_atm/$nproc_atmche/
g/ncpl_atm/s/ncpl_atm/$ncpl_atmche/
g/nproc_che/s/nproc_che/$nproc_atmche/
g/ncpl_che/s/ncpl_che/$ncpl_atmche/
w
q
EOF

  if [ $comp_para == 1 ] || [ $run_para == 1 ] ; then
      mv $rundir/namcouple $rundir/namcouple_0
  fi
#
  if [ $run_para == 1 ] ;   then
      ed $rundir/namcouple_1 <<EOF
      g/mpixx/s/mpixx/$chan/
      g/ini_date/s/ini_date/$date/
      g/nproc_atm/s/nproc_atm/$nproc_atmche/
      g/ncpl_atm/s/ncpl_atm/$ncpl_atmche/
      g/nproc_che/s/nproc_che/$nproc_atmche/
      g/ncpl_che/s/ncpl_che/$ncpl_atmche/
      w
      q
EOF
  fi

######################################################################
###
### 2. Create configuration scripts
if [ $arch == crayx1 ] && [ $chan == MPI1 ]; then

  #
  # crayx1  
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche )) 

if  [ $run_para == 1 ] ; then
  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -S /bin/ksh
#PBS -N toyoasis3
#PBS -l mppe=8
#PBS -l walltime=00:59:00
#PBS -j oe
/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
aprun -n 1 ./$oa3_exe1 : -n 1 ./$oa3_exe2 : -n $nproc_atmche ./$atm_exe : -n 1 ./$oce_exe : -n $nproc_atmche ./$che_exe
/bin/echo " job completed at: " \\c
/bin/date
EOF

else

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -S /bin/ksh
#PBS -N toyoasis3
#PBS -l mppe=8
#PBS -l walltime=00:59:00
#PBS -j oe

/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host

cd $rundir
aprun -n 1 ./$oa3_exe1 : -n $nproc_atmche ./$atm_exe : -n 1 ./$oce_exe : -n $nproc_atmche ./$che_exe
/bin/echo " job completed at: " \\c
/bin/date
EOF

fi

elif [ $arch == crayxt ] && [ $chan == MPI1 ]; then

  #
  # cayxt
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche ))

if  [ $run_para == 1 ] ; then

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/bash
#PBS -l size=$nproc
#PBS -N toyoasis3.$jobnum
#PBS -j oe
/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
cat <<EOF1 > loadfile
yod  -sz 1 -small_pages $oa3_exe1
yod  -sz 1 -small_pages $oa3_exe2
yod -sz $nproc_atmche -small_pages $atm_exe
yod -sz 1 -small_pages $oce_exe
yod -sz $nproc_atmche -small_pages $che_exe
EOF1
yod -F loadfile
/bin/echo " job completed at: " \\c
/bin/date
EOF

else

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/bash
#PBS -l size=$nproc
#PBS -N toyoasis3.$jobnum
#PBS -j oe
/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
cat <<EOF1 > loadfile
yod  -sz 1 -small_pages $oa3_exe1
yod -sz $nproc_atmche -small_pages $atm_exe
yod -sz 1 -small_pages $oce_exe
yod -sz $nproc_atmche -small_pages $che_exe
EOF1
yod -F loadfile
/bin/echo " job completed at: " \\c
/bin/date
EOF

fi

elif [ $arch == crayxd1 ] && [ $chan == MPI1 ]; then

  #
  # crayxd1
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche ))

if  [ $run_para == 1 ] ; then 
  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/csh
#PBS -S /bin/csh
#PBS -l nodes=2:ppn=2
#PBS -N toyoasis3.$jobnum
#PBS -j oe
/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
cat <<EOF1 > loadfile
-n 1 : $XD1LAUNCHER $oa3_exe1
-n 1 : $XD1LAUNCHER $oa3_exe2
-n $nproc_atmche : $XD1LAUNCHER $atm_exe
-n 1 : $XD1LAUNCHER $oce_exe
-n $nproc_atmche : $XD1LAUNCHER $che_exe
EOF1
mpiexec  -config  loadfile
/bin/echo " job completed at: " \\c
/bin/date
EOF

else

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/csh
#PBS -S /bin/csh
#PBS -l nodes=2:ppn=2
#PBS -N toyoasis3.$jobnum
#PBS -j oe
/bin/echo " job started at: " \\c
/bin/date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
cat <<EOF1 > loadfile
-n 1 : $XD1LAUNCHER $oa3_exe1
-n $nproc_atmche : $XD1LAUNCHER $atm_exe
-n 1 : $XD1LAUNCHER $oce_exe
-n $nproc_atmche : $XD1LAUNCHER $che_exe
EOF1
mpiexec  -config  loadfile
/bin/echo " job completed at: " \\c
/bin/date
EOF

fi

elif [ $arch == Linux ]; then
  #
  # Linux
  #

  if [ $chan == MPI2 ]; then
      echo 'Running on Linux with MPI2 not implemeted here!'
      EXIT
  fi

  cat <<EOF> $rundir/appl-linux.conf
$host 0 $rundir/$oa3_exe1
EOF

  if [ $run_para == 1 ] ;   then
 cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$oa3_exe1
EOF
  fi

  if [ $cmcc_run_para == 1 ] ;   then
  count=1
  while [[ $count -lt $nproc_oa3 ]];do
 cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$oa3_exe1
EOF
   (( count += 1 ))
  done
  fi

  count=0
  while [[ $count -lt $nproc_atmche ]];do
   cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$atm_exe
EOF
   (( count += 1 ))
  done

   cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$oce_exe
EOF

  count=0
  while [[ $count -lt $nproc_atmche ]];do
   cat <<EOF>> $rundir/appl-linux.conf
$host 1 $rundir/$che_exe
EOF
   (( count += 1 ))
  done

elif [ $arch == SX ] && [ $chan == MPI1 ]; then

  #
  # NEX SX-6
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche ))
 
if  [ $run_para == 1 ] ; then

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
mpiexec -n 1 $oa3_exe1 : -n 1 $oa3_exe2 : -n $nproc_atmche $atm_exe : -n 1 $oce_exe : -n $nproc_atmche $che_exe
/bin/echo " job completed at: " \\c
date

EOF

else

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
cd $rundir
mpiexec -n $nproc_oa3 $oa3_exe1 : -n $nproc_atmche $atm_exe : -n 1 $oce_exe : -n $nproc_atmche $che_exe
/bin/echo " job completed at: " \\c
date

EOF

fi

elif [ $arch == SX ] && [ $chan == MPI2 ]; then

  #
  # NEX SX-6
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche )) 

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -l cpunum_prc=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l cputim_job=00:30:00 # 30 min cputime
#PBS -l memsz_job=3gb       # 3 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host


cd $rundir

mpiexec -v -np 1 -max_np $nproc $oa3_exe1

/bin/echo " job completed at: " \\c
date

EOF

elif  [ $arch == SX8 ] || [ $arch == SX9 ] && [ $chan == MPI2 ]; then

  #
  # NEX SX-9
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche )) 

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -T mpisx
#PBS -q standard
#PBS -b 1
#PBS -l cpunum_job=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l elapstim_req=00:25:00   # real time
#PBS -l cputim_job=01:30:00 # 30 min cputime
#PBS -l memsz_job=10gb       # 10 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host


cd $rundir
mv atmoa3 toyatm
mv oceoa3 toyoce
mv cheoa3 toyche

mpiexec -v -np $nproc_oa3 -max_np $nproc $oa3_exe1

/bin/echo " job completed at: " \\c
date

EOF

elif [ $arch == SX8 ] || [ $arch == SX9 ] && [ $chan == MPI1 ]; then

  #
  # NEC SX-8 or SX9
  #
 (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche ))
 
if  [ $run_para == 1 ] ; then

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -T mpisx
#PBS -q vector
#PBS -b 1
#PBS -l cpunum_job=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l elapstim_req=00:59:00   # real time
#PBS -l cputim_job=00:59:00 # 30 min cputime
#PBS -l memsz_job=10gb       # 10 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
MPISUSPEND=ON; export MPISUSPEND
cd $rundir
mpiexec -n $nproc_oa3 $oa3_exe1 : -n 1 $oa3_exe2 : -n $nproc_atmche $atm_exe : -n 1 $oce_exe : -n $nproc_atmche $che_exe
/bin/echo " job completed at: " \\c
date
/usr/local/bin/ja
EOF

else

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh
#PBS -T mpisx
#PBS -q vector
#PBS -b 1
#PBS -l cpunum_job=$nproc        # number of CPUs (maximum number of CPUs 8)
#PBS -l elapstim_req=00:59:00   # real time
#PBS -l cputim_job=00:59:00 # 30 min cputime
#PBS -l memsz_job=10gb       # 10 Gbyte memory
#PBS -j o                   # join err and out to out
#PBS -N toyoasis3              # job name
###PBS -M youremail         # email address for error messages etc
#
/bin/echo " job started at: " \\c
date
/bin/echo " ExecutionHost : " \\c
hostname                   # print name of current host
MPISUSPEND=ON; export MPISUSPEND
cd $rundir
mpiexec -n $nproc_oa3 $oa3_exe1 : -n $nproc_atmche $atm_exe : -n 1 $oce_exe : -n $nproc_atmche $che_exe
/bin/echo " job completed at: " \\c
date
/usr/local/bin/ja
EOF

fi

elif  [ $arch == ibm_power4 ]; then

  if [ $chan == MPI2 ]; then
      echo 'MPI2 not possible at IBM'
      EXIT
  fi

  #
  # IBM power4
  #

  cat <<EOF >$rundir/appl-aix.conf
$oa3_exe1
EOF

 if [ $run_para == 1 ] ;   then
  cat <<EOF >$rundir/appl-aix.conf
$oa3_exe2
EOF
fi

  count=0
  while [[ $count -lt $nproc_atmche ]];do
   cat <<EOF>> $rundir/appl-aix.conf
$atm_exe
EOF
   (( count += 1 ))
  done

   cat <<EOF>> $rundir/appl-aix.conf
$oce_exe
EOF

  count=0
  while [[ $count -lt $nproc_atmche ]];do
   cat <<EOF>> $rundir/appl-aix.conf
$che_exe
EOF
   (( count += 1 ))
  done


  (( nproc = $nproc_oa3 + $nproc_atmche + 1 + $nproc_atmche )) 

  cat <<EOF >$rundir/run_toyoasis3.$arch
#!/bin/ksh 
#
#  This script will launch toyoasis3 with oasis3
#
#@ shell           = /usr/bin/ksh 
#@ class           = np 
#@ job_type        = parallel 
#@ job_name        = toyoasis3
#@ output          = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ error           = \$(job_name).\$(schedd_host).\$(jobid).out 
#@ notification    = never  
#@ resources       = ConsumableCpus(1) ConsumableMemory(100mb) 
#@ node            = 1
#@ tasks_per_node  = $nproc
#@ cpu_limit        = 00:30:00 
#@ wall_clock_limit = 00:30:00 
#@ queue 

MPI_DSM_VERBOSE=1; export MPI_DSM_VERBOSE
poe -pgmmodel mpmd -cmdfile appl-aix.conf

exit 0  

EOF


else 
  echo 'For architecture='$arch ' you have to add your execution'
  exit
fi 


######################################################################
###
### 3. Execute the model

if [ $arch == crayx1 ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_toyoasis3.$arch
  qstat

elif [ $arch == crayxd1 ]; then

  echo 'Submitting the job to queue using qsub'
  qsub -q test $rundir/run_toyoasis3.$arch
  qstat

elif [ $arch == crayxt ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_toyoasis3.$arch
  qstat

elif [ $arch == Linux ]; then
  echo 'Executing the model using '$MPIRUN 

  $MPIRUN -p4pg appl-linux.conf ./$oa3_exe1 > runjob.err

elif [ $arch == SX ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_toyoasis3.$arch
  qstat

elif [ $arch == SX8 ] || [ $arch == SX9 ]; then

  echo 'Submitting the job to queue using qsub'
  qsub $rundir/run_toyoasis3.$arch
  qstat

elif  [ $arch == ibm_power4 ]; then
 
  echo 'Submitting the job to queue using llsubmit'    
  llsubmit $rundir/run_toyoasis3.$arch
    
fi

echo 'toyoasis3 '$run'run is executed or submitted to queue.'
echo 'Results are found in rundir : '$rundir 



exit

