#!/bin/sh
################################################################################
# This script submits parallel jobs.
# Usage: psub config cdate cdump cstep
# Imported variables:
#   CONFIG
#   CDATE
#   CDUMP
#   CSTEP
# Configuration variables:
#   PSLOT
#   USER
#   CUE2RUN
#   CUE2RUN1
#   NUMPROCPREP
#   NUMPROCANAL
#   MANALhhddd (hh=00,06,12,18;ddd=fnl,gfs)
#   MFCSThhddd (hh=00,06,12,18;ddd=fnl,gfs)
#   CYINC
#   LEVSFCSThhddd
#   NUMPROCFCSThhddd
#   NUMPROCPOSThhddd
#   ckshSH (cksh=dump,etc)
#   RLIST
#   COMDMP
#   COMRES
#   COMROT
#   NDATE
#   SUB
#   RUNLOG
#   PLOG
#   PCNE
################################################################################
set -ux


################################################################################
# Validate arguments

export CONFIG=${1:-${CONFIG:?}}
[[ $(dirname $CONFIG) = . ]]&&CONFIG=$(pwd)/$CONFIG
if [[ ! -r $CONFIG ]];then
  echo $0: inaccessible configuration file $CONFIG >&2
  exit 4
fi
set -a;. $CONFIG;set +a

export CDATE=${2:-${CDATE:?}}
NDATE=${NDATE:-/nwprod/util/exec/ndate}
if [[ $($NDATE 0 $CDATE) != $CDATE ]];then
  echo $0: invalid current date $CDATE >&2
  exit 4
fi

export CDUMP=${3:-${CDUMP:?}}
if [[ $CDUMP != fnl && $CDUMP != gfs ]];then
  echo $0: invalid current dump $CDUMP >&2
  exit 4
fi

export CSTEP=${4:-${CSTEP:?}}
export CKSH=$(echo $CSTEP|cut -c-4)
export CKND=$(echo $CSTEP|cut -c5-)

################################################################################
# Set other variables

export NDATE=${NDATE:-/nwprod/util/exec/ndate}
export SUB=${SUB:-/u/wx20mi/bin/sub}
export LLQU=${LLQU:-/u/wx20mi/bin/llqu}
export CCPOST=${CCPOST:-NO}
export USER=${USER:-$LOGNAME}
export CUE2RUN=${CUE2RUN:-"dev"}
export CUE2RUN1=${CUE2RUN1:-"1/R"}
export CYINC=${CYINC:-06}
export PLOG=${PLOG:-$SHDIR/plog}
export PCNE=${PCNE:-$SHDIR/pcne}
export GDATE=$($NDATE -$CYINC $CDATE)
export GDUMP=${GDUMP:-fnl}
cycle=$(echo $CDATE|cut -c9-10)
cdump=$(echo $CDUMP|tr [a-z] [A-Z])
eval manal=\${MANAL$cycle$cdump:-1}
eval mfcst=\${MFCST$cycle$cdump:-1}

################################################################################
# Set prerequisites and submit parameters based on script name

case $CKSH in

copy)
  prereqs="true"
  numproc="1/1"
  cue2run=$CUE2RUN1
  timelim=1:00:00
  whenrun=+0000
;;

dump)
  prereqs="true"
  numproc="1/1"
  cue2run=$CUE2RUN1
  timelim=8:00:00
  case $CDUMP in
  gfs)
    case $(echo $CDATE|cut -c9-10) in
      00) whenrun=$(echo $CDATE|cut -c1-8)0250;;
      06) whenrun=$(echo $CDATE|cut -c1-8)0850;;
      12) whenrun=$(echo $CDATE|cut -c1-8)1450;;
      18) whenrun=$(echo $CDATE|cut -c1-8)2050;;
    esac
  ;;
  fnl)
    case $(echo $CDATE|cut -c9-10) in
      00)whenrun=$(echo $CDATE|cut -c1-8)0610;;
      06)whenrun=$(echo $CDATE|cut -c1-8)1150;;
      12)whenrun=$(echo $CDATE|cut -c1-8)2010;;
      18)whenrun=$(echo $CDATE|cut -c1-8)2350;;
    esac
  ;;
  esac
;;

prep)
  prereqs="[[ -s $COMDMP/statup.$CDUMP.$CDATE && $manal -gt 0  ]]"
  numproc=${NUMPROCPREP:-"12"}
  cue2run=$CUE2RUN
  timelim=1:00:00
  whenrun=+0000
;;

anal)
  prereqs="[[ $manal -gt 0 ]]"
  numproc=${NUMPROCANAL:-"$(($JCAP/2+3))"}
  cue2run=$CUE2RUN
  timelim=3:00:00
  whenrun=+0000
;;

fcst)
  nknd=${CKND:-1}
  prereqs="[[ $nknd -le $mfcst ]]"
  numproc=$(eval echo \${NUMPROCFCST$cycle$cdump:-$LEVS}|cut -f$nknd -d,)
  numproc=${numproc:-$LEVS}
  cue2run=$CUE2RUN
  timelim=4:00:00
  whenrun=+0000
;;

post)
  nknd=${CKND:-1}
  prereqs="[[ $nknd -le $mfcst ]]"
  numproc=${numproc:-$(eval echo \${NUMPROCPOST$cycle$cdump:-4}|cut -f$nknd -d,)}
  numproc=${numproc:-4}
  cue2run=$CUE2RUN
  timelim=4:00:00
  if [[ $CCPOST = YES ]];then
    whenrun=+0010
  else
    whenrun=+0000
  fi
;;

vrfy)
  prereqs="true"
  numproc="1/1"
  cue2run=$CUE2RUN1
  timelim=2:00:00
  whenrun=+0000
;;

arch)
  prereqs="true"
  numproc="1/1"
  cue2run=$CUE2RUN1
  timelim=2:00:00
  whenrun=+0000
;;

esac

################################################################################
# Check prerequisites

if [[ $CDATE -gt $EDATE || $CDATE$CDUMP$CSTEP = $EDATE$EDUMP$ESTEP ]];then
  echo $0: end of experiment reached >&2
  exit 1
elif eval $prereqs;then
  $PCNE $CDATE/$CDUMP/$CSTEP/RSI $COMRES <$RLIST
  if [[ $? -eq 0 ]];then
    echo $0: all prerequisites satisfied for step $CSTEP >&2
  else
    echo $0: RS prerequisites not satisfied for step $CSTEP >&2
    exit 1
  fi
else
  echo $0: IN prerequisites not satisfied for step $CSTEP >&2
  exit 1
fi

################################################################################
# Submit job

en=CONFIG,CDATE,CDUMP,CSTEP
jn=$PSLOT$CDATE$CDUMP$CSTEP
df=$COMROT/$jn.dayfile
qn=$cue2run
np=$numproc
tl=$timelim
us=$USER
wr=$whenrun
eval job=\${$(echo $CKSH|tr [a-z] [A-Z])SH:-$SHDIR/$CKSH.sh}
if $LLQU $us|grep $jn;then
  $PLOG "$RUNLOG" NOTE "$jn already queued"
else
  $SUB -e $en -j $jn -o $df -p $np -q $qn -t $tl -u $us -w $wr $job
  if [[ $? -ne 0 ]];then
    $PLOG "$RUNLOG"  NOTE "$jn submit failure"
    sleep 60
    if $LLQU $us|grep $jn;then
      $PLOG "$RUNLOG" NOTE "$jn already queued"
    else
      $SUB -e $en -j $jn -o $df -p $np -q $qn -t $tl -u $us -w $wr $job
      if [[ $? -ne 0 ]];then
        $PLOG "$RUNLOG" ERROR "$jn submit failure"
        exit 8
      fi
      $PLOG "$RUNLOG" OK "$jn submitted"
    fi
  else
    $PLOG "$RUNLOG" OK "$jn submitted"
  fi
fi
exit