#!/bin/ksh # dumpmb - 3 August 2005 # # Abstract: # This script creates MODS formatted BUFR data files according to user # specified date range and data types. Source data is located in /dcom or # MODS online holdings. If found in /dcom, data is dumped and converted to # MODS format. If found already existing in the online MODS archive, data # is simply copied. The destination of output data files is configurable # via the DATA environment variable -- if blank, current directory is used # as default. # # # Script history log: # 2004-07-08 JWhiting Original script # 7/13/04 JAW - added user env vars printout # 7/22/04 JAW - default BTAB set to chkout/dcom.jifs/ version (temporary) # 2/09/05 JAW - fixed dumpjb rc, now passed thru; # altered some diag output. # 3/16/05 JAW - enabled NDATE in place of FDATE; # added RC_ARG error return processing. # 5/06/05 JAW - default MODX set to chkout/exec.jifs/ version (temporary) # 7/27/05 JAW - upgraded for operational use: TMPDIR=$DATA, etc # 8/04/05 JAW - default BTAB=/nwprod/fix/bufrtab.mods # # # Usage: # dumpmb ymd1 ymd2 dType1 [dType2 [... [dTypeN]]] # where ymd1 == Start date # ymd2 == End date # dType* == data types (eg, metar, ships, bathy, etc) # # Date strings (ymd*) are expected in YYYYMMDD format. # A value of 'all' for dType1 will cause all valid types to be processed. # # # Environment variables ( [ default value] ): # TANK - directory path to online data [ /dcom/us007003 ] # BTAB - path to MODS bufrtable [ /nwprod/fix/bufrtab.mods ] # DATA - directory path for MODS output [ pwd ] # MODX - path to bufr_prepmods exec [ /nwprod/exec/bufr_prepmods ] # DUMP - path to general dump script [ /nwprod/ush/dumpjb ] # LIST - path to dumplist file [ /nwprod/fix/bufr_dumplist ] # # # Exit conditions: # This script will exit with a return code equal to the greatest # value returned from any of the called dump or conversion # processes. For certain fatal errors, specific error codes are # returned: # # code reason # 00 - No problem encountered # 77 - Incorrect usage (arguments) # 78 - Invalid date (arguments) # 79 - No valid data types specified # 88 - Invalid Directory (DATA or TANK env vars) # 66 - Error creating/using working dir # # # Modules and files referenced: # user scripts: dumpjb # fixed fields: bufr_dumplist bufrtab.mods # executables : bufr_prepmods /nwprod/util/exec/ndate # # # Issues/caveats: # - Data is currently handled in full day (24 hour) files only. No # subsetting on a shorter time nor combining on a longer time is # supported. # - No geographic subsetting is supported. # - Will overwrite existing files in destination directory. # ver='08.04.05' usage="\ dumpmb: This script generates MODS formatted BUFR files. Usage: dumpmb ymd1 ymd2 dType1 [dType2 [... [dTypeN]]] where ymd1 == Start date ymd2 == End date dType* == data types (eg, metar, ships, bathy, etc) Date strings (ymd*) are expected in YYYYMMDD format. A value of 'all' for dType1 will cause all valid types to be processed. This is version $ver" [ $# -lt 3 ] && { echo "$usage" ; exit 77 ; } echo 'dumpmb: ============================================================' echo 'dumpmb: Starting execution of MODS dump script -- dumpmb' echo "dumpmb: Version $ver" echo "dumpmb: Start time: `date`" echo 'dumpmb: ============================================================' RC_ARG=0 # Arguments testing return code cymd1=${1:?"$usage"} cymd2=${2:?"$usage"} #echo "debug: $# args='$*'" #--- set -a # auto export variables # Parse args for valid dates # -------------------------- #FDATE='/bin/sh /nwprod/util/ush/finddate.sh' NDATE=/nwprod/util/exec/ndate for cymd in $cymd1 $cymd2 ; do $NDATE 24 ${cymd}12 > /dev/null 2>&1 ; RC=$? if [ $RC -ne 0 ] ; then echo "dumpmb: ERROR: - invalid Date argument '$cymd' -- exiting (78)" RC_ARG=78 fi # RC != 0 done # cymd if [ $cymd1 -gt $cymd2 ] ; then echo "dumpmb: ERROR: - invalid Dates '$cymd1' > '$cymd2' -- exiting (78)" RC_ARG=78 fi # cymd1 > cymd2 if [ $RC_ARG -eq 0 ] ; then echo "dumpmb: Creating MODS formatted BUFR files for" echo "dumpmb: Date range: $cymd1\c" [ $cymd2 = $cymd1 ] || echo " thru $cymd2" && echo "" # Parse args for valid data types # ------------------------------- t000=' metar ' t001=' ships dbuoy mbuoy lcman tideg slpbg cstgd ' t012=' sstns sstnvh ssmipn ' # ersal t031=' bathy tesac trkob nersal ngfoal ntpxal njsnal ' alltypes=${t000}${t001}${t012}${t031} dtypes="$3" badtypes=' ' if [ ".$dtypes" = . -o ".$dtypes" = '.all' ] ; then dtypes=$alltypes else shift 2 dtypes='' while [ $# -ge 1 ] ; do arg1=$1 shift 1 for DTYP in $arg1 ; do DTYP=`echo $DTYP | tr [A-Z] [a-z]` if [ -z "`echo \" $alltypes \" | grep \" $DTYP \"`" ] ; then badtypes=$badtypes" $DTYP" continue fi # valid dtype dtypes="$dtypes $DTYP" done # for DTYP done # while args exist fi # dtypes = blank|all if [ -n "$dtypes" ] ; then echo "dumpmb: Data types: $dtypes" ; fi for BTYP in $badtypes ; do msg="dumpmb: ! Input argument '$BTYP' not a supported datatype." echo $msg done # badtypes if [ -z "$dtypes" ] ; then echo "dumpmb: ERROR: - No valid data types listed - exiting (79)" RC_ARG=79 fi # no valid dtypes fi # RC_ARG = 0 if [ $RC_ARG -eq 0 ] ; then # Set up environment parms # ------------------------ # ---make list (UVARs) of non-blank (user set) env vars eVARs='TANK BTAB DATA MODX DUMP LIST' UVARs='' for eVAR in $eVARs ; do eval var=\$$eVAR [ -n "$var" ] && UVARs="$UVARs $eVAR" done # for env Vars (eVar) # ---set default values userTANK='' [ -n "$TANK" ] && userTANK=$TANK BTAB=${BTAB:-"/nwprod/fix/bufrtab.mods"} userDATA="`pwd`" [ -n "$DATA" ] && userDATA="$DATA" DATA=${DATA:-"`pwd`"} MODX=${MODX:-"/nwprod/exec/bufr_prepmods"} DUMP=${DUMP:-"/nwprod/ush/dumpjb"} LIST=${LIST:-"/nwprod/fix/bufr_dumplist"} pgm=`basename $MODX` # ---report User-specified values (if any) if [ -n "$UVARs" ] ; then echo 'dumpmb: ------------------------------------------------------------' echo 'dumpmb: User supplied environment variables:' fi # UVARs not blank for UVAR in $UVARs ; do eval var=\$$UVAR echo "dumpmb: $UVAR == $var" done # for UVAR # ---trap errors in directory settings for eVAR in DATA TANK ; do eval var=\$$eVAR if [ -n "$var" -a ! -d "$var" ] ; then echo ; echo "dumpmb: ! Directory '$var' not found" echo "dumpmb: ERROR: - Bad value ($var) for env variable $eVAR" echo "dumpmb: Exiting with RC=88" ; echo RC_ARG=88 fi # dir not exist done # for eVAR fi # RC_ARG = 0 #echo "debug exit" ; exit if [ $RC_ARG -eq 0 ] ; then # Set up working dir # ------------------ uid=$(echo "$USER" | cut -c1-6) if [ .$uid = .nwprod -o .$uid = .climpr ] ; then # for production runs TMPDIR=$DATA else # prod vs chkout # for checkout runs [ .$TMPDIR = . -o .$TMPDIR = ./nfstmp -o .$TMPDIR = ./gpfstmp ] && \ TMPDIR=/gpfstmp/$USER fi # prod vs chkout WRKDIR=$TMPDIR/dumpmb_`date -u +%H%M`.$$ [ -d $WRKDIR ] && /bin/rm -rf $WRKDIR mkdir -p $WRKDIR || \ { echo "ERROR: mkdir $WRKDIR -- exiting (66)" ; RC_ARG=66 ; } cd $WRKDIR || { echo "ERROR: cd $WRKDIR -- exiting (66)" ; RC_ARG=66 ; } #echo " debug: DATA='$DATA' TMPDIR='$TMPDIR'" #; RC_ARG=45 fi # RC_ARG = 0 if [ $RC_ARG -eq 0 ] ; then RC_MAX=0 # Global return code # Loop over input dates & valid data types # ---------------------------------------- cymd=$cymd1 while [ $cymd -le $cymd2 ] ; do #d echo "dm2: c='$cymd' c2='$cymd2'" ; if [ skip = it ] ; then for DTYP in $dtypes ; do mfile=$DTYP.$cymd echo echo "dumpmb: =========== Creating MODS file $mfile ===========" # Get source data file # Look first in dcom # ------------------- #--Build dcom file spec case $DTYP in cstgd) DCOM=dcomdev ;; *) DCOM=dcom ;; esac TANK=/$DCOM/us007003 [ -n "$userTANK" ] && TANK=$userTANK mtst=`grep "_$DTYP " $LIST|cut -c 16-500|cut -f 1 -d#` if [ -z "$mtst" ] ; then echo "dumpmb: ! dType '$DTYP' not found in dumplist file ($LIST)" echo "dumpmb: ...skipping processing of $DTYP.$cymd" [ $RC_MAX -lt 33 ] && RC_MAX=33 continue fi # mtst blank (DTYP not found) mtyp="`echo $mtst | cut -c1-3" styp="`echo $mtst | cut -c4-6" date_spec=$cymd if [ $mtyp = '031' -a $styp -lt 100 ] ; then date_spec=`echo $cymd | cut -c1-6` fi DSUB=$date_spec/b$mtyp dspec=$TANK/$DSUB/xx$styp #--Build MODS file spec YYYY=`echo $cymd | cut -c1-4` MM=`echo $cymd | cut -c5-6` mspec=$TANK/mods/$YYYY/$MM/$DTYP/$mfile #--Check if dcom file exists if [ -s $dspec ] ; then #echo "dmb debug: dspec='$dspec'" ; ls -log $dspec ; echo 'debug exit' ; exit echo "dumpmb: ! Found dType '$DTYP' as dcom type b$mtyp-$styp" echo "dumpmb: ...generating MODS $mfile from this DCOM source" #---Generate MODS file from dcom source # --Run DUMPJB echo echo "dumpmb: Running DUMPJB script w/ args: ${cymd}12 11.999 $DTYP" # ----save value of DATA then set to blank; this allows dumpjb # to use the current, MODS tmp working dir (pwd) so that the # following bufr_prepmods step will be able to find the dump # files as input. userDATA=$DATA export DATA='' ### this allows dumpjb to use its own WRKDIR # ----run dump script [ -f $DTYP.ibm ] && /bin/rm $DTYP.ibm > /dev/null 2>&1 echo "timex $DUMP ${cymd}12 11.999 $DTYP" timex $DUMP ${cymd}12 11.999 $DTYP >/dev/null 2>&1 rc_dump=$? DATA=$userDATA echo "dumpmb: DUMPJB finished $DTYP $cymd - RC=$rc_dump" if [ $rc_dump -gt 0 ] ; then if [ $rc_dump -gt $RC_MAX ] ; then RC_MAX=$rc_dump ; fi if [ $rc_dump -le 22 ] ; then msg="dumpmb: ...Dump ERROR: empty source for $DTYP $cymd" else msg="dumpmb: ...Dump FAILED for $DTYP $cymd" fi # dump rc =< 22 echo $msg echo "dumpmb: (see $WRKDIR/$DTYP.out)" echo "dumpmb: Skipping creation of MODS file $mfile" else # rc_dump not > 0 msg="dumpmb: DUMPJB completed successfully - $DTYP $cymd" echo $msg # --Run converter program (bufr_prepmods) echo echo "dumpmb: Running $pgm program to create MODS file $mfile" if [ -s $mfile ] ; then echo "dumpmb: !!! MODS file already exists - skipping" else # mfile not already exist export XLFUNITS=0 unset `env | grep XLFUNIT | awk -F= '{print $1}'` if [ -z "$XLFRTEOPTS" ]; then export XLFRTEOPTS="unit_vars=yes" else export XLFRTEOPTS="${XLFRTEOPTS}:unit_vars=yes" fi export XLFUNIT_20="$DTYP.ibm" export XLFUNIT_21="$BTAB" export XLFUNIT_50="$DTYP.$cymd" echo "dumpmb: exec = '$MODX'" echo "dumpmb: BUFR table = '$XLFUNIT_21'" if [ ! -s $DTYP.ibm -o ! -s $BTAB ] ; then echo echo "dumpmb: ERROR: - bufr_prepmods input ($DTYP.ibm) missing" echo "dumpmb: Skipping creation of MODS file $mfile" rc_modx=55 else # modx inputs not found echo "timex $MODX" timex $MODX rc_modx=$? echo "dumpmb: $pgm finished $DTYP $cymd - RC=$rc_modx" fi # modx inputs not found if [ $rc_modx -ne 0 ];then if [ $rc_modx -gt $RC_MAX ] ; then RC_MAX=$rc_modx ; fi msg="dumpmb: $pgm FAILED for $DTYP $cymd" echo $msg msg="dumpmb: ...saving output as `pwd`/$mfile.err" echo $msg mv $mfile $mfile.err else # rc_modx = 0 msg="dumpmb: $pgm completed successfully - $DTYP $cymd" echo $msg echo #---Save MODS file to user space echo "dumpmb: Copying MODS archive file to user dir" if [ -s $DATA/$mfile ] ; then echo "dumpmb: ...overwriting local file ($DATA/$mfile)" fi # local mfile exists echo " cp $mfile $DATA/" cp $mfile $DATA/ # chmod 644 $DATA/$mfile fi # mfile exists fi # rc_modx != 0 fi # rc_dump > 0 else # dspec not exist echo "dumpmb: ! dType '$DTYP' ($cymd) not found as dcom type b$mtyp-$styp" # With no dcom file available, look in MODS # ----------------------------------------- #--Check if MODS file exists if [ -s $mspec ] ; then echo "dumpmb: ! Found MODS file ($mfile) in online archive" echo "dumpmb: ...copying this archive file to user dir" if [ -s $DATA/`basename $mspec` ] ; then echo "dumpmb: ...overwriting local file ($DATA/`basename $mspec`)" fi # local basename mfile exists echo " cp -p $mspec $DATA/" cp -p $mspec $DATA/ chmod 644 $DATA/$mfile else # MODS file not exist echo "dumpmb: ! MODS file ($mfile) not found in archive" # Data not found in dcom nor MODS holdings # ---------------------------------------- echo "dumpmb: !!! No online data available for dType '$DTYP' on $cymd" fi # MODS file exists fi # dcom file exists echo "dumpmb: ======= Finished creating MODS file $mfile =======" echo done # for DTYP #d fi # skip it cymd=`$NDATE 24 ${cymd}12 | cut -c1-8` # cymd=`$FDATE ${cymd}12 P+1` # sh /nwprod/util/ush/finddate.sh `today` P+1 done # while in date range echo echo "dumpmb: MODS Dump processing done w/ highest ret code RC=$RC_MAX" if [ $RC_MAX -eq 0 ] ; then cd $userDATA [ -n "$WRKDIR" ] && /bin/rm -rf $WRKDIR else # echo "dumpmb: Error Return was non-zero somewhere in job." echo "dumpmb: ...keeping Working Dir ($WRKDIR)." echo fi # RC_MAX = 0 # Pass error return from argument parsing # --------------------------------------- else # RC_ARG != 0 RC_MAX=$RC_ARG echo "dumpmb: ERROR - MODS Dump processing aborted w/ highest ret code RC=$RC_MAX" fi # RC_ARG = 0 # Close script # ------------ echo 'dumpmb: ============================================================' echo 'dumpmb: This execution of dumpmb is finished' echo "dumpmb: End time: `date`" echo 'dumpmb: ============================================================' exit $RC_MAX #---end of dumpmb.ksh