#### UNIX Script Documentation Block # # Script name: tranjb Tranfer job to put data in BUFR tanks # # JIF contact: Woollen org: w/nmc2 date: 1996-09-24 # # Abstract: The tranjb script controls the transfer of BUFR data # into the BUFR database tanks. The parameters point it to the # database parent directory and the BUFR input file. Output from # the transfer executable is scrolled into the top of the logfile, # tranjb.out, in the database parent directory. An errlog.out file is # also located there. # # Script history log: # 1996-09-06 J. Woollen Original version for implementation. # 1996-12-10 J. Woollen Modified error exit so errlog file is smaller. # 1996-12-11 J. Woollen Added time stamp to heading with input filename. # 1997-12-15 Bill Facey Convert to IBM. # 1998-05-13 Bill Facey Add new variable "cword" that controls the # execution of the script defined in CWORD. For unblocked BUFR files # (e.g., all decoder files and NESDIS ATOVS retrievals) cword should be # set to "yes" (default) in order to FORTRAN block them because this # platform uses BUFRLIB with FORTRAN blocked BUFR files (tanks). If the # BUFR file is already FORTRAN blocked (e.g., all SHEF and NESDIS data # except for ATOVS retrievals) cword must be exported as "NO" or "no". # 1999-02-02 Bill Facey Modified to give individual users the options of # receiving a return code from the script and to turn off the time # screening of the incoming BUFR data. # 2000-03-13 Bill Facey Modified to generate a return code unless otherwise # specified. # 2001-04-26 D. Keyser Modified to accept the imported script variable # "EXECbufr" to set the path to the BUFR_TRANJB executable (if not # found, still defaults to "/nwprod/exec"). # 2001-02-13 D. Keyser Modified to increase the maximum number of lines in # the output file /dcom/us007003/tranjb.out from 20K to 80K. The error # log file /dcom/us007003/errlog.out is now limited to 80K lines and new # output is scrolled from the beginning of the file rather than appended # to the end of the file. # 2001-06-05 D. Keyser Modified to post a message to the joblog file prior # to the execution of TRANX. # 2003-08-19 D. Keyser Add new variable "CHGRP_RSTPROD" which when "YES" # (default) forces TRANX to check to see if a BUFR database file # qualifies as a "RESTRICTED" data type. Also now allows the variable # "CWORD" to be imported with a value different than the default # "/nwprod/ush/cwordsh" # 2005-01-25 D. Keyser Modified to run in the working directory (DATA) # 2005-11-01 D. Keyser Removed echo of SCREEN into temporary file, # removed stdin of this file into TRANX since TRANX now obtains value of # SCREEN directly via call to GETENV. Now writes error for zero-length # or non-existent file to the top of file errlog.out instead of to the # bottom (to be consistent with bufr_tranjb errors) # 2007-08-24 D. Keyser Add new variable "MESSAGE_LENGTH" which when imported # with a value of "10001" or larger forces TRANX to open all new output # BUFR messages with length $MESSAGE_LENGTH bytes {maximum value for # MESSAGE_LENGTH is 50000 (i.e., 50,000 byte messages opened}. If # MESSAGE_LENGTH is not imported here, it is set to the default value of # "-99999". If the value for MESSAGE_LENGTH passed into TRANX is less # than "10001" (e.g., the default value of "-99999"), TRANX opens all # new output BUFR messages with the default BUFRLIB length of 10,000 # bytes. Also, variable "CHGRP_RSTPROD" can now be imported with a # value of "ALL" which forces TRANX to set all input BUFR database files # to "RESTRICTED" data types (regardless of whether or not they # explicitly qualify to be a "RESTRICTED" data type when "CHGRP_RSTPROD" # is imported as "YES"). # 2008-01-02 D. Keyser Added a new section which, when new script variable # COPYFILES is imported as YES, attempts to copy individual files pulled # over from the remote unix machine to directory # $tank_dir/YYYYMMDD/wbufbul (obtaining YYYYMMDD from the date qualifier # in the remote filename), after their translation into NCEP BUFR and # subsequent standardization into WMO BUFR (Note: This works only for # remote files with date qualifiers the following forms: *.Dyyddd.S*.*, # *.Dyyddd.T*.*, *.Dyyyymmddhh.T*.* and *.Dyyyymmddhh.S*.*). If script # variable SENDDBN is imported as YES, a DBN alert will be submitted to # post files to a remote machine (currently only possible for WindSat # files). # # Usage: tranjb db-parent bufrfile # # Script parameters: # db-parent - the database parent directory # bufrfile - full path to a BUFR file to be ingested to the database # # Modules and files referenced: # scripts : $utilscript/date2jday.sh # data cards : none # executables: $TRANX # $utilexec/tocsbufr # # Remarks: # # Condition codes: # 00 - no problem encountered # 99 - more or less than two arguments passed # # This script passes the imported variable "SUBDATE_CHECK" into # bufr_tranjb (which reads it via a call to "GETARG"). SUBDATE_CHECK # is set (optionally) in a parent script. # # Attributes: # # Language: AIX script # Machine: IBM # #### [ $# -ne 2 ] && { echo "$0: "; exit 99; } set -xa tank_dir=$1 file=$2 # unset `env | grep XLFUNIT | awk '{print $1}'` export XLFRTEOPTS="unit_vars=yes" [ -z "$CHGRP_RSTPROD" ] && CHGRP_RSTPROD=YES [ -z "$MESSAGE_LENGTH" ] && MESSAGE_LENGTH=-99999 [ $MESSAGE_LENGTH -gt 50000 ] && MESSAGE_LENGTH=50000 [ -z "$SCREEN" ] && SCREEN=YES [ -z "${give_rc}" ] && give_rc=YES [ -z "${cword}" ] && cword=yes RC_FLAG=` echo $give_rc | tr '[a-z]' '[A-Z]' ` tmp=$tank_dir/tranjb.out.$$ EXECbufr=${EXECbufr:-/nwprod/exec} FIXbufr=${FIXbufr:-/nwprod/fix} # 2nd choice location for BUFR mnemonic table # (1st choice is tank_dir) # controlled inside program bufr_tranjb TRANX=${TRANX:-$EXECbufr/bufr_tranjb} CWORD=${CWORD:-/nwprod/ush/cwordsh} err=$tank_dir/errlog.out log=${log:-$tank_dir/tranjb.out} # just exit quietly if the input file doesn't exist # ------------------------------------------------- # ------------------------------------------------- if [ ! -s $file ]; then sleep 10 if [ ! -s $file ]; then if [ ! -f $file ]; then msg="$file does not exist" else msg="$file has zero length" fi tmp_err=$tank_dir/errlog.out.$$ echo "****************************************************************\ *****************************************************************" > $tmp_err echo "****************************************************************\ *****************************************************************" >> $tmp_err echo "`date -u`" >> $tmp_err echo "Input file $msg -- tranjb cannot run" >> $tmp_err [ -s $err ] || >$err head -n 80000 $err | cat >> $tmp_err cp $tmp_err $err rm $file $tmp_err exit 0 fi fi # put a filename header at the top of the ingest report # ----------------------------------------------------- echo "*******************************************************************\ **************************************************************" >> $tmp echo "*******************************************************************\ **************************************************************" >> $tmp echo "-------------------------------------------------------------------\ --------------" >> $tmp echo "`date +%r` $file" >> $tmp echo "-------------------------------------------------------------------\ --------------" >> $tmp # # check to see if need to run cwordsh to block the bufrfile if [ $cword = yes ]; then $CWORD block $file $tank_dir/F8_blocked.$$ err_cword=$? if [ $err_cword -eq 0 ]; then echo "unblocked file successfully blocked by cword and copied to \ $tank_dir/F8_blocked.$$" >> $tmp else rm $tank_dir/F8_blocked.$$ [ $RC_FLAG = YES ] && exit 1 exit 0 fi else echo "blocked file successfully copied to $tank_dir/F8_blocked.$$" >> $tmp cp $file $tank_dir/F8_blocked.$$ fi # run the bufr_tranjb database ingest program # ------------------------------------------- pgm=`basename $TRANX` msg="$pgm has BEGUN" [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg" export XLFUNIT_8=$tank_dir/F8_blocked.$$ $TRANX >> $tmp 2>&1 rc=$? if [ -n "$COPYFILES" ]; then if [ $COPYFILES = YES ]; then ############################################################################### # This section attempts to copy individual files pulled over from the remote # unix machine to directory $tank_dir/YYYYMMDD/wbufbul (obtaining YYYYMMDD # from the date qualifier in the remote filename), after their translation # into NCEP BUFR and subsequent standardization into WMO BUFR # # Note: This works only for remote files with date qualifiers in the following # form: *.Dyyddd.S*.* # *.Dyyddd.T*.* # *.Dyyyymmddhh.T*.* # *.Dyyyymmddhh.S*.* ############################################################################### iflag=0 file_base=`basename $file | awk -F[.]tmpout '{print $1}' | cut -c12-` msg="attempt to standardize and save this NCEP BUFR file" [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg" # Convert NCEP BUFR file to "standard" WMO BUFR file # -------------------------------------------------- export pgm=tocsbufr cwd=`pwd` cd $DATA . $DATA/prep_step cd $cwd export XLFUNIT_11=$tank_dir/F8_blocked.$$ export XLFUNIT_51=$tank_dir/bufrout.$$ $DATA/startmsg $utilexec/tocsbufr << EOF &INPUT BULHED="DUMMY ",KWBX="DUMM", NCEP2STD=.TRUE., SEPARATE=.TRUE., / EOF err_tocsbufr=$? if [ $err_tocsbufr -eq 0 ]; then # If conversion successful, determine YYYYMMDD from date qualifier in file # name (to get correct path $tank_dir/YYYYMMDD/wbufbul to copy file to) # ------------------------------------------------------------------------ yyyyddd=0 # initialize YYYYMMDD as ZERO for char in S T; do date_qual=\ `echo $file_base | awk -F[.]D '{print $2}' | awk -F[.]$char '{print $1}'` if [ "${#date_qual}" -eq '5' ]; then # .... come here if 5-character date qualifier in filename, make sure it # is numeric (thus in the form YYDDD) # ----------------------------------------------------------------- if [ "$(echo $date_qual | grep "^[[:digit:]]*$")" ]; then # ........ make sure YY and DDD pass sanity checks # --------------------------------------- ddd=`echo $date_qual | cut -c3-5` if [ $ddd -eq 0 -o $ddd -gt 366 ];then iflag=3 # day-of-year in date qualifier in invalid else yy=`echo $date_qual | cut -c1-2` # ............ convert 2-digit to 4-digit yr using Y2K windowing technique # (yy=21-99, assume 1921-1999; yy=00-20, assume 2000-2020) # ----------------------------------------------------------- if [ $yy -gt 20 ]; then yyyyddd=19${yy}${ddd} else yyyyddd=20${yy}${ddd} fi # ............ convert year and day-of-year to year, month and day # --------------------------------------------------- yyyymmdd=`$utilscript/date2jday.sh $yyyyddd` break # have YYYYMMDD, all done fi else iflag=2 # date qualifier is not all numeric fi elif [ "${#date_qual}" -eq '10' ]; then # .... come here if 10-character date qualifier in filename, make sure it # is numeric (thus in the form YYYYMMDDHH) # ------------------------------------------------------------------ if [ "$(echo $date_qual | grep "^[[:digit:]]*$")" ]; then # ........ make sure YYYY, MM, DD and HH pass sanity checks # ----------------------------------------------- yyyy=`echo $date_qual | cut -c1-4` mm=`echo $date_qual | cut -c5-6` dd=`echo $date_qual | cut -c7-8` hh=`echo $date_qual | cut -c9-10` if [ $yyyy -lt 1900 -o $yyyy -gt 2100 ];then iflag=3 # year invalid elif [ $mm -lt 1 -o $mm -gt 12 ];then iflag=3 # month invalid elif [ $dd -lt 1 -o $dd -gt 31 ];then iflag=3 # day-of-month invalid elif [ $hh -lt 0 -o $hh -gt 23 ];then iflag=3 # hour invalid else yyyymmdd=${yyyy}${mm}${dd}${hh} break # have YYYYMMDD, all done fi else iflag=2 # date qualifier is not all numeric fi else iflag=1 # date qualifier is neither 5- nor 10-characters fi done if [ $yyyymmdd -ne 0 ]; then mkdir -p $tank_dir/$yyyymmdd/wbufbul mv $tank_dir/bufrout.$$ $tank_dir/$yyyymmdd/wbufbul/$file_base msg="$file_base successfully standardized and saved in \ $tank_dir/$yyyymmdd/wbufbul" [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg" if [ -n "$SENDDBN" ]; then if [ $SENDDBN = YES ]; then # ........ post WindSat files to remote machine via DBN alert # -------------------------------------------------- $DBNROOT/bin/dbn_alert BUFR WINDSAT $job \ $tank_dir/$yyyymmdd/wbufbul/$file_base fi fi else if [ $iflag -eq 1 ]; then msg="***WARNING: $file_base standardized but not saved because \ target directory could not be obtained (date qualifier neither 5- nor 10-char)" elif [ $iflag -eq 2 ]; then msg="***WARNING: $file_base standardized but not saved because \ target directory could not be obtained (date qualifier is not numeric)" elif [ $iflag -eq 3 ]; then msg="***WARNING: $file_base standardized but not saved because \ target directory could not be obtained (date qualifier out of range)" elif [ $iflag -eq 0 ]; then msg="***WARNING: $file_base standardized but not saved because \ target directory could not be obtained (no valid date qualifier found)" fi [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg" fi else msg="***WARNING: standardization of NCEP BUFR file FAILED, thus it \ is not saved" [ -n "$jlogfile" ] && $DATA/postmsg "$jlogfile" "$msg" fi ############################################################################### fi fi # scroll the report into the log file(s) and exit # ----------------------------------------------- if [ $rc -ne 0 ]; then tmp_err=$tank_dir/errlog.out.$$ cp $tmp $tmp_err [ -s $err ] || >$err head -n 80000 $err | cat >> $tmp_err cp $tmp_err $err fi [ -s $log ] || >$log head -n 80000 $log | cat >> $tmp cp $tmp $log rm $tmp $tmp_err # test to see if return code from script is desired. # -------------------------------------------------- rm $tank_dir/F8_blocked.$$ [ $RC_FLAG = YES ] && exit $rc exit 0