;+ ; $Id: secchi_reduce.pro,v 1.299 2009/01/23 21:40:02 secchia Exp $ ; ; Project : STEREO SECCHI ; ; Name : secchi_reduce ; ; Purpose : scan input directory for 8.3 files and do pipeline processing ; ; Explanation : ; ; Use : IDL> secchi_reduce ; ; Inputs : ; ; Outputs : ; ; Keywords : WAITRATE Set to number of seconds to wait before checking for new ; science files. ; /NOWAIT If set, don't wait for new files after done ; /NOTELDIR Do not put day directories in telescope subdirs ; /FAST If set, skip internal waits ; _extra set to indicate pre flight test for SPICE ie sim2='sim2' ; /NO_SPICE If set add do NOT compute SPICE coords to image header ; IPver set Flight software IP version for images v46 = V4.6 ; /TESTLZ Set to use islz=1 for testing when lz not in $SECCHI ; /NOHBTEMPS Set to skip temperatures in header ; /NOSUMMARY Set to skip writing to summary file; also writes delete commands ; in sql script. ; /NOEXTHDR Skips parts that write extended ASCII header ; /NOSORT Do not sort by timestamp ; /HDR_ONLY Get image from outfile rather than scifile ; ** WARNING ** does not work yet with HI ; /GZIP Copy output file to $SECCHI_GZ and gzip it there ; /NOPRETTIES Do not run make_daily_pretties or sccdopolar ; /NOBKG Do not update SECCHI_BKG ; ENDBADAH= time to start reloading stereo_spice if att_file='NotFound' ; /NOEVENT_NOTIFY Default is to send message to sccevent@cronus if there is ; an event detection. Set to disable. ; /FIND_HDR_SCRIPT Search for shell script to modify header. ; /RESUME Use cadence.sav to determine what file in input list to start with ; ; Calls from LASCO : getenv_slash.pro ; ; Common : ; ; Restrictions: Requires env variables defined in spacecraft- and pipeline-specific *.env ; file in $SSW_SECCHI/idl/pipeline ; ; Side effects: ; ; Category : Pipeline ; ; Prev. Hist. : None. ; ; Written : Nathan Rich, NRL/I2, Sep 2004 ; - ; Modified : ; ; $Log: secchi_reduce.pro,v $ ; Revision 1.299 2009/01/23 21:40:02 secchia ; nr - fixed bug if inputfilelist; only mkdir and/or link if it does not exist ; ; Revision 1.298 2009/01/14 21:02:38 nathan ; run new pro touchdatelink to refresh symlink after processing ; ; Revision 1.297 2009/01/09 22:49:35 secchia ; nr - moved telimgdir check so does not run for hdr-only ; ; Revision 1.296 2009/01/09 21:45:00 nathan ; Requires new env var definition for telimgdir; allows spec of separate ; outdir for each tel and each type. ; ; Revision 1.295 2009/01/09 20:15:31 nathan ; added code to create symbolic links for daydir ; ; Revision 1.294 2009/01/08 14:39:28 mcnutt ; date_fname_utc to date_cmd to complete v1.293 mods ; ; Revision 1.293 2009/01/08 13:52:07 mcnutt ; corrected date_obs and avg for summed images using the image save buffers ; ; Revision 1.292 2009/01/07 23:08:29 nathan ; To speed up all processing, remove wait before processing each file, and instead ; wait only when number of found files is <3. Also will not exit automatically. ; ; Revision 1.291 2008/12/11 19:27:55 mcnutt ; calls scc_aplly_mask if mask_tbl ne none ; ; Revision 1.290 2008/11/05 19:21:56 nathan ; changed sccevent@calliope to sccevent@cronus ; ; Revision 1.289 2008/11/03 16:30:51 mcnutt ; changed scc_daily_pretties calls hi1 and 2 with histars keyword ; ; Revision 1.288 2008/11/03 15:37:42 mcnutt ; added calls to scc_daily_pretties hi enhance hi images ; ; Revision 1.287 2008/10/07 16:54:48 nathan ; implement method to key off of imgctr to determine when schedule tables were read ; ; Revision 1.286 2008/09/15 19:01:40 secchia ; nr - committed changes from 9/4/08 ; ; 9/4/08, nr - Changed /DOBKG to /NOBKG; limit effect of /NOPRETTIES to dopolar, cor2 mk_monthly min, and pretties ; ; Revision 1.285 2008/07/15 19:07:21 nathan ; reduce qual threshold for icer misslist (bug 319) ; ; Revision 1.284 2008/05/20 14:50:38 secchia ; added cor1 daily pretties copied from ssc ahead hi processing ; ; Revision 1.283 2008/04/28 13:14:53 mcnutt ; add daily pretties for yyyymmdd-30 cor2 and his only ; ; Revision 1.282 2008/04/22 18:08:42 mcnutt ; change name of hdr only files to seb_prog h instead of s for database fits file name ; ; Revision 1.281 2008/04/16 20:54:41 nathan ; implemented cor2_point.pro ; ; Revision 1.280 2008/04/16 15:30:32 secchib ; corrected Syntax error ; ; Revision 1.279 2008/04/14 18:31:30 mcnutt ; moved Error message to after make_scc_hdr to set scch values ; ; Revision 1.278 2008/04/08 11:57:16 mcnutt ; corrected image error message if exptime eq 0 and no IP error ; ; Revision 1.277 2008/03/28 17:59:54 mcnutt ; added email notification for IP_ERROR,SKIPPED, or ABORT ; ; Revision 1.276 2008/03/18 14:35:34 secchib ; add sort summary at end of batch for j=0 only ; ; Revision 1.275 2008/03/14 15:21:37 mcnutt ; moves mpegframes to line 780 to run at the start of a new day to avoid running twice ; ; Revision 1.274 2008/03/12 14:36:14 mcnutt ; added call to mpegframes if A and HI1 to create mpeg movie list files ; ; Revision 1.273 2008/02/20 18:43:40 nathan ; update cdelt in HI FITS header after running hi_point_v2 (bug 237) ; ; Revision 1.272 2008/02/08 17:23:48 secchia ; nr - updated end-of-day logic ; ; Revision 1.271 2008/02/07 13:50:55 bewsher ; Changed call to hi_point_v2 ; ; Revision 1.270 2008/01/31 19:10:25 nathan ; fix type conv error in last edit and move to ahead only for njpeg msg ; ; Revision 1.269 2008/01/29 22:51:25 secchia ; nr - fix weird hanging error by separating msg strings ; ; Revision 1.268 2008/01/29 21:55:39 nathan ; debugging messages in ipmsg.txt for njpegs lists ; ; Revision 1.267 2008/01/16 21:00:04 nathan ; more of previous ; ; Revision 1.266 2008/01/16 20:49:38 nathan ; implement daily_select_njpegs for mpeg movies (Bug 275) ; ; Revision 1.265 2008/01/14 19:38:48 nathan ; changed npd threshold to 50 from 500 (bug 276) ; ; Revision 1.264 2008/01/07 23:25:26 nathan ; updated scch when EXPTIME from summed (Bug 273) ; ; Revision 1.263 2007/12/18 16:24:56 nathan ; do not send email to dennis if /delete_old ; ; Revision 1.262 2007/12/13 16:38:12 nathan ; updated expcmt for DOUBLE images (bug 268) ; ; Revision 1.261 2007/12/13 13:23:22 mcnutt ; Bug 270 COR2 double n_images ; ; Revision 1.260 2007/12/05 21:21:02 nathan ; removed mail and some writes to impmsg.txt if isrt ; ; Revision 1.259 2007/12/04 14:24:15 secchib ; added final cor summed image to SIsttime incase all other headers are missing ; ; Revision 1.258 2007/11/09 16:58:22 nathan ; div2corr set in scc_icerdiv2 (Bug 252) ; ; Revision 1.257 2007/11/02 18:40:07 nathan ; workaround to Bug 252 BIASMEAN for summed SPWX; added /DOBKG ; ; Revision 1.256 2007/10/30 14:51:26 secchib ; removed exposure adjustment for COR2 summed SPW images ; ; Revision 1.255 2007/10/26 16:16:07 mcnutt ; added final COR2 summed images to the extended header and corrected summed SPW exposure times ; ; Revision 1.254 2007/10/25 11:16:48 mcnutt ; corrected estimate and actual exposure times for cor2 summed images ; ; Revision 1.253 2007/10/16 19:29:40 mcnutt ; corrected SIbasehdr typo when begin reset to -1 ; ; Revision 1.252 2007/10/02 19:35:17 secchia ; added HIxseq_m1.sav to save previous extended header incase sort is out of order ; ; Revision 1.251 2007/10/01 19:21:52 secchib ; fix nmissing bug 243 ; ; Revision 1.250 2007/09/24 22:06:02 nathan ; This is the version used to reprocess SECCHI through 12/06. In includes: ; Added /RESUME; fixed EPHEMFIL call; removed history logs to decrease length; ; moved make_scc_hdr call; correct sebx(y)sum for HI images in 1996 which do ; not show summing properly; use numinseq instead of clear-hi-sumbuf to ; determine start of new HIseqtbl; always set missing block vars ; ; Revision 1.249 2007/09/18 22:46:46 nathan ; removed condition for write_db_scripts; added /FIND_HDR_SCRIPT; ; put more information in subject line for sccevents email ; ; Revision 1.248 2007/09/07 16:31:49 mcnutt ; divid hibias by 1000 ; ; Revision 1.247 2007/09/06 21:03:19 secchib ; change hibias arry to long to avoid negatives ; ; Revision 1.246 2007/09/06 18:08:41 mcnutt ; fills offsetcr for HI images with average meanbias Bug 223 ; ; Revision 1.245 2007/09/04 21:43:08 nathan ; implement scc_mk_all_monthly.pro ; ; Revision 1.244 2007/08/31 21:31:14 nathan ; left in debug lines ... Doh! ; ; Revision 1.243 2007/08/31 19:41:05 nathan ; slight mod to previous ; ; Revision 1.242 2007/08/31 19:29:58 nathan ; put in workaround for Bug 81 (idecomp output) ; ; Revision 1.241 2007/08/31 15:25:58 secchib ; nr - better check for nseg in idecomp ; ; Revision 1.240 2007/08/30 16:39:16 nathan ; implement parsefcmetafile.pro and new idecomp output (Bug 186) ; ; Revision 1.239 2007/08/28 21:00:35 nathan ; Update gzdestdir; implement sc_yaw,pitch for cor1 and euvi; ; implement INS_[xyr]0 (Bug 91); begin implementation of ; parsing new idecomp output (Bug 186) ; ; Revision 1.238 2007/08/24 17:09:45 mcnutt ; added call to comp_archive ; ; Revision 1.237 2007/08/22 18:39:34 nathan ; hilow write_db_scripts in secchi_hi_low_merge ; ; Revision 1.236 2007/08/20 19:00:16 nathan ; Set sdspath earlier and use in calls to make_scc_ext_tbl and hi_point_v2 (Bug 178) ; ; Revision 1.235 2007/08/14 19:09:52 mcnutt ; move make_scc_hdr to before hiseqtbl to correctly fill extend hdr ; ; Revision 1.234 2007/08/13 19:27:39 nathan ; update to do HI summary and db in secchi_reduce, and make sure all HI images go to hi_point_v2 ; ; Revision 1.233 2007/08/08 17:20:36 mcnutt ; added APID report to EUVI processing ; ; Revision 1.232 2007/08/07 17:04:06 nathan ; label pipeline rz for reprocessing ; ; Revision 1.231 2007/08/03 19:44:53 nathan ; update hi_point_v2 implementation (Bug 178) ; ; Revision 1.230 2007/08/03 17:12:08 mcnutt ; Bug 210 corrects biasmean for event detecet images ; ; Revision 1.229 2007/08/01 15:37:36 secchib ; corrected hicadence in cases where we dont have two consecutive headers ; ; Revision 1.228 2007/07/25 17:02:19 nathan ; remove common stereo_spice after conflict with load_stereo_spice.pro ; ; Revision 1.227 2007/07/19 19:50:25 nathan ; fix typo near mm ; ; Revision 1.226 2007/07/19 16:29:06 nathan ; Implement hi_point_v2.pro (Bug 178); update SDS_FIN_DIR ; ; Revision 1.225 2007/07/10 15:02:43 nathan ; Add event detection notification (Bug 148) ; ; Revision 1.224 2007/06/26 22:12:03 nathan ; implement SDS_FIN_DIR for COR1 reprocessing ; ; Revision 1.223 2007/06/25 17:49:26 nathan ; add pointing logging for EUVI ; ; Revision 1.222 2007/06/21 14:42:48 nathan ; fix cebtemps conflict ; ; Revision 1.221 2007/06/20 21:58:51 nathan ; fix error in cadencefile near beginning ; ; Revision 1.220 2007/06/20 15:55:35 secchia ; nr - fix typo ; ; Revision 1.219 2007/06/20 15:09:10 nathan ; Implement ENDBADAH keyword ; ; Revision 1.218 2007/06/19 16:34:23 nathan ; put scc_daily_pretties after mk_daily_med ; ; Revision 1.217 2007/06/19 15:39:23 nathan ; add logging for CRVAL,CRPIX,CRVALA; add /DELETE_OLD ; ; Revision 1.216 2007/06/19 15:02:28 mcnutt ; added Make daily median backgrounds after daily processing Bug 188 ; ; Revision 1.215 2007/06/14 16:50:20 nathan ; use base34 for icer MISSLIST (Bug 183) ; ; Revision 1.214 2007/06/13 14:56:52 mcnutt ; Bug 70 calls get_ictemps to fill CEB_T ; ; Revision 1.213 2007/06/11 15:18:15 secchia ; added if statement to skip cosmic report if missing form image ; ; Revision 1.212 2007/06/08 20:17:03 mcnutt ; corrcted extended header when files are missing Bug 182 and 179 ; ; Revision 1.211 2007/06/08 15:41:09 secchia ; nr - use nsubsi not subsi[0] for subsi tests; negate subsi and addsi in case of event detect (FSW 5.03) ; ; Revision 1.210 2007/06/06 17:25:21 nathan ; Remove comments for EUVI crpix; fix writing HI headers from last update; ; do sccimagestats. ; ; Revision 1.209 2007/06/05 16:40:55 nathan ; add counter arguments for testing HI IP values; create HIfile if HI and it ; does not exist; change location of and add logging if header-only and skipping ; to writedb; add previous values of pointing info for EUVI and COR1 ; ; Revision 1.208 2007/06/04 21:25:11 mcnutt ; Bug 138 fills extended header with skipped image headers and estimates exposure time when headers are missing and corrects filename when first header is missing ; ; Revision 1.207 2007/05/21 20:48:30 nathan ; check cadence.sav in case of input list (Bug 144); do not cpimgdb if /TESTLZ; ; reload stereo_spice only if not /NOWAIT ; ; Revision 1.206 2007/05/14 16:51:02 nathan ; move dospice to after hbtemps, and only do load_stereo_spice 1x if not lz ; ; Revision 1.205 2007/05/04 21:48:53 nathan ; implement cor1_point.pro; add info from proc computer to db filename ; ; Revision 1.204 2007/05/03 14:28:26 mcnutt ; Bug 100 and Bug 84 added misslist and nmissing for rice and HC ; ; Revision 1.203 2007/05/02 15:47:07 secchib ; nr - make sure to /RELOAD load_stereo_spice if att_file=NotFound (Bug 129) ; ; Revision 1.202 2007/04/27 22:55:03 secchib ; nr - do not set img_dir in secchi_rectify section ; ; Revision 1.201 2007/04/26 19:23:56 secchib ; fix typo ; ; Revision 1.200 2007/04/26 19:17:11 secchib ; nr - add /NOPRETTIES to avoid writing to earth ;__________________________________________________________________________________________________________ ; function getgndtstfile, basehdrstruct common scc_reduce, lulog, basehdr, nom_hdr, ext_hdr, scch, leapsecs, enums, resetvars, $ ipdat, nom_tlr, prevbasename, readfileidline, wavefileidline, $ setupfileidline, exposfileidline, maskfileidline, ipfileidline, icerdiv2flag, $ rotblidlines,hbtimes,hbtemps,cebtimes,cebtemps, subdir, date_fname_utc, ispb, islz, isrt, aorb, $ iserror, ephemfil, att_file filename1=strmid(string(basehdrstruct.filename),5,3) basehdrstruct.filename=[48,48,48,48,48,48,48,48,48,48,48,48] scilist=findfile("sw*.sci") nlist=n_elements(scilist) help,scilist IF (scilist[0] NE '') THEN for ii=0,nlist-1 do begin openr,3,scilist[ii] a = assoc(3,basehdrstruct) basehdrcopy = a[0] close,3 print,'Saw sci file ',scilist[ii],' for ',string(basehdrcopy.filename) printf,lulog,'Saw sci file ',scilist[ii],' for ',string(basehdrcopy.filename) if (strmid(string(basehdrcopy.filename),5,3) EQ filename1) THEN return,scilist[ii] ii=ii+1 endfor amessage="WARNING: Spacewire file for "+filename1+" not found !!!" print,' ' print,amessage print,' ' wait,5 return,amessage end ;__________________________________________________________________________________________________________ ; ; pro for recreating date link after processing so timestamp is current pro touchdatelink, destdir, img_dir,telj,yyyymmdd teldir=strlowcase(telj) IF teldir EQ 'hi1' THEN teldir='hi_1' IF teldir EQ 'hi2' THEN teldir='hi_2' linkdir='none' telimgdir=getenv(strupcase(teldir)+'_'+strupcase(img_dir)) IF strmid(telimgdir,0,25) NE strmid(destdir,0,25) THEN BEGIN ; create symbolic link in directory pointed to in $secchi linkdir=filepath( yyyymmdd,ROOT=destdir+'/'+img_dir+'/'+teldir) outdir =filepath( yyyymmdd,ROOT=telimgdir) ENDIF IF file_exist(outdir) THEN BEGIN IF linkdir NE 'none' THEN BEGIN spawn,'/bin/rm -f '+linkdir cmd='ln -s '+outdir+' '+linkdir print,cmd spawn,cmd,/sh ENDIF ENDIF end ; pro secchi_reduce, infilelist, WAITRATE=waitrate, NOWAIT=nowait, NOTELDIR=noteldir, TESTLZ=testlz, $ FAST=fast, NO_SPICE=no_spice, IPver=IPver, _extra=_extra,NOHBTEMPS=NOHBTEMPS, NOSUMMARY=nosummary, $ NOEXTHDR=noexthdr, NOSORT=nosort, HDR_ONLY=hdr_only, GZIP=gzip, NOPRETTIES=nopretties, $ DELETE_OLD=delete_old, ENDBADAH=endbadah, NOEVENT_NOTIFY=noevent_notify, FIND_HDR_SCRIPT=find_hdr_script, $ RESUME=resume, NOBKG=nobkg ;_extra is need for stereo spice to load attitude files from Mission Sim2 not needed after launch common scc_reduce;, lulog, basehdr, nom_hdr, ext_hdr, scch, leapsecs, enums, $ ; ipdat, nom_tlr,hbtimes,hbtemps ;common stereo_spice, def_ephem, ephem, attitude common stereo_spice_conic, mu, maxdate, conic common dbms,ludb, dbfile, delflag, img_seb_hdr, img_seb_hdr_ext, img_cal_bias, $ img_cal_led, img_cal_dark, img_hist_cmnt, img_euvi_gt ; ludb used by scc_db_insert.pro IF keyword_set(NO_SPICE) THEN dospice=0 ELSE dospice=1 version = '$Id: secchi_reduce.pro,v 1.299 2009/01/23 21:40:02 secchia Exp $' len=strlen(version) version = strmid(version,1,len-2) SPAWN,'hostname',machine,/sh machine = machine(N_ELEMENTS(machine)-1) src=strmid(machine,0,1)+rstrmid(machine,0,1) yyyymmdd='UND' lastyyyymmdd='UND' lastdobsyymmdd='UND' ymdd='UND' lastymdd='UND' resetvars=1 lulog=9 ;lusummary=10 HI1sttime=0 HI2sttime=0 HI1cmdtime=0 HI2cmdtime=0 HISTTIME=0 HISCANT='F' SIscant='F' delflag=0 ; Controls whether all database rows of date_obs are deleted first icerdiv2flag=0 iserror=0 ; Image Scheduling error / skipped image scitelvals=['HI1','COR2','COR1','EUVI','HI2','HI1','COR2','COR1','EUVI','HI2'] IF keyword_set(WAITRATE) THEN waittime = waitrate ELSE waittime = 60 keeplooking=1 seb_img = getenv ('SEB_IMG') destdir = getenv ('SECCHI') gzdestdir=getenv ('SECCHI_GZ') IF gzdestdir NE '' and keyword_set(GZIP) THEN dogz=1 ELSE dogz=0 ;sci_attic = getenv_slash('SCC_ATTIC') sci_attic = getenv('SCC_ATTIC') ;aorb=getenv('SCC_PLATFORM') sci_attic=sci_attic+'/' ;sci_attic= '/home/reduce/SECCHI/sci/attic/sci' ; *********KB unblock_cmd = concat_dir(getenv('UTIL_DIR'),'unblockSciFile') ;unblock_cmd= '/home/reduce/SECCHI/unblockSciFile' ; *********KB uncompr_cmd = concat_dir(getenv('UTIL_DIR'),'ricerecon64 ') huncompr_cmd = concat_dir(getenv('UTIL_DIR'),'hd64 -v ') iuncompr_cmd = concat_dir(getenv('UTIL_DIR'),'idecomp -v ') cpdb_cmd = concat_dir(getenv('ITOS_GROUPDIR'),'bin/cpimgdb ') source = getenv ('SCI_SOURCE') ; a string: p, r, or 0 if datatype(scch) NE 'STC' THEN scch = def_secchi_hdr() cadencefile= concat_dir(getenv('SCC_LOG'),'cadence.sav') IF strpos(seb_img,'redo') GT 0 THEN Begin cmd='rm '+cadencefile print,'REDO: ',cmd wait,5 spawn,cmd,/sh ; first CADENCE value will be incorrect ENDIF ; check savefile for last-processed file if input list if (findfile(cadencefile) NE '' and n_params() GT 0) and keyword_set(RESUME) then begin restore, cadencefile ;From make_scc_hdr.pro: save,filename=cadencefile,cadence,iphicrs,basename lastsci=where(strpos(infilelist,basename) GE 0,nlast) IF nlast GT 0 THEN BEGIN message,'Resuming where previously stopped with '+infilelist[lastsci[0]],/info wait,10 infilelist=infilelist[lastsci[0]:n_elements(infilelist)-1] ENDIF endif islz=0 ispb=0 isrt=0 pipeline='spw' IF strpos(destdir,'lz') GE 0 OR keyword_set(TESTLZ) THEN BEGIN islz=1 pipeline='lz' IF keyword_set(DELETE_OLD) THEN pipeline ='rz' ENDIF IF strpos(destdir,'pb') GE 0 THEN BEGIN ispb=1 pipeline='pb' ENDIF IF strpos(destdir,'rt') GE 0 THEN BEGIN isrt=1 pipeline='rt' ENDIF ;stop ndone=0L ng=0L nowaitset=0 IF n_params() GT 0 THEN BEGIN help,infilelist list=infilelist ng=n_elements(infilelist) ;nowaitset=1 if infilelist[0] EQ '' THEN BEGIN message,'Empty input; returning...',/info wait,5 return endif ENDIF IF keyword_set(NOWAIT) THEN nowaitset=1 ;--always work in $seb_img cd,seb_img spawn,'pwd',/sh wait,2 repeat begin IF n_params() GT 0 AND ndone LT ng THEN BEGIN ; ++++ file(s) to process is an argument ++++ infile=infilelist[ndone] break_file, infile, dlog, dir, sciroot, ext scifile=sciroot+ext fileorig=scifile help,ndone,ng,scifile ;keeplooking=0 ENDIF ELSE BEGIN ; ++++ Look for new files to process. ++++ again: IF (ndone EQ ng) THEN BEGIN ; either we just started, or we finished a batch IF ng GT 0 and (islz) THEN BEGIN ; ok, we just finished a batch ; get days in list pos0=rstrpos(list[0],'/')+1 allfnymdd=strmid(list,pos0,4) uymdd=uniq(allfnymdd) ndays=n_elements(uymdd) npd=lonarr(ndays) ; number of files per day FOR k=0,ndays-1 DO BEGIN if k EQ 0 THEN x=0 ELSE x=uymdd[k-1] npd[k]=uymdd[k]-x ENDFOR ymdds=allfnymdd[uymdd] ; get telescopes in list alltelid =strmid(list,pos0+11,1) tels=fix(alltelid[uniq(alltelid,sort(alltelid))]) ntl=n_elements(tels) ; for each day and tel in list, do daily processing FOR i=0,ndays-1 DO BEGIN tm=ymdd2utc(ymdds[i]) doy=utc2doy(tm) yyyymmdd=utc2yymmdd(tm,/yyyy) IF (doy+3) mod 7 EQ 0 THEN day4=1 ELSE day4=0 IF doy mod 7 EQ 0 THEN day7=1 ELSE day7=0 ; get days for scc_mk_all_monthly.pro tm1=tm ; get today minus 5 days tm1.mjd=tm1.mjd-5 tm5d=tm1 ; get today minus 30 days tm1.mjd=tm1.mjd-25 ; only subtract 25 since we already took off 5 tm30d=tm1 ; convert those dates to YYYYMMDD dend=utc2yymmdd(tm5d,/yyyy) dstart=utc2yymmdd(tm30d,/yyyy) FOR j=0,ntl-1 DO BEGIN telj=scitelvals[tels[j]] teln=strlowcase(telj) IF teln EQ 'hi1' THEN teln='hi_1' IF teln EQ 'hi2' THEN teln='hi_2' IF teln EQ 'euvi' THEN teln='e*' ;--Recreate link if it exists touchdatelink, destdir, 'cal',telj,yyyymmdd touchdatelink, destdir, 'img',telj,yyyymmdd touchdatelink, destdir, 'seq',telj,yyyymmdd IF npd[i] GT 50 THEN BEGIN IF j eq 0 then sort_summary,yyyymmdd,telj,/logdiff IF telj EQ 'COR2' THEN begin ; dopolar required for mk_daily_med TBR IF ~keyword_set(NOPRETTIES) THEN $ sccdopolar,utc2str(tm),/COR2, AHEAD=aheadf, BEHIND=behindf IF ~keyword_set(NOBKG) THEN BEGIN scc_mk_daily_med,'COR2',aorb,yyyymmdd,polar='0' scc_mk_daily_med,'COR2',aorb,yyyymmdd,polar='120' scc_mk_daily_med,'COR2',aorb,yyyymmdd,polar='240' IF ~keyword_set(NOPRETTIES) THEN BEGIN scc_mk_daily_med,'COR2',aorb,yyyymmdd,polar='TBR' IF (day4) or (day7) THEN $ scc_mk_all_monthly,'cor2',aorb,dstart,dend,'all' ENDIF ENDIF ENDIF IF telj EQ 'HI1' or telj EQ 'HI2' and ~keyword_set(NOBKG) THEN BEGIN scc_mk_daily_med,telj,aorb,yyyymmdd,polar='TBR' IF (day4) or (day7) THEN $ scc_mk_all_monthly,telj,aorb,dstart,dend,'tbr' ENDIF IF ~keyword_set(NOPRETTIES) THEN BEGIN scc_daily_pretties,yyyymmdd,telj,aorb IF telj EQ 'COR2' or telj EQ 'HI2' THEN scc_daily_pretties,yyyymmdd,telj,aorb,/minus30 IF telj EQ 'HI1' or telj EQ 'HI2' THEN begin scc_daily_pretties,yyyymmdd,telj,aorb,/minus30,/histars scc_daily_pretties,yyyymmdd,telj,aorb,/histars ENDIF ; Implement list of files for mpegs, Ahead only ; IF (aheadf and telj EQ 'HI1') THEN BEGIN moved to line 780 ; mpegframes,yyyymmdd,['euvi','cor2','hi1','hi2'] ;do all telescopes in hi1 ; daily_select_njpegs,aorb,yyyymmdd,telj ; foutdir=concat_dir(getenv('SECCHI_JPG'),'mpegframes') ; cmd='mv '+concat_dir(getenv('SCC_LOG'),'mpegframes')+'/'+teln+'.lst '+foutdir ; print,cmd ; IF (day7) THEN spawn,cmd,/sh ; IF ~ishi THEN IF (day4) THEN spawn,cmd,/sh ; openw,elun,'ipmsg.txt',/get_lun,/append ; ;printf,elun,'dbfile/session= '+aorb+' '+pipeline+' '+dbroot ; msg='njpegs: '+yyyymmdd+' '+telj+' '+trim(doy)+' day4='+trim(day4)+' day7='+trim(day7)+' i='+trim(i) ; msg2=' npd='+arr2str(trim(npd)) ; printf,elun,msg+' '+msg2 ; printf,elun,'' ; close,elun ; free_lun,elun ; ;printf,lulog,msg ; print,msg,' ',msg2 ; wait,5 ; ENDIF ENDIF ; END if ~keyword_set(NOPRETTIES) ENDIF ELSE BEGIN ; END if npd[i] GT 50 openw,elun,'ipmsg.txt',/get_lun,/append ;printf,elun,'dbfile/session= '+aorb+' '+pipeline+' '+dbroot msg='skipped pretties: '+yyyymmdd+' i='+trim(i)+' npd='+arr2str(trim(npd)) msg2='list='+arr2str(strmid(list,pos0,12)) printf,elun,msg printf,elun,msg2 printf,elun,'' close,elun free_lun,elun ;printf,lulog,msg print,msg wait,5 ENDELSE ENDFOR ; telescopes ENDFOR ; END FOR i=0,ndays-1 w=where(scitelvals[tels] EQ 'EUVI',iseuvi) IF (iseuvi) and (date_fname_utc.time GT 84600000) THEN begin ; assuming EUVI is always last to finish, this is when to compute archive stats, at the end of a day (after 23:30) sccimagestats,date_fname_utc,ahead=aheadf,behind=behindf ; create daily apid reports run only with the EUVI processing daily_ssr1_report,aorb,yyyymmdd,seb_img+'/pcktcnt/' call_comp_archive,aorb,yyyymmdd endif ENDIF ; do .0 files first, they are oldest list1 = file_search('????????.???', count=ng1) list2 = file_search('????????.???.?', count=ng2) IF ng1 EQ 0 THEN list=list2 ELSE $ IF ng2 EQ 0 THEN list=list1 ELSE list=[list2,list1] ng=ng1+ng2 IF ~keyword_set(NOSORT) THEN BEGIN fileinfo=file_info(list) msl=sort(fileinfo.mtime) ; sort by time of last mod list=list[msl] ENDIF ;g = where(strlen(list) EQ 12, ng) help,ng ndone=0L ; ++++ IF ng LT 1 THEN BEGIN ;IF nowaitset THEN return $ ;ELSE BEGIN ;print,'Waiting ',trim(string(waittime/60.)),' minutes before trying again ...' wait,waittime ndone=0L goto, again ;ENDELSE ENDIF ; END just finished a batch ENDIF ; END just started or finished a batch scifile=list[ndone] help,ndone,ng,scifile infile=scifile fileorig=scifile ENDELSE ; ++++ End new files to process. ++++ dir='' spawn,'pwd',/sh ndone=ndone+1 ;IF ng-ndone EQ 0 and nowaitset THEN keeplooking=0 parts = str_sep(scifile,'.') sciroot=parts[0] basename=sciroot scisfx =parts[1] dashpos=rstrpos(scifile,'.') strput,scifile,'-',dashpos ; ++++ Make sure file is done being written. ++++ waitcnt=0 IF ng LT 3 THEN REPEAT BEGIN lastsize=file_size(infile) print,'Waiting to make sure file is complete ...' wait, 2 metasize=file_size(infile+'.meta') IF metasize EQ 0 THEN REPEAT BEGIN print,'Waiting 20 sec for ',infile+'.meta' wait,20 waitcnt=waitcnt+1 metasize=file_size(infile+'.meta') ENDREP UNTIL (metasize GT 100) OR (waitcnt GT 30) ; wait 10 minutes for realtime image to come down IF waitcnt GT 30 THEN BEGIN openw,elun,'ipmsg.txt',/get_lun,/append msg=infile+'.meta did not complete' printf,elun,msg printf,elun,'' close,elun free_lun,elun ;printf,lulog,msg print,msg wait,2 ENDIF wait, 2 ENDREP UNTIL (file_size(infile) EQ lastsize) ; ++++ Process file header. ++++ scidat = scifile+'.dat' ; always written in current directory scihdr = scifile+'.hdr' scitlr = scifile+'.tlr' sciimg = scifile+'.img' ; always written in current directory spawn, unblock_cmd+' '+infile,/sh, unblockresult print,unblockresult spawn,'$UTIL_DIR/rdSh '+scihdr+ ' | grep ctrl.Version',versionline,/sh parts=str_sep(versionline,':') ; If there is a read error, core is dumped and output is empty. Assume hdr version 4. IF n_elements(parts) LT 2 THEN parts=[0,4] IF fix(parts[1]) EQ 0 THEN BEGIN spawn,'$UTIL_DIR/rdShv1 '+scihdr+ ' | grep ctrl.Version',versionline,/sh parts=str_sep(versionline,':') ENDIF IF fix(parts[1]) LT 3 THEN BEGIN versionline=versionline+': loading sebhdr_struct.inc' nomoffset=80 extoffset=192 @./sebhdr_struct.inc ENDIF ELSE IF fix(parts[1]) EQ 3 THEN BEGIN ;if datatype(ext_hdr) EQ 'UND' THEN $ versionline=versionline+': loading sebhdr3_struct.inc' nomoffset=96 extoffset=232 @./sebhdr3_struct.inc ENDIF ELSE BEGIN ;if datatype(ext_hdr) EQ 'UND' THEN $ versionline=versionline+': loading sebhdr4_struct.inc' nomoffset=88 extoffset=216 @./sebhdr4_struct.inc ENDELSE print,versionline IF NOT(file_exist(scihdr)) THEN spawn,'touch '+scihdr,/sh IF NOT(file_exist(scitlr)) THEN spawn,'touch '+scitlr,/sh print,'Reading ',scihdr openr,1,scihdr scitlrmissing='no' openr,2,scitlr hdrstat=fstat(1) tlrstat=fstat(2) readhdrinfo="hdrsize="+string(hdrstat.size)+" tlrsize="+string(tlrstat.size) IF (tlrstat.size GT 500) OR (tlrstat.size LT 10) THEN BEGIN close,2 command='cp '+scihdr+' '+scitlr print,command scitlrmissing='Trailer wrong size: '+command spawn, command,/sh openr,2,scitlr tlrstat=fstat(2) ENDIF IF tlrstat.size GT hdrstat.size THEN BEGIN mainlun=2 readhdrinfo=readhdrinfo+": Using "+scitlr+" for main header." ENDIF ELSE BEGIN mainlun=1 ENDELSE print,readhdrinfo a = assoc(mainlun,m_baseheader) basehdr = a[0] ; so it works on linux PCs: swap_endian_inplace,basehdr,/SWAP_IF_LITTLE_ENDIAN IF (tlrstat.size LT hdrstat.size) THEN BEGIN command='cp '+scihdr+' '+scitlr print,command spawn, command,/sh ENDIF ;b = assoc(2,m_baseheader) ;basetlr = b[0] IF ((hdrstat.size>tlrstat.size) GT 100) THEN BEGIN a = assoc(mainlun,m_nominalheader,nomoffset) nom_hdr = a[0] ; so it works on linux PCs: swap_endian_inplace,nom_hdr,/SWAP_IF_LITTLE_ENDIAN print,'Reading ',scitlr b = assoc(2,m_nominalheader,nomoffset) nom_tlr = b[0] ; so it works on linux PCs: swap_endian_inplace,nom_tlr,/SWAP_IF_LITTLE_ENDIAN ENDIF IF ((hdrstat.size>tlrstat.size) GT 300) THEN BEGIN a = assoc(mainlun,m_extendedheader,extoffset) ext_hdr = a[0] ; so it works on linux PCs: swap_endian_inplace,ext_hdr,/SWAP_IF_LITTLE_ENDIAN ;b = assoc(2,m_extendedheader,192) ;ext_tlr = b[0] ENDIF close,1 close,2 date_fname_utc =tai2utc(nom_hdr.cmdexptime, /nocorrect) yyyymmdd_hhmmss=utc2yymmdd(date_fname_utc,/yyyy,/hhmmss) yyyymmdd=strmid(yyyymmdd_hhmmss,0,8) yymmdd=strmid(yyyymmdd,2,6) IF yyyymmdd NE lastyyyymmdd THEN spawn,'mkdir -p '+sci_attic+yymmdd,/sh ymdd=strmid(scifile,0,4) ePlatform= [ $ 'STEREO' $ , 'STEREO_A' $ , 'STEREO_B'] aorb=strmid(ePlatform[nom_hdr.platformid],7,1) aorblc=strlowcase(aorb) IF aorb EQ 'A' THEN BEGIN ; these are used in call to sccdopolar aheadf=1 behindf=0 scname='ahead' ENDIF ELSE BEGIN aheadf=0 behindf=1 scname='behind' ENDELSE ; need to set actualexptime if Skipped image skippedimg=where(basehdr.ipcmdlog ge 254,skpim) IF (skpim gt 0) OR basehdr.actualexptime EQ 0 THEN BEGIN ; skipped image basehdr.actualexptime=nom_hdr.ipprocessingtime ; So it goes in database iserror=1 ENDIF date_obs_utc=tai2utc(basehdr.actualexptime, /nocorrect) dobsyymmdd=utc2yymmdd(date_obs_utc,/yyyy) yyyy=strmid(dobsyymmdd,0,4) mm =strmid(dobsyymmdd,4,2) ; ++++ start log file ++++ IF (dobsyymmdd NE lastdobsyymmdd) THEN BEGIN ; limit IP ERROR, SKIPPED and ABORT emails to 1 iserrormail=intarr(4) +0 ;remove hilow dir for day-2 rmhilow=filepath( utc2yymmdd(tai2utc(basehdr.actualexptime-(48l*60*60), /nocorrect),/yyyy),ROOT=getenv('seb_img')+'/hilow' ) spawn,'/bin/rm -r '+rmhilow if (aheadf and nom_hdr.TELESCOPEID ge 4 and ~keyword_set(NOPRETTIES)) then begin mpegframes,yyyymmdd,['euvi','cor2','hi1','hi2'] ;do all telescopes in hi processing scc_get_cor1_daily,yyyymmdd endif ; spawn,'mkdir -p '+sci_attic+yymmdd,/sh ;close,lulog ;free_lun,lulog ;++++ Copy previous day db scripts to lambda ++++ IF (islz OR isrt) THEN $ IF ( ~keyword_set(NOHBTEMPS) and $ ~keyword_set(TESTLZ) and $ strpos(seb_img,'testimg') LT 0 and $ strpos(seb_img,'edm') LT 0) THEN BEGIN print,'Running '+cpdb_cmd spawn,cpdb_cmd,/sh ENDIF ELSE BEGIN print,'Not running ',cpdb_cmd wait,4 ENDELSE ; go ahead and make db files for SPWX or PB, even though not (yet) using them dbroot=yyyymmdd+strmid(yyyymmdd_hhmmss,9,4) dbfile=concat_dir(getenv('SCC_DB'),'scc'+aorb+pipeline+src+dbroot+'sql.script') openw,ludb,dbfile,/get_lun dbfstat=fstat(ludb) IF dbfstat.size EQ 0 THEN BEGIN ; only at beginning of each script file printf,ludb,'use secchi_flight;' printf,ludb,'start transaction;' ENDIF get_utc,ustarttime starttime=utc2str(ustarttime,/ecs) printf,ludb,'# starting dbfile at '+starttime close,ludb free_lun,ludb logfile= concat_dir(getenv('SCC_LOG'),'scc'+aorb+pipeline+dobsyymmdd+'.log') openw,lulog,logfile,/append printf,lulog,version printf,lulog,' ' ;text=readlist('sebhdr_struct.inc') text=[';* DB Table IDL structure definition created by CPP2IDLSTRUCT.PRO', $ ';* from $stereo/fsw/sw_dev/utils/rdSh/hdr.h on Thu May 19 15:11:05 2005'] printf,lulog,'NOTE: sebhdr info is static -- need to generate from sebhdr4_struct.inc' printf,lulog,text[0] printf,lulog,text[1] printf,lulog,'starting log file at '+starttime printf,lulog,' ' printf,lulog,'SEB_IMG ',seb_img printf,lulog,'SCI_SOURCE ',source printf,lulog,'SECCHI ',destdir printf,lulog,'SCC_ATTIC ',sci_attic printf,lulog,unblock_cmd printf,lulog,uncompr_cmd,' or ' printf,lulog,huncompr_cmd,' or ' printf,lulog,iuncompr_cmd printf,lulog,'IDL_PATH ',getenv('IDL_PATH') printf,lulog,'TABLES ',getenv('TABLES') printf,lulog,'PT ',getenv('PT') printf,lulog,'DATE: ',scch.date printf,lulog,'===============================' ; Move any existing .sh files for previous day to attic spawn, 'ls -l '+lastymdd+'*.sh', lsres,/sh,COUNT=nls IF nls GT 0 THEN BEGIN ;openw,55,concat_dir(getenv('SCC_LOG'),'extra_sh_files.log'),/append ;for ii=0,nls-1 do printf,55,lsres[ii] ;close,55 ;command='mv *.sh '+sci_attic+strmid(lastyyyymmdd,2,6) ;print,command for ii=0,nls-1 do print,lsres[ii] print print,'Waiting 5 ...' wait, 5 ;spawn,command,/sh ;printf,lulog,command for ii=0,nls-1 do printf,lulog,lsres[ii] ENDIF ; If playback, Copy any .sh files from realtime attic to current directory ;IF (ispb) or (islz) THEN BEGIN IF not (islz) or keyword_set(FIND_HDR_SCRIPT) THEN BEGIN ; nr - matches what is below rtattic=getenv_slash('SCC_RTATTIC') command='cp -p '+rtattic+yymmdd+'/*.sh .' print,'%% secchi_reduce: ',command wait,5 spawn,command,/sh wait,5 ENDIF ; End of day activities IF file_exist('ipmsg.txt') and ~isrt THEN BEGIN maillist='nathan.rich@nrl.navy.mil lynn.mcnutt@nrl.navy.mil' IF pipeline EQ 'lz' THEN maillist=maillist+' dennis.wang@nrl.navy.mil' IF keyword_set(TESTLZ) THEN maillist='nathan.rich@nrl.navy.mil' spawn,'mailx -s Image_Processing_Errors '+maillist+' < ipmsg.txt',/sh spawn,'rm ipmsg.txt',/sh ENDIF lastymdd=strmid(sciroot,0,4) ; get one days worth of HB temp data for lz only and test_img for testing mode only IF (keyword_set(NOHBTEMPS) or NOT(islz) ) THEN BEGIN message,"WARNING: HB temps not retrieved from database!!!",/info print,'' print,'' IF not(keyword_set(FAST)) then wait,2 ENDIF ELSE begin euvitemps='HBEUVIAFTSHTRT,HBEUVIZONET,HBEUVIAFTMNTT,HBEUVISECMIRT,HBEUVIENTRT,HBEUVIFWDMNTT' cor1temps='HBCOR1ZONE2T,HBCOR1DOUB2T,HBCOR1POLDOUB1T,HBCOR1TUBEOCCT,HBCOR1ZONE1T' cor2temps='HBCOR2ZONE3T,HBCOR2RLYLNST,HBCOR2FLDLNST,HBCOR2HRMRRT,HBCOR2ZONE2T,HBCOR2ZONE1T' hitemps='HBHIBACKSTRT,HBHIFINT,HBHIZONE1T,HBHIZONE2T,HBHIFRNTSTRT,HBHIBASESTRT' ccdtemps='HBEUVICCDT,HBCOR1CCDT,HBCOR2CCDT,HBHI1CCDT,HBHI2CCDT' cebenctemps='HBSCIPCEBENCLT,HBHICEBENCLT' teltemp=euvitemps+','+cor1temps+','+cor2temps+','+hitemps+','+ccdtemps+','+cebenctemps stemps=get_hbtemps(aorb,teltemp,dobsyymmdd,res=0.1) if(datatype(stemps) ne 'STR')then begin tags=tag_names(stemps) & ntags=n_Elements(tags) timez=strmid(stemps.pckt_time,0,4)+strmid(stemps.pckt_time,5,2)+strmid(stemps.pckt_time,8,2)+'_'$ +strmid(stemps.pckt_time,11,2)+strmid(stemps.pckt_time,14,2)+strmid(stemps.pckt_time,17,2) hbtimes=yymmdd2utc(timez) hbtemps=fltarr(ntags-2,n_elements(stemps.(0))) for n=0,ntags-3 do hbtemps(n,*)=stemps.(n+2) endif ictemps=get_ictemps(aorb,dobsyymmdd,res=0.1) if(datatype(ictemps) ne 'STR')then begin tags=tag_names(ictemps) & ntags=n_Elements(tags) timez=strmid(ictemps.pckt_time,0,4)+strmid(ictemps.pckt_time,5,2)+strmid(ictemps.pckt_time,8,2)+'_'$ +strmid(ictemps.pckt_time,11,2)+strmid(ictemps.pckt_time,14,2)+strmid(ictemps.pckt_time,17,2) cebtimes=yymmdd2utc(timez) cebtemps=fltarr(ntags-2,n_elements(ictemps.(0))) for n=0,ntags-3 do cebtemps(n,*)=ictemps.(n+2) endif ENDELSE resetvars=1 ; re-query events database lastimgctr=basehdr.imgctr ; if (dospice eq 1) then begin IF ndone LE 1 OR islz THEN BEGIN print,'Calling load_stereo_spice' load_stereo_spice, _extra=_extra, /verbose, Reload=~nowaitset ENDIF date_obs =utc2str(date_obs_utc,/noz) print,'Calling get_stereo_spice_kernel /ATTITUDE...' ATT_FILE =get_stereo_spice_kernel(date_obs,aorb,/ATTITUDE) IF ATT_FILE EQ '' THEN ATT_FILE="NotFound" print,'Calling get_stereo_spice_kernel ...' EPHEMFIL =get_stereo_spice_kernel(date_obs,aorb) IF EPHEMFIL EQ '' THEN EPHEMFIL="NotFound" help,ephemfil,att_file IF ephemfil EQ 'NotFound' OR att_file EQ 'NotFound' THEN BEGIN msg=date_obs+': EPHEMFIL='+ephemfil+', ATT_FILE='+att_file print,'Waiting 10...' print wait,10 openw,elun,'attmsg.txt',/get_lun printf,elun,msg printf,elun,'' close,elun free_lun,elun IF (islz) and ~keyword_set(TESTLZ) THEN $ spawn,'mailx -s ATT_FILE_NotFound nathan.rich@nrl.navy.mil lynn.mcnutt@nrl.navy.mil < attmsg.txt',/sh ENDIF endif ;--For cor?_point and hi_point_v2 (gt slew data) ;--older than 45 days, then should look in SDS_FIN_DIR ; currently appears MOC is keeping 30 days of .fin files online, so last 60 days total. nr,6/26/07 IF date_obs_utc.mjd LT ustarttime.mjd-45 THEN $ sdspath=getenv('SDS_FIN_DIR')+'/'+aorblc+'/'+yyyy+'/'+mm ELSE $ sdspath=getenv('SDS_DIR')+'/'+scname+'/data_products/level_0_telemetry/secchi' endif ELSE BEGIN openw,lulog,logfile,/append ;openw,ludb,dbfile,/append,/get_lun IF basehdr.imgctr LT lastimgctr THEN resetvars=1 ; This means schedule and tables were read in lastimgctr=basehdr.imgctr ENDELSE ; daily activities printf,lulog,' ' printf,lulog,infile FOR i=0,n_elements(unblockresult)-1 DO printf,lulog,unblockresult[i] printf,lulog,versionline printf,lulog,readhdrinfo printf,lulog,'Processing time: ',scch.date printf,lulog,' ' lastyyyymmdd=yyyymmdd lastdobsyymmdd=dobsyymmdd ;++++ markers for two part images IP prog 120 and 121++++ lowhilow=where(basehdr.ipcmdlog eq 121, ishilow) ; use low word from HI summing buffer lowhihi=where(basehdr.ipcmdlog eq 120, ishihigh) ; use high word from HI summing buffer ; - img num is low num + 2000 ;NOTE that highword image will have BOTH lowhilow and lowhihi true ;BUG 78 to correct FSW error "What should be 51 is 101 and what should be 2098 is 2020" if ((lowhihi(0) gt -1) and (nom_hdr.p1col eq 101)) then nom_hdr.p1col=51 if ((lowhihi(0) gt -1) and (nom_hdr.p2col eq 2020)) then nom_hdr.p2col=2098 ;++++ check for CRS sigma to be set and save in cadence.sav++++ CRS2im=where(basehdr.ipcmdlog eq 31, crs2) IF (crs2) THEN BEGIN ; this only matters for 2-image CRS IF datatype(iphicrs) EQ 'UND' THEN BEGIN ; first time through ; check savefile for last known value cadencefile= concat_dir(getenv('SCC_LOG'),'cadence.sav') cadencetemp=findfile(cadencefile) if (cadencetemp eq '') then begin ; no cadence file, so set sigma to default iphicrs=124b ; 1 sigma crs filter cadence=dblarr(3,32767) -1 ; set var to complete savset endif ELSE restore, cadencefile ; cadence file exists and iphicrs has previous value or default value ENDIF ; else cadence and iphicrs are set with previous values and cadencefile must exist ; now we have a valid value for iphicrs in secchi_reduce level CRSval=where(basehdr.ipcmdlog ge 124 and basehdr.ipcmdlog le 128, sigmapresent) ;IF ~(sigmapresent) THEN use previous value for iphicrs, and no need to update saveset ; but do need to update ipcmdlog IF (sigmapresent) THEN BEGIN IF (CRSval[0] NE iphicrs) then begin ;only update if changed iphicrs=basehdr.ipcmdlog[CRSval[0]] save,filename=cadencefile,cadence,iphicrs,basename ENDIF ENDIF ELSE BEGIN ipar=basehdr.ipcmdlog ; 1st is always a destination, and last value is always compression ; so need to insert iphicrs in the middle basehdr.ipcmdlog=[ipar[0],iphicrs,ipar[1:18]] ENDELSE endif ;++++ check to see if image is part of sequence and set up ASCII table++++ hdronly=where(basehdr.ipcmdlog eq 5) ; Header Only clrHI1=where(basehdr.ipcmdlog eq 33,nch1) ; Clear HI-1 Summing Buffer clrHI2=where(basehdr.ipcmdlog eq 35,nch2) ; Clear HI-2 Summing Buffer HI1sumF=where(basehdr.ipcmdlog eq 114,nfsh1) ; Filtered_Add_hi_sumBuf1 HI2sumF=where(basehdr.ipcmdlog eq 115,nfsh2) ; Filtered_Add_hi_sumBuf2 HI1sum=where(basehdr.ipcmdlog eq 34,nsh1) ; Add image to HI-1 Summing Buffer HI2sum=where(basehdr.ipcmdlog eq 36,nsh2) ; Add image to HI-2 Summing Buffer HI1seqimg=where(basehdr.ipcmdlog eq 37, nhi1seqimg) ; Use HI-1 Summing Buffer as Output HI2seqimg=where(basehdr.ipcmdlog eq 38, nhi2seqimg) ; Use HI-2 Summing Buffer as Output SpWximg=where(basehdr.ipcmdlog eq 40, isbeacon) ; use SpaceWeather APID seqhdronly=0 dateobscmnt='' ;define dateobscmnt for sequence images get used if date obs is estimated. IF (nhi1seqimg+nhi2seqimg+nsh1+nsh2) GT 0 THEN BEGIN ; nr - correct for bad sebxsum in pre-?? images ; MUST be before make_scc_hdr basehdr.sebxsum = (basehdr.sebxsum)>2 basehdr.sebysum = (basehdr.sebysum)>2 ENDIF ;++++ populate FITS header ++++ fits_hdr = make_scc_hdr(source, FAST=fast, spice=dospice, IPver=IPver, ENDBADAH=endbadah) iperror=where(basehdr.ipcmdlog ge 253) IF ((iserror) or iperror(0) gt -1)THEN BEGIN if iperror eq -1 then isiperr=3 else isiperr=basehdr.ipcmdlog(iperror(0))-253 iserrmsgs=['ERROR','SKIPPED','ABORTED','HDR ONLY EXPTIME = 0'] iserrmsg=iserrmsgs(isiperr) date_cmd_utc =tai2utc(nom_hdr.CMDEXPTIME, /nocorrect) datecmd=utc2str(date_cmd_utc,/noz) iptime=utc2str(tai2utc(double(nom_hdr.ipprocessingtime), /nocorrect)) msg=string(basehdr.filename)+' DATE-CMD='+datecmd+', IPTIME='+iptime+' was '+iserrmsg+'!!!' openw,elun,'ipmsg.txt',/get_lun,/append ;printf,elun,'dbfile/session= '+aorb+' '+pipeline+' '+dbroot printf,elun,msg printf,elun,'' close,elun free_lun,elun printf,lulog,msg print,msg wait,5 ;goto, skipscifile ; Should be handled as header only! (ie, goes in database) IF (~keyword_set(NOEVENT_NOTIFY) and iserrormail(isiperr) eq 0)THEN BEGIN openw,evlun,'errmsg.txt',/get_lun printf,evlun,'******* IP '+iserrmsg+' *******' printf,evlun,'' printf,evlun,'pipeline=',string(pipeline,'(a8)') printf,evlun,'obsrvtry= ',scch.obsrvtry printf,evlun,'filename= ',scch.filename printf,evlun,'date_obs= ',scch.date_obs printf,evlun,'fileorig= ',scch.fileorig printf,evlun,'Critical Event= ',string(basehdr.CRITEVENT,'(z4)') if iperror(0) gt -1 then for nip=0,n_Elements(iperror)-1 do printf,evlun,'ipcmdlog',string(iperror(nip),'(i2.2)'),'=',basehdr.ipcmdlog(iperror(nip)) if iperror(0) eq -1 then printf,evlun,'Header only no exposure time recorded' printf,evlun,'' close,evlun free_lun,evlun print,'' spawn,'cat errmsg.txt',/sh wait,10 maillist='nathan.rich@nrl.navy.mil lynn.mcnutt@nrl.navy.mil dennis.wang@nrl.navy.mil' spawn,'mailx -s SECCHI-'+aorb+'_IP_'+iserrmsg+'_'+yymmdd+' '+maillist+' < errmsg.txt',/sh iserrormail(isiperr)=1 ENDIF ENDIF IF strmid(scch.detector,0,2) EQ 'HI' THEN ishi=1 ELSE ishi=0 IF (iserror) then begin IF (~keyword_set(NOEXTHDR) and basehdr.NUMINSEQ gt 1) then begin IF(basehdr.IMGSEQ gt 0 and nom_hdr.TELESCOPEID eq 5) then HI1sum=(1) IF(basehdr.IMGSEQ gt 0 and nom_hdr.TELESCOPEID eq 4) then HI2sum=(1) IF(basehdr.IMGSEQ eq 0 and nom_hdr.TELESCOPEID eq 5) then nch1=(1) ;nbr, 6/5/07 IF(basehdr.IMGSEQ eq 0 and nom_hdr.TELESCOPEID eq 4) then nch2=(1) ENDIF ENDIF IF (scch.detector EQ 'HI1' and basehdr.numinseq GT 1) THEN BEGIN HIfile=concat_dir(getenv('SCC_LOG'),'HI1seqtbl.sav') HIfile_m1=concat_dir(getenv('SCC_LOG'),'HI1seqtbl_m1.sav') ;ishi=1 ; set earlier ENDIF IF (scch.detector EQ 'HI2' and basehdr.numinseq GT 1) THEN BEGIN HIfile=concat_dir(getenv('SCC_LOG'),'HI2seqtbl.sav') HIfile_m1=concat_dir(getenv('SCC_LOG'),'HI2seqtbl_m1.sav') ;ishi=1 ENDIF IF ~keyword_set(NOEXTHDR) THEN $ IF (ishi and basehdr.numinseq GT 1 and basehdr.imgseq EQ 0) then begin print,'Initting HIseqtbl' wait,3 restore,HIfile save,filename=HIfile_m1,HIseqtbl,HIsttime,HIcmdtime,HIscant,HIfstimg,HIBias HIseqtbl=strarr(basehdr.numinseq) HIsttime=basehdr.actualexptime HIcmdtime=nom_hdr.cmdexptime HIfstimg=basehdr.TELESCOPEIMGCNT HIBias=lonarr(basehdr.numinseq) save,filename=HIfile,HIseqtbl,HIsttime,HIcmdtime,HIscant,HIfstimg,HIBias endif IF (isbeacon + ishihigh) EQ 0 THEN $ if (nfsh1+nsh1+nfsh2+nsh2) GT 0 or (ishi and basehdr.numinseq GT 1) then begin IF ~keyword_set(NOEXTHDR) THEN BEGIN restore,HIfile if(basehdr.TELESCOPEIMGCNT lt HIfstimg and basehdr.actualexptime gt HIsttime)then HIcurimg=basehdr.TELESCOPEIMGCNT+long(65535) else HIcurimg=basehdr.TELESCOPEIMGCNT if(nch1+nch2 EQ 0)then begin ;nbr, 6/5/07 ; not first image zx=where(hiseqtbl ne '') seqtime =-1 & clrhiseq=0 if(n_elements(zx) gt 1)then seqtime=(float(strmid(hiseqtbl(zx(1)),0,8))-float(strmid(hiseqtbl(zx(0)),0,8)))*n_elements(hiseqtbl) if(seqtime gt -1 and basehdr.actualexptime gt HIsttime+seqtime)then clrhiseq=1 if(HIcurimg gt HIfstimg+n_elements(hiseqtbl))then clrhiseq=1 if(clrhiseq eq 1 and ~(iserror))then begin save,filename=HIfile_m1,HIseqtbl,HIsttime,HIcmdtime,HIscant,HIfstimg,HIBias HIseqtbl=strarr(basehdr.numinseq) HIsttime=basehdr.actualexptime HIcmdtime=nom_hdr.cmdexptime HIfstimg=basehdr.TELESCOPEIMGCNT HIBias=lonarr(basehdr.numinseq) endif if(basehdr.actualexptime lt HIsttime and ~(iserror))then begin HIfile=HIfile_m1 restore,HIfile if(basehdr.TELESCOPEIMGCNT lt HIfstimg and basehdr.actualexptime gt HIsttime)then HIcurimg=basehdr.TELESCOPEIMGCNT+long(65535) else HIcurimg=basehdr.TELESCOPEIMGCNT zx=where(hiseqtbl ne '') seqtime =-1 & clrhiseq=0 if(n_elements(zx) gt 1)then seqtime=(float(strmid(hiseqtbl(zx(1)),0,8))-float(strmid(hiseqtbl(zx(0)),0,8)))*n_elements(hiseqtbl) if(seqtime gt -1 and basehdr.actualexptime gt HIsttime+seqtime)then clrhiseq=1 if(HIcurimg gt HIfstimg+n_elements(hiseqtbl))then clrhiseq=1 if(basehdr.actualexptime lt HIsttime)then clrhiseq=1 if(clrhiseq eq 1)then begin HIseqtbl=strarr(basehdr.numinseq) HIsttime=basehdr.actualexptime HIcmdtime=nom_hdr.cmdexptime HIfstimg=basehdr.TELESCOPEIMGCNT HIBias=lonarr(basehdr.numinseq) endif endif endif HIBias(basehdr.imgseq)=long(basehdr.meanbias/1000.) if(lowhihi(0) eq -1) then make_Scc_ext_tbl,HIseqtbl,HIsttime,basehdr,nom_hdr,basehdr.imgseq, FAST=fast,dir=sdspath if(HIcmdtime ne 0)then date_fname_utc=tai2utc(HIcmdtime, /nocorrect) if(HI1seqimg(0) gt -1 or HI2seqimg(0) gt -1)then begin seqhdronly=0 ;the following is done twice once here to get the file name correct when the first header is missing then again for exposure after the header is created if(hiseqtbl(0) eq '')then begin hifilled=where(hiseqtbl ne '') if (n_Elements(hifilled) gt 1)then hicadence=(float(strmid(hiseqtbl(hifilled(0)+1),0,8))-float(strmid(hiseqtbl(hifilled(0)),0,8)))/(hifilled(1)-hifilled(0)) if (n_Elements(hifilled) le 1)then hicadence=60. ;set default if all image headers are missing HIcmdtime=nom_hdr.cmdexptime-(hicadence*(basehdr.numinseq-1)) HIsttime=basehdr.actualexptime-(hicadence*(basehdr.numinseq-1)) date_fname_utc=tai2utc(HIsttime, /nocorrect) endif nom_hdr.cmdexptime=HIcmdtime date_cmd_utc =tai2utc(nom_hdr.cmdexptime, /nocorrect) fxaddpar,fits_hdr,'DATE-CMD',utc2str(date_cmd_utc,/noz) scch.date_cmd=utc2str(date_cmd_utc,/noz) fxaddpar,fits_hdr,'filename',utc2yymmdd(date_fname_utc,/yyyy,/hhmmss)+strmid(scch.filename,15,10) scch.filename = utc2yymmdd(date_fname_utc,/yyyy,/hhmmss)+strmid(scch.filename,15,10) fxaddpar,fits_hdr,'offsetcr',total(HIBias(where(HIBias gt 0)))/n_elements(where(HIBias gt 0)) scch.offsetcr = total(HIBias(where(HIBias gt 0)))/n_elements(where(HIBias gt 0)) endif ENDIF if ((strmid(string(nom_hdr.PREEXPSCSTATUS, '(b16.16)'),8,1) EQ 1) or $ (strmid(string(nom_hdr.POSTEXPSCSTATUS,'(b16.16)'),8,1) EQ 1)) then HISCANT='T' if(hdronly(0) gt -1)then begin seqhdronly=1 if(HI1sumF(0) gt -1 or HI1sum(0) gt -1)then lsthi1img=basehdr.imgseq else lsthi2img=basehdr.imgseq endif IF ~keyword_set(NOEXTHDR) THEN save,filename=HIfile,HIseqtbl,HIsttime,HIcmdtime,HIscant,HIfstimg,HIBias endif ;IF (ishi and basehdr.numinseq GT 1 and basehdr.imgseq EQ 0) then stop ; if(seqhdronly eq 1)then goto, skipscifile ;check to see if image is stored or created from SAVE area tagSI=where(basehdr.ipcmdlog ge 56 and basehdr.ipcmdlog le 60) ; Tag as Saved Image 0-4 addSI=where(basehdr.ipcmdlog ge 61 and basehdr.ipcmdlog le 65, naddsi) ; Add Saved Image 0-4 to Current Image subSI=where(basehdr.ipcmdlog ge 66 and basehdr.ipcmdlog le 70, nsubsi) ; Subtract Saved Image 0-4 from Cur Img evtd =where(basehdr.ipcmdlog EQ 4, nevtd) nsubsi=nsubsi-nevtd ; nr - Negate presence of subsi if Event Detect exists (FSW 5.03) naddsi=naddsi-nevtd relSI=where(basehdr.ipcmdlog ge 71 and basehdr.ipcmdlog le 75, nrelSI) ; Release Saved Image 0-4 IF (~keyword_set(NOEXTHDR) and SpWximg(0) eq -1) THEN if(tagSI(0) gt -1)then begin ;save header structures for use in extended table SIbasehdr=basehdr & SInom_hdr=nom_hdr SIfile=concat_dir(getenv('SCC_LOG'),'SI'+string(basehdr.ipcmdlog(tagSI)-56,'(i1.1)')+'_hdr.sav') save,filename=SIfile,SIbasehdr,SInom_hdr endif IF (~keyword_set(NOEXTHDR) and SpWximg(0) eq -1) THEN if (naddsi+nsubsi GT 0)then begin tmpSI=([addSi,subSI]) & tmpSI=tmpSI(where(tmpSI gt -1)) tmpSI=basehdr.ipcmdlog(tmpSI)-61 if(max(tmpSI) gt 4)then tmpSI(where(tmpSI gt 4))=tmpSI(where(tmpSI gt 4))-5 SIseqtbl=strarr(n_Elements(tmpSI)+1) SIsttime=dblarr(n_elements(tmpSI)) SIcmdtime=dblarr(n_elements(tmpSI)) for nadd=0,n_elements(tmpSI)-1 do begin ;read in time first to get expo order SIfile=concat_dir(getenv('SCC_LOG'),'SI'+string(tmpSI(nadd),'(i1.1)')+'_hdr.sav') print,'Searching for ',sifile flchck=findfile(SIfile) if(flchck ne '')then begin SIbasehdr=-1 & SInom_hdr=-1 restore,SIfile SIsize=size(SIbasehdr) if(SIsize(2) eq 8) then SIsttime(nadd)=SIbasehdr.actualexptime SIsize=size(SInom_hdr) if(SIsize(2) eq 8) then SIcmdtime(nadd)=SInom_hdr.cmdexptime endif endfor SIsttime=[SIsttime,basehdr.actualexptime] ;add current image to starttime incase other images are missing. SIcmdtime=[SIcmdtime,nom_hdr.cmdexptime] ;add current image to cmndtime incase other images are missing. gdsi=where(SIsttime gt 0) ;check for and fill in missing hdr times if n_elements(gdsi) le n_elements(tmpSI) then begin tbtwnexp=-1 if n_Elements(gdsi eq 1)then begin ;use OBS_ID to get time between exposures if scch.obs_id eq 1693 or scch.obs_id eq 1694 or scch.obs_id eq 1715 then tbtwnexp=12 if scch.obs_id eq 1695 or scch.obs_id eq 1698 or scch.obs_id eq 1701 then tbtwnexp=10 if scch.obs_id eq 1707 then tbtwnexp=20 endif else tbtwnexp=round((SIsttime(gdsi(1))-SIsttime(gdsi(0)))/(gdsi(1)-gdsi(0))) if tbtwnexp eq -1 then begin openw,evlun,'missedhdr.txt',/get_lun printf,evlun,'******* unknown OBS_ID for summed image *******' printf,evlun,'Can not estimate times!!!' printf,evlun,'Using 10 secs, Fits file may need correcting' printf,evlun,'pipeline=',string(pipeline,'(a8)') printf,evlun,'obsrvtry= ',scch.obsrvtry printf,evlun,'filename= ',scch.filename printf,evlun,'date_obs= ',scch.date_obs printf,evlun,'fileorig= ',scch.fileorig printf,evlun,'' close,evlun free_lun,evlun wait,10 maillist='nathan.rich@nrl.navy.mil lynn.mcnutt@nrl.navy.mil' spawn,'mailx -s SECCHI-'+aorb+'_'+yymmdd+'_unknown_obs-id_COR2 '+maillist+' < missedhdr.txt',/sh tbtwnexp=10 endif ztsi=where(SIsttime eq 0) SIsttime(ztsi)=SIsttime(n_elements(SIsttime)-1)-tbtwnexp*(n_elements(SIsttime)-(ztsi+1)) SIcmdtime(ztsi)=SIcmdtime(n_elements(SIcmdtime)-1)-tbtwnexp*(n_elements(SIcmdtime)-(ztsi+1)) if ztsi(0) eq 0 then begin dateobscmnt='Estimated time - first HDR missing' fxaddpar,fits_hdr,'COMMENT','DATE-OBS and DATE-CMD are estimated first HDR missing' endif endif sttime=min(SIsttime) SIcmdtime=min(SIcmdtime) SIsttime=SIsttime(0:n_elements(SIsttime)-2); remove current image starttime before sorting tmpSI=tmpSI(sort(SIsttime)) SISCANT='F' for nadd=0,n_elements(tmpSI)-1 do begin ;read in header save files for extended table SIfile=concat_dir(getenv('SCC_LOG'),'SI'+string(tmpSI(nadd),'(i1.1)')+'_hdr.sav') print,'Searching for ',sifile flchck=findfile(SIfile) if(flchck ne '')then begin SIbasehdr=-1 & SInom_hdr=-1 restore,SIfile SIsize=size(SIbasehdr) if(SIsize(2) eq 8)then begin make_Scc_ext_tbl,SIseqtbl,sttime,SIbasehdr,SInom_hdr,nadd,Fast=fast,dir=sdspath if ((strmid(string(SInom_hdr.PREEXPSCSTATUS,'(b16.16)'),8,1) EQ 1) $ or (strmid(string(SInom_hdr.POSTEXPSCSTATUS,'(b16.16)'),8,1) EQ 1))then SISCANT='T' endif endif endfor ;add current image to extended header make_Scc_ext_tbl,SIseqtbl,sttime,basehdr,nom_hdr,n_elements(tmpSI),Fast=fast,dir=sdspath ; update cmdexptime to first exposure of summed image nom_hdr.cmdexptime=SIcmdtime date_cmd_utc =tai2utc(nom_hdr.cmdexptime, /nocorrect) date_fname_utc =tai2utc(nom_hdr.cmdexptime, /nocorrect) if dateobscmnt eq '' then fxaddpar,fits_hdr,'DATE-CMD',utc2str(date_cmd_utc,/noz) else fxaddpar,fits_hdr,'DATE-CMD',utc2str(date_cmd_utc,/noz),dateobscmnt scch.date_cmd=utc2str(date_cmd_utc,/noz) fxaddpar,fits_hdr,'filename',utc2yymmdd(date_fname_utc,/yyyy,/hhmmss)+strmid(scch.filename,15,10) scch.filename = utc2yymmdd(date_fname_utc,/yyyy,/hhmmss)+strmid(scch.filename,15,10) endif ;++++correct bais for eventdetect images bug 210 if ((evtd(0) gt -1) and (scch.biasmean lt 400)) then fxaddpar,fits_hdr,'BIASMEAN',scch.biasmean*2 ;+++ notify of IP event +++ IF ~keyword_set(NOEVENT_NOTIFY) THEN BEGIN ;evfile=getenv_slash('SCC_LOG')+'critevent.sav' ;restore,evfile ; = lastcritevent critevent=string(basehdr.CRITEVENT,'(z4)') ; DW says event does not persist. IF scch.event EQ 'T' THEN BEGIN openw,evlun,'evmsg.txt',/get_lun printf,evlun,'******* CME EVENT DETECTED *******' printf,evlun,'' printf,evlun,'pipeline=',string(pipeline,'(a8)') printf,evlun,'obsrvtry= ',scch.obsrvtry printf,evlun,'filename= ',scch.filename printf,evlun,'date_obs= ',scch.date_obs printf,evlun,'fileorig= ',scch.fileorig printf,evlun,'ip_00_19=',scch.ip_00_19 printf,evlun,'' printf,evlun,'event= ',string(scch.event,'(a8)') printf,evlun,'evcount= ',string(scch.evcount,'(a8)') printf,evlun,'evrow= ',string(scch.evrow,'(i8)') printf,evlun,'evcol= ',string(scch.evcol,'(i8)') printf,evlun,'critEvent=',string(critevent,'(a7)') close,evlun free_lun,evlun print,'' spawn,'cat evmsg.txt',/sh wait,10 spawn,'mailx -s SECCHI-'+aorb+'_Event_Detected'+yymmdd+' sccevent@cronus.nrl.navy.mil < evmsg.txt',/sh ENDIF ;lastcritevent=critevent ;save,lastcritevent,filename=evfile ENDIF ; -- end notify of IP event -- fxaddpar,fits_hdr,'HISTORY',version fxaddpar,fits_hdr,'HISTORY','Processed on '+machine+': '+!version.arch+' '+!version.os_name+' IDL'+!version.release IF (scitlrmissing NE 'no') THEN fxaddpar,fits_hdr,'HISTORY',scitlrmissing IF (iserror) THEN fxaddpar,fits_hdr,'HISTORY','IS error SKIPPED detected' fxaddpar,fits_hdr,'FILEORIG',fileorig ;if (dospice eq 1) then fxaddpar,fits_hdr,'ATT_FILE',ATT_FILE if ((nhi1seqimg+nhi2seqimg+naddsi+nsubsi GT 0) and SpWximg(0) eq -1 and lowhihi(0) eq -1) then begin exttblhdr = def_secchi_ext_hdr() ; fxaddpar,exttblhdr,'XTENSION','TABLE ',before='SIMPLE' ; sxdelpar,exttblhdr,'SIMPLE' if (HI1seqimg(0) gt -1 or HI2seqimg(0) gt -1)then numin_seq=basehdr.numinseq*1 else numin_seq=n_Elements(tmpSI) fxaddpar,exttblhdr,'NAXIS2',numin_seq fxaddpar,fits_hdr,'EXTEND','T' fxaddpar,fits_hdr,'N_IMAGES',numin_seq ; defaults to 1 in make_scc_hdr exptime=fxpar(fits_hdr,'EXPTIME',comment=expcmt) if (HI1seqimg(0) gt -1 or HI2seqimg(0) gt -1) then BEGIN if(HISTTIME ne 0)then firsttime=HIsttime else firsttime=basehdr.actualexptime IF datatype(HIseqtbl) EQ 'UND' THEN BEGIN exp0=(scch.imgseq+1)*scch.exptime expcmt='ESTIMATED -- some HDR0 files missing' ENDIF ELSE IF (n_Elements(hiseqtbl) ne n_elements(where(hiseqtbl ne '')))then begin hiunfilled=where(hiseqtbl eq '') hifilled=where(hiseqtbl ne '') ; ushdrs=hifilled(where(shift(hifilled,-1)-hifilled eq 1)) hiexp0=strmid(hiseqtbl(hifilled(0)),9,7) ; hicadence=float(strmid(hiseqtbl(ushdrs(0)+1),0,8))-float(strmid(hiseqtbl(ushdrs(0)),0,8)) if (n_Elements(hifilled) gt 1)then hicadence=(float(strmid(hiseqtbl(hifilled(0)+1),0,8))-float(strmid(hiseqtbl(hifilled(0)),0,8)))/(hifilled(1)-hifilled(0)) if (n_Elements(hifilled) le 1)then hicadence=60. ;set default if all image headers are missing if(hiseqtbl(0) eq '')then firsttime=basehdr.actualexptime-(hicadence*(basehdr.numinseq-1)) hiseqtbl(hiunfilled)=string(hicadence*hiunfilled,'(f8.3)')+string(hiexp0,'(f7.3)')+string(' ','(a169)') if (where(hiunfilled eq 0) gt -1) then hiseqtbl(hifilled)=string(hicadence*hifilled,'(f8.3)')+strmid(hiseqtbl(hifilled),8,strlen(hiseqtbl(hifilled(0)))-8) exp0=(scch.imgseq+1)*scch.exptime expcmt='ESTIMATED -- some HDR0 files missing' ENDIF ELSE exp0=total(float(strmid(HIseqtbl,9,7))) fxaddpar,fits_hdr,'EXPTIME',exp0,expcmt scch.exptime=exp0 endif if (naddsi+nsubsi GT 0) then BEGIN if(STTIME ne 0)then firsttime=sttime else firsttime=basehdr.actualexptime IF datatype(SIseqtbl) EQ 'UND' THEN BEGIN exp0=(naddsi+nsubsi+1)*scch.exptime expcmt='ESTIMATED -- some HDR0 files missing' ENDIF ELSE IF (n_Elements(SIseqtbl) ne n_Elements(where(SIseqtbl ne '')))then begin exp0=n_Elements(SIseqtbl)*scch.exptime expcmt='ESTIMATED -- some HDR0 files missing' ENDIF ELSE BEGIN exp0=total(double(strmid(SIseqtbl,9,10))) expcmt=' sum of'+expcmt ENDELSE scch.N_images=naddsi+nsubsi+1 fxaddpar,fits_hdr,'N_IMAGES',naddsi+nsubsi+1 fxaddpar,fits_hdr,'EXPTIME',exp0,expcmt scch.exptime=exp0 endif first_date_obs=utc2str(tai2utc(firsttime, /nocorrect),/noz) if dateobscmnt eq '' then fxaddpar,fits_hdr,'DATE-OBS',first_date_obs else fxaddpar,fits_hdr,'DATE-OBS',first_date_obs,dateobscmnt scch.DATE_OBS=first_date_obs date_av=tai2utc(((firsttime+basehdr.actualexptime+(basehdr.actualexpduration*4.0e-6))/2), /nocorrect) DATE_AVG=utc2str(date_av,/noz) scch.date_avg=date_avg fxaddpar,fits_hdr,'DATE-AVG',DATE_AVG if (HI1seqimg(0) gt -1 or HI2seqimg(0) gt -1 and hiscant eq 'T') then fxaddpar,fits_hdr,'SCANT_ON','T' if (naddsi+nsubsi GT 0 and SIscant eq 'T') then fxaddpar,fits_hdr,'SCANT_ON','T' endif ;COR2 space weather images not summed 10-30-2007 if (naddsi-nsubsi GT 0 and SpWximg(0) gt -1) then BEGIN ; exp0=(naddsi+nsubsi+1)*scch.exptime ; fxaddpar,fits_hdr,'EXPTIME',exp0 fac=(naddsi-nsubsi+1) newbias=scch.biasmean/fac IF newbias GT 500 THEN BEGIN msg='% SECCHI_REDUCE correcting '+infile+' bias '+trim(scch.biasmean)+' to '+trim(newbias) printf,lulog,msg print,msg wait,2 scch.biasmean=newbias fxaddpar,fits_hdr,'BIASMEAN',newbias,' divided by '+trim(fac)+' from basehdr' fxaddpar,fits_hdr,'COMMENT','workaround correction for Bug 252 applied to BIASMEAN' ENDIF endif ;++++ Begin derive outdir ;select SUB directory for image img_dir set to img ; if (seqhdronly eq 0) then begin img_dir='img' case strmid(scch.filename,16,1) of 'k' : IF (scch.detector NE 'HI1') AND (scch.detector NE 'HI2') THEN img_dir='cal' ; DARK 'e' : img_dir='cal' ; LED 'c' : img_dir='cal' ; CONT else : endcase if(trim(strlowcase(scch.detector)) eq 'cor1' or trim(strlowcase(scch.detector)) eq 'cor2')then begin case strmid(scch.filename,16,1) of 'n' : img_dir='seq' 's' : img_dir='seq' else: endcase endif ;create SUB directory name and make hin directories hi_n ;eDoorStatus : $ ;[ 'CLOSED', $ ; 'IN_TRANSIT', $ ; 'OPEN ', $ ; 'OFF', $ ; 'ERROR', $ ; replicate('invalid',250), $ ; 'NO_DATA' ] } ; SCIP door CLOSED or IN_TRANSIT go in cal; HI door CLOSED (only) go in cal IF ((scch.doorstat NE 2) AND (scch.detector NE 'HI1') AND (scch.detector NE 'HI2')) OR $ ((scch.doorstat EQ 0) AND ((scch.detector EQ 'HI1') OR (scch.detector EQ 'HI2'))) $ THEN img_dir='cal' case trim(strlowcase(scch.detector)) of 'hi1' : BEGIN teldir='hi_1' END 'hi2' : BEGIN teldir='hi_2' END else : teldir=trim(strlowcase(scch.detector)) endcase subdir=img_dir+'/'+teldir IF keyword_set(NOTELDIR) THEN $ outdir =filepath( yyyymmdd,ROOT=destdir ) ELSE $ outdir =filepath( yyyymmdd,ROOT=destdir,SUBD=SUBDir ) ;++++ End derive outdir imsize=0 ; by default no image misslist='' nmissing=-1 missn=-1 nblocks=0 IF keyword_set(HDR_ONLY) and (scch.comprssn NE 5) THEN BEGIN indir=getenv_slash('secchi')+'lz/L0/'+strlowcase(aorb)+rstrmid(outdir,0,18) ;stop finfil=concat_dir(indir,scch.filename) print,finfil printf,lulog,'Reading '+finfil IF file_exist(finfil) THEN BEGIN image=sccreadfits(concat_dir(indir,scch.filename),khdr) ;scch.rectify=khdr.rectify goto,afterdecomp ENDIF ELSE BEGIN print,'File does not exist--continuing with decomp.' printf,lulog,'File does not exist--continuing with decomp.' ENDELSE ENDIF IF (scch.comprssn EQ 5) OR NOT(file_exist(scidat)) THEN BEGIN ; Header only image=0 fxaddpar,fits_hdr,'naxis',0 fxaddpar,fits_hdr,'naxis1',0 fxaddpar,fits_hdr,'naxis2',0 scch.filename=strmid(scch.filename,0,16)+'h'+strmid(scch.filename,17,8) ;change fits file name to h for header only database entry fxaddpar,fits_hdr,'filename',scch.filename printf,lulog,sciimg+' is header only' print,sciimg+' is header only' IF (scch.comprssn NE 5) THEN BEGIN msg=sciimg+' ('+scch.filename+') is not supposed to be header only!!!' IF ~isrt THEN BEGIN openw,elun,'ipmsg.txt',/get_lun,/append printf,elun,'dbfile/session= '+aorb+' '+pipeline+' '+dbroot printf,elun,msg printf,elun,'' close,elun free_lun,elun ENDIF printf,lulog,msg print,msg wait,5 ENDIF msg='Not writing '+scch.fileorig+'/'+scch.filename+'; skipping to db scripts' print,msg IF ~keyword_set(FAST) THEN wait,2 printf,lulog,msg goto, writedb ENDIF IF (strmid(sciimg,0,7) EQ 'WARNING') THEN BEGIN ; Bad spacewire file image=0 fxaddpar,fits_hdr,'naxis1',0 fxaddpar,fits_hdr,'naxis2',0 printf,lulog,sciimg goto, skipscifile ENDIF ELSE BEGIN ; start process Normal image ; ++++ process image ++++ fc_counts=parsefcmetafile(infile) fcmissing=fc_counts.packets_expected-fc_counts.packets_found fcpkts=fc_counts.packets_expected fcerrs=fc_counts.nerrors segqual=0 cfline='No compression error message detected.' IF keyword_set(HDR_ONLY) THEN BEGIN message,'HDR_ONLY set -- NMISSING and MISSLIST not derived from decomp output!',/info ENDIF ELSE IF scch.comprssn EQ 7 THEN BEGIN ; ++++ Rice compression command=uncompr_cmd+' '+scidat+' '+sciimg print,command spawn,command ,uncompresult,/sh print,uncompresult nres=n_elements(uncompresult) FOR i=0L,nres-1 DO BEGIN IF strlen(uncompresult[i]) GT 500 $ THEN prlin=strmid(uncompresult[i],0,120)+'...' $ ELSE prlin=uncompresult[i] printf,lulog,prlin IF strpos(prlin,'error') GE 0 THEN cfline='ricerecon64: '+prlin ENDFOR nblkx=(scch.naxis1 / 64) nblky=(scch.naxis2 / 64) ;scch.naxis1 = nblkx * 64 ; if compressed, naxis must be mult of 64 ;scch.naxis2 = nblky * 64 ; ++++ parse result from ricerecon64 ++++ IF scch.naxis1 LE 0 or scch.naxis1 GT 2176 THEN BEGIN stmts=str_sep(uncompresult[2],' ') ; Method=1 BCol1=0 BCol2=15 BRow1=0 BRow2=15 ncol=1024 nrow=1024 IF n_elements(stmts) EQ 7 THEN BEGIN res1=execute(stmts[5]) res2=execute(stmts[6]) IF (res1 and res2) THEN BEGIN scch.naxis1=ncol scch.naxis2=nrow ENDIF ENDIF ENDIF ; ++++ extract missing blocks ++++ parts=strsplit(strmid(uncompresult[nres-3],2,strlen(uncompresult[nres-3])),/extract) ; --iiii out of iiii blocks were good-- gdbk=FIX(parts(0)) nblocks=FIX(parts(3)) bfac=1. IF ((nhi1seqimg+nhi2seqimg GT 0) and (lowhilow(0) eq -1) and (lowhihi(0) eq -1) and (SpWximg(0) eq -1)) THEN bfac=2. ; Array is square and has 4-byte elements if(gdbk ne nblocks)then nmissing = ceil((bfac*nblkx*nblky+bfac - gdbk)/bfac) else nmissing=0 scch.nmissing=nmissing ; stop nblocks=bfac*nblkx*nblky fxaddpar,fits_hdr,'NMISSING',scch.nmissing ; ++++ extract compression factor ++++ cfline=uncompresult[nres-4] ; Without packet overhead bpp=ff.ffffff CF=f.ffffff parts=str_sep(cfline,'=') IF (scch.nmissing EQ 0) THEN fxaddpar,fits_hdr,'COMPFACT', float(parts[2]), ' Without packet overhead' ENDIF ELSE IF scch.comprssn GT 7 and scch.comprssn LT 18 THEN BEGIN ; ++++ H-compress command=huncompr_cmd+' < '+scidat+' > '+sciimg+' 2> hd64.out' print,command spawn, command,/sh uncompresult=readlist('hd64.out') print,uncompresult nres=n_elements(uncompresult) FOR i=0L,nres-1 DO BEGIN prlin=uncompresult[i] printf,lulog,prlin IF strpos(prlin,'error') GE 0 THEN cfline='hd64: '+prlin ENDFOR nblkx=ceil(scch.naxis1 / 64.) nblky=ceil(scch.naxis2 / 64.) nblocks=nblkx*nblky ;stop ;scch.naxis1 = nblkx * 64 ; if compressed, naxis must be mult of 64 ;scch.naxis2 = nblky * 64 ; it doesn't work this way -- NR, 1/4/07 ; ++++ extract missing blocks ++++ ;gdbkline=uncompresult[nres-3] ; --iiii out of iiii blocks were good-- ;pos=strpos(gdbkline,' ') ;gdbk=FIX(strmid(gdbkline,2,pos)) ;scch.nmissing = nblkx*nblky+1 - gdbk ;fxaddpar,fits_hdr,'NMISSING',scch.nmissing ; ++++ extract compression factor ++++ cfline=uncompresult[nres-1] ; Compression Factor = 0.474 bpp = 33.74951 parts=strsplit(cfline,/extract) np=n_elements(parts) ;IF (scch.nmissing EQ 0) THEN ;for i=0,np-1 do print,i,' ',parts[i] IF np GE 7 THEN $ fxaddpar,fits_hdr,'COMPFACT', float(parts[6]) , ' from hd64 output' ELSE $ IF (not(keyword_set(FAST))) THEN wait, 5 ENDIF ELSE IF scch.comprssn GT 89 and scch.comprssn LT 102 THEN BEGIN ; ++++ Icer expectedbytes=1 readbytes=0 sgmt=0 icererr=0 nseg=0 msg3='' command=iuncompr_cmd+' -i '+scidat+' -o '+sciimg print,command spawn, command,uncompresult,/sh ;print,uncompresult nres=n_elements(uncompresult) IF nres EQ 1 THEN icererr=1 IF nres EQ 1 THEN goto, icererrstuff ;--Parse uncompresult print ; 0 Verbose ON(1) ; 1 Input file: 7602012C-548.dat ; 2 Output file: 7602012C-548.img ; 3 nseg: 32 status: 0 status1: 0 i=3 print,uncompresult[i] parts=strsplit(uncompresult[i],/extract) nseg=fix(parts[1]) ;help,nseg nmbk=nseg IF nseg GT 64 or nseg LE 0 THEN icererr=1 IF nseg GT 64 or nseg LE 0 THEN goto, icererrstuff nbits=fltarr(nseg) dataseg=lonarr(nseg+1) ; bytes segstat=intarr(nseg) segqual=dblarr(nseg) ; 4 Status: ; 5 Status1: ; 6 nbits[0]: 3040 i=5 print,uncompresult[i] IF strpos(uncompresult[i],'ICER_DIV2_FLAG') GT 0 THEN BEGIN icerdiv2flag=1 cfline='ICER_DIV2_FLAG detected in idecomp' ENDIF i=i+icerdiv2flag i0=i+1 ; If there is a Status1 message, there is a blank line after it REPEAT BEGIN i=i+1 parts=strsplit(uncompresult[i],/extract) IF strmid(parts[0],0,4) EQ 'nbit' THEN nbits[i-i0]=long(parts[1]) ENDREP UNTIL strmid(parts[0],0,4) NE 'nbit' ; 37 nbits[31]: 3488 ; 38 tot_nbits: 3257120 bytes: 407140 ; 39 Read 474452 bytes ; 40 dataseg[0] 0 bytes IF strpos(uncompresult[i],'error') GE 0 THEN icererr=1 IF (icererr) THEN goto, icererrstuff print,uncompresult[i:i+1] parts=strsplit(uncompresult[i],/extract) expectedbytes=long(parts[3]) i=i+1 parts=strsplit(uncompresult[i],/extract) readbytes=long(parts[1]) dataseg[nseg]=readbytes i0=i+1 REPEAT BEGIN i=i+1 parts=strsplit(uncompresult[i],/extract) IF strmid(parts[0],0,7) EQ 'dataseg' THEN dataseg[i-i0]=long(parts[1]) ENDREP UNTIL strmid(parts[0],0,7) NE 'dataseg' IF strpos(uncompresult[i],'error') GE 0 THEN icererr=1 IF (icererr) THEN goto, icererrstuff ; 71 dataseg[31] 406704 bytes ; 72 ; 73 decompressing... ; 74 Segment statuses: ; 75 segment 0: 0 print,uncompresult[i+1:i+2] i=i+2 i0=i+1 REPEAT BEGIN i=i+1 parts=strsplit(uncompresult[i],/extract) sgmt=i-i0 IF strmid(parts[0],0,7) EQ 'segment' THEN segstat[sgmt]=fix(parts[2]) ENDREP UNTIL strmid(parts[0],0,7) NE 'segment' IF strpos(uncompresult[i],'error') GE 0 THEN icererr=1 IF (icererr) THEN goto, icererrstuff ; 106 segment 31: 2 ; 107 Status word: 0 ; 108 Width: 2048 height: 2048 ; 109 Wavelet filter: 0 Number of decompositions: 4 Number of segments: 32 ; 110 (ystart, xstart, height, width) ; 111 segment 0: (0, 0, 320, 400); quality: 5.000000e+00 print,uncompresult[i:i+2] i=i+3 i0=i+1 REPEAT BEGIN i=i+1 parts=strsplit(uncompresult[i],/extract) IF strmid(parts[0],0,7) EQ 'segment' THEN segqual[i-i0]=double(parts[7]) ENDREP UNTIL strmid(parts[0],0,7) NE 'segment' IF strpos(uncompresult[i],'error') GE 0 THEN icererr=1 IF (icererr) THEN goto, icererrstuff print,'' nmissing=0 FOR j=1,nseg DO BEGIN sgmt=j-1 ; --Test each segment completeness=100.*8*(dataseg[j]-dataseg[j-1])/nbits[j-1] qual=segqual[sgmt] print,'dataseg',string(j-1,'(i3)'),' is ', $ string(completeness,'(f6.2)'),'% of nbits ', $ 'and quality ',trim(segqual[j-1]),' and status ',trim(segstat[j-1]) ; --Compute MISSLIST here IF (qual GT 7 or qual LT 0.1) $ or segstat[sgmt] NE 0 $ or completeness NE 100 THEN BEGIN IF segstat[sgmt] EQ 0 $ or (qual LT 7 and qual GT 1) THEN BEGIN msg3=' *** Check seg '+trim(sgmt) ;icererr=1 ENDIF ;--Do not count as missing if no file_capture errors are detected (Bug 81) IF fc_counts.packets_found EQ 0 or fcerrs GT 0 THEN BEGIN miss34=inttobn(sgmt,34) misslist=misslist+string(miss34,format='(a2)') nmissing=nmissing+1 ENDIF ELSE msg3=' *** bad quality detected but 0 FC errors (not marked as missing)' ENDIF ENDFOR ;IF ~icererr THEN goto, after_icererr icererrstuff: FOR i=0L,nres-1 DO BEGIN prlin=uncompresult[i] IF (icererr) THEN print,prlin printf,lulog,prlin IF strpos(prlin,'error') GE 0 THEN cfline='idecomp: '+prlin IF nres EQ 1 and prlin EQ '' THEN cfline='idecomp: Segmentation fault?' ENDFOR after_icererr: IF expectedbytes NE readbytes and nmissing EQ 0 THEN BEGIN icererr=1 nmissing=1 misslist='0' ENDIF ;stop IF nmissing LT 0 THEN nmissing=nseg fxaddpar,fits_hdr,'NMISSING',nmissing,'out of '+trim(string(nseg))+' segments' scch.nmissing=nmissing IF nmissing GT 0 THEN BEGIN fxaddpar,fits_hdr,'MISSLIST',misslist,'base34 fmt=(a2); use bntoint(x,34)' ENDIF IF sgmt+1 NE nseg THEN icererr=1 IF (icererr) or (msg3 NE '') THEN BEGIN msg1='segments counted='+trim(sgmt+1)+', segments reported='+trim(string(nseg)) msg2='expectedbytes='+trim(expectedbytes)+', readbytes='+trim(readbytes) print,scidat,' ',msg1 print,scidat,' ',msg2 print,scidat,' ',cfline+msg3 print,scidat,' NMISSING='+trim(nmissing)+': '+misslist print IF ~isrt THEN BEGIN print,'Waiting 10...' wait,10 openw,elun,'ipmsg.txt',/get_lun,/append printf,elun,scidat+' idecomp error: dbfile/session= '+aorb+' '+pipeline+' '+dbroot printf,elun,msg1 printf,elun,msg2 printf,elun,cfline+msg3 printf,elun,' NMISSING='+trim(nmissing)+': '+misslist printf,elun,'' close,elun free_lun,elun ENDIF ENDIF nblkx=ceil(scch.naxis1 / 64.) nblky=ceil(scch.naxis2 / 64.) ;scch.naxis1 = nblkx * 64 ; if compressed, naxis must be mult of 64 ;scch.naxis2 = nblky * 64 IF (not(keyword_set(FAST))) THEN wait, 5 ENDIF ELSE IF scch.comprssn EQ 43 THEN BEGIN ; Gnd Test image print,scch.fileorig,' IPCMD:',nom_hdr.ipcmd IF (not(keyword_set(FAST))) THEN wait, 5 sciimg=getgndtstfile(basehdr) ENDIF ELSE sciimg=scidat ; ++++ End uncompress image msg1=fileorig+', '+ipdat[scch.comprssn]+', nmissing=, '+trim(nmissing)+', numblk=, '+trim(nmbk)+', fcpkts=, '+trim(fcpkts)+', fcmissing=, '+trim(fcmissing)+', fcerrs=, '+trim(fcerrs)+', maxqual=, '+trim(max(segqual)) printf,lulog,msg1 IF scch.nmissing EQ 0 and (fcmissing GT 0 or fcerrs GT 0) THEN BEGIN openw,elun,'ipmsg.txt',/get_lun,/append printf,elun,msg1 printf,elun,'' close,elun free_lun,elun ENDIF ;--Treat missing sciimg file and sciimg file of size zero the same sciimgok=0 IF FILE_EXIST(SCIIMG) THEN BEGIN openr,1,sciimg imgstat=fstat(1) IF imgstat.size GT 0 THEN sciimgok=1 ENDIF IF (sciimgok) THEN BEGIN ;*** Added by Thompson *** ; ++++ Space Weather case where pcol/prow may be undefined IF (hdrstat.size LT 100) THEN BEGIN scch.naxis1=64*(fix(sqrt(imgstat.size/2)/64)) ; SPWX images should always be square scch.naxis2=scch.naxis1 ENDIF ;stop IF ((nhi1seqimg+nhi2seqimg GT 0) and (lowhilow(0) eq -1) and (lowhihi(0) eq -1) and (SpWximg(0) eq -1)) THEN BEGIN ; Array is square and has 4-byte elements ;scch.naxis1=64*(fix(sqrt(imgstat.size/4)/64)) ;--updated 2/14/07, nbr side=scch.naxis1