;======================== header =============================================== ;+ ; NAME: ; image_select ; PURPOSE: ; FUNCTION that selects the good c2 and c3 files and delete the bad ones ; it keeps a log file of the deleted + reason of refusal ; CATEGORY: ; cactus ver 4.00 (paperA&A-version, May 2004) ; CALLING SEQUENCE: ; (meant to be used in batchmode) ; INPUTS: ; ; ; KEYWORD PARAMETERS: ; /logfile: to print diagnostics to a logfile (default=stdout=screen) ; OUTPUTS: ; ; COMMON BLOCKS: ; none ; SIDE EFFECTS: ; unknown ; RESTRICTIONS: ; unknown ; PROCEDURE: ; ; CALLS: ; ; ; TO DO: ; ; current version crashes when all tested files are bad ; ; MODIFICATION HISTORY: ; 2002/04/25, ver 1.00, David Berghmans ; created on the basis of db_c2_select ; 2002/11/21, DB, now doing both c2 and c3 ; 2004/01/23, DB & ER, ratio testing added ; bug corrected, now moving bad files to archdir/'telescope'/ ; 2005/03/17, ER Extra test on header.datap01 (first percentile of the image) ; + rewrite log-file ; 2006/05/05 GRL Re-named cactus_select from image_select; cast 'properly' for SSW ;- ;======================== end header =========================================== FUNCTION cactus_select, filelist, logfile=logfile, telescope COMMON CACTUS_ENVS ; ------------------------ preamble ------------------------------------ ; Who am I? myname ='image_select' ; main input/output ; cactusdir=getenv('CACTUSDIR') ; archdir = cactusdir+'/archive/' ; the output directory were we put cleaned images ; logdir = cactusdir+'/log/' CASE (logfile) OF 0 : log=-1 1 : openw, log, cactuslog+'cactus_log.txt' , /get_lun, /append 2 : openw, log, cactuslog+ myname+'_log.txt', /get_lun, /append ENDCASE ; log information ; mylogfile=logdir+myname+'.log' ; if logfile keyword is set, open the logfile, if not send to screen: ; IF keyword_set(logfile) THEN openw, log, mylogfile, /get_lun , /append $ ; ELSE log=-1 ; say hello printf, log, ' >>>> '+myname+'.pro started at '+ systime() ; sanity check IF (size(telescope))(1) ne 7 THEN BEGIN printf, log, 'Sanity check failure: telescope not properly defined' exit ENDIF IF (telescope ne 'c2') and (telescope ne 'c3') and $ (telescope ne 'C2') and (telescope ne 'C3') THEN BEGIN printf, log, 'Sanity check failure: telescope unknown' printf, log, '***** telescope=', telescope,' ***************************************' exit ENDIF telescope=STRLOWCASE(telescope) printf, log, '***** telescope=', telescope,' ***************************************' ;------------------------- Selection criteria ------------------------------- IF telescope eq 'c2' THEN BEGIN ; setting for checking type of image nx = 1024 ; required x size of input images ny = 1024 ; required y size of input images filt ='Orange' ; required filter wheel position of input images polar ='Clear' ; required polar wheel position of input images lp_num ='Normal' ; required lp_num ;setting for sanity check min_exptime = 25.0 ; minimal exposure time required ; (lowest I have ever seen was a corrupted image with 17.0854) max_datazer = 186368 ; maximum number of datazer ; ; note that missing blocks in itself do not hurt cactus but a large number of ; missing blocks is an indication for possible corrupteness of the image ; ; (good image has hdr.datazer EQ 62464 which is the ; equivalent of 61 blocks) min_dataavg = 2350 ; minimal data average ; lowest 'good image' I have ever seen 2461 ; highest 'corrupted image' I have ever seen 2317.65 min_ratio = 0. ; mimimal value for the ratio dataavg/exptime ; unused max_ratio = 9999. ; maximal value for the ratio dataavg/exptime ; unused min_datap01 = 850. ; First percentile of the image is typically 918. ; This extra test is to exclude images which have many ; dark pixelc which are not black ENDIF; telescope eq 'c2' IF telescope eq 'c3' THEN BEGIN ; setting for checking type of image nx = 1024 ; required x size of input images ny = 1024 ; required y size of input images filt ='Clear' ; required filter wheel position of input images polar ='Clear' ; required polar wheel position of input images lp_num ='Normal' ; required lp_num ;setting for sanity check min_exptime = 18.0 ; minimal exposure time required ; (lowest I have ever seen was a corrupted image with 17.0854) max_datazer = 186368 ; maximum number of datazer ; ; note that missing blocks in itself do not hurt cactus but a large number of ; missing blocks is an indication for possible corrupteness of the image ; ; (good image has hdr.datazer EQ 101376 which is the ; equivalent of 99 blocks) ; (120000 threshold removed 78 out of 4206 image in a test) min_dataavg = 1800 ; minimal data average ; .....unused..... ; .....unused..... min_ratio = 87. ; mimimal value for the ratio dataavg/exptime ; (typical values are between 97 and 108) max_ratio = 118. ; maximal value for the ratio dataavg/exptime ; (typical values are between 97 and 108) min_datap01 = 365. ; ; This extra test is to exclude images which have many ; dark pixelc which are not black ENDIF; telescope eq 'c3' Printf, log, 'Quality is 1 if the following conditions are satisfied: ' printf, log, ' A : exptime > ',STRTRIM(min_exptime,2) printf, log, ' B : datazer < ',STRTRIM(max_datazer,2) printf, log, ' C : dataavg > ',STRTRIM(min_dataavg,2) printf, log, ' D : dataavg/exptime in [',STRTRIM(min_ratio,2),',',STRTRIM(max_ratio,2),']' printf, log, ' E : datap01 > ',STRTRIM(min_datap01,2) printf, log, 'Filename | date | time ||Accept||Type |Quality|| A B C D E | ' printf, log, '--------------------------------------------------------------------------' ;----------------------------------------------------------------------------- ; some variables maxnimages=n_elements(filelist) ; this is a max since there some files will be dropped good_names = ' ' ; will contain the selected filenames nbad=0 ntype=0 nnormal=0 ; loop over all candidate files FOR i= 0,maxnimages-1 DO BEGIN filename=filelist[i] ; printf, log, 'filename : ', filename a=lasco_readfits(filename,hdr, /NO_IMG,/SILENT) IF (DATATYPE(hdr) EQ 'STC') THEN BEGIN ; check if image is of the correct type type_ok = (hdr.naxis1 EQ nx) and (hdr.naxis2 EQ ny) and $ (hdr.filter EQ filt) and (hdr.polar EQ polar) and $ (hdr.lp_num EQ lp_num) ; calculate ratio ( of data average to exposure time) ratio = hdr.dataavg/hdr.exptime ; ratio is Floating point number ; check if main parameters are within expectations normal=(hdr.exptime GT min_exptime) and $ (hdr.datazer LT max_datazer) and $ (hdr.dataavg GT min_dataavg) and $ (ratio GT min_ratio) and (ratio LT max_ratio) and $ (hdr.datap01 GT min_datap01) ; keep file or remove it IF (type_ok AND normal) THEN good_names= [good_names,filename] $ ELSE BEGIN ; spawn, 'mv '+filelist[i]+' '+archdir+telescope+'/' nbad=nbad+1 ntype=ntype+(1-type_ok*1) nnormal=nnormal+(1-normal*1) ENDELSE ; log info ; printf, log, ' obstime : '+hdr.date_obs+' '+hdr.time_obs ; printf, log, ' Accepted? :', (type_ok AND normal) ; printf, log, ' type ok? :', type_ok ; printf, log, ' within specs? :', normal ; printf, log, ' exptime = ',(hdr.exptime GT min_exptime) , ' ',hdr.exptime ; printf, log, ' datazer = ',(hdr.datazer LT max_datazer) , ' ',hdr.datazer ; printf, log, ' dataavg = ',(hdr.dataavg GT min_dataavg) , ' ',hdr.dataavg ; printf, log, ' ratio = ',(ratio GT min_ratio) , ' ',ratio ; printf, log, ' ratio = ',(ratio LT max_ratio) , ' ',ratio ; printf, log, ' datap01 = ',(hdr.datap01 GT min_datap01) , ' ',hdr.datap01 printf, log, hdr.filename,' | ',hdr.date_obs,' | ',STRMID(hdr.time_obs,0,8),' || '$ , STRTRIM((type_ok AND normal)*1,2),' || ',STRTRIM((type_ok)*1,2),' | ',STRTRIM((normal)*1,2),' || '$ , STRTRIM((hdr.exptime GT min_exptime)*1,2) ,' '$ , STRTRIM((hdr.datazer LT max_datazer)*1,2) ,' '$ , STRTRIM((hdr.dataavg GT min_dataavg)*1,2) ,' '$ , STRTRIM(((ratio GT min_ratio) AND (ratio LT max_ratio))*1,2),' ' $ , STRTRIM((hdr.datap01 GT min_datap01)*1,2) ,' |' ENDIF ELSE printf, log, ' fitsfile error' ENDFOR; i printf, log, '--------------------------------------------------------------------------' printf, log, 'Removed ||',STRMID(100000000+nbad,7,5),' ||',STRMID(100000000+ntype,7,5),'| ' $ ,STRMID(100000000+nnormal,7,5),' || ' ; chop unused part of array if n_elements(good_names) eq 1 then undefine,filelist else filelist = good_names[1:*] ; ------------------------closing remarks------------------------------- ; say goodbye printf, log, ' <<<< '+myname+'.pro finished at '+ systime() ; close log file if needed IF keyword_set(logfile) THEN BEGIN close, log free_lun, log ENDIF ; ---------------------------------------------------------------------- return, filelist END