-----Original Message----- From: Jim Sullivan [mailto:sullivan@cluster.bu.edu] Sent: Tuesday, September 10, 2002 9:43 AM To: Theodore Fritz Subject: I have automated the downloading of level 0 and its processing into BU level 2 IDL savesets: PRO GETPWG, CAMMICE=cammice, NOL2=nol2, ORBIT=orbit, ATTITUDE=attitude, $ DEF=def, STOP=stop, CEPPAD=ceppad ; ; Fetch new POLAR ATTITUDE, ORBIT, CAMMICE, or CEPPAD level zero data from PWG ; Save level zero files locally and make level two savesets ; lkw = KEYWORD_SET(attitude) OR KEYWORD_SET(cammice) OR KEYWORD_SET(ceppad) OR $ KEYWORD_SET(orbit) IF NOT lkw THEN BEGIN PRINT,'GETPWG: Fetch POLAR (compressed) level zero data from PWGdata' PRINT,' and make BU level two savesets' PRINT,' Arguments:' PRINT,' Keywords: Only one data set may be selected at at time' PRINT,' ATTITUDE If set, fetch (predict) attitude data (cdf files)' PRINT,' CAMMICE If set, fetch CAMMICE level zero data' PRINT,' CEPPAD If set, fetch CEPPAD level zero data (D)' PRINT,' DEF If set, fetch definitive orbit/attitude data' PRINT,' NOL2 If set, do not make level two savesets' PRINT,' ORBIT If set, fetch (predict) orbit data (cdf files)' PRINT,' Environment variables:' PRINT,' CAML2PATH CAMMICE level two path '+GETENV('CAML2PATH') PRINT,' CAMLZPATH CAMMICE level zero path '+GETENV('CAMLZPATH') PRINT,' CEPL2PATH CEPPAD level two path '+GETENV('CEPL2PATH') PRINT,' CEPLZPATH CEPPAD level zero path '+GETENV('CEPLZPATH') PRINT,' ORATPATH Orbit/attitude path '+GETENV('ORATPATH') PRINT,'' RETURN ENDIF ; Get list of files currently on hand orat = 0 clzpath = GETENV('CEPLZPATH') IF KEYWORD_SET(cammice) THEN clzpath=GETENV('CAMLZPATH') IF KEYWORD_SET(orbit) OR KEYWORD_SET(attitude) THEN BEGIN orat = 1 ; Assumes only one directory in orbit/attitude path clzpath = GETENV('ORATPATH') IF KEYWORD_SET(def) THEN clzpath=clzpath+'/def' ELSE clzpath=clzpath+'/pre' IF KEYWORD_SET(orbit) THEN clzpath=clzpath+'/or' ELSE clzpath=clzpath+'/at' ENDIF IF KEYWORD_SET(ceppad) THEN clzpath=GETENV('CEPLZPATH') lzpath = STR_SEP(clzpath,':') ; get full list of local files (excluding sfdu files) kkcnt = 0L FOR i=0,N_ELEMENTS(lzpath)-1 DO BEGIN SPAWN,'\ls -l '+lzpath[i]+'/*',list,COUNT=lcnt,/SH kcnt = 0L FOR j=0L,lcnt-1 DO BEGIN indx = STRPOS(list[j],'sfdu') IF orat EQ 1 THEN jndx=STRPOS(list[j],'.dat') ELSE jndx=-1 IF indx EQ -1 AND jndx EQ -1 AND $ N_ELEMENTS(STR_SEP(STRCOMPRESS(list[j]),' ')) EQ 9 THEN BEGIN fname = STR_SEP(list[j],'/') IF kcnt EQ 0 THEN $ filename=fname[N_ELEMENTS(fname)-1] ELSE $ filename=[filename,fname[N_ELEMENTS(fname)-1]] kcnt = kcnt + 1L ENDIF ENDFOR IF kcnt GT 0 THEN IF kkcnt EQ 0 THEN $ datafile=filename ELSE datafile=[datafile,filename] kkcnt = kkcnt + kcnt IF N_ELEMENTS(lzpath) GT 1 THEN PRINT,lzpath[i],kcnt ENDFOR ; datafile contains the (nonprefixed) file names already held IF N_ELEMENTS(lzpath) EQ 1 THEN $ PRINT,kkcnt,' local LZ files in '+lzpath[0] ELSE $ PRINT,'Total number of local LZ files',N_ELEMENTS(datafile) IF kkcnt EQ 0 THEN GOTO,Return ; Check PWG ground system over all years since launch for new data files CALDAT,SYSTIME(/JULIAN),moy,dom,curyear,hod,moh,som ; fetch year by year to mimic PWG storage scheme FOR year=1996L,LONG(curyear) DO BEGIN ; nfile is the number of new (not locally available) files for this year nfile = 0L ; prepare an ftp command file to get the list of files OPENW,lun,/GET_LUN,'/usr/local/home/data/pwgftp.cmd' PRINTF,lun,'open pwgdata.gsfc.nasa.gov' ; the file .netrc contains all login information PRINTF,lun,'cd pub/compressed/po' IF KEYWORD_SET(cammice) THEN BEGIN PRINTF,lun,'cd cam/lz' ENDIF ELSE IF orat EQ 1 THEN BEGIN IF KEYWORD_SET(def) THEN PRINTF,lun,'cd def' ELSE PRINTF,lun,'cd pre' IF KEYWORD_SET(orbit) THEN PRINTF,lun,'cd or' ELSE PRINTF,lun,'cd at' ENDIF ELSE BEGIN PRINTF,lun,'cd cep/lz' ENDELSE PRINTF,lun,'cd '+STRING(year,FORMAT='(I4.4)') PRINTF,lun,'dir' PRINTF,lun,'quit' CLOSE,lun FREE_LUN,lun cmd = '/usr/bin/ftp < /usr/local/home/data/pwgftp.cmd' Spawn,cmd,dirlist,COUNT=cnt,/SH ; Scan for new files FOR i=0L,cnt-1L DO BEGIN inline = STRCOMPRESS(dirlist[i]) list = STR_SEP(inline,' ') IF N_ELEMENTS(list) NE 9 THEN GOTO,Skip lastfile = list[8] ; check file name against current list tabulate new files ; note version numbers are within the file names ; so don't need to check creation dates or checksums ; ORBIT/ATTITUDE files stored ungzipped locally so remove .gz sufffix IF orat EQ 1 THEN BEGIN indx = STRPOS(list[8],'cdf') IF indx EQ -1 THEN GOTO,Skip tlist = STRMID(list[8],0,STRPOS(list[8],'.gz')) ENDIF ELSE tlist=list[8] indx = WHERE(tlist EQ datafile,icnt) IF icnt EQ 0 AND STRLEN(list[8]) GT 26 THEN BEGIN IF nfile EQ 0 THEN filelist=list[8] ELSE filelist=[filelist,list[8]] nfile = nfile + 1L ENDIF Skip: ENDFOR ; log year, number of files, number of new files, and last file name PRINT,year,cnt,nfile,' '+lastfile ; prepare an ftp command file to get the new files for this year IF nfile GT 0 THEN BEGIN OPENW,lun,/GET_LUN,'/usr/local/home/data/pwgftp.cmd' PRINTF,lun,'open pwgdata.gsfc.nasa.gov' PRINTF,lun,'cd pub/compressed/po' IF KEYWORD_SET(cammice) THEN BEGIN PRINTF,lun,'cd cam/lz' ENDIF ELSE IF orat EQ 1 THEN BEGIN IF KEYWORD_SET(def) THEN PRINTF,lun,'cd def' ELSE PRINTF,lun,'cd pre' IF KEYWORD_SET(orbit) THEN PRINTF,lun,'cd or' ELSE PRINTF,lun,'cd at' ENDIF ELSE BEGIN PRINTF,lun,'cd pub/compressed/po/cep/lz' ENDELSE PRINTF,lun,'cd '+STRING(year,FORMAT='(I4.4)') PRINTF,lun,'binary' ; stage data through local disk /RAID4 PRINTF,lun,'lcd /RAID4' FOR j=0L,nfile-1 DO PRINTF,lun,'get '+filelist[j] PRINTF,lun,'quit' CLOSE,lun FREE_LUN,lun cmd = '/usr/bin/ftp < /usr/local/home/data/pwgftp.cmd' Spawn,cmd,result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] ; files contains the list of all new files downloaded IF N_ELEMENTS(files) EQ 0 THEN files=filelist ELSE files=[files,filelist] ENDIF ENDFOR SPAWN,'\rm /usr/local/home/data/pwgftp.cmd',result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] ; if there are new files process (into BU level-two savesets) IF N_ELEMENTS(files) GT 0 THEN BEGIN IF KEYWORD_SET(cammice) THEN BEGIN ; CAMMICE SPAWN,'\mv /RAID4/po_*cam*gz '+lzpath[0],result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] caml2path = STR_SEP(GETENV('CAML2PATH'),':') FOR k=0L,N_ELEMENTS(files)-1 DO BEGIN list = STR_SEP(files[k],'_') READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)' ydoy = Ydoyymd([yy,mm,dd]) IF NOT KEYWORD_SET(nol2) THEN $ Makemicsl2,yy,ydoy(1),ydoy(1),PATH=caml2path[0] ENDFOR ENDIF ELSE IF orat EQ 1 THEN BEGIN ; ORBIT/ATTITUDE SPAWN,'gunzip /RAID4/po_*cdf.gz',result,COUNT=cnt,/SH FOR k=0L,cnt-1L DO PRINT,result[k] SPAWN,'\mv /RAID4/po_*cdf '+lzpath[0],result,COUNT=cnt,/SH FOR k=0L,cnt-1L DO PRINT,result[k] cepl2path = STR_SEP(GETENV('CEPL2PATH'),':') FOR k=0L,N_ELEMENTS(files)-1L DO BEGIN list = STR_SEP(files[k],'_') READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)' IF NOT KEYWORD_SET(nol2) THEN $ Makepapco,[yy,mm,dd],[yy,mm,dd],LOG=75,/LZ,PATH=cepl2path[0] ENDFOR ENDIF ELSE BEGIN ; CEPPAD SPAWN,'gunzip /RAID4/po_*cep*gz',result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] SETENV,'CEPLZPATH=/RAID4' cepl2path = STR_SEP(GETENV('CEPL2PATH'),':') FOR k=0L,N_ELEMENTS(files)-1 DO BEGIN list = STR_SEP(files[k],'_') READS,list[3],yy,mm,dd,FORMAT='(I4,2I2)' IF NOT KEYWORD_SET(nol2) THEN $ Makepapco,[yy,mm,dd],[yy,mm,dd],LOG=75,/LZ,PATH=cepl2path[0] ENDFOR SETENV,'CEPLZPATH='+clzpath SPAWN,'gzip /RAID4/po_*cep*',result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] SPAWN,'\mv /RAID4/po_*cep*gz '+lzpath[0],result,COUNT=cnt,/SH FOR k=0,cnt-1 DO PRINT,result[k] ENDELSE ENDIF Return: IF KEYWORD_SET(stop) THEN STOP RETURN END