#!/usr/bin/env python # # $Id: stateUtil.py,v 1.28 2003/07/31 16:38:08 gug Exp $ # beginning of a state saving utility module for dlsam ##### section 1 : import statements import mstatic from Report import * report=mstatic.report import imp import sys import os from types import * import re import string from threading import * import exceptions import thSafeUtils ###### section 2 : global and class definitions ### Global Lock Instance: priRLock --- ### A recursive blocking lock for use in stateUtil module for ### thread safety. ### priRLock=RLock() ### Class Definition: stateUtil Class --- ### Allows saving and storing of dictionary based state information. ### class stateUtil: ### Class Method Definition: stateUtil.__init__ --- ### Initialization of class instance for bufferSetManager Class. ### @param topLevel Top directory level for state tree ### @returns None ### def __init__(self,topLevel): self.topLevel=topLevel self.calendar={'Jan':'01', 'Feb':'02', 'Mar':'03', 'Apr':'04', 'May':'05', 'Jun':'06', 'Jul':'07', 'Aug':'08', 'Sep':'09', 'Oct':'10', 'Nov':'11', 'Dec':'12'} ### Class Method Definition: stateUtil.makeSubDir --- ### Function for making a run number subdirectory. ### @param lpath Path to directory where sub directory will reside ### @param name Name of sub directory ### @returns None ### def makeSubDir(self,lpath,name): priRLock.acquire() found=None lruns=thSafeUtils.listdir(lpath) sname="%s" % (name) # force this to be a string if len(sname)==1: sname="00%s" % (sname) elif len(sname)==2: sname="0%s" % (sname) if type(lruns) != ListType: msg="Unable to make state subdirectory--something is severely wrong.\n" msg="%sReply from listdir of %s == %s" % (msg,lpath,lruns) report.submit(ERROR,msg) return for run in lruns: if run == sname: found=1 if found == None: path="%s/%s" % (lpath,sname) thSafeUtils.mkdir(path) priRLock.release() return ### Class Method Definition: stateUtil.buildExportPath --- ### Build the export path string based on the type of state information to ### be saved. ### @param state State object to build export path for ### @returns Path ### def buildExportPath(self,state): path=self.topLevel # first check for the basic keys needed if state.has_key("StateName")==0: msg="State has no StateName key! Save to toplevel directory\n" report.submit(ERROR,msg) return path stateName=state['StateName'] runState=state if stateName!='PartitionFileInfo' and stateName!="BufferSetInfo": msg= "%s is not a supported stateName. Saving to toplevel directory\n" % (stateName) report.submit(ERROR,msg) # handle Partition and BufferSet states if stateName=='PartitionFileInfo': runState=state['DataDictionary'] if runState.has_key("RunYear")==0: msg= "State has no RunYear key! Save to toplevel directory" report.submit(ERROR,msg) return path elif runState.has_key("RunMonth")==0: msg="State has no RunMonth key! Save to toplevel directory" report.submit(ERROR,msg) return path elif runState.has_key("RunNumber")==0: msg="State has no RunNumber key! Save to toplevel directory" report.submit(ERROR,msg) return path runYear=runState['RunYear'] runMonth=runState['RunMonth'] runNumber=runState['RunNumber'] # if run directory does not exist, create it lpath="%s/%s/%s" % (path,runYear,runMonth) self.makeSubDir(lpath,runNumber) if runState.has_key("StreamName")==0: msg="State has no StreamName key! Save to toplevel directory" report.submit(ERROR,msg) return path elif runState.has_key("Partition")==0: msg="State has no Partition key! Save to toplevel directory" report.submit(ERROR,msg) return path streamName=runState['StreamName'] partition="%s" % (runState['Partition']) if len(partition)==1: partition="00%s" % (partition) elif len(partition)==2: partition="0%s" % (partition) lpath="%s/%s" % (lpath,runNumber) self.makeSubDir(lpath,streamName) lpath="%s/%s" % (lpath,streamName) self.makeSubDir(lpath,partition) path="%s/%s" % (lpath,partition) if stateName=='BufferSetInfo': idname=state['IDName'] lpath="%s/disks" % (path) self.makeSubDir(lpath,idname) path="%s/disks/%s" % (path,idname) return path ### Class Method Definition: stateUtil.exportState --- ### Export state to file system for requested state and save with file prefix ### of contained in name. ### @param name Type of state to export (partition|bufferSet) ### @param state State object to export ### @returns None ### def exportState(self,name,state): # we will write to tmp file, move export to save and move tmp to export priRLock.acquire() lastDir=thSafeUtils.getcwd() path=self.buildExportPath(state) exportName="%s/%s.py" % (path,name) saveName="%s/%s.save" % (path,name) tmpName="%s/%s.tmp" % (path,name) fd=open(tmpName,'w') # create a little header of comments fd.write("#state information for ") fd.write(name) fd.write('\n') fd.write("state=") self.writeDict(fd,state) fd.write('\n') fd.close() # move files around now that writing is finished allFiles=thSafeUtils.listdir(path) for file in allFiles: xfile="%s/%s" % (path,file) if xfile==exportName: thSafeUtils.rename(exportName,saveName) tmp=thSafeUtils.rename(tmpName,exportName) priRLock.release() return ### Class Method Definition: stateUtil.writeDict --- ### Write a dictionary object to a file using recursive calls for trees. ### @param fd File descriptor for output file ### @param state State object to write ### @returns None ### def writeDict(self,fd,state): fd.write('{') comma=None for key in state.keys(): if comma==None: comma=1 else: fd.write(",\n") fd.write("'") fd.write(key) fd.write("':") item=state[key] valType=type(item) if valType is TupleType: self.writeSequence(fd,item,'Tuple') elif valType is ListType: self.writeSequence(fd,item,'List') elif valType is DictType: self.writeDict(fd,item) elif valType is ClassType: msg="No support for User defined classes in exportState\n" report.submit(ERROR,msg) elif valType is StringType: fd.write("'") fd.write(item) fd.write("'") else: # force this to StringType before write---since not a string we shouldn't need # quotes sitem="%s" % (item) fd.write(sitem) fd.write('}') ### Class Method Definition: stateUtil.writeSequence --- ### Write a sequence object (list,tuple) to a file using recursive calls for ### trees. ### @param fd File descriptor for output file ### @param state State object to write ### @param seqType Type of python sequence object to write ### @returns None ### def writeSequence(self,fd,state,seqType): if seqType == 'List' : fd.write('[') elif seqType=='Tuple': fd.write('(') comma=None for item in state: if comma==None: comma=1 else: fd.write(",\n") valType=type(item) if valType is TupleType: self.writeSequence(fd,item,'Tuple') elif valType is ListType: self.writeSequence(fd,item,'List') elif valType is DictType: self.writeDict(fd,item) elif valType is ClassType: msg="No support for User defined classes in exportState\n" report.submit(ERROR,msg) elif valType is StringType: fd.write("'") fd.write(item) fd.write("'") else: # force this to StringType before write---since not a string we shouldn't need # quotes sitem="%s" % (item) fd.write(sitem) if seqType == 'List' : fd.write(']') elif seqType=='Tuple': fd.write(')') ### Class Method Definition: stateUtil.importState --- ### Snagged from an online python documentation example for the imp module ### and modified. Import or reimport a python module. ### @param path The absolute path to directory for state file. ### @param modName Type of state to import (partition|bufferSet) ### @returns Imported state object if successful, else None ### def importState(self,path,modName): # If any of the following calls raises an exception, # there's a problem we can't handle -- let the caller handle it. priRLock.acquire() state=None mod=None fp=None try: pathname="%s/%s.py" % (path,modName) fp=open(pathname,'r') description=('','r',1) mod=imp.load_module(modName, fp, pathname, description) if not mod.__dict__.has_key('state'): msg="Import state %s from %s has no state object." % (modName,path) msg="%s. State file corrupted!" % (msg) report.submit(WARN,msg) raise exceptions.ImportError except: # try loading tmp file instead try: pathname="%s/%s.tmp" % (path,modName) fp=open(pathname,'r') description=('','r',1) mod=imp.load_module(modName, fp, pathname, description) if not mod.__dict__.has_key('state'): msg="Import state %s from %s has no state object." % (modName,path) msg="%s. State file corrupted!" % (msg) report.submit(WARN,msg) raise exceptions.ImportError except: # finaly try the save extension try: pathname="%s/%s.save" % (path,modName) fp=open(pathname,'r') description=('','r',1) mod=imp.load_module(modName, fp, pathname, description) if not mod.__dict__.has_key('state'): msg="Import state %s from %s has no state object." % (modName,path) msg="%s. State file corrupted!" % (msg) report.submit(WARN,msg) raise exceptions.ImportError except: msg="Unable to import state %s from %s\n" % (modName,path) report.submit(ERROR,msg) if fp: fp.close() # check to see if it worked (for debugging only) if mod!=None: state=mod.state priRLock.release() return state ### Class Method Definition: stateUtil.createDiskAreas --- ### Create dlsam disk state area directory tree. ### @returns None ### def createDiskAreas(self): area1="%s/disks" % (self.topLevel) area2="%s/disklists" % (self.topLevel) thSafeUtils.mkdir(area1) thSafeUtils.mkdir(area2) return ### Class Method Definition: stateUtil.createYearTree --- ### Create dlsam state year directory tree. ### @param firstYear Start year to create tree for ### @param lastYear End year to create tree for ### @returns None ### def createYearTree(self,firstYear,lastYear): priRLock.acquire() # check to see if year already exists, if so skip that year since we don't # want to risk clobbering existing state information definedYears=thSafeUtils.listdir(self.topLevel) definedMonths=self.calendar.values() definedMonths.sort() for year in range(firstYear,lastYear+1): found=None syear="%s" % (year) for prior in definedYears: if prior == syear: found=1 if found==None: xsyear="%s/%s" % (self.topLevel,syear) thSafeUtils.mkdir(xsyear) for month in definedMonths: subdir="%s/%s" % (xsyear,month) thSafeUtils.mkdir(subdir) priRLock.release() return ### Class Method Definition: stateUtil.getTemplateState --- ### Get a template of type name for building a state object. ### @param location Location of template state directory ### @param name Type of template to import ### @returns Imported template state object if successful, else None ### def getTemplateState(self,location,name): priRLock.acquire() opwd=thSafeUtils.getcwd() theFile=self.importState(location,name) priRLock.release() return theFile ### Class Method Definition: stateUtil.importMetaFile --- ### Import meta data file. ### @param path Absolute path of directory where meta files is. ### @param name Type of metafile to import ### @returns Imported meta data file object if successful, else None ### def importMetaFile(self,path,name): mod=None priRLock.acquire() modName="metaMod" fp=None msg="importMetaFile: %s: %s\n" % (path,name) report.submit(DBUG2,msg) try: pathname="%s/%s" % (path,name) fp=open(pathname,'r') description=('','r',1) mod=imp.load_module(modName, fp, pathname, description) if not mod.__dict__.has_key('TheRawDataFile'): msg="Import %s from %s has no TheRawDataFile object." % (name,path) msg="%s. State file corrupted!" % (msg) report.submit(WARN,msg) raise exceptions.ImportError except: msg="Cannot import meta file %s from location %s\n" % (name,path) report.submit(ERROR,msg) mod=None if fp: fp.close() priRLock.release() # del sys.modules[modName] return mod ### Class Method Definition: stateUtil.getMetaValues --- ### Get a select set of meta data from meta data file avoiding import. ### @param path Absolute path of directory where meta files is. ### @param name Type of metafile to import ### @returns List of part_nr, stream, start_time ### def getMetaValues(self,path,name): mod=None priRLock.acquire() fp=None msg="getMetaValues: %s: %s\n" % (path,name) # report.submit(DBUG2,msg) try: pathname="%s/%s" % (path,name) fp=open(pathname,'r') except: msg="Cannot open meta file %s from location %s\n" % (name,path) # report.submit(ERROR,msg) else: mod=[] mdata=fp.readlines() part_nr=None stream=None start_time=None run_number=None TheFileFamily=None for line in mdata: zline=None zstart=None zequal=string.find(line,'=') if string.find(line,'part_nr')>-1: zstart=string.find(line,'part_nr') elif string.find(line,'stream')>-1: zstart=string.find(line,'stream') elif string.find(line,'TheFileFamily')>-1: zstart=string.find(line,'TheFileFamily') elif string.find(line,'start_time')>-1: zstart=string.find(line,'start_time') elif string.find(line,'run_id')>-1: # getting run number is tricky idx1=1+string.find(line,"(") idx2=string.find(line,"L,") run_number="%010d" % int(line[idx1:idx2]) if zstart!=None: if zequal>zstart: zline=string.replace(line,',','') exec zline[zstart:] mod.append(part_nr) mod.append(stream) mod.append(start_time) mod.append(TheFileFamily) mod.append(run_number) if fp: fp.close() priRLock.release() return mod ##### section 3 if __name__ == '__main__': print " Use: ../stateUtilTest.py for testing "