SamFile.SamDataFile.SamDataFile
python wrapper around the IDL data structure SAM.SamDataFileStruct
This documentation was generated by and for sam_common_pylib v8_5_0.
IDL definition:
struct DataFileAttributesStruct {
DataFileRequiredAttributesStruct requiredAttributes;
SAM::CorbaCaseInsensitiveDictionaryStruct additionalAttributes;
}; // struct DataFileAttributesStruct
API Usage Example:
# importing the data structure:
from SamFile.SamDataFile import SamDataFile
Usage:
a) To create a SamDataFile object for an EXISTING file
(one that has already been declared in the SAM database):
myFile = SamDataFile('fileNameHere')
myFile = SamDataFile(fileId)
myFile = SamDataFile(fileName='fileNameHere')
myFile = SamDataFile(fileId=fileIdHere)
b) To create a SamDataFile object for a NEW file
(one that does NOT exist in the SAM database),
you must create the instance with more args than
above (so the constructor knows you're creating a
NEW file), and you pass in the file metadata,
either as keyword args, or as a dictionary.
myFile = SamDataFile(fileName='myFileName.tar',
fileType='nonPhysicsGeneric',
fileFormat='tar',
fileSize='2G',
fileContentStatus='good',
...)
myFile = SamDataFile({SAM.attrFileName: 'myFileName.tar',
SAM.attrFileType: 'nonPhysicsGeneric',
SAM.attrFileFormat: 'tar',
SAM.attrFileSize : '2g',
SAM.attrFileContentStatus:'good',
...})
Or, you may create a new SamDataFile and set the metadata
attributes later, by giving just the fileName and fileType
when you create the instance, and using the various setXXX
methods to update the file later:
myFile = SamDataFile(fileName='myFileName.tar',
fileType='nonPhysicsGeneric')
myFile.setFileFormat('tar')
myFile.setFileSize('10g')
myFile.setFileContentStatus('good')
# kwargs:
f = SamDataFile(fileName='myFileName',
fileType=SAM.DataFileType_PhysicsGeneric,
fileSize='10g',
crcValue='12345L', crcType=SAM.CRC_Adler32Type,
runType='detector', runNumber=12345,
...)
# argDict:
f = SamDataFile({SAM.attrFileName : 'myFileName',
SAM.attrFileSize : '10g',
SAM.attrFileType : SAM.DataFileType_DerivedDetector,
SAM.attrCrc : ('12345L', SAM.CRC_Adler32Type),
...})
################################################################################
For educational purposes, the selftest used in validating this
data structure is shown below. Hopefully it will prove to be
illuminating and useful; but for further details, RTFC.
################################################################################
class selftest(SamStructSelftestBaseClass):
def runTest(self):
print("Commencing unit tests of SamFile and related utilities.")
from SamFile.SamDataFile import SamDataFile
from SamCorba.SamServerProxyManager import getSamServerProxyManager, DB_DATA_FILE_PROXY, DB_BASE_TEST_PROXY
proxyMgr = getSamServerProxyManager()
self.dbFile = proxyMgr.getProxy(DB_DATA_FILE_PROXY)
self.dbTest = proxyMgr.getProxy(DB_BASE_TEST_PROXY)
# minimum required attributes for ALL files:
self.SAMPLE_GOOD_NON_PHYSICS_GENERIC_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_NON_PHYSICS_GENERIC_METADATA.update({
SAM.attrFileType : SAM.DataFileType_NonPhysicsGeneric,
SAM.attrFileFormat : SAM.DataFileFormat_Unknown,
SAM.attrFileContentStatus : SAM.DataFileContentStatus_Unknown,
SAM.attrFileId : SAM.SamDefault_DbId_NotSet,
SAM.attrFileName : 'NonPhysicsFile_12345',
SAM.attrFileSize : '10g',
SAM.attrCrcValue : SAM.CRC_UnknownValue,
SAM.attrCrcType : SAM.CRC_UnknownType,
SAM.attrWorkGroup : 'workGroupInCharge',
})
# generic physics files add dataTier:
self.SAMPLE_GOOD_PHYSICS_GENERIC_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_PHYSICS_GENERIC_METADATA.update(self.SAMPLE_GOOD_NON_PHYSICS_GENERIC_METADATA)
self.SAMPLE_GOOD_PHYSICS_GENERIC_METADATA.update(
{SAM.attrFileType : SAM.DataFileType_PhysicsGeneric,
SAM.attrFileName : 'PhysicsGenericFile_alpha',
SAM.attrDataTier : 'DataTierHere',
})
# non-generic physics files add application info, event info, run info
self.PHYSICS_DATA_BASE_INFO = CaseInsensitiveDictionary()
self.PHYSICS_DATA_BASE_INFO.update(self.SAMPLE_GOOD_PHYSICS_GENERIC_METADATA)
self.PHYSICS_DATA_BASE_INFO.update(
{SAM.attrApplicationFamily : ApplicationFamily('appFamily', 'appName', 'appVersion'),
SAM.attrFirstEvent : 10,
SAM.attrLastEvent : 20,
SAM.attrEventCount : 11,
SAM.attrRunNumber : [1,2,3,4,5],
SAM.attrRunType : SAM.RunType_Unknown,
})
# imported detector (aka raw data) files:
self.SAMPLE_GOOD_IMPORTED_DETECTOR_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_IMPORTED_DETECTOR_METADATA.update(self.PHYSICS_DATA_BASE_INFO)
self.SAMPLE_GOOD_IMPORTED_DETECTOR_METADATA.update(
{SAM.attrFileType : SAM.DataFileType_ImportedDetector,
SAM.attrFileName : 'DetectorImportFile_123',
SAM.attrPhysicalDatastream : 'physicalDatastreamHere',
SAM.attrStartTime : '10-Aug-2003',
SAM.attrEndTime : '12-Aug-2003',
SAM.attrFilePartition : 1,
SAM.attrLumMin : 100,
SAM.attrLumMax : 200,
})
# derived detector (aka analyzed files from real data):
self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA.update(self.PHYSICS_DATA_BASE_INFO)
self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA.update(
{SAM.attrFileType : SAM.DataFileType_DerivedDetector,
SAM.attrFileName : 'DerivedDetectorFile_abc',
SAM.attrPhysicalDatastream : 'physicalDatastreamHere',
SAM.attrConsumerProcessId : 100,
SAM.attrParents : ['parentFile1', 'parentFile2'],
})
# simulated import (aka MC raw):
self.SAMPLE_GOOD_IMPORTED_SIMULATED_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_IMPORTED_SIMULATED_METADATA.update(self.PHYSICS_DATA_BASE_INFO)
self.SAMPLE_GOOD_IMPORTED_SIMULATED_METADATA.update(
{SAM.attrFileType : SAM.DataFileType_ImportedSimulated,
SAM.attrFileName : 'SimulatedImportFile_246',
SAM.attrParams : {'Global' : {'global' : 'params', 'go':'here'},
'Simulated' : {'other' : 'params', 'go':'here'},
},
})
# simulated derived (aka processed MC files):
self.SAMPLE_GOOD_DERIVED_SIMULATED_METADATA = CaseInsensitiveDictionary()
self.SAMPLE_GOOD_DERIVED_SIMULATED_METADATA.update(self.PHYSICS_DATA_BASE_INFO)
self.SAMPLE_GOOD_DERIVED_SIMULATED_METADATA.update(
{SAM.attrFileType : SAM.DataFileType_DerivedSimulated,
SAM.attrFileName : 'SimulatedDerivedFile_beta',
SAM.attrConsumerProcessId : 105,
SAM.attrParents : ['simFile1', 'simFile2',],
})
try:
print("make a mdAttributeDescriptor...")
from SamUtility.MetadataRequirementHandler import getMetadataRequirementHandler
mdRequirementHandler = getMetadataRequirementHandler()
self.mdAttributeDescriptor = mdRequirementHandler.getMetadataRequirements()
print("Ok, now let's have some fun. Let's make some files.")
for doTheTest in (self.testNonPhysicsGeneric,
self.testPhysicsGeneric,
self.testImportedDetector,
self.testDerivedDetector,
self.testImportedSimulated,
self.testDerivedSimulated,
self.testFaultyMetadata):
print("===========================================================================")
doTheTest()
print("===========================================================================")
print("\nAll tests passed. Congratulations.")
except Exception, exc:
print("UNEXPECTED EXCEPTION: %s %s" % (sys.exc_info()[0], sys.exc_info()[1]))
try:
if hasattr(exc, 'argDict'):
print("Remote exception args:")
for (k,v) in exc.argDict.items():
if k == 'formattedTracebackString':
pass
else:
print("\t%s = %s" % (k,v))
print("Remote traceback:\n%s" % exc.argDict.get('formattedTracebackString'))
except:
pass
print("Local traceback:")
traceback.print_tb(sys.exc_info()[2])
self.fail("TEST FAILED. Bummer.")
def testNonPhysicsGeneric(self):
print("Testing NON-PHYSICS-GENERIC METADATA")
metadata = self.SAMPLE_GOOD_NON_PHYSICS_GENERIC_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile,),
(PhysicsGenericFile,
ImportedDetectorFile, DerivedDetectorFile,
ImportedSimulatedFile, DerivedSimulatedFile))
def testPhysicsGeneric(self):
print("Testing PHYSICS-GENERIC METADATA")
metadata = self.SAMPLE_GOOD_PHYSICS_GENERIC_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile, PhysicsGenericFile),
(ImportedDetectorFile, DerivedDetectorFile,
ImportedSimulatedFile, DerivedSimulatedFile))
def testImportedSimulated(self):
print("Testing IMPORTED SIMULATED METADATA")
metadata = self.SAMPLE_GOOD_IMPORTED_SIMULATED_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile, PhysicsGenericFile,
ImportedSimulatedFile),
(DerivedSimulatedFile,
ImportedDetectorFile, DerivedDetectorFile))
def testDerivedSimulated(self):
print("Testing DERIVED SIMULATED METADATA")
metadata = self.SAMPLE_GOOD_DERIVED_SIMULATED_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile, PhysicsGenericFile,
DerivedSimulatedFile),
(ImportedSimulatedFile,
DerivedDetectorFile, ImportedDetectorFile))
def testImportedDetector(self):
print("Testing IMPORTED DETECTOR METADATA")
metadata = self.SAMPLE_GOOD_IMPORTED_DETECTOR_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile, PhysicsGenericFile,
ImportedDetectorFile),
(DerivedDetectorFile,
ImportedSimulatedFile, DerivedSimulatedFile))
def testDerivedDetector(self):
print("Testing DERIVED DETECTOR METADATA")
metadata = self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA
self.verifyMetadataAgainstAllDataFileTypes(metadata,
(NonPhysicsGenericFile, PhysicsGenericFile,
DerivedSimulatedFile, DerivedDetectorFile),
(ImportedDetectorFile, ImportedSimulatedFile))
def testFaultyMetadata(self):
print("NOW, the hardest part of all: faulty (rather than missing) metadata.")
class NotAnythingValid:
def __str__(self):
return 'NotAnythingValid'
# a base dictionary to work from
baseMetadata = CaseInsensitiveDictionary()
baseMetadata.update(self.SAMPLE_GOOD_IMPORTED_DETECTOR_METADATA)
baseMetadata.update(self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA)
baseMetadata.update(self.SAMPLE_GOOD_IMPORTED_SIMULATED_METADATA)
baseMetadata.update(self.SAMPLE_GOOD_DERIVED_DETECTOR_METADATA)
for attrName in self.mdAttributeDescriptor.getAllAttributeNames():
# make a copy of the baseMetadata to mess with
faultyMetadata = CaseInsensitiveDictionary()
faultyMetadata.update(baseMetadata)
# find out what type this attribute SHOULD be:
shouldBeType = self.mdAttributeDescriptor.getRequiredDataType(attrName)
# if it should be a STRING, remove it from the metadata dict entirely;
# otherwise, *everything* has a string representation
if shouldBeType == SAM.SamDataType_StringType:
del(faultyMetadata[attrName])
# otherwise, turn it into something that cannot POSSIBLY be used:
else:
faultyMetadata[attrName] = NotAnythingValid()
# now make sure that this fails for all cases where it is required
requiredBy = self.mdAttributeDescriptor.getRequiredBy(attrName)
print("testing faulty metadata for attribute '%s'" % attrName)
for c in (NonPhysicsGenericFile, PhysicsGenericFile,
ImportedSimulatedFile, DerivedSimulatedFile,
ImportedDetectorFile, DerivedDetectorFile):
try:
f = c(faultyMetadata)
f.validateMetadata()
# we should ALWAYS fail!
raise AssertionError("FAIL --> this should NOT be valid!")
except SamExceptions.ArgumentError, ex:
print("ok %s:\n%s" % (c.__name__, str(ex.args)))
except SamExceptions.InvalidMetadata, ex:
print("ok %s:\n%s" % (c.__name__, str(ex.args)))
except AssertionError:
print("EXCEPTION: SHOULD NOT BE HERE! %s %s" % (sys.exc_info()[0], sys.exc_info()[1]))
traceback.print_tb(sys.exc_info()[2])
raise
print("===========================================================")
# utility routine for checking various forms of metadata:
def verifyMetadataAgainstAllDataFileTypes(self, metadata, shouldPassList, shouldFailList):
print("INCOMING METADATA DICTIONARY: %s" % metadata)
print("\nCheck the fileTypes for which this should be VALID metadata:")
for c in shouldPassList:
print("\nMetadata for class '%s' being tested against class '%s'" %
(metadata.get('fileType'), c.__name__))
newFile = c(metadata)
try:
newFile.verifyMetadata()
print("Metadata locally verified (existence requirements pass as predicted) for class '%s'" % c.__name__)
except SamExceptions.InvalidMetadata, ex:
print("\n\n==========================================================================\n" + \
"Metadata LOCAL VERIFICATION (existence requirements) FAIL, which is unexpected, for class '%s'"
% c.__name__)
print("%s" % str(ex.args))
print("INCOMING metadata WAS: %s" % str(metadata))
print("newFile.getMetadata(): %s" % str(newFile.getMetadata()))
raise
print("... test marshalling/unmarshalling, make sure it's still the same file ...")
result = self.dbTest().testArbitrarySamDataStructure(newFile.__class__.__name__, Any(newFile))
try:
assert(result == newFile)
except AssertionError:
print("RETURN FROM DBSERVER NOT THE SAME:")
for k in newFile.getMetadata().keys():
if newFile.get(k) != result.get(k):
print("DIFFERENT for key = %s: original = %s, db = %s" % (k, newFile.get(k),
result.get(k)))
raise
print("... check validation...")
try:
self.dbFile().validateMetadata(newFile)
raise Exception("OOPS, I don't think we really have valid data here!")
except SamExceptions.InvalidMetadata, ex:
print("... metadata is invalid (as could be expected), and here's str(ex.args):\n%s" % str(ex.args))
except Exception:
# aha, one special case: the degenerate NonPhysicsGeneric case.
print("%s" % sys.exc_info()[1])
if newFile.getFileType() == SAM.DataFileType_NonPhysicsGeneric:
print("Degenerate case: metadata is valid for NonPhysicsGeneric file.")
else:
raise
print("\nCheck the fileTypes for which this should NOT be LOCALLY VALID:")
for c in shouldFailList:
print("\nMetadata for class '%s' being tested against class '%s'" % (metadata.get('fileType'),
c.__name__))
newFile = c(metadata)
print("... test marshalling/unmarshalling, make sure it's still the same file ...")
result = self.dbTest().testArbitrarySamDataStructure(newFile.__class__.__name__, Any(newFile))
try:
assert(result == newFile)
except AssertionError:
print("RETURN FROM DBSERVER NOT THE SAME:")
for k in newFile.getMetadata().keys():
if newFile.get(k) != result.get(k):
print("DIFFERENT for key = %s: original = %s, db = %s" % (k, newFile.get(k),
result.get(k)))
raise
print("... check local verification (NOT validation)...")
try:
newFile.verifyMetadata()
raise Exception("Did NOT catch InvalidMetadata for metadata type '%s' used in class '%s'" %
(metadata.get('fileType'), c.__name__))
except SamExceptions.InvalidMetadata, ex:
print("Metadata existence requirements fail as prediced for class '%s':\n%s" % (c.__name__, str(ex.args)))
if __name__ == "__main__":
from SamFile.SamDataFile import selftest
from SamUtility.SamUnitTest import unitTestRunner
sys.exit(unitTestRunner(selftest))