checkCalibrationCSV.py

Go to the documentation of this file.

#!/usr/bin/python
# This is meant to be a simple class to test the CSV files created for NOvA calibration.
# Currently there are only a couple of checks that are performed
#   1) Check for trailing commas (which causes the calibration code to die)
#   2) Check that there are calibration files for all runs

import sys
import glob

def usage():
    print("Usage:\n"+sys.argv[0]+" <new dir> <old dir>[Optional]")

class NovaCalibTest:

    def __init__(self, newDir):
        self.dirName = newDir
        self.ndDataAbsDict={}
        self.fdDataAbsDict={}
        self.ndMCAbsDict={}
        self.fdMCAbsDict={}
        self.ndDataAttenDict={}
        self.fdDataAttenDict={}
        self.ndMCAttenDict={}
        self.fdMCAttenDict={}

        self.getAbsFiles('nd','data')
        self.getAbsFiles('fd','data')
        self.getAbsFiles('nd','mc')
        self.getAbsFiles('fd','mc')
        self.getAttenFiles('fd','data')
        self.getAttenFiles('fd','mc')
        self.getAttenFiles('nd','data')
        self.getAttenFiles('nd','mc')

        self.badTrailingCommas=False
        self.badMissingRuns=False


    def checkForTrailingCommas(self,fileName):
        gotTrailingCommas=False
        with open(fileName) as fp:
            line = fp.readline()
            while line:
                line = fp.readline()
                if "," in line[len(line)-2:]:
                    gotTrailingCommas=True
                    print("Oh no the line ends in a comma: "+line)
        return gotTrailingCommas

    def checkAbsFilesForTrailingCommas(self):
        for dict in [self.ndDataAbsDict,self.ndMCAbsDict,self.fdDataAbsDict,self.fdMCAbsDict]:
            for i in sorted (dict.keys()) : 
                if(self.checkForTrailingCommas(dict[i])):
                    print("Broken file is "+dict[i])
                    self.badTrailingCommas=True
    
    def checkAttenFilesForTrailingCommas(self):
        for dict in [self.ndDataAttenDict,self.ndMCAttenDict,self.fdDataAttenDict,self.fdMCAttenDict]:
            for i in sorted (dict.keys()) : 
                if(self.checkForTrailingCommas(dict[i])):
                    print("Broken file is "+dict[i])
                    self.badTrailingCommas=True

    def checkAbsRunCoverage(self):
        return self.checkRunCoverage([self.ndDataAbsDict,self.ndMCAbsDict,self.fdDataAbsDict,self.fdMCAbsDict])

    def checkAttenRunCoverage(self):
        return self.checkRunCoverage([self.ndDataAttenDict,self.ndMCAttenDict,self.fdDataAttenDict,self.fdMCAttenDict])
                    
    def checkRunCoverage(self,dictList):
        countBad=0
        for dict in dictList:
            lastRun=-1;
            countFb=0
            for tuple in sorted (dict.keys()) :
                checkThis=True 
                if(len(tuple)==2):
                    (first,last)=tuple
                else:
                    (first,last,fb)=tuple
                    if(fb!=0):
                        checkThis=False
                        countFb=countFb+1
                    else: 
                        if(countFb>0):
                            print("Fibre brightness bins "+str(countFb))
                            print("in dictionary: "+str(dict[tuple]))                    
                        countFb=1
                if(checkThis and first!=lastRun+1):
                    print("Missing run gap: "+str(lastRun)+" to "+str(first)+" before "+dict[tuple])
                    countBad+=abs(first-(lastRun+1))
                lastRun=last
            if(countFb>0):
                print("Fibre brightness bins "+str(countFb))
                print("in dictionary: "+str(dict[tuple]))   
                
        if(countBad==0): 
            print("Found CSV files for all runs in range ")
        else:
            print("Missing "+str(countBad)+" files")
            self.badMissingRuns=True

  

    def getAbsFileForThisRun(self,whichDet,dataOrMC,thisRun):
        thisDict={}
        if whichDet == 'nd' and dataOrMC=='data':
            thisDict=self.ndDataAbsDict
        elif whichDet == 'nd' and dataOrMC=='mc':
            thisDict=self.ndMCAbsDict
        elif whichDet == 'fd' and dataOrMC=='data':
            thisDict=self.fdDataAbsDict
        elif whichDet == 'fd' and dataOrMC=='mc':
            thisDict=self.fdMCAbsDict
        for (k1,k2) in thisDict:
            if(k1<=thisRun and k2>=thisRun):
                return thisDict[(k1,k2)]

    def getAttenFileForThisRun(self,whichDet,dataOrMC,thisRun,whichFb=-1):
        thisDict={}
        if whichDet == 'nd' and dataOrMC=='data':
            thisDict=self.ndDataAttenDict
        elif whichDet == 'nd' and dataOrMC=='mc':
            thisDict=self.ndMCAttenDict
        elif whichDet == 'fd' and dataOrMC=='data':
            thisDict=self.fdDataAttenDict
        elif whichDet == 'fd' and dataOrMC=='mc':
            thisDict=self.fdMCAttenDict
        if(whichFb<0):
            for (k1,k2) in thisDict:
                if(k1<=thisRun and k2>=thisRun):
                    return thisDict[(k1,k2)]
        else:
            for (k1,k2,k3) in thisDict:
                if(k1<=thisRun and k2>=thisRun and k3==whichFb):
                    return thisDict[(k1,k2,k3)]



    def parseRunString(self,runString):
            runBoundsList=runString.split('-')
            minStr=runBoundsList[0]
            maxStr=runBoundsList[1]
            minRun=0
            maxRun=sys.maxint
            if(len(minStr[1:])>0):
                minRun=int(minStr[1:])
            if(len(maxStr[1:])>0):
                maxRun=int(maxStr[1:])
            return (minRun,maxRun)


    def getAbsFiles(self,whichDet,dataOrMC):
        preString=self.dirName+'/calib_abs_consts.'+whichDet+'.'+dataOrMC
        absFiles=glob.glob(preString+'*.csv')
        for absFile in absFiles:
            verString=absFile[len(preString)+1:].split('.')[0]
            runString=absFile[len(preString)+1:].split('.')[1]
            runTuple=self.parseRunString(runString)
            if whichDet == 'nd' and dataOrMC=='data':
                self.ndDataAbsDict[runTuple]=absFile
            elif whichDet == 'nd' and dataOrMC=='mc':
                self.ndMCAbsDict[runTuple]=absFile
            elif whichDet == 'fd' and dataOrMC=='data':
                self.fdDataAbsDict[runTuple]=absFile
            elif whichDet == 'fd' and dataOrMC=='mc':
                self.fdMCAbsDict[runTuple]=absFile
            else: 
                print("Nothing defined for "+whichDet+ ' with '+dataOrMC)

    def getAttenFiles(self,whichDet,dataOrMC):
        preString=self.dirName+'/calib_atten_consts.'+whichDet+'.'+dataOrMC
        attenFiles=glob.glob(preString+'*.csv')
        for attenFile in attenFiles:
            verString=attenFile[len(preString)+1:].split('.')[0]
            runString=attenFile[len(preString)+1:].split('.')[1]
            fbString=attenFile[len(preString)+1:].split('.')[2]
            hasFb=False
            runTuple=self.parseRunString(runString)
            if "fb" in fbString:
                hasFb=True
                runTuple=runTuple+(int(fbString[2:]),) 
            if whichDet == 'nd' and dataOrMC=='data':
                self.ndDataAttenDict[runTuple]=attenFile
            elif whichDet == 'nd' and dataOrMC=='mc':
                self.ndMCAttenDict[runTuple]=attenFile
            elif whichDet == 'fd' and dataOrMC=='data':
                self.fdDataAttenDict[runTuple]=attenFile
            elif whichDet == 'fd' and dataOrMC=='mc':
                self.fdMCAttenDict[runTuple]=attenFile
            else: 
                print("Nothing defined for "+whichDet+ ' with '+dataOrMC)

    def printSummary(self):
        if self.badMissingRuns:
            for i in range(0,10):
                print("There are missing runs!!!!!!!!!!!!!!!!")  
                
        if self.badTrailingCommas:
            for i in range(0,10):
                print("There are traling commas!!!!!!!!!!!!!!!!")       


def main():
    if(len(sys.argv)<2):
        usage()
        return

    newDir=sys.argv[1]
    print("Using files from: "+newDir)
    nct = NovaCalibTest(newDir)
    nct.checkAbsRunCoverage()
    nct.checkAttenRunCoverage()
    nct.checkAbsFilesForTrailingCommas()
    nct.checkAttenFilesForTrailingCommas()

    nct.printSummary()
    return nct.badMissingRuns+nct.badTrailingCommas

if __name__ == "__main__":
    main()