checkCalibrationCSV.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 # This is meant to be a simple class to test the CSV files created for NOvA calibration.
3 # Currently there are only a couple of checks that are performed
4 # 1) Check for trailing commas (which causes the calibration code to die)
5 # 2) Check that there are calibration files for all runs
6 
7 import sys
8 import glob
9 
10 def usage():
11  print("Usage:\n"+sys.argv[0]+" <new dir> <old dir>[Optional]")
12 
14 
15  def __init__(self, newDir):
16  self.dirName = newDir
17  self.ndDataAbsDict={}
18  self.fdDataAbsDict={}
19  self.ndMCAbsDict={}
20  self.fdMCAbsDict={}
23  self.ndMCAttenDict={}
24  self.fdMCAttenDict={}
25 
26  self.getAbsFiles('nd','data')
27  self.getAbsFiles('fd','data')
28  self.getAbsFiles('nd','mc')
29  self.getAbsFiles('fd','mc')
30  self.getAttenFiles('fd','data')
31  self.getAttenFiles('fd','mc')
32  self.getAttenFiles('nd','data')
33  self.getAttenFiles('nd','mc')
34 
35  self.badTrailingCommas=False
36  self.badMissingRuns=False
37 
38 
39  def checkForTrailingCommas(self,fileName):
40  gotTrailingCommas=False
41  with open(fileName) as fp:
42  line = fp.readline()
43  while line:
44  line = fp.readline()
45  if "," in line[len(line)-2:]:
46  gotTrailingCommas=True
47  print("Oh no the line ends in a comma: "+line)
48  return gotTrailingCommas
49 
51  for dict in [self.ndDataAbsDict,self.ndMCAbsDict,self.fdDataAbsDict,self.fdMCAbsDict]:
52  for i in sorted (dict.keys()) :
53  if(self.checkForTrailingCommas(dict[i])):
54  print("Broken file is "+dict[i])
55  self.badTrailingCommas=True
56 
58  for dict in [self.ndDataAttenDict,self.ndMCAttenDict,self.fdDataAttenDict,self.fdMCAttenDict]:
59  for i in sorted (dict.keys()) :
60  if(self.checkForTrailingCommas(dict[i])):
61  print("Broken file is "+dict[i])
62  self.badTrailingCommas=True
63 
65  return self.checkRunCoverage([self.ndDataAbsDict,self.ndMCAbsDict,self.fdDataAbsDict,self.fdMCAbsDict])
66 
69 
70  def checkRunCoverage(self,dictList):
71  countBad=0
72  for dict in dictList:
73  lastRun=-1;
74  countFb=0
75  for tuple in sorted (dict.keys()) :
76  checkThis=True
77  if(len(tuple)==2):
78  (first,last)=tuple
79  else:
80  (first,last,fb)=tuple
81  if(fb!=0):
82  checkThis=False
83  countFb=countFb+1
84  else:
85  if(countFb>0):
86  print("Fibre brightness bins "+str(countFb))
87  print("in dictionary: "+str(dict[tuple]))
88  countFb=1
89  if(checkThis and first!=lastRun+1):
90  print("Missing run gap: "+str(lastRun)+" to "+str(first)+" before "+dict[tuple])
91  countBad+=abs(first-(lastRun+1))
92  lastRun=last
93  if(countFb>0):
94  print("Fibre brightness bins "+str(countFb))
95  print("in dictionary: "+str(dict[tuple]))
96 
97  if(countBad==0):
98  print("Found CSV files for all runs in range ")
99  else:
100  print("Missing "+str(countBad)+" files")
101  self.badMissingRuns=True
102 
103 
104 
105  def getAbsFileForThisRun(self,whichDet,dataOrMC,thisRun):
106  thisDict={}
107  if whichDet == 'nd' and dataOrMC=='data':
108  thisDict=self.ndDataAbsDict
109  elif whichDet == 'nd' and dataOrMC=='mc':
110  thisDict=self.ndMCAbsDict
111  elif whichDet == 'fd' and dataOrMC=='data':
112  thisDict=self.fdDataAbsDict
113  elif whichDet == 'fd' and dataOrMC=='mc':
114  thisDict=self.fdMCAbsDict
115  for (k1,k2) in thisDict:
116  if(k1<=thisRun and k2>=thisRun):
117  return thisDict[(k1,k2)]
118 
119  def getAttenFileForThisRun(self,whichDet,dataOrMC,thisRun,whichFb=-1):
120  thisDict={}
121  if whichDet == 'nd' and dataOrMC=='data':
122  thisDict=self.ndDataAttenDict
123  elif whichDet == 'nd' and dataOrMC=='mc':
124  thisDict=self.ndMCAttenDict
125  elif whichDet == 'fd' and dataOrMC=='data':
126  thisDict=self.fdDataAttenDict
127  elif whichDet == 'fd' and dataOrMC=='mc':
128  thisDict=self.fdMCAttenDict
129  if(whichFb<0):
130  for (k1,k2) in thisDict:
131  if(k1<=thisRun and k2>=thisRun):
132  return thisDict[(k1,k2)]
133  else:
134  for (k1,k2,k3) in thisDict:
135  if(k1<=thisRun and k2>=thisRun and k3==whichFb):
136  return thisDict[(k1,k2,k3)]
137 
138 
139 
140  def parseRunString(self,runString):
141  runBoundsList=runString.split('-')
142  minStr=runBoundsList[0]
143  maxStr=runBoundsList[1]
144  minRun=0
145  maxRun=sys.maxint
146  if(len(minStr[1:])>0):
147  minRun=int(minStr[1:])
148  if(len(maxStr[1:])>0):
149  maxRun=int(maxStr[1:])
150  return (minRun,maxRun)
151 
152 
153  def getAbsFiles(self,whichDet,dataOrMC):
154  preString=self.dirName+'/calib_abs_consts.'+whichDet+'.'+dataOrMC
155  absFiles=glob.glob(preString+'*.csv')
156  for absFile in absFiles:
157  verString=absFile[len(preString)+1:].split('.')[0]
158  runString=absFile[len(preString)+1:].split('.')[1]
159  runTuple=self.parseRunString(runString)
160  if whichDet == 'nd' and dataOrMC=='data':
161  self.ndDataAbsDict[runTuple]=absFile
162  elif whichDet == 'nd' and dataOrMC=='mc':
163  self.ndMCAbsDict[runTuple]=absFile
164  elif whichDet == 'fd' and dataOrMC=='data':
165  self.fdDataAbsDict[runTuple]=absFile
166  elif whichDet == 'fd' and dataOrMC=='mc':
167  self.fdMCAbsDict[runTuple]=absFile
168  else:
169  print("Nothing defined for "+whichDet+ ' with '+dataOrMC)
170 
171  def getAttenFiles(self,whichDet,dataOrMC):
172  preString=self.dirName+'/calib_atten_consts.'+whichDet+'.'+dataOrMC
173  attenFiles=glob.glob(preString+'*.csv')
174  for attenFile in attenFiles:
175  verString=attenFile[len(preString)+1:].split('.')[0]
176  runString=attenFile[len(preString)+1:].split('.')[1]
177  fbString=attenFile[len(preString)+1:].split('.')[2]
178  hasFb=False
179  runTuple=self.parseRunString(runString)
180  if "fb" in fbString:
181  hasFb=True
182  runTuple=runTuple+(int(fbString[2:]),)
183  if whichDet == 'nd' and dataOrMC=='data':
184  self.ndDataAttenDict[runTuple]=attenFile
185  elif whichDet == 'nd' and dataOrMC=='mc':
186  self.ndMCAttenDict[runTuple]=attenFile
187  elif whichDet == 'fd' and dataOrMC=='data':
188  self.fdDataAttenDict[runTuple]=attenFile
189  elif whichDet == 'fd' and dataOrMC=='mc':
190  self.fdMCAttenDict[runTuple]=attenFile
191  else:
192  print("Nothing defined for "+whichDet+ ' with '+dataOrMC)
193 
194  def printSummary(self):
195  if self.badMissingRuns:
196  for i in range(0,10):
197  print("There are missing runs!!!!!!!!!!!!!!!!")
198 
199  if self.badTrailingCommas:
200  for i in range(0,10):
201  print("There are traling commas!!!!!!!!!!!!!!!!")
202 
203 
204 def main():
205  if(len(sys.argv)<2):
206  usage()
207  return
208 
209  newDir=sys.argv[1]
210  print("Using files from: "+newDir)
211  nct = NovaCalibTest(newDir)
212  nct.checkAbsRunCoverage()
213  nct.checkAttenRunCoverage()
214  nct.checkAbsFilesForTrailingCommas()
215  nct.checkAttenFilesForTrailingCommas()
216 
217  nct.printSummary()
218  return nct.badMissingRuns+nct.badTrailingCommas
219 
220 if __name__ == "__main__":
221  main()
void split(double tt, double *fr)
void abs(TH1 *hist)
def getAttenFiles(self, whichDet, dataOrMC)
if(dump)
def checkForTrailingCommas(self, fileName)
bool print
procfile open("FD_BRL_v0.txt")
def getAbsFiles(self, whichDet, dataOrMC)
def getAbsFileForThisRun(self, whichDet, dataOrMC, thisRun)
def getAttenFileForThisRun(self, whichDet, dataOrMC, thisRun, whichFb=-1)