TB_WatchdogFx.py
Go to the documentation of this file.
1 # Python std stuff
2 import sys, datetime
3 import os
4 # all of root
5 from ROOT import *
6 # My utilities
7 import canMan, HTMLTools as html
8 # Hist set for early data plots
9 from collections import OrderedDict
10 import psycopg2
11 import subprocess
12 import samweb_client
13 
14 pubContext = os.environ["SRT_PUBLIC_CONTEXT"]
15 priContext = os.environ["SRT_PRIVATE_CONTEXT"]
16 rootLogon=os.path.join(pubContext,"Utilities/rootlogon.C")
17 gROOT.Macro(rootLogon)
18 
19 #runHistoryHists = [rawRunsHist, rootRunsHist, recoRunsHist, cafRunsHist, dbRunsHist, onMonRunsHist]
20 #numiRunHistoryHists = [numiRawRunsHist, numiRootRunsHist, numiRecoRunsHist, numiCafRunsHist, dbRunsHist, onMonRunsHist]
21 #calRunHistoryHists = [rawRunsHist, rootRunsHist, pcHitRunsHist, attenProfRunsHist, dbRunsHist, onMonRunsHist]
22 """
23 runHistoryFillStyles = [1001, 1001, 1001, 1001, 1001, 0, 0]
24 runHistoryLineWidths = [1, 1, 1, 1, 1, 2, 2]
25 runHistoryLineStyles = [0, 0, 0, 0, 0, 1, 7]
26 runHistoryColors = [kRed-9, kAzure+1, kOrange+1, kMagenta - 6, kGreen - 7, kBlack, kRed]
27 varOpts = ["][", "][", "][", "][", "][", "", "]["]
28 """
29 
30 emptyFileColor = kGreen-9
31 rawColor = kRed-3
32 rootColor = kOrange+1
33 recoColor = kBlue-3
34 pidColor = kAzure+1
35 pcColor = kAzure+2
36 cafColor = kTeal + 9
37 dbColor = kBlack
38 onMonColor = kGray + 2
39 #onMonColor = kBlue -4
40 
41 runHistoryFillStyles = [1001, 1001, 1001, 1001, 1001, 0, 0]
42 runHistoryLineWidths = [1, 1, 1, 1, 1, 2, 2]
43 runHistoryLineStyles = [0, 1, 0, 0, 0, 1, 7]
44 runHistoryColors = [emptyFileColor, rawColor, rootColor, recoColor, cafColor, dbColor, onMonColor]
45 varOpts = ["][", "][", "][", "][", "][", "", "]["]
46 
47 
48 runHistoryFillStylesWithExtra = [1001, 1001, 1001, 1001, 1001, 1001, 0, 0]
49 runHistoryLineWidthsWithExtra = [1, 1, 1, 1, 1, 0, 2, 2]
50 runHistoryLineStylesWithExtra = [0, 0, 0, 0, 0, 0, 1, 7]
51 runHistoryColorsWithExtra = [emptyFileColor, rawColor, rootColor, recoColor, pidColor, cafColor, dbColor, onMonColor]
52 varOptsWithExtra = ["][", "][", "][", "][", "][", "][", "", "]["]
53 
54 
55 
56 
57 samweb = samweb_client.SAMWebClient(experiment="nova")
58 
59 
60 class Connection:
61  def __init__(self, config, pw):
62  try:
63  self.conn = psycopg2.connect(config + " password=" + pw)
64  except:
65  print "I am unable to connect to the database"
66  exit(0)
67 
68  self.cur = self.conn.cursor()
69 
70 
71 #Establish a connection to the DB.
72 pw = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
73 
74 dbPort = OrderedDict()
75 dbPort["testbeam"] = " 5434 "
76 
77 dbConfig = OrderedDict()
78 dbConfig["testbeam"] = "dbname=nova_prod host=ifdbrep.fnal.gov user=nova_reader port=5434"
79 
80 db = OrderedDict()
81 for det, config in dbConfig.items():
82  db[det] = Connection(config, pw)
83 
84 
85 
86 onMonDirs = OrderedDict()
87 onMonDirs["testbeam"] = "/nova/data/nearline-OnMon/TestBeam/S19-10-22/"
88 
89 onMonTriggerDirs = OrderedDict()
90 onMonTriggerDirs["testbeam"] = "DDActivity1/"
91 
92 
93 def fetchTable(cur, query):
94  cur.execute(query)
95  return cur.fetchall()
96 
97 
98 def isNumber(s):
99  try:
100  float(s)
101  return True
102  except ValueError:
103  return False
104 
105 
106 
107 
108 
109 def getSAMRunsHist(name, maxRunNum, minRunNum, dataSet):
110 
111  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1, minRunNum, maxRunNum + 1)
112 
113 
114  command = dataSet + " " + " AND Online.RunNumber >= " + str(minRunNum) + " AND Online.RunNumber <= " + str(maxRunNum)
115  print "about to make samweb query:", command
116  files = samweb.listFiles(command)
117  print "done"
118  for file in files:
119  if len(file) <= 0:
120  continue
121  try:
122  begRun = file.find("_r")
123  endRun = file.find("_", begRun + 1)
124  run = int(file[begRun+2:endRun])
125  hist.Fill(run)
126  except:
127  print "oops : ", file
128  print "number of files returned: ", len(files)
129  return hist
130 
131 
132 def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="", skipStr=None,
133  additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack):
134 
135  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1 , minRunNum, maxRunNum + 1)
136  hist.SetLineStyle(lineStyle)
137  hist.SetLineColor(lineColor)
138 
139  multiRunDirs = sorted(os.listdir(path))
140  # Loop to get info on recent runs
141  for multiRunDir in multiRunDirs:
142  if not isNumber(multiRunDir):
143  continue
144  multiRunPath = path + multiRunDir + "/"
145  if not os.path.isdir(multiRunPath):
146  continue
147  runDirs = sorted(os.listdir(multiRunPath))
148  for runDir in runDirs:
149  try:
150  runNum = float(runDir)
151  except:
152  print "Could not parse runDir", runDir
153  continue
154  if runNum >= minRunNum and runNum <= maxRunNum:
155  runPath = multiRunPath + runDir + "/" + additionalDir
156  if not os.path.isdir(runPath):
157  continue
158  fileNames = sorted(os.listdir(runPath))
159  for fileName in fileNames:
160  if checkKeys:
161  filePath = runPath + fileName
162  rootFile = TFile(filePath)
163  if rootFile.IsZombie() or rootFile.TestBit(TFile.kRecovered):
164  continue
165  if not testStr in fileName:
166  continue
167  if skipStr:
168  if skipStr in fileName:
169  continue
170  hist.Fill(runNum)
171  if printRunNum:
172  print runNum
173  return hist
174 
175 def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7):
176  # Get maximum run
177  maxRunNumDB = fetchTable(db[detector].cur, "select max(run) from "+detector+".runs")[0][0] #This returns a two by two table, get the 0,0 entry
178  #Get run from 7 days ago
179  #cur.execute("select min(run) from fardet.runs where tstart >= (CURRENT_TIMESTAMP - interval '7' day);")
180 
181  minRunNumDB = fetchTable(db[detector].cur, "select min(run) from "+detector+".runs where tstart >= ("+maxTime+" - interval '"+str(interval)+"' day);")[0][0] #This returns a two by two table, get the 0,0 entry
182  return (maxRunNumDB, minRunNumDB)
183 
184 def getDBRunsHist(minRun, maxRun,table="testbeam", lineStyle=1, lineColor=dbColor):
185  dbRunData = fetchTable(db[table].cur, "select run, nsubruns from "+table+".runs where run >= "+str(minRun)+" and run <=" + str(maxRun) + " order by run asc;")
186  dbRunsHist = TH1F("Number of Subruns in DB", ";Run Number;Number of Subruns", maxRun - minRun+1, minRun, maxRun+1)
187  dbRunsHist.SetLineStyle(lineStyle)
188  dbRunsHist.SetLineColor(lineColor)
189  for line in dbRunData:
190  run = line[0]
191  subrun = line[1]
192  if run and subrun: dbRunsHist.Fill(float(run), float(subrun))
193  return dbRunsHist
194 
195 
196 
197 
198 def countFiles(dir, testStrs=[]):
199  files = os.listdir(dir)
200  count = 0
201  for file in files:
202  failed = False
203  for str in testStrs:
204  if not str in file:
205  failed = True
206  break
207  if not failed:
208  count += 1
209  return count
210 
211 
212 class Tier:
213  def __init__(self, name, query, color=kBlue, fillStyle=1001, lineWidth=0, lineStyle=0):
214  self.name = name
215  self.query = query
216  self.color = color
217  self.fillStyle = fillStyle
218  self.lineWidth = lineWidth
219  self.lineStyle = lineStyle
220  self.cache = dict()
221 
222  def getHist(self,maxRunNum, minRunNum):
223  histId = str(maxRunNum) + "-" + str(minRunNum)
224  if histId in self.cache.keys():
225  return self.cache[histId]
226  hist = getSAMRunsHist(self.name, maxRunNum, minRunNum, self.query)
227  hist.SetFillColor(self.color)
228  hist.SetLineColor(self.color)
229  hist.SetFillStyle(self.fillStyle)
230  hist.SetLineWidth(self.lineWidth)
231  hist.SetLineStyle(self.lineStyle)
232  for axis in [hist.GetXaxis(), hist.GetYaxis()]:
233  axis.SetNoExponent()
234  self.cache[histId] = hist
235  return hist
236 
237 
238 
239 
240 
241 class Path:
242  def __init__(self, name, title, legWidth=0.4, legHeight=0.3, padFactor=2):
243  # Things about this path histogram set
244  self.name = name
245  self.title = title
246  self.legWidth = legWidth
247  self.legHeight = legHeight
248  self.padFactor = padFactor
249 
250  # List of tier objects
251  self.tiers = []
252 
253 
254  # Add tier to path
255  def addTier(self, tier):
256  self.tiers.append(tier)
257 
258  # Get all of the histograms for this path
259  def getHists(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
260  hists = []
261  for tier in self.tiers:
262  hists.append(tier.getHist(maxRunNum, minRunNum))
263  hists.append(dbRunsHist)
264 # hists.append(onMonRunsHist)
265  return hists
266  def getPlotCanvas(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
267  hists = self.getHists(maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
268  # Hardcoding legend position. autoPlaceLegend() was putting the legend on top of the data for ND.
269  can = canMan.drawOverlay(hists, varOpts=varOpts, name=self.name, legPos = [0.15,0.55], legWidth=0.4, legHeight=0.3, padFactor=2, preserveStyle=True, yOffset=0.9)
270  return (can,hists)
271 
272  # Generator for tier names
273  def tierNames(self):
274  for tier in self.tiers:
275  yield tier.name
276 
277  # Generator for queries
278  def queries(self):
279  for tier in self.tiers:
280  yield tier.query
281 
282  # Generator for colors
283  def colors(self):
284  for tier in self.tiers:
285  yield tier.color
286 
287  # Generator for fillStyles
288  def fillStyles(self):
289  for tier in self.tiers:
290  yield tier.fillStyle
291 
292  # Generator for lineWidths
293  def lineWidths(self):
294  for tier in self.tiers:
295  yield tier.lineWidths
296 
297  # Generator for lineStyles
298  def lineStyles(self):
299  for tier in self.tiers:
300  yield tier.lineStyle
301 
302 
303 
304 paths = OrderedDict()
305 
306 paths["testbeam_act"] = Path("testbeam_act", "TB DDActivity1 Triggers")
307 
308 paths["testbeam_act"].addTier(Tier("DDActivity1 Empty Raw Files",
309  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1',
310  color=emptyFileColor))
311 paths["testbeam_act"].addTier(Tier("DDActivity1 Raw Files",
312  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1 AND Online.TotalEvents > 0',
313  color=rawColor))
314 paths["testbeam_act"].addTier(Tier("DDActivity1 ROOT Files",
315  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=DDActivity1 AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c OR daq2rawdigit.base_release R21-01-28-testbeam-production.a)',
316  color=rootColor))
317 paths["testbeam_act"].addTier(Tier("DDActivity1 PCList Files",
318  'Online.Detector="testbeam" AND data_tier="pclist" AND Online.Stream=DDActivity1 AND (NOVA.Release R21-01-28-testbeam-production.a)',
319  color=pcColor))
320 
321 #-------------------------------------------------------
322 
323 paths["testbeam_beamline"] = Path("testbeam_beamline", "TB Beamline Triggers")
324 
325 tbBeamlineRawEmpty = Tier("Beamline Empty Raw Files",
326  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline',
327  color=emptyFileColor )
328 paths["testbeam_beamline"].addTier(tbBeamlineRawEmpty)
329 
330 tbBeamlineRaw = Tier("Beamline Raw Files",
331  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline AND Online.TotalEvents > 0',
332  color=rawColor )
333 paths["testbeam_beamline"].addTier(tbBeamlineRaw)
334 tbBeamlineRoot = Tier("Beamline ROOT Files",
335  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Beamline AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c OR daq2rawdigit.base_release R21-01-28-testbeam-production.a)',
336  color=rootColor )
337 paths["testbeam_beamline"].addTier(tbBeamlineRoot)
338 paths["testbeam_beamline"].addTier(Tier("Beamline Reco Files",
339  'Online.Detector="testbeam" AND data_tier="tbreco" AND Online.Stream=Beamline AND (NOVA.Release R21-01-28-testbeam-production.a)',
340  color=recoColor))
341 #-------------------------------------------------------
342 
343 paths["testbeam_spill"] = Path("testbeam_spill", "TB Spill Triggers")
344 
345 tbSpillRawEmpty = Tier("TBSpill Empty Raw Files",
346  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill',
347  color=emptyFileColor )
348 paths["testbeam_spill"].addTier(tbSpillRawEmpty)
349 
350 tbSpillRaw = Tier("TBSpill Raw Files",
351  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill AND Online.TotalEvents > 0',
352  color=rawColor )
353 paths["testbeam_spill"].addTier(tbSpillRaw)
354 tbSpillRoot = Tier("TBSpill ROOT Files",
355  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Spill AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c OR daq2rawdigit.base_release R21-01-28-testbeam-production.a)',
356  color=rootColor )
357 paths["testbeam_spill"].addTier(tbSpillRoot)
358 paths["testbeam_spill"].addTier(Tier("Spill Reco Files",
359  'Online.Detector="testbeam" AND data_tier="tbreco" AND Online.Stream=Spill AND (NOVA.Release R21-01-28-testbeam-production.a)',
360  color=recoColor))
361 
362 #-------------------------------------------------------
def fetchTable(cur, query)
def getHist(self, maxRunNum, minRunNum)
def __init__(self, name, title, legWidth=0.4, legHeight=0.3, padFactor=2)
def __init__(self, config, pw)
def getPlotCanvas(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
def addTier(self, tier)
def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7)
procfile open("FD_BRL_v0.txt")
def countFiles(dir, testStrs=[])
def __init__(self, name, query, color=kBlue, fillStyle=1001, lineWidth=0, lineStyle=0)
def getHists(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
exit(0)
def getSAMRunsHist(name, maxRunNum, minRunNum, dataSet)
def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="", skipStr=None, additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack)
def drawOverlay(listOfHists, name="", opt="hist", varOpts=[""], colors=[kRed, kBlue, kGreen, kMagenta, kOrange, kBlack, kViolet, lineStyles=[1], legEntries=[], logY=False, axisMin=0, legPos="right", legTextSize=0, inputCan=None, legHeight=0.15, legWidth=0.2, supressZero=False, padFactor=1.1, lineWidths=[gStyle.GetHistLineWidth()], fillStyles=[0], preserveStyle=False, yOffset=1.05)
Definition: canMan.py:64
def getDBRunsHist(minRun, maxRun, table="testbeam", lineStyle=1, lineColor=dbColor)