TB_WatchdogFx.py
Go to the documentation of this file.
1 # Python std stuff
2 import sys, datetime
3 import os
4 # all of root
5 from ROOT import *
6 # My utilities
7 import canMan, HTMLTools as html
8 # Hist set for early data plots
9 from collections import OrderedDict
10 import psycopg2
11 import subprocess
12 import samweb_client
13 
14 pubContext = os.environ["SRT_PUBLIC_CONTEXT"]
15 priContext = os.environ["SRT_PRIVATE_CONTEXT"]
16 rootLogon=os.path.join(pubContext,"Utilities/rootlogon.C")
17 gROOT.Macro(rootLogon)
18 
19 #runHistoryHists = [rawRunsHist, rootRunsHist, recoRunsHist, cafRunsHist, dbRunsHist, onMonRunsHist]
20 #numiRunHistoryHists = [numiRawRunsHist, numiRootRunsHist, numiRecoRunsHist, numiCafRunsHist, dbRunsHist, onMonRunsHist]
21 #calRunHistoryHists = [rawRunsHist, rootRunsHist, pcHitRunsHist, attenProfRunsHist, dbRunsHist, onMonRunsHist]
22 """
23 runHistoryFillStyles = [1001, 1001, 1001, 1001, 1001, 0, 0]
24 runHistoryLineWidths = [1, 1, 1, 1, 1, 2, 2]
25 runHistoryLineStyles = [0, 0, 0, 0, 0, 1, 7]
26 runHistoryColors = [kRed-9, kAzure+1, kOrange+1, kMagenta - 6, kGreen - 7, kBlack, kRed]
27 varOpts = ["][", "][", "][", "][", "][", "", "]["]
28 """
29 
30 emptyFileColor = kGreen-9
31 rawColor = kRed-3
32 rootColor = kOrange+1
33 recoColor = kMagenta - 7
34 pidColor = kAzure+1
35 cafColor = kTeal + 9
36 dbColor = kBlack
37 onMonColor = kGray + 2
38 #onMonColor = kBlue -4
39 
40 runHistoryFillStyles = [1001, 1001, 1001, 1001, 1001, 0, 0]
41 runHistoryLineWidths = [1, 1, 1, 1, 1, 2, 2]
42 runHistoryLineStyles = [0, 1, 0, 0, 0, 1, 7]
43 runHistoryColors = [emptyFileColor, rawColor, rootColor, recoColor, cafColor, dbColor, onMonColor]
44 varOpts = ["][", "][", "][", "][", "][", "", "]["]
45 
46 
47 runHistoryFillStylesWithExtra = [1001, 1001, 1001, 1001, 1001, 1001, 0, 0]
48 runHistoryLineWidthsWithExtra = [1, 1, 1, 1, 1, 0, 2, 2]
49 runHistoryLineStylesWithExtra = [0, 0, 0, 0, 0, 0, 1, 7]
50 runHistoryColorsWithExtra = [emptyFileColor, rawColor, rootColor, recoColor, pidColor, cafColor, dbColor, onMonColor]
51 varOptsWithExtra = ["][", "][", "][", "][", "][", "][", "", "]["]
52 
53 
54 
55 
56 samweb = samweb_client.SAMWebClient(experiment="nova")
57 
58 
59 class Connection:
60  def __init__(self, config, pw):
61  try:
62  self.conn = psycopg2.connect(config + " password=" + pw)
63  except:
64  print "I am unable to connect to the database"
65  exit(0)
66 
67  self.cur = self.conn.cursor()
68 
69 
70 #Establish a connection to the DB.
71 pw = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
72 
73 dbPort = OrderedDict()
74 dbPort["testbeam"] = " 5434 "
75 
76 dbConfig = OrderedDict()
77 dbConfig["testbeam"] = "dbname=nova_prod host=ifdbrep.fnal.gov user=nova_reader port=5434"
78 
79 db = OrderedDict()
80 for det, config in dbConfig.items():
81  db[det] = Connection(config, pw)
82 
83 
84 
85 onMonDirs = OrderedDict()
86 onMonDirs["testbeam"] = "/bluearc/nova/data/nearline-OnMon/TestBeam/S19-10-22/"
87 
88 onMonTriggerDirs = OrderedDict()
89 onMonTriggerDirs["testbeam"] = "DDActivity1/"
90 
91 
92 def fetchTable(cur, query):
93  cur.execute(query)
94  return cur.fetchall()
95 
96 
97 def isNumber(s):
98  try:
99  float(s)
100  return True
101  except ValueError:
102  return False
103 
104 
105 
106 
107 
108 def getSAMRunsHist(name, maxRunNum, minRunNum, dataSet):
109 
110  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1, minRunNum, maxRunNum + 1)
111 
112 
113  command = dataSet + " " + " AND Online.RunNumber >= " + str(minRunNum) + " AND Online.RunNumber <= " + str(maxRunNum)
114  print "about to make samweb query:", command
115  files = samweb.listFiles(command)
116  print "done"
117  for file in files:
118  if len(file) <= 0:
119  continue
120  try:
121  begRun = file.find("_r")
122  endRun = file.find("_", begRun + 1)
123  run = int(file[begRun+2:endRun])
124  hist.Fill(run)
125  except:
126  print "oops : ", file
127  print "number of files returned: ", len(files)
128  return hist
129 
130 
131 def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="", skipStr=None,
132  additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack):
133 
134  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1 , minRunNum, maxRunNum + 1)
135  hist.SetLineStyle(lineStyle)
136  hist.SetLineColor(lineColor)
137 
138  multiRunDirs = sorted(os.listdir(path))
139  # Loop to get info on recent runs
140  for multiRunDir in multiRunDirs:
141  if not isNumber(multiRunDir):
142  continue
143  multiRunPath = path + multiRunDir + "/"
144  if not os.path.isdir(multiRunPath):
145  continue
146  runDirs = sorted(os.listdir(multiRunPath))
147  for runDir in runDirs:
148  try:
149  runNum = float(runDir)
150  except:
151  print "Could not parse runDir", runDir
152  continue
153  if runNum >= minRunNum and runNum <= maxRunNum:
154  runPath = multiRunPath + runDir + "/" + additionalDir
155  if not os.path.isdir(runPath):
156  continue
157  fileNames = sorted(os.listdir(runPath))
158  for fileName in fileNames:
159  if checkKeys:
160  filePath = runPath + fileName
161  rootFile = TFile(filePath)
162  if rootFile.IsZombie() or rootFile.TestBit(TFile.kRecovered):
163  continue
164  if not testStr in fileName:
165  continue
166  if skipStr:
167  if skipStr in fileName:
168  continue
169  hist.Fill(runNum)
170  if printRunNum:
171  print runNum
172  return hist
173 
174 def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7):
175  # Get maximum run
176  maxRunNumDB = fetchTable(db[detector].cur, "select max(run) from "+detector+".runs")[0][0] #This returns a two by two table, get the 0,0 entry
177  #Get run from 7 days ago
178  #cur.execute("select min(run) from fardet.runs where tstart >= (CURRENT_TIMESTAMP - interval '7' day);")
179 
180  minRunNumDB = fetchTable(db[detector].cur, "select min(run) from "+detector+".runs where tstart >= ("+maxTime+" - interval '"+str(interval)+"' day);")[0][0] #This returns a two by two table, get the 0,0 entry
181  return (maxRunNumDB, minRunNumDB)
182 
183 def getDBRunsHist(minRun, maxRun,table="testbeam", lineStyle=1, lineColor=dbColor):
184  dbRunData = fetchTable(db[table].cur, "select run, nsubruns from "+table+".runs where run >= "+str(minRun)+" and run <=" + str(maxRun) + " order by run asc;")
185  dbRunsHist = TH1F("Number of Subruns in DB", ";Run Number;Number of Subruns", maxRun - minRun+1, minRun, maxRun+1)
186  dbRunsHist.SetLineStyle(lineStyle)
187  dbRunsHist.SetLineColor(lineColor)
188  for line in dbRunData:
189  run = line[0]
190  subrun = line[1]
191  if run and subrun: dbRunsHist.Fill(float(run), float(subrun))
192  return dbRunsHist
193 
194 
195 
196 
197 def countFiles(dir, testStrs=[]):
198  files = os.listdir(dir)
199  count = 0
200  for file in files:
201  failed = False
202  for str in testStrs:
203  if not str in file:
204  failed = True
205  break
206  if not failed:
207  count += 1
208  return count
209 
210 
211 class Tier:
212  def __init__(self, name, query, color=kBlue, fillStyle=1001, lineWidth=0, lineStyle=0):
213  self.name = name
214  self.query = query
215  self.color = color
216  self.fillStyle = fillStyle
217  self.lineWidth = lineWidth
218  self.lineStyle = lineStyle
219  self.cache = dict()
220 
221  def getHist(self,maxRunNum, minRunNum):
222  histId = str(maxRunNum) + "-" + str(minRunNum)
223  if histId in self.cache.keys():
224  return self.cache[histId]
225  hist = getSAMRunsHist(self.name, maxRunNum, minRunNum, self.query)
226  hist.SetFillColor(self.color)
227  hist.SetLineColor(self.color)
228  hist.SetFillStyle(self.fillStyle)
229  hist.SetLineWidth(self.lineWidth)
230  hist.SetLineStyle(self.lineStyle)
231  for axis in [hist.GetXaxis(), hist.GetYaxis()]:
232  axis.SetNoExponent()
233  self.cache[histId] = hist
234  return hist
235 
236 
237 
238 
239 
240 class Path:
241  def __init__(self, name, title, legWidth=0.4, legHeight=0.3, padFactor=2):
242  # Things about this path histogram set
243  self.name = name
244  self.title = title
245  self.legWidth = legWidth
246  self.legHeight = legHeight
247  self.padFactor = padFactor
248 
249  # List of tier objects
250  self.tiers = []
251 
252 
253  # Add tier to path
254  def addTier(self, tier):
255  self.tiers.append(tier)
256 
257  # Get all of the histograms for this path
258  def getHists(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
259  hists = []
260  for tier in self.tiers:
261  hists.append(tier.getHist(maxRunNum, minRunNum))
262  hists.append(dbRunsHist)
263 # hists.append(onMonRunsHist)
264  return hists
265  def getPlotCanvas(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
266  hists = self.getHists(maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
267  # Hardcoding legend position. autoPlaceLegend() was putting the legend on top of the data for ND.
268  can = canMan.drawOverlay(hists, varOpts=varOpts, name=self.name, legPos = [0.15,0.55], legWidth=0.4, legHeight=0.3, padFactor=2, preserveStyle=True, yOffset=0.9)
269  return (can,hists)
270 
271  # Generator for tier names
272  def tierNames(self):
273  for tier in self.tiers:
274  yield tier.name
275 
276  # Generator for queries
277  def queries(self):
278  for tier in self.tiers:
279  yield tier.query
280 
281  # Generator for colors
282  def colors(self):
283  for tier in self.tiers:
284  yield tier.color
285 
286  # Generator for fillStyles
287  def fillStyles(self):
288  for tier in self.tiers:
289  yield tier.fillStyle
290 
291  # Generator for lineWidths
292  def lineWidths(self):
293  for tier in self.tiers:
294  yield tier.lineWidths
295 
296  # Generator for lineStyles
297  def lineStyles(self):
298  for tier in self.tiers:
299  yield tier.lineStyle
300 
301 
302 
303 paths = OrderedDict()
304 
305 paths["testbeam_act"] = Path("testbeam_act", "TB DDActivity1 Triggers")
306 
307 paths["testbeam_act"].addTier(Tier("DDActivity1 Empty Raw Files",
308  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1',
309  color=emptyFileColor))
310 paths["testbeam_act"].addTier(Tier("DDActivity1 Raw Files",
311  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1 AND Online.TotalEvents > 0',
312  color=rawColor))
313 paths["testbeam_act"].addTier(Tier("DDActivity1 ROOT Files",
314  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=DDActivity1 AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
315  color=rootColor))
316 #-------------------------------------------------------
317 
318 paths["testbeam_beamline"] = Path("testbeam_beamline", "TB Beamline Triggers")
319 
320 tbBeamlineRawEmpty = Tier("Beamline Empty Raw Files",
321  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline',
322  color=emptyFileColor )
323 paths["testbeam_beamline"].addTier(tbBeamlineRawEmpty)
324 
325 tbBeamlineRaw = Tier("Beamline Raw Files",
326  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline AND Online.TotalEvents > 0',
327  color=rawColor )
328 paths["testbeam_beamline"].addTier(tbBeamlineRaw)
329 tbBeamlineRoot = Tier("Beamline ROOT Files",
330  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Beamline AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
331  color=rootColor )
332 paths["testbeam_beamline"].addTier(tbBeamlineRoot)
333 
334 #-------------------------------------------------------
335 
336 paths["testbeam_spill"] = Path("testbeam_spill", "TB Spill Triggers")
337 
338 tbSpillRawEmpty = Tier("TBSpill Empty Raw Files",
339  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill',
340  color=emptyFileColor )
341 paths["testbeam_spill"].addTier(tbSpillRawEmpty)
342 
343 tbSpillRaw = Tier("TBSpill Raw Files",
344  'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill AND Online.TotalEvents > 0',
345  color=rawColor )
346 paths["testbeam_spill"].addTier(tbSpillRaw)
347 tbSpillRoot = Tier("TBSpill ROOT Files",
348  'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Spill AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
349  color=rootColor )
350 paths["testbeam_spill"].addTier(tbSpillRoot)
351 
352 #-------------------------------------------------------
def fetchTable(cur, query)
def getHist(self, maxRunNum, minRunNum)
def __init__(self, name, title, legWidth=0.4, legHeight=0.3, padFactor=2)
def __init__(self, config, pw)
def getPlotCanvas(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
def addTier(self, tier)
def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7)
procfile open("FD_BRL_v0.txt")
def countFiles(dir, testStrs=[])
def __init__(self, name, query, color=kBlue, fillStyle=1001, lineWidth=0, lineStyle=0)
def getHists(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
exit(0)
def getSAMRunsHist(name, maxRunNum, minRunNum, dataSet)
def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="", skipStr=None, additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack)
def drawOverlay(listOfHists, name="", opt="hist", varOpts=[""], colors=[kRed, kBlue, kGreen, kMagenta, kOrange, kBlack, kViolet, lineStyles=[1], legEntries=[], logY=False, axisMin=0, legPos="right", legTextSize=0, inputCan=None, legHeight=0.15, legWidth=0.2, supressZero=False, padFactor=1.1, lineWidths=[gStyle.GetHistLineWidth()], fillStyles=[0], preserveStyle=False, yOffset=1.05)
Definition: canMan.py:64
def getDBRunsHist(minRun, maxRun, table="testbeam", lineStyle=1, lineColor=dbColor)