TB_WatchdogFx.py

Go to the documentation of this file.

# Python std stuff
import sys, datetime
import os
# all of root 
from ROOT import *
# My utilities
import canMan, HTMLTools as html 
# Hist set for early data plots
from collections import OrderedDict
import psycopg2
import subprocess
import samweb_client 

pubContext = os.environ["SRT_PUBLIC_CONTEXT"]
priContext = os.environ["SRT_PRIVATE_CONTEXT"]
rootLogon=os.path.join(pubContext,"Utilities/rootlogon.C")
gROOT.Macro(rootLogon)

#runHistoryHists       = [rawRunsHist,     rootRunsHist,     recoRunsHist,     cafRunsHist,      dbRunsHist, onMonRunsHist]
#numiRunHistoryHists   = [numiRawRunsHist, numiRootRunsHist, numiRecoRunsHist, numiCafRunsHist,  dbRunsHist, onMonRunsHist]
#calRunHistoryHists   =  [rawRunsHist,     rootRunsHist,     pcHitRunsHist,    attenProfRunsHist,  dbRunsHist, onMonRunsHist]
"""
runHistoryFillStyles  = [1001,      1001,          1001,             1001,             1001,             0,          0]
runHistoryLineWidths  = [1,         1,             1,                1,                1,                2,          2]
runHistoryLineStyles  = [0,         0,             0,                0,                0,                1,          7]
runHistoryColors      = [kRed-9,  kAzure+1,      kOrange+1,        kMagenta - 6,     kGreen - 7,       kBlack,     kRed]
varOpts               = ["][",      "][",          "][",             "][",             "][",             "",          "]["]
"""

emptyFileColor  = kGreen-9
rawColor        = kRed-3
rootColor       = kOrange+1
recoColor       = kMagenta - 7
pidColor        = kAzure+1
cafColor        = kTeal + 9
dbColor         = kBlack
onMonColor      = kGray + 2
#onMonColor      = kBlue -4 

runHistoryFillStyles  = [1001,               1001,          1001,             1001,             1001,                 0,          0]
runHistoryLineWidths  = [1,                     1,             1,                1,                1,                 2,          2]
runHistoryLineStyles  = [0,                     1,             0,                0,                0,                 1,          7]
runHistoryColors      = [emptyFileColor, rawColor,     rootColor,        recoColor,         cafColor,           dbColor,     onMonColor]
varOpts               = ["][",      "][",          "][",             "][",             "][",             "",          "]["]


runHistoryFillStylesWithExtra  = [1001,                  1001,             1001,             1001,              1001,              1001,           0,          0]
runHistoryLineWidthsWithExtra  = [1,                        1,                1,                1,                  1,                0,           2,          2]
runHistoryLineStylesWithExtra  = [0,                        0,                0,                0,                  0,                0,           1,          7]
runHistoryColorsWithExtra      = [emptyFileColor,    rawColor,        rootColor,        recoColor,           pidColor,          cafColor,     dbColor,     onMonColor]
varOptsWithExtra               = ["][",                       "][",             "][",             "][",               "][",             "][",             "",          "]["]




samweb = samweb_client.SAMWebClient(experiment="nova")


class Connection:
  def __init__(self, config, pw):
    try:
        self.conn = psycopg2.connect(config + " password=" + pw)
    except:
        print "I am unable to connect to the database"
        exit(0)
        
    self.cur = self.conn.cursor()


#Establish a connection to the DB. 
pw = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()

dbPort = OrderedDict()
dbPort["testbeam"] = " 5434 "

dbConfig = OrderedDict()
dbConfig["testbeam"] = "dbname=nova_prod host=ifdbrep.fnal.gov user=nova_reader port=5434"

db = OrderedDict()
for det, config in dbConfig.items(): 
  db[det] = Connection(config, pw)



onMonDirs = OrderedDict()
onMonDirs["testbeam"] = "/bluearc/nova/data/nearline-OnMon/TestBeam/S19-10-22/"

onMonTriggerDirs = OrderedDict()
onMonTriggerDirs["testbeam"] = "DDActivity1/"


def fetchTable(cur, query):
  cur.execute(query)
  return cur.fetchall()


def isNumber(s):
  try:
    float(s)
    return True
  except ValueError:
      return False





def getSAMRunsHist(name, maxRunNum, minRunNum, dataSet):
  
  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1, minRunNum, maxRunNum + 1) 
  

  command =  dataSet + " " + " AND Online.RunNumber >= " + str(minRunNum) + " AND Online.RunNumber <= " + str(maxRunNum) 
  print "about to make samweb query:", command
  files = samweb.listFiles(command)
  print "done"
  for file in files:
    if len(file) <= 0:
      continue
    try:
      begRun = file.find("_r")
      endRun = file.find("_", begRun + 1)
      run = int(file[begRun+2:endRun])
      hist.Fill(run)
    except:
      print "oops : ", file
  print "number of files returned: ", len(files)   
  return hist


def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="",  skipStr=None,
                         additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack):
  
  hist = TH1F(name, ";Run Number;Number of Subruns", maxRunNum - minRunNum + 1 , minRunNum, maxRunNum + 1) 
  hist.SetLineStyle(lineStyle)
  hist.SetLineColor(lineColor)

  multiRunDirs = sorted(os.listdir(path))
  # Loop to get info on recent runs
  for multiRunDir in multiRunDirs:
    if not isNumber(multiRunDir):
      continue
    multiRunPath = path + multiRunDir + "/"
    if not os.path.isdir(multiRunPath):
      continue
    runDirs = sorted(os.listdir(multiRunPath))
    for runDir in runDirs:
      try:
        runNum = float(runDir)
      except:
        print "Could not parse runDir", runDir
        continue
      if runNum >= minRunNum and runNum <= maxRunNum:
        runPath = multiRunPath + runDir + "/" + additionalDir
        if not os.path.isdir(runPath):
          continue
        fileNames = sorted(os.listdir(runPath))
        for fileName in fileNames:    
          if checkKeys: 
            filePath = runPath + fileName
            rootFile = TFile(filePath)
            if rootFile.IsZombie() or rootFile.TestBit(TFile.kRecovered): 
              continue
          if not testStr in fileName:
            continue
          if skipStr:
            if skipStr in fileName:
              continue
          hist.Fill(runNum)
          if printRunNum:  
            print runNum
  return hist  

def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7):
  # Get maximum run 
  maxRunNumDB =  fetchTable(db[detector].cur, "select max(run) from "+detector+".runs")[0][0]  #This returns a two by two table, get the 0,0 entry
  #Get run from 7 days ago
  #cur.execute("select min(run) from fardet.runs where tstart >= (CURRENT_TIMESTAMP - interval '7' day);")
  
  minRunNumDB = fetchTable(db[detector].cur, "select min(run) from "+detector+".runs where tstart >= ("+maxTime+" - interval '"+str(interval)+"' day);")[0][0]  #This returns a two by two table, get the 0,0 entry
  return (maxRunNumDB, minRunNumDB)

def getDBRunsHist(minRun, maxRun,table="testbeam",  lineStyle=1, lineColor=dbColor):
  dbRunData = fetchTable(db[table].cur, "select run, nsubruns  from "+table+".runs where run >= "+str(minRun)+" and run <=" + str(maxRun) + " order by run asc;")
  dbRunsHist = TH1F("Number of Subruns in DB", ";Run Number;Number of Subruns", maxRun - minRun+1, minRun, maxRun+1) 
  dbRunsHist.SetLineStyle(lineStyle)
  dbRunsHist.SetLineColor(lineColor)
  for line in dbRunData:
    run = line[0]
    subrun = line[1]
    if run and subrun: dbRunsHist.Fill(float(run), float(subrun))
  return dbRunsHist
  

  
    
def countFiles(dir, testStrs=[]):
  files =  os.listdir(dir)
  count = 0
  for file in files:
    failed = False
    for str in testStrs:
      if not str in file:
        failed = True
        break 
    if not failed:
      count += 1
  return count


class Tier:
  def __init__(self, name, query, color=kBlue, fillStyle=1001, lineWidth=0, lineStyle=0):
    self.name = name
    self.query = query 
    self.color = color 
    self.fillStyle = fillStyle 
    self.lineWidth = lineWidth 
    self.lineStyle = lineStyle 
    self.cache = dict()

  def getHist(self,maxRunNum, minRunNum):
    histId = str(maxRunNum) + "-" + str(minRunNum)
    if histId in self.cache.keys():
      return self.cache[histId]
    hist = getSAMRunsHist(self.name, maxRunNum, minRunNum, self.query)
    hist.SetFillColor(self.color)
    hist.SetLineColor(self.color) 
    hist.SetFillStyle(self.fillStyle)
    hist.SetLineWidth(self.lineWidth)
    hist.SetLineStyle(self.lineStyle)
    for axis in [hist.GetXaxis(), hist.GetYaxis()]:
      axis.SetNoExponent()
    self.cache[histId] = hist
    return hist 





class Path:
  def __init__(self, name, title, legWidth=0.4, legHeight=0.3, padFactor=2):
    # Things about this path histogram set 
    self.name           = name
    self.title          = title 
    self.legWidth       = legWidth 
    self.legHeight      = legHeight
    self.padFactor      = padFactor

    # List of tier objects 
    self.tiers          = []
  
  
  # Add tier to path 
  def addTier(self, tier):
    self.tiers.append(tier)

  # Get all of the histograms for this path 
  def getHists(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
    hists = []
    for tier in self.tiers: 
      hists.append(tier.getHist(maxRunNum, minRunNum))
    hists.append(dbRunsHist)
#    hists.append(onMonRunsHist)
    return hists
  def getPlotCanvas(self, maxRunNum, minRunNum, dbRunsHist, onMonRunsHist):
    hists = self.getHists(maxRunNum, minRunNum, dbRunsHist, onMonRunsHist)
    # Hardcoding legend position. autoPlaceLegend() was putting the legend on top of the data for ND.
    can = canMan.drawOverlay(hists, varOpts=varOpts, name=self.name, legPos = [0.15,0.55], legWidth=0.4, legHeight=0.3, padFactor=2, preserveStyle=True, yOffset=0.9)
    return (can,hists)

  # Generator for tier names 
  def tierNames(self):
    for tier in self.tiers:
      yield tier.name     

  # Generator for queries 
  def queries(self):
    for tier in self.tiers:
      yield tier.query 

  # Generator for colors  
  def colors(self):
    for tier in self.tiers:
      yield tier.color 

  # Generator for fillStyles  
  def fillStyles(self):
    for tier in self.tiers:
      yield tier.fillStyle  
  
  # Generator for lineWidths  
  def lineWidths(self):
    for tier in self.tiers:
      yield tier.lineWidths  

  # Generator for lineStyles  
  def lineStyles(self):
    for tier in self.tiers:
      yield tier.lineStyle  



paths = OrderedDict()

paths["testbeam_act"] = Path("testbeam_act", "TB DDActivity1 Triggers")

paths["testbeam_act"].addTier(Tier("DDActivity1 Empty Raw Files",
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1', 
           color=emptyFileColor))
paths["testbeam_act"].addTier(Tier("DDActivity1 Raw Files",      
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=DDActivity1 AND Online.TotalEvents > 0',
           color=rawColor))
paths["testbeam_act"].addTier(Tier("DDActivity1 ROOT Files",     
           'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=DDActivity1  AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
           color=rootColor))
#-------------------------------------------------------

paths["testbeam_beamline"] = Path("testbeam_beamline", "TB Beamline Triggers")

tbBeamlineRawEmpty = Tier("Beamline Empty Raw Files",  
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline',
           color=emptyFileColor )
paths["testbeam_beamline"].addTier(tbBeamlineRawEmpty)

tbBeamlineRaw =  Tier("Beamline Raw Files",        
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Beamline AND Online.TotalEvents > 0',
           color=rawColor )
paths["testbeam_beamline"].addTier(tbBeamlineRaw)
tbBeamlineRoot = Tier("Beamline ROOT Files",       
           'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Beamline AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
           color=rootColor )
paths["testbeam_beamline"].addTier(tbBeamlineRoot)

#-------------------------------------------------------

paths["testbeam_spill"] = Path("testbeam_spill", "TB Spill Triggers")

tbSpillRawEmpty = Tier("TBSpill Empty Raw Files",  
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill',
           color=emptyFileColor )
paths["testbeam_spill"].addTier(tbSpillRawEmpty)

tbSpillRaw =  Tier("TBSpill Raw Files",        
           'Online.Detector="testbeam" AND data_tier="raw" AND Online.Stream=Spill AND Online.TotalEvents > 0',
           color=rawColor )
paths["testbeam_spill"].addTier(tbSpillRaw)
tbSpillRoot = Tier("TBSpill ROOT Files",       
           'Online.Detector="testbeam" AND data_tier="artdaq" AND Online.Stream=Spill AND (daq2rawdigit.base_release R19-09-24-testbeam-production.a OR daq2rawdigit.base_release R19-09-24-testbeam-production.b OR daq2rawdigit.base_release R19-09-24-testbeam-production.c)',
           color=rootColor )
paths["testbeam_spill"].addTier(tbSpillRoot)

#-------------------------------------------------------