recentWatchdog.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 # Python std stuff
4 import sys, datetime
5 import os
6 # all of root
7 from ROOT import *
8 # My utilities
9 import canMan, HTMLTools as html
10 # Hist set for early data plots
11 from collections import OrderedDict
12 import psycopg2
13 from WatchdogFx import *
14 
15 
16 gEnv.SetValue("TFile.Recover", 0)
17 
18 
19 webDir = "/nusoft/app/web/htdoc/nova/datacheck/watchdogs/"
20 
21 
22 
23 
24 fullNames = OrderedDict()
25 fullNames["fardet"] = "Far Detector"
26 fullNames["neardet"] = "Near Detector"
27 
28 now = datetime.datetime.now()
29 dateStr = str(now.year) + "-"+ str(now.month) + "-" + str(now.day) + "_" + str(now.hour) + "-" + str(now.minute)
30 outputDir = webDir + dateStr + "/"
31 print "Output results in : ", outputDir
32 if not os.path.isdir(outputDir):
33  os.mkdir(outputDir)
34 
35 url = "http://nusoft.fnal.gov/nova/datacheck/watchdogs/" + dateStr + "/watchdog.html"
36 
37 recoveryUrl = "http://cdcvs.fnal.gov/redmine/projects/datacheck/wiki/Recovering_Missing_Files_from_FTS"
38 
39 
40 outputHtml = open(outputDir + "watchdog.html", 'w')
41 outputHtml.write(html.preventCaching())
42 
43 outputHtml.write(html.bootStrap())
44 
45 files = os.listdir(outputDir)
46 outputHtml.write(html.title("Watchdog page for " + dateStr))
47 jumboText = html.heading1("NOvA Offline Data Watchdog FTS Summary")
48 jumboText += html.paragraph("Generated on " + now.strftime("%Y/%m/%d at %H:%M") + " central time. ")
49 jumboText += html.paragraph("Updates every 4 hours.")
50 jumboText += html.button("Permalink", url)
51 jumboText += html.button("Latest page", "http://nusoft.fnal.gov/nova/datacheck/watchdogs/latest/watchdog.html")
52 
53 outputHtml.write(html.jumbo(jumboText))
54 outputHtml.write(html.paragraph("If you notice missing files are more than 48 hours old, then there may be a problem with FTS or the network. Please send an email to nova_daq_support@fnal.gov"))
55 #outputHtml.write(html.paragraph("If you notice missing files, to find what are the issues and more info are here " + html.link("recovery instructions", recoveryUrl, newTab=True)+"."))
56 
57 outputHtml.write(html.paragraph("<b>Missing files in SAM:</b> If the color fill is lower than the black solid line (ie. if there is a white gap between the color fill and the solid line), this means not all the raw files have been transferred by FTS. If the color fill is lower than the black solid line to the left of the dashed grey line, there may be a problem with FTS. Check the relevant FTS plots on ganglia."))
58 
59 outputHtml.write(html.paragraph("<b>Missing files from the database:</b> If the black line is below the color fill, this means that the run history database has no (or incorrect) information for the number of subruns in that run. There is a script that runs every day to backfill the database. If the black line is below the color fill on the left half of this plot, there is a problem with the database."))
60 
61 outputHtml.write(html.paragraph("<b>Missing root files:</b> If the plot is showing red fill instead of orange, this means not all files have been raw2rooted yet. If there is any red on the left half of this plot, please post in the #keepup channel in Slack about the issue. Be sure to use @here, as most people keep the channel muted."))
62 
63 outputHtml.write(html.paragraph("For older runs, see the " + html.link("All-time Processing History", "/nova/datacheck/processinghistory/history.html", newTab=True)+"."))
64 
65 for detector, onMonDir in onMonDirs.items():
66  (maxRunNumDB, minRunNumDB) = getMaxMinRunNumDB(detector)
67  onMonRunsHist = getCompletedRunsHist(onMonDir, "OnMon Files", maxRunNumDB, minRunNumDB, lineStyle=7, lineColor=onMonColor, additionalDir=onMonTriggerDirs[detector], skipStr=".SMALL")
68  dbRunsHist = getDBRunsHist(minRunNumDB, maxRunNumDB, detector, lineStyle=1, lineColor=dbColor)
69  print 'export PGPASSWORD=' + pw + '; psql -h ifdbprod.fnal.gov -d nova_prod -p '+dbPort[detector] + ' -U nova_reader -H -c "select run, nsubruns, nevents, partition, tstart, shifter, begin_comment, end_comment from fardet.runs where run > '+str(minRunNumDB) +' and run < '+str(maxRunNumDB) +' order by run asc;" > ' + outputDir + 'dbDump_'+detector+'.html'
70  #os.system('export PGPASSWORD=' + pw + '; psql -h ifdbprod.fnal.gov -d nova_prod -p '+dbPort[detector] + ' -U nova_reader -H -c "select run, nsubruns, nevents, partition, tstart, shifter, begin_comment, end_comment from fardet.runs where run > '+str(minRunNumDB) +' and run < '+str(maxRunNumDB) +' order by run asc;" > ' + outputDir + 'dbDump_'+detector+'.html')
71  outputHtml.write(html.heading2(fullNames[detector] + " Processing History - Last Seven Days"))
72  #outputHtml.write(html.paragraph(html.link("Database table dump", "dbDump_"+detector+".html") + " for these runs."))
73 
74 
75  for key, path in paths.items():
76  if not detector in key:
77  continue
78  plotName = path.name + ".png"
79  (can,hists) = path.getPlotCanvas(maxRunNumDB, minRunNumDB, dbRunsHist, onMonRunsHist)
80  can.SaveAs(os.path.join(outputDir, plotName))
81  onMonDiffHist = (onMonRunsHist - hists[0])
82  dbDiffHist = dbRunsHist - hists[0]
83  onMonDiff = onMonDiffHist.Integral(1, onMonDiffHist.GetNbinsX() - 1)
84 # dbDiff = dbDiffHist.Integral(1, dbDiffHist.GetNbinsX() - 1)
85  dbDiff = dbDiffHist.Integral(1, dbDiffHist.GetNbinsX() - 3)
86  text = ""
87 # if dbDiff > 0 or onMonDiff > 0:
88  # text+= html.alertDanger(html.bold("Warning! ")+" Files appear to be missing, please follow the " + html.alertLink("recovery instructions", recoveryUrl, newTab=True)+".")
89  text += html.img(plotName,"")
90 
91 # text += html.paragraph("Missing raw files according to OnMon files: " + str(onMonDiff))
92  text += html.paragraph("Total number of files according samweb: " + str(hists[0].Integral(1, hists[0].GetNbinsX() - 3)))
93  text += html.paragraph("Total number of files according data base runhistory: " + str(dbRunsHist.Integral(1, dbRunsHist.GetNbinsX() - 3)))
94 # text += html.paragraph("Missing raw files according to database: " + str(dbDiff))
95  outputHtml.write(html.panel(path.title, text))
96 
97 
98 
99 
100 
101 outputHtml.close()
102 redirectUrl = dateStr + "/watchdog.html"
103 
104 apacheFile = open(webDir + ".htaccess", 'w')
105 apacheFile.write("RewriteEngine on \n")
106 apacheFile.write("RewriteRule latest.html /nova/datacheck/watchdogs/" + redirectUrl + " [R] \n")
107 apacheFile.write("RewriteRule latest/(.*) /nova/datacheck/watchdogs/"+ dateStr +"/$1 [PT] \n")
108 apacheFile.close()
109 
110 
double Integral(const Spectrum &s, const double pot, int cvnregion)
def getMaxMinRunNumDB(detector="testbeam", maxTime="CURRENT_TIMESTAMP", interval=7)
procfile open("FD_BRL_v0.txt")
def getCompletedRunsHist(path, name, maxRunNum, minRunNum, testStr="", skipStr=None, additionalDir="", checkKeys=False, printRunNum=False, lineStyle=1, lineColor=kBlack)
def getDBRunsHist(minRun, maxRun, table="testbeam", lineStyle=1, lineColor=dbColor)