Functions | Variables
ViewHTML Namespace Reference

Functions

def stringToHTML (string)
 Convert a string to html. More...
 
def showMetric (metric, output_name="metric.html")
 print metric More...
 
def showMetricsSummary (chain, skip_unused_tiers=True, output_name="summary.html")
 print metrics summary More...
 
def rreplace (s, old, new, occurrence)
 recursive replace based on occurance from: http://stackoverflow.com/questions/2556108/how-to-replace-the-last-occurence-of-an-expression-in-a-string More...
 
def metricsToHTMLTable (chain, skip_unused_tiers=False, fcl_links=True, log_links=True, metric_links=True)
 Display metrics as an HTML table. More...
 
def masthead (text, subtext="")
 print a masthead More...
 
def footer ()
 footer More...
 
def collectEnvironment (environ, time_str)
 collect environment More...
 
def parseInfo (info, folder)
 parse info More...
 
def foldersToButtonGroups (folder_list, output, url, page="index.html", messages={})
 convert a folder list to a html button group More...
 
def tierConfigurationsTOC (chains)
 TierConfigurations TOC. More...
 
def tierConfigurationsTables (chains)
 TierConfigurations table. More...
 
def assumptionsToTable (ass, eff, n_per_file)
 Tier assumptions to table. More...
 
def samFileInfoSummaryTable (definitions, names=False, configs=False, fcls=False, status=False)
 samFileInfoSummaryTable More...
 
def samDefinitionDescriptionsModal (def_descriptions)
 SAM definition descriptions to modals (for fun) More...
 
def directoryToTable (wildcard, base_url, status=False, sam_status=False, to_skip=False, verbose=False)
 Scan a directory and return a table. More...
 
def samProjectParametersToUL (parameters)
 convert the sam project parameters to a UL More...
 
def samProjectSummary (config)
 convert the sam project parameters to a UL More...
 
def samProjectCollectionToTable (collection, project_name, check_n_files=False, reduced_files=False, skip_projects=False, verbose=False)
 convert the sam project collection to a table More...
 
def samDefinitionAlert (input_def, guess_child=False)
 Convert a SAM input definition into an alert. More...
 
def samStatusToHTML (sam_status, processes=0)
 Convert a sam status code to HTML. More...
 
def getOutputDSStatus (config, expected_n_files, verbose=False)
 Check the status of a sam project being migrated to SAM. More...
 
def getOutputCloneStatus (config, expected_n_files, verbose=False)
 Check the status of a sam project being migrated to SAM. More...
 
def makeSAMQueryStatusBadge (status, time=0., date="???")
 A dictionary of sam query status badges. More...
 
def definitionListToExplorerDivs (definitions)
 Turn a list of definitions into a list of divs for the explorer JS. More...
 
def definitionListToExplorerData (definitions, filter_on_mrcc=False, skip_counts=False)
 Turn a list of definitions into a list of JS objects for the explorer JS. More...
 
def findDetector (definition)
 find detector from definition name More...
 
def isSimulated (definition)
 is a file simulated based on it's definition name More...
 
def findTrigger (definition)
 find trigger from definition name More...
 
def findTier (definition)
 find tier from definition name More...
 
def findRelease (definition)
 find release from definition name More...
 
def findMRCC (definition)
 find MRCC from definition name More...
 
def makeBadge (this_class, text, icon)
 Make a badge with a popup over. More...
 

Variables

 base_html = \
 base html More...
 
dictionary statusToGlyph = {}
 convert a status into a glyph-icon More...
 

Function Documentation

def ViewHTML.assumptionsToTable (   ass,
  eff,
  n_per_file 
)

Tier assumptions to table.

Definition at line 614 of file ViewHTML.py.

References novadaq::HexUtils.format(), and makeTrainCVSamples.int.

614 def assumptionsToTable(ass, eff, n_per_file):
615  html = \
616  """
617  <table class="table table-striped">
618  <tr>
619  <th>Chain</th>
620  <th>Tier</th>
621  <th>Number of events to produce</th>
622  <th>Efficiency [%]</th>
623  <th>Number of events to run</th>
624  <th>Number of events per file</th>
625  </tr>
626  """
627 
628  chains = ass.keys()
629  chains.sort()
630  for chain in chains:
631  chain_name = chain.split(":")[0]
632  if ":" in chain:
633  tier = chain.split(":")[1]
634  else:
635  tier = "default"
636  #print chain_name, tier
637  events = ass[chain]
638  prod_events = events
639  events_per_file = n_per_file[chain]
640  this_eff = 100
641  if chain in eff.keys():
642  this_eff = (eff[chain])*100.
643  prod_events = prod_events * (100./this_eff)
644  row = \
645  """
646  <tr>
647  <td>%s</td>
648  <td>%s</td>
649  <td>%s</td>
650  <td>%.0f</td>
651  <td>%s</td>
652  <td>%s</td>
653  </tr>
654  """%(chain_name.replace("_"," "),
655  tier,"{:,}".format(events),
656  this_eff,"{:,}".format(int(prod_events)),
657  "{:,}".format(events_per_file))
658  html += row
659 
660 
661  html +="\n</table>"
662  return html
def assumptionsToTable(ass, eff, n_per_file)
Tier assumptions to table.
Definition: ViewHTML.py:614
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def ViewHTML.collectEnvironment (   environ,
  time_str 
)

collect environment

Definition at line 320 of file ViewHTML.py.

Referenced by metricsToHTMLTable().

320 def collectEnvironment(environ, time_str):
321  env = \
322  """
323  <div class="alert alert-info bs-alert-old-docs">
324  This test was run using:
325  <ul>
326  """
327  #env = "<dl>\n"
328  #import time
329  #the_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
330  #env += "<dt>Local time:</dt>\n<dd>%s</dd>"%the_time
331  #import os
332  env += "<li><strong>Time:</strong> %s</li>"%(time_str)
333  variables = ["USER", "HOSTNAME", "HOSTTYPE", "SRT_BASE_RELEASE","SRT_QUAL", "SRT_PUBLIC_CONTEXT", "SRT_PRIVATE_CONTEXT"]
334  for variable in variables:
335  if variable in environ.keys():
336  env += "<li><strong>%s:</strong> %s</li>"%(variable,environ[variable])
337  env += "</ul></div>"
338  return env
def collectEnvironment(environ, time_str)
collect environment
Definition: ViewHTML.py:320
def ViewHTML.definitionListToExplorerData (   definitions,
  filter_on_mrcc = False,
  skip_counts = False 
)

Turn a list of definitions into a list of JS objects for the explorer JS.

Definition at line 1165 of file ViewHTML.py.

References ProductionTestTools.convertSize(), dir, findDetector(), findMRCC(), findRelease(), findTier(), findTrigger(), novadaq::HexUtils.format(), makeTrainCVSamples.int, and isSimulated().

1165 def definitionListToExplorerData(definitions,filter_on_mrcc=False,skip_counts=False):
1166  import json
1167  if "convertSize" not in dir(): from ProductionTestTools import convertSize
1168  html = \
1169  """
1170  <script>
1171  var DATA = {}
1172  DATA.datasets = ["""
1173  # html += ",\n ".join('"%s"'%b for b in definitions)
1174  unique_definitions = [ a for a in set(definitions) ]
1175  unique_definitions.sort(key=lambda x: x.name)
1176  detectors = ["All"]
1177  data_mcs = ["All"]
1178  triggers = ["All"]
1179  tiers = ["All"]
1180  releases = ["All"]
1181  mrccs = ["All"]
1182 
1183  for definition in unique_definitions:
1184  if (definition.file_count == None): definition.file_count = 0
1185  if (definition.total_event_count == None): definition.total_event_count = 0
1186  if (definition.total_file_size == None): definition.total_file_size = 0
1187  detector = findDetector(definition)
1188  is_mc = isSimulated(definition)
1189  trigger = findTrigger(definition)
1190  tier = findTier(definition)
1191  release = findRelease(definition)
1192  mrcc = findMRCC(definition)
1193 
1194  if not trigger: continue
1195  if not tier: continue
1196  if tier == "ccaf": continue
1197 
1198  if detector not in detectors: detectors.append(detector)
1199  if is_mc not in data_mcs: data_mcs.append(is_mc)
1200  if trigger not in triggers: triggers.append(trigger)
1201  if tier not in tiers: tiers.append(tier)
1202  if release not in releases: releases.append(release)
1203  if mrcc not in mrccs: mrccs.append(mrcc)
1204 
1205  html += '\n {"name":"%s","detector":"%s","is_mc":"%s","trigger":"%s","tier":"%s","release":"%s","mrcc":"%s",description:%s,"file_count":"%s","total_event_count":"%s","total_file_size":"%s"},'%\
1206  (definition.name,detector,is_mc,trigger,tier,release,mrcc,\
1207  json.dumps(definition.description),\
1208  "{:,}".format(definition.file_count),\
1209  "{:,}".format(int(definition.total_event_count)),\
1210  convertSize(definition.total_file_size))
1211  html += "\n ];"
1212 
1213  html += "\n DATA.detectors = ["
1214  html += ",\n ".join('"%s"'%b for b in detectors)
1215  html += "\n ];"
1216 
1217  html += "\n DATA.is_mcs = ["
1218  html += ",\n ".join('"%s"'%b for b in data_mcs)
1219  html += "\n ];"
1220 
1221  html += "\n DATA.triggers = ["
1222  html += ",\n ".join('"%s"'%b for b in triggers)
1223  html += "\n ];"
1224 
1225  tiers.sort()
1226  html += "\n DATA.tiers = ["
1227  html += ",\n ".join('"%s"'%b for b in tiers)
1228  html += "\n ];"
1229 
1230  releases.sort()
1231  html += "\n DATA.releases = ["
1232  html += ",\n ".join('"%s"'%b for b in releases)
1233  html += "\n ];"
1234 
1235  html += "\n DATA.mrccs = ["
1236  html += ",\n ".join('"%s"'%b for b in mrccs)
1237  html += "\n ];"
1238 
1239  if filter_on_mrcc: html += "\n DATA.include_mrcc = true;"
1240  if skip_counts: html += "\n DATA.skip_counts = true;"
1241  html += "\n </script>"
1242  return html
def findTier(definition)
find tier from definition name
Definition: ViewHTML.py:1279
def findTrigger(definition)
find trigger from definition name
Definition: ViewHTML.py:1261
def isSimulated(definition)
is a file simulated based on it&#39;s definition name
Definition: ViewHTML.py:1253
def findDetector(definition)
find detector from definition name
Definition: ViewHTML.py:1245
def convertSize(size)
Parse file sizes into human readable format.
def findRelease(definition)
find release from definition name
Definition: ViewHTML.py:1291
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def definitionListToExplorerData(definitions, filter_on_mrcc=False, skip_counts=False)
Turn a list of definitions into a list of JS objects for the explorer JS.
Definition: ViewHTML.py:1165
TDirectory * dir
Definition: macro.C:5
def findMRCC(definition)
find MRCC from definition name
Definition: ViewHTML.py:1300
def ViewHTML.definitionListToExplorerDivs (   definitions)

Turn a list of definitions into a list of divs for the explorer JS.

Definition at line 1154 of file ViewHTML.py.

1155  html = \
1156  """
1157  <div id="searchable_datasets">
1158  """
1159  for definition in definitions:
1160  html += "\n<div>%s</div>"%definition
1161  html += "\n</div>"
1162  return html
def definitionListToExplorerDivs(definitions)
Turn a list of definitions into a list of divs for the explorer JS.
Definition: ViewHTML.py:1154
def ViewHTML.directoryToTable (   wildcard,
  base_url,
  status = False,
  sam_status = False,
  to_skip = False,
  verbose = False 
)

Scan a directory and return a table.

Definition at line 781 of file ViewHTML.py.

References ProductionTestTools.convertSize(), check_time_usage.float, novadaq::HexUtils.format(), makeTrainCVSamples.int, caf_analysis.keys, makeBadge(), and samStatusToHTML().

781 def directoryToTable(wildcard, base_url, status=False, sam_status=False, to_skip=False, verbose=False):
782  from ProductionTestTools import convertSize
783  html = \
784  """
785  <table class="table table-striped">
786  <tr>
787  <th>File name</th>
788  <th>Modification time</th>
789  HEADERS
790  </tr>
791  ROW
792  </table>
793  """
794  if status:
795  html = html.replace("HEADERS","<th>Completed</th><th>Run time</th>")
796  elif sam_status:
797  html = html.replace("HEADERS","<th>Message</th><th>SAM jobs</th><th>Files</th><th>Size</th><th>Definition</th><th>Reduced CAF</th>")
798  else:
799  html = html.replace("HEADERS","")
800 
801  import glob
802  import os.path
803  import time
804 
805  file_list = glob.glob(wildcard)
806  file_list.sort()
807  for f in file_list:
808  skip = False
809  if to_skip:
810  for s in to_skip:
811  if (s in f): skip = True
812  if skip: continue
813  try:
814  name = f.split("/")[-1]
815  link = base_url+name.replace("txt","html")
816  if link[-5:] != ".html": link = link + ".html"
817  name_link = '<a href="%s">%s</a>'%(link,name)
818  mod_time = time.ctime(os.path.getmtime(f))
819  except OSError:
820  print f,"is not a valid file. Skipping"
821  continue
822 
823  line = "<tr><td>%s</td><td>%s</td>"%(name_link,mod_time)
824  if status:
825  this_status = status[f]
826  this_status_html = '<td align="center">%s</td>'
827  if (this_status[0] != False) and (this_status[1] ==False):
828  line = line + this_status_html%makeBadge("alert-success","Completed successfully","ok")
829  elif (this_status[1] == False):
830  line = line + this_status_html%makeBadge("","Ongoing","cog")
831  else:
832  line = line + this_status_html%makeBadge("alert-danger",this_status[1],"remove")
833 
834  if this_status[2] != None:
835  dt = divmod(this_status[2].total_seconds(), 60)
836  line = line + '<td>%i m %i s</td>'%(dt[0],dt[1])
837  else:
838  line = line +'<td></td>'
839 
840 
841  elif sam_status:
842  this_status = sam_status[f]
843  project_status_dict = this_status[0]
844  # message
845  line += "<td>%s</td>"%this_status[1]
846  # SAM jobs
847  line += "<td>"
848  for project in project_status_dict.keys():
849  if len(project_status_dict[project]["sam_status"]) > 1: line += " ["
850  for i,this_sam_status in enumerate(project_status_dict[project]["sam_status"]):
851  line += samStatusToHTML(this_sam_status,processes=project_status_dict[project]["processes"][i])
852  if len(project_status_dict[project]["sam_status"]) > 1: line += "] "
853  line += "</td>"
854  # n_files
855  progress = int((float(this_status[3]) / this_status[4])*100)
856  # print progress, this_status[3], this_status[4]
857  line += \
858  """
859  <td>
860  <div class="progress">
861  <div class="progress-bar progress-bar-success mypopover" role="progressbar" aria-valuenow="%i" aria-valuemin="0" aria-valuemax="100" style="width: %i%%" data-toggle="tooltip" data-original-title="%s / %s" rel="tooltip">
862  <span class="sr-only">%i%% Complete (success)</span>
863  </div>
864  </div>
865  </td>
866  """%(progress,progress,"{:,}".format(this_status[3]), "{:,}".format(this_status[4]), progress)
867  # size
868  line += "<td nowrap>%s</td>"%convertSize(this_status[5])
869 
870  # SAM definition
871  line += "<td>"
872  for project in project_status_dict.keys():
873  line += project_status_dict[project]["output_ds"]
874  line += "</td>"
875 
876  # # dCache clone
877  # line += "<td>"
878  # for project in project_status_dict.keys():
879  # line += project_status_dict[project]["dcache"]
880  # line += "</td>"
881 
882  # reduced cafs
883  line += "<td>"
884  for c in this_status[2].keys():
885  if "success" in this_status[2][c]: line += '<span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span>'
886  elif "warning" in this_status[2][c]: line += '<span class="badge alert-warning"><span class="glyphicon glyphicon-cog"></span></span>'
887  else: line += '<span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>'
888  line += "</td>"
889 
890  html = html.replace("ROW",line+"</tr>\nROW")
891  return html.replace("ROW","")
keys
Reco plots.
Definition: caf_analysis.py:46
def makeBadge(this_class, text, icon)
Make a badge with a popup over.
Definition: ViewHTML.py:1306
def convertSize(size)
Parse file sizes into human readable format.
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def directoryToTable(wildcard, base_url, status=False, sam_status=False, to_skip=False, verbose=False)
Scan a directory and return a table.
Definition: ViewHTML.py:781
def samStatusToHTML(sam_status, processes=0)
Convert a sam status code to HTML.
Definition: ViewHTML.py:1084
def ViewHTML.findDetector (   definition)

find detector from definition name

Definition at line 1245 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1245 def findDetector(definition):
1246  tokens = definition.name.split("_")
1247  for t in tokens:
1248  if t in ["nd","neardet"]:
1249  return "Near"
1250  return "Far"
def findDetector(definition)
find detector from definition name
Definition: ViewHTML.py:1245
def ViewHTML.findMRCC (   definition)

find MRCC from definition name

Definition at line 1300 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1300 def findMRCC(definition):
1301  if "mrcc" in definition.name: return "mrcc"
1302  if "mre" in definition.name: return "mre"
1303  return "none"
def findMRCC(definition)
find MRCC from definition name
Definition: ViewHTML.py:1300
def ViewHTML.findRelease (   definition)

find release from definition name

Definition at line 1291 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1291 def findRelease(definition):
1292  i = 2
1293  if ("contain_decaf" in definition.name) or ("with_lempart" in definition.name): i=4
1294  tokens = definition.name.split("_")
1295  if (tokens[i][0] != "S") and (tokens[i][:2] != "FA"): return "unknown"
1296  if tokens[i][-3:] == "-nd": return tokens[i][:-3]
1297  return tokens[i]
def findRelease(definition)
find release from definition name
Definition: ViewHTML.py:1291
def ViewHTML.findTier (   definition)

find tier from definition name

Definition at line 1279 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1279 def findTier(definition):
1280  tokens = definition.name.split("_")
1281  if tokens[1][0] == "S": return False
1282  if "mrcc" in tokens[1]: return tokens[1][4:]
1283  if "mre" in tokens[1]: return tokens[1][3:]
1284  try:
1285  if tokens[2]+"_"+tokens[3] == "contain_decaf": return "decaf"
1286  except:
1287  pass
1288  return tokens[1]
def findTier(definition)
find tier from definition name
Definition: ViewHTML.py:1279
def ViewHTML.findTrigger (   definition)

find trigger from definition name

Definition at line 1261 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1261 def findTrigger(definition):
1262  tokens = definition.name.split("_")
1263  for t in tokens:
1264  if t in ["numi","t00","genie","crygenie"]:
1265  return "NuMI"
1266  if t in ["cosmic","t02","cry","cosmics"]:
1267  return "Cosmic"
1268  if t in ["DDActivity1"]:
1269  return "DD-Activity"
1270  if t in ["DDCalMu"]:
1271  return "DD-CalMu"
1272  # if we haven't returned by now try some desperate measures:
1273  for t in tokens:
1274  if t in ["fhc","rhc"]: return "NuMI"
1275  # assert False,"Cannot discern trigger type for definition name: %s"%definition
1276  return False
def findTrigger(definition)
find trigger from definition name
Definition: ViewHTML.py:1261
def ViewHTML.foldersToButtonGroups (   folder_list,
  output,
  url,
  page = "index.html",
  messages = {} 
)

convert a folder list to a html button group

Definition at line 438 of file ViewHTML.py.

438 def foldersToButtonGroups(folder_list, output, url, page="index.html", messages={}):
439  import os
440  import ProductionTestTools as tools
441 
442  button_groups = {}
443  for folder in folder_list:
444  if not os.path.isdir("%s%s"%(output,folder)) : continue
445  message = "No message"
446  if folder in messages.keys():
447  message = messages[folder]
448  if message == "": message = "No message"
449  # catagorise folders
450  if tools.isLogicalFolder(folder):
451  release = tools.releaseFromFolder(folder)
452  f_date = tools.dateFromFolder(folder)
453  if release not in button_groups.keys():
454  button_groups[release] = []
455  button_groups[release].append((url,folder,f_date, message))
456  else:
457  if "Misc" not in button_groups.keys():
458  button_groups["Misc"] = []
459  button_groups["Misc"].append((url,folder,folder, message))
460 
461  button_html = \
462  """
463  <div>
464  <h3>Development</h3>
465  <!--DEVELOPMENT_BUTTON_HOOK-->
466  <br><br>
467  <!--DEVELOPMENT_HOOK-->
468  </div>
469  <hr>
470 
471  <div>
472  <h3>Snapshot/FA snapshot</h3>
473  <!--S_BUTTON_HOOK-->
474  <br><br>
475  <!--S_HOOK-->
476  </div>
477  <hr>
478 
479  <div>
480  <h3>R-style release</h3>
481  <!--R_BUTTON_HOOK-->
482  <br><br>
483  <!--R_HOOK-->
484  </div>
485  """
486 
487  groups = button_groups.keys()
488  groups.sort()
489  patterns = { "DEVELOPMENT_BUTTON_HOOK":["development"],"S_BUTTON_HOOK":["FA","S"], "R_BUTTON_HOOK":["R"]}
490  for pattern_key in patterns.keys():
491  for group in groups:
492  matches = False
493  for sub_pattern in patterns[pattern_key]:
494  if group[:len(sub_pattern)] == sub_pattern: matches=True
495  if not matches: continue
496 
497  html = \
498  """
499  <div class="btn-group">
500  <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown"><span class="badge">%i</span> %s <span class="caret"></span></button>
501  <ul class="dropdown-menu" role="menu">
502  TEST
503  </ul>
504  </div><!-- /btn-group -->
505  """%(len(button_groups[group]),group)
506  button_groups[group].sort()
507  for s in button_groups[group]:
508  html = html.replace("TEST",
509  '<li><a href="%s%s/%s" data-toggle="tooltip" data-original-title="%s" rel="tooltip">%s</a></li>\nTEST'%(s[0],s[1],page,s[3],s[2]))
510  html = html.replace("TEST","")
511  button_html = button_html.replace("<!--%s-->"%pattern_key,"%s\n<!--%s-->"%(html,pattern_key))
512 
513  return button_html
def foldersToButtonGroups(folder_list, output, url, page="index.html", messages={})
convert a folder list to a html button group
Definition: ViewHTML.py:438
def ViewHTML.footer ( )

footer

Definition at line 304 of file ViewHTML.py.

304 def footer():
305  import time
306  t_string = time.strftime("%Y-%m-%d %H:%M:%S (%Z)", time.localtime())
307  #t_string = t_string.replace(t_string[-3:],":%s"%str(t_string[-3:]))
308  to_return = \
309  """
310  <hr>
311  <div id="footer">
312  <div class="container-fluid">
313  <p class="muted credit">Page autogenerated at <font color="#0066CC">%s</font> by Production's <a href="https://www.youtube.com/watch?v=WGoi1MSGu64">robots</a>.</p>
314  </div>
315  </div>
316  """%t_string
317  return to_return
def footer()
footer
Definition: ViewHTML.py:304
def ViewHTML.getOutputCloneStatus (   config,
  expected_n_files,
  verbose = False 
)

Check the status of a sam project being migrated to SAM.

Definition at line 1118 of file ViewHTML.py.

1118 def getOutputCloneStatus(config,expected_n_files,verbose=False):
1119  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1120  if not hasattr(config,"pnfs_output_dir"):
1121  return html%("danger","No PNFS clone","remove")
1122 
1123  import glob
1124  import os
1125 
1126  output_dir = config.pnfs_output_dir
1127  if verbose: print "vhtml: definition has been cloned to: %s"%config.pnfs_output_dir
1128 
1129  if not os.path.exists(output_dir):
1130  return html%("danger","Clone location: %s doesn't exist"%output_dir,"fire")
1131 
1132  files = glob.glob(output_dir+"/*")
1133  info = "location: %s (%i/%i files)"%(output_dir,len(files),expected_n_files)
1134  if len(files) == expected_n_files:
1135  return html%("success",info,"ok")
1136  else:
1137  return html%("warning",info,"cog")
def getOutputCloneStatus(config, expected_n_files, verbose=False)
Check the status of a sam project being migrated to SAM.
Definition: ViewHTML.py:1118
def ViewHTML.getOutputDSStatus (   config,
  expected_n_files,
  verbose = False 
)

Check the status of a sam project being migrated to SAM.

Definition at line 1100 of file ViewHTML.py.

References ProductionTestTools.samDefinitionSummary().

Referenced by samProjectCollectionToTable().

1100 def getOutputDSStatus(config,expected_n_files,verbose=False):
1101  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1102  if not hasattr(config,"output_sam_defname"):
1103  return html%("danger","No output SAM dataset","remove")
1104  from ProductionTestTools import samDefinitionSummary
1105 
1106  def_name = config.output_sam_defname
1107 
1108  if verbose: print "vhtml: definition has been declared as: %s"%config.output_sam_defname
1109 
1110  output_files = samDefinitionSummary(config.output_sam_defname)
1111  info = "defname: %s (%i/%i files)"%(def_name,output_files["file_count"],expected_n_files)
1112  if output_files["file_count"] == expected_n_files:
1113  return html%("success",info,"ok")
1114  else:
1115  return html%("warning",info,"cog")
def samDefinitionSummary(input_def)
Get the sam definition summary.
def getOutputDSStatus(config, expected_n_files, verbose=False)
Check the status of a sam project being migrated to SAM.
Definition: ViewHTML.py:1100
def ViewHTML.isSimulated (   definition)

is a file simulated based on it's definition name

Definition at line 1253 of file ViewHTML.py.

Referenced by definitionListToExplorerData().

1253 def isSimulated(definition):
1254  tokens = definition.name.split("_")
1255  for t in tokens:
1256  if t in ["sim","genie","cry","fhc","rhc"]:
1257  return "MC"
1258  return "Data"
def isSimulated(definition)
is a file simulated based on it&#39;s definition name
Definition: ViewHTML.py:1253
def ViewHTML.makeBadge (   this_class,
  text,
  icon 
)

Make a badge with a popup over.

Definition at line 1306 of file ViewHTML.py.

Referenced by directoryToTable().

1306 def makeBadge(this_class,text,icon):
1307  html = '<span class="badge %s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1308  return html%(this_class,text,icon)
1309 
def makeBadge(this_class, text, icon)
Make a badge with a popup over.
Definition: ViewHTML.py:1306
def ViewHTML.makeSAMQueryStatusBadge (   status,
  time = 0.,
  date = "???" 
)

A dictionary of sam query status badges.

Definition at line 1140 of file ViewHTML.py.

1140 def makeSAMQueryStatusBadge(status,time=0.,date="???"):
1141  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1142  if status == "good":
1143  return html%("success","SAM query returned in %.2f s on %s"%(time,date),"ok")
1144  elif status == "timed out":
1145  return html%("danger","SAM timed out after %.2f s on %s"%(time,date),"remove")
1146  elif status == "cached":
1147  return html%("info","Cached query from %s used"%(date),"cog")
1148  elif "error" in status:
1149  return html%("danger","SAM Error %s on %s"%(status,date),"remove")
1150  else:
1151  return html%("warning","Unknown status, %s, on %s"%(status,date),"cog")
def makeSAMQueryStatusBadge(status, time=0., date="???")
A dictionary of sam query status badges.
Definition: ViewHTML.py:1140
def ViewHTML.masthead (   text,
  subtext = "" 
)

print a masthead

Definition at line 283 of file ViewHTML.py.

283 def masthead(text,subtext=""):
284  to_return = \
285  """
286  <!-- Masthead ================================================== -->
287  <header class="mastheader row" id="header">
288 
289  <div class="span12">
290  <div class="page-header">
291  <h1>
292  TEXT
293  <small>SUBT</small>
294  </h1>
295  </div>
296  </div>
297  </header>
298  """
299  to_return = to_return.replace("TEXT",text)
300  to_return = to_return.replace("SUBT",subtext)
301  return to_return
def masthead(text, subtext="")
print a masthead
Definition: ViewHTML.py:283
def ViewHTML.metricsToHTMLTable (   chain,
  skip_unused_tiers = False,
  fcl_links = True,
  log_links = True,
  metric_links = True 
)

Display metrics as an HTML table.

Definition at line 70 of file ViewHTML.py.

References collectEnvironment(), ProductionTestTools.convertSize(), dir, check_time_usage.float, min(), PandAna.Demos.demo1.range, and rreplace().

Referenced by rreplace(), and showMetricsSummary().

70  fcl_links=True, log_links=True, metric_links=True):
71  if "convertSize" not in dir(): from ProductionTestTools import convertSize
72  if "Metric" not in dir(): import Metric
73  status = False
74  table_base = \
75  """
76  <div class="page-header"><h2 id="%s"> %s </h2></div>
77 
78  """%(chain.name,chain.name.replace("_"," "))
79 
80  table_base += "\n"+collectEnvironment(chain.env, chain.time_str)+"\n"
81  if not log_links:
82  table_base = table_base.replace("</ul></div>",
83  "</ul>\n<strong>No logs requested</strong>\n</div>")
84  if not fcl_links:
85  table_base = table_base.replace("</ul></div>",
86  "</ul>\n<strong>No FHiCLs requested</strong>\n</div>")
87  if not metric_links:
88  table_base = table_base.replace("</ul></div>",
89  "</ul>\n<strong>No metrics requested</strong>\n</div>")
90 
91  header = \
92  """
93  <div class="panel panel-default">
94  <table class="table table-striped">
95  <tr>
96  <th>Tier</th>
97  <th>In evt</th>
98  <th>User CPU (/in evt) [s]</th>
99  <th>Memory</th>
100  <th>DB queries</th>
101  <th>Query time [s]</th>
102  <th>Child</th>
103  <th>Events (efficiency)</th>
104  <th>Size (/out evt)</th>
105  </tr>
106  """
107 
108  table_base += header
109  rows = ""
110  any_run = False
111  for tier in chain.tiers:
112  if skip_unused_tiers and (tier.metric.run == False): continue
113  # name
114  t = tier.short_name
115 
116  if (tier.metric.run == False):
117  rows += '<tr class="warning"><td id=%s_%s>%s - was not run</td>'%(chain.name,tier.short_name,tier.short_name)
118  rows += "<td></td>"*8
119  rows += "</tr>\n"
120  if tier.metric.return_code == 1: status = 5
121  continue
122  any_run = True
123 
124  continue_after_metrics = False
125  if (tier.metric.return_code != 0):
126  row = '<tr class="danger"><td id=%s_%s>%s - failed, got return code: %i</td>'%(chain.name,tier.short_name,tier.short_name, tier.metric.return_code)
127  row += "<td></td>"*8
128  row += "</tr>\n"
129  status = 5
130  continue_after_metrics = True
131  else:
132  row = "<tr>\n"
133  row += "<td id=%s_%s>%s</td>\n"%(chain.name,tier.short_name,t)
134 
135  if log_links:
136  row = row.replace(">%s"%t,'><pre>%s\n<a href="logs/%s_%s.html">log</a></pre>'%\
137  (t,chain.name,tier.short_name))
138 
139  if metric_links:
140  if log_links:
141  row = row.replace("log</a>",'log</a>, <a href="metrics/%s_%s.html">metrics</a>'%\
142  (chain.name,tier.short_name))
143  else:
144  row = row.replace(">%s"%t,'><pre>%s\n<a href="metrics/%s_%s.html">metrics</a></pre>'%\
145  (t,chain.name,tier.short_name))
146  if fcl_links:
147  if log_links:
148  row = row.replace("log</a>",'log</a>, <a href="fcls/%s_%s.html">fcl</a>'%\
149  (chain.name,tier.short_name))
150  elif metric_links:
151  row = row.replace("metric</a>",'metric</a>, <a href="fcls/%s_%s.html">fcl</a>'%\
152  (chain.name,tier.short_name))
153  else:
154  row = row.replace(">%s"%t,'><pre>%s\n<a href="fcls/%s_%s.html">fcl</a></pre>'%\
155  (t,chain.name,tier.short_name))
156 
157  if continue_after_metrics:
158  rows += row
159  continue
160  metric = tier.metric
161  #print metric
162 
163  # check on if this tier was run
164  # This should probably be extended to check is this tier completed.
165  if (tier.metric.run == False):
166  row += "<td>%s</td>\n"%"Not run"
167  rows+=row
168  continue
169 
170  # input/used events
171  in_events = None
172  if (metric.used_events == None) and (metric.input_events == None):
173  t = "n/a"
174  elif metric.used_events and metric.input_events:
175  t = "%i"%min(metric.used_events,metric.input_events)
176  in_events = min(metric.used_events,metric.input_events)
177  elif metric.used_events:
178  t = "%i"%metric.used_events
179  in_events = metric.used_events
180  elif metric.input_events:
181  t = "%i"%metric.input_events
182  in_events = metric.input_events
183  row += "<td>%s</td>\n"%(t)
184 
185  # user cpu
186  if metric.user_cpu == None:
187  cpu = "n/a"
188  e_cpu = "n/a"
189  else:
190  cpu = "%.2f"%metric.user_cpu
191  if (in_events != None) and (in_events != 0):
192  e_cpu = "%.2f"%(metric.user_cpu/float(in_events))
193  else:
194  e_cpu = "n/a"
195  row += '<td><a href="timings/%s_%s.html">%s (%s)</a></td>\n'%(chain.name,tier.short_name,cpu,e_cpu)
196 
197  # memory
198  if metric.peak_memory == None: mem = "n/a"
199  else: mem = "%s"%convertSize(metric.peak_memory)
200  row += "<td>%s</td>\n"%(mem)
201 
202  # db queries
203  if (metric.db_usage != None) and (metric.db_usage.n_queries != None):
204  q = "%i"%metric.db_usage.n_queries
205  else:
206  q = "n/a"
207  row += "<td>%s</td>\n"%q
208 
209  # query time
210  if (metric.db_usage != None):
211  q = "%.2f"%(metric.db_usage.sumQueryTime()/1000)
212  else:
213  q = "n/a"
214  row += "<td>%s</td>\n"%q
215 
216  # parsing time
217  #if (metric.db_usage != None):
218  #q = "%16.2f"%(metric.db_usage.sumParsingTime()/1000)
219  #else:
220  #q = "n/a"
221  #line += "%16s | "%q
222 
223 
224  if metric.output_name != None:
225  width = 0
226  for i, out in enumerate(metric.output_name):
227  # For the secondary children start a new line in the
228  # table which is indented to the depth of the first.
229  # Take the indentation depth from the first child.
230  if (i == 0):
231  width = row.count("</td>")
232  if (i != 0):
233  row += "</tr>\n<tr>\n"
234  for j in range(width):
235  row+='<td align="center">``</td>\n'
236  # name
237  if len(out) > 27:
238  out_name = "%s...%s"%(out[:12],out[-12:])
239  else:
240  out_name = out
241  row += "<td>%s</td>\n"%out_name
242 
243  # events
244  if metric.output_events[i] == None:
245  e = "n/a"
246  eff = "n/a"
247  else:
248  e = "%i"%metric.output_events[i]
249  if (in_events != None) and (in_events != 0):
250  eff = "%.0f [%%]"%(100*float(metric.output_events[i])/float(in_events))
251  else:
252  eff = "n/a"
253  row += "<td>%s (%s)</td>\n"%(e,eff)
254 
255  # size
256  if metric.output_size[i] == None:
257  e = "n/a"
258  eff = "n/a"
259  else:
260  e = "%s"%convertSize(metric.output_size[i])
261  if (metric.output_events != None) and (metric.output_events[i] != 0):
262  eff = "%s"%convertSize(float(metric.output_size[i])/float(metric.output_events[i]))
263  else:
264  eff = "n/a"
265  row += '<td><a href="file_sizes/%s_%s_%s.html">%s (%s)</a></td>\n'%(chain.name,tier.short_name,out.replace(".","_"),e,eff)
266 
267  # check for any output events
268  if ((metric.output_events[i] in [None,0]) and ("stop" not in out)):
269  row = rreplace(row, "<tr>", '<tr class="danger">', 1)
270  status = 5
271 
272  else:
273  row += "<td>No children</td>\n"
274 
275  row +="</tr>\n"
276  rows+=row
277  if not any_run: status = 5
278  table_base+=rows
279  table_base+="\n</table>\n</div>"
280  return table_base, status
def rreplace(s, old, new, occurrence)
recursive replace based on occurance from: http://stackoverflow.com/questions/2556108/how-to-replace-...
Definition: ViewHTML.py:64
def collectEnvironment(environ, time_str)
collect environment
Definition: ViewHTML.py:320
def convertSize(size)
Parse file sizes into human readable format.
static float min(const float a, const float b, const float c)
Definition: absgeo.cxx:45
TDirectory * dir
Definition: macro.C:5
def ViewHTML.parseInfo (   info,
  folder 
)

parse info

Definition at line 341 of file ViewHTML.py.

References dir.

341 def parseInfo(info,folder):
342  import ProductionTestTools as tools
343  html = \
344  """
345  <div class="row">
346  <div class="col-md-3">
347  <div class="panel panel-default">
348  <div class="panel-heading">
349  <h3 class="panel-title">Test parameters</h3>
350  </div>
351  <div class="panel-body">
352  <ul>
353  <li>TIME</li>
354  <li>RELEASE</li>
355  <li>MESSAGE</li>
356  </ul>
357  </div>
358  </div>
359  </div>
360 
361  <div class="col-md-6">
362  <div class="panel panel-default">
363  <div class="panel-heading">
364  <h3 class="panel-title">Chains</h3>
365  </div>
366  <!-- <div class="panel-body"> -->
367  <table class="table table-striped">
368  <!-- <tr> <th>Tier</th> <th>Logs</th> <th>Status</th> </tr> -->
369  TIER
370  </table>
371  <!-- </div> -->
372  </div>
373  </div>
374 
375  <div class="col-md-3">
376  <div class="panel panel-default">
377  <div class="panel-heading">
378  <h3 class="panel-title">Batch job status keys</h3>
379  </div>
380  <div class="panel-body">
381  <span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span>
382  &nbsp&nbsp job ended successfully<br>
383 
384  <span class="badge"><span class="glyphicon glyphicon-cog"></span></span>
385  &nbsp&nbsp job ongoing<br>
386 
387  <span class="badge alert-warning"><span class="glyphicon glyphicon-flag"></span></span>
388  &nbsp&nbsp STDERR not empty<br>
389 
390  <span class="badge alert-danger"><span class="glyphicon glyphicon-wrench"></span></span>
391  &nbsp&nbsp job was killed by batch robots<br>
392 
393  <span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>
394  &nbsp&nbsp error in run tier<br>
395 
396  <span class="badge alert-danger"><span class="glyphicon glyphicon-floppy-remove"></span></span>
397  &nbsp&nbsp no pkl file for a completed chain<br>
398 
399  <span class="badge alert-danger"><span class="glyphicon glyphicon-send"></span></span>
400  &nbsp&nbsp batch submission failed<br>
401 
402  </div>
403  </div>
404  </div>
405 
406  </div>
407  """
408  #env = "<dl>\n"
409  #import time
410  #the_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
411  #env += "<dt>Local time:</dt>\n<dd>%s</dd>"%the_time
412  #import os
413  html = html.replace("TIME","<strong>Time:</strong> %s"%(info.time_str))
414  html = html.replace("RELEASE","<strong>Release:</strong> %s"%(info.release))
415  message = ""
416  if "message" in dir(info):
417  html = html.replace("MESSAGE","<strong>Message:</strong> %s"%(info.message))
418  message = info.message
419  else:
420  html = html.replace("MESSAGE","<strong>Message:</strong> none")
421  message = "No message"
422  logs = info.logs
423  commands = info.commands
424  commands.sort()
425  for i, command in enumerate(commands):
426  chain, chain_name = tools.findChainNameFromCommand(command)
427  this_html = '<tr><td><a href="#%s">%s</a></td>'%(chain_name,chain)
428  this_html += '<td><a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_cmd.html">cmd</a>'%(folder,chain)
429  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_log.html">log</a>'%(folder,chain)
430  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_out.html">out</a>'%(folder,chain)
431  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_err.html">err</a></td>'%(folder,chain)
432  this_html += ' <td><!--BATCH_STATUS_HOOK_%s--><br></td></tr>\n'%chain_name
433  html = html.replace("TIER","%s\nTIER"%this_html)
434  html = html.replace("TIER","")
435  return html,message
def parseInfo(info, folder)
parse info
Definition: ViewHTML.py:341
TDirectory * dir
Definition: macro.C:5
def ViewHTML.rreplace (   s,
  old,
  new,
  occurrence 
)

recursive replace based on occurance from: http://stackoverflow.com/questions/2556108/how-to-replace-the-last-occurence-of-an-expression-in-a-string

Definition at line 64 of file ViewHTML.py.

References metricsToHTMLTable().

Referenced by metricsToHTMLTable().

64 def rreplace(s, old, new, occurrence):
65  li = s.rsplit(old, occurrence)
66  return new.join(li)
def rreplace(s, old, new, occurrence)
recursive replace based on occurance from: http://stackoverflow.com/questions/2556108/how-to-replace-...
Definition: ViewHTML.py:64
def ViewHTML.samDefinitionAlert (   input_def,
  guess_child = False 
)

Convert a SAM input definition into an alert.

Definition at line 1042 of file ViewHTML.py.

References ProductionTestTools.samDefinitionSummary().

1042 def samDefinitionAlert(input_def,guess_child=False):
1043  children = {"prod_daq_FA14-10-03a_nd_genie_fhc_nonswap" : "prod_reco_FA15-02-09_nd_genie_fhc_nonswap",
1044  "prod_daq_FA14-10-03x.a_fd_genie_fhc_nonswap" : "prod_reco_S15-05-04_fd_genie_fhc_nonswap",
1045  "prod_daq_FA14-10-03x.a_fd_genie_fhc_fluxswap" : "prod_reco_S15-05-04_fd_genie_fhc_fluxswap",
1046  "prod_artdaq_S15-03-11_nd_numi" : 3533,
1047  "tamsett_snap_140515_prod_daq_FA14-10-03x.a_nd_genie_fhc_nonswap" : 10401}
1048 
1049  from ProductionTestTools import samDefinitionSummary
1050  html = \
1051  """
1052  <div class="alert alert-success bs-alert-old-docs" role="alert">
1053  <p>input: <strong>%s</strong>
1054  """%input_def
1055 
1056  summary = samDefinitionSummary(input_def)
1057  expected_n_events = False
1058 
1059  html += " has <strong>%i</strong> files</p>"%summary["file_count"]
1060 
1061  if guess_child:
1062  if input_def in children.keys():
1063  child_def = children[input_def]
1064  else:
1065  release = input_def.split("_")[2]
1066  child_def = input_def.replace("daq","reco").replace(release,guess_child)
1067  if type(child_def) == int:
1068  html += "<p>forced number of children expected: <strong>%i</strong> files</p>"%(child_def)
1069  expected_n_events = child_def
1070  else:
1071  try:
1072  summary = samDefinitionSummary(child_def)
1073  html += "<p>child: <strong>%s</strong> has <strong>%i</strong> files</p>"%(child_def,summary["file_count"])
1074  expected_n_events = summary["file_count"]
1075  except:
1076  html += "<p><strong>No child definition found</strong>, tried: %s</p>"%(child_def)
1077  expected_n_events = 1
1078  else:
1079  expected_n_events = samDefinitionSummary(input_def)["file_count"]
1080  html += "</div>"
1081  return html, expected_n_events
def samDefinitionSummary(input_def)
Get the sam definition summary.
::xsd::cxx::tree::type type
Definition: Database.h:110
def samDefinitionAlert(input_def, guess_child=False)
Convert a SAM input definition into an alert.
Definition: ViewHTML.py:1042
def ViewHTML.samDefinitionDescriptionsModal (   def_descriptions)

SAM definition descriptions to modals (for fun)

Definition at line 746 of file ViewHTML.py.

References ProductionTestTools.makeAlphanumeric(), and runNovaSAM.str.

746 def samDefinitionDescriptionsModal(def_descriptions):
747  from ProductionTestTools import makeAlphanumeric
748  html = ""
749  modal_base = \
750  """
751  <div class="modal fade" id="IDDEF" tabindex="-1" role="dialog" aria-labelledby="DEFLabel" aria-hidden="true">
752  <div class="modal-dialog modal-lg">
753  <div class="modal-content">
754  <div class="modal-body">
755  <ul>BODY</ul>
756  </div>
757  <div class="modal-footer">
758  <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
759  </div>
760  </div><!-- /.modal-content -->
761  </div><!-- /.modal-dialog -->
762  </div><!-- /.modal -->
763  """
764 
765  for definition in def_descriptions.keys():
766  this_modal = modal_base.replace("IDDEF", makeAlphanumeric(definition))
767  this_modal = this_modal.replace("DEF", definition)
768  #this_modal = this_modal.replace("TITLE", "%s description"%definition)
769  this_body = ""
770  description = def_descriptions[definition]
771  for key in description.keys():
772  this_body += "<li><strong>%s:</strong> %s</li>"%(key,str(description[key]))
773  this_body += '<a href="http://samweb.fnal.gov:8480/sam/nova/api/files/list?format=html&dims=%%20%%28%%20defname%%3A%%20%s%%20%%29">SAM-web link</a>'%description["defname"]
774  this_modal = this_modal.replace("BODY",this_body)
775  html+=this_modal
776 
777 
778  return html
def samDefinitionDescriptionsModal(def_descriptions)
SAM definition descriptions to modals (for fun)
Definition: ViewHTML.py:746
def makeAlphanumeric(input)
convert a string into a purely alphanumeric one adapted from: http://stackoverflow.com/questions/1276764/stripping-everything-but-alphanumeric-chars-from-a-string-in-python
def ViewHTML.samFileInfoSummaryTable (   definitions,
  names = False,
  configs = False,
  fcls = False,
  status = False 
)

samFileInfoSummaryTable

Definition at line 665 of file ViewHTML.py.

References novadaq::HexUtils.format(), and makeTrainCVSamples.int.

665 def samFileInfoSummaryTable(definitions,names=False,configs=False,fcls=False,status=False):
666  import ProductionTestTools as tools
667  html = \
668  """
669  <div class="panel panel-default">
670  <table class="table table-striped">
671  <tr>
672  <th>Definition</th>
673  <th>Files</th>
674  <th>Events</th>
675  <th>Size</th>
676  </tr>
677  """
678  if fcls:
679  html = html.replace("<tr>","<tr><th>FHiCL</th>")
680  if configs:
681  html = html.replace("<tr>","<tr><th>Configuration</th>")
682  if names:
683  html = html.replace("<tr>","<tr><th>Chain</th><th>Tier</th>")
684  if status:
685  html = html.replace("<th>Definition</th>","<th>Definition</th><th>SAM query status</th>")
686 
687  totals = [0,0,0]
688  for i,definition in enumerate(definitions.keys()):
689  file_info = definitions[definition]
690  n_files = definitions[definition]["file_count"]
691  events = definitions[definition]["total_event_count"]
692  if events == None: events = 0
693  size = definitions[definition]["total_file_size"]
694  if size == None: size = 0
695  totals[0]+=n_files
696  totals[1]+=int(events)
697  totals[2]+=size
698 
699  row = \
700  """
701  <tr>
702  <td>%s</td>
703  <td>%s</td>
704  <td>%s</td>
705  <td>%s</td>
706  </tr>
707  """%(definition,
708  "{:,}".format(n_files),
709  "{:,}".format(int(events)),
710  tools.convertSize(size)
711  )
712  if configs:
713  row = row.replace("<tr>",'<tr><td><a href="%s_config.html">config</a></td>'%(definition))
714  if fcls:
715  row = row.replace("<tr>",'<tr><td><a href="%s.html">fcl</a></td>'%(definition))
716  if names:
717  chain = names[i][0]
718  if (i > 0) and (chain == names[i-1][0]):
719  chain = ""
720  row = row.replace("<tr>","<tr><td>%s</td><td>%s</td>"%(chain,names[i][1]))
721  if status:
722  row = row.replace("<td>%s</td>"%definition,"<td>%s</td><td>%s</td>"%(definition,definitions[definition]["status"]))
723  html += row
724  row = \
725  """
726  <tr>
727  <td><strong>Total</strong></td>
728  <td>%s</td>
729  <td>%s</td>
730  <td>%s</td>
731  </tr>
732  """%("{:,}".format(totals[0]),
733  "{:,}".format(totals[1]),
734  tools.convertSize(totals[2])
735  )
736  if configs: row = row.replace("<tr>","<tr><td></td>")
737  if fcls: row = row.replace("<tr>","<tr><td></td>")
738  if status: row = row.replace("<tr>","<tr><td></td>")
739  if names: row = row.replace("<td><strong>Total</strong></td>","<td><strong>Total</strong></td><td></td><td></td>")
740  html += row
741 
742  html +="\n</table></div>"
743  return html, totals
def samFileInfoSummaryTable(definitions, names=False, configs=False, fcls=False, status=False)
samFileInfoSummaryTable
Definition: ViewHTML.py:665
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def ViewHTML.samProjectCollectionToTable (   collection,
  project_name,
  check_n_files = False,
  reduced_files = False,
  skip_projects = False,
  verbose = False 
)

convert the sam project collection to a table

Definition at line 929 of file ViewHTML.py.

References ProductionTestTools.convertSize(), ProductionTestTools.countFilesInDirectory(), ProductionTestTools.flatDirectorySize(), getOutputDSStatus(), ProductionTestTools.getSAMProjectStatus(), ProductionTestTools.getStationMonitorLink(), and samStatusToHTML().

929 def samProjectCollectionToTable(collection,project_name,check_n_files=False,reduced_files=False,skip_projects=False,verbose=False):
930  status = {}
931  expected_files = 0
932  total_files = 0
933  total_size = 0
934  from ProductionTestTools import getStationMonitorLink, countFilesInDirectory, convertSize, flatDirectorySize, getSAMProjectStatus
935  html = \
936  """
937  <div class="panel panel-default">
938  <table class="table table-striped">
939  <tr>
940  <th>Name</th>
941  <th>Description</th>
942  <th>fcl</th>
943  <th>SAM config</th>
944  <th>Sub log</th>
945  <th>monitor</th>
946  <th>status</th>
947  <th>Busy jobs</th>
948  <th>Output files</th>
949  <th>Size</th>
950  <th>Dataset</th>
951  <!--<th>dCache</th>-->
952  <!--reduced_files-->
953  </tr>
954  ROW
955  </table>
956  </div>
957  """
958  if reduced_files:
959  html = html.replace("<!--reduced_files-->","<th>Reduced CAF</th>")
960 
961  for config in collection.configs:
962  n_files = countFilesInDirectory("%s%s"%(collection.output_dir,config.name))
963  total_files += n_files
964  if check_n_files: expected_files += check_n_files
965  line = '<tr>'
966  line += "<td>%s</td>"%config.name
967  line += "<td>%s</td>"%config.description
968  line += '<td><a href="%s/fcl/%s.html">fcl</a></td>'%(project_name,config.name)
969  line += '<td><a href="%s/grid/%s.html">config</a></td>'%(project_name,config.name)
970  line += '<td><a href="%s/log/%s.html">log</a></td>'%(project_name,config.name)
971  line += '<td><a href="%s">monitor</a></td>'%getStationMonitorLink(config.submission_log)
972 
973  # SAM status
974  sam_monitor = config.submission_log
975  sam_status, processes = getSAMProjectStatus(sam_monitor,skip=skip_projects)
976  line += "<td>%s</td>"%samStatusToHTML(sam_status, processes=processes)
977  status[config.name] = {"sam_status":[sam_status],"processes":[processes]}
978 
979  # Busy processes
980  line += "<td>%i</td>"%processes
981  all_files_done = False
982  if check_n_files:
983  if n_files == check_n_files:
984  line += '<td id="n_files_%s" class="success">%i</td>'%(config.name,n_files)
985  status[config.name]["sam_status"][0] = 0
986  all_files_done = True
987  else:
988  line += '<td id="n_files_%s" class="warning">%i</td>'%(config.name,n_files)
989  else:
990  line += '<td id="n_files_%s" >%i</td>'%(config.name,n_files)
991  this_size = flatDirectorySize("%s%s"%(collection.output_dir,config.name))
992  total_size += this_size
993  line += '<td id="n_files_%s">%s</td>'%(config.name,convertSize(this_size))
994 
995  # Output DS
996  ds_html = getOutputDSStatus(config,n_files,verbose=verbose)
997  line += "<td>%s</td>"%ds_html
998  status[config.name]["output_ds"] = ds_html
999 
1000  # # dCache clone
1001  # dcache_html = getOutputCloneStatus(config,n_files,verbose=verbose)
1002  # line += "<td>%s</td>"%dcache_html
1003  # status[config.name]["dcache"] = dcache_html
1004 
1005  if reduced_files:
1006  # if reduced_files[config.name] == 2:
1007  # line += '<td id="n_files_%s"><span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span></td>'%(config.name)
1008  # elif reduced_files[config.name] == 1:
1009  # line += '<td id="n_files_%s"><span class="badge alert-warning"><span class="glyphicon glyphicon-cog"></span></span></td>'%(config.name)
1010  # else:
1011  # line += '<td id="n_files_%s"><span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span></td>'%(config.name)
1012  line += reduced_files[config.name]
1013  line += "</tr>"
1014 
1015 
1016  if hasattr(config,"retry_config"):
1017  if verbose: print "vhmtl: Config: %s has retry: %s"%(config.name,config.retry_config)
1018  line = line.replace("<td>%s</td>"%config.name,'<td rowspan="2">%s</td>'%config.name)
1019  line += "<tr>"
1020  line += "<td>&nbsp; retry</td>"
1021  line += '<td><a href="%s/fcl/%s.html">fcl</a></td>'%(project_name,config.name)
1022  line += '<td><a href="%s/grid/retry_%s.html">config</a></td>'%(project_name,config.name)
1023  line += '<td><a href="%s/log/retry_%s.html">log</a></td>'%(project_name,config.name)
1024  line += '<td><a href="%s">monitor</a></td>'%getStationMonitorLink(config.retry_config.submission_log)
1025  line = line.replace('<td id="n_files_%s"'%config.name,'<td rowspan="2" id="n_files_%s"'%config.name)
1026  sam_monitor = config.retry_config.submission_log
1027  sam_status, processes = getSAMProjectStatus(sam_monitor,skip=skip_projects)
1028  line += "<td>%s</td>"%samStatusToHTML(sam_status, processes=processes)
1029  # Busy processes
1030  line += "<td>%i</td>"%processes
1031  if not all_files_done:
1032  status[config.name]["sam_status"].append(sam_status)
1033  status[config.name]["processes"].append(processes)
1034  line += "</tr>"
1035 
1036  # assert False
1037  html = html.replace("ROW",line+"\nROW")
1038 
1039  return html.replace("ROW",""), status, total_files, expected_files, total_size
def countFilesInDirectory(directory, wildcard="*")
count the number of files in a directory
def getSAMProjectStatus(monitor_text, skip=False)
determine SAM project status from submission log
def convertSize(size)
Parse file sizes into human readable format.
def getStationMonitorLink(log)
get the link to a station monitor from a log
def getOutputDSStatus(config, expected_n_files, verbose=False)
Check the status of a sam project being migrated to SAM.
Definition: ViewHTML.py:1100
def samProjectCollectionToTable(collection, project_name, check_n_files=False, reduced_files=False, skip_projects=False, verbose=False)
convert the sam project collection to a table
Definition: ViewHTML.py:929
def flatDirectorySize(dir_path)
Get the size of a FLAT directory in bytes Sauce: http://stackoverflow.com/questions/1392413/calculati...
def samStatusToHTML(sam_status, processes=0)
Convert a sam status code to HTML.
Definition: ViewHTML.py:1084
def ViewHTML.samProjectParametersToUL (   parameters)

convert the sam project parameters to a UL

Definition at line 894 of file ViewHTML.py.

894 def samProjectParametersToUL(parameters):
895  html = "<ul>\n"
896  params = ["config", "base_fcl", "base_grid", "input_def", "test_rel", "tag", "output_dir", "message"]
897  for p in params:
898  html += " <li><strong>%s:</strong> %s</li>\n"%(p,getattr(parameters,p))
899  html += "</ul>\n"
900  return html
def samProjectParametersToUL(parameters)
convert the sam project parameters to a UL
Definition: ViewHTML.py:894
def ViewHTML.samProjectSummary (   config)

convert the sam project parameters to a UL

Definition at line 903 of file ViewHTML.py.

903 def samProjectSummary(config):
904  html = \
905  """
906  <section>
907  <div class="page-header"><h2 id="%s">%s</h2></div>
908  <div class="panel panel-default">
909  <div class="panel-body">
910  <p>
911  <strong>%s</strong>
912  </p>
913  """%(config.name,config.name.replace("_"," "),config.description)
914 
915  html += '<p><a href="%s/fcl/">fcl</a></p>'%config.name
916  html += '<p><a href="%s/grid/">grid submission script</a></p>'%config.name
917 
918  html += \
919  """
920  </div>
921  </div>
922  </section>
923  <hr>
924  """
925 
926  return html
def samProjectSummary(config)
convert the sam project parameters to a UL
Definition: ViewHTML.py:903
def ViewHTML.samStatusToHTML (   sam_status,
  processes = 0 
)

Convert a sam status code to HTML.

Definition at line 1084 of file ViewHTML.py.

Referenced by directoryToTable(), and samProjectCollectionToTable().

1084 def samStatusToHTML(sam_status, processes=0):
1085  html = False
1086  if sam_status == 1: html = '<span class="badge"><span class="glyphicon glyphicon-cog"></span></span>'
1087  elif sam_status == 2: html = '<span class="badge alert-warning"><span class="glyphicon glyphicon-refresh"></span></span>'
1088  elif sam_status == 3: html = '<span class="badge alert-info"><span class="glyphicon glyphicon-ok"></span></span>'
1089  elif sam_status == 4: html = '<span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>'
1090  elif sam_status == 5: html = '<span class="badge alert-danger"><span class="glyphicon glyphicon-fire"></span></span>'
1091  elif sam_status == 0: html = '<span class="badge alert-success"><span class="glyphicon glyphicon-file"></span></span>'
1092  else:
1093  print "vhtml: Unknown sam status given: ",sam_status
1094  assert False
1095  if processes != 0:
1096  html = html.replace('"><span',' mypopover" data-toggle="tooltip" data-original-title="%i" rel="tooltip"><span'%processes)
1097  return html
def samStatusToHTML(sam_status, processes=0)
Convert a sam status code to HTML.
Definition: ViewHTML.py:1084
def ViewHTML.showMetric (   metric,
  output_name = "metric.html" 
)

print metric

Definition at line 42 of file ViewHTML.py.

References open(), and stringToHTML().

42 def showMetric(metric, output_name="metric.html"):
43  repr_metric = repr(metric)
44  repr_metric = stringToHTML(repr_metric)
45  to_return = base_html
46  to_return = to_return.replace("<!--BODY_HOOK-->", repr_metric)
47  output_file = open(output_name, "wb")
48  output_file.write(to_return)
49  output_file.close()
50  #return to_return
def stringToHTML(string)
Convert a string to html.
Definition: ViewHTML.py:36
procfile open("FD_BRL_v0.txt")
def showMetric(metric, output_name="metric.html")
print metric
Definition: ViewHTML.py:42
def ViewHTML.showMetricsSummary (   chain,
  skip_unused_tiers = True,
  output_name = "summary.html" 
)

print metrics summary

Definition at line 53 of file ViewHTML.py.

References metricsToHTMLTable(), and open().

53 def showMetricsSummary(chain, skip_unused_tiers=True, output_name="summary.html"):
54  repr_metric = metricsToHTMLTable(chain, skip_unused_tiers=skip_unused_tiers)
55  to_return = base_html
56  to_return = to_return.replace("<!--BODY_HOOK-->", repr_metric)
57  output_file = open(output_name, "wb")
58  output_file.write(to_return)
59  output_file.close()
def metricsToHTMLTable(chain, skip_unused_tiers=False, fcl_links=True, log_links=True, metric_links=True)
Display metrics as an HTML table.
Definition: ViewHTML.py:70
def showMetricsSummary(chain, skip_unused_tiers=True, output_name="summary.html")
print metrics summary
Definition: ViewHTML.py:53
procfile open("FD_BRL_v0.txt")
def ViewHTML.stringToHTML (   string)

Convert a string to html.

Definition at line 36 of file ViewHTML.py.

Referenced by showMetric().

36 def stringToHTML(string):
37  string = string.replace("\n","<br>")
38  string = string+"<br>"
39  return string
def stringToHTML(string)
Convert a string to html.
Definition: ViewHTML.py:36
def ViewHTML.tierConfigurationsTables (   chains)

TierConfigurations table.

Definition at line 557 of file ViewHTML.py.

References novadaq::HexUtils.format().

558  import os
559  c_html = ""
560  chain_names = chains.keys()
561  chain_names.sort()
562  # loop through chains and print each tiers configuration
563  for name in chain_names:
564  chain = chains[name]
565  c_html += '<div class="page-header"><h2 id="%s"> %s </h2></div>\n'%(chain.name,chain.name.replace("_"," "))
566  # c_html += '<div class="panel panel-default">\n'
567  c_html += '<table class="table table-striped">\n'
568  c_html += """
569  <tr>
570  <th>Tier</th>
571  <th>inspect only</th>
572  <th>parent in SAM</th>
573  <th>input</th>
574  <th>fcl</th>
575  <th>outputs</th>
576  <th>precommand</th>
577  <th>multiply events</th>
578  <th>multiple outputs</th>
579  </tr>\n
580  """
581  tiers = chain.tiers
582  for tier in tiers:
583  c_html += "<tr>"
584  c_html += "<td id=%s_%s>%s</td>"%(chain.name,tier.short_name,tier.short_name)
585  c_html += "<td>%s</td>"%repr(tier.inspect_only)
586  c_html += "<td>%s</td>"%repr(tier.parent_in_sam)
587  c_html += "<td>%s</td>"%repr(tier.input_name)
588  c_html += "<td>%s</td>"%repr(tier.fcl)
589  c_html += "<td>%s</td>"%repr(tier.output_name)
590  c_html += "<td>%s</td>"%repr(tier.precommand)
591  c_html += "<td>%s</td>"%"{:,}".format(tier.multiply_events)
592  c_html += "<td>%s</td>"%repr(tier.multiple_outputs)
593  c_html += "</tr>\n"
594  c_html += "\n</table>"
595  # c_html += "\n</div>"
596  c_html += "\n<hr>"
597  c_html = c_html.replace("'","")
598  c_html = c_html.replace("%s/FD_raw/"%os.environ["NOVAPRODVALID_DATA"],"")
599  c_html = c_html.replace("%s/ND_raw/"%os.environ["NOVAPRODVALID_DATA"],"")
600  return c_html
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def tierConfigurationsTables(chains)
TierConfigurations table.
Definition: ViewHTML.py:557
def ViewHTML.tierConfigurationsTOC (   chains)

TierConfigurations TOC.

Definition at line 516 of file ViewHTML.py.

517  toc = '<div>'
518  chain_names = chains.keys()
519  chain_names.sort()
520  making_group = False
521  for name in chain_names:
522  line = ""
523  chain = chains[name]
524  b = "primary"
525  if "FD" in name: b = "info"
526  if "genie" in name:
527  if not making_group:
528  making_group = True
529  # open drop down
530  line += \
531  """
532  <div class="btn-group">
533  <button type="button" class="btn btn-%s dropdown-toggle" data-toggle="dropdown">%s <span class="caret"></span></button>
534  <ul class="dropdown-menu" role="menu">
535  <li><a href="#%s">%s</a></li>
536  """%(b,name.split("genie")[0].replace("_"," ")+" genie",chain.name,chain.name.replace("_"," "))
537  else:
538  line += '<li><a href="#%s">%s</a></li>'%(chain.name,chain.name.replace("_"," "))
539  else:
540  if making_group:
541  making_group = False
542  # close drop down group
543  line += "</ul></div>"
544  line +=\
545  """
546  <div class="btn-group">
547  <a class="btn btn-%s" href="#%s">%s</a>
548  </div>
549  """%(b, chain.name,chain.name.replace("_"," "))
550  toc += line
551  if making_group:
552  toc += '</ul></div>'
553  toc += "\n</div>"
554  return toc
def tierConfigurationsTOC(chains)
TierConfigurations TOC.
Definition: ViewHTML.py:516

Variable Documentation

ViewHTML.base_html = \

base html

Definition at line 3 of file ViewHTML.py.

dictionary ViewHTML.statusToGlyph = {}

convert a status into a glyph-icon

Definition at line 603 of file ViewHTML.py.