ViewHTML.py
Go to the documentation of this file.
1 ###########################
2 # base html
3 base_html = \
4 """
5 <!DOCTYPE html>
6 <html lang="en">
7 <head>
8  <meta charset="utf-8">
9  <title>base html</title>
10  <meta name="viewport" content="width=device-width, initial-scale=1.0">
11  <meta name="description" content="">
12  <meta name="author" content="Matthew Tamsett">
13  <link rel="shortcut icon" href="http://nusoft.fnal.gov/nova/production/static/nu_bkg.ico"/>
14 
15  <!-- Bootstrap styling -->
16  <link href="http://nusoft.fnal.gov/nova/production/static/bootstrap/css/bootstrap.css" rel="stylesheet">
17  <style>
18  body {
19  padding-top: 0px; /* 60px to make the container go all the way to the bottom of the topbar */
20  }
21  </style>
22  <link href="http://nusoft.fnal.gov/nova/production/static/bootstrap/css/bootstrap-responsive.css" rel="stylesheet">
23 
24  <!--HEADER_HOOK-->
25 </head>
26 <body>
27 <div class="container">
28 <!--BODY_HOOK-->
29 </div> <!-- /container -->
30 </body>
31 </body>
32 </html>
33 """
34 ###########################
35 # Convert a string to html
36 def stringToHTML(string):
37  string = string.replace("\n","<br>")
38  string = string+"<br>"
39  return string
40 ###########################
41 # print metric
42 def showMetric(metric, output_name="metric.html"):
43  repr_metric = repr(metric)
44  repr_metric = stringToHTML(repr_metric)
45  to_return = base_html
46  to_return = to_return.replace("<!--BODY_HOOK-->", repr_metric)
47  output_file = open(output_name, "wb")
48  output_file.write(to_return)
49  output_file.close()
50  #return to_return
51 ###########################
52 # print metrics summary
53 def showMetricsSummary(chain, skip_unused_tiers=True, output_name="summary.html"):
54  repr_metric = metricsToHTMLTable(chain, skip_unused_tiers=skip_unused_tiers)
55  to_return = base_html
56  to_return = to_return.replace("<!--BODY_HOOK-->", repr_metric)
57  output_file = open(output_name, "wb")
58  output_file.write(to_return)
59  output_file.close()
60 ###########################
61 # recursive replace based on occurance
62 # from:
63 # http://stackoverflow.com/questions/2556108/how-to-replace-the-last-occurence-of-an-expression-in-a-string
64 def rreplace(s, old, new, occurrence):
65  li = s.rsplit(old, occurrence)
66  return new.join(li)
67 ###########################
68 # Display metrics as an HTML table
69 def metricsToHTMLTable(chain, skip_unused_tiers=False,
70  fcl_links=True, log_links=True, metric_links=True):
71  if "convertSize" not in dir(): from ProductionTestTools import convertSize
72  if "Metric" not in dir(): import Metric
73  status = False
74  table_base = \
75  """
76  <div class="page-header"><h2 id="%s"> %s </h2></div>
77 
78  """%(chain.name,chain.name.replace("_"," "))
79 
80  table_base += "\n"+collectEnvironment(chain.env, chain.time_str)+"\n"
81  if not log_links:
82  table_base = table_base.replace("</ul></div>",
83  "</ul>\n<strong>No logs requested</strong>\n</div>")
84  if not fcl_links:
85  table_base = table_base.replace("</ul></div>",
86  "</ul>\n<strong>No FHiCLs requested</strong>\n</div>")
87  if not metric_links:
88  table_base = table_base.replace("</ul></div>",
89  "</ul>\n<strong>No metrics requested</strong>\n</div>")
90 
91  header = \
92  """
93  <div class="panel panel-default">
94  <table class="table table-striped">
95  <tr>
96  <th>Tier</th>
97  <th>In evt</th>
98  <th>User CPU (/in evt) [s]</th>
99  <th>Memory</th>
100  <th>DB queries</th>
101  <th>Query time [s]</th>
102  <th>Child</th>
103  <th>Events (efficiency)</th>
104  <th>Size (/out evt)</th>
105  </tr>
106  """
107 
108  table_base += header
109  rows = ""
110  any_run = False
111  for tier in chain.tiers:
112  if skip_unused_tiers and (tier.metric.run == False): continue
113  # name
114  t = tier.short_name
115 
116  if (tier.metric.run == False):
117  rows += '<tr class="warning"><td id=%s_%s>%s - was not run</td>'%(chain.name,tier.short_name,tier.short_name)
118  rows += "<td></td>"*8
119  rows += "</tr>\n"
120  if tier.metric.return_code == 1: status = 5
121  continue
122  any_run = True
123 
124  continue_after_metrics = False
125  if (tier.metric.return_code != 0):
126  row = '<tr class="danger"><td id=%s_%s>%s - failed, got return code: %i</td>'%(chain.name,tier.short_name,tier.short_name, tier.metric.return_code)
127  row += "<td></td>"*8
128  row += "</tr>\n"
129  status = 5
130  continue_after_metrics = True
131  else:
132  row = "<tr>\n"
133  row += "<td id=%s_%s>%s</td>\n"%(chain.name,tier.short_name,t)
134 
135  if log_links:
136  row = row.replace(">%s"%t,'><pre>%s\n<a href="logs/%s_%s.html">log</a></pre>'%\
137  (t,chain.name,tier.short_name))
138 
139  if metric_links:
140  if log_links:
141  row = row.replace("log</a>",'log</a>, <a href="metrics/%s_%s.html">metrics</a>'%\
142  (chain.name,tier.short_name))
143  else:
144  row = row.replace(">%s"%t,'><pre>%s\n<a href="metrics/%s_%s.html">metrics</a></pre>'%\
145  (t,chain.name,tier.short_name))
146  if fcl_links:
147  if log_links:
148  row = row.replace("log</a>",'log</a>, <a href="fcls/%s_%s.html">fcl</a>'%\
149  (chain.name,tier.short_name))
150  elif metric_links:
151  row = row.replace("metric</a>",'metric</a>, <a href="fcls/%s_%s.html">fcl</a>'%\
152  (chain.name,tier.short_name))
153  else:
154  row = row.replace(">%s"%t,'><pre>%s\n<a href="fcls/%s_%s.html">fcl</a></pre>'%\
155  (t,chain.name,tier.short_name))
156 
157  if continue_after_metrics:
158  rows += row
159  continue
160  metric = tier.metric
161  #print metric
162 
163  # check on if this tier was run
164  # This should probably be extended to check is this tier completed.
165  if (tier.metric.run == False):
166  row += "<td>%s</td>\n"%"Not run"
167  rows+=row
168  continue
169 
170  # input/used events
171  in_events = None
172  if (metric.used_events == None) and (metric.input_events == None):
173  t = "n/a"
174  elif metric.used_events and metric.input_events:
175  t = "%i"%min(metric.used_events,metric.input_events)
176  in_events = min(metric.used_events,metric.input_events)
177  elif metric.used_events:
178  t = "%i"%metric.used_events
179  in_events = metric.used_events
180  elif metric.input_events:
181  t = "%i"%metric.input_events
182  in_events = metric.input_events
183  row += "<td>%s</td>\n"%(t)
184 
185  # user cpu
186  if metric.user_cpu == None:
187  cpu = "n/a"
188  e_cpu = "n/a"
189  else:
190  cpu = "%.2f"%metric.user_cpu
191  if (in_events != None) and (in_events != 0):
192  e_cpu = "%.2f"%(metric.user_cpu/float(in_events))
193  else:
194  e_cpu = "n/a"
195  row += '<td><a href="timings/%s_%s.html">%s (%s)</a></td>\n'%(chain.name,tier.short_name,cpu,e_cpu)
196 
197  # memory
198  if metric.peak_memory == None: mem = "n/a"
199  else: mem = "%s"%convertSize(metric.peak_memory)
200  row += "<td>%s</td>\n"%(mem)
201 
202  # db queries
203  if (metric.db_usage != None) and (metric.db_usage.n_queries != None):
204  q = "%i"%metric.db_usage.n_queries
205  else:
206  q = "n/a"
207  row += "<td>%s</td>\n"%q
208 
209  # query time
210  if (metric.db_usage != None):
211  q = "%.2f"%(metric.db_usage.sumQueryTime()/1000)
212  else:
213  q = "n/a"
214  row += "<td>%s</td>\n"%q
215 
216  # parsing time
217  #if (metric.db_usage != None):
218  #q = "%16.2f"%(metric.db_usage.sumParsingTime()/1000)
219  #else:
220  #q = "n/a"
221  #line += "%16s | "%q
222 
223 
224  if metric.output_name != None:
225  width = 0
226  for i, out in enumerate(metric.output_name):
227  # For the secondary children start a new line in the
228  # table which is indented to the depth of the first.
229  # Take the indentation depth from the first child.
230  if (i == 0):
231  width = row.count("</td>")
232  if (i != 0):
233  row += "</tr>\n<tr>\n"
234  for j in range(width):
235  row+='<td align="center">``</td>\n'
236  # name
237  if len(out) > 27:
238  out_name = "%s...%s"%(out[:12],out[-12:])
239  else:
240  out_name = out
241  row += "<td>%s</td>\n"%out_name
242 
243  # events
244  if metric.output_events[i] == None:
245  e = "n/a"
246  eff = "n/a"
247  else:
248  e = "%i"%metric.output_events[i]
249  if (in_events != None) and (in_events != 0):
250  eff = "%.0f [%%]"%(100*float(metric.output_events[i])/float(in_events))
251  else:
252  eff = "n/a"
253  row += "<td>%s (%s)</td>\n"%(e,eff)
254 
255  # size
256  if metric.output_size[i] == None:
257  e = "n/a"
258  eff = "n/a"
259  else:
260  e = "%s"%convertSize(metric.output_size[i])
261  if (metric.output_events != None) and (metric.output_events[i] != 0):
262  eff = "%s"%convertSize(float(metric.output_size[i])/float(metric.output_events[i]))
263  else:
264  eff = "n/a"
265  row += '<td><a href="file_sizes/%s_%s_%s.html">%s (%s)</a></td>\n'%(chain.name,tier.short_name,out.replace(".","_"),e,eff)
266 
267  # check for any output events
268  if ((metric.output_events[i] in [None,0]) and ("stop" not in out)):
269  row = rreplace(row, "<tr>", '<tr class="danger">', 1)
270  status = 5
271 
272  else:
273  row += "<td>No children</td>\n"
274 
275  row +="</tr>\n"
276  rows+=row
277  if not any_run: status = 5
278  table_base+=rows
279  table_base+="\n</table>\n</div>"
280  return table_base, status
281 ###########################
282 # print a masthead
283 def masthead(text,subtext=""):
284  to_return = \
285  """
286  <!-- Masthead ================================================== -->
287  <header class="mastheader row" id="header">
288 
289  <div class="span12">
290  <div class="page-header">
291  <h1>
292  TEXT
293  <small>SUBT</small>
294  </h1>
295  </div>
296  </div>
297  </header>
298  """
299  to_return = to_return.replace("TEXT",text)
300  to_return = to_return.replace("SUBT",subtext)
301  return to_return
302 ###########################
303 # footer
304 def footer():
305  import time
306  t_string = time.strftime("%Y-%m-%d %H:%M:%S (%Z)", time.localtime())
307  #t_string = t_string.replace(t_string[-3:],":%s"%str(t_string[-3:]))
308  to_return = \
309  """
310  <hr>
311  <div id="footer">
312  <div class="container-fluid">
313  <p class="muted credit">Page autogenerated at <font color="#0066CC">%s</font> by Production's <a href="https://www.youtube.com/watch?v=WGoi1MSGu64">robots</a>.</p>
314  </div>
315  </div>
316  """%t_string
317  return to_return
318 ###########################
319 # collect environment
320 def collectEnvironment(environ, time_str):
321  env = \
322  """
323  <div class="alert alert-info bs-alert-old-docs">
324  This test was run using:
325  <ul>
326  """
327  #env = "<dl>\n"
328  #import time
329  #the_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
330  #env += "<dt>Local time:</dt>\n<dd>%s</dd>"%the_time
331  #import os
332  env += "<li><strong>Time:</strong> %s</li>"%(time_str)
333  variables = ["USER", "HOSTNAME", "HOSTTYPE", "SRT_BASE_RELEASE","SRT_QUAL", "SRT_PUBLIC_CONTEXT", "SRT_PRIVATE_CONTEXT"]
334  for variable in variables:
335  if variable in environ.keys():
336  env += "<li><strong>%s:</strong> %s</li>"%(variable,environ[variable])
337  env += "</ul></div>"
338  return env
339 ###########################
340 # parse info
341 def parseInfo(info,folder):
342  import ProductionTestTools as tools
343  html = \
344  """
345  <div class="row">
346  <div class="col-md-3">
347  <div class="panel panel-default">
348  <div class="panel-heading">
349  <h3 class="panel-title">Test parameters</h3>
350  </div>
351  <div class="panel-body">
352  <ul>
353  <li>TIME</li>
354  <li>RELEASE</li>
355  <li>MESSAGE</li>
356  </ul>
357  </div>
358  </div>
359  </div>
360 
361  <div class="col-md-6">
362  <div class="panel panel-default">
363  <div class="panel-heading">
364  <h3 class="panel-title">Chains</h3>
365  </div>
366  <!-- <div class="panel-body"> -->
367  <table class="table table-striped">
368  <!-- <tr> <th>Tier</th> <th>Logs</th> <th>Status</th> </tr> -->
369  TIER
370  </table>
371  <!-- </div> -->
372  </div>
373  </div>
374 
375  <div class="col-md-3">
376  <div class="panel panel-default">
377  <div class="panel-heading">
378  <h3 class="panel-title">Batch job status keys</h3>
379  </div>
380  <div class="panel-body">
381  <span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span>
382  &nbsp&nbsp job ended successfully<br>
383 
384  <span class="badge"><span class="glyphicon glyphicon-cog"></span></span>
385  &nbsp&nbsp job ongoing<br>
386 
387  <span class="badge alert-warning"><span class="glyphicon glyphicon-flag"></span></span>
388  &nbsp&nbsp STDERR not empty<br>
389 
390  <span class="badge alert-danger"><span class="glyphicon glyphicon-wrench"></span></span>
391  &nbsp&nbsp job was killed by batch robots<br>
392 
393  <span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>
394  &nbsp&nbsp error in run tier<br>
395 
396  <span class="badge alert-danger"><span class="glyphicon glyphicon-floppy-remove"></span></span>
397  &nbsp&nbsp no pkl file for a completed chain<br>
398 
399  <span class="badge alert-danger"><span class="glyphicon glyphicon-send"></span></span>
400  &nbsp&nbsp batch submission failed<br>
401 
402  </div>
403  </div>
404  </div>
405 
406  </div>
407  """
408  #env = "<dl>\n"
409  #import time
410  #the_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
411  #env += "<dt>Local time:</dt>\n<dd>%s</dd>"%the_time
412  #import os
413  html = html.replace("TIME","<strong>Time:</strong> %s"%(info.time_str))
414  html = html.replace("RELEASE","<strong>Release:</strong> %s"%(info.release))
415  message = ""
416  if "message" in dir(info):
417  html = html.replace("MESSAGE","<strong>Message:</strong> %s"%(info.message))
418  message = info.message
419  else:
420  html = html.replace("MESSAGE","<strong>Message:</strong> none")
421  message = "No message"
422  logs = info.logs
423  commands = info.commands
424  commands.sort()
425  for i, command in enumerate(commands):
426  chain, chain_name = tools.findChainNameFromCommand(command)
427  this_html = '<tr><td><a href="#%s">%s</a></td>'%(chain_name,chain)
428  this_html += '<td><a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_cmd.html">cmd</a>'%(folder,chain)
429  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_log.html">log</a>'%(folder,chain)
430  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_out.html">out</a>'%(folder,chain)
431  this_html += ', <a href="http://nusoft.fnal.gov/nova/production/testing/%s/batch_logs/%s_err.html">err</a></td>'%(folder,chain)
432  this_html += ' <td><!--BATCH_STATUS_HOOK_%s--><br></td></tr>\n'%chain_name
433  html = html.replace("TIER","%s\nTIER"%this_html)
434  html = html.replace("TIER","")
435  return html,message
436 ###########################
437 # convert a folder list to a html button group
438 def foldersToButtonGroups(folder_list, output, url, page="index.html", messages={}):
439  import os
440  import ProductionTestTools as tools
441 
442  button_groups = {}
443  for folder in folder_list:
444  if not os.path.isdir("%s%s"%(output,folder)) : continue
445  message = "No message"
446  if folder in messages.keys():
447  message = messages[folder]
448  if message == "": message = "No message"
449  # catagorise folders
450  if tools.isLogicalFolder(folder):
451  release = tools.releaseFromFolder(folder)
452  f_date = tools.dateFromFolder(folder)
453  if release not in button_groups.keys():
454  button_groups[release] = []
455  button_groups[release].append((url,folder,f_date, message))
456  else:
457  if "Misc" not in button_groups.keys():
458  button_groups["Misc"] = []
459  button_groups["Misc"].append((url,folder,folder, message))
460 
461  button_html = \
462  """
463  <div>
464  <h3>Development</h3>
465  <!--DEVELOPMENT_BUTTON_HOOK-->
466  <br><br>
467  <!--DEVELOPMENT_HOOK-->
468  </div>
469  <hr>
470 
471  <div>
472  <h3>Snapshot/FA snapshot</h3>
473  <!--S_BUTTON_HOOK-->
474  <br><br>
475  <!--S_HOOK-->
476  </div>
477  <hr>
478 
479  <div>
480  <h3>R-style release</h3>
481  <!--R_BUTTON_HOOK-->
482  <br><br>
483  <!--R_HOOK-->
484  </div>
485  """
486 
487  groups = button_groups.keys()
488  groups.sort()
489  patterns = { "DEVELOPMENT_BUTTON_HOOK":["development"],"S_BUTTON_HOOK":["FA","S"], "R_BUTTON_HOOK":["R"]}
490  for pattern_key in patterns.keys():
491  for group in groups:
492  matches = False
493  for sub_pattern in patterns[pattern_key]:
494  if group[:len(sub_pattern)] == sub_pattern: matches=True
495  if not matches: continue
496 
497  html = \
498  """
499  <div class="btn-group">
500  <button type="button" class="btn btn-default dropdown-toggle" data-toggle="dropdown"><span class="badge">%i</span> %s <span class="caret"></span></button>
501  <ul class="dropdown-menu" role="menu">
502  TEST
503  </ul>
504  </div><!-- /btn-group -->
505  """%(len(button_groups[group]),group)
506  button_groups[group].sort()
507  for s in button_groups[group]:
508  html = html.replace("TEST",
509  '<li><a href="%s%s/%s" data-toggle="tooltip" data-original-title="%s" rel="tooltip">%s</a></li>\nTEST'%(s[0],s[1],page,s[3],s[2]))
510  html = html.replace("TEST","")
511  button_html = button_html.replace("<!--%s-->"%pattern_key,"%s\n<!--%s-->"%(html,pattern_key))
512 
513  return button_html
514 ###########################
515 # TierConfigurations TOC
517  toc = '<div>'
518  chain_names = chains.keys()
519  chain_names.sort()
520  making_group = False
521  for name in chain_names:
522  line = ""
523  chain = chains[name]
524  b = "primary"
525  if "FD" in name: b = "info"
526  if "genie" in name:
527  if not making_group:
528  making_group = True
529  # open drop down
530  line += \
531  """
532  <div class="btn-group">
533  <button type="button" class="btn btn-%s dropdown-toggle" data-toggle="dropdown">%s <span class="caret"></span></button>
534  <ul class="dropdown-menu" role="menu">
535  <li><a href="#%s">%s</a></li>
536  """%(b,name.split("genie")[0].replace("_"," ")+" genie",chain.name,chain.name.replace("_"," "))
537  else:
538  line += '<li><a href="#%s">%s</a></li>'%(chain.name,chain.name.replace("_"," "))
539  else:
540  if making_group:
541  making_group = False
542  # close drop down group
543  line += "</ul></div>"
544  line +=\
545  """
546  <div class="btn-group">
547  <a class="btn btn-%s" href="#%s">%s</a>
548  </div>
549  """%(b, chain.name,chain.name.replace("_"," "))
550  toc += line
551  if making_group:
552  toc += '</ul></div>'
553  toc += "\n</div>"
554  return toc
555 ###########################
556 # TierConfigurations table
558  import os
559  c_html = ""
560  chain_names = chains.keys()
561  chain_names.sort()
562  # loop through chains and print each tiers configuration
563  for name in chain_names:
564  chain = chains[name]
565  c_html += '<div class="page-header"><h2 id="%s"> %s </h2></div>\n'%(chain.name,chain.name.replace("_"," "))
566  # c_html += '<div class="panel panel-default">\n'
567  c_html += '<table class="table table-striped">\n'
568  c_html += """
569  <tr>
570  <th>Tier</th>
571  <th>inspect only</th>
572  <th>parent in SAM</th>
573  <th>input</th>
574  <th>fcl</th>
575  <th>outputs</th>
576  <th>precommand</th>
577  <th>multiply events</th>
578  <th>multiple outputs</th>
579  </tr>\n
580  """
581  tiers = chain.tiers
582  for tier in tiers:
583  c_html += "<tr>"
584  c_html += "<td id=%s_%s>%s</td>"%(chain.name,tier.short_name,tier.short_name)
585  c_html += "<td>%s</td>"%repr(tier.inspect_only)
586  c_html += "<td>%s</td>"%repr(tier.parent_in_sam)
587  c_html += "<td>%s</td>"%repr(tier.input_name)
588  c_html += "<td>%s</td>"%repr(tier.fcl)
589  c_html += "<td>%s</td>"%repr(tier.output_name)
590  c_html += "<td>%s</td>"%repr(tier.precommand)
591  c_html += "<td>%s</td>"%"{:,}".format(tier.multiply_events)
592  c_html += "<td>%s</td>"%repr(tier.multiple_outputs)
593  c_html += "</tr>\n"
594  c_html += "\n</table>"
595  # c_html += "\n</div>"
596  c_html += "\n<hr>"
597  c_html = c_html.replace("'","")
598  c_html = c_html.replace("%s/FD_raw/"%os.environ["NOVAPRODVALID_DATA"],"")
599  c_html = c_html.replace("%s/ND_raw/"%os.environ["NOVAPRODVALID_DATA"],"")
600  return c_html
601 ###########################
602 # convert a status into a glyph-icon
603 statusToGlyph = {}
604 statusToGlyph[0] = ""
605 statusToGlyph[1] = '<span class="badge pull-right alert-success"><span class="glyphicon glyphicon-ok"></span></span>'
606 statusToGlyph[2] = '<span class="badge pull-right"><span class="glyphicon glyphicon-cog"></span></span>'
607 statusToGlyph[3] = '<span class="badge pull-right alert-danger"><span class="glyphicon glyphicon-wrench"></span></span>'
608 statusToGlyph[4] = '<span class="badge pull-right alert-warning"><strong><span class="glyphicon glyphicon-flag"></span></span>'
609 statusToGlyph[5] = '<span class="badge pull-right alert-danger"><span class="glyphicon glyphicon-remove"></span></span>'
610 statusToGlyph[6] = '<span class="badge pull-right alert-danger"><span class="glyphicon glyphicon-floppy-remove"></span></span>'
611 statusToGlyph[7] = '<span class="badge pull-right alert-danger"><span class="glyphicon glyphicon-send"></span></span>'
612 ###########################
613 # Tier assumptions to table
614 def assumptionsToTable(ass, eff, n_per_file):
615  html = \
616  """
617  <table class="table table-striped">
618  <tr>
619  <th>Chain</th>
620  <th>Tier</th>
621  <th>Number of events to produce</th>
622  <th>Efficiency [%]</th>
623  <th>Number of events to run</th>
624  <th>Number of events per file</th>
625  </tr>
626  """
627 
628  chains = ass.keys()
629  chains.sort()
630  for chain in chains:
631  chain_name = chain.split(":")[0]
632  if ":" in chain:
633  tier = chain.split(":")[1]
634  else:
635  tier = "default"
636  #print chain_name, tier
637  events = ass[chain]
638  prod_events = events
639  events_per_file = n_per_file[chain]
640  this_eff = 100
641  if chain in eff.keys():
642  this_eff = (eff[chain])*100.
643  prod_events = prod_events * (100./this_eff)
644  row = \
645  """
646  <tr>
647  <td>%s</td>
648  <td>%s</td>
649  <td>%s</td>
650  <td>%.0f</td>
651  <td>%s</td>
652  <td>%s</td>
653  </tr>
654  """%(chain_name.replace("_"," "),
655  tier,"{:,}".format(events),
656  this_eff,"{:,}".format(int(prod_events)),
657  "{:,}".format(events_per_file))
658  html += row
659 
660 
661  html +="\n</table>"
662  return html
663 ###########################
664 # samFileInfoSummaryTable
665 def samFileInfoSummaryTable(definitions,names=False,configs=False,fcls=False,status=False):
666  import ProductionTestTools as tools
667  html = \
668  """
669  <div class="panel panel-default">
670  <table class="table table-striped">
671  <tr>
672  <th>Definition</th>
673  <th>Files</th>
674  <th>Events</th>
675  <th>Size</th>
676  </tr>
677  """
678  if fcls:
679  html = html.replace("<tr>","<tr><th>FHiCL</th>")
680  if configs:
681  html = html.replace("<tr>","<tr><th>Configuration</th>")
682  if names:
683  html = html.replace("<tr>","<tr><th>Chain</th><th>Tier</th>")
684  if status:
685  html = html.replace("<th>Definition</th>","<th>Definition</th><th>SAM query status</th>")
686 
687  totals = [0,0,0]
688  for i,definition in enumerate(definitions.keys()):
689  file_info = definitions[definition]
690  n_files = definitions[definition]["file_count"]
691  events = definitions[definition]["total_event_count"]
692  if events == None: events = 0
693  size = definitions[definition]["total_file_size"]
694  if size == None: size = 0
695  totals[0]+=n_files
696  totals[1]+=int(events)
697  totals[2]+=size
698 
699  row = \
700  """
701  <tr>
702  <td>%s</td>
703  <td>%s</td>
704  <td>%s</td>
705  <td>%s</td>
706  </tr>
707  """%(definition,
708  "{:,}".format(n_files),
709  "{:,}".format(int(events)),
710  tools.convertSize(size)
711  )
712  if configs:
713  row = row.replace("<tr>",'<tr><td><a href="%s_config.html">config</a></td>'%(definition))
714  if fcls:
715  row = row.replace("<tr>",'<tr><td><a href="%s.html">fcl</a></td>'%(definition))
716  if names:
717  chain = names[i][0]
718  if (i > 0) and (chain == names[i-1][0]):
719  chain = ""
720  row = row.replace("<tr>","<tr><td>%s</td><td>%s</td>"%(chain,names[i][1]))
721  if status:
722  row = row.replace("<td>%s</td>"%definition,"<td>%s</td><td>%s</td>"%(definition,definitions[definition]["status"]))
723  html += row
724  row = \
725  """
726  <tr>
727  <td><strong>Total</strong></td>
728  <td>%s</td>
729  <td>%s</td>
730  <td>%s</td>
731  </tr>
732  """%("{:,}".format(totals[0]),
733  "{:,}".format(totals[1]),
734  tools.convertSize(totals[2])
735  )
736  if configs: row = row.replace("<tr>","<tr><td></td>")
737  if fcls: row = row.replace("<tr>","<tr><td></td>")
738  if status: row = row.replace("<tr>","<tr><td></td>")
739  if names: row = row.replace("<td><strong>Total</strong></td>","<td><strong>Total</strong></td><td></td><td></td>")
740  html += row
741 
742  html +="\n</table></div>"
743  return html, totals
744 ###########################
745 # SAM definition descriptions to modals (for fun)
746 def samDefinitionDescriptionsModal(def_descriptions):
747  from ProductionTestTools import makeAlphanumeric
748  html = ""
749  modal_base = \
750  """
751  <div class="modal fade" id="IDDEF" tabindex="-1" role="dialog" aria-labelledby="DEFLabel" aria-hidden="true">
752  <div class="modal-dialog modal-lg">
753  <div class="modal-content">
754  <div class="modal-body">
755  <ul>BODY</ul>
756  </div>
757  <div class="modal-footer">
758  <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
759  </div>
760  </div><!-- /.modal-content -->
761  </div><!-- /.modal-dialog -->
762  </div><!-- /.modal -->
763  """
764 
765  for definition in def_descriptions.keys():
766  this_modal = modal_base.replace("IDDEF", makeAlphanumeric(definition))
767  this_modal = this_modal.replace("DEF", definition)
768  #this_modal = this_modal.replace("TITLE", "%s description"%definition)
769  this_body = ""
770  description = def_descriptions[definition]
771  for key in description.keys():
772  this_body += "<li><strong>%s:</strong> %s</li>"%(key,str(description[key]))
773  this_body += '<a href="http://samweb.fnal.gov:8480/sam/nova/api/files/list?format=html&dims=%%20%%28%%20defname%%3A%%20%s%%20%%29">SAM-web link</a>'%description["defname"]
774  this_modal = this_modal.replace("BODY",this_body)
775  html+=this_modal
776 
777 
778  return html
779 ###########################
780 # Scan a directory and return a table
781 def directoryToTable(wildcard, base_url, status=False, sam_status=False, to_skip=False, verbose=False):
782  from ProductionTestTools import convertSize
783  html = \
784  """
785  <table class="table table-striped">
786  <tr>
787  <th>File name</th>
788  <th>Modification time</th>
789  HEADERS
790  </tr>
791  ROW
792  </table>
793  """
794  if status:
795  html = html.replace("HEADERS","<th>Completed</th><th>Run time</th>")
796  elif sam_status:
797  html = html.replace("HEADERS","<th>Message</th><th>SAM jobs</th><th>Files</th><th>Size</th><th>Definition</th><th>Reduced CAF</th>")
798  else:
799  html = html.replace("HEADERS","")
800 
801  import glob
802  import os.path
803  import time
804 
805  file_list = glob.glob(wildcard)
806  file_list.sort()
807  for f in file_list:
808  skip = False
809  if to_skip:
810  for s in to_skip:
811  if (s in f): skip = True
812  if skip: continue
813  try:
814  name = f.split("/")[-1]
815  link = base_url+name.replace("txt","html")
816  if link[-5:] != ".html": link = link + ".html"
817  name_link = '<a href="%s">%s</a>'%(link,name)
818  mod_time = time.ctime(os.path.getmtime(f))
819  except OSError:
820  print f,"is not a valid file. Skipping"
821  continue
822 
823  line = "<tr><td>%s</td><td>%s</td>"%(name_link,mod_time)
824  if status:
825  this_status = status[f]
826  this_status_html = '<td align="center">%s</td>'
827  if (this_status[0] != False) and (this_status[1] ==False):
828  line = line + this_status_html%makeBadge("alert-success","Completed successfully","ok")
829  elif (this_status[1] == False):
830  line = line + this_status_html%makeBadge("","Ongoing","cog")
831  else:
832  line = line + this_status_html%makeBadge("alert-danger",this_status[1],"remove")
833 
834  if this_status[2] != None:
835  dt = divmod(this_status[2].total_seconds(), 60)
836  line = line + '<td>%i m %i s</td>'%(dt[0],dt[1])
837  else:
838  line = line +'<td></td>'
839 
840 
841  elif sam_status:
842  this_status = sam_status[f]
843  project_status_dict = this_status[0]
844  # message
845  line += "<td>%s</td>"%this_status[1]
846  # SAM jobs
847  line += "<td>"
848  for project in project_status_dict.keys():
849  if len(project_status_dict[project]["sam_status"]) > 1: line += " ["
850  for i,this_sam_status in enumerate(project_status_dict[project]["sam_status"]):
851  line += samStatusToHTML(this_sam_status,processes=project_status_dict[project]["processes"][i])
852  if len(project_status_dict[project]["sam_status"]) > 1: line += "] "
853  line += "</td>"
854  # n_files
855  progress = int((float(this_status[3]) / this_status[4])*100)
856  # print progress, this_status[3], this_status[4]
857  line += \
858  """
859  <td>
860  <div class="progress">
861  <div class="progress-bar progress-bar-success mypopover" role="progressbar" aria-valuenow="%i" aria-valuemin="0" aria-valuemax="100" style="width: %i%%" data-toggle="tooltip" data-original-title="%s / %s" rel="tooltip">
862  <span class="sr-only">%i%% Complete (success)</span>
863  </div>
864  </div>
865  </td>
866  """%(progress,progress,"{:,}".format(this_status[3]), "{:,}".format(this_status[4]), progress)
867  # size
868  line += "<td nowrap>%s</td>"%convertSize(this_status[5])
869 
870  # SAM definition
871  line += "<td>"
872  for project in project_status_dict.keys():
873  line += project_status_dict[project]["output_ds"]
874  line += "</td>"
875 
876  # # dCache clone
877  # line += "<td>"
878  # for project in project_status_dict.keys():
879  # line += project_status_dict[project]["dcache"]
880  # line += "</td>"
881 
882  # reduced cafs
883  line += "<td>"
884  for c in this_status[2].keys():
885  if "success" in this_status[2][c]: line += '<span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span>'
886  elif "warning" in this_status[2][c]: line += '<span class="badge alert-warning"><span class="glyphicon glyphicon-cog"></span></span>'
887  else: line += '<span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>'
888  line += "</td>"
889 
890  html = html.replace("ROW",line+"</tr>\nROW")
891  return html.replace("ROW","")
892 ###########################
893 # convert the sam project parameters to a UL
894 def samProjectParametersToUL(parameters):
895  html = "<ul>\n"
896  params = ["config", "base_fcl", "base_grid", "input_def", "test_rel", "tag", "output_dir", "message"]
897  for p in params:
898  html += " <li><strong>%s:</strong> %s</li>\n"%(p,getattr(parameters,p))
899  html += "</ul>\n"
900  return html
901 ###########################
902 # convert the sam project parameters to a UL
903 def samProjectSummary(config):
904  html = \
905  """
906  <section>
907  <div class="page-header"><h2 id="%s">%s</h2></div>
908  <div class="panel panel-default">
909  <div class="panel-body">
910  <p>
911  <strong>%s</strong>
912  </p>
913  """%(config.name,config.name.replace("_"," "),config.description)
914 
915  html += '<p><a href="%s/fcl/">fcl</a></p>'%config.name
916  html += '<p><a href="%s/grid/">grid submission script</a></p>'%config.name
917 
918  html += \
919  """
920  </div>
921  </div>
922  </section>
923  <hr>
924  """
925 
926  return html
927 ###########################
928 # convert the sam project collection to a table
929 def samProjectCollectionToTable(collection,project_name,check_n_files=False,reduced_files=False,skip_projects=False,verbose=False):
930  status = {}
931  expected_files = 0
932  total_files = 0
933  total_size = 0
934  from ProductionTestTools import getStationMonitorLink, countFilesInDirectory, convertSize, flatDirectorySize, getSAMProjectStatus
935  html = \
936  """
937  <div class="panel panel-default">
938  <table class="table table-striped">
939  <tr>
940  <th>Name</th>
941  <th>Description</th>
942  <th>fcl</th>
943  <th>SAM config</th>
944  <th>Sub log</th>
945  <th>monitor</th>
946  <th>status</th>
947  <th>Busy jobs</th>
948  <th>Output files</th>
949  <th>Size</th>
950  <th>Dataset</th>
951  <!--<th>dCache</th>-->
952  <!--reduced_files-->
953  </tr>
954  ROW
955  </table>
956  </div>
957  """
958  if reduced_files:
959  html = html.replace("<!--reduced_files-->","<th>Reduced CAF</th>")
960 
961  for config in collection.configs:
962  n_files = countFilesInDirectory("%s%s"%(collection.output_dir,config.name))
963  total_files += n_files
964  if check_n_files: expected_files += check_n_files
965  line = '<tr>'
966  line += "<td>%s</td>"%config.name
967  line += "<td>%s</td>"%config.description
968  line += '<td><a href="%s/fcl/%s.html">fcl</a></td>'%(project_name,config.name)
969  line += '<td><a href="%s/grid/%s.html">config</a></td>'%(project_name,config.name)
970  line += '<td><a href="%s/log/%s.html">log</a></td>'%(project_name,config.name)
971  line += '<td><a href="%s">monitor</a></td>'%getStationMonitorLink(config.submission_log)
972 
973  # SAM status
974  sam_monitor = config.submission_log
975  sam_status, processes = getSAMProjectStatus(sam_monitor,skip=skip_projects)
976  line += "<td>%s</td>"%samStatusToHTML(sam_status, processes=processes)
977  status[config.name] = {"sam_status":[sam_status],"processes":[processes]}
978 
979  # Busy processes
980  line += "<td>%i</td>"%processes
981  all_files_done = False
982  if check_n_files:
983  if n_files == check_n_files:
984  line += '<td id="n_files_%s" class="success">%i</td>'%(config.name,n_files)
985  status[config.name]["sam_status"][0] = 0
986  all_files_done = True
987  else:
988  line += '<td id="n_files_%s" class="warning">%i</td>'%(config.name,n_files)
989  else:
990  line += '<td id="n_files_%s" >%i</td>'%(config.name,n_files)
991  this_size = flatDirectorySize("%s%s"%(collection.output_dir,config.name))
992  total_size += this_size
993  line += '<td id="n_files_%s">%s</td>'%(config.name,convertSize(this_size))
994 
995  # Output DS
996  ds_html = getOutputDSStatus(config,n_files,verbose=verbose)
997  line += "<td>%s</td>"%ds_html
998  status[config.name]["output_ds"] = ds_html
999 
1000  # # dCache clone
1001  # dcache_html = getOutputCloneStatus(config,n_files,verbose=verbose)
1002  # line += "<td>%s</td>"%dcache_html
1003  # status[config.name]["dcache"] = dcache_html
1004 
1005  if reduced_files:
1006  # if reduced_files[config.name] == 2:
1007  # line += '<td id="n_files_%s"><span class="badge alert-success"><span class="glyphicon glyphicon-ok"></span></span></td>'%(config.name)
1008  # elif reduced_files[config.name] == 1:
1009  # line += '<td id="n_files_%s"><span class="badge alert-warning"><span class="glyphicon glyphicon-cog"></span></span></td>'%(config.name)
1010  # else:
1011  # line += '<td id="n_files_%s"><span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span></td>'%(config.name)
1012  line += reduced_files[config.name]
1013  line += "</tr>"
1014 
1015 
1016  if hasattr(config,"retry_config"):
1017  if verbose: print "vhmtl: Config: %s has retry: %s"%(config.name,config.retry_config)
1018  line = line.replace("<td>%s</td>"%config.name,'<td rowspan="2">%s</td>'%config.name)
1019  line += "<tr>"
1020  line += "<td>&nbsp; retry</td>"
1021  line += '<td><a href="%s/fcl/%s.html">fcl</a></td>'%(project_name,config.name)
1022  line += '<td><a href="%s/grid/retry_%s.html">config</a></td>'%(project_name,config.name)
1023  line += '<td><a href="%s/log/retry_%s.html">log</a></td>'%(project_name,config.name)
1024  line += '<td><a href="%s">monitor</a></td>'%getStationMonitorLink(config.retry_config.submission_log)
1025  line = line.replace('<td id="n_files_%s"'%config.name,'<td rowspan="2" id="n_files_%s"'%config.name)
1026  sam_monitor = config.retry_config.submission_log
1027  sam_status, processes = getSAMProjectStatus(sam_monitor,skip=skip_projects)
1028  line += "<td>%s</td>"%samStatusToHTML(sam_status, processes=processes)
1029  # Busy processes
1030  line += "<td>%i</td>"%processes
1031  if not all_files_done:
1032  status[config.name]["sam_status"].append(sam_status)
1033  status[config.name]["processes"].append(processes)
1034  line += "</tr>"
1035 
1036  # assert False
1037  html = html.replace("ROW",line+"\nROW")
1038 
1039  return html.replace("ROW",""), status, total_files, expected_files, total_size
1040 ###########################
1041 # Convert a SAM input definition into an alert
1042 def samDefinitionAlert(input_def,guess_child=False):
1043  children = {"prod_daq_FA14-10-03a_nd_genie_fhc_nonswap" : "prod_reco_FA15-02-09_nd_genie_fhc_nonswap",
1044  "prod_daq_FA14-10-03x.a_fd_genie_fhc_nonswap" : "prod_reco_S15-05-04_fd_genie_fhc_nonswap",
1045  "prod_daq_FA14-10-03x.a_fd_genie_fhc_fluxswap" : "prod_reco_S15-05-04_fd_genie_fhc_fluxswap",
1046  "prod_artdaq_S15-03-11_nd_numi" : 3533,
1047  "tamsett_snap_140515_prod_daq_FA14-10-03x.a_nd_genie_fhc_nonswap" : 10401}
1048 
1049  from ProductionTestTools import samDefinitionSummary
1050  html = \
1051  """
1052  <div class="alert alert-success bs-alert-old-docs" role="alert">
1053  <p>input: <strong>%s</strong>
1054  """%input_def
1055 
1056  summary = samDefinitionSummary(input_def)
1057  expected_n_events = False
1058 
1059  html += " has <strong>%i</strong> files</p>"%summary["file_count"]
1060 
1061  if guess_child:
1062  if input_def in children.keys():
1063  child_def = children[input_def]
1064  else:
1065  release = input_def.split("_")[2]
1066  child_def = input_def.replace("daq","reco").replace(release,guess_child)
1067  if type(child_def) == int:
1068  html += "<p>forced number of children expected: <strong>%i</strong> files</p>"%(child_def)
1069  expected_n_events = child_def
1070  else:
1071  try:
1072  summary = samDefinitionSummary(child_def)
1073  html += "<p>child: <strong>%s</strong> has <strong>%i</strong> files</p>"%(child_def,summary["file_count"])
1074  expected_n_events = summary["file_count"]
1075  except:
1076  html += "<p><strong>No child definition found</strong>, tried: %s</p>"%(child_def)
1077  expected_n_events = 1
1078  else:
1079  expected_n_events = samDefinitionSummary(input_def)["file_count"]
1080  html += "</div>"
1081  return html, expected_n_events
1082 ###########################
1083 # Convert a sam status code to HTML
1084 def samStatusToHTML(sam_status, processes=0):
1085  html = False
1086  if sam_status == 1: html = '<span class="badge"><span class="glyphicon glyphicon-cog"></span></span>'
1087  elif sam_status == 2: html = '<span class="badge alert-warning"><span class="glyphicon glyphicon-refresh"></span></span>'
1088  elif sam_status == 3: html = '<span class="badge alert-info"><span class="glyphicon glyphicon-ok"></span></span>'
1089  elif sam_status == 4: html = '<span class="badge alert-danger"><span class="glyphicon glyphicon-remove"></span></span>'
1090  elif sam_status == 5: html = '<span class="badge alert-danger"><span class="glyphicon glyphicon-fire"></span></span>'
1091  elif sam_status == 0: html = '<span class="badge alert-success"><span class="glyphicon glyphicon-file"></span></span>'
1092  else:
1093  print "vhtml: Unknown sam status given: ",sam_status
1094  assert False
1095  if processes != 0:
1096  html = html.replace('"><span',' mypopover" data-toggle="tooltip" data-original-title="%i" rel="tooltip"><span'%processes)
1097  return html
1098 ###########################
1099 # Check the status of a sam project being migrated to SAM
1100 def getOutputDSStatus(config,expected_n_files,verbose=False):
1101  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1102  if not hasattr(config,"output_sam_defname"):
1103  return html%("danger","No output SAM dataset","remove")
1104  from ProductionTestTools import samDefinitionSummary
1105 
1106  def_name = config.output_sam_defname
1107 
1108  if verbose: print "vhtml: definition has been declared as: %s"%config.output_sam_defname
1109 
1110  output_files = samDefinitionSummary(config.output_sam_defname)
1111  info = "defname: %s (%i/%i files)"%(def_name,output_files["file_count"],expected_n_files)
1112  if output_files["file_count"] == expected_n_files:
1113  return html%("success",info,"ok")
1114  else:
1115  return html%("warning",info,"cog")
1116 ###########################
1117 # Check the status of a sam project being migrated to SAM
1118 def getOutputCloneStatus(config,expected_n_files,verbose=False):
1119  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1120  if not hasattr(config,"pnfs_output_dir"):
1121  return html%("danger","No PNFS clone","remove")
1122 
1123  import glob
1124  import os
1125 
1126  output_dir = config.pnfs_output_dir
1127  if verbose: print "vhtml: definition has been cloned to: %s"%config.pnfs_output_dir
1128 
1129  if not os.path.exists(output_dir):
1130  return html%("danger","Clone location: %s doesn't exist"%output_dir,"fire")
1131 
1132  files = glob.glob(output_dir+"/*")
1133  info = "location: %s (%i/%i files)"%(output_dir,len(files),expected_n_files)
1134  if len(files) == expected_n_files:
1135  return html%("success",info,"ok")
1136  else:
1137  return html%("warning",info,"cog")
1138 ###########################
1139 # A dictionary of sam query status badges
1140 def makeSAMQueryStatusBadge(status,time=0.,date="???"):
1141  html = '<span class="badge alert-%s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1142  if status == "good":
1143  return html%("success","SAM query returned in %.2f s on %s"%(time,date),"ok")
1144  elif status == "timed out":
1145  return html%("danger","SAM timed out after %.2f s on %s"%(time,date),"remove")
1146  elif status == "cached":
1147  return html%("info","Cached query from %s used"%(date),"cog")
1148  elif "error" in status:
1149  return html%("danger","SAM Error %s on %s"%(status,date),"remove")
1150  else:
1151  return html%("warning","Unknown status, %s, on %s"%(status,date),"cog")
1152 ###########################
1153 # Turn a list of definitions into a list of divs for the explorer JS
1155  html = \
1156  """
1157  <div id="searchable_datasets">
1158  """
1159  for definition in definitions:
1160  html += "\n<div>%s</div>"%definition
1161  html += "\n</div>"
1162  return html
1163 ###########################
1164 # Turn a list of definitions into a list of JS objects for the explorer JS
1165 def definitionListToExplorerData(definitions,filter_on_mrcc=False,skip_counts=False):
1166  import json
1167  if "convertSize" not in dir(): from ProductionTestTools import convertSize
1168  html = \
1169  """
1170  <script>
1171  var DATA = {}
1172  DATA.datasets = ["""
1173  # html += ",\n ".join('"%s"'%b for b in definitions)
1174  unique_definitions = [ a for a in set(definitions) ]
1175  unique_definitions.sort(key=lambda x: x.name)
1176  detectors = ["All"]
1177  data_mcs = ["All"]
1178  triggers = ["All"]
1179  tiers = ["All"]
1180  releases = ["All"]
1181  mrccs = ["All"]
1182 
1183  for definition in unique_definitions:
1184  if (definition.file_count == None): definition.file_count = 0
1185  if (definition.total_event_count == None): definition.total_event_count = 0
1186  if (definition.total_file_size == None): definition.total_file_size = 0
1187  detector = findDetector(definition)
1188  is_mc = isSimulated(definition)
1189  trigger = findTrigger(definition)
1190  tier = findTier(definition)
1191  release = findRelease(definition)
1192  mrcc = findMRCC(definition)
1193 
1194  if not trigger: continue
1195  if not tier: continue
1196  if tier == "ccaf": continue
1197 
1198  if detector not in detectors: detectors.append(detector)
1199  if is_mc not in data_mcs: data_mcs.append(is_mc)
1200  if trigger not in triggers: triggers.append(trigger)
1201  if tier not in tiers: tiers.append(tier)
1202  if release not in releases: releases.append(release)
1203  if mrcc not in mrccs: mrccs.append(mrcc)
1204 
1205  html += '\n {"name":"%s","detector":"%s","is_mc":"%s","trigger":"%s","tier":"%s","release":"%s","mrcc":"%s",description:%s,"file_count":"%s","total_event_count":"%s","total_file_size":"%s"},'%\
1206  (definition.name,detector,is_mc,trigger,tier,release,mrcc,\
1207  json.dumps(definition.description),\
1208  "{:,}".format(definition.file_count),\
1209  "{:,}".format(int(definition.total_event_count)),\
1210  convertSize(definition.total_file_size))
1211  html += "\n ];"
1212 
1213  html += "\n DATA.detectors = ["
1214  html += ",\n ".join('"%s"'%b for b in detectors)
1215  html += "\n ];"
1216 
1217  html += "\n DATA.is_mcs = ["
1218  html += ",\n ".join('"%s"'%b for b in data_mcs)
1219  html += "\n ];"
1220 
1221  html += "\n DATA.triggers = ["
1222  html += ",\n ".join('"%s"'%b for b in triggers)
1223  html += "\n ];"
1224 
1225  tiers.sort()
1226  html += "\n DATA.tiers = ["
1227  html += ",\n ".join('"%s"'%b for b in tiers)
1228  html += "\n ];"
1229 
1230  releases.sort()
1231  html += "\n DATA.releases = ["
1232  html += ",\n ".join('"%s"'%b for b in releases)
1233  html += "\n ];"
1234 
1235  html += "\n DATA.mrccs = ["
1236  html += ",\n ".join('"%s"'%b for b in mrccs)
1237  html += "\n ];"
1238 
1239  if filter_on_mrcc: html += "\n DATA.include_mrcc = true;"
1240  if skip_counts: html += "\n DATA.skip_counts = true;"
1241  html += "\n </script>"
1242  return html
1243 ###########################
1244 # find detector from definition name
1245 def findDetector(definition):
1246  tokens = definition.name.split("_")
1247  for t in tokens:
1248  if t in ["nd","neardet"]:
1249  return "Near"
1250  return "Far"
1251 ###########################
1252 # is a file simulated based on it's definition name
1253 def isSimulated(definition):
1254  tokens = definition.name.split("_")
1255  for t in tokens:
1256  if t in ["sim","genie","cry","fhc","rhc"]:
1257  return "MC"
1258  return "Data"
1259 ###########################
1260 # find trigger from definition name
1261 def findTrigger(definition):
1262  tokens = definition.name.split("_")
1263  for t in tokens:
1264  if t in ["numi","t00","genie","crygenie"]:
1265  return "NuMI"
1266  if t in ["cosmic","t02","cry","cosmics"]:
1267  return "Cosmic"
1268  if t in ["DDActivity1"]:
1269  return "DD-Activity"
1270  if t in ["DDCalMu"]:
1271  return "DD-CalMu"
1272  # if we haven't returned by now try some desperate measures:
1273  for t in tokens:
1274  if t in ["fhc","rhc"]: return "NuMI"
1275  # assert False,"Cannot discern trigger type for definition name: %s"%definition
1276  return False
1277 ###########################
1278 # find tier from definition name
1279 def findTier(definition):
1280  tokens = definition.name.split("_")
1281  if tokens[1][0] == "S": return False
1282  if "mrcc" in tokens[1]: return tokens[1][4:]
1283  if "mre" in tokens[1]: return tokens[1][3:]
1284  try:
1285  if tokens[2]+"_"+tokens[3] == "contain_decaf": return "decaf"
1286  except:
1287  pass
1288  return tokens[1]
1289 ###########################
1290 # find release from definition name
1291 def findRelease(definition):
1292  i = 2
1293  if ("contain_decaf" in definition.name) or ("with_lempart" in definition.name): i=4
1294  tokens = definition.name.split("_")
1295  if (tokens[i][0] != "S") and (tokens[i][:2] != "FA"): return "unknown"
1296  if tokens[i][-3:] == "-nd": return tokens[i][:-3]
1297  return tokens[i]
1298 ###########################
1299 # find MRCC from definition name
1300 def findMRCC(definition):
1301  if "mrcc" in definition.name: return "mrcc"
1302  if "mre" in definition.name: return "mre"
1303  return "none"
1304 ###########################
1305 # Make a badge with a popup over
1306 def makeBadge(this_class,text,icon):
1307  html = '<span class="badge %s mypopover" data-toggle="tooltip" data-original-title="%s" rel="tooltip"><span class="glyphicon glyphicon-%s"></span></span>'
1308  return html%(this_class,text,icon)
def samDefinitionSummary(input_def)
Get the sam definition summary.
keys
Reco plots.
Definition: caf_analysis.py:46
def samDefinitionDescriptionsModal(def_descriptions)
SAM definition descriptions to modals (for fun)
Definition: ViewHTML.py:746
def countFilesInDirectory(directory, wildcard="*")
count the number of files in a directory
def stringToHTML(string)
Convert a string to html.
Definition: ViewHTML.py:36
def getSAMProjectStatus(monitor_text, skip=False)
determine SAM project status from submission log
def samProjectSummary(config)
convert the sam project parameters to a UL
Definition: ViewHTML.py:903
def samProjectParametersToUL(parameters)
convert the sam project parameters to a UL
Definition: ViewHTML.py:894
def makeBadge(this_class, text, icon)
Make a badge with a popup over.
Definition: ViewHTML.py:1306
def metricsToHTMLTable(chain, skip_unused_tiers=False, fcl_links=True, log_links=True, metric_links=True)
Display metrics as an HTML table.
Definition: ViewHTML.py:70
def findTier(definition)
find tier from definition name
Definition: ViewHTML.py:1279
def footer()
footer
Definition: ViewHTML.py:304
def assumptionsToTable(ass, eff, n_per_file)
Tier assumptions to table.
Definition: ViewHTML.py:614
def rreplace(s, old, new, occurrence)
recursive replace based on occurance from: http://stackoverflow.com/questions/2556108/how-to-replace-...
Definition: ViewHTML.py:64
::xsd::cxx::tree::type type
Definition: Database.h:110
def collectEnvironment(environ, time_str)
collect environment
Definition: ViewHTML.py:320
def tierConfigurationsTOC(chains)
TierConfigurations TOC.
Definition: ViewHTML.py:516
def getOutputCloneStatus(config, expected_n_files, verbose=False)
Check the status of a sam project being migrated to SAM.
Definition: ViewHTML.py:1118
def findTrigger(definition)
find trigger from definition name
Definition: ViewHTML.py:1261
def showMetricsSummary(chain, skip_unused_tiers=True, output_name="summary.html")
print metrics summary
Definition: ViewHTML.py:53
def isSimulated(definition)
is a file simulated based on it&#39;s definition name
Definition: ViewHTML.py:1253
def findDetector(definition)
find detector from definition name
Definition: ViewHTML.py:1245
def samFileInfoSummaryTable(definitions, names=False, configs=False, fcls=False, status=False)
samFileInfoSummaryTable
Definition: ViewHTML.py:665
def convertSize(size)
Parse file sizes into human readable format.
def parseInfo(info, folder)
parse info
Definition: ViewHTML.py:341
def findRelease(definition)
find release from definition name
Definition: ViewHTML.py:1291
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def definitionListToExplorerData(definitions, filter_on_mrcc=False, skip_counts=False)
Turn a list of definitions into a list of JS objects for the explorer JS.
Definition: ViewHTML.py:1165
def getStationMonitorLink(log)
get the link to a station monitor from a log
def masthead(text, subtext="")
print a masthead
Definition: ViewHTML.py:283
procfile open("FD_BRL_v0.txt")
static float min(const float a, const float b, const float c)
Definition: absgeo.cxx:45
def directoryToTable(wildcard, base_url, status=False, sam_status=False, to_skip=False, verbose=False)
Scan a directory and return a table.
Definition: ViewHTML.py:781
def foldersToButtonGroups(folder_list, output, url, page="index.html", messages={})
convert a folder list to a html button group
Definition: ViewHTML.py:438
def makeAlphanumeric(input)
convert a string into a purely alphanumeric one adapted from: http://stackoverflow.com/questions/1276764/stripping-everything-but-alphanumeric-chars-from-a-string-in-python
TDirectory * dir
Definition: macro.C:5
def getOutputDSStatus(config, expected_n_files, verbose=False)
Check the status of a sam project being migrated to SAM.
Definition: ViewHTML.py:1100
def tierConfigurationsTables(chains)
TierConfigurations table.
Definition: ViewHTML.py:557
def definitionListToExplorerDivs(definitions)
Turn a list of definitions into a list of divs for the explorer JS.
Definition: ViewHTML.py:1154
def samProjectCollectionToTable(collection, project_name, check_n_files=False, reduced_files=False, skip_projects=False, verbose=False)
convert the sam project collection to a table
Definition: ViewHTML.py:929
def flatDirectorySize(dir_path)
Get the size of a FLAT directory in bytes Sauce: http://stackoverflow.com/questions/1392413/calculati...
def samStatusToHTML(sam_status, processes=0)
Convert a sam status code to HTML.
Definition: ViewHTML.py:1084
def findMRCC(definition)
find MRCC from definition name
Definition: ViewHTML.py:1300
def samDefinitionAlert(input_def, guess_child=False)
Convert a SAM input definition into an alert.
Definition: ViewHTML.py:1042
def showMetric(metric, output_name="metric.html")
print metric
Definition: ViewHTML.py:42
def makeSAMQueryStatusBadge(status, time=0., date="???")
A dictionary of sam query status badges.
Definition: ViewHTML.py:1140