Functions
PageMakers Namespace Reference

Functions

def makeFrontPage (options, status={})
 Make the testing front page. More...
 
def makeValidationPages (options)
 Make the validation pages. More...
 
def makeValidationPage (options, controller_path, validation, version)
 Make an individual validation page. More...
 
def makeComponentTypePage (options, parent_dir, view, component_type, validation_name, version_tag)
 Make a component page. More...
 
def makeComponentSubmissionPage (options, parent_dir, component, validation_name, version_tag)
 Make a component submission log page. More...
 
def makeComponentViewPages (options, view, url, validation_name, version_tag)
 Make a component view page. More...
 
def makeComponentLogPages (options, parent_dir, component, validation_name, version_tag)
 Make a component submission log page. More...
 
def makeControllerConfigPage (options, controller, parent_dir, validation_name, version_tag)
 Make a component submission log page. More...
 
def makeFrontPage (options)
 Make the testing front page. More...
 
def makeResultsIndexPage (options, messages={})
 Results page. More...
 
def makeConfigPage (options)
 Configuration page. More...
 
def makeResultsPage (options, return_pickles=False)
 Results page. More...
 
def makeMetricPage (chain, tier, options, folder)
 Metrics page. More...
 
def makeLogPage (chain, tier, options, folder)
 Log page. More...
 
def makeFileSizePage (chain, tier, options, folder)
 FileSize page. More...
 
def makeTimingPage (chain, tier, options, folder)
 FileSize page. More...
 
def makeFHiCLPage (chain, tier, options, folder)
 FHiCL page. More...
 
def makeProjectionsIndexPage (options, messages={})
 Projections index page. More...
 
def makeProjectionsPage (options, unpickled_chains={})
 Projections page. More...
 
def makeBatchLogPages (info, options, folder)
 Batch log pages. More...
 
def makeLogBookPage (options)
 Log book page. More...
 

Function Documentation

def PageMakers.makeBatchLogPages (   info,
  options,
  folder 
)

Batch log pages.

Definition at line 938 of file PageMakers.py.

References open().

Referenced by makeResultsPage().

938 def makeBatchLogPages(info,options,folder):
939  if options.verbose: print "PMS : --- Making batch log pages"
940  statuses = {}
941  ########################### Imports
942  import PageMaker as PM
943  import ViewSTDOUT as stdout_view
944  import ProductionTestTools as tools
945  import os
946  import getpass
947  import glob
948  import subprocess
949  from datetime import datetime
950  ########################### Loop over chains
951  logs = info.logs
952  commands = info.commands
953  alt_log_dir = "%s/production_log_archive/"%os.environ["NOVAPRODVALID_GRID"]
954  #commands.sort()
955  for i, command in enumerate(commands):
956  print "PMS : command: %s"%command
957  print logs[i]
958  chain, chain_name = tools.findChainNameFromCommand(command)
959  # jobsub tools logs
960  log_base = False
961  for l in logs[i]:
962  print l
963  if (".cmd" in l) and ("condor-tmp" in l):
964  log_base = l[:-3]
965  # jobsub client logs
966  this_status = []
967  if not log_base:
968  jobsub_client_id = tools.getJobID(logs[i])
969  print "PMS : jobsub client job id: %s"%jobsub_client_id
970  if not jobsub_client_id:
971  # assert False, "PMS : ERROR no jobsub client ID found"
972  print "PMS : ERROR no jobsub client ID found"
973  this_status.append(7)
974  statuses[chain_name] = this_status
975  continue
976  client_log_dir = "%s%s"%(alt_log_dir,jobsub_client_id)
977  print "PMS : client log dir: %s"%client_log_dir
978  fetch_logs = False
979  if os.path.exists(client_log_dir):
980  print "PMS : - log directory exists"
981  folder_datetime = tools.datetimeFromFolder(folder)
982  now = datetime.now()
983  d_t = datetime.now() - folder_datetime
984  print "PMS : folder datetime: %s, now: %s, days: %i"%(folder_datetime,now,d_t.days)
985 
986  contains_logs = False
987  for l in glob.glob("%s/*"%client_log_dir):
988  if (".cmd" in l): contains_logs = True
989 
990  if (d_t.days < 5) or (not contains_logs):
991  print "PMS : re-fetching logs"
992  rm_command = "rm %s/*"%client_log_dir
993  os.system(rm_command)
994  fetch_logs = True
995  else:
996  print "PMS : using existing logs"
997  else:
998  print "PMS : - making new directory to fetch logs into"
999  # mkdir_command = "mkdir %s"%client_log_dir
1000  # os.system(mkdir_command)
1001  os.mkdir(client_log_dir)
1002  assert os.path.exists(client_log_dir), "PMS : mkdir for log files failed"
1003  fetch_logs = True
1004  if fetch_logs:
1005  print "PMS : - fetching logs"
1006  fetch_command = "jobsub_fetchlog -G nova --job=%s --unzipdir=%s"%(jobsub_client_id,client_log_dir)
1007  if "novapro" in getpass.getuser(): fetch_command+=" --role=Production"
1008  print "PMS : fetch log command to be run: %s"%fetch_command
1009  # os.system(fetch_command)
1010  subprocess.check_call(fetch_command.split())
1011  file_list = glob.glob("%s/*"%client_log_dir)
1012  for l in file_list:
1013  if (".cmd" in l): log_base = l[:-3]
1014  if not log_base:
1015  assert False, "PMS : ERROR no log base provided"
1016  # print "PMS : ERROR no log base provided"
1017  # statuses[chain_name] = this_status
1018  # this_status.append(4)
1019  # continue
1020  log_types = ["cmd","log","out","err"]
1021 
1022  for log in log_types:
1023  ########################### Base HTML
1024  batch_log_base_html = \
1025  """
1026  <div class="page-header">
1027  <h1><!--NAME_HOOK-->
1028  </h1>
1029  </div>
1030  """
1031  ########################### Insert content
1032  name="testing/%s/batch_logs/%s_%s.html"%(folder, chain, log)
1033  page = PM.PageMaker(name,batch_log_base_html, options)
1034  nav_html = '<li><a href="http://nusoft.fnal.gov/nova/production/testing/%s/index.html">%s</a></li>'%(folder,folder)
1035  page.insertContent("<!--NAVBAR_HOOK-->", nav_html)
1036  page.insertContent("<!--NAME_HOOK-->",
1037  "%s %s %s"%(folder,
1038  chain,
1039  log))
1040  if options.verbose: print "PMS : Finding log file: %s*%s"%(log_base[:-1],log)
1041  log_files = glob.glob("%s*%s"%(log_base[:-1],log))
1042  if len(log_files) == 1:
1043  log_path = log_files[0]
1044  if options.verbose: print "PMS : - using file: %s"%log_path
1045  else:
1046  print "PMS : Found not one log file: ",log_files
1047  log_path = "fake path"
1048  try:
1049  this_log = open(log_path,"rb").read()
1050  except:
1051  print "PMS : log file: %s can't be found"%(log_path)
1052  this_log = ""
1053  page.insertContent("<!--BODY_HOOK-->",
1054  "<pre>%s</pre>"%this_log)
1055  if log == "log": this_status.append(tools.batchLogStatus(this_log))
1056  if log == "err":
1057  if (this_log != ""):
1058  err_log_status = tools.parseErrorLog(this_log)
1059  if err_log_status != None:
1060  this_status.append(err_log_status)
1061  ########################## Write
1062  page.write()
1063  statuses[chain_name] = this_status
1064  return statuses
procfile open("FD_BRL_v0.txt")
def makeBatchLogPages(info, options, folder)
Batch log pages.
Definition: PageMakers.py:938
def PageMakers.makeComponentLogPages (   options,
  parent_dir,
  component,
  validation_name,
  version_tag 
)

Make a component submission log page.

Definition at line 333 of file PageMakers.py.

References naming_schema.componentName(), and view_tools.toHTML().

Referenced by makeValidationPage().

333 def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag):
334  ########################### Imports
335  import PageMaker as PM
336  import common_tools as tools
337  import view_tools
338  import os
339  from BatchLog import BatchLog
340  import naming_schema
341  ########################### Insert content
342  if options.testing:
343  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
344  else:
345  base_dir = os.environ["NOVAANAVALID_WEB"]
346  # get logs
347  job_sub_id = tools.getJobID(component.lines)
348  component.logs = BatchLog(tools.fetchLogs(job_sub_id, component.run_time))
349  if job_sub_id == False: return
350 
351  log_types = ["cmd","log","out","err"]
352  for log_type in log_types:
353  url = tools.directoryToURL(parent_dir)
354  name = "%s_%s_%s_batch_%s.html"%(component.name, component.version, component.dataset, log_type)
355  page = PM.PageMaker(name, parent_dir, url)
356  breadcrumb = \
357  """
358  <li><a href="%s/index.html">Validation</a></li>
359  <li><a href="%s/index.html">%s %s</a></li>
360  <li class="active">%s %s %s batch %s</li>
361  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
362  component.name, component.version, component.label, log_type)
363  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
364 
365  if hasattr(component.logs, log_type): content = getattr(component.logs, log_type)
366  else: content = [""]
367 
368  html = \
369  """
370  <div class="col-lg-12">
371  <h2 class="page-header">%s: <small>%s, batch %s</small></h1>
372  <pre>%s</pre>
373  </div>
374  """%(naming_schema.componentName(component.name), component.label, log_type, view_tools.toHTML("".join(content)))
375  page.addContent("<!--BODY_HOOK-->",html)
376  ########################### Write
377  page.write()
def toHTML(text)
convert some text to safe HTML
Definition: view_tools.py:69
def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:333
def componentName(name)
get a component name
def PageMakers.makeComponentSubmissionPage (   options,
  parent_dir,
  component,
  validation_name,
  version_tag 
)

Make a component submission log page.

Definition at line 273 of file PageMakers.py.

References naming_schema.componentName(), and view_tools.toHTML().

Referenced by makeValidationPage().

273 def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag):
274  ########################### Imports
275  import PageMaker as PM
276  import common_tools as tools
277  import view_tools
278  import os
279  import naming_schema
280  ########################### Insert content
281  url = tools.directoryToURL(parent_dir)
282  name = "%s_%s_%s_submission_log.html"%(component.name, component.version, component.dataset)
283  page = PM.PageMaker(name, parent_dir, url)
284  if options.testing:
285  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
286  else:
287  base_dir = os.environ["NOVAANAVALID_WEB"]
288  breadcrumb = \
289  """
290  <li><a href="%s/index.html">Validation</a></li>
291  <li><a href="%s/index.html">%s %s</a></li>
292  <li class="active">%s %s %s submission log</li>
293  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
294  component.name, component.version, component.label)
295  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
296 
297  html = \
298  """
299  <div class="col-lg-12">
300  <h2 class="page-header">%s: <small>%s, submission log</small></h1>
301  <pre>%s</pre>
302  </div>
303  """%(naming_schema.componentName(component.name), component.label, view_tools.toHTML("".join(component.lines)))
304  page.addContent("<!--BODY_HOOK-->",html)
305  ########################### Write
306  page.write()
def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:273
def toHTML(text)
convert some text to safe HTML
Definition: view_tools.py:69
def componentName(name)
get a component name
def PageMakers.makeComponentTypePage (   options,
  parent_dir,
  view,
  component_type,
  validation_name,
  version_tag 
)

Make a component page.

Definition at line 228 of file PageMakers.py.

References naming_schema.componentName().

Referenced by makeValidationPage().

228 def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag):
229  ########################### Imports
230  import PageMaker as PM
231  import common_tools as tools
232  import view_tools
233  import os
234  import naming_schema
235  ########################### Insert content
236  parent_url = tools.directoryToURL(parent_dir)
237  output_dir = parent_dir+"/%s"%component_type
238  url = tools.directoryToURL(output_dir)
239  name = "index.html"
240  page = PM.PageMaker(name, output_dir, url)
241  if options.testing:
242  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
243  else:
244  base_dir = os.environ["NOVAANAVALID_WEB"]
245  breadcrumb = \
246  """
247  <li><a href="%s/index.html">Validation</a></li>
248  <li><a href="%s/index.html">%s %s</a></li>
249  <li class="active">%s</li>
250  """%(tools.directoryToURL(base_dir), parent_url, validation_name, version_tag, \
251  naming_schema.componentName(component_type))
252  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
253 
254  html = \
255  """
256  <div class="col-lg-12">
257  <h1 class="page-header">%s</h1>
258  <p>The following plot groups have been identified:
259  <ul>
260  """%(naming_schema.componentName(component_type))
261 
262  # view.find_complete_components(component_type)
263  # view.group_components()
264  if component_type in view.plot_groups.keys():
265  for group in view.plot_groups[component_type]:
266  html += '<li><a href="%s/%s">%s</a></li>\n'%(url,group.replace(" ","_")+".html",group)
267  html += "</ul></p><hr></div>"
268  page.addContent("<!--BODY_HOOK-->",html)
269  ########################### Write
270  page.write()
def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag)
Make a component page.
Definition: PageMakers.py:228
def componentName(name)
get a component name
def PageMakers.makeComponentViewPages (   options,
  view,
  url,
  validation_name,
  version_tag 
)

Make a component view page.

Definition at line 309 of file PageMakers.py.

Referenced by makeValidationPage().

309 def makeComponentViewPages(options, view, url, validation_name, version_tag):
310  ########################### Imports
311  from HTMLView import HTMLView
312  from Canvas import Canvas
313  import os
314  import common_tools as tools
315  ########################### Insert content
316  canvas = Canvas()
317  view.component_view(canvas)
318  if options.testing:
319  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
320  else:
321  base_dir = os.environ["NOVAANAVALID_WEB"]
322 
323  breadcrumb = \
324  """
325  <li><a href="%s/index.html">Validation</a></li>
326  <li><a href="%s/index.html">%s %s</a></li>
327  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
328 
329  html_view = HTMLView(view, breadcrumb=breadcrumb, reuse_histograms=options.reuse, verbose=options.verbose, testing=options.testing)
330  html_view.write_component_html()
Definition: Canvas.py:1
def makeComponentViewPages(options, view, url, validation_name, version_tag)
Make a component view page.
Definition: PageMakers.py:309
def PageMakers.makeConfigPage (   options)

Configuration page.

Definition at line 181 of file PageMakers.py.

181 def makeConfigPage(options):
182  ########################### Imports
183  import PageMaker as PM
184  import TierConfigurations as TC
185  import ViewHTML as html_view
186  import subprocess
187  ########################### Base HTML
188  config_base_html = \
189  """
190  <div class="page-header">
191  <h1>Configuration overview
192  </h1>
193  </div>
194 
195  <p>
196  This page summarises the tier configurations used to run the
197  production tests. This page was generated using
198  TierConfigurations.py revision:
199  <strong><!--REVISION_HOOK--></strong>. The user should note
200  that this is not necessarily the same configuration as was
201  used to run the individual tests, that information is
202  displayed on the appropriate results page along with details
203  of the NOvA release used.
204  </p>
205  <p>
206  The configured chains are:
207  </p>
208  """
209  ########################### Insert content
210  page = PM.PageMaker("config.html",config_base_html, options)
211  # revision
212  a = subprocess.check_output(["svn","info","%s/../TierConfigurations.py"%options.path])
213  t = a.split("\n")
214  rev = "UNKNOWN"
215  for tk in t:
216  words = tk.split(": ")
217  if len(words) != 2: continue
218  if words[0] == "Revision":
219  rev = '<a href="https://cdcvs.fnal.gov/redmine/projects/novaart/repository/revisions/%s/entry/trunk/Validation/production/testing/TierConfigurations.py">%s</a>'%\
220  (words[1], words[1])
221  # check for local revisions
222  if subprocess.check_output(["svn","status","%sTierConfigurations.py"%options.path]) != "":
223  rev += " - WARNING, local revisions detected"
224  page.insertContent("<!--REVISION_HOOK-->",rev)
225  # contents
226  toc = html_view.tierConfigurationsTOC(TC.chains)
227  page.insertContent("<!--BODY_HOOK-->",
228  "%s\n<!--BODY_HOOK-->"%toc)
229  c_html = html_view.tierConfigurationsTables(TC.chains)
230  page.insertContent("<!--BODY_HOOK-->",
231  "%s\n<!--BODY_HOOK-->"%c_html)
232  ########################### Write
233  page.write()
def makeConfigPage(options)
Configuration page.
Definition: PageMakers.py:181
def PageMakers.makeControllerConfigPage (   options,
  controller,
  parent_dir,
  validation_name,
  version_tag 
)

Make a component submission log page.

Definition at line 380 of file PageMakers.py.

Referenced by makeValidationPage().

380 def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag):
381  ########################### Imports
382  import PageMaker as PM
383  import common_tools as tools
384  import os
385  ########################### Insert content
386  url = tools.directoryToURL(parent_dir)
387  name = "config.html"
388  page = PM.PageMaker(name, parent_dir, url)
389  if options.testing:
390  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
391  else:
392  base_dir = os.environ["NOVAANAVALID_WEB"]
393  breadcrumb = \
394  """
395  <li><a href="%s/index.html">Validation</a></li>
396  <li><a href="%s/index.html">%s %s</a></li>
397  <li class="active">configuration</li>
398  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
399  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
400 
401  config = repr(controller.config)
402  config = config.replace("u'","'")
403  config = config.replace(',',',\n')
404 
405  html = \
406  """
407  <div class="col-lg-12">
408  <h2 class="page-header">Configuration</h1>
409  <pre>%s</pre>
410  </div>
411  """%(config)
412  page.addContent("<!--BODY_HOOK-->",html)
413  ########################### Write
414  page.write()
415 
416 
def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:380
def PageMakers.makeFHiCLPage (   chain,
  tier,
  options,
  folder 
)

FHiCL page.

Definition at line 724 of file PageMakers.py.

Referenced by makeResultsPage().

724 def makeFHiCLPage(chain,tier,options,folder):
725  ########################### Imports
726  import PageMaker as PM
727  ########################### Base HTML
728  fcl_base_html = \
729  """
730  <div class="page-header">
731  <h1><!--NAME_HOOK--> FHiCL
732  </h1>
733  </div>
734  """
735  ########################### Insert content
736  cwd = "%s%s"%(options.output,folder)
737  name="testing/%s/fcls/%s_%s.html"%(folder, chain.name, tier.short_name)
738  page = PM.PageMaker(name,fcl_base_html, options)
739  nav_html = '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html">%s</a></li>'%(folder,folder)
740  nav_html += '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html#%s_%s">%s_%s</a></li>'%\
741  (folder, chain.name, tier.short_name, chain.name, tier.short_name)
742  page.insertContent("<!--NAVBAR_HOOK-->", nav_html)
743  page.insertContent("<!--NAME_HOOK-->",
744  "%s %s %s"%(folder,
745  chain.name.replace("_"," "),
746  tier.short_name.replace("_"," ")))
747 
748  page.insertContent("<!--BODY_HOOK-->",
749  '<pre class="prettyprint linenums">\n%s</pre>'%tier.full_fcl)
750  page.insertContent("<!--JS_HOOK-->",
751  '<script src="https://google-code-prettify.googlecode.com/svn/loader/run_prettify.js"></script>')
752  #page.insertContent("<!--BODY_HOOK-->",
753  #'<pre>\n%s</pre>'%tier.full_fcl)
754  ########################### Write
755  page.write()
def makeFHiCLPage(chain, tier, options, folder)
FHiCL page.
Definition: PageMakers.py:724
def PageMakers.makeFileSizePage (   chain,
  tier,
  options,
  folder 
)

FileSize page.

Definition at line 482 of file PageMakers.py.

References open().

Referenced by makeResultsPage().

482 def makeFileSizePage(chain,tier,options,folder):
483  ########################### Imports
484  import PageMaker as PM
485  import ViewSTDOUT as stdout_view
486  import ProductionTestTools as tools
487  ########################### Base HTML
488  file_size_base_html = \
489  """
490  <div class="page-header">
491  <h1><!--NAME_HOOK--> file size <!--TEST_HOOK-->
492  </h1>
493  </div>
494  """
495  ########################### Insert content
496  cwd = "%s%s"%(options.output,folder)
497  pretty_name = "%s %s"%(tools.releaseFromFolder(folder), tools.dateFromFolder(folder))
498 
499  metric = tier.metric
500  if metric.output_name == None:
501  return
502 
503  for i, output in enumerate(metric.output_name):
504  name="testing/%s/file_sizes/%s_%s_%s.html"%(folder, chain.name, tier.short_name,output.replace(".","_"))
505  page = PM.PageMaker(name,file_size_base_html, options)
506 
507  page.insertContent("<!--NAVBAR_HOOK-->",
508  '<li><a href="http://nusoft.fnal.gov/nova/production/testing/%s/index.html">%s</a></li>'%(folder,pretty_name))
509  page.insertContent("<!--NAME_HOOK-->",
510  "%s %s %s"%\
511  (chain.name.replace("_"," "),tier.short_name.replace("_"," "),output))
512  page.insertContent("<!--TEST_HOOK-->",
513  "<small>%s %s </small>"%\
514  (tools.releaseFromFolder(folder),tools.dateFromFolder(folder)))
515 
516  body_html = ""
517  try:
518  file_sizes = getattr(metric,"file_size_ana")
519  except:
520  file_sizes = None
521  pass
522 
523  if file_sizes != None:
524  assert len(file_sizes) == len(metric.output_name),"PMS : length of file size vector: %i != length of output_names: %i"%(len(file_sizes),len(metric.output_names))
525  file_size = file_sizes[i]
526  container_name = "container_%s"%(metric.output_name[i].replace(".","_"))
527  body_html += \
528  """
529  <form>
530  <label><input type="radio" name="mode" value="sunburst" checked> Sunburst</label>
531  <label><input type="radio" name="mode" value="treemap"> Treemap</label>
532  </form>
533  <div id="outer">
534  <div id="%s"></div>
535  </div>
536  """%(container_name)
537  # if d3:
538  body_html += \
539  """
540  <script src="http://d3js.org/d3.v3.min.js"></script>
541  <script src="http://labratrevenge.com/d3-tip/javascripts/d3.tip.v0.6.3.js"></script>
542  """
543  the_json = file_size[0]
544  file_name = "file_sizes/%s_%s_%s.json"%(chain.name, tier.short_name, metric.output_name[i])
545  to_write = open("%s/%s"%(cwd,file_name),"w")
546  to_write.write(the_json)
547  to_write.close()
548 
549  page.insertContent("<!--STYLE_HOOK-->",
550  """
551  path {
552  stroke: #fff;
553  fill-rule: evenodd;
554  }
555 
556  div.tooltip {
557  position: absolute;
558  text-align: center;
559  color: #fff;
560  padding: 10px;
561  font: 14px sans-serif;
562  background: rgba(0, 0, 0, 0.8);
563  border: 5px;
564  border-radius: 8px;
565  pointer-events: none;
566  }
567  rect {
568  stroke-width:3;
569  stroke:rgb(255,255,255);
570  }
571  """
572  )
573  file_size_ana_base = open(options.path+"file_size_ana_base.html","r").read()
574  file_size_ana_base = file_size_ana_base.replace("FILENAME","http://nusoft.fnal.gov/nova/production/testing/%s/%s"%(folder,file_name))
575  # get the number of output events for this tier
576  n_events = 1
577  if metric.output_events[i] != None:
578  n_events = metric.output_events[i]
579  file_size_ana_base = file_size_ana_base.replace("NEVENTS","%i"%n_events)
580  file_size_ana_base = file_size_ana_base.replace("CONTAINERNAME",container_name)
581 
582  body_html += file_size_ana_base
583  # else:
584  # # if we don't want to display the d3, just input the pngs
585  # body_html += ' <div class="row">\n'
586  # body_html += ' <div class="col-md-6">\n'
587  # radial_png = file_size[1]
588  # file_name = "file_sizes/%s_%s_%s_radial.png"%(chain.name, tier.short_name, metric.output_name[i])
589  # to_write = open("%s/%s"%(cwd,file_name),"w")
590  # to_write.write(radial_png)
591  # to_write.close()
592  # body_html += ' <img src="http://nusoft.fnal.gov/nova/production/testing/%s/%s" class="img-responsive" alt="%s" />\n'%(folder,file_name,file_name)
593  # body_html += " </div>"
594 
595  # body_html += ' <div class="col-md-6">'
596  # bar_png = file_size[2]
597  # file_name = "file_sizes/%s_%s_%s_bar.png"%(chain.name, tier.short_name, metric.output_name[i])
598  # to_write = open("%s/%s"%(cwd,file_name),"w")
599  # to_write.write(bar_png)
600  # to_write.close()
601  # body_html += ' <img src="http://nusoft.fnal.gov/nova/production/testing/%s/%s" class="img-responsive" alt="%s" />\n'%(folder,file_name,file_name)
602  # body_html += " </div>"
603  # body_html += " </div>"
604  # body_html += "<hr>"
605  page.insertContent("<!--BODY_HOOK-->",
606  body_html)
607  ########################### Write
608  page.write()
procfile open("FD_BRL_v0.txt")
def makeFileSizePage(chain, tier, options, folder)
FileSize page.
Definition: PageMakers.py:482
def PageMakers.makeFrontPage (   options)

Make the testing front page.

Definition at line 3 of file PageMakers.py.

References makeFrontPage().

3 def makeFrontPage(options):
4  ########################### Imports
5  import PageMaker as PM
6  import TierConfigurations as TC
7  import os
8  ########################### Base HTML
9  url = "http://nusoft.fnal.gov/nova/production/testing/"
10  if options.local: url = options.output
11  front_base_html = \
12  """
13  <div class="page-header">
14  <h1>Production test results
15  <small>For all your test based needs.</small>
16  </h1>
17  </div>
18 
19  <div class="alert alert-info bs-alert-old-docs">
20  <strong>Heads up!</strong> This is a beta version.
21  </div>
22 
23  <div class="page-header">
24  <h2>Overview</h2>
25  </div>
26  <p>
27  This page serves as a gateway to the results of the production
28  testing framework. It aims to provide the experiment with an
29  overview of how a given NOvA software release performs in terms
30  of a number of simple metrics.
31  </p>
32  <p>
33  This site indexes the contents of
34  <strong><!--PATH_HOOK--></strong>. Any folders that contain
35  test results are then
36  <a href="%sresults.html">displayed</a>. The plan is to
37  eventually have tests be run (semi-)automatically. This page
38  itself is generated using a cron job. In case of any issues
39  contact Matthew Tamsett.
40  </p>
41  <div class="page-header">
42  <h2>Testing methods</h2>
43  </div>
44  <p>
45  NOvA data processing can be divided into a number of logical
46  chains based on the type of data being handled. The currently
47  <a href="%sconfig.html">configured</a> chains are:
48  </p>
49  <ul>
50  <!--CHAINS_HOOK-->
51  </ul>
52  <p>
53  Each of these chains is subdivided into a number of tiers. Each
54  tier is a single NOvA art job configured using a single FHiCL
55  file. The available chains and tiers are defined in
56  <a href="https://cdcvs.fnal.gov/redmine/projects/novaart/repository/entry/trunk/Validation/production/TierConfigurations.py">
57  TierConfigurations.py</a>. Tier configurations are used so that the
58  execution of multiple tiers can be processed inside a single
59  loop. Tier-to-tier differences are small and can be configured
60  based on a small number of options. The currently configured
61  tiers are:
62  </p>
63  <ul>
64  <!--TIERS_HOOK-->
65  </ul>
66  <p>
67  Tests are run on the requested chain and tiers using a
68  python wrapper program to spawn NOvA ART jobs using the sub-
69  process module. The performance of jobs is monitored during
70  their run-time using the psutil module to periodically
71  record metrics such as CPU and memory usage. Simultaneously
72  the STDOUT and STDERR generated by the job is captured and
73  interlaced with time stamps. After the completion of the job
74  this log is then parsed to extract information about the ART
75  process, such as when each event began processing and when
76  calls were made to the database.
77  </p>
78  <p>
79  The resultant metrics and logs are stored in a pickle file
80  by the python job. After the fact these metrics are
81  visualised and formatted for web display using pickle,
82  HTML parsing, Google charts and Bokeh.
83  </p>
84  """%(url,url)
85  ########################### Insert content
86  page = PM.PageMaker("index.html",front_base_html, options)
87  # environment
88  folder_location = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
89  if options.local: folder_location = options.output
90  page.insertContent("<!--PATH_HOOK-->",
91  "%s:%s"%(os.uname()[1],folder_location))
92  # Currently configured chains and tiers
93  chains = TC.chains
94  chain_names = []
95  for chain in TC.chains.keys():
96  chain_names.append(chain.replace("_"," "))
97  chain_names.sort()
98  for name in chain_names:
99  page.insertContent("<!--CHAINS_HOOK-->",
100  '<li><a href="%sconfig.html#%s">%s</a></li>\n<!--CHAINS_HOOK-->'%\
101  (url,name.replace(" ","_"),name))
102  chain = chains[name.replace(" ","_")]
103  tiers = chain.GetTierShortNames()
104  page.insertContent("<!--TIERS_HOOK-->",
105  '<li><strong>%s</strong>: <!--TIERS_HOOK-->'%name)
106  for tier in tiers:
107  page.insertContent("<!--TIERS_HOOK-->",
108  '<a href="%sconfig.html#%s_%s">%s</a>, <!--TIERS_HOOK-->'%\
109  (url,name.replace(" ","_"),tier,tier.replace("_"," ")))
110  page.insertContent(", <!--TIERS_HOOK-->",
111  "</li>\n<!--TIERS_HOOK-->",destructive=True)
112  ########################### Write
113  page.write()
def makeFrontPage(options, status={})
Make the testing front page.
Definition: PageMakers.py:7
def PageMakers.makeFrontPage (   options,
  status = {} 
)

Make the testing front page.

Definition at line 7 of file PageMakers.py.

References view_tools.classToGlyphicon(), caf_analysis.keys, parse_dependency_file_t.list, and plotROC.title.

Referenced by makeFrontPage().

7 def makeFrontPage(options, status={}):
8  ########################### Imports
9  import PageMaker as PM
10  import os
11  import common_tools as tools
12  from glob import glob
13  import view_tools
14  ########################### Base HTML
15  if options.testing:
16  output_dir = os.environ["NOVAANAVALID_WEB_TEST"]
17  else:
18  output_dir = os.environ["NOVAANAVALID_WEB"]
19  url = tools.directoryToURL(output_dir)
20  page = PM.PageMaker("index.html", output_dir, url)
21  ########################### Insert content
22  breadcrumb = '<li class="active">Validation</li>'
23  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
24 
25  html = \
26  """
27  <div class="col-lg-12">
28  <h1 class="page-header">Analysis validation</h1>
29  """
30  # find all instances of validation
31  if options.testing:
32  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
33  else:
34  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
35  if options.verbose: print "PMS : found %i validations"%len(validations)
36  validations.sort()
37  # Make a list of contents based on these
38  for validation in validations:
39  validation_base = validation.split("/")[-1]
40  validation_name = validation_base.replace("_"," ").title()
41  html += "<h2>%s</h2><ul>"%validation_name
42  versions = glob("%s/*"%validation)
43  if options.verbose: print "PMS : Validation: %s, %i versions"%(validation_name, len(versions))
44  versions.sort()
45  for version in versions:
46  version = version.split("/")[-1]
47  html += "<li><a href=%s/%s>%s</a> "%(validation_base,version,version)
48  v_key = "%s_%s"%(validation_base, version)
49  if (v_key in status.keys()):
50  all_statuses = []
51  for component in status[v_key].keys():
52  for c in status[v_key][component]:
53  all_statuses.append(view_tools.classToGlyphicon(c))
54  all_statuses = " ".join(list(set(all_statuses)))
55  html += all_statuses
56  html += "</li>"
57 
58  html += "</ul><hr>"
59  html += "</div>"
60  page.addContent("<!--BODY_HOOK-->",html)
61  ########################### Write
62  page.write()
keys
Reco plots.
Definition: caf_analysis.py:46
def classToGlyphicon(class_label)
convert a class to a glyphicon
Definition: view_tools.py:152
def makeFrontPage(options, status={})
Make the testing front page.
Definition: PageMakers.py:7
def PageMakers.makeLogBookPage (   options)

Log book page.

Definition at line 1067 of file PageMakers.py.

1067 def makeLogBookPage(options):
1068  ########################### Imports
1069  import PageMaker as PM
1070  ########################### Base HTML
1071  url = "http://nusoft.fnal.gov/nova/production/testing/"
1072  if options.local: url = options.output
1073  base_html = \
1074  """
1075  <div class="page-header">
1076  <h1>Production log book</h1>
1077  </div>
1078 
1079  <p>
1080  This is the log book it should contain entries documenting what changes
1081  with the production testing framework over time.
1082  </p>
1083  <p>
1084  Eventually I'll expand this to include tags and searchable content,
1085  but for now it is just plain HTML.
1086  </p>
1087  <hr>
1088 
1089  <h2>01/05/15</h2>
1090  <p><span class="badge">development-300415</span><span class="badge">development-010515</span><span class="badge">S15-04-30</span></p>
1091  <p>
1092  All simulation jobs (ND, FD) x (genie, cosmics) are failing with an error of the form:
1093  <pre>Looks like a MC run number - assume perfect + any mask
1094 terminate called after throwing an instance of 'std::length_error'
1095  what(): vector::_M_default_append
1096 Aborted</pre>
1097  I've contacted simulation to inform them.
1098  </p>
1099  <hr>
1100 
1101  <h2>29/04/15</h2>
1102  <p><span class="badge">development-280415</span></p>
1103  <p>
1104  There was a typo in the hough vertex addition. Fixed in commit
1105  <a href="https://cdcvs.fnal.gov/redmine/projects/novaart/repository/revisions/14365">r14365</a>.
1106  The issue with the PC-hits tier failing is still present.
1107  I'll email the calibration list when today's tests are complete.
1108  </p>
1109  <hr>
1110 
1111  <h2>28/04/15</h2>
1112  <p><span class="badge">development-270415</span></p>
1113  <p>
1114  Yesterdays development tests for FD data NuMI, data cosmics and MC cosmics all
1115  failed. There are two observed pathologies:
1116  <ul>
1117  <li>
1118  The pclist tier is failing due to an issue in the PClist module in the stopper
1119  path. I think this will be related to changes made by Dan Pershey. I've
1120  emailed him.
1121  </li>
1122  <li>
1123  The pid and recopid tiers are failing due to hitting the test memory limit of
1124  2.4Gb. This is probably due to the switch to using v7 CSV files for calibration.
1125  Either the new method or the new constants are responsible. I've raised the limit
1126  to 2.9Gb for now to remedy this.
1127  </li>
1128  </ul>
1129  </p>
1130  <hr>
1131  """
1132  ########################### Insert content
1133  page = PM.PageMaker("log_book.html",base_html, options)
1134  ########################### Write
1135  page.write()
1136 
1137 
def makeLogBookPage(options)
Log book page.
Definition: PageMakers.py:1067
def PageMakers.makeLogPage (   chain,
  tier,
  options,
  folder 
)

Log page.

Definition at line 451 of file PageMakers.py.

Referenced by makeResultsPage().

451 def makeLogPage(chain,tier,options,folder):
452  ########################### Imports
453  import PageMaker as PM
454  import ViewSTDOUT as stdout_view
455  ########################### Base HTML
456  log_base_html = \
457  """
458  <div class="page-header">
459  <h1><!--NAME_HOOK--> log
460  </h1>
461  </div>
462  """
463  ########################### Insert content
464  cwd = "%s%s"%(options.output,folder)
465  name="testing/%s/logs/%s_%s.html"%(folder, chain.name, tier.short_name)
466  page = PM.PageMaker(name,log_base_html, options)
467  nav_html = '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html">%s</a></li>'%(folder,folder)
468  nav_html += '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html#%s_%s">%s_%s</a></li>'%\
469  (folder, chain.name, tier.short_name, chain.name, tier.short_name)
470  page.insertContent("<!--NAVBAR_HOOK-->", nav_html)
471  page.insertContent("<!--NAME_HOOK-->",
472  "%s %s %s"%(folder,
473  chain.name.replace("_"," "),
474  tier.short_name.replace("_"," ")))
475 
476  page.insertContent("<!--BODY_HOOK-->",
477  "<pre>%s</pre>"%stdout_view.stripTimingMarkersFromLog(tier.log))
478  ########################### Write
479  page.write()
def makeLogPage(chain, tier, options, folder)
Log page.
Definition: PageMakers.py:451
def PageMakers.makeMetricPage (   chain,
  tier,
  options,
  folder 
)

Metrics page.

Definition at line 386 of file PageMakers.py.

Referenced by makeResultsPage().

386 def makeMetricPage(chain,tier,options,folder):
387  ########################### Imports
388  import PageMaker as PM
389  import ViewBokeh as bokeh_view
390  import os
391  ########################### Base HTML
392  metric_base_html = \
393  """
394  <div class="page-header">
395  <h1>Metrics
396  </h1>
397  </div>
398  """
399  ########################### Attributes to plot
400  attributes = ["memory",
401  "memory_percent",
402  "vms_memory",
403  "cpu_percent",
404  "u_cpu_times",
405  #"s_cpu_times",
406  #"read_bytes",
407  #"write_bytes",
408  #"rss_memory", # should be redundant w/ memory
409  #"n_open_files",
410  #"n_connections",
411  ]
412  ########################### Insert content
413  cwd = "%s%s"%(options.output,folder)
414  name="testing/%s/metrics/%s_%s.html"%(folder, chain.name, tier.short_name)
415  page = PM.PageMaker(name,metric_base_html, options)
416  nav_html = '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html">%s</a></li>'%(folder,folder.replace("_"," "))
417  nav_html += '<li><a href="http://nusoft.fnal.gov/nova/production/%s/index.html#%s_%s">%s %s</a></li>'%\
418  (folder, chain.name, tier.short_name, chain.name.replace("_"," "), tier.short_name.replace("_"," "))
419  page.insertContent("<!--NAVBAR_HOOK-->", nav_html)
420 
421  intro = """\
422  <section>
423  <p>
424  These metrics correspond to the test: <strong>%s</strong>, the
425  chain: <strong>%s</strong> and the tier: <strong>%s</strong>.
426  </p>
427  </section>
428  """%(folder, chain.name.replace("_"," "),tier.short_name.replace("_"," "))
429  page.insertContent("<!--BODY_HOOK-->",
430  "%s\n<!--BODY_HOOK-->"%intro)
431  bokeh_path = "http://nusoft.fnal.gov/nova/production/static/bokeh_static/"
432  base_url = "http://nusoft.fnal.gov/nova/production/testing/%s/embed/"%folder
433  if options.local:
434  bokeh_path = "/Users/mt354/Analysis/web/static/bokeh_static/"
435  base_url = "%s/embed/"%cwd
436  if options.verbose: print "PMS : - making bokeh metrics"
437  this_html = bokeh_view.tierToHTML(tier, attributes,
438  title=chain.name.replace("_"," "),
439  bokeh_path=bokeh_path,
440  base_url =base_url,
441  verbose =options.verbose)
442  page.insertContent("<!--BODY_HOOK-->",
443  this_html)
444  ########################### Write
445  page.write()
446  # now copy the embed.js objects over
447  js_list = bokeh_view.findJSList(this_html)
448  for js in js_list: os.system("mv %s %s/embed/"%(js,cwd))
def makeMetricPage(chain, tier, options, folder)
Metrics page.
Definition: PageMakers.py:386
def PageMakers.makeProjectionsIndexPage (   options,
  messages = {} 
)

Projections index page.

Definition at line 758 of file PageMakers.py.

758 def makeProjectionsIndexPage(options, messages={}):
759  ########################### Imports
760  import PageMaker as PM
761  import ViewHTML as html_view
762  import os
763  ########################### Base HTML
764  results_base_html = \
765  """
766  <div class="page-header">
767  <h1>Projections
768  </h1>
769  </div>
770 
771  <p>
772  Searching for test results in
773  <strong><!--PATH_HOOK--></strong>. The following test
774  results have been found:
775  </p>
776  <!--BUTTON_HOOK-->
777  """
778  ########################### Insert content
779  page = PM.PageMaker("projections.html",results_base_html, options)
780  # List directory content
781  folder_location = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
782  if options.local: folder_location = options.output
783  url = "http://nusoft.fnal.gov/nova/production/testing/"
784  if options.local: url = options.output
785  page.insertContent("<!--PATH_HOOK-->",
786  "%s:%s"%(os.uname()[1],folder_location))
787  folder_list = os.listdir(folder_location)
788  button_html = html_view.foldersToButtonGroups(folder_list, options.output, url, page="projections.html", messages=messages)
789  page.insertContent("<!--BUTTON_HOOK-->", button_html)
790  ########################### Write
791  page.write()
def makeProjectionsIndexPage(options, messages={})
Projections index page.
Definition: PageMakers.py:758
def PageMakers.makeProjectionsPage (   options,
  unpickled_chains = {} 
)

Projections page.

Definition at line 794 of file PageMakers.py.

794 def makeProjectionsPage(options, unpickled_chains = {}):
795  ########################### Imports
796  import PageMaker as PM
797  import cPickle
798  import ViewHTML as html_view
799  import ViewGoogleCharts as google_view
800  import os
801  import ProductionTestTools as tools
802  import glob
803  import Assumptions
804  ########################### Base HTML
805  result_base_html = \
806  """
807  <div class="page-header">
808  <h1><!--FOLDER_HOOK-->
809  </h1>
810  </div>
811 
812  <p>
813  This page interprets the results of the production tests in terms
814  of the <a href="#per_event">measured running time per event</a>
815  and the <a href="#projection">projected running time per file and for
816  an ensemble of chains</a> representing a typical production group.
817  Finally we repeat this with the measured <a href="#sizes">sizes of
818  files</a>.
819  </p>
820  <p>
821  The <a href="index.html">results section</a> contains information on the
822  tests used to make these projections, their current status and
823  a detailed break down of their results. That page should be read before
824  interpreting any of the information below.
825  </p>
826  <p>
827  The times shown here and used to make projections are the integrated user
828  and system CPU time for a given tier divided by the number
829  of events run over. This averages any overheads, such as ART job
830  initialisation, first event instantiations, data-base calls and
831  job finalisation, over the course of the tier. In addition tiers are run
832  sequentially as part of the same job, this reduces overheads for later
833  tiers relative to running in isolation. As such these times are to be
834  taken with a pinch of salt. In order to be in any way representative
835  the test jobs must have been configured to run over a number of events
836  typical of a single job in the production running.
837  </p>
838  <p>
839  The graphs shown here are interactive. You can click on any tier on the
840  legend to toggle it on or off.
841  </p>
842  """
843  ########################### Get all the results pages
844  # List directory content
845  folder_location = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
846  if options.local: folder_location = options.output
847  folder_list = os.listdir(folder_location)
848  ########################### Make content
849  for folder in folder_list:
850  if options.wildcard and (options.wildcard not in folder): continue
851  cwd = "%s%s"%(options.output,folder)
852  if not os.path.isdir(cwd) : continue
853  if options.verbose: print "PMS : Making projections page for folder: %s"%cwd
854  tmp_file_list = glob.glob("%s/*.pkl"%cwd)
855  file_list = []
856  for f in tmp_file_list:
857  if "info.pkl" in f: continue
858  file_list.append(f)
859  ########################### Cacheing
860  # Two options for cacheing exist:
861  # 1. All chains for this test have existing metric pages. In this case we can exit now.
862  # 2. Some chains for this test have existing metric pages. In this case we have to remake the results page.
863  if options.full:
864  cached_chains = []
865  else:
866  cached_chains = tools.detectCaching(cwd)
867  if len(cached_chains) and (len(cached_chains) == len(file_list)):
868  if options.verbose: print "PMS : all chains have metrics pages already made for them, skipping this folder."
869  continue
870  ########################### Base page
871  page = PM.PageMaker("%s/projections.html"%folder,result_base_html, options)
872  if tools.isLogicalFolder(folder):
873  pretty_name = "%s %s"%(tools.releaseFromFolder(folder),tools.dateFromFolder(folder))
874  else:
875  pretty_name = folder.replace("_"," ")
876  page.insertContent("<!--NAVBAR_HOOK-->",
877  '<li><a href="http://nusoft.fnal.gov/nova/production/testing/%s/projections.html">%s</a></li>'%(folder,pretty_name))
878  if tools.isLogicalFolder(folder):
879  page.insertContent("<!--FOLDER_HOOK-->",
880  "%s projections <small>%s</small>"%(tools.releaseFromFolder(folder),tools.dateFromFolder(folder)))
881  else:
882  page.insertContent("<!--FOLDER_HOOK-->",
883  "%s"%pretty_name)
884  ########################### Get pickles
885  if (len(file_list) == 0):
886  print "PMS : No .pkl files found in found: %s"%cwd
887  page.insertContent("<!--BODY_HOOK-->",
888  "<strong>No pickle files found.</strong>")
889  page.write()
890  continue
891  ########################### Projections
892  head = '<script type="text/javascript" src="https://www.google.com/jsapi"></script>\n'
893  head_html, body_sections = google_view.directoryProjectionBarChart(folder, pretty_name, cwd, options.path, unpickled_chains = unpickled_chains, rounding=True)
894  head += head_html
895  body = """
896  <div class="page-header">
897  <h2 id="per_event">Per event times</h2>
898  </div>
899  %s
900  <hr>
901  <hr>
902  <div class="page-header">
903  <h2 id="projection">Projected times</h2>
904  </div>
905  <p>
906  The projections were made using the assumptions detailed in the table below.
907  Where a tier is not explicitly mentioned it follows the pattern of the default
908  tier of it's chain. The efficiencies quoted should reflect that measured in
909  the <a href="index.html">results section</a> of this test.
910  </p>
911  <!--ASSUMPTIONS_HOOK-->
912  <hr>
913  %s
914  <hr>
915  %s
916  <hr>
917  <div class="page-header">
918  <h2 id="sizes">Projected sizes</h2>
919  </div>
920  <p>
921  Similarly we project the size of the final output files based on the measured
922  size per output event.
923  </p>
924  %s
925  <hr>
926  %s
927  <hr>
928  %s
929  """%body_sections
930  table = html_view.assumptionsToTable(Assumptions.n_events, Assumptions.eff, Assumptions.events_per_file)
931  body = body.replace("<!--ASSUMPTIONS_HOOK-->",table)
932  page.insertContent("<!--BODY_HOOK-->",body)
933  page.insertContent("<!--HEADER_HOOK-->",head)
934  ########################### Write
935  page.write()
def makeProjectionsPage(options, unpickled_chains={})
Projections page.
Definition: PageMakers.py:794
def PageMakers.makeResultsIndexPage (   options,
  messages = {} 
)

Results page.

Definition at line 116 of file PageMakers.py.

116 def makeResultsIndexPage(options, messages={}):
117  ########################### Imports
118  import PageMaker as PM
119  import ViewHTML as html_view
120  import os
121  import ViewHighcharts as VHC
122  ########################### Base HTML
123  results_base_html = \
124  """
125  <div class="page-header">
126  <h1>Test results
127  </h1>
128  </div>
129 
130  <p>
131  Searching for test results in
132  <strong><!--PATH_HOOK--></strong>. The following test
133  results have been found:
134  </p>
135  <!--BUTTON_HOOK-->
136  """
137  ########################### Insert content
138  page = PM.PageMaker("results.html",results_base_html, options)
139  # List directory content
140  folder_location = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
141  if options.local: folder_location = options.output
142  url = "http://nusoft.fnal.gov/nova/production/testing/"
143  if options.local: url = options.output
144  page.insertContent("<!--PATH_HOOK-->",
145  "%s:%s"%(os.uname()[1],folder_location))
146  folder_list = os.listdir(folder_location)
147  button_html = html_view.foldersToButtonGroups(folder_list, options.output, url, messages=messages)
148  page.insertContent("<!--BUTTON_HOOK-->", button_html)
149  ########################### Add in the summary highcharts
150  extra_js = \
151  """
152  <script src="http://code.highcharts.com/stock/highstock.js"></script>
153  <script src="http://code.highcharts.com/stock/modules/exporting.js"></script>
154  <!--JS_HOOK-->
155  """
156  page.insertContent("<!--JS_HOOK-->", extra_js)
157 
158  # fa_body, fa_timeseries = VHC.testSummary("FA",tagged_release=True)
159  # page.insertContent("<!--FA_HOOK-->", fa_body)
160  # page.insertContent("<!--JS_HOOK-->", fa_timeseries+"\n<!--JS_HOOK-->")
161 
162  r_body, r_timeseries = VHC.testSummary("R",tagged_release=True)
163  page.insertContent("<!--R_HOOK-->", r_body)
164  page.insertContent("<!--JS_HOOK-->", r_timeseries+"\n<!--JS_HOOK-->")
165 
166  s_body, s_timeseries = VHC.testSummary("S",tagged_release=True)
167  page.insertContent("<!--S_HOOK-->", s_body)
168  page.insertContent("<!--JS_HOOK-->", s_timeseries+"\n<!--JS_HOOK-->")
169 
170  dev_body, dev_timeseries = VHC.testSummary("development_")
171  page.insertContent("<!--DEVELOPMENT_HOOK-->",dev_body)
172  page.insertContent("<!--JS_HOOK-->", dev_timeseries+"\n<!--JS_HOOK-->")
173 
174  # fa_body, fa_timeseries = VHC.testSummary("first-ana_")
175  # page.insertContent("<!--FIRST_ANA_HOOK-->",fa_body)
176  # page.insertContent("<!--JS_HOOK-->", fa_timeseries+"\n<!--JS_HOOK-->")
177  ########################### Write
178  page.write()
def makeResultsIndexPage(options, messages={})
Results page.
Definition: PageMakers.py:116
def PageMakers.makeResultsPage (   options,
  return_pickles = False 
)

Results page.

Definition at line 236 of file PageMakers.py.

References makeBatchLogPages(), makeFHiCLPage(), makeFileSizePage(), makeLogPage(), makeMetricPage(), makeTimingPage(), and open().

236 def makeResultsPage(options,return_pickles=False):
237  # the return pickles option will return the open pickles to the user
238  # the use of this option will speed up downstream functions that reuse these
239  # however it also means that all pickles remain in memory
240  # - this can cause exhaust the available memory on a machine in complex cases
241  # USE WITH CAUTION
242  unpickled_chains = {}
243  messages = {}
244  ########################### Imports
245  import PageMaker as PM
246  import cPickle
247  import ViewHTML as html_view
248  import os
249  import ProductionTestTools as tools
250  import glob
251  import re
252  ########################### Base HTML
253  result_base_html = \
254  """
255  <div class="page-header">
256  <h1><!--FOLDER_HOOK-->
257  </h1>
258  </div>
259  """
260  ########################### Get all the results pages
261  # List directory content
262  folder_location = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
263  if options.local: folder_location = options.output
264  folder_list = os.listdir(folder_location)
265  folder_list.sort()
266  for folder in folder_list:
267  if options.wildcard and (options.wildcard not in folder): continue
268  # if "development_2016-01-20-05-10-02" not in folder: continue
269  cwd = "%s%s"%(options.output,folder)
270  if not os.path.isdir(cwd) : continue
271  if not tools.isLogicalFolder(folder): continue
272  if options.verbose: print "PMS : Making results page for folder: %s"%cwd
273  file_list = [g for g in glob.glob("%s/*.pkl"%cwd) if re.search("(ND|FD)", g)]
274  ########################### Cacheing
275  # Two options for cacheing exist:
276  # 1. All chains for this test have existing metric pages. In this case we can exit now.
277  # 2. Some chains for this test have existing metric pages. In this case we have to remake the results page.
278  if options.full:
279  cached_chains = []
280  os.system("rm %s/embed/*.js"%cwd)
281  else:
282  cached_chains = tools.detectCaching(cwd)
283  ########################### Base page
284  page = PM.PageMaker("testing/%s/index.html"%folder,result_base_html, options)
285 
286  pretty_name = "%s %s"%(tools.releaseFromFolder(folder), tools.dateFromFolder(folder))
287 
288  page.insertContent("<!--NAVBAR_HOOK-->",
289  '<li><a href="http://nusoft.fnal.gov/nova/production/testing/%s/index.html">%s</a></li>'%(folder,pretty_name))
290  page.insertContent("<!--FOLDER_HOOK-->",
291  "%s <small>%s</small>"%(tools.releaseFromFolder(folder),tools.dateFromFolder(folder)))
292  ########################### Make link to projections
293  proj = '<p>The <a href="projection.html">projections section</a> interperates any results displayed here.</p>'
294  page.insertContent("<!--BODY_HOOK-->","%s\n<!--BODY_HOOK-->"%proj)
295  ########################### Make subfolders
296  tools.makeFolder("%s/logs"%cwd)
297  tools.makeFolder("%s/fcls"%cwd)
298  tools.makeFolder("%s/metrics"%cwd)
299  tools.makeFolder("%s/embed"%cwd)
300  tools.makeFolder("%s/batch_logs"%cwd)
301  tools.makeFolder("%s/file_sizes"%cwd)
302  tools.makeFolder("%s/timings"%cwd)
303  ########################### Look for info.pkl
304  info_pkl = "%s/info.pkl"%cwd
305  status_glyphs = {}
306  if os.path.exists(info_pkl):
307  if options.verbose: print "PMS : Making results page info: %s"%info_pkl
308  info = cPickle.load(open(info_pkl, "rb"))
309  info_html, message = html_view.parseInfo(info,folder)
310  messages[folder] = message
311  page.insertContent("<!--BODY_HOOK-->",
312  "%s\n<!--BODY_HOOK-->"%info_html)
313  statuses = makeBatchLogPages(info,options,folder)
314  for chain_name in statuses.keys():
315  this_glyphs = ""
316  # check for completed job but no pkl
317  if 1 in statuses[chain_name]:
318  found_pkl = False
319  for i_p, pkl_file in enumerate(file_list):
320  pkl_chain_name = tools.chainNameFromFileName(pkl_file)
321  if chain_name == pkl_chain_name:
322  found_pkl = True
323  if not found_pkl:
324  statuses[chain_name].append(6)
325  for s in statuses[chain_name]:
326  this_glyphs += html_view.statusToGlyph[s]
327  status_glyphs[chain_name] = this_glyphs
328  del info
329  ########################### Get pickles
330  if (len(file_list) == 0):
331  print "PMS : No .pkl files found in found: %s"%cwd
332  page.insertContent("<!--BODY_HOOK-->",
333  "<strong>No pickle files found.</strong>")
334  ########################### Add in status glyphs
335  chain_names = status_glyphs.keys()
336  chain_names.sort()
337  for chain_name in chain_names:
338  page.insertContent("<!--BATCH_STATUS_HOOK_%s-->"%chain_name,status_glyphs[chain_name])
339  page.write()
340  continue
341  file_list.sort()
342  ########################### Add content
343  if len(cached_chains) == len(file_list):
344  if options.verbose: print "PMS : all chains have metrics pages already made for them, skipping this folder."
345  continue
346  body = ""
347  ########################### Generate content
348  file_list.sort()
349  pickled_chains = []
350  for i_p, pkl_file in enumerate(file_list):
351  if options.verbose: print "PMS : Making results page for file: %s"%pkl_file
352  if options.verbose: print "PMS : - unpickling"
353  chain = cPickle.load(open(pkl_file, "rb"))
354  if return_pickles: unpickled_chains[pkl_file] = chain
355  if options.verbose: print "PMS : - unpickled"
356 
357  html, status = html_view.metricsToHTMLTable(chain)
358  if status: status_glyphs[chain.name] += html_view.statusToGlyph[status]
359  body += html
360  body += "\n\n"
361  ########################### Metric pages
362  # Caching option 2
363  if tools.chainNameFromFileName(pkl_file) in cached_chains:
364  if options.verbose: print "PMS : - has cached metric"
365  continue
366  for tier in chain.tiers:
367  if options.verbose: print "PMS : Making results page for tier: %s"%tier.short_name
368  # if (not options.quick) or ("S15-02-05_2015-02-05-12-26-41" in folder):
369  if (not options.quick):
370  makeMetricPage(chain,tier,options,folder)
371  makeLogPage(chain,tier,options,folder)
372  makeFHiCLPage(chain,tier,options,folder)
373  makeFileSizePage(chain,tier,options,folder)
374  makeTimingPage(chain,tier,options,folder)
375  if not return_pickles: del chain
376  page.insertContent("<!--BODY_HOOK-->",body)
377  ########################### Add in status glyphs
378  for chain_name in status_glyphs.keys():
379  page.insertContent("<!--BATCH_STATUS_HOOK_%s-->"%chain_name,status_glyphs[chain_name])
380  ########################### Write
381  page.write()
382  if return_pickles: return unpickled_chains, messages
383  return messages
def makeFHiCLPage(chain, tier, options, folder)
FHiCL page.
Definition: PageMakers.py:724
def makeLogPage(chain, tier, options, folder)
Log page.
Definition: PageMakers.py:451
def makeMetricPage(chain, tier, options, folder)
Metrics page.
Definition: PageMakers.py:386
procfile open("FD_BRL_v0.txt")
def makeFileSizePage(chain, tier, options, folder)
FileSize page.
Definition: PageMakers.py:482
def makeResultsPage(options, return_pickles=False)
Results page.
Definition: PageMakers.py:236
def makeBatchLogPages(info, options, folder)
Batch log pages.
Definition: PageMakers.py:938
def makeTimingPage(chain, tier, options, folder)
FileSize page.
Definition: PageMakers.py:611
def PageMakers.makeTimingPage (   chain,
  tier,
  options,
  folder 
)

FileSize page.

Definition at line 611 of file PageMakers.py.

Referenced by makeResultsPage().

611 def makeTimingPage(chain,tier,options,folder):
612  ########################### Imports
613  import PageMaker as PM
614  import ViewTimingDB as view
615  import ProductionTestTools as tools
616  import os
617  ########################### Base HTML
618  timing_base_html = \
619  """
620  <div class="page-header">
621  <h1>Time Tracker <small><!--NAME_HOOK--></small>
622  </h1>
623  </div>
624  """
625  ########################### Insert content
626  cwd = "%s%s"%(options.output,folder)
627  pretty_name = "%s %s"%(tools.releaseFromFolder(folder), tools.dateFromFolder(folder))
628 
629  name="testing/%s/timings/%s_%s.html"%(folder, chain.name, tier.short_name)
630  page = PM.PageMaker(name,timing_base_html, options)
631 
632  page.insertContent("<!--NAVBAR_HOOK-->",
633  '<li><a href="http://nusoft.fnal.gov/nova/production/testing/%s/index.html">%s</a></li>'%(folder,pretty_name))
634  page.insertContent("<!--NAME_HOOK-->",
635  "%s %s %s %s"%\
636  (chain.name.replace("_"," "),tier.short_name.replace("_"," "),
637  tools.releaseFromFolder(folder),tools.dateFromFolder(folder)))
638 
639  body_html = \
640  """
641  <div id="box_plot"></div>
642  <div id="line_plot"></div>
643  """
644  page.insertContent("<!--BODY_HOOK-->", body_html)
645 
646  db_name = "timing_db_%s_%s"%(chain.name, tier.short_name)
647  input_name = "%s/production_testing/%s/%s"%(os.environ["NOVAPRODVALID_GRID"],folder,db_name)
648  found_timing = False
649  if os.path.exists("%s.csv"%input_name):
650  print "PMS : - found timing CSV"
651  os.system("cp %s.csv %s/%s/timings/%s.csv"%(input_name, options.output, folder, db_name))
652  found_timing = True
653  # elif os.path.exists("%s.json"%input_name):
654  # print "PMS : - found timing JSON, converting to CSV"
655  # output_name = "%s/%s/timings/%s.csv"%(options.output,folder,db_name)
656  # timing_db = view.ViewTimingDB("%s.json"%input_name, verbose=False)
657  # timing_db.write_csv(output_name)
658  # found_timing = True
659 
660  if found_timing:
661  js_html = \
662  """
663  <script src="http://nusoft.fnal.gov/nova/production/static/d3.v3.min.js"></script>
664  <script type="text/javascript">
665  INPUT_FILE = "%s.csv";
666  </script>
667  <script src="http://nusoft.fnal.gov/nova/production/static/d3_timing_db_box.js"></script>
668  </body>
669  """%db_name
670  page.insertContent("<!--JS_HOOK-->", js_html)
671  else:
672  page.insertContent("<!--BODY_HOOK-->", "No JSON found")
673  ########################### Style
674  style_css = \
675  """
676  path {
677  stroke: steelblue;
678  stroke-width: 2;
679  fill: none;
680  }
681 
682  .quantile {
683  stroke: grey;
684  stroke-width: 2;
685  opacity: 0.5;
686  stroke-dasharray: 3,3;
687  }
688 
689  .axis path,
690  .axis line {
691  fill: none;
692  stroke: #000;
693  shape-rendering: crispEdges;
694  }
695 
696  .box {
697  fill: #fff;
698  stroke: #000;
699  stroke-width: 1.5px;
700  }
701 
702  .whisker {
703  fill: #fff;
704  stroke: #000;
705  stroke-width: 1.5px;
706  stroke-dasharray: 3,3;
707  }
708 
709  circle {
710  fill: none;
711  stroke: #ccc;
712  stroke-width: 1.5px;
713  }
714 
715  .first{
716  stroke: red;
717  }
718  """
719  page.insertContent("/*STYLE_HOOK*/",style_css)
720  ########################### Write
721  page.write()
def makeTimingPage(chain, tier, options, folder)
FileSize page.
Definition: PageMakers.py:611
def PageMakers.makeValidationPage (   options,
  controller_path,
  validation,
  version 
)

Make an individual validation page.

Definition at line 86 of file PageMakers.py.

References naming_schema.componentName(), view_tools.componentStatusRow(), view_tools.componentStatusSummary(), view_tools.controllerInfo(), makeComponentLogPages(), makeComponentSubmissionPage(), makeComponentTypePage(), makeComponentViewPages(), makeControllerConfigPage(), open(), and plotROC.title.

Referenced by makeValidationPages().

86 def makeValidationPage(options, controller_path, validation, version):
87  ########################### Imports
88  import PageMaker as PM
89  import os
90  import common_tools as tools
91  from glob import glob
92  import cPickle
93  import view_tools
94  from BatchLog import BatchLog
95  from ControllerView import ControllerView
96  import naming_schema
97  import samweb_client
98  ########################### Checks
99  if (not os.path.exists(controller_path)):
100  print "PMS : Controller %s not found"%controller_path
101  return
102  validation_base = validation.split("/")[-1]
103  validation_name = validation_base.replace("_"," ").title()
104  version_tag = version.split("/")[-1]
105  ########################### SAM web
106  sam = samweb_client.SAMWebClient("nova")
107  ########################### Insert content
108  print "PMS : --- %s (%s)"%(validation_name, version_tag)
109  controller = cPickle.load(open(controller_path,"r"))
110  view_path = controller_path.replace("controller","controller_view")
111  if (options.cache):
112  print "PMS : trying to load view from: %s"%view_path
113  if os.path.exists(view_path):
114  print "PMS : - loading"
115  view = cPickle.load(open(view_path, "r"))
116  else:
117  print "PMS : - not found, making a new view"
118  view = ControllerView(controller, verbose=options.verbose, testing=options.testing)
119  else:
120  view = ControllerView(controller, verbose=options.verbose, testing=options.testing)
121 
122  if options.testing:
123  output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB_TEST"],validation_base,version_tag)
124  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
125  else:
126  output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB"],validation_base,version_tag)
127  base_dir = os.environ["NOVAANAVALID_WEB"]
128  url = tools.directoryToURL(output_dir)
129  page = PM.PageMaker("index.html", output_dir, url)
130  breadcrumb = \
131  """
132  <li><a href="%s/index.html">Validation</a></li>
133  <li class="active">%s %s</li>
134  """%(tools.directoryToURL(base_dir),validation_name, version_tag)
135  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
136 
137  html = \
138  """
139  <div class="col-lg-12">
140  <h1 class="page-header">%s <small>%s</small></h1>
141  """%(validation_name, version_tag)
142 
143  html += view_tools.controllerInfo(controller, url)
144  makeControllerConfigPage(options, controller, output_dir, validation_name, version_tag)
145 
146  # Make the component view pages
147  if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
148  makeComponentViewPages(options, view, url, validation_name, version_tag)
149 
150  # write configuration page
151  print "PMS : - %i component types"%len(view.component_types)
152  component_types = view.component_types.keys()
153  component_types.sort()
154  component_status = {}
155  for component_type in component_types:
156  component_example = view.component_types[component_type][0]
157  if hasattr(component_example, "sam") and component_example.sam:
158  html += \
159  """
160  <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div>
161  <div class="panel panel-default">
162  <table class="table table-striped">
163  <tr>
164  <th>Dataset</th>
165  <th>Start time</th>
166  <th>Run time</th>
167  <th>Submission log</th>
168  <th>Batch logs</th>
169  <th>SAM project</th>
170  <th>SAM status</th>
171  <th>SAM merge</th>
172  <th>Merged ROOT file</th>
173  </tr>
174  """%(component_type,url,component_type,naming_schema.componentName(component_type))
175  else:
176  html += \
177  """
178  <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div>
179  <div class="panel panel-default">
180  <table class="table table-striped">
181  <tr>
182  <th>Dataset</th>
183  <th>Start time</th>
184  <th>Run time</th>
185  <th>Submission log</th>
186  <th>Batch logs</th>
187  <th>Log status</th>
188  <th>Err status</th>
189  <th>Root file</th>
190  </tr>
191  """%(component_type,url,component_type,naming_schema.componentName(component_type))
192 
193  if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
194  makeComponentTypePage(options, output_dir, view, component_type, validation_name, version_tag)
195 
196  component_status[component_type] = []
197  for i_c,component in enumerate(view.component_types[component_type]):
198  # html += view_tools.componentSummary(component, url)
199  ########################### Child pages
200  makeComponentSubmissionPage(options, output_dir, component, validation_name, version_tag)
201  makeComponentLogPages(options, output_dir, component, validation_name, version_tag)
202  row, status = view_tools.componentStatusRow(component, url, testing=options.testing, sam=sam)
203  html += row
204  component_status[component_type].append(status)
205  html += "</table></div>"
206  html += "<hr></div>"
207 
208  page.addContent("<!--BODY_HOOK-->",html)
209 
210  component_summary = view_tools.componentStatusSummary(component_status, url)
211  page.addContent("<!--CONTROLLER_SUMMARY_HOOK-->", component_summary)
212  ########################### Write
213  page.write()
214  # print "PMS : will save view if all components succeeded"
215  n_pass = 0
216  n_all = 0
217  for c in component_status.keys():
218  for s in component_status[c]:
219  n_all += 1
220  if s == "success": n_pass +=1
221  print "PMS : - %i/%i succeeded"%(n_pass, n_all)
222  if (n_all > 0) and (n_pass == n_all) and (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
223  # print "PMS : - writing as: %s"%view_path
224  view.write(view_path)
225  return component_status
def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:273
def componentStatusRow(component, url, testing=False, sam=False)
component status table row
Definition: view_tools.py:75
def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:333
def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag)
Make a component page.
Definition: PageMakers.py:228
def makeValidationPage(options, controller_path, validation, version)
Make an individual validation page.
Definition: PageMakers.py:86
def controllerInfo(controller, url)
controller HTML info summary
Definition: view_tools.py:3
def componentName(name)
get a component name
procfile open("FD_BRL_v0.txt")
def makeComponentViewPages(options, view, url, validation_name, version_tag)
Make a component view page.
Definition: PageMakers.py:309
def componentStatusSummary(status_dict, url)
component status summary
Definition: view_tools.py:130
def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:380
def PageMakers.makeValidationPages (   options)

Make the validation pages.

Definition at line 65 of file PageMakers.py.

References makeValidationPage().

65 def makeValidationPages(options):
66  ########################### Imports
67  from glob import glob
68  import os
69  ########################### Make one page per validation
70  statuses = {}
71  if options.testing:
72  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
73  else:
74  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
75  for validation in validations:
76  validation_base = validation.split("/")[-1]
77  versions = glob("%s/*"%validation)
78  versions.sort()
79  for version in versions:
80  version_tag = version.split("/")[-1]
81  controller_path = "%s/controller.pkl"%version
82  statuses["%s_%s"%(validation_base, version_tag)] = makeValidationPage(options, controller_path, validation, version)
83  return statuses
def makeValidationPage(options, controller_path, validation, version)
Make an individual validation page.
Definition: PageMakers.py:86
def makeValidationPages(options)
Make the validation pages.
Definition: PageMakers.py:65