readPickles.py
Go to the documentation of this file.
1 #!/bin/env python
2 print "run : --- Read multiple production test pickles"
3 ########################### Imports
4 import cPickle
5 import ViewSTDOUT as view
6 import ViewHTML as html_view
7 import ViewBokeh as bokeh_view
8 import ProductionTestTools as tools
9 import glob
10 import os
11 ########################### Option parser
12 from optparse import OptionParser
13 parser = OptionParser()
14 parser.add_option("-f", "--folder", help="input folder", action="store", type=str, dest="folder", default=False)
15 parser.add_option("-o", "--output", help="output folder", action="store", type=str, dest="output", default="html_output")
16 parser.add_option("-w", "--html", help="write html output", action="store_true", dest="html", default=False)
17 parser.add_option("-m", "--metrics", help="write metrics", action="store_true", dest="metrics", default=False)
18 parser.add_option("-v", "--verbose", help="turn on verbose mode", action="store_true", dest="verbose", default=False)
19 (options, args) = parser.parse_args()
20 if options.verbose:
21  print "run : --- Options"
22  print "run : folder: ",options.folder
23  print "run : output: ",options.output
24  print "run : html: ",options.html
25  print "run : metrics: ",options.metrics
26  print "run : verbose mode: ",options.verbose
27 assert (options.folder), "No folder provided to unpickle, specify one with -f MYFILE"
28 if "/nusoft/app/web/htdoc/nova/production/" in options.output:
29  base_url = options.output.replace("/nusoft/app/web/htdoc/nova/production/",
30  "http://nusoft.fnal.gov/nova/production/")
31 else:
32  base_url = options.output
33 ########################### Get files
34 file_list = glob.glob("%s/*.pkl"%options.folder)
35 assert(len(file_list)), "No .pkl files found in found: %s"%options.folder
36 file_list.sort()
37 ########################### Unpickle
38 attributes = ["memory",
39  "memory_percent",
40  "cpu_percent",
41  "u_cpu_times",
42  "s_cpu_times",
43  "read_bytes",
44  "write_bytes",
45  "rss_memory", # should be redundant w/ memory
46  "vms_memory",
47  #"n_open_files",
48  #"n_connections",
49 ]
50 if options.html:
51  base_html = html_view.base_html.replace("<title>base html</title>","<title>prod-test</title>")
52  if "/nusoft/app/web/htdoc/nova/production/" not in options.output:
53  base_html = base_html.replace("http://nusoft.fnal.gov/nova/production/static/","/Users/mt354/Analysis/web/")
54  body = ""
55  #header = html_view.collectEnvironment()
56  header = html_view.masthead("Production testing results","")
57  #toc = '<div class="page-header"><h2>Chains</h2></div><ul>\n'
58  toc = ''
59  footer = html_view.footer()
60  # check that the output folder exists
61  tools.makeFolder(options.output)
62  tools.makeFolder(options.output+"logs")
63  tools.makeFolder(options.output+"fcls")
64  tools.makeFolder(options.output+"metrics")
65  tools.makeFolder(options.output+"embed")
66 
67 for i_p, pkl_file in enumerate(file_list):
68  print "run : Opening pickle file %s"%pkl_file
69  chain = cPickle.load(open(pkl_file, "rb"))
70  view.showMetricsSummary(chain)
71 
72  if options.html:
73  this_body = html_view.metricsToHTMLTable(chain, metric_links=options.metrics)
74  body += this_body
75  body += "\n\n"
76  #toc += """<li><a href="#%s">%s</a></li>"""%(chain.name,chain.name.replace("_"," "))
77  toc += """<a class="btn btn-info" href="#%s">%s</a>"""%(chain.name,chain.name.replace("_"," "))
78 
79  if options.metrics:
80  print "run : --- Writing Bokeh metrics"
81  for tier in chain.tiers:
82  if "/nusoft/app/web/htdoc/nova/production/" not in options.output:
83  bokeh_path="/Users/mt354/Analysis/web/static/bokeh_static/"
84  else:
85  bokeh_path="http://nusoft.fnal.gov/nova/production/static/bokeh_static/"
86  this_html = bokeh_view.tierToHTML(tier, attributes,
87  title=chain.name.replace("_"," "),
88  bokeh_path=bokeh_path,
89  base_url ="%sembed/"%base_url)
90  this_html = this_html.replace("<!--FOOTER_HOOK-->", footer)
91  if "/nusoft/app/web/htdoc/nova/production/" not in options.output:
92  this_html = this_html.replace("http://nusoft.fnal.gov/nova/production/static/","/Users/mt354/Analysis/web/")
93  output_file = open("%smetrics/%s_%s.html"%(options.output,chain.name,tier.short_name), "wb")
94  output_file.write(this_html)
95  #print "run : Written tier metric to folder: %s"%options.output
96  output_file.close()
97  # now copy the embed.js objects over
98  js_list = bokeh_view.findJSList(this_html)
99  # make a directory
100  for js in js_list: os.system("mv %s %s/embed/"%(js,options.output))
101 if options.html:
102  print "run : Writing output to folder: %s"%options.output
103  # dump logs
104  #view.dumpLogFilesToFolder(chain, options.output+"logs/", skip_unused_tiers=True)
105  # dump fhcils
106  #view.dumpFHiCLFilesToFolder(chain, options.output+"fcls/", skip_unused_tiers=True)
107 
108  body = header+"\n\n"+toc+"\n\n"+body+footer
109  base_html = base_html.replace("<!--BODY_HOOK-->", body)
110  output_file = open("%s/index.html"%options.output, "wb")
111  output_file.write(base_html)
112  #print "run : Written output to folder: %s"%options.output
113  output_file.close()
114 ########################### Done
115 print "run : --- Done\n"
procfile open("FD_BRL_v0.txt")
assert(nhit_max >=nhit_nbins)