PageMakers.py
Go to the documentation of this file.
1 # These classes are used to make the HTML pages we care about
2 # any useful functionality related to status checking should
3 # be contained in the classes queried. Any large pieces of
4 # HTML should live in view tools
5 ###########################
6 # Make the testing front page
7 def makeFrontPage(options, status={}):
8  ########################### Imports
9  import PageMaker as PM
10  import os
11  import common_tools as tools
12  from glob import glob
13  import view_tools
14  ########################### Base HTML
15  if options.testing:
16  output_dir = os.environ["NOVAANAVALID_WEB_TEST"]
17  else:
18  output_dir = os.environ["NOVAANAVALID_WEB"]
19  url = tools.directoryToURL(output_dir)
20  page = PM.PageMaker("index.html", output_dir, url)
21  ########################### Insert content
22  breadcrumb = '<li class="active">Validation</li>'
23  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
24 
25  html = \
26  """
27  <div class="col-lg-12">
28  <h1 class="page-header">Analysis validation</h1>
29  """
30  # find all instances of validation
31  if options.testing:
32  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
33  else:
34  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
35  if options.verbose: print "PMS : found %i validations"%len(validations)
36  validations.sort()
37  # Make a list of contents based on these
38  for validation in validations:
39  validation_base = validation.split("/")[-1]
40  validation_name = validation_base.replace("_"," ").title()
41  html += "<h2>%s</h2><ul>"%validation_name
42  versions = glob("%s/*"%validation)
43  if options.verbose: print "PMS : Validation: %s, %i versions"%(validation_name, len(versions))
44  versions.sort()
45  for version in versions:
46  version = version.split("/")[-1]
47  html += "<li><a href=%s/%s>%s</a> "%(validation_base,version,version)
48  v_key = "%s_%s"%(validation_base, version)
49  if (v_key in status.keys()):
50  all_statuses = []
51  for component in status[v_key].keys():
52  for c in status[v_key][component]:
53  all_statuses.append(view_tools.classToGlyphicon(c))
54  all_statuses = " ".join(list(set(all_statuses)))
55  html += all_statuses
56  html += "</li>"
57 
58  html += "</ul><hr>"
59  html += "</div>"
60  page.addContent("<!--BODY_HOOK-->",html)
61  ########################### Write
62  page.write()
63 ###########################
64 # Make the validation pages
65 def makeValidationPages(options):
66  ########################### Imports
67  from glob import glob
68  import os
69  ########################### Make one page per validation
70  statuses = {}
71  if options.testing:
72  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
73  else:
74  validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
75  for validation in validations:
76  validation_base = validation.split("/")[-1]
77  versions = glob("%s/*"%validation)
78  versions.sort()
79  for version in versions:
80  version_tag = version.split("/")[-1]
81  controller_path = "%s/controller.pkl"%version
82  statuses["%s_%s"%(validation_base, version_tag)] = makeValidationPage(options, controller_path, validation, version)
83  return statuses
84 ###########################
85 # Make an individual validation page
86 def makeValidationPage(options, controller_path, validation, version):
87  ########################### Imports
88  import PageMaker as PM
89  import os
90  import common_tools as tools
91  from glob import glob
92  import cPickle
93  import view_tools
94  from BatchLog import BatchLog
95  from ControllerView import ControllerView
96  import naming_schema
97  import samweb_client
98  ########################### Checks
99  if (not os.path.exists(controller_path)):
100  print "PMS : Controller %s not found"%controller_path
101  return
102  validation_base = validation.split("/")[-1]
103  validation_name = validation_base.replace("_"," ").title()
104  version_tag = version.split("/")[-1]
105  ########################### SAM web
106  sam = samweb_client.SAMWebClient("nova")
107  ########################### Insert content
108  print "PMS : --- %s (%s)"%(validation_name, version_tag)
109  controller = cPickle.load(open(controller_path,"r"))
110  view_path = controller_path.replace("controller","controller_view")
111  if (options.cache):
112  print "PMS : trying to load view from: %s"%view_path
113  if os.path.exists(view_path):
114  print "PMS : - loading"
115  view = cPickle.load(open(view_path, "r"))
116  else:
117  print "PMS : - not found, making a new view"
118  view = ControllerView(controller, verbose=options.verbose, testing=options.testing)
119  else:
120  view = ControllerView(controller, verbose=options.verbose, testing=options.testing)
121 
122  if options.testing:
123  output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB_TEST"],validation_base,version_tag)
124  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
125  else:
126  output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB"],validation_base,version_tag)
127  base_dir = os.environ["NOVAANAVALID_WEB"]
128  url = tools.directoryToURL(output_dir)
129  page = PM.PageMaker("index.html", output_dir, url)
130  breadcrumb = \
131  """
132  <li><a href="%s/index.html">Validation</a></li>
133  <li class="active">%s %s</li>
134  """%(tools.directoryToURL(base_dir),validation_name, version_tag)
135  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
136 
137  html = \
138  """
139  <div class="col-lg-12">
140  <h1 class="page-header">%s <small>%s</small></h1>
141  """%(validation_name, version_tag)
142 
143  html += view_tools.controllerInfo(controller, url)
144  makeControllerConfigPage(options, controller, output_dir, validation_name, version_tag)
145 
146  # Make the component view pages
147  if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
148  makeComponentViewPages(options, view, url, validation_name, version_tag)
149 
150  # write configuration page
151  print "PMS : - %i component types"%len(view.component_types)
152  component_types = view.component_types.keys()
153  component_types.sort()
154  component_status = {}
155  for component_type in component_types:
156  component_example = view.component_types[component_type][0]
157  if hasattr(component_example, "sam") and component_example.sam:
158  html += \
159  """
160  <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div>
161  <div class="panel panel-default">
162  <table class="table table-striped">
163  <tr>
164  <th>Dataset</th>
165  <th>Start time</th>
166  <th>Run time</th>
167  <th>Submission log</th>
168  <th>Batch logs</th>
169  <th>SAM project</th>
170  <th>SAM status</th>
171  <th>SAM merge</th>
172  <th>Merged ROOT file</th>
173  </tr>
174  """%(component_type,url,component_type,naming_schema.componentName(component_type))
175  else:
176  html += \
177  """
178  <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div>
179  <div class="panel panel-default">
180  <table class="table table-striped">
181  <tr>
182  <th>Dataset</th>
183  <th>Start time</th>
184  <th>Run time</th>
185  <th>Submission log</th>
186  <th>Batch logs</th>
187  <th>Log status</th>
188  <th>Err status</th>
189  <th>Root file</th>
190  </tr>
191  """%(component_type,url,component_type,naming_schema.componentName(component_type))
192 
193  if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
194  makeComponentTypePage(options, output_dir, view, component_type, validation_name, version_tag)
195 
196  component_status[component_type] = []
197  for i_c,component in enumerate(view.component_types[component_type]):
198  # html += view_tools.componentSummary(component, url)
199  ########################### Child pages
200  makeComponentSubmissionPage(options, output_dir, component, validation_name, version_tag)
201  makeComponentLogPages(options, output_dir, component, validation_name, version_tag)
202  row, status = view_tools.componentStatusRow(component, url, testing=options.testing, sam=sam)
203  html += row
204  component_status[component_type].append(status)
205  html += "</table></div>"
206  html += "<hr></div>"
207 
208  page.addContent("<!--BODY_HOOK-->",html)
209 
210  component_summary = view_tools.componentStatusSummary(component_status, url)
211  page.addContent("<!--CONTROLLER_SUMMARY_HOOK-->", component_summary)
212  ########################### Write
213  page.write()
214  # print "PMS : will save view if all components succeeded"
215  n_pass = 0
216  n_all = 0
217  for c in component_status.keys():
218  for s in component_status[c]:
219  n_all += 1
220  if s == "success": n_pass +=1
221  print "PMS : - %i/%i succeeded"%(n_pass, n_all)
222  if (n_all > 0) and (n_pass == n_all) and (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
223  # print "PMS : - writing as: %s"%view_path
224  view.write(view_path)
225  return component_status
226 ###########################
227 # Make a component page
228 def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag):
229  ########################### Imports
230  import PageMaker as PM
231  import common_tools as tools
232  import view_tools
233  import os
234  import naming_schema
235  ########################### Insert content
236  parent_url = tools.directoryToURL(parent_dir)
237  output_dir = parent_dir+"/%s"%component_type
238  url = tools.directoryToURL(output_dir)
239  name = "index.html"
240  page = PM.PageMaker(name, output_dir, url)
241  if options.testing:
242  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
243  else:
244  base_dir = os.environ["NOVAANAVALID_WEB"]
245  breadcrumb = \
246  """
247  <li><a href="%s/index.html">Validation</a></li>
248  <li><a href="%s/index.html">%s %s</a></li>
249  <li class="active">%s</li>
250  """%(tools.directoryToURL(base_dir), parent_url, validation_name, version_tag, \
251  naming_schema.componentName(component_type))
252  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
253 
254  html = \
255  """
256  <div class="col-lg-12">
257  <h1 class="page-header">%s</h1>
258  <p>The following plot groups have been identified:
259  <ul>
260  """%(naming_schema.componentName(component_type))
261 
262  # view.find_complete_components(component_type)
263  # view.group_components()
264  if component_type in view.plot_groups.keys():
265  for group in view.plot_groups[component_type]:
266  html += '<li><a href="%s/%s">%s</a></li>\n'%(url,group.replace(" ","_")+".html",group)
267  html += "</ul></p><hr></div>"
268  page.addContent("<!--BODY_HOOK-->",html)
269  ########################### Write
270  page.write()
271 ###########################
272 # Make a component submission log page
273 def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag):
274  ########################### Imports
275  import PageMaker as PM
276  import common_tools as tools
277  import view_tools
278  import os
279  import naming_schema
280  ########################### Insert content
281  url = tools.directoryToURL(parent_dir)
282  name = "%s_%s_%s_submission_log.html"%(component.name, component.version, component.dataset)
283  page = PM.PageMaker(name, parent_dir, url)
284  if options.testing:
285  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
286  else:
287  base_dir = os.environ["NOVAANAVALID_WEB"]
288  breadcrumb = \
289  """
290  <li><a href="%s/index.html">Validation</a></li>
291  <li><a href="%s/index.html">%s %s</a></li>
292  <li class="active">%s %s %s submission log</li>
293  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
294  component.name, component.version, component.label)
295  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
296 
297  html = \
298  """
299  <div class="col-lg-12">
300  <h2 class="page-header">%s: <small>%s, submission log</small></h1>
301  <pre>%s</pre>
302  </div>
303  """%(naming_schema.componentName(component.name), component.label, view_tools.toHTML("".join(component.lines)))
304  page.addContent("<!--BODY_HOOK-->",html)
305  ########################### Write
306  page.write()
307 ###########################
308 # Make a component view page
309 def makeComponentViewPages(options, view, url, validation_name, version_tag):
310  ########################### Imports
311  from HTMLView import HTMLView
312  from Canvas import Canvas
313  import os
314  import common_tools as tools
315  ########################### Insert content
316  canvas = Canvas()
317  view.component_view(canvas)
318  if options.testing:
319  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
320  else:
321  base_dir = os.environ["NOVAANAVALID_WEB"]
322 
323  breadcrumb = \
324  """
325  <li><a href="%s/index.html">Validation</a></li>
326  <li><a href="%s/index.html">%s %s</a></li>
327  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
328 
329  html_view = HTMLView(view, breadcrumb=breadcrumb, reuse_histograms=options.reuse, verbose=options.verbose, testing=options.testing)
330  html_view.write_component_html()
331 ###########################
332 # Make a component submission log page
333 def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag):
334  ########################### Imports
335  import PageMaker as PM
336  import common_tools as tools
337  import view_tools
338  import os
339  from BatchLog import BatchLog
340  import naming_schema
341  ########################### Insert content
342  if options.testing:
343  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
344  else:
345  base_dir = os.environ["NOVAANAVALID_WEB"]
346  # get logs
347  job_sub_id = tools.getJobID(component.lines)
348  component.logs = BatchLog(tools.fetchLogs(job_sub_id, component.run_time))
349  if job_sub_id == False: return
350 
351  log_types = ["cmd","log","out","err"]
352  for log_type in log_types:
353  url = tools.directoryToURL(parent_dir)
354  name = "%s_%s_%s_batch_%s.html"%(component.name, component.version, component.dataset, log_type)
355  page = PM.PageMaker(name, parent_dir, url)
356  breadcrumb = \
357  """
358  <li><a href="%s/index.html">Validation</a></li>
359  <li><a href="%s/index.html">%s %s</a></li>
360  <li class="active">%s %s %s batch %s</li>
361  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
362  component.name, component.version, component.label, log_type)
363  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
364 
365  if hasattr(component.logs, log_type): content = getattr(component.logs, log_type)
366  else: content = [""]
367 
368  html = \
369  """
370  <div class="col-lg-12">
371  <h2 class="page-header">%s: <small>%s, batch %s</small></h1>
372  <pre>%s</pre>
373  </div>
374  """%(naming_schema.componentName(component.name), component.label, log_type, view_tools.toHTML("".join(content)))
375  page.addContent("<!--BODY_HOOK-->",html)
376  ########################### Write
377  page.write()
378 ###########################
379 # Make a component submission log page
380 def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag):
381  ########################### Imports
382  import PageMaker as PM
383  import common_tools as tools
384  import os
385  ########################### Insert content
386  url = tools.directoryToURL(parent_dir)
387  name = "config.html"
388  page = PM.PageMaker(name, parent_dir, url)
389  if options.testing:
390  base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
391  else:
392  base_dir = os.environ["NOVAANAVALID_WEB"]
393  breadcrumb = \
394  """
395  <li><a href="%s/index.html">Validation</a></li>
396  <li><a href="%s/index.html">%s %s</a></li>
397  <li class="active">configuration</li>
398  """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
399  page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)
400 
401  config = repr(controller.config)
402  config = config.replace("u'","'")
403  config = config.replace(',',',\n')
404 
405  html = \
406  """
407  <div class="col-lg-12">
408  <h2 class="page-header">Configuration</h1>
409  <pre>%s</pre>
410  </div>
411  """%(config)
412  page.addContent("<!--BODY_HOOK-->",html)
413  ########################### Write
414  page.write()
415 
keys
Reco plots.
Definition: caf_analysis.py:46
Definition: Canvas.py:1
def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:273
def componentStatusRow(component, url, testing=False, sam=False)
component status table row
Definition: view_tools.py:75
def toHTML(text)
convert some text to safe HTML
Definition: view_tools.py:69
def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:333
def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag)
Make a component page.
Definition: PageMakers.py:228
def makeValidationPage(options, controller_path, validation, version)
Make an individual validation page.
Definition: PageMakers.py:86
def controllerInfo(controller, url)
controller HTML info summary
Definition: view_tools.py:3
def componentName(name)
get a component name
procfile open("FD_BRL_v0.txt")
def makeComponentViewPages(options, view, url, validation_name, version_tag)
Make a component view page.
Definition: PageMakers.py:309
def classToGlyphicon(class_label)
convert a class to a glyphicon
Definition: view_tools.py:152
def makeFrontPage(options, status={})
Make the testing front page.
Definition: PageMakers.py:7
def componentStatusSummary(status_dict, url)
component status summary
Definition: view_tools.py:130
def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag)
Make a component submission log page.
Definition: PageMakers.py:380
def makeValidationPages(options)
Make the validation pages.
Definition: PageMakers.py:65