11 import common_tools
as tools
16 output_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
18 output_dir = os.environ[
"NOVAANAVALID_WEB"]
19 url = tools.directoryToURL(output_dir)
20 page = PM.PageMaker(
"index.html", output_dir, url)
22 breadcrumb =
'<li class="active">Validation</li>' 23 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
27 <div class="col-lg-12"> 28 <h1 class="page-header">Analysis validation</h1> 32 validations = glob(
"%s/validation/*"%os.environ[
"NOVAANAVALID_GRID_TEST"])
34 validations = glob(
"%s/validation/*"%os.environ[
"NOVAANAVALID_GRID"])
35 if options.verbose:
print "PMS : found %i validations"%len(validations)
38 for validation
in validations:
39 validation_base = validation.split(
"/")[-1]
40 validation_name = validation_base.replace(
"_",
" ").
title()
41 html +=
"<h2>%s</h2><ul>"%validation_name
42 versions = glob(
"%s/*"%validation)
43 if options.verbose:
print "PMS : Validation: %s, %i versions"%(validation_name, len(versions))
45 for version
in versions:
46 version = version.split(
"/")[-1]
47 html +=
"<li><a href=%s/%s>%s</a> "%(validation_base,version,version)
48 v_key =
"%s_%s"%(validation_base, version)
49 if (v_key
in status.keys()):
51 for component
in status[v_key].
keys():
52 for c
in status[v_key][component]:
54 all_statuses =
" ".join(
list(set(all_statuses)))
60 page.addContent(
"<!--BODY_HOOK-->",html)
72 validations = glob(
"%s/validation/*"%os.environ[
"NOVAANAVALID_GRID_TEST"])
74 validations = glob(
"%s/validation/*"%os.environ[
"NOVAANAVALID_GRID"])
75 for validation
in validations:
76 validation_base = validation.split(
"/")[-1]
77 versions = glob(
"%s/*"%validation)
79 for version
in versions:
80 version_tag = version.split(
"/")[-1]
81 controller_path =
"%s/controller.pkl"%version
82 statuses[
"%s_%s"%(validation_base, version_tag)] =
makeValidationPage(options, controller_path, validation, version)
88 import PageMaker
as PM
90 import common_tools
as tools
94 from BatchLog
import BatchLog
95 from ControllerView
import ControllerView
99 if (
not os.path.exists(controller_path)):
100 print "PMS : Controller %s not found"%controller_path
102 validation_base = validation.split(
"/")[-1]
103 validation_name = validation_base.replace(
"_",
" ").
title()
104 version_tag = version.split(
"/")[-1]
106 sam = samweb_client.SAMWebClient(
"nova")
108 print "PMS : --- %s (%s)"%(validation_name, version_tag)
109 controller = cPickle.load(
open(controller_path,
"r")) 110 view_path = controller_path.replace("controller",
"controller_view")
112 print "PMS : trying to load view from: %s"%view_path
113 if os.path.exists(view_path):
114 print "PMS : - loading" 115 view = cPickle.load(
open(view_path,
"r")) 117 print "PMS : - not found, making a new view" 118 view =
ControllerView(controller, verbose=options.verbose, testing=options.testing)
120 view =
ControllerView(controller, verbose=options.verbose, testing=options.testing)
123 output_dir =
"%s/%s/%s"%(os.environ[
"NOVAANAVALID_WEB_TEST"],validation_base,version_tag)
124 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
126 output_dir =
"%s/%s/%s"%(os.environ[
"NOVAANAVALID_WEB"],validation_base,version_tag)
127 base_dir = os.environ[
"NOVAANAVALID_WEB"]
128 url = tools.directoryToURL(output_dir)
129 page = PM.PageMaker(
"index.html", output_dir, url)
132 <li><a href="%s/index.html">Validation</a></li> 133 <li class="active">%s %s</li> 134 """%(tools.directoryToURL(base_dir),validation_name, version_tag)
135 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
139 <div class="col-lg-12"> 140 <h1 class="page-header">%s <small>%s</small></h1> 141 """%(validation_name, version_tag)
147 if (
not options.status)
and (tools.filterValidation(options.filter, validation_base, version_tag)):
151 print "PMS : - %i component types"%len(view.component_types)
152 component_types = view.component_types.keys()
153 component_types.sort()
154 component_status = {}
155 for component_type
in component_types:
156 component_example = view.component_types[component_type][0]
157 if hasattr(component_example,
"sam")
and component_example.sam:
160 <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div> 161 <div class="panel panel-default"> 162 <table class="table table-striped"> 167 <th>Submission log</th> 172 <th>Merged ROOT file</th> 178 <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div> 179 <div class="panel panel-default"> 180 <table class="table table-striped"> 185 <th>Submission log</th> 193 if (
not options.status)
and (tools.filterValidation(options.filter, validation_base, version_tag)):
196 component_status[component_type] = []
197 for i_c,component
in enumerate(view.component_types[component_type]):
204 component_status[component_type].
append(status)
205 html +=
"</table></div>" 208 page.addContent(
"<!--BODY_HOOK-->",html)
211 page.addContent(
"<!--CONTROLLER_SUMMARY_HOOK-->", component_summary)
217 for c
in component_status.keys():
218 for s
in component_status[c]:
220 if s ==
"success": n_pass +=1
221 print "PMS : - %i/%i succeeded"%(n_pass, n_all)
222 if (n_all > 0)
and (n_pass == n_all)
and (
not options.status)
and (tools.filterValidation(options.filter, validation_base, version_tag)):
224 view.write(view_path)
225 return component_status
230 import PageMaker
as PM
231 import common_tools
as tools
236 parent_url = tools.directoryToURL(parent_dir)
237 output_dir = parent_dir+
"/%s"%component_type
238 url = tools.directoryToURL(output_dir)
240 page = PM.PageMaker(name, output_dir, url)
242 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
244 base_dir = os.environ[
"NOVAANAVALID_WEB"]
247 <li><a href="%s/index.html">Validation</a></li> 248 <li><a href="%s/index.html">%s %s</a></li> 249 <li class="active">%s</li> 250 """%(tools.directoryToURL(base_dir), parent_url, validation_name, version_tag, \
252 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
256 <div class="col-lg-12"> 257 <h1 class="page-header">%s</h1> 258 <p>The following plot groups have been identified: 264 if component_type
in view.plot_groups.keys():
265 for group
in view.plot_groups[component_type]:
266 html +=
'<li><a href="%s/%s">%s</a></li>\n'%(url,group.replace(
" ",
"_")+
".html",group)
267 html +=
"</ul></p><hr></div>" 268 page.addContent(
"<!--BODY_HOOK-->",html)
275 import PageMaker
as PM
276 import common_tools
as tools
281 url = tools.directoryToURL(parent_dir)
282 name =
"%s_%s_%s_submission_log.html"%(component.name, component.version, component.dataset)
283 page = PM.PageMaker(name, parent_dir, url)
285 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
287 base_dir = os.environ[
"NOVAANAVALID_WEB"]
290 <li><a href="%s/index.html">Validation</a></li> 291 <li><a href="%s/index.html">%s %s</a></li> 292 <li class="active">%s %s %s submission log</li> 293 """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
294 component.name, component.version, component.label)
295 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
299 <div class="col-lg-12"> 300 <h2 class="page-header">%s: <small>%s, submission log</small></h1> 304 page.addContent(
"<!--BODY_HOOK-->",html)
311 from HTMLView
import HTMLView
312 from Canvas
import Canvas
314 import common_tools
as tools
317 view.component_view(canvas)
319 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
321 base_dir = os.environ[
"NOVAANAVALID_WEB"]
325 <li><a href="%s/index.html">Validation</a></li> 326 <li><a href="%s/index.html">%s %s</a></li> 327 """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
329 html_view =
HTMLView(view, breadcrumb=breadcrumb, reuse_histograms=options.reuse, verbose=options.verbose, testing=options.testing)
330 html_view.write_component_html()
335 import PageMaker
as PM
336 import common_tools
as tools
339 from BatchLog
import BatchLog
343 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
345 base_dir = os.environ[
"NOVAANAVALID_WEB"]
347 job_sub_id = tools.getJobID(component.lines)
348 component.logs =
BatchLog(tools.fetchLogs(job_sub_id, component.run_time))
349 if job_sub_id ==
False:
return 351 log_types = [
"cmd",
"log",
"out",
"err"]
352 for log_type
in log_types:
353 url = tools.directoryToURL(parent_dir)
354 name =
"%s_%s_%s_batch_%s.html"%(component.name, component.version, component.dataset, log_type)
355 page = PM.PageMaker(name, parent_dir, url)
358 <li><a href="%s/index.html">Validation</a></li> 359 <li><a href="%s/index.html">%s %s</a></li> 360 <li class="active">%s %s %s batch %s</li> 361 """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
362 component.name, component.version, component.label, log_type)
363 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
365 if hasattr(component.logs, log_type): content = getattr(component.logs, log_type)
370 <div class="col-lg-12"> 371 <h2 class="page-header">%s: <small>%s, batch %s</small></h1> 375 page.addContent(
"<!--BODY_HOOK-->",html)
382 import PageMaker
as PM
383 import common_tools
as tools
386 url = tools.directoryToURL(parent_dir)
388 page = PM.PageMaker(name, parent_dir, url)
390 base_dir = os.environ[
"NOVAANAVALID_WEB_TEST"]
392 base_dir = os.environ[
"NOVAANAVALID_WEB"]
395 <li><a href="%s/index.html">Validation</a></li> 396 <li><a href="%s/index.html">%s %s</a></li> 397 <li class="active">configuration</li> 398 """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
399 page.addContent(
"<!--BREADCRUMB_HOOK-->", breadcrumb)
401 config = repr(controller.config)
402 config = config.replace(
"u'","'")
403 config = config.replace(
',',
',\n')
407 <div class="col-lg-12"> 408 <h2 class="page-header">Configuration</h1> 412 page.addContent(
"<!--BODY_HOOK-->",html)
def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag)
Make a component submission log page.
def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag)
Make a component page.
def makeValidationPage(options, controller_path, validation, version)
Make an individual validation page.
def componentName(name)
get a component name
procfile open("FD_BRL_v0.txt")
def makeComponentViewPages(options, view, url, validation_name, version_tag)
Make a component view page.
def makeFrontPage(options, status={})
Make the testing front page.
def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag)
Make a component submission log page.
def makeValidationPages(options)
Make the validation pages.