PageMakers.py

Go to the documentation of this file.

# These classes are used to make the HTML pages we care about
# any useful functionality related to status checking should 
# be contained in the classes queried. Any large pieces of 
# HTML should live in view tools
########################### 
# Make the testing front page
def makeFrontPage(options, status={}):
    ########################### Imports
    import PageMaker as PM
    import os
    import common_tools as tools
    from glob import glob
    import view_tools
    ########################### Base HTML
    if options.testing:
        output_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        output_dir = os.environ["NOVAANAVALID_WEB"]
    url        = tools.directoryToURL(output_dir)
    page = PM.PageMaker("index.html", output_dir, url)
    ########################### Insert content
    breadcrumb = '<li class="active">Validation</li>'
    page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

    html = \
    """
    <div class="col-lg-12">
      <h1 class="page-header">Analysis validation</h1>
    """
    # find all instances of validation
    if options.testing:
        validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
    else:
        validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
    if options.verbose: print "PMS  :     found %i validations"%len(validations)
    validations.sort()
    # Make a list of contents based on these
    for validation in validations:
        validation_base = validation.split("/")[-1]
        validation_name = validation_base.replace("_"," ").title()
        html += "<h2>%s</h2><ul>"%validation_name
        versions = glob("%s/*"%validation)
        if options.verbose: print "PMS  :         Validation: %s, %i versions"%(validation_name, len(versions))
        versions.sort()
        for version in versions:
            version = version.split("/")[-1]
            html += "<li><a href=%s/%s>%s</a> "%(validation_base,version,version)
            v_key = "%s_%s"%(validation_base, version)
            if (v_key in status.keys()):
                all_statuses = []
                for component in status[v_key].keys():
                    for c in status[v_key][component]:
                        all_statuses.append(view_tools.classToGlyphicon(c))
                all_statuses = " ".join(list(set(all_statuses)))
                html += all_statuses
            html += "</li>"

        html += "</ul><hr>"
    html += "</div>"
    page.addContent("<!--BODY_HOOK-->",html)
    ########################### Write
    page.write()
########################### 
# Make the validation pages
def makeValidationPages(options):
    ########################### Imports
    from glob import glob
    import os
    ########################### Make one page per validation
    statuses = {}
    if options.testing:
        validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID_TEST"])
    else:
        validations = glob("%s/validation/*"%os.environ["NOVAANAVALID_GRID"])
    for validation in validations:
        validation_base = validation.split("/")[-1]
        versions = glob("%s/*"%validation)
        versions.sort()
        for version in versions:
            version_tag = version.split("/")[-1]
            controller_path = "%s/controller.pkl"%version
            statuses["%s_%s"%(validation_base, version_tag)] = makeValidationPage(options, controller_path, validation, version)
    return statuses
########################### 
# Make an individual validation page
def makeValidationPage(options, controller_path, validation, version):
    ########################### Imports
    import PageMaker as PM
    import os
    import common_tools as tools
    from glob import glob
    import cPickle
    import view_tools
    from BatchLog import BatchLog
    from ControllerView import ControllerView
    import naming_schema
    import samweb_client
    ########################### Checks
    if (not os.path.exists(controller_path)):
        print "PMS  : Controller %s not found"%controller_path
        return
    validation_base = validation.split("/")[-1]
    validation_name = validation_base.replace("_"," ").title()
    version_tag = version.split("/")[-1]
    ########################### SAM web
    sam = samweb_client.SAMWebClient("nova")
    ########################### Insert content
    print "PMS  : --- %s (%s)"%(validation_name, version_tag)
    controller = cPickle.load(open(controller_path,"r"))
    view_path = controller_path.replace("controller","controller_view")
    if (options.cache):
        print "PMS  :     trying to load view from: %s"%view_path
        if os.path.exists(view_path):
            print "PMS  :       - loading"
            view = cPickle.load(open(view_path, "r"))
        else:
            print "PMS  :       - not found, making a new view"
            view = ControllerView(controller, verbose=options.verbose, testing=options.testing)
    else:
        view = ControllerView(controller, verbose=options.verbose, testing=options.testing)

    if options.testing:
        output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB_TEST"],validation_base,version_tag)
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        output_dir = "%s/%s/%s"%(os.environ["NOVAANAVALID_WEB"],validation_base,version_tag)
        base_dir = os.environ["NOVAANAVALID_WEB"]
    url        = tools.directoryToURL(output_dir)
    page       = PM.PageMaker("index.html", output_dir, url)
    breadcrumb = \
    """
    <li><a href="%s/index.html">Validation</a></li>
    <li class="active">%s %s</li>
    """%(tools.directoryToURL(base_dir),validation_name, version_tag)
    page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

    html = \
    """
    <div class="col-lg-12">
      <h1 class="page-header">%s <small>%s</small></h1>
    """%(validation_name, version_tag)

    html += view_tools.controllerInfo(controller, url)
    makeControllerConfigPage(options, controller, output_dir, validation_name, version_tag)

    # Make the component view pages
    if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
        makeComponentViewPages(options, view, url, validation_name, version_tag)

    # write configuration page
    print "PMS  :       - %i component types"%len(view.component_types)
    component_types = view.component_types.keys()
    component_types.sort()
    component_status = {}
    for component_type in component_types:
        component_example = view.component_types[component_type][0]
        if hasattr(component_example, "sam") and component_example.sam:
            html += \
            """
            <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div> 
            <div class="panel panel-default">
                <table class="table table-striped">
                <tr>
                    <th>Dataset</th>
                    <th>Start time</th>
                    <th>Run time</th>
                    <th>Submission log</th>
                    <th>Batch logs</th>
                    <th>SAM project</th>
                    <th>SAM status</th>
                    <th>SAM merge</th>
                    <th>Merged ROOT file</th>
                </tr>
            """%(component_type,url,component_type,naming_schema.componentName(component_type))
        else:
            html += \
            """
            <div class="page-header"><h2 id="%s"><a href="%s/%s/index.html">%s</a></h2></div> 
            <div class="panel panel-default">
                <table class="table table-striped">
                <tr>
                    <th>Dataset</th>
                    <th>Start time</th>
                    <th>Run time</th>
                    <th>Submission log</th>
                    <th>Batch logs</th>
                    <th>Log status</th>
                    <th>Err status</th>
                    <th>Root file</th>
                </tr>
            """%(component_type,url,component_type,naming_schema.componentName(component_type))

        if (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
            makeComponentTypePage(options, output_dir, view, component_type, validation_name, version_tag)

        component_status[component_type] = []
        for i_c,component in enumerate(view.component_types[component_type]):
            # html += view_tools.componentSummary(component, url)
            ########################### Child pages
            makeComponentSubmissionPage(options, output_dir, component, validation_name, version_tag)
            makeComponentLogPages(options, output_dir, component, validation_name, version_tag)
            row, status = view_tools.componentStatusRow(component, url, testing=options.testing, sam=sam)
            html += row
            component_status[component_type].append(status)
        html += "</table></div>"
    html += "<hr></div>"

    page.addContent("<!--BODY_HOOK-->",html)

    component_summary = view_tools.componentStatusSummary(component_status, url)
    page.addContent("<!--CONTROLLER_SUMMARY_HOOK-->", component_summary)
    ########################### Write
    page.write()
    # print "PMS  :     will save view if all components succeeded"
    n_pass = 0
    n_all  = 0
    for c in component_status.keys():
        for s in component_status[c]:
            n_all += 1
            if s == "success": n_pass +=1
    print "PMS  :       - %i/%i succeeded"%(n_pass, n_all)
    if (n_all > 0) and (n_pass == n_all) and (not options.status) and (tools.filterValidation(options.filter, validation_base, version_tag)):
        # print "PMS  :       - writing as: %s"%view_path
        view.write(view_path)
    return component_status
########################### 
# Make a component page
def makeComponentTypePage(options, parent_dir, view, component_type, validation_name, version_tag):
    ########################### Imports
    import PageMaker as PM
    import common_tools as tools
    import view_tools
    import os
    import naming_schema
    ########################### Insert content
    parent_url = tools.directoryToURL(parent_dir)
    output_dir = parent_dir+"/%s"%component_type
    url        = tools.directoryToURL(output_dir)
    name = "index.html"
    page       = PM.PageMaker(name, output_dir, url)
    if options.testing:
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        base_dir = os.environ["NOVAANAVALID_WEB"]
    breadcrumb = \
    """
    <li><a href="%s/index.html">Validation</a></li>
    <li><a href="%s/index.html">%s %s</a></li>
    <li class="active">%s</li>
    """%(tools.directoryToURL(base_dir), parent_url, validation_name, version_tag, \
         naming_schema.componentName(component_type))
    page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

    html = \
    """
    <div class="col-lg-12">
      <h1 class="page-header">%s</h1>
      <p>The following plot groups have been identified: 
        <ul>
    """%(naming_schema.componentName(component_type))

    # view.find_complete_components(component_type)
    # view.group_components()
    if component_type in view.plot_groups.keys():
        for group in view.plot_groups[component_type]:
            html += '<li><a href="%s/%s">%s</a></li>\n'%(url,group.replace(" ","_")+".html",group)
    html += "</ul></p><hr></div>"
    page.addContent("<!--BODY_HOOK-->",html)
    ########################### Write
    page.write()
########################### 
# Make a component submission log page
def makeComponentSubmissionPage(options, parent_dir, component, validation_name, version_tag):
    ########################### Imports
    import PageMaker as PM
    import common_tools as tools
    import view_tools
    import os
    import naming_schema
    ########################### Insert content
    url        = tools.directoryToURL(parent_dir)
    name = "%s_%s_%s_submission_log.html"%(component.name, component.version, component.dataset)
    page       = PM.PageMaker(name, parent_dir, url)
    if options.testing:
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        base_dir = os.environ["NOVAANAVALID_WEB"]
    breadcrumb = \
    """
    <li><a href="%s/index.html">Validation</a></li>
    <li><a href="%s/index.html">%s %s</a></li>
    <li class="active">%s %s %s submission log</li>
    """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
         component.name, component.version, component.label)
    page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

    html = \
    """
    <div class="col-lg-12">
      <h2 class="page-header">%s: <small>%s, submission log</small></h1>
      <pre>%s</pre>
    </div>
    """%(naming_schema.componentName(component.name), component.label, view_tools.toHTML("".join(component.lines)))
    page.addContent("<!--BODY_HOOK-->",html)
    ########################### Write
    page.write()
########################### 
# Make a component view page
def makeComponentViewPages(options, view, url, validation_name, version_tag):
    ########################### Imports
    from HTMLView import HTMLView
    from Canvas import Canvas
    import os
    import common_tools as tools
    ########################### Insert content
    canvas = Canvas()
    view.component_view(canvas)
    if options.testing:
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        base_dir = os.environ["NOVAANAVALID_WEB"]

    breadcrumb = \
    """
    <li><a href="%s/index.html">Validation</a></li>
    <li><a href="%s/index.html">%s %s</a></li>
    """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)

    html_view = HTMLView(view, breadcrumb=breadcrumb, reuse_histograms=options.reuse, verbose=options.verbose, testing=options.testing)
    html_view.write_component_html()    
########################### 
# Make a component submission log page
def makeComponentLogPages(options, parent_dir, component, validation_name, version_tag):
    ########################### Imports
    import PageMaker as PM
    import common_tools as tools
    import view_tools
    import os
    from BatchLog import BatchLog
    import naming_schema
    ########################### Insert content
    if options.testing:
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        base_dir = os.environ["NOVAANAVALID_WEB"]
    # get logs
    job_sub_id = tools.getJobID(component.lines)
    component.logs = BatchLog(tools.fetchLogs(job_sub_id, component.run_time))
    if job_sub_id == False: return

    log_types = ["cmd","log","out","err"]
    for log_type in log_types:
      url        = tools.directoryToURL(parent_dir)
      name = "%s_%s_%s_batch_%s.html"%(component.name, component.version, component.dataset, log_type)
      page       = PM.PageMaker(name, parent_dir, url)
      breadcrumb = \
      """
      <li><a href="%s/index.html">Validation</a></li>
      <li><a href="%s/index.html">%s %s</a></li>
      <li class="active">%s %s %s batch %s</li>
      """%(tools.directoryToURL(base_dir), url, validation_name, version_tag, \
           component.name, component.version, component.label, log_type)
      page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

      if hasattr(component.logs, log_type): content = getattr(component.logs, log_type)
      else:                                 content = [""]

      html = \
      """
      <div class="col-lg-12">
        <h2 class="page-header">%s: <small>%s, batch %s</small></h1>
        <pre>%s</pre>
      </div>
      """%(naming_schema.componentName(component.name), component.label, log_type, view_tools.toHTML("".join(content)))
      page.addContent("<!--BODY_HOOK-->",html)
      ########################### Write
      page.write()
########################### 
# Make a component submission log page
def makeControllerConfigPage(options, controller, parent_dir, validation_name, version_tag):
    ########################### Imports
    import PageMaker as PM
    import common_tools as tools
    import os
    ########################### Insert content
    url        = tools.directoryToURL(parent_dir)
    name       = "config.html"
    page       = PM.PageMaker(name, parent_dir, url)
    if options.testing:
        base_dir = os.environ["NOVAANAVALID_WEB_TEST"]
    else:
        base_dir = os.environ["NOVAANAVALID_WEB"]
    breadcrumb = \
    """
    <li><a href="%s/index.html">Validation</a></li>
    <li><a href="%s/index.html">%s %s</a></li>
    <li class="active">configuration</li>
    """%(tools.directoryToURL(base_dir), url, validation_name, version_tag)
    page.addContent("<!--BREADCRUMB_HOOK-->", breadcrumb)

    config = repr(controller.config)
    config = config.replace("u'","'")
    config = config.replace(',',',\n')

    html = \
    """
    <div class="col-lg-12">
      <h2 class="page-header">Configuration</h1>
      <pre>%s</pre>
    </div>
    """%(config)
    page.addContent("<!--BODY_HOOK-->",html)
    ########################### Write
    page.write()