ControllerView.py

Go to the documentation of this file.

import Controller
import common_tools as tools
import view_tools
from BatchLog import BatchLog
import os
import Canvas
import styles
import naming_schema
import cPickle
import datetime
import copy
from ROOT import TH1D, TH1F, TH2F, TH2D, TProfile2D
import samweb_client


class ControllerView():

    def __init__(self, controller, verbose=False, testing=False):
        self.verbose    = verbose
        self.controller = controller
        self.testing    = testing
        if self.testing:
            self.plot_base  = "%s/figures/%s/v%s/"%(os.environ["NOVAANAVALID_GRID_TEST"],
                                                   self.controller.name,
                                                   self.controller.version)
        else:
            self.plot_base  = "%s/figures/%s/v%s/"%(os.environ["NOVAANAVALID_GRID"],
                                                   self.controller.name,
                                                   self.controller.version)
        tools.mkdir(self.plot_base)
        # if self.verbose: print "view : ControllerView plot base: %s"%self.plot_base
        self.find_component_types()
        self.creation_time = datetime.datetime.now()
        self.component_view_run = False
        self.standard_true_suffixes = naming_schema.standard_truth_suffixes

        self.plot_groups = {}

    def component_view(self, canvas):
        """
        This will compose a view of each component of the controller
        individually.

        These plots are order by category (reco, true, mixed), with
        one page provided for each dataset and one page provided
        overlaying all datasets.

        Components that are not complete are skipped 
        """
        if self.component_view_run:
            if self.verbose: print "view :     component view has already been run"
            return

        if self.verbose: print "view : Controller %s has %i component types"%(self.controller.name,len(self.component_type_keys))
        self.plot_information = {}

        for component_type in self.component_type_keys:
            if self.verbose:
                print "view :     Component type: %s, has %i instances"%(component_type, len(self.component_types[component_type]))

            self.find_complete_components(component_type)
            if self.verbose: print "view :     found %i complete components"%len(self.complete_components)

            if len(self.complete_components):
                self.plot_information[component_type] = {}

                self.plot_groups[component_type] = []
                self.draw_component_plots(canvas, component_type)

            if component_type in self.plot_groups.keys():
                if self.verbose: print "view :     plot groups found: %s"%repr(self.plot_groups[component_type])

        self.component_view_run = True

    def draw_component_plots(self, canvas, component_type):
        plots = {}
        open_files = []
        plot_names = []

        # loop over the current complete components extracting plots and drawing them stand-alone
        for component in self.complete_components:
            plots[component.dataset] = {}
            directories = hasattr(component,"directories") and component.directories
            plot_dict, open_root_file = tools.openPlots(component.root_file, directories=directories)
            if self.verbose: print "view : component %s, dataset: %s, opened %i histograms"%(component.name, component.dataset, len(plot_dict.keys()))
            open_files.append(open_root_file)

            component.plot_dir = "%s/%s/v%s/%s"%(self.plot_base, component.name, component.version, component.dataset)
            tools.mkdir(component.plot_dir)

            # collect the POT from the file
            if "sensitivity" in component.name:
                component.pot = False
                area_norm     = False
            else:
                component.pot = self.get_pot(plot_dict)
                area_norm     = True
            if self.verbose: print "view :     PoT: %.2e"%component.pot

            # first draw all plots
            for plot_name in plot_dict.keys():
                if ("true" in plot_name) and ("data" in naming_schema.datasetToLabel(component.dataset)): continue

                plot_names.append(plot_name)
                plots[component.dataset][plot_name] = plot_dict[plot_name]
                # don't draw truth components individually
                if (tools.getHistogramTrueLevel(plot_name) != ""): continue

                if self.verbose: print "view :     plot name: %s"%(plot_name)
                
                if plot_name.count("_vs_") > 1:
                    this_area_norm = False
                    this_pot_norm  = False
                else:
                    this_area_norm = area_norm
                    this_pot_norm  = component.pot

                # draw the plot
                path = self.draw_plots( [plot_dict[plot_name]], component.plot_dir, plot_name, [component], canvas)
                if (this_area_norm):
                    self.draw_plots( [plot_dict[plot_name]], component.plot_dir, plot_name, [component], canvas, normalisation="area")
                if (this_pot_norm):
                    self.draw_plots( [plot_dict[plot_name]], component.plot_dir, plot_name, [component], canvas, normalisation="pot")

                # fill the plot information
                self.plot_information[component_type][path] = self.get_plot_information([plot_dict[plot_name]], [component], area_norm=this_area_norm, pot_norm=this_pot_norm)
                if self.plot_information[component_type][path].group not in self.plot_groups[component.name]: self.plot_groups[component.name].append(self.plot_information[component_type][path].group)

                # If we have the standard truth suffixes, draw them
                if (not self.has_standard_truth_types(plot_name, plot_dict.keys())): continue
                if self.verbose: print "view :       - has standard truth components"
                self.plot_information[component_type][path].true_components = True

                sub_plots   = []
                sub_comps   = []
                fraction_of = []
                for suffix in self.standard_true_suffixes:
                    sub_plots.append(plot_dict["%s%s"%(plot_name,suffix)])
                    sub_comps.append(component)
                    fraction_of.append(0)
                plot_name = "%s-truth_components"%plot_name
                plots[component.dataset][plot_name] = sub_plots

                self.draw_plots( sub_plots, component.plot_dir, plot_name, sub_comps, canvas)
                if (this_area_norm):
                    self.draw_plots( sub_plots, component.plot_dir, plot_name, sub_comps, canvas, normalisation="area", fraction_of=fraction_of)
                if (this_pot_norm):
                    self.draw_plots( sub_plots, component.plot_dir, plot_name, sub_comps, canvas, normalisation="pot")

        plot_names = list(set(plot_names))
        # next draw them overlaid
        self.group_components()
        for plot_name in plot_names:
            for component_group in self.grouped_components:
                components = []
                for component in component_group:
                    if (plot_name not in plots[component.dataset].keys()): continue
                    if ("true" in plot_name) and ("data" in naming_schema.datasetToLabel(component.dataset)): continue
                    components.append(component)
                if len(components) < 2: continue

                to_show = [plots[c.dataset][plot_name] for c in components]
                joined_name = "_".join([c.dataset for c in components])
                if len(joined_name) > 25: joined_name = joined_name[:25]

                overlay_dir = "%s/%s/v%s/%s"%(self.plot_base, component.name, component.version, joined_name)
                tools.mkdir(overlay_dir)

                path = self.draw_plots( to_show, overlay_dir, plot_name, [c for c in components], canvas, ratios=True)
                self.draw_plots( to_show, overlay_dir, plot_name, [c for c in components], canvas, normalisation="area", ratios=True)
                if (component.pot):
                    self.draw_plots( to_show, overlay_dir, plot_name, [c for c in components], canvas, normalisation="pot", ratios=True)

                # fill the plot information
                self.plot_information[component_type][path] = self.get_plot_information(to_show, components)
                if self.plot_information[component_type][path].group not in self.plot_groups[component.name]: self.plot_groups[component.name].append(self.plot_information[component_type][path].group)

                # If any components have truth sub-divisions add these
                any_truth     = False
                truth_to_show = []
                comps         = []
                fraction_of   = []
                for i,component in enumerate(components):
                    component = copy.copy(component)
                    if ("%s-truth_components"%plot_name in plots[component.dataset].keys()):
                        any_truth = True
                        if self.verbose: print "view :     plot: %s, component: %s has truth components"%(plot_name, component.dataset)
                        
                        truth_plots = plots[component.dataset]["%s-truth_components"%plot_name]
                        component.true_components = True
                        for j,plot in enumerate(truth_plots):
                            truth_to_show.append(plot)
                            fraction_of.append(i)
                            comps.append(component)
                    else:
                        truth_to_show.append(plots[component.dataset][plot_name])
                        fraction_of.append(i)
                        comps.append(component)

                if any_truth:
                    self.plot_information[component_type][path].true_components = True
                    plot_name = "%s-truth_components"%(plot_name)                    
                    self.draw_plots( truth_to_show, overlay_dir, plot_name, comps, canvas)
                    self.draw_plots( truth_to_show, overlay_dir, plot_name, comps, canvas, normalisation="area", fraction_of=fraction_of)
                    if (component.pot):
                        self.draw_plots( truth_to_show, overlay_dir, plot_name, comps, canvas, normalisation="pot")


        for open_file in open_files:
            open_file.Close()

    def has_standard_truth_types(self, name, all_names):
        for suffix in self.standard_true_suffixes:
            if "%s%s"%(name,suffix) not in all_names: return False
        return True

    def get_plot_information(self, plots, components, area_norm=False, pot_norm=True):
        info = tools.getPlotStats(plots, area_norm=area_norm)

        info.datasets           = [c.dataset for c in components]
        info.names              = [naming_schema.datasetToLabel(c.label) for c in components]
        info.component_name     = components[0].name
        info.component_version  = components[0].version
        info.controller_name    = self.controller.name
        info.controller_version = self.controller.version
        if len(components) ==1:
            info.group          = naming_schema.datasetToLabel(components[0].label)
        elif "_nd_" in components[0].dataset:
            info.group          = "ND overlays"
        else:
            info.group          = "FD overlays"

        if components[0].pot and pot_norm: info.normalisations.append(("PoT","pot_norm_"))
        return info

    def draw_plots(self, \
                   plots, output_dir, plot_name, components, canvas, \
                   normalisation=False, fraction_of=False, \
                   ratios=False, \
                   ratio_ranges=[ ("",[]), ("zoom_",[0.81,1.19]) ], \
                   log=True):
        draw_config = styles.parseDrawingConfig(plot_name, components)
        try:
            latex_label = naming_schema.recoLevelName(tools.getHistogramRecoLevel(plot_name))
        except:
            latex_label = ""

        if (normalisation == "area"):
            new_plots = []
            for i,plot in enumerate(plots):
                if (type(plot) not in [TH1F, TH1D]): return
                base_area = 100.
                if fraction_of:
                    try:
                        area = base_area * (plot.GetSumOfWeights() / plots[fraction_of[i]].GetSumOfWeights())
                    except ZeroDivisionError:
                        area = 0.
                else:
                    area = base_area
                new_plots.append(view_tools.normaliseHistogram(plot, area=area))
            plots = new_plots
            draw_config.axis_labels = "%s;Normalised Entries [%%]"%(draw_config.axis_labels.split(";")[0])
            plot_name = "area_norm_%s"%(plot_name)
        
        elif (normalisation == "pot"):
            new_plots = []
            for i,plot in enumerate(plots):
                if (type(plot) not in [TH1F, TH1D]): return
                try:
                    area = plot.GetSumOfWeights() * (tools.STANDARD_POT / components[i].pot)
                except ZeroDivisionError:
                    area = 0.
                new_plots.append(view_tools.normaliseHistogram(plot, area=area))
            plots = new_plots
            draw_config.axis_labels = "%s;Entries / %.0e PoT"%(draw_config.axis_labels.split(";")[0],tools.STANDARD_POT)
            plot_name = "pot_norm_%s"%(plot_name)

        elif (normalisation):
            print "view : Unknown normalisation: %s"%repr(normalisation)
            exit(1)
       
        name = "%s/%s"%(output_dir,plot_name)

        canvas.draw(plots, 
                    draw_config.labels, 
                    draw_config.axis_labels, 
                    colours=draw_config.colours, 
                    lines=draw_config.lines,
                    markers=draw_config.markers,
                    draw_options=draw_config.draw_options,
                    latex_label=latex_label,
                    save_as=name)

        if log:
            log_name = "%s/log_%s"%(output_dir,plot_name)
            canvas.draw(plots, 
                        draw_config.labels, 
                        draw_config.axis_labels, 
                        colours=draw_config.colours, 
                        lines=draw_config.lines,
                        markers=draw_config.markers,
                        draw_options=draw_config.draw_options,
                        latex_label=latex_label,
                        log_y=True,
                        save_as=log_name)

        if ratios and (len(plots) > 1) and (type(plots[0]) in [TH1F, TH1D, TH2D, TH2F, TProfile2D]):
            ratios = view_tools.makeRatioHistograms(plots[1:],[plots[0]])
            if (normalisation == "area"):   prefix = "Area normalised ratio"
            elif (normalisation == "pot"):  prefix = "PoT normalised ratio"
            else:                           prefix = "Ratio"
            ratio_axis_labels = "%s;%s to %s"%(draw_config.axis_labels.split(";")[0], prefix, draw_config.labels[0])
            
            for ratio_range in ratio_ranges:
                prefix  = ratio_range[0]
                if type(plots[0]) in [TH1F, TH1D]:
                    y_range = ratio_range[1]
                    z_range = []
                else:
                    y_range = []
                    z_range = ratio_range[1]

                if (len(ratios) == 1) and (type(ratios[0]) in [TH2D, TH2F, TProfile2D]):
                    draw_config.draw_options[0] = "COLZ"

                ratio_name = "%s/ratio_%s%s"%(output_dir,prefix,plot_name)
                canvas.draw(ratios, 
                            draw_config.labels[1:], 
                            ratio_axis_labels,
                            y_range=y_range,
                            z_range=z_range,
                            colours=draw_config.colours[1:], 
                            lines=draw_config.lines[1:], 
                            markers=draw_config.markers[1:],
                            draw_options=draw_config.draw_options[1:],
                            latex_label=latex_label,
                            save_as=ratio_name)
        return name
            

    def get_pot(self, plot_dict):
        pot_name = "meta-TotalPOT"
        if pot_name in plot_dict.keys():
            return plot_dict[pot_name].GetBinContent(1)
        return False

    def component_status(self, component_type):
        for i_c,component in enumerate(self.component_types[component_type]):
            if self.verbose: print "view :         [%i]: dataset: %s"%(i_c, component.dataset)

            self.check_log_status(component)
            self.check_root_files(component)

    def find_complete_components(self, component_type):
        self.complete_components = []
        for i_c,component in enumerate(self.component_types[component_type]):
            if self.verbose: print "view :         [%i]: dataset: %s"%(i_c, component.dataset)

            if component.sam:
                if (not self.check_sam_status(component)): continue
            else:
                if (not self.check_log_status(component)): continue
            if (not self.check_root_files(component)): continue

            self.complete_components.append(component)

    def check_root_files(self, component):
        # for now only accept components that produce exactly one root file
        root_files = tools.findRootOutput(component, testing=self.testing)
        if len(root_files) != 1:
            print "view: component %s, has %i root files"%(component.name, len(root_files))
            tools.findRootOutput(component, verbose=True, testing=self.testing)
            return False
        # to avoid repeated calls lets assign the root file to the component
        component.root_file = root_files[0]
        return True

    def check_log_status(self, component):
        job_sub_id = tools.getJobID(component.lines)
        logs = BatchLog(tools.fetchLogs(job_sub_id, component.run_time))
        
        if (logs.log_status() != "completed") or (logs.err_status() != "no error"):
            # print "view :         component hasn't completed successfully"
            print "view :           - log status: ",logs.log_status()
            print "view :           - err status: ",logs.err_status()
            return False
        
        return True

    def check_sam_status(self, component):
        # has the SAM job completed
        if (not hasattr(component, "sam_view")):
            component.setup_sam_view()

        sam_status = component.get_sam_status(self.sam)
        if (sam_status != "complete") and ("ended complete" not in sam_status):
            component.sam_view.print_summary(self.sam)
            print "view : SAM status: ",sam_status
            return False 

        # have the output files been merged
        if self.verbose: print "view :     Merge status: %s"%component.get_sam_merge_status()
        if component.get_sam_merge_status() == "complete":
            return True

        return False

    def find_component_types(self):
        """
        return a mapping of component types to components
        """
        self.component_types = {}
        self.sam = False
        for component in self.controller.components:
            if component.name in ["hello_world"]: continue
            if component.name not in self.component_types.keys():
                self.component_types[component.name] = []
            self.component_types[component.name].append(component)
            if hasattr(component, "sam") and component.sam and (not self.sam):
                self.sam = samweb_client.SAMWebClient("nova")
            
        self.component_type_keys = self.component_types.keys()
        self.component_type_keys.sort()

    def group_components(self):
        """
        this looks at the current complete components and groups them
        """
        self.grouped_components = []
        # ND and FD
        nd_components = [c for c in self.complete_components if "_nd_" in c.dataset]
        if len(nd_components): self.grouped_components.append(nd_components)

        fd_components = [c for c in self.complete_components if "_fd_" in c.dataset]
        if len(fd_components): self.grouped_components.append(fd_components)
    
    def find_plot_types(self, plots):
        """
        plots can have different truth levels and different reco levels, group them here
        """
        plots.sort()
        self.plot_types  = []
        self.reco_levels = []
        self.true_levels = []
        for plot in plots:
            category   = tools.getHistogramCategory(plot)
            observable = tools.getHistogramObservable(plot)
            plot_type = "%s-%s"%(category,observable)
            if (plot_type not in self.plot_types): self.plot_types.append(plot_type)
            reco_level = tools.getHistogramRecoLevel(plot)
            if (reco_level not in self.reco_levels): self.reco_levels.append(reco_level)
            true_level = tools.getHistogramTrueLevel(plot)
            if (true_level not in self.true_levels): self.true_levels.append(true_level)
        if self.verbose: print "view :     found %i plot types, %i reco-levels and %i true-levels"%(len(self.plot_types),len(self.reco_levels),len(self.true_levels))

    def write(self, output_name):
        self.file_name = output_name
        print "view : Saving as: %s"%self.file_name
        cPickle.dump(self,open(self.file_name,"w"))