time_estimates.py

Go to the documentation of this file.

#!/bin/env python
from __future__ import print_function
from __future__ import division
from builtins import str
from builtins import map
from past.utils import old_div
import os, sys, re
import bz2
from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool

try:
    from fnal import *
except ImportError:
    def FetchLog(filename):
        pass

def mean(numbers):
    return float(sum(numbers)) / max(len(numbers), 1)



nfiles = 20

definitions = {
    "FD Numi data": "prod_pid_R16-03-03-prod2reco.d_fd_numi_fhc_full_v1_goodruns_prod2-snapshot",
    "FD Numi mc nonswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_nonswap_fhc_nova_v08_full_v1_prod2-snapshot",
    "FD Numi mc fluxswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_fluxswap_fhc_nova_v08_full_v1_prod2-snapshot",
    "FD Numi mc tauswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_tau_fhc_nova_v08_full_v1_prod2-snapshot",
    "FD Numi mc overlay": "prod_pid_R16-03-03-mixedcalib.a_fd_genie_nonswap_fhc_nova_v08_full_v1_cosmic-overlay-removenoise",
    "FD Cosmic data": "prod_pidpart_R16-03-03-prod2reco.a_fd_cosmic_full_nueveto_v1_goodruns",
    "ND Numi data": "prod_pid_R16-03-03-prod2reco.a_nd_numi_fhc_full_v1_goodruns_prod2-snapshot",
    "ND Numi MC": "prod_pidpart_R16-03-03-prod2reco.a_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
    "ND Numi MC LEM": "prod_pid_R16-03-03-prod2reco.d_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
    "ND Numi MC fluxswap": "prod_pid_R16-03-03-prod2reco.d_nd_genie_fluxswap_nogenierw_fhc_nova_v08_full_v1",
    "ND Numi MC overlay": "prod_pid_R16-03-03-mixedcalib.a_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1_cosmic-overlay-removenoise",
    "ND RHC MC": "prod_pid_R16-03-03-prod2reco.h_nd_genie_nonswap_genierw_rhc_nova_v08_epoch4a_v1",
    "MRE data artdaq": "prod_mreartdaq_R16-03-03-prod2reco.b_nd_numi_fhc_full_v1",
    "MRE data pid": "prod_mrepid_R16-03-03-prod2reco.b_nd_numi_fhc_full_v1",
    "MRE MC artdaq": "prod_mreartdaq_R16-03-03-prod2reco.b_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1",
    "MRE MC pid": "prod_mrepid_R16-03-03-prod2reco.b_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1"
#    "ND Numi MC gen": "prod_artdaq_R16-02-11-prod2genie.a_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
#    "FD Numi MC nonswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_nonswap_fhc_nova_v08_full_v1_prod2-snapshot",
#    "FD Numi MC fluxswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_fluxswap_fhc_nova_v08_full_v1_prod2-snapshot",
#    "FD Numi MC tauswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_tau_fhc_nova_v08_full_v1_prod2-snapshot"
}

names = sorted(definitions.keys())

########################################################

files = {}
for name, defname in list(definitions.items()):
    filelist = name.replace(" ","_")+".files"
    if os.path.exists(filelist):
        files[name] = []
        fin = open(filelist,"r")
        for line in fin:
            files[name].append(line.strip())
    else:
        if not samweb:
            import samweb_client
            samweb = samweb_client.SAMWebClient(experiment='nova')        
        print("Listing files for",name)
        files[name] = samweb.listFiles("defname: {0} with limit {1:d}".format(defname, nfiles))
        fout = open(filelist,"w")
        for f in files[name]:
            print(f, file=fout)


########################################################
            
logfiles = {}
alllogfiles = []
for name, filelist in list(files.items()):
    logfiles[name] = []
    for filename in filelist:
        logfilename = filename.replace(".root",".log.bz2")
        logfiles[name].append(logfilename)
        alllogfiles.append(logfilename)


pool = ThreadPool(8)
results = pool.map(FetchLog, alllogfiles)
pool.close()
pool.join()


########################################################

redigit = re.compile("\d+\.\d*")
maxlen = str(max(list(map(len, names))))

print(("{0:"+maxlen+"}  {1:>5}  {2:>5}  {3:>5}  {4:>5}").format("Sample", "Total", "CVN", "LEM", "Others"))
for name in names:
    logfilelist = logfiles[name]
    CPUPerEvent = []
    LEMPerEvent = []
    CVNPerEvent = []
    OtherPerEvent = []
    CPUTotal = 0
    nevents = 0
    for logfilename in logfilelist:
        summary_found = False
        
        if not os.path.exists(logfilename):
            continue
        f = bz2.BZ2File(logfilename)
        try:
            for line in f:
                if "TrigReport Events" in line:
                    matches = re.findall("\d+", line)
                    nevents = float(matches[0])

                if "TimeReport" not in line:
                    continue
                line = line.strip()

                if "TimeReport CPU/event" in line:
                    matches = redigit.findall(line)
                    CPUPerEvent.append(float(matches[0]))
                    summary_found = True

                if re.search("lem|lemmre$", line):
                    matches = redigit.findall(line)
                    if len(matches) == 4:
                        LEMPerEvent.append(float(matches[1]))

                if re.search("cvneval|cvnevalmre$", line):
                    matches = redigit.findall(line)
                    if len(matches) == 4:
                        CVNPerEvent.append(float(matches[1]))

                if "TimeReport CPU" in line:
                    matches = redigit.findall(line)
                    CPUTotal = float(matches[0])

            if not summary_found and CPUTotal > 0:
                CPUPerEvent.append(old_div(CPUTotal,nevents))

        except EOFError:
            continue
    if len(CPUPerEvent) == 0:
        continue
    print(("{0:"+maxlen+"}  {1:5.2f}  {2:5.2f}  {3:5.2f}  {4:5.2f}").format(name, 
                                                                            mean(CPUPerEvent),  mean(CVNPerEvent),  mean(LEMPerEvent),
                                                                            mean(CPUPerEvent) - mean(CVNPerEvent) - mean(LEMPerEvent)))