time_estimates.py
Go to the documentation of this file.
1 #!/bin/env python
2 from __future__ import print_function
3 from __future__ import division
4 from builtins import str
5 from builtins import map
6 from past.utils import old_div
7 import os, sys, re
8 import bz2
9 from multiprocessing import Pool
10 from multiprocessing.dummy import Pool as ThreadPool
11 
12 try:
13  from fnal import *
14 except ImportError:
15  def FetchLog(filename):
16  pass
17 
18 def mean(numbers):
19  return float(sum(numbers)) / max(len(numbers), 1)
20 
21 
22 
23 nfiles = 20
24 
25 definitions = {
26  "FD Numi data": "prod_pid_R16-03-03-prod2reco.d_fd_numi_fhc_full_v1_goodruns_prod2-snapshot",
27  "FD Numi mc nonswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_nonswap_fhc_nova_v08_full_v1_prod2-snapshot",
28  "FD Numi mc fluxswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_fluxswap_fhc_nova_v08_full_v1_prod2-snapshot",
29  "FD Numi mc tauswap": "prod_pid_R16-03-03-prod2reco.a_fd_genie_tau_fhc_nova_v08_full_v1_prod2-snapshot",
30  "FD Numi mc overlay": "prod_pid_R16-03-03-mixedcalib.a_fd_genie_nonswap_fhc_nova_v08_full_v1_cosmic-overlay-removenoise",
31  "FD Cosmic data": "prod_pidpart_R16-03-03-prod2reco.a_fd_cosmic_full_nueveto_v1_goodruns",
32  "ND Numi data": "prod_pid_R16-03-03-prod2reco.a_nd_numi_fhc_full_v1_goodruns_prod2-snapshot",
33  "ND Numi MC": "prod_pidpart_R16-03-03-prod2reco.a_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
34  "ND Numi MC LEM": "prod_pid_R16-03-03-prod2reco.d_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
35  "ND Numi MC fluxswap": "prod_pid_R16-03-03-prod2reco.d_nd_genie_fluxswap_nogenierw_fhc_nova_v08_full_v1",
36  "ND Numi MC overlay": "prod_pid_R16-03-03-mixedcalib.a_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1_cosmic-overlay-removenoise",
37  "ND RHC MC": "prod_pid_R16-03-03-prod2reco.h_nd_genie_nonswap_genierw_rhc_nova_v08_epoch4a_v1",
38  "MRE data artdaq": "prod_mreartdaq_R16-03-03-prod2reco.b_nd_numi_fhc_full_v1",
39  "MRE data pid": "prod_mrepid_R16-03-03-prod2reco.b_nd_numi_fhc_full_v1",
40  "MRE MC artdaq": "prod_mreartdaq_R16-03-03-prod2reco.b_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1",
41  "MRE MC pid": "prod_mrepid_R16-03-03-prod2reco.b_nd_genie_nonswap_genierw_fhc_nova_v08_full_v1"
42 # "ND Numi MC gen": "prod_artdaq_R16-02-11-prod2genie.a_nd_genie_nonswap_nogenierw_fhc_nova_v08_epoch1-3c_v1_prod2-snapshot",
43 # "FD Numi MC nonswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_nonswap_fhc_nova_v08_full_v1_prod2-snapshot",
44 # "FD Numi MC fluxswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_fluxswap_fhc_nova_v08_full_v1_prod2-snapshot",
45 # "FD Numi MC tauswap gen": "prod_artdaq_R16-02-11-prod2genie.a_fd_genie_tau_fhc_nova_v08_full_v1_prod2-snapshot"
46 }
47 
48 names = sorted(definitions.keys())
49 
50 ########################################################
51 
52 files = {}
53 for name, defname in list(definitions.items()):
54  filelist = name.replace(" ","_")+".files"
55  if os.path.exists(filelist):
56  files[name] = []
57  fin = open(filelist,"r")
58  for line in fin:
59  files[name].append(line.strip())
60  else:
61  if not samweb:
62  import samweb_client
63  samweb = samweb_client.SAMWebClient(experiment='nova')
64  print("Listing files for",name)
65  files[name] = samweb.listFiles("defname: {0} with limit {1:d}".format(defname, nfiles))
66  fout = open(filelist,"w")
67  for f in files[name]:
68  print(f, file=fout)
69 
70 
71 ########################################################
72 
73 logfiles = {}
74 alllogfiles = []
75 for name, filelist in list(files.items()):
76  logfiles[name] = []
77  for filename in filelist:
78  logfilename = filename.replace(".root",".log.bz2")
79  logfiles[name].append(logfilename)
80  alllogfiles.append(logfilename)
81 
82 
83 pool = ThreadPool(8)
84 results = pool.map(FetchLog, alllogfiles)
85 pool.close()
86 pool.join()
87 
88 
89 ########################################################
90 
91 redigit = re.compile("\d+\.\d*")
92 maxlen = str(max(list(map(len, names))))
93 
94 print(("{0:"+maxlen+"} {1:>5} {2:>5} {3:>5} {4:>5}").format("Sample", "Total", "CVN", "LEM", "Others"))
95 for name in names:
96  logfilelist = logfiles[name]
97  CPUPerEvent = []
98  LEMPerEvent = []
99  CVNPerEvent = []
100  OtherPerEvent = []
101  CPUTotal = 0
102  nevents = 0
103  for logfilename in logfilelist:
104  summary_found = False
105 
106  if not os.path.exists(logfilename):
107  continue
108  f = bz2.BZ2File(logfilename)
109  try:
110  for line in f:
111  if "TrigReport Events" in line:
112  matches = re.findall("\d+", line)
113  nevents = float(matches[0])
114 
115  if "TimeReport" not in line:
116  continue
117  line = line.strip()
118 
119  if "TimeReport CPU/event" in line:
120  matches = redigit.findall(line)
121  CPUPerEvent.append(float(matches[0]))
122  summary_found = True
123 
124  if re.search("lem|lemmre$", line):
125  matches = redigit.findall(line)
126  if len(matches) == 4:
127  LEMPerEvent.append(float(matches[1]))
128 
129  if re.search("cvneval|cvnevalmre$", line):
130  matches = redigit.findall(line)
131  if len(matches) == 4:
132  CVNPerEvent.append(float(matches[1]))
133 
134  if "TimeReport CPU" in line:
135  matches = redigit.findall(line)
136  CPUTotal = float(matches[0])
137 
138  if not summary_found and CPUTotal > 0:
139  CPUPerEvent.append(old_div(CPUTotal,nevents))
140 
141  except EOFError:
142  continue
143  if len(CPUPerEvent) == 0:
144  continue
145  print(("{0:"+maxlen+"} {1:5.2f} {2:5.2f} {3:5.2f} {4:5.2f}").format(name,
146  mean(CPUPerEvent), mean(CVNPerEvent), mean(LEMPerEvent),
147  mean(CPUPerEvent) - mean(CVNPerEvent) - mean(LEMPerEvent)))
148 
149 
150 
void append()
Definition: append.C:24
bool print
def mean(numbers)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
def FetchLog(filename)
Double_t sum
Definition: plot.C:31
T max(sqlite3 *const db, std::string const &table_name, std::string const &column_name)
Definition: statistics.h:68