ViewSTDOUT.py
Go to the documentation of this file.
1 ###########################
2 # print metric
3 def showMetric(metric):
4  if "convertSize" not in dir(): from ProductionTestTools import convertSize
5  if metric.user_cpu != None:
6  print "tools: User CPU time: %f s"%metric.user_cpu
7  if metric.system_cpu != None:
8  print "tools: System CPU time: %f s"%metric.system_cpu
9  if metric.peak_memory != None:
10  print "tools: Memory usage: %s"%metric.peak_memory
11  if metric.input_events != None:
12  print "tools: Input events: %i"%metric.input_events
13  if metric.output_events != None:
14  for i_out, output in enumerate(metric.output_name):
15  print "tools: Output name: %s"%metric.output_name[i_out]
16  print "tools: Output events: %i"%metric.output_events[i_out]
17  if metric.input_events != None:
18  try:
19  print "tools: Efficiency: %5.1f %%"%((float(metric.output_events[i_out])/float(metric.input_events)) * 100)
20  except:
21  pass
22  print "tools: Output size: %s"%convertSize(metric.output_size[i_out])
23  try:
24  print "tools: Output event size:%s"%convertSize(metric.output_size[i_out] / float(metric.output_events[i_out]))
25  except:
26  pass
27  if metric.db_usage != None:
28  print "tools: DB queries: %s"%repr(metric.db_usage.n_queries)
29  print "tools: vmem peak: %s"%repr(metric.db_usage.peakVMUsage())
30  print "tools: mem peak: %s"%repr(metric.db_usage.peakMUsage())
31  print "tools: sum rows: %s"%repr(metric.db_usage.sumRows())
32  print "tools: sum query time: %s"%repr(metric.db_usage.sumQueryTime())
33  print "tools: sum parsing time: %s"%repr(metric.db_usage.sumParsingTime())
34 ###########################
35 # print metrics summary
36 # I tried to make this simpler but it didn't work
37 def showMetricsSummaryComplex(chain, skip_unused_tiers=True):
38  if "Metric" not in dir(): import Metric
39  if "convertSize" not in dir(): from ProductionTestTools import convertSize
40 
41  # a dictionary of what to show
42  # the format is heading:(attribute of tier, attribute name, width padding, per input event, per output event, efficiency, is a size, is a vector)
43  d_to_show = { "tier" :(True, "short_name", 6, False, False, False, False, False),
44  "input events" :(False, "input_events", 0, False, False, False, False, False),
45  "user cpu [s]" :(False, "user_cpu", 0, False, True, False, False, False),
46  "system cpu [s]" :(False, "system_cpu", 0, False, True, False, False, False),
47  "memory" :(False, "peak_memory", 4, False, False, False, True, False),
48  "output name" :(False, "output_name", 5, False, False, False, False, True),
49  "output events" :(False, "output_events", 0, False, False, True, False, True),
50  "output size" :(False, "output_size", 4, False, True, False, True, True),
51  "db queries" :("db", "n_queries", 0, False, False, False, False, False),
52  "query time [ms]":("db", "sum_query_time", 0, False, False, False, False, False),
53  "parsing time [ms]":("db", "sum_parsing_time", 0, False, False, False, False, False),
54  }
55 
56  to_show = ["tier",
57  "input events",
58  "user cpu [s]",
59  #"system cpu [s]",
60  "memory",
61  "db queries",
62  "query time [ms]",
63  "parsing time [ms]",
64  "output name",
65  "output events",
66  "output size",
67  ]
68 
69  header = ""
70  for i_s,s in enumerate(to_show):
71  if (i_s == 0):
72  format_string = "%%-%is"%(len(s)+d_to_show[s][2])
73  else:
74  format_string = "%%%is"%(len(s)+d_to_show[s][2])
75  header += format_string%(s)
76 
77  if d_to_show[s][3]: header += " (/in evt)"
78  if d_to_show[s][4]: header += " (/out evt)"
79  if d_to_show[s][5]: header += " (eff [%])"
80 
81  header += " | "
82  header = header[:-3]
83 
84  lines = ["\ntools: --- Summary"]
85  lines.append("\ntools: Chain: %s"%chain.name)
86  lines.append(header)
87  line_break = len(header)*"-"
88  lines.append(line_break)
89 
90  for tier in chain.tiers:
91  if skip_unused_tiers and (tier.metric.run == False): continue
92  line = ""
93  found_vector = False
94  len_vector = None
95  for i_s, s in enumerate(to_show):
96  if (i_s == 0):
97  format_string = "%%-%is"%(len(s)+d_to_show[s][2])
98  else:
99  format_string = "%%%is"%(len(s)+d_to_show[s][2])
100 
101  if d_to_show[s][0] == "db":
102  if tier.metric.db_usage != None:
103  tier.metric.db_usage.refresh()
104  #print tier.metric
105  attr = getattr(tier.metric.db_usage, d_to_show[s][1])
106  else:
107  attr = "n/a"
108  elif d_to_show[s][0]:
109  attr = getattr(tier, d_to_show[s][1])
110  else:
111  attr = getattr(tier.metric, d_to_show[s][1])
112 
113  if d_to_show[s][7] and attr != None:
114  found_vector = True
115  if len_vector == None:
116  len_vector = len(attr)
117  else:
118  assert(len(attr) == len_vector)
119  attr = attr[0]
120 
121  #print s, attr
122  if d_to_show[s][6] and (attr != None):
123  line += format_string%convertSize(attr)
124  else:
125  if s == "output name":
126  line += format_string%(repr(attr)[-15:])
127  else:
128  line += format_string%repr(attr)
129 
130  # per input event
131  if d_to_show[s][3]:
132  if (tier.metric.input_events != None) and (attr != None):
133  if d_to_show[s][6]: line += " (%7s)"%convertSize((attr / float(tier.metric.input_events)))
134  else: line += " (%7.2f)"%(attr / float(tier.metric.input_events))
135  else:
136  line += " ( n/a)"
137  # per output event
138  if d_to_show[s][4]:
139  if (tier.metric.output_events != None) and (attr != None):
140  if d_to_show[s][6]: line += " (%8s)"%convertSize((attr / float(tier.metric.output_events[0])))
141  else: line += " (%8.2f)"%(attr / float(tier.metric.output_events[0]))
142  else:
143  line += " ( n/a)"
144  # efficiency per output event
145  if d_to_show[s][5]:
146  if (tier.metric.output_events != None) and (attr != None):
147  line += " (%7.2f)"%((attr / float(tier.metric.output_events[0]))*100)
148  else:
149  line += " ( n/a)"
150  line += " | "
151  line = line[:-3]
152  lines.append(line)
153 
154  if found_vector and len_vector > 1:
155  for i in range(1,len_vector):
156  line = ""
157  for i_s, s in enumerate(to_show):
158  if not d_to_show[s][7]:
159  line += " "*(len(s)+d_to_show[s][2])
160  if d_to_show[s][3]: line += " "*9
161  if d_to_show[s][4]: line += " "*11
162  if d_to_show[s][5]: line += " "*9
163  else:
164  format_string = "%%%is"%(len(s)+d_to_show[s][2])
165  if d_to_show[s][0]:
166  attr = getattr(tier, d_to_show[s][1])
167  else:
168  attr = getattr(tier.metric, d_to_show[s][1])
169 
170  print s, attr, i
171  attr = attr[i]
172 
173  if d_to_show[s][6] and (attr != None):
174  line += format_string%convertSize(attr)
175  else:
176  if s == "output name":
177  line += format_string%(repr(attr)[-15:])
178  else:
179  line += format_string%repr(attr)
180 
181  if d_to_show[s][3]:
182  if (tier.metric.input_events != None) and (attr != None):
183  if d_to_show[s][6]: line += " (%7s)"%convertSize((attr / float(tier.metric.input_events)))
184  else: line += " (%7.2f)"%(attr / float(tier.metric.input_events))
185  else:
186  line += " ( n/a)"
187 
188  if d_to_show[s][4]:
189  if (tier.metric.input_events != None) and (attr != None):
190  if d_to_show[s][6]: line += " (%8s)"%convertSize((attr / float(tier.metric.output_events[0])))
191  else: line += " (%8.2f)"%(attr / float(tier.metric.output_events[0]))
192  else:
193  line += " ( n/a)"
194 
195  if d_to_show[s][5]:
196  if (tier.metric.input_events != None) and (attr != None):
197  line += " (%7.2f)"%((attr / float(tier.metric.output_events[0]))*100)
198  else:
199  line += " ( n/a)"
200  line += " | "
201  line = line[:-3]
202  lines.append(line)
203 
204  lines.append(line_break)
205  lines.append("\n")
206  for line in lines: print line
207 ###########################
208 # print metrics summary
209 def showMetricsSummary(chain, skip_unused_tiers=True):
210  if "Metric" not in dir(): import Metric
211  if "convertSize" not in dir(): from ProductionTestTools import convertSize
212 
213  header = "%-10s | %8s | %19s | %10s | %10s | %8s | %15s | %20s | %23s"%\
214  ("tier",
215  "in evts",
216  "usr cpu (/in evt) [s]",
217  "memory",
218  "db queries",
219  "time [s]",
220  #"parsing time [s]",
221  "child",
222  "events (eff)",
223  "size (/out evt)")
224 
225  lines = ["\ntools: --- Summary"]
226  lines.append("\ntools: Chain: %s"%chain.name)
227  lines.append(header)
228  line_break = len(header)*"-"
229  lines.append(line_break)
230 
231  for tier in chain.tiers:
232  if skip_unused_tiers and (tier.metric.run == False): continue
233 
234  # name
235  t = tier.short_name
236  line = "%-10s | "%t
237  metric = tier.metric
238  #print metric
239 
240  # check on if this tier was run
241  # This should probably be extended to check is this tier completed.
242  if (tier.metric.run == False):
243  line += "Not run"
244  lines.append(line)
245  continue
246 
247  # input/used events
248  in_events = None
249  if (metric.used_events == None) and (metric.input_events == None):
250  t = "n/a"
251  elif metric.used_events and metric.input_events:
252  t = "%6i"%min(metric.used_events,metric.input_events)
253  in_events = min(metric.used_events,metric.input_events)
254  elif metric.used_events:
255  t = "%6i"%metric.used_events
256  in_events = metric.used_events
257  elif metric.input_events:
258  t = "%6i"%metric.input_events
259  in_events = metric.input_events
260  line += " %6s | "%(t)
261 
262  # user cpu
263  if metric.user_cpu == None:
264  cpu = "n/a"
265  e_cpu = "n/a"
266  else:
267  cpu = "%5.2f"%metric.user_cpu
268  if (in_events != None) and (in_events != 0):
269  e_cpu = "%5.2f"%(metric.user_cpu/float(in_events))
270  else:
271  e_cpu = "n/a"
272  line += " %8s (%8s) | "%(cpu,e_cpu)
273 
274  # memory
275  if metric.peak_memory == None: mem = "n/a"
276  else: mem = "%10s"%convertSize(metric.peak_memory)
277  line += "%10s | "%(mem)
278 
279  # db queries
280  if (metric.db_usage != None) and (metric.db_usage.n_queries != None):
281  q = "%8i"%metric.db_usage.n_queries
282  else:
283  q = "n/a"
284  line += "%10s | "%q
285 
286  # query time
287  if (metric.db_usage != None):
288  q = "%8.2f"%(metric.db_usage.sumQueryTime()/1000)
289  else:
290  q = "n/a"
291  line += "%8s | "%q
292 
293  # parsing time
294  #if (metric.db_usage != None):
295  #q = "%16.2f"%(metric.db_usage.sumParsingTime()/1000)
296  #else:
297  #q = "n/a"
298  #line += "%16s | "%q
299 
300  if metric.output_name != None:
301  width = 0
302  for i, out in enumerate(metric.output_name):
303  # For the secondary children start a new line in the
304  # table which is indented to the depth of the first.
305  # Take the indentation depth from the first child.
306  if (i == 0): width = len(line)
307  if (i != 0):
308  line += "\n"
309  for j in range(width):
310  if header[j] == "|":
311  line += "|"
312  else:
313  line += " "
314  # name
315  line += "%15s | "%out[-15:]
316 
317  # events
318  if metric.output_events[i] == None:
319  e = "n/a"
320  eff = "n/a"
321  else:
322  e = "%5i"%metric.output_events[i]
323  if (in_events != None) and (in_events != 0):
324  eff = "%5.2f"%(float(metric.output_events[i])/float(in_events))
325  else:
326  eff = "n/a"
327  line += " %8s (%8s) | "%(e,eff)
328 
329  # size
330  if metric.output_size[i] == None:
331  e = "n/a"
332  eff = "n/a"
333  else:
334  e = "%8s"%convertSize(metric.output_size[i])
335  if (metric.output_events != None) and (metric.output_events[i] != 0):
336  eff = "%8s"%convertSize(float(metric.output_size[i])/float(metric.output_events[i]))
337  else:
338  eff = "n/a"
339  line += " %10s (%10s)"%(e,eff)
340 
341  else:
342  line += " No children"
343 
344  lines.append(line)
345 
346 
347  lines.append(line_break)
348  lines.append("\n")
349  for line in lines: print line
350 ###########################
351 # dump log files to folder
352 def dumpLogFilesToFolder(chain, folder, skip_unused_tiers=True):
353  import os
354  try:
355  os.stat(folder)
356  except:
357  print "vstdo: Making folder: %s"%folder
358  os.mkdir(folder)
359  for tier in chain.tiers:
360  if skip_unused_tiers and (tier.metric.run == False): continue
361  log = stripTimingMarkersFromLog(tier.log)
362  output_file = open("%s/%s_%s.log"%(folder,chain.name,tier.short_name), "wb")
363  output_file.write(log)
364  output_file.close()
365 ###########################
366 # dump FHiCL files to folder
367 def dumpFHiCLFilesToFolder(chain, folder, skip_unused_tiers=True):
368  import os
369  try:
370  os.stat(folder)
371  except:
372  print "vstdo: Making folder: %s"%folder
373  os.mkdir(folder)
374  for tier in chain.tiers:
375  if skip_unused_tiers and (tier.metric.run == False): continue
376  if tier.fcl == False:
377  fcl = "No FHiCL file used"
378  else:
379  fcl = "# FHiCL file: %s\n"%tier.fcl_path
380  fcl += tier.full_fcl
381  output_file = open("%s/%s_%s.fcl"%(folder,chain.name,tier.short_name), "wb")
382  output_file.write(fcl)
383  output_file.close()
384 ###########################
385 # Strip a log of timing markers
386 def stripTimingMarkersFromLog(log, remove_markup=True):
387  return_log = ""
388  for line in log.split("\n"):
389  if "TimeMarker" in line: continue
390  if remove_markup:
391  line = line.replace("<","&lt;")
392  line = line.replace(">","&gt;")
393  return_log += "%s\n"%line
394  return return_log
def showMetric(metric)
print metric
Definition: ViewSTDOUT.py:3
def showMetricsSummaryComplex(chain, skip_unused_tiers=True)
print metrics summary I tried to make this simpler but it didn&#39;t work
Definition: ViewSTDOUT.py:37
def dumpLogFilesToFolder(chain, folder, skip_unused_tiers=True)
dump log files to folder
Definition: ViewSTDOUT.py:352
def convertSize(size)
Parse file sizes into human readable format.
procfile open("FD_BRL_v0.txt")
static float min(const float a, const float b, const float c)
Definition: absgeo.cxx:45
def showMetricsSummary(chain, skip_unused_tiers=True)
print metrics summary
Definition: ViewSTDOUT.py:209
def dumpFHiCLFilesToFolder(chain, folder, skip_unused_tiers=True)
dump FHiCL files to folder
Definition: ViewSTDOUT.py:367
TDirectory * dir
Definition: macro.C:5
def stripTimingMarkersFromLog(log, remove_markup=True)
Strip a log of timing markers.
Definition: ViewSTDOUT.py:386
assert(nhit_max >=nhit_nbins)