Functions
ViewGoogleCharts Namespace Reference

Functions

def tierProjectionBarChart (chain, options)
 Projection bar charts. More...
 
def directoryProjectionBarChart (name, title, directory, path, unpickled_chains={}, verbose=False, rounding=False)
 Entire test Projection bar charts. More...
 
def samFileInfoSummaryCharts (definitions, path)
 SAM file summary info into a chart. More...
 
def stackedBarChart (name="name", legend=[], results=[], title="title", x_title="x", y_title="y", options="", s_options="", dashboard=False, checkbox=False, for_definitions=True, rounding=False)
 Simple stacked bar charts. More...
 
def stackedColumnChart (name="name", legend=[], results=[], title="title", x_title="y", y_title="y", options="", s_options="", dashboard=False, checkbox=False, rounding=False)
 Simple stacked column charts. More...
 

Function Documentation

def ViewGoogleCharts.directoryProjectionBarChart (   name,
  title,
  directory,
  path,
  unpickled_chains = {},
  verbose = False,
  rounding = False 
)

Entire test Projection bar charts.

Definition at line 22 of file ViewGoogleCharts.py.

References open(), Projections.projectionSizes(), Projections.projectionTimes(), and stackedBarChart().

22 def directoryProjectionBarChart(name,title,directory,path,unpickled_chains={},verbose=False,rounding=False):
23  head_html = ""
24  import cPickle
25  import glob
26  import Projections
27  import Assumptions
28  if verbose:
29  print "vwGC : directoryProjectionBarChart"
30  print "vwGC : name: %s"%name
31  print "vwGC : title: %s"%title
32  print "vwGC : directory: %s"%directory
33  print "vwGC : path: %s"%path
34  all_files = glob.glob("%s/*.pkl"%directory)
35  all_files.sort()
36  if verbose:
37  print "vwGC : - files found: %s"%repr(all_files)
38  files = [f for f in all_files if "info" not in f]
39  tiers = ['raw2root',
40  'genie', 'cry', 'rock', 'detector', 'mix',
41  'pchits',
42  'reco', 'pid', 'caf'
43  ]
44  colours = ["#08804C",
45  "#0A166B", "#111F88", "#1729B0", "#3B49B4", "#5C68C3",
46  "#6C0382",
47  "#9B0000", "#FF0000", "#FF6363"
48  ]
49  s_colours = "[COL]"
50  for colour in colours:
51  s_colours = s_colours.replace("COL","%s,COL"%repr(colour))
52  s_colours = s_colours.replace(",COL","")
53 
54  chains = []
55  for f in files:
56  if f in unpickled_chains.keys():
57  chains.append(unpickled_chains[f])
58  else:
59  chains.append(cPickle.load(open(f,"rb")))
60  times = {}
61  sizes = {}
62  for chain in chains:
63  these_tiers = chain.GetTierShortNames()
64  times[chain.name] = []
65  times[chain.name] = Projections.projectionTimes(chain, per_input_event=True)
66  if verbose: print "vwGC : - chain: %s, times: %s"%(chain.name,repr(times[chain.name]))
67  sizes[chain.name] = []
68  sizes[chain.name] = Projections.projectionSizes(chain, per_output_event=True)
69  if verbose: print "vwGC : - chain: %s, sizes: %s"%(chain.name,repr(sizes[chain.name]))
70 
71  # collect data
72  legend = ["tier"]
73  for tier in tiers: legend.append(tier)
74  options = "isStacked: true, colors: %s"%s_colours
75  s_options = "'isStacked': true, 'colors': %s"%s_colours
76 
77  time_per_event = []
78  time_per_file = []
79  total_time = []
80  size_per_event = []
81  size_per_file = []
82  total_size = []
83  S_TO_KILOWEEKS = 1. / (60.*60.*24.*7.*1000.)
84  S_TO_KILODAYS = 1. / (60.*60.*24.*1000.)
85  for chain in chains:
86  this_time_per_event = [chain.name.replace("_"," ")]
87  this_time_per_file = [chain.name.replace("_"," ")]
88  this_total_time = [chain.name.replace("_"," ")]
89  this_size_per_event = [chain.name.replace("_"," ")]
90  this_size_per_file = [chain.name.replace("_"," ")]
91  this_total_size = [chain.name.replace("_"," ")]
92 
93  these_tiers = chain.GetTierShortNames()
94  if verbose: print "vwGC : - chain: %s"%chain.name
95  for i_t, t in enumerate(tiers):
96  if verbose: print "vwGC : - tier: %s, %i"%(t,i_t)
97  # Get per event time
98  if t in these_tiers:
99  if verbose: print "vwGC : - index: ",these_tiers.index(t)
100  this_time = times[chain.name][these_tiers.index(t)]
101  else:
102  this_time = 0.
103  #Get per event size
104  if t in these_tiers:
105  this_size = sizes[chain.name][these_tiers.index(t)]
106  else:
107  this_size = 0.
108 
109  # Get events per file
110  if "%s:%s"%(chain.name,t) in Assumptions.events_per_file.keys():
111  this_events_per_file = Assumptions.events_per_file["%s:%s"%(chain.name,t)]
112  else:
113  this_events_per_file = Assumptions.events_per_file["%s"%(chain.name)]
114 
115  # Get event per production
116  if "%s:%s"%(chain.name,t) in Assumptions.n_events.keys():
117  this_events_per_production = Assumptions.n_events["%s:%s"%(chain.name,t)]
118  else:
119  this_events_per_production = Assumptions.n_events["%s"%(chain.name)]
120 
121  # Get inverse efficiency
122  this_inverse_eff = 1.
123  if "%s:%s"%(chain.name,t) in Assumptions.eff.keys():
124  this_inverse_eff = (1. / Assumptions.eff["%s:%s"%(chain.name,t)])
125 
126 
127  this_time_per_event.append(this_time)
128  this_time_per_file.append(this_time * this_events_per_file * this_inverse_eff)
129  this_total_time.append(this_time * this_events_per_production * this_inverse_eff * S_TO_KILODAYS)
130 
131  this_size_per_event.append(this_size / (1024) )
132  this_size_per_file.append(this_size * this_events_per_file / (1024*1024) )
133  this_total_size.append(this_size * this_events_per_production / (1024*1024*1024) )
134 
135  time_per_event.append(this_time_per_event)
136  time_per_file.append(this_time_per_file)
137  total_time.append(this_total_time)
138  size_per_event.append(this_size_per_event)
139  size_per_file.append(this_size_per_file)
140  total_size.append(this_total_size)
141 
142  tpe_head_html, tpe_base_html = \
143  stackedBarChart(name=name+"_per_event",
144  legend=legend,results=time_per_event,
145  title='Chain execution time per event [s] for test: %s'%title,
146  x_title="Time per event [s]",
147  y_title="Chain",
148  options=options,s_options=s_options,
149  checkbox=True,
150  for_definitions=False,
151  rounding=rounding,
152  )
153  head_html += tpe_head_html
154  head_html +="\n\n"
155 
156  tpf_head_html, tpf_base_html = \
157  stackedBarChart(name=name+"_per_file",
158  legend=legend,results=time_per_file,
159  title='Chain execution time for each file [CPU s] for test: %s'%title,
160  x_title="Time for each file [CPU s]",
161  y_title="Chain",
162  options=options,
163  checkbox=True,
164  for_definitions=False,
165  rounding=rounding,
166  )
167  head_html += tpf_head_html
168  head_html +="\n\n"
169 
170  tpp_head_html, tpp_base_html = \
171  stackedBarChart(name=name+"_full_sample",
172  legend=legend,results=total_time,
173  title='Chain execution time for all events [k-CPU days] for test: %s'%title,
174  x_title="Time for all events [k-CPU days]",
175  y_title="Chain",
176  options=options,
177  checkbox=True,
178  for_definitions=False,
179  rounding=rounding,
180  )
181  head_html += tpp_head_html
182  head_html +="\n\n"
183 
184  # Sizes
185  spe_head_html, spe_base_html = \
186  stackedBarChart(name=name+"_size_per_event",
187  legend=legend,results=size_per_event,
188  title='Chain size per event [kB] for test: %s'%title,
189  x_title="Size per event [kB]",
190  y_title="Chain",
191  options=options,s_options=s_options,
192  checkbox=True,
193  for_definitions=False,
194  rounding=rounding,
195  )
196  head_html += spe_head_html
197  head_html +="\n\n"
198 
199  spf_head_html, spf_base_html = \
200  stackedBarChart(name=name+"_size_per_file",
201  legend=legend,results=size_per_file,
202  title='Chain size for each file [MB] for test: %s'%title,
203  x_title="Size for each file [MB]",
204  y_title="Chain",
205  options=options,
206  checkbox=True,
207  for_definitions=False,
208  rounding=rounding,
209  )
210  head_html += spf_head_html
211  head_html +="\n\n"
212 
213  spp_head_html, spp_base_html = \
214  stackedBarChart(name=name+"_size_full_sample",
215  legend=legend,results=total_size,
216  title='Chain size for all events [GB] for test: %s'%title,
217  x_title="Size for all events [GB]",
218  y_title="Chain",
219  options=options,
220  checkbox=True,
221  for_definitions=False,
222  rounding=rounding,
223  )
224  head_html += spp_head_html
225  head_html +="\n\n"
226  return head_html, (tpe_base_html, tpf_base_html, tpp_base_html, spe_base_html, spf_base_html, spp_base_html)
def stackedBarChart(name="name", legend=[], results=[], title="title", x_title="x", y_title="y", options="", s_options="", dashboard=False, checkbox=False, for_definitions=True, rounding=False)
Simple stacked bar charts.
def projectionTimes(chain, per_input_event=False, verbose=False)
Build a projection of the time it will take to build a dataset.
Definition: Projections.py:3
def projectionSizes(chain, per_output_event=False, verbose=False)
Build a projection of the size of each tier.
Definition: Projections.py:32
procfile open("FD_BRL_v0.txt")
def directoryProjectionBarChart(name, title, directory, path, unpickled_chains={}, verbose=False, rounding=False)
Entire test Projection bar charts.
def ViewGoogleCharts.samFileInfoSummaryCharts (   definitions,
  path 
)

SAM file summary info into a chart.

Definition at line 229 of file ViewGoogleCharts.py.

References open(), and stackedBarChart().

229 def samFileInfoSummaryCharts(definitions, path):
230 
231  colours = ["#08804C",
232  "#0A166B", "#111F88", "#1729B0", "#3B49B4", "#5C68C3",
233  "#6C0382",
234  "#9B0000", "#FF0000", "#FF6363"
235  ]
236  s_colours = "[COL]"
237  for colour in colours:
238  s_colours = s_colours.replace("COL","%s,COL"%repr(colour))
239  s_colours = s_colours.replace(",COL","")
240 
241  # legend = ""
242  # for definition in definitions.keys():
243  # legend += repr(definition)+","
244  # legend = legend[:-1]
245 
246  n_files_html = open("%s/base_google_bar_chart.html"%path,"rb").read()
247  n_files_html = n_files_html.replace("NAME","n_files")
248  n_files_html = n_files_html.replace("Y_TITLE","Definition")
249  n_files_html = n_files_html.replace("X_TITLE","Number of files")
250  n_files_html = n_files_html.replace("TITLE", "Number of files per definition")
251  n_files_html = n_files_html.replace("<!--OPTIONS-->", "colors: ['#08804C']")
252  n_files_html = n_files_html.replace("VAR","'definitions','Files'")
253 
254  n_events_html = open("%s/base_google_bar_chart.html"%path,"rb").read()
255  n_events_html = n_events_html.replace("NAME","n_events")
256  n_events_html = n_events_html.replace("Y_TITLE","Definition")
257  n_events_html = n_events_html.replace("X_TITLE","Number of events")
258  n_events_html = n_events_html.replace("TITLE", "Number of events per definition")
259  n_events_html = n_events_html.replace("<!--OPTIONS-->", "colors: ['#1729B0']")
260  n_events_html = n_events_html.replace("VAR","'definitions','Events'")
261 
262  size_html = open("%s/base_google_bar_chart.html"%path,"rb").read()
263  size_html = size_html.replace("NAME","size")
264  size_html = size_html.replace("Y_TITLE","Definition")
265  size_html = size_html.replace("X_TITLE","Size [MB]")
266  size_html = size_html.replace("TITLE", "Size of definition")
267  size_html = size_html.replace("<!--OPTIONS-->", "colors: ['#FF0000']")
268  size_html = size_html.replace("VAR","'definitions','Size [MB]'")
269 
270  for definition in definitions.keys():
271  file_info = definitions[definition]
272  n_files = len(file_info)
273  events = 0
274  size = 0.
275  for f in file_info:
276  events += f.event_count
277  size += f.file_size
278 
279  this_n_files = "[%s,%s],"%(repr(definition),n_files)
280  n_files_html = n_files_html.replace("ENTRY","%s\nENTRY"%this_n_files)
281 
282  this_n_events = "[%s,%s],"%(repr(definition),events)
283  n_events_html = n_events_html.replace("ENTRY","%s\nENTRY"%this_n_events)
284 
285  this_size = "[%s,%s],"%(repr(definition),size/(1024.*1024.))
286  size_html = size_html.replace("ENTRY","%s\nENTRY"%this_size)
287 
288 
289  html = n_files_html
290  html += "\n%s"%n_events_html
291  html += "\n%s"%size_html
292  html = html.replace("ENTRY","")
293  return html
def samFileInfoSummaryCharts(definitions, path)
SAM file summary info into a chart.
procfile open("FD_BRL_v0.txt")
def ViewGoogleCharts.stackedBarChart (   name = "name",
  legend = [],
  results = [],
  title = "title",
  x_title = "x",
  y_title = "y",
  options = "",
  s_options = "",
  dashboard = False,
  checkbox = False,
  for_definitions = True,
  rounding = False 
)

Simple stacked bar charts.

Definition at line 304 of file ViewGoogleCharts.py.

References open(), and stackedColumnChart().

Referenced by directoryProjectionBarChart(), samFileInfoSummaryCharts(), and stackedColumnChart().

304  ):
305  import os
306  path = "%s/web/templates"%os.environ["NOVAPRODVALID_DIR"]
307  if for_definitions:
308  if dashboard: base_html = "template_google_bar_chart_dashboard.html"
309  else: base_html = "template_google_bar_chart.html"
310  else:
311  if dashboard: base_html = "template_testing_google_bar_chart_dashboard.html"
312  else: base_html = "template_testing_google_bar_chart.html"
313  head_html = open("%s/%s"%(path,base_html), "rb").read()
314  head_html = head_html.replace("NAME", name)
315  head_html = head_html.replace("X_TITLE", x_title)
316  head_html = head_html.replace("Y_TITLE", y_title)
317  head_html = head_html.replace("TITLE", title)
318  head_html = head_html.replace("[VAR]", repr(legend))
319  if dashboard:
320  head_html = head_html.replace("// OPTIONS", options)
321  head_html = head_html.replace("// S_OPTIONS", s_options)
322  else:
323  head_html = head_html.replace("<!--OPTIONS-->", options)
324  for result in results:
325  head_html = head_html.replace("ENTRY","\n[ENTRY")
326  for i_r,r in enumerate(result):
327  if r == None: r = 0
328  if i_r == 0: head_html = head_html.replace("ENTRY","%s,ENTRY"%repr(r))
329  elif rounding:head_html = head_html.replace("ENTRY","%.2g,ENTRY"%r)
330  else: head_html = head_html.replace("ENTRY","%f,ENTRY"%r)
331  head_html = head_html.replace("ENTRY","],ENTRY")
332 
333  head_html = head_html.replace("ENTRY","")
334 
335  if dashboard:
336  body_html = \
337  """
338  <div id="dashboard_div_%s">
339  <div class="row">
340  <div class="col-md-6"><div id="chart_div_%s"></div></div>
341  <div class="col-md-6"><div id="chain_filter_div_%s"></div></div>
342  </div>
343  </div>
344  """%(name,name,name)
345 
346  elif checkbox:
347  body_html = \
348  """
349  <div id="chart_div_%s" style="width: 900px; height: 500px;"></div>
350  <div class="row">
351  <div class="col-md-2 col-md-offset-1">
352  """%name
353 
354  for i,r in enumerate(results):
355  l = r[0].replace(" ","_")
356 
357  body_html += \
358  """
359  <div class="checkbox">
360  <label><input type="checkbox" id="%s_%s" checked="true" autocomplete="off">%s</label>
361  </div>
362  """%(name,l,l.replace("_"," "))
363 
364  head_html = head_html.replace("<!--CHECK_BOX_HOOK-->",\
365  """
366  var hide_%s = document.getElementById("%s_%s");
367  hide_%s.onclick = function()
368  {
369  view = new google.visualization.DataView(data);
370  if (hide_%s.checked) {
371  var index = rows.indexOf(%i);
372  if (index == -1){
373  rows.push(%i);
374  }
375  } else {
376  var index = rows.indexOf(%i);
377  if (index != -1){
378  rows.splice(index, 1);
379  }
380  }
381  rows.sort(function(a, b){return a-b});
382  console.log(rows);
383  view.setRows(rows);
384  view.setColumns(columns);
385  chart.draw(view, options);
386  //document.getElementById('png_%s').outerHTML = '<a href="' + chart.getImageURI() + '">Printable version</a>';
387  }
388  <!--CHECK_BOX_HOOK-->
389  """%(l,name,l,l,l,i,i,i,name)
390  )
391 
392  if ((i+1)%5 == 0) and ((i+1) != len(results)):
393  body_html += '</div><div class="col-md-2">'
394  body_html += "</div></div>"
395  body_html += "<div id='png_%s'></div>"%name
396  else:
397  body_html = \
398  """
399  <div id="chart_div_%s" style="width: 900px; height: 500px;"></div>
400  """%name
401 
402 
403  return head_html, body_html
procfile open("FD_BRL_v0.txt")
def ViewGoogleCharts.stackedColumnChart (   name = "name",
  legend = [],
  results = [],
  title = "title",
  x_title = "y",
  y_title = "y",
  options = "",
  s_options = "",
  dashboard = False,
  checkbox = False,
  rounding = False 
)

Simple stacked column charts.

Definition at line 411 of file ViewGoogleCharts.py.

References stackedBarChart().

Referenced by stackedBarChart().

411  rounding=False):
412 
413  head_html, body_html = stackedBarChart(name=name,legend=legend,results=results,
414  title=title, x_title=x_title, y_title=y_title,
415  options=options,s_options=s_options,
416  dashboard=dashboard,checkbox=checkbox,rounding=rounding)
417  head_html = head_html.replace("BarChart","ColumnChart")
418  return head_html, body_html
def stackedBarChart(name="name", legend=[], results=[], title="title", x_title="x", y_title="y", options="", s_options="", dashboard=False, checkbox=False, for_definitions=True, rounding=False)
Simple stacked bar charts.
def ViewGoogleCharts.tierProjectionBarChart (   chain,
  options 
)

Projection bar charts.

Definition at line 3 of file ViewGoogleCharts.py.

References open().

3 def tierProjectionBarChart(chain, options):
4  import Projections as proj
5  import os
6  base_html = open("%s/web/templates/template_google_bar_chart.html"%os.environ["NOVAPRODVALID_DIR"],"r").read()
7  base_html = base_html.replace("NAME",chain.name)
8  base_html = base_html.replace("Y_TITLE","Tier")
9  base_html = base_html.replace("TITLE", "'%s execution time [s]'"%chain.name.replace("_"," "))
10  projections = proj.projectionTimes(chain)
11  base_html = base_html.replace("VAR","'tier','time'")
12  i = -1
13  for tier in chain.tiers:
14  if not tier.metric.run: continue
15  i+=1
16  base_html = base_html.replace("ENTRY",
17  "['%s',%f],ENTRY"%(tier.short_name.replace("_"," "),projections[i]))
18  base_html = base_html.replace("ENTRY","")
19  return base_html
procfile open("FD_BRL_v0.txt")
def tierProjectionBarChart(chain, options)
Projection bar charts.