ViewBokeh.py
Go to the documentation of this file.
1 ###########################
2 # Dictionary of nice attribute names
3 attribute_names = {
4  "memory":"Memory usage [MB]",
5  "memory_percent":"Memory usage [%]",
6  "cpu_percent":"CPU usage [%]",
7  "read_bytes": "Bytes read in [MB]",
8  "write_bytes": "Bytes written out [MB]",
9  "u_cpu_times": "Integrated CPU time (user) [s]",
10  "s_cpu_times": "Integrated CPU time (system) [s]",
11  "rss_memory" : "RSS memory [MB]",
12  "vms_memory" : "VMS memory [MB]",
13  "n_open_files": "Number of open files",
14  "n_connections": "Number of open connections",
15 }
16 ###########################
17 # base html
18 old_metric_base_html = \
19 """
20 <!DOCTYPE html>
21 <html lang="en">
22 <head>
23  <meta charset="utf-8">
24  <title>prod-metric</title>
25  <meta name="viewport" content="width=device-width, initial-scale=1.0">
26  <meta name="description" content="">
27  <meta name="author" content="Matthew Tamsett">
28  <link rel="shortcut icon" href="http://nusoft.fnal.gov/nova/production/static/nu_bkg.ico"/>
29 
30  <!-- Bootstrap styling -->
31  <link href="http://nusoft.fnal.gov/nova/production/static/bootstrap/css/bootstrap.css" rel="stylesheet">
32  <style>
33  body {
34  padding-top: 0px; /* 60px to make the container go all the way to the bottom of the topbar */
35  }
36  </style>
37  <!--<link href="http://nusoft.fnal.gov/nova/production/static/bootstrap/css/dashboard.css" rel="stylesheet">
38  -->
39  <!--HEADER_HOOK-->
40 </head>
41 
42 <body>
43  <div class="container-fluid">
44  <div class="row">
45  <div class="col-sm-3 col-md-2 sidebar">
46  <ul class="nav nav-sidebar">
47  <h2 class="page-header">Metrics</h2>
48  <!--SIDEBAR_HOOK-->
49  </ul>
50  </div> <!-- /sidebar -->
51  <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
52  <h1 class="page-header">Dashboard</h1>
53  <!--BODY_HOOK-->
54  </div> <!-- /body -->
55  </div>
56  <!--FOOTER_HOOK-->
57  </div>
58 </body>
59 </html>
60 """
61 metric_base_html = \
62 """
63  <div class="dropdown">
64  <button class="btn btn-default btn-lg dropdown-toggle" type="button" id="dropdownMenu1" data-toggle="dropdown">
65  <span class="glyphicon glyphicon-stats"></span>
66  Metrics
67 
68  <span class="caret"></span>
69  </button>
70  <ul class="dropdown-menu" role="menu" aria-labelledby="dropdownMenu1">
71  <!--SIDEBAR_HOOK-->
72  </ul>
73  </div>
74  <section>
75  <!--BODY_HOOK-->
76  </section>
77 """
78 ###########################
79 # draw all metrics from a chain
80 def drawChain(name, chain, attributes,
81  title="chain example",
82  verbose=True,):
83  if verbose: print "vbokh: Drawing Chain"
84  import numpy as np
85  import bokeh.plotting as bk
86  import Tier, Chain, Metric, ViewBokeh, ParserArtEvents
87  TOOLS = "hover"
88  # bk.hold()
89  bk.output_file(name, title=title)
90 
91  for tier in chain.tiers:
92  if verbose: print "vbokh: --- Drawing tier: %s"%tier.short_name
93  metric = tier.metric
94  if (tier.metric.run == False): continue
95  metric.refresh()
96 
97  all_events = False
98  if tier.log != "":
99  art_events = ParserArtEvents.parseLines(tier.log)
100  events = art_events.events
101  queries = art_events.queries
102  parsing = art_events.parsing
103  all_events = [events, queries, parsing]
104  events_legend= ["Events","DB queries","DB parsing"]
105 
106  times = np.array(metric.sample_times)
107 
108  for attribute in attributes:
109  values = np.array(getattr(metric,attribute))
110  value_range = findRange(values)
111 
112  bk_figure = bk.figure(title="%s: %s"%(tier.short_name, attribute))
113 
114  bk_figure.line(times,values,
115  color="#0000FF",
116  line_width=3,
117  tools=TOOLS,
118  #x_axis_label=x_axis_label,
119  #y_axis_label=y_axis_label,
120  y_range=value_range,
121  legend=attribute)
122 
123  if all_events:
124  colours = ["red","green","#800080"]
125  for i, event in enumerate(all_events):
126  drawEventOnCurve(bk_figure, event, times, values,
127  colour=colours[i],
128  legend=events_legend[i],
129  tools=TOOLS)
130  bk.show()
131 ###########################
132 # draw all metrics from a tier
133 def drawTier(name, tier, attributes,
134  title="tier example",
135  verbose=True):
136  if verbose: print "vbokh: Drawing Tier"
137  import numpy as np
138  import bokeh.plotting as bk
139  import Tier, Metric, ParserArtEvents
140  TOOLS = "hover"
141  # bk.hold(value=False)
142  bk.output_file(name, title=title)
143 
144  if verbose: print "vbokh: --- Drawing tier: %s"%tier.short_name
145  metric = tier.metric
146  if (tier.metric.run == False): return
147  metric.refresh()
148 
149  all_events = False
150  if tier.log != "":
151  art_events = ParserArtEvents.parseLines(tier.log)
152  events = art_events.events
153  queries = art_events.queries
154  parsing = art_events.parsing
155  all_events = [events, queries, parsing]
156  events_legend= ["Events","DB queries","DB parsing"]
157 
158  times = np.array(metric.sample_times)
159 
160  for i_a, attribute in enumerate(attributes):
161  values = np.array(getattr(metric,attribute))
162  value_range = findRange(values)
163 
164  bk_figure = bk.figure(title="%s: %s"%(tier.short_name, attribute))
165 
166  bk_figure.line(times,values,
167  color="#0000FF",
168  line_width=3,
169  tools=TOOLS,
170  #x_axis_label=x_axis_label,
171  #y_axis_label=y_axis_label,
172  y_range=value_range,
173  legend=attribute)
174  # if i_a > 0: bk.hold()
175 
176  if all_events:
177  colours = ["red","green","#800080"]
178  for i, event in enumerate(all_events):
179  drawEventOnCurve(bk_figure, event, times, values,
180  colour=colours[i],
181  legend=events_legend[i],
182  tools=TOOLS)
183  bk.show()
184 ###########################
185 # draw a single metric
186 def drawSingleMetric(name, metric, attribute_name,
187  title="metric example",
188  x_axis_label="x",
189  y_axis_label="y",
190  legend="metric",
191  events=False, events_legend=False,
192  verbose=False):
193  import numpy as np
194 
195  times = np.array(metric.sample_times)
196  values = np.array(getattr(metric,attribute_name))
197  value_range = findRange(values)
198 
199  import bokeh.plotting as bk
200  TOOLS = "hover"
201  # bk.hold()
202  bk_figure = bk.figure(title=title)
203  bk.output_file(name, title=title)
204  bk_figure.line(times,values,
205  color="#0000FF",
206  line_width=3,
207  tools=TOOLS,
208  x_axis_label=x_axis_label,
209  y_axis_label=y_axis_label,
210  y_range=value_range,
211  legend=legend)
212 
213  if events:
214  if type(events) != list: events = [events]
215  if type(events_legend) != list: events_legend = [events_legend]
216  if events_legend != [False]: assert(len(events) == len(events_legend))
217  if events_legend == [False]:
218  for e in events[1:]: events_legend.append(False)
219  colours = ["red","green","#800080"]
220  for i, event in enumerate(events):
221  drawEventOnCurve(bk_figure,event, times, values,
222  colour=colours[i],
223  legend=events_legend[i],
224  tools=TOOLS)
225  bk.show()
226 ###########################
227 # draw all metrics from a tier
228 def tierToHTML(tier, attributes,
229  title="",
230  bokeh_path="/Users/mt354/anaconda/lib/python2.7/site-packages/bokeh/server/static/",
231  base_url ="/Users/mt354/Analysis/Production/",
232  verbose=False):
233  unit_conversion = {}
234  unit_conversion["memory"] = 1./(1024**2)
235  unit_conversion["rss_memory"] = 1./(1024**2)
236  unit_conversion["vms_memory"] = 1./(1024**2)
237  unit_conversion["read_bytes"] = 1./(1024**2)
238  unit_conversion["write_bytes"] = 1./(1024**2)
239  if verbose: print "vbokh: --- Writing html for tier %s"%tier.short_name
240  import numpy as np
241  import bokeh.plotting as bk
242  from bokeh.resources import CDN
243  from bokeh.embed import file_html
244  import Tier, Metric, ParserArtEvents, ViewHTML
245 
246  #base_html = metric_base_html.replace('<h1 class="page-header">Dashboard</h1>',
247  #'<h1 class="page-header">%s tier: %s</h1>'%(title, tier.short_name))
248  base_html = metric_base_html
249 
250  TOOLS = "hover,previewsave"
251  # bk.hold()
252  #bk.output_file(name, title=title)
253 
254  snippets = {}
255 
256  #if verbose: print "vbokh: --- Drawing tier: %s"%tier.short_name
257  metric = tier.metric
258  if (tier.metric.run == False): return base_html
259  metric.refresh()
260 
261  all_events = []
262  events_legend= []
263  if verbose: print "vbokh: parsing log"
264  if tier.log != "":
265  art_events = ParserArtEvents.parseLines(tier.log)
266  events = art_events.events
267  queries = art_events.queries
268  parsing = art_events.parsing
269  if len(events.keys()) > 0:
270  if verbose: print "vbokh: events found: %i"%len(events.keys())
271  if len(events.keys()) > 20:
272  tmp_events = {}
273  keys = events.keys()
274  keys.sort()
275  for k in keys[:19]:
276  tmp_events[k] = events[k]
277  tmp_events[keys[-1]] = events[keys[-1]]
278  events = tmp_events
279  if verbose: print "vbokh: events used: %i"%len(events.keys())
280  all_events.append(events)
281  events_legend.append("Events")
282  if len(queries.keys()) > 0:
283  all_events.append(queries)
284  events_legend.append("DB queries")
285  if len(parsing.keys()) > 0:
286  all_events.append(parsing)
287  events_legend.append("DB parsing")
288 
289  times = np.array(metric.sample_times)
290 
291  for i_a, attribute in enumerate(attributes):
292  if verbose: print "vbokh: - making array for %s"%attribute
293  this_attr = getattr(metric,attribute)
294  if this_attr == None: continue
295  if attribute in unit_conversion.keys():
296  for i in range(len(this_attr)):
297  this_attr[i] = this_attr[i] * unit_conversion[attribute]
298  values = np.array(this_attr)
299  value_range = findRange(values)
300 
301  #bk.figure(title="%s: %s"%(tier.short_name, attribute))
302  bk_figure = bk.figure(title="", tools=TOOLS)
303  if verbose: print "vbokh: - making figure"
304  bk_figure.line(times,values,
305  color="#0000FF",
306  line_width=3,
307  tools=TOOLS,
308  x_axis_label="Time [s]",
309  y_axis_label=attribute_names[attribute],
310  y_range=value_range,
311  legend=attribute)
312  if verbose: print "vbokh: - adding events"
313  if len(all_events):
314  colours = ["red","green","#800080"]
315  for i, event in enumerate(all_events):
316  drawEventOnCurve(bk_figure, event, times, values,
317  colour=colours[i],
318  legend=events_legend[i],
319  tools=TOOLS)
320  if verbose: print "vbokh: - making html snippet"
321  this_html = file_html(bk_figure, CDN, attribute_names[attribute])
322  base_html = base_html.replace("<!--BODY_HOOK-->",
323  '<h2 class="page-header" id="%s">%s</h2>\n%s\n<!--BODY_HOOK-->'%(attribute, attribute_names[attribute], this_html))
324  base_html = base_html.replace("<!--SIDEBAR_HOOK-->",
325  '<li role="presentation"><a role="menuitem" tabindex="-1" href="#%s">%s</a></li>\n<!--SIDEBAR_HOOK-->'%(attribute, attribute_names[attribute]))
326  return base_html.replace("<!--BODY_HOOK-->","")
327 ###########################
328 # draw art events onto curve
329 def drawEventOnCurve(bk_figure, events, times, values,
330  colour="red",
331  legend=False,
332  tools=[]):
333  import numpy as np
334  import bokeh.plotting as bk
335  from bokeh.models import HoverTool
336  from collections import OrderedDict
337 
338  the_times = np.array(events.values())
339  the_values = fastFindValuesFromCurve(the_times, times, values)
340 
341  # make mouse hover values
342  hover_info = events.keys()
343  source = bk.ColumnDataSource( data=dict( time=the_times,
344  value=the_values,
345  hover_info=hover_info
346  )
347  )
348  hover = bk_figure.select(dict(type=HoverTool))
349  hover.tooltips = OrderedDict([ ("(x,y)", "(@time, @value)"),
350  ("info", "@hover_info"),
351  ])
352 
353  bk_figure.scatter(the_times, the_values,
354  source=source, size=12, color=colour,
355  legend=legend,
356  alpha=0.5, tools=tools)
357 ###########################
358 # evaluate the value of a curve at a given series of time points
359 def findValuesFromCurve(times, curve_times, curve_values):
360  import numpy as np
361  tmp_values = []
362  for time in times:
363  found = False
364  for i,curve_time in enumerate(curve_times):
365  if compareFloats(curve_time,time):
366  tmp_values.append(curve_values[i])
367  found = True
368  break
369  if curve_time > time:
370  print "Curve time[%i]: %f exceeds current time: %f"%\
371  (i, curve_time, time)
372  print "Curve time[%i]: %f"%(i-1, curve_times[i-1])
373  assert False
374  values = np.array(tmp_values)
375  return values
376 ###########################
377 # quick search
378 from bisect import bisect_left
379 def takeClosest(myList, myNumber):
380  """
381  Assumes myList is sorted. Returns closest value to myNumber.
382 
383  If two numbers are equally close, return the smallest number.
384  """
385  pos = bisect_left(myList, myNumber)
386  if pos == 0:
387  return 0
388  if pos == len(myList):
389  return len(myList)-1
390  before = myList[pos - 1]
391  after = myList[pos]
392  if after - myNumber < myNumber - before:
393  return pos
394  else:
395  return pos - 1
396 ###########################
397 # FAST VERSION to evaluate the value of a curve at a given series of time points
398 def fastFindValuesFromCurve(times, curve_times, curve_values):
399  import numpy as np
400  tmp_values = []
401  for time in times:
402  time_idx = takeClosest(curve_times, time)
403  try:
404  tmp_values.append(curve_values[time_idx])
405  except IndexError:
406  print "vbokh: --- takeClosest threw an index error"
407  print "vbokh: given time: ",time
408  print "vbokh: %i curve times"%len(curve_times)
409  print "vbokh: curve times: ",curve_times
410  print "vbokh: %i curve values"%len(curve_values)
411  print "vbokh: curve values: ",curve_values
412  print "vbokh: got index: %i"%time_idx
413  assert False
414  values = np.array(tmp_values)
415  return values
416 ###########################
417 # compare floats
418 def compareFloats(a, b):
419  from math import fabs
420  if (fabs(a - b) < (a*0.000001)): return True
421  return False
422 ###########################
423 # find the spread of some values
424 def findRange(array):
425  minimum = min(array)
426  maximum = max(array)
427  spread = maximum - minimum
428  return [0, maximum*(1.5)]
429 ###########################
430 # Find all the embeded js objects in a html file
431 def findJSList(html):
432  lines = html.split("\n")
433  js_list = []
434  for line in lines:
435  if ".embed.js" not in line: continue
436  tokens = line.split('"')
437  path = False
438  for token in tokens:
439  if ".embed.js" in token:
440  path = token.split("/")[-1]
441  assert(".embed.js" in path)
442  js_list.append(path)
443  assert(path)
444  return js_list
fvar< T > fabs(const fvar< T > &x)
Definition: fabs.hpp:15
def tierToHTML(tier, attributes, title="", bokeh_path="/Users/mt354/anaconda/lib/python2.7/site-packages/bokeh/server/static/", base_url="/Users/mt354/Analysis/Production/", verbose=False)
draw all metrics from a tier
Definition: ViewBokeh.py:232
def findJSList(html)
Find all the embeded js objects in a html file.
Definition: ViewBokeh.py:431
def takeClosest(myList, myNumber)
Definition: ViewBokeh.py:379
def compareFloats(a, b)
compare floats
Definition: ViewBokeh.py:418
def drawChain(name, chain, attributes, title="chain example", verbose=True)
draw all metrics from a chain
Definition: ViewBokeh.py:82
def findRange(array)
find the spread of some values
Definition: ViewBokeh.py:424
def drawEventOnCurve(bk_figure, events, times, values, colour="red", legend=False, tools=[])
draw art events onto curve
Definition: ViewBokeh.py:332
static float min(const float a, const float b, const float c)
Definition: absgeo.cxx:45
def parseLines(lines, verbose=False)
Parse output file.
def fastFindValuesFromCurve(times, curve_times, curve_values)
FAST VERSION to evaluate the value of a curve at a given series of time points.
Definition: ViewBokeh.py:398
def drawTier(name, tier, attributes, title="tier example", verbose=True)
draw all metrics from a tier
Definition: ViewBokeh.py:135
assert(nhit_max >=nhit_nbins)
def drawSingleMetric(name, metric, attribute_name, title="metric example", x_axis_label="x", y_axis_label="y", legend="metric", events=False, events_legend=False, verbose=False)
draw a single metric
Definition: ViewBokeh.py:192
T max(sqlite3 *const db, std::string const &table_name, std::string const &column_name)
Definition: statistics.h:68
def findValuesFromCurve(times, curve_times, curve_values)
evaluate the value of a curve at a given series of time points
Definition: ViewBokeh.py:359