Functions | Variables
POTGetData Namespace Reference

Functions

def unix_timestamp (input)
 
def database_connect (beam)
 
def get_active_chans (run, beam)
 
def get_runs (day, deltaDay, beam)
 
def get_spills (day, deltaDay, beam)
 
def get_uptime (day, runs)
 
def run_startstop (runs)
 
def pot_between (t0, t1, spills)
 
def pot (day, runs, spills)
 
def append_json (jsonEntry, beam)
 
def write_json (jsonData, beam)
 
def options (option_arg)
 
def main ()
 

Variables

int ndays = 1
 
float GOOD_SPILL_POT = 0.05
 
string JSON_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'
 
string JSON_BNB_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'
 

Detailed Description

Compute operations statistics (uptime, POT recorded, delivered, etc.)
for NOvA and append to existing json file.

Based on Commissioning/nova_uptime.py

Fernanda Psihas (psihas@fnal.gov)
Justin Vasel (jvasel@fnal.gov)

Function Documentation

def POTGetData.append_json (   jsonEntry,
  beam 
)
Appends today's entry to JSON file.

  Args:
  jsonEntry : A Python object

Definition at line 378 of file POTGetData.py.

References open(), runNovaSAM.str, and write_json().

Referenced by main().

378 def append_json(jsonEntry,beam):
379  """ Appends today's entry to JSON file.
380 
381  Args:
382  jsonEntry : A Python object
383 
384  """
385 
386  # Open file for reading and store contents
387  if beam == str("numi"):
388  file = open(JSON_FILE_PATH, 'r')
389  elif beam == str("bnb"):
390  file = open(JSON_BNB_FILE_PATH, 'r')
391  contents = file.read()
392 
393  # Decode JSON as a Python object
394  jsonData = json.loads(contents)
395 
396  # Append jsonEntry to jsonData
397  jsonData.append(jsonEntry)
398 
399  # Close file for reading
400  file.close()
401 
402  # Write file
403  write_json(jsonData,beam)
404 
405 
406 #.......................................................................
def append_json(jsonEntry, beam)
Definition: POTGetData.py:378
def write_json(jsonData, beam)
Definition: POTGetData.py:407
procfile open("FD_BRL_v0.txt")
def POTGetData.database_connect (   beam)
Initiate connection to psql database

Definition at line 64 of file POTGetData.py.

References open(), and runNovaSAM.str.

Referenced by main().

64 def database_connect(beam):
65  """ Initiate connection to psql database
66  """
67 
68  DB_NAME = os.environ['NOVADBNAME'] # 'nova_prod'
69  DB_HOST = os.environ['NOVADBHOST'] # 'ifdbrep.fnal.gov'
70  DB_USER = os.environ['NOVADBUSER'] # 'nova_reader'
71  DB_PASS = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
72  DB_FAR_PORT = os.environ['NOVAFARDAQDBPORT'] # '5436'
73  DB_PORT = os.environ['NOVADBPORT'] # '5433'
74 
75  # Initiate connection to database
76  if beam == str("numi"):
77  try:
78  db = psycopg2.connect(\
79  "dbname=%s host=%s user=%s password=%s port=%s" % \
80  (DB_NAME, DB_HOST, DB_USER, DB_PASS, DB_FAR_PORT))
81  except:
82  print "Unable to connect to the database"
83 
84  elif beam == str("bnb"):
85  try:
86  db = psycopg2.connect(\
87  "dbname=%s host=%s user=%s password=%s port=%s" % \
88  (DB_NAME, DB_HOST, DB_USER, DB_PASS, DB_PORT))
89  except:
90  print "Unable to connect to the database"
91  # Create cursor object so that we can send queries to the db.
92  # Note: The `cursor_factory' argument will create a dictionary
93  # cursor so that columns are returned as a dictionary so that we
94  # can access columns by their name instead of index.
95  global db_cursor
96  db_cursor = db.cursor(cursor_factory = psycopg2.extras.DictCursor)
97 
98 #.......................................................................
procfile open("FD_BRL_v0.txt")
def database_connect(beam)
Definition: POTGetData.py:64
def POTGetData.get_active_chans (   run,
  beam 
)
Return how many active channels there we at start of run

  Args:
  run: Run number

  Returns:
  Number of active channels (APD pixels)

Definition at line 99 of file POTGetData.py.

References runNovaSAM.str.

Referenced by POTUptimeData.do_detector_mass().

99 def get_active_chans(run, beam):
100  """ Return how many active channels there we at start of run
101 
102  Args:
103  run: Run number
104 
105  Returns:
106  Number of active channels (APD pixels)
107  """
108 
109  if (run == 0):
110  return 0;
111 
112  # Build the SQL query to be performed
113  if beam == str("numi"):
114  query = "select nactivechan from fardet.runs where run=" + str(run)
115  elif beam == str("bnb"):
116  query = "select nactivechan from neardet.runs where run=" + str(run)
117  # Submit query to SQL. Store response as a Python object.
118  db_cursor.execute(query)
119  number = db_cursor.fetchall()
120 
121  # Return the number of active channels
122  return number[0]['nactivechan']
123 
124 
125 #.......................................................................
def get_active_chans(run, beam)
Definition: POTGetData.py:99
def POTGetData.get_runs (   day,
  deltaDay,
  beam 
)
Get info about runs that start/stop within a particular date range

  Args:
  day : A datetime object
  deltaDay : Number of days to consider

  Returns:
  runs : A Python object with the following information:
          run number, run start time, and run stop time

Definition at line 126 of file POTGetData.py.

References novadaq::HexUtils.format(), and runNovaSAM.str.

Referenced by POTUptimeData.main(), and main().

126 def get_runs(day, deltaDay, beam):
127  """ Get info about runs that start/stop within a particular date range
128 
129  Args:
130  day : A datetime object
131  deltaDay : Number of days to consider
132 
133  Returns:
134  runs : A Python object with the following information:
135  run number, run start time, and run stop time
136  """
137 
138  day1 = day + timedelta(days = deltaDay)
139 
140  # Build the SQL query to be performed
141  if beam == str("numi"):
142  select = "select run,tstart,tstop from fardet.runs"
143  elif beam == str("bnb"):
144  select = "select run,tstart,tstop from neardet.runs"
145 
146  cut1 = " where partition=1"
147  cut2 = " tstop is not null"
148 
149  cut3 = "tstart >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
150  cut4 = "tstart < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
151 
152  cut5 = "tstop >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
153  cut6 = "tstop < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
154 
155  sort = "order by tstart asc";
156 
157  query = select + cut1 + " and " + cut2 + " and " + \
158  "((" + cut3+" and "+cut4+") or ("+cut5+" and "+cut6+")) " + \
159  sort
160 
161  # Submit query to SQL. Store response as a Python object.
162  db_cursor.execute(query)
163  runs = db_cursor.fetchall()
164 
165  # Return run info
166  return runs
167 
168 
169 #.......................................................................
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def get_runs(day, deltaDay, beam)
Definition: POTGetData.py:126
def POTGetData.get_spills (   day,
  deltaDay,
  beam 
)
Get info about spills during a particular day

  Args:
  day : A datetime object
  deltaDay : Number of days to consider

  Returns:
  spills : A list of spills (time in sec, pot number in 1E+12)

Definition at line 170 of file POTGetData.py.

References check_time_usage.float, novadaq::HexUtils.format(), runNovaSAM.str, and unix_timestamp().

Referenced by POTUptimeData.do_pot(), and main().

170 def get_spills(day, deltaDay, beam):
171  """ Get info about spills during a particular day
172 
173  Args:
174  day : A datetime object
175  deltaDay : Number of days to consider
176 
177  Returns:
178  spills : A list of spills (time in sec, pot number in 1E+12)
179  """
180 
181  spills = []
182 
183  # Convert start and end of day to UNIX timestamp
184  t0 = unix_timestamp(day)
185  t1 = unix_timestamp(day + timedelta(days = deltaDay))
186 
187  # URL where the spill data live
188  if beam == str("numi"):
189  url = 'http://ifb-data.fnal.gov:8099/ifbeam/data/data?'\
190  'v=E:TRTGTD&e=e,a9&'\
191  't0={}&t1={}&&f=csv'.format(t0, t1)
192  elif beam == str("bnb"):
193  url = 'http://ifb-data.fnal.gov:8099/ifbeam/data/data?'\
194  'v=E:TOR860&e=e,1d&'\
195  't0={}&t1={}&&f=csv'.format(t0, t1)
196 
197  # Store the HTTP GET response here
198  response = None
199 
200  # Try to retrieve the URL contents. If this fails three times, then
201  # give up and return an empty array.
202  try:
203  response = urllib2.urlopen(url, None, 200)
204  except:
205  try:
206  response = urllib2.urlopen(url, None, 200)
207  except:
208  try:
209  response = urllib2.urlopen(url, None, 200)
210  except:
211  return spills
212 
213  if (response == None):
214  return spills
215 
216  # Use csv module to parse the response as CSV
217  csvdata = csv.reader(response)
218 
219  # The response will look like this:
220  # "e,a9", E:TRTGTD, 1406851201047, E12, 22.0598346211
221  # We are interested in the third column (unix timestamp in ms)
222  # and the fifth column (value). The list will be populated with these
223  # values.
224  if beam == str("numi"):
225  for row in csvdata:
226  if (row.count('E:TRTGTD') > 0):
227  if (row[4] != 'null'):
228  spills.append( (float(row[2]) / 1000., float(row[4])) )
229 
230  elif beam == str("bnb"):
231  for row in csvdata:
232  if (row.count('E:TOR860') > 0):
233  if (row[4] != 'null'):
234  spills.append( (float(row[2]) / 1000., float(row[4])) )
235  # Return spill info
236  return spills
237 
238 
239 #.......................................................................
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def unix_timestamp(input)
Definition: POTGetData.py:46
def get_spills(day, deltaDay, beam)
Definition: POTGetData.py:170
def POTGetData.get_uptime (   day,
  runs 
)
Compute total and fractional run uptime

  Args:
  day  : A datetime object
  runs : A list of runs for the day

  Returns:
  (uptimeTotal, uptimeFraction) : (datetime object, int)

Definition at line 240 of file POTGetData.py.

References min().

Referenced by main().

240 def get_uptime(day, runs):
241  """ Compute total and fractional run uptime
242 
243  Args:
244  day : A datetime object
245  runs : A list of runs for the day
246 
247  Returns:
248  (uptimeTotal, uptimeFraction) : (datetime object, int)
249  """
250 
251  t0 = day
252  t1 = day + timedelta(days = 1)
253 
254  # Instantiate total uptime as datetime object
255  uptimeTotal = timedelta(0)
256 
257  # Loop over runs
258  for run in runs:
259  tStart = run['tstart']
260  tStop = run['tstop']
261 
262  # If the run started before the day began, we should truncate
263  # to ensure we're only looking at the portion from today. Likewise
264  # for runs that end after the day ended.
265  if (tStart < t0):
266  tStart = t0
267  if (tStop > t1):
268  tStop = t1
269 
270  uptimeTotal += (tStop - tStart)
271 
272  # Calculate fractional uptime
273  # NOTE: In some weird edge cases, the daily uptime can be a couple seconds
274  # longer than one day. Instead of reporting a fractional uptime of >100%,
275  # we just set it to 100% by taking the min of the calculation and 1.
276  uptimeFraction = min(uptimeTotal.total_seconds() / (24.0 * 60.0 * 60.0), 1)
277 
278  # Return both the total and fractional uptime
279  return (uptimeTotal, uptimeFraction)
280 
281 
282 #.......................................................................
def get_uptime(day, runs)
Definition: POTGetData.py:240
static float min(const float a, const float b, const float c)
Definition: absgeo.cxx:45
def POTGetData.main ( )

1D plots hits vs w //

Definition at line 466 of file POTGetData.py.

References append_json(), database_connect(), get_runs(), get_spills(), get_uptime(), makeTrainCVSamples.int, options(), pot(), PandAna.Demos.demo1.range, run_startstop(), runNovaSAM.str, and write_json().

466 def main():
467 
468  # Grab user-chosen values from options
469  ndays = options(int(1))
470  beam = str("numi")
471  if len(sys.argv) > 3:
472  beam = options(int(2))
473 
474  print "Beam type: ", beam
475 
476  # Initiate connection to psql database
477  database_connect(beam)
478 
479  # Find today's date. This will serve as the final day to be included.
480  # Subtract off the number of days to be included, to find the start
481  # date.
482  t1 = datetime.utcnow()
483  t1 = t1.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
484  t0 = t1 - timedelta(days = ndays)
485 
486  # Print output column labels
487  print " "
488  print "No. Date Run1 Run2 Uptime "\
489  "Uptime (frac) POT Recorded POT Delivered POT Rec / Del"
490  print "====================================================="\
491  "==================================================================="
492 
493  # Prepare JSON object if re-writing data
494  if (ndays != 1):
495  jsonData = []
496 
497  print "ndays is ", ndays
498 
499  # Loop over days
500  for i in range(0, ndays):
501 
502  # Find the ith date
503  day = t0 + timedelta(days = i)
504 
505  # Get the list of run information for this day
506  runs = get_runs(day, 1, beam)
507 
508  # Find the first and final run number for this day
509  runInitial = 0
510  runFinal = 0
511  if (runs):
512  (runInitial, runFinal) = run_startstop(runs)
513 
514  # Compute uptime (total and fractional) for this day
515  uptimeTotal = 0
516  uptimeFrac = 0
517  if (runs):
518  (uptimeTotal, uptimeFrac) = get_uptime(day, runs)
519 
520  # Get the list of spill information for this day
521  spills = get_spills(day, 1, beam)
522 
523  # Compute how many protons were delivered and recorded
524  (potRecorded, potDelivered) = pot(day, runs, spills)
525 
526  # Compute the fraction of POT recorded. In cases where the POT
527  # delivered is zero, record the fraction recorded as the uptime
528  # computed above
529  potFraction = 0.0
530  if (potDelivered == 0.0):
531  potFraction = uptimeFrac
532  else:
533  potFraction = potRecorded / potDelivered
534 
535  # Reassign day so that the timestamp (00:00:00) doesn't appear
536  day = date(day.year, day.month, day.day)
537 
538  # Print the results to the screen
539  print "",i + 1," ",\
540  day," ",\
541  runInitial," ",\
542  runFinal," ",\
543  uptimeTotal," ",\
544  uptimeFrac," ",\
545  potRecorded," ",\
546  potDelivered," ",\
547  potFraction
548 
549  # Build the JSON entry for this day
550  jsonEntry = {"date" : str(day),\
551  "runStart" : str(runInitial),\
552  "runEnd" : str(runFinal),\
553  "uptime_hours" : str(uptimeTotal),\
554  "uptime_frac" : str(uptimeFrac),\
555  "pot_record" : str(potRecorded),\
556  "pot_deliv" : str(potDelivered),\
557  "pot_daily_frac" : str(potFraction)}
558 
559  # Append to jsonData if re-writing data
560  if (ndays != 1):
561  jsonData.append(jsonEntry)
562 
563  # Append jsonEntry to JSON file
564  if (ndays == 1):
565  append_json(jsonEntry,beam)
566  else:
567  write_json(jsonData,beam)
568 
569  print " "
570 
571 
572 #.......................................................................
573 
574 # Prevent execution on import
::xsd::cxx::tree::date< char, simple_type > date
Definition: Database.h:186
def main()
Definition: POTGetData.py:466
def append_json(jsonEntry, beam)
Definition: POTGetData.py:378
def run_startstop(runs)
Definition: POTGetData.py:283
def options(option_arg)
Definition: POTGetData.py:434
def get_uptime(day, runs)
Definition: POTGetData.py:240
def write_json(jsonData, beam)
Definition: POTGetData.py:407
def get_runs(day, deltaDay, beam)
Definition: POTGetData.py:126
def database_connect(beam)
Definition: POTGetData.py:64
def get_spills(day, deltaDay, beam)
Definition: POTGetData.py:170
def pot(day, runs, spills)
Definition: POTGetData.py:337
def POTGetData.options (   option_arg)
Parse the command line options

  Returns:
  n (int) : Number of days to report
  b {bnb,numi} : Which beam to run POT plots for

Definition at line 434 of file POTGetData.py.

References makeTrainCVSamples.int, and runNovaSAM.str.

Referenced by main().

434 def options(option_arg):
435  """ Parse the command line options
436 
437  Returns:
438  n (int) : Number of days to report
439  b {bnb,numi} : Which beam to run POT plots for
440  """
441 
442  # Get options and arguments supplied by user
443  # options that require an argument have a : attached
444  try:
445  (opts, args) = getopt.gnu_getopt(sys.argv[1:], "n:b:", ["ndays=","beam="])
446  except getopt.GetoptError as err:
447  print str(err)
448  sys.exit(2)
449 
450  # Loop over arguments supplied
451  for opt, arg in opts:
452  if opt in ("-n", "--ndays"):
453  ndays = int(arg)
454  elif opt in ("-b", "--beam"):
455  beam = str(arg)
456  if option_arg == 1:
457  return ndays
458  if option_arg == 2:
459  return beam
460 
461 
462 #.......................................................................
463 # "MAIN" FUNCTION
464 #
465 #.......................................................................
def options(option_arg)
Definition: POTGetData.py:434
def POTGetData.pot (   day,
  runs,
  spills 
)
Compute the POT recorded and delivered for a given day

  Args:
  day    : A datetime object
  runs   : A Python object containing run information
  spills : A list of spills

  Returns:
  potRecorded  : int
  potDelivered : int

Definition at line 337 of file POTGetData.py.

References pot_between().

Referenced by main().

337 def pot(day, runs, spills):
338  """ Compute the POT recorded and delivered for a given day
339 
340  Args:
341  day : A datetime object
342  runs : A Python object containing run information
343  spills : A list of spills
344 
345  Returns:
346  potRecorded : int
347  potDelivered : int
348  """
349 
350  t0 = day
351  t1 = day + timedelta(days = 1)
352 
353  # Calculate how many POT were delivered on this day
354  potDelivered = pot_between(t0, t1, spills)
355 
356  # Loop over runs to determine how many POT were actually recorded
357  potRecorded = 0.0
358  for run in runs:
359  tStart = run['tstart']
360  tStop = run['tstop']
361 
362  # If the run started before the day began, we should truncate
363  # to ensure we're only looking at the portion from today. Likewise
364  # for runs that end after the day ended.
365  if (tStart < t0):
366  tStart = t0
367  if (tStop > t1):
368  tStop = t1
369 
370  # Calculate how many POT were recorded on this day
371  potRecorded += pot_between(tStart, tStop, spills)
372 
373  # Return both POT recorded and delivered
374  return (potRecorded, potDelivered)
375 
376 
377 #.......................................................................
def pot_between(t0, t1, spills)
Definition: POTGetData.py:309
def pot(day, runs, spills)
Definition: POTGetData.py:337
def POTGetData.pot_between (   t0,
  t1,
  spills 
)
Determines the number of POT in a given time interval

  Args:
  t0     : A datetime object
  t1     : A datetime object
  spills : A list of spills

  Returns:
  potTotal : int

Definition at line 309 of file POTGetData.py.

References unix_timestamp().

Referenced by uptime_graphic.do_pot(), POTUptimeData.do_pot(), and pot().

309 def pot_between(t0, t1, spills):
310  """ Determines the number of POT in a given time interval
311 
312  Args:
313  t0 : A datetime object
314  t1 : A datetime object
315  spills : A list of spills
316 
317  Returns:
318  potTotal : int
319  """
320 
321  # Convert to unix timestamps
322  t0 = unix_timestamp(t0)
323  t1 = unix_timestamp(t1)
324 
325  # Instantiate total
326  potTotal = 0.0
327 
328  for (time, value) in spills:
329  if ( (time >= t0) & (time <= t1) & (value > GOOD_SPILL_POT) ):
330  potTotal += value
331 
332  # Value from database is in units of 1E+12. Convert to 1E+18
333  return potTotal * (1.0E12 / 1.0E18)
334 
335 
336 #.......................................................................
def unix_timestamp(input)
Definition: POTGetData.py:46
def pot_between(t0, t1, spills)
Definition: POTGetData.py:309
def POTGetData.run_startstop (   runs)
Determine the first and final runs of the day

  Args:
  runs : A Python object with run information

  Returns:
  (runInitial, runFinal) : (int, int)

Definition at line 283 of file POTGetData.py.

Referenced by main().

283 def run_startstop(runs):
284  """ Determine the first and final runs of the day
285 
286  Args:
287  runs : A Python object with run information
288 
289  Returns:
290  (runInitial, runFinal) : (int, int)
291  """
292 
293  # Assign initial values
294  runInitial = runs[0]['run']
295  runFinal = runs[0]['run']
296 
297  # Loop through runs
298  for run in runs:
299  if (run['run'] < runInitial):
300  runInitial = run['run']
301  if (run['run'] > runFinal):
302  runFinal = run['run']
303 
304  # Return initial and final run numbers
305  return (runInitial, runFinal)
306 
307 
308 #.......................................................................
def run_startstop(runs)
Definition: POTGetData.py:283
def POTGetData.unix_timestamp (   input)
Convert a datetime to a unix timestamp

  Args:
  input : A datetime object

  Returns:
  difference : A unix timestamp

Definition at line 46 of file POTGetData.py.

Referenced by get_spills(), POTSpillRate.num_spills_between(), and pot_between().

46 def unix_timestamp(input):
47  """ Convert a datetime to a unix timestamp
48 
49  Args:
50  input : A datetime object
51 
52  Returns:
53  difference : A unix timestamp
54  """
55 
56  t0 = datetime(year = 1970, month = 1, day = 1)
57  difference = input - t0
58 
59  # Return unix timestamp
60  return difference.total_seconds()
61 
62 
63 #.......................................................................
def unix_timestamp(input)
Definition: POTGetData.py:46
def POTGetData.write_json (   jsonData,
  beam 
)
Writes jsonData to JSON file.

  Args:
  jsonData : A Python object

Definition at line 407 of file POTGetData.py.

References open(), and runNovaSAM.str.

Referenced by append_json(), main(), and novaddt::HotMapMaker.saveMetaData().

407 def write_json(jsonData,beam):
408  """ Writes jsonData to JSON file.
409 
410  Args:
411  jsonData : A Python object
412 
413  """
414 
415  # Open file for writing
416  if beam == str("numi"):
417  file = open(JSON_FILE_PATH, 'w')
418  elif beam == str("bnb"):
419  file = open(JSON_BNB_FILE_PATH, 'w')
420 
421  # Encode JSON and write to file
422  file.write(json.dumps(jsonData, indent = 2, sort_keys = True))
423 
424  # Close file
425  file.close()
426 
427  print "JSON file written"
428 
429 
430 #.......................................................................
431 # OPTIONS
432 #
433 #.......................................................................
def write_json(jsonData, beam)
Definition: POTGetData.py:407
procfile open("FD_BRL_v0.txt")

Variable Documentation

float POTGetData.GOOD_SPILL_POT = 0.05

Definition at line 39 of file POTGetData.py.

string POTGetData.JSON_BNB_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'

Definition at line 42 of file POTGetData.py.

string POTGetData.JSON_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'

Definition at line 40 of file POTGetData.py.

int POTGetData.ndays = 1

Definition at line 38 of file POTGetData.py.