POTPopulateData.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 """
3  Compute operations statistics (uptime, POT recorded, delivered, etc.)
4  for NOvA and overwrite json file. This script will be merged with
5  POTGetData.py in the future.
6 
7  Based on Commissioning/nova_uptime.py
8 
9  Fernanda Psihas (psihas@fnal.gov)
10  Justin Vasel (jvasel@fnal.gov)
11 
12  """
13 
14 #.......................................................................
15 # INCLUDES
16 #
17 #.......................................................................
18 # Retrieving options
19 import sys, os
20 import getopt
21 
22 import urllib2 # Retrieves contents of file at URL
23 import csv # CSV file i/o
24 
25 import json # JSON file i/o
26 
27 # Date and time stuff
28 from datetime import date, datetime, timedelta
29 
30 # PostgreSQL database adapter
31 import psycopg2
32 import psycopg2.extras
33 
34 #.......................................................................
35 # DEFINITIONS
36 #
37 #.......................................................................
38 # Real spills will have more than this number (time 1e12) POT recorded
39 GOOD_SPILL_POT = 0.05
40 JSON_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'\
41  'dynamicPOT/POTmon.json'
42 
43 
44 #.......................................................................
45 def unix_timestamp(input):
46  """ Convert a datetime to a unix timestamp
47 
48  Args:
49  input : A datetime object
50 
51  Returns:
52  difference : A unix timestamp
53  """
54 
55  t0 = datetime(year = 1970, month = 1, day = 1)
56  difference = input - t0
57 
58  # Return unix timestamp
59  return difference.total_seconds()
60 
61 
62 #.......................................................................
64  """ Initiate connection to psql database
65  """
66 
67  DB_NAME = os.environ['NOVADBNAME'] # 'nova_prod'
68  DB_HOST = os.environ['NOVADBHOST'] # 'ifdbrep.fnal.gov'
69  DB_USER = os.environ['NOVADBUSER'] # 'nova_reader'
70  DB_PASS = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
71  DB_PORT = os.environ['NOVADBPORT'] # '5433'
72 
73 
74  # Initiate connection to database
75  try:
76  db = psycopg2.connect(\
77  "dbname=%s host=%s user=%s password=%s port=%s" % \
78  (DB_NAME, DB_HOST, DB_USER, DB_PASS, DB_PORT))
79  except:
80  print "Unable to connect to the database"
81 
82  # Create cursor object so that we can send queries to the db.
83  # Note: The `cursor_factory' argument will create a dictionary
84  # cursor so that columns are returned as a dictionary so that we
85  # can access columns by their name instead of index.
86  global db_cursor
87  db_cursor = db.cursor(cursor_factory = psycopg2.extras.DictCursor)
88 
89 #.......................................................................
91  """ Return how many active channels there we at start of run
92 
93  Args:
94  run: Run number
95 
96  Returns:
97  Number of active channels (APD pixels)
98  """
99 
100  if (run == 0):
101  return 0;
102 
103  # Build the SQL query to be performed
104  query = "select nactivechan from fardet.runs where run=" + str(run)
105 
106  # Submit query to SQL. Store response as a Python object.
107  db_cursor.execute(query)
108  number = db_cursor.fetchall()
109 
110  # Return the number of active channels
111  return number[0]['nactivechan']
112 
113 
114 #.......................................................................
115 def get_runs(day, deltaDay):
116  """ Get info about runs that start/stop within a particular date range
117 
118  Args:
119  day : A datetime object
120  deltaDay : Number of days to consider
121 
122  Returns:
123  runs : A Python object with the following information:
124  run number, run start time, and run stop time
125  """
126 
127  day1 = day + timedelta(days = deltaDay)
128 
129  # Build the SQL query to be performed
130  select = "select run,tstart,tstop from fardet.runs"
131 
132  cut1 = " where partition=1"
133  cut2 = " tstop is not null"
134 
135  cut3 = "tstart >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
136  cut4 = "tstart < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
137 
138  cut5 = "tstop >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
139  cut6 = "tstop < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
140 
141  sort = "order by tstart asc";
142 
143  query = select + cut1 + " and " + cut2 + " and " + \
144  "((" + cut3+" and "+cut4+") or ("+cut5+" and "+cut6+")) " + \
145  sort
146 
147  # Submit query to SQL. Store response as a Python object.
148  db_cursor.execute(query)
149  runs = db_cursor.fetchall()
150 
151  # Return run info
152  return runs
153 
154 
155 #.......................................................................
156 def get_spills(day, deltaDay):
157  """ Get info about spills during a particular day
158 
159  Args:
160  day : A datetime object
161  deltaDay : Number of days to consider
162 
163  Returns:
164  spills : A list of spills (time in sec, pot number in 1E+12)
165  """
166 
167  spills = []
168 
169  # Convert start and end of day to UNIX timestamp
170  t0 = unix_timestamp(day)
171  t1 = unix_timestamp(day + timedelta(days = deltaDay))
172 
173  # URL where the spill data live
174  url = 'http://ifb-data.fnal.gov:8099/ifbeam/data/data?'\
175  'v=E:TRTGTD&e=e,a9&'\
176  't0={}&t1={}&&f=csv'.format(t0, t1)
177 
178  # Store the HTTP GET response here
179  response = None
180 
181  # Try to retrieve the URL contents. If this fails three times, then
182  # give up and return an empty array.
183  try:
184  response = urllib2.urlopen(url, None, 200)
185  except:
186  try:
187  response = urllib2.urlopen(url, None, 200)
188  except:
189  try:
190  response = urllib2.urlopen(url, None, 200)
191  except:
192  return spills
193 
194  if (response == None):
195  return spills
196 
197  # Use csv module to parse the response as CSV
198  csvdata = csv.reader(response)
199 
200  # The response will look like this:
201  # "e,a9", E:TRTGTD, 1406851201047, E12, 22.0598346211
202  # We are interested in the third column (unix timestamp in ms)
203  # and the fifth column (value). The list will be populated with these
204  # values.
205  for row in csvdata:
206  if (row.count('E:TRTGTD') > 0):
207  if (row[4] != 'null'):
208  spills.append( (float(row[2]) / 1000., float(row[4])) )
209 
210  # Return spill info
211  return spills
212 
213 
214 #.......................................................................
215 def get_uptime(day, runs):
216  """ Compute total and fractional run uptime
217 
218  Args:
219  day : A datetime object
220  runs : A list of runs for the day
221 
222  Returns:
223  (uptimeTotal, uptimeFraction) : (datetime object, int)
224  """
225 
226  t0 = day
227  t1 = day + timedelta(days = 1)
228 
229  # Instantiate total uptime as datetime object
230  uptimeTotal = timedelta(0)
231 
232  # Loop over runs
233  for run in runs:
234  tStart = run['tstart']
235  tStop = run['tstop']
236 
237  # If the run started before the day began, we should truncate
238  # to ensure we're only looking at the portion from today. Likewise
239  # for runs that end after the day ended.
240  if (tStart < t0):
241  tStart = t0
242  if (tStop > t1):
243  tStop = t1
244 
245  uptimeTotal += (tStop - tStart)
246 
247  # Calculate fractional uptime
248  uptimeFraction = uptimeTotal.seconds / (24.0 * 60.0 * 60.0)
249 
250  # Return both the total and fractional uptime
251  return (uptimeTotal, uptimeFraction)
252 
253 
254 #.......................................................................
255 def run_startstop(runs):
256  """ Determine the first and final runs of the day
257 
258  Args:
259  runs : A Python object with run information
260 
261  Returns:
262  (runInitial, runFinal) : (int, int)
263  """
264 
265  # Assign initial values
266  runInitial = runs[0]['run']
267  runFinal = runs[0]['run']
268 
269  # Loop through runs
270  for run in runs:
271  if (run['run'] < runInitial):
272  runInitial = run['run']
273  if (run['run'] > runFinal):
274  runFinal = run['run']
275 
276  # Return initial and final run numbers
277  return (runInitial, runFinal)
278 
279 
280 #.......................................................................
281 def pot_between(t0, t1, spills):
282  """ Determines the number of POT in a given time interval
283 
284  Args:
285  t0 : A datetime object
286  t1 : A datetime object
287  spills : A list of spills
288 
289  Returns:
290  potTotal : int
291  """
292 
293  # Convert to unix timestamps
294  t0 = unix_timestamp(t0)
295  t1 = unix_timestamp(t1)
296 
297  # Instantiate total
298  potTotal = 0.0
299 
300  for (time, value) in spills:
301  if ( (time >= t0) & (time <= t1) & (value > GOOD_SPILL_POT) ):
302  potTotal += value
303 
304  # Value from database is in units of 1E+12. Convert to 1E+18
305  return potTotal * (1.0E12 / 1.0E18)
306 
307 
308 #.......................................................................
309 def pot(day, runs, spills):
310  """ Compute the POT recorded and delivered for a given day
311 
312  Args:
313  day : A datetime object
314  runs : A Python object containing run information
315  spills : A list of spills
316 
317  Returns:
318  potRecorded : int
319  potDelivered : int
320  """
321 
322  t0 = day
323  t1 = day + timedelta(days = 1)
324 
325  # Calculate how many POT were delivered on this day
326  potDelivered = pot_between(t0, t1, spills)
327 
328  # Loop over runs to determine how many POT were actually recorded
329  potRecorded = 0.0
330  for run in runs:
331  tStart = run['tstart']
332  tStop = run['tstop']
333 
334  # If the run started before the day began, we should truncate
335  # to ensure we're only looking at the portion from today. Likewise
336  # for runs that end after the day ended.
337  if (tStart < t0):
338  tStart = t0
339  if (tStop > t1):
340  tStop = t1
341 
342  # Calculate how many POT were recorded on this day
343  potRecorded += pot_between(tStart, tStop, spills)
344 
345  # Return both POT recorded and delivered
346  return (potRecorded, potDelivered)
347 
348 
349 #.......................................................................
350 # OPTIONS
351 #
352 #.......................................................................
353 def options():
354  """ Parse the command line options
355 
356  Returns:
357  n (int) : Number of days to report
358  """
359 
360  ndays = 30
361 
362  # Get options and arguments supplied by user
363  try:
364  (opts, args) = getopt.getopt(sys.argv[1:], "n:", ["ndays="])
365  except getopt.GetoptError as err:
366  print str(err)
367  sys.exit(2)
368 
369  # Loop over arguments supplied
370  for opt, arg in opts:
371  if opt in ("-n", "--ndays"):
372  ndays = int(arg)
373 
374  return ndays
375 
376 
377 #.......................................................................
378 # "MAIN" FUNCTION
379 #
380 #.......................................................................
381 def main():
382 
383  # Grab user-chosen values from options
384  ndays = options()
385 
386  # Initiate connection to psql database
388 
389  # Find today's date. This will serve as the final day to be included.
390  # Subtract off the number of days to be included, to find the start
391  # date.
392  t1 = datetime.utcnow()
393  t1 = t1.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
394  t0 = t1 - timedelta(days = ndays)
395 
396  # Initialize the array that will hold the data for each day to
397  # be written to a JSON file
398  jsonData = []
399 
400  # Print output column labels
401  print " "
402  print "No. Date Run1 Run2 Uptime "\
403  "Uptime (frac) POT Recorded POT Delivered POT Rec / Del"
404  print "====================================================="\
405  "==================================================================="
406 
407  # Loop over days
408  for i in range(0, ndays):
409 
410  # Find the ith date
411  day = t0 + timedelta(days = i)
412 
413  # Get the list of run information for this day
414  runs = get_runs(day, 1)
415 
416  # Find the first and final run number for this day
417  runInitial = 0
418  runFinal = 0
419  if (runs):
420  (runInitial, runFinal) = run_startstop(runs)
421 
422  # Compute uptime (total and fractional) for this day
423  uptimeTotal = 0
424  uptimeFrac = 0
425  if (runs):
426  (uptimeTotal, uptimeFrac) = get_uptime(day, runs)
427 
428  # Get the list of spill information for this day
429  spills = get_spills(day, 1)
430 
431  # Compute how many protons were delivered and recorded
432  (potRecorded, potDelivered) = pot(day, runs, spills)
433 
434  # Compute the fraction of POT recorded. In cases where the POT
435  # delivered is zero, record the fraction recorded as the uptime
436  # computed above
437  potFraction = 0.0
438  if (potDelivered == 0.0):
439  potFraction = uptimeFrac
440  else:
441  potFraction = potRecorded / potDelivered
442 
443  # Reassign day so that the timestamp (00:00:00) doesn't appear
444  day = date(day.year, day.month, day.day)
445 
446  # Print the results to the screen
447  print "",i + 1," ",\
448  day," ",\
449  runInitial," ",\
450  runFinal," ",\
451  uptimeTotal," ",\
452  uptimeFrac," ",\
453  potRecorded," ",\
454  potDelivered," ",\
455  potFraction
456 
457  # Build the JSON entry for this day
458  jsonEntry = {"date" : str(day),\
459  "runStart" : str(runInitial),\
460  "runEnd" : str(runFinal),\
461  "uptime_hours" : str(uptimeTotal),\
462  "uptime_frac" : str(uptimeFrac),\
463  "pot_record" : str(potRecorded),\
464  "pot_deliv" : str(potDelivered),\
465  "pot_daily_frac" : str(potFraction)}
466 
467  # Append the JSON entry to the jsonData array
468  jsonData.append(jsonEntry)
469 
470  # Encode the array into JSON and write to a file
471  file = open(JSON_FILE_PATH, 'w')
472  file.write(json.dumps(jsonData, indent = 2))
473  file.close()
474 
475  print " "
476 
477 
478 #.......................................................................
479 
480 # Prevent execution on import
481 if __name__ == "__main__":
482  main()
483 ########################################################################
484 
::xsd::cxx::tree::date< char, simple_type > date
Definition: Database.h:186
def run_startstop(runs)
def get_runs(day, deltaDay)
def get_active_chans(run)
def get_spills(day, deltaDay)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
def pot_between(t0, t1, spills)
def unix_timestamp(input)
def pot(day, runs, spills)
def get_uptime(day, runs)