Functions | Variables
POTPopulateData Namespace Reference

Functions

def unix_timestamp (input)
 
def database_connect ()
 
def get_active_chans (run)
 
def get_runs (day, deltaDay)
 
def get_spills (day, deltaDay)
 
def get_uptime (day, runs)
 
def run_startstop (runs)
 
def pot_between (t0, t1, spills)
 
def pot (day, runs, spills)
 
def options ()
 
def main ()
 

Variables

float GOOD_SPILL_POT = 0.05
 
string JSON_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'
 

Detailed Description

  Compute operations statistics (uptime, POT recorded, delivered, etc.)
  for NOvA and overwrite json file. This script will be merged with 
  POTGetData.py in the future.
  
  Based on Commissioning/nova_uptime.py
  
  Fernanda Psihas (psihas@fnal.gov)
  Justin Vasel (jvasel@fnal.gov)

Function Documentation

def POTPopulateData.database_connect ( )
Initiate connection to psql database

Definition at line 63 of file POTPopulateData.py.

References open().

Referenced by main().

64  """ Initiate connection to psql database
65  """
66 
67  DB_NAME = os.environ['NOVADBNAME'] # 'nova_prod'
68  DB_HOST = os.environ['NOVADBHOST'] # 'ifdbrep.fnal.gov'
69  DB_USER = os.environ['NOVADBUSER'] # 'nova_reader'
70  DB_PASS = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
71  DB_PORT = os.environ['NOVADBPORT'] # '5433'
72 
73 
74  # Initiate connection to database
75  try:
76  db = psycopg2.connect(\
77  "dbname=%s host=%s user=%s password=%s port=%s" % \
78  (DB_NAME, DB_HOST, DB_USER, DB_PASS, DB_PORT))
79  except:
80  print "Unable to connect to the database"
81 
82  # Create cursor object so that we can send queries to the db.
83  # Note: The `cursor_factory' argument will create a dictionary
84  # cursor so that columns are returned as a dictionary so that we
85  # can access columns by their name instead of index.
86  global db_cursor
87  db_cursor = db.cursor(cursor_factory = psycopg2.extras.DictCursor)
88 
89 #.......................................................................
procfile open("FD_BRL_v0.txt")
def POTPopulateData.get_active_chans (   run)
Return how many active channels there we at start of run

  Args:
  run: Run number

  Returns:
  Number of active channels (APD pixels)

Definition at line 90 of file POTPopulateData.py.

References runNovaSAM.str.

91  """ Return how many active channels there we at start of run
92 
93  Args:
94  run: Run number
95 
96  Returns:
97  Number of active channels (APD pixels)
98  """
99 
100  if (run == 0):
101  return 0;
102 
103  # Build the SQL query to be performed
104  query = "select nactivechan from fardet.runs where run=" + str(run)
105 
106  # Submit query to SQL. Store response as a Python object.
107  db_cursor.execute(query)
108  number = db_cursor.fetchall()
109 
110  # Return the number of active channels
111  return number[0]['nactivechan']
112 
113 
114 #.......................................................................
def get_active_chans(run)
def POTPopulateData.get_runs (   day,
  deltaDay 
)
Get info about runs that start/stop within a particular date range
  
  Args:
  day : A datetime object
  deltaDay : Number of days to consider

  Returns:
  runs : A Python object with the following information:
          run number, run start time, and run stop time

Definition at line 115 of file POTPopulateData.py.

References novadaq::HexUtils.format().

Referenced by main().

115 def get_runs(day, deltaDay):
116  """ Get info about runs that start/stop within a particular date range
117 
118  Args:
119  day : A datetime object
120  deltaDay : Number of days to consider
121 
122  Returns:
123  runs : A Python object with the following information:
124  run number, run start time, and run stop time
125  """
126 
127  day1 = day + timedelta(days = deltaDay)
128 
129  # Build the SQL query to be performed
130  select = "select run,tstart,tstop from fardet.runs"
131 
132  cut1 = " where partition=1"
133  cut2 = " tstop is not null"
134 
135  cut3 = "tstart >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
136  cut4 = "tstart < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
137 
138  cut5 = "tstop >= '{}'".format(day.strftime("%Y-%m-%dT%H:%M:%S"))
139  cut6 = "tstop < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
140 
141  sort = "order by tstart asc";
142 
143  query = select + cut1 + " and " + cut2 + " and " + \
144  "((" + cut3+" and "+cut4+") or ("+cut5+" and "+cut6+")) " + \
145  sort
146 
147  # Submit query to SQL. Store response as a Python object.
148  db_cursor.execute(query)
149  runs = db_cursor.fetchall()
150 
151  # Return run info
152  return runs
153 
154 
155 #.......................................................................
def get_runs(day, deltaDay)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def POTPopulateData.get_spills (   day,
  deltaDay 
)
Get info about spills during a particular day
  
  Args:
  day : A datetime object
  deltaDay : Number of days to consider

  Returns:
  spills : A list of spills (time in sec, pot number in 1E+12)

Definition at line 156 of file POTPopulateData.py.

References check_time_usage.float, novadaq::HexUtils.format(), and unix_timestamp().

Referenced by main().

156 def get_spills(day, deltaDay):
157  """ Get info about spills during a particular day
158 
159  Args:
160  day : A datetime object
161  deltaDay : Number of days to consider
162 
163  Returns:
164  spills : A list of spills (time in sec, pot number in 1E+12)
165  """
166 
167  spills = []
168 
169  # Convert start and end of day to UNIX timestamp
170  t0 = unix_timestamp(day)
171  t1 = unix_timestamp(day + timedelta(days = deltaDay))
172 
173  # URL where the spill data live
174  url = 'http://ifb-data.fnal.gov:8099/ifbeam/data/data?'\
175  'v=E:TRTGTD&e=e,a9&'\
176  't0={}&t1={}&&f=csv'.format(t0, t1)
177 
178  # Store the HTTP GET response here
179  response = None
180 
181  # Try to retrieve the URL contents. If this fails three times, then
182  # give up and return an empty array.
183  try:
184  response = urllib2.urlopen(url, None, 200)
185  except:
186  try:
187  response = urllib2.urlopen(url, None, 200)
188  except:
189  try:
190  response = urllib2.urlopen(url, None, 200)
191  except:
192  return spills
193 
194  if (response == None):
195  return spills
196 
197  # Use csv module to parse the response as CSV
198  csvdata = csv.reader(response)
199 
200  # The response will look like this:
201  # "e,a9", E:TRTGTD, 1406851201047, E12, 22.0598346211
202  # We are interested in the third column (unix timestamp in ms)
203  # and the fifth column (value). The list will be populated with these
204  # values.
205  for row in csvdata:
206  if (row.count('E:TRTGTD') > 0):
207  if (row[4] != 'null'):
208  spills.append( (float(row[2]) / 1000., float(row[4])) )
209 
210  # Return spill info
211  return spills
212 
213 
214 #.......................................................................
def get_spills(day, deltaDay)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def unix_timestamp(input)
def POTPopulateData.get_uptime (   day,
  runs 
)
Compute total and fractional run uptime
  
  Args:
  day  : A datetime object
  runs : A list of runs for the day

  Returns:
  (uptimeTotal, uptimeFraction) : (datetime object, int)

Definition at line 215 of file POTPopulateData.py.

Referenced by main().

215 def get_uptime(day, runs):
216  """ Compute total and fractional run uptime
217 
218  Args:
219  day : A datetime object
220  runs : A list of runs for the day
221 
222  Returns:
223  (uptimeTotal, uptimeFraction) : (datetime object, int)
224  """
225 
226  t0 = day
227  t1 = day + timedelta(days = 1)
228 
229  # Instantiate total uptime as datetime object
230  uptimeTotal = timedelta(0)
231 
232  # Loop over runs
233  for run in runs:
234  tStart = run['tstart']
235  tStop = run['tstop']
236 
237  # If the run started before the day began, we should truncate
238  # to ensure we're only looking at the portion from today. Likewise
239  # for runs that end after the day ended.
240  if (tStart < t0):
241  tStart = t0
242  if (tStop > t1):
243  tStop = t1
244 
245  uptimeTotal += (tStop - tStart)
246 
247  # Calculate fractional uptime
248  uptimeFraction = uptimeTotal.seconds / (24.0 * 60.0 * 60.0)
249 
250  # Return both the total and fractional uptime
251  return (uptimeTotal, uptimeFraction)
252 
253 
254 #.......................................................................
def get_uptime(day, runs)
def POTPopulateData.main ( )

1D plots hits vs w //

Definition at line 381 of file POTPopulateData.py.

References database_connect(), get_runs(), get_spills(), get_uptime(), open(), options(), pot(), PandAna.Demos.demo1.range, run_startstop(), and runNovaSAM.str.

381 def main():
382 
383  # Grab user-chosen values from options
384  ndays = options()
385 
386  # Initiate connection to psql database
388 
389  # Find today's date. This will serve as the final day to be included.
390  # Subtract off the number of days to be included, to find the start
391  # date.
392  t1 = datetime.utcnow()
393  t1 = t1.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
394  t0 = t1 - timedelta(days = ndays)
395 
396  # Initialize the array that will hold the data for each day to
397  # be written to a JSON file
398  jsonData = []
399 
400  # Print output column labels
401  print " "
402  print "No. Date Run1 Run2 Uptime "\
403  "Uptime (frac) POT Recorded POT Delivered POT Rec / Del"
404  print "====================================================="\
405  "==================================================================="
406 
407  # Loop over days
408  for i in range(0, ndays):
409 
410  # Find the ith date
411  day = t0 + timedelta(days = i)
412 
413  # Get the list of run information for this day
414  runs = get_runs(day, 1)
415 
416  # Find the first and final run number for this day
417  runInitial = 0
418  runFinal = 0
419  if (runs):
420  (runInitial, runFinal) = run_startstop(runs)
421 
422  # Compute uptime (total and fractional) for this day
423  uptimeTotal = 0
424  uptimeFrac = 0
425  if (runs):
426  (uptimeTotal, uptimeFrac) = get_uptime(day, runs)
427 
428  # Get the list of spill information for this day
429  spills = get_spills(day, 1)
430 
431  # Compute how many protons were delivered and recorded
432  (potRecorded, potDelivered) = pot(day, runs, spills)
433 
434  # Compute the fraction of POT recorded. In cases where the POT
435  # delivered is zero, record the fraction recorded as the uptime
436  # computed above
437  potFraction = 0.0
438  if (potDelivered == 0.0):
439  potFraction = uptimeFrac
440  else:
441  potFraction = potRecorded / potDelivered
442 
443  # Reassign day so that the timestamp (00:00:00) doesn't appear
444  day = date(day.year, day.month, day.day)
445 
446  # Print the results to the screen
447  print "",i + 1," ",\
448  day," ",\
449  runInitial," ",\
450  runFinal," ",\
451  uptimeTotal," ",\
452  uptimeFrac," ",\
453  potRecorded," ",\
454  potDelivered," ",\
455  potFraction
456 
457  # Build the JSON entry for this day
458  jsonEntry = {"date" : str(day),\
459  "runStart" : str(runInitial),\
460  "runEnd" : str(runFinal),\
461  "uptime_hours" : str(uptimeTotal),\
462  "uptime_frac" : str(uptimeFrac),\
463  "pot_record" : str(potRecorded),\
464  "pot_deliv" : str(potDelivered),\
465  "pot_daily_frac" : str(potFraction)}
466 
467  # Append the JSON entry to the jsonData array
468  jsonData.append(jsonEntry)
469 
470  # Encode the array into JSON and write to a file
471  file = open(JSON_FILE_PATH, 'w')
472  file.write(json.dumps(jsonData, indent = 2))
473  file.close()
474 
475  print " "
476 
477 
478 #.......................................................................
479 
480 # Prevent execution on import
::xsd::cxx::tree::date< char, simple_type > date
Definition: Database.h:186
def run_startstop(runs)
def get_runs(day, deltaDay)
def get_spills(day, deltaDay)
procfile open("FD_BRL_v0.txt")
def pot(day, runs, spills)
def get_uptime(day, runs)
def POTPopulateData.options ( )
Parse the command line options

  Returns:
  n (int) : Number of days to report

Definition at line 353 of file POTPopulateData.py.

References makeTrainCVSamples.int, and runNovaSAM.str.

Referenced by main().

353 def options():
354  """ Parse the command line options
355 
356  Returns:
357  n (int) : Number of days to report
358  """
359 
360  ndays = 30
361 
362  # Get options and arguments supplied by user
363  try:
364  (opts, args) = getopt.getopt(sys.argv[1:], "n:", ["ndays="])
365  except getopt.GetoptError as err:
366  print str(err)
367  sys.exit(2)
368 
369  # Loop over arguments supplied
370  for opt, arg in opts:
371  if opt in ("-n", "--ndays"):
372  ndays = int(arg)
373 
374  return ndays
375 
376 
377 #.......................................................................
378 # "MAIN" FUNCTION
379 #
380 #.......................................................................
def POTPopulateData.pot (   day,
  runs,
  spills 
)
Compute the POT recorded and delivered for a given day
  
  Args:
  day    : A datetime object
  runs   : A Python object containing run information
  spills : A list of spills

  Returns:
  potRecorded  : int
  potDelivered : int

Definition at line 309 of file POTPopulateData.py.

References pot_between().

Referenced by main().

309 def pot(day, runs, spills):
310  """ Compute the POT recorded and delivered for a given day
311 
312  Args:
313  day : A datetime object
314  runs : A Python object containing run information
315  spills : A list of spills
316 
317  Returns:
318  potRecorded : int
319  potDelivered : int
320  """
321 
322  t0 = day
323  t1 = day + timedelta(days = 1)
324 
325  # Calculate how many POT were delivered on this day
326  potDelivered = pot_between(t0, t1, spills)
327 
328  # Loop over runs to determine how many POT were actually recorded
329  potRecorded = 0.0
330  for run in runs:
331  tStart = run['tstart']
332  tStop = run['tstop']
333 
334  # If the run started before the day began, we should truncate
335  # to ensure we're only looking at the portion from today. Likewise
336  # for runs that end after the day ended.
337  if (tStart < t0):
338  tStart = t0
339  if (tStop > t1):
340  tStop = t1
341 
342  # Calculate how many POT were recorded on this day
343  potRecorded += pot_between(tStart, tStop, spills)
344 
345  # Return both POT recorded and delivered
346  return (potRecorded, potDelivered)
347 
348 
349 #.......................................................................
350 # OPTIONS
351 #
352 #.......................................................................
def pot_between(t0, t1, spills)
def pot(day, runs, spills)
def POTPopulateData.pot_between (   t0,
  t1,
  spills 
)
Determines the number of POT in a given time interval
  
  Args:
  t0     : A datetime object
  t1     : A datetime object
  spills : A list of spills

  Returns:
  potTotal : int

Definition at line 281 of file POTPopulateData.py.

References unix_timestamp().

Referenced by pot().

281 def pot_between(t0, t1, spills):
282  """ Determines the number of POT in a given time interval
283 
284  Args:
285  t0 : A datetime object
286  t1 : A datetime object
287  spills : A list of spills
288 
289  Returns:
290  potTotal : int
291  """
292 
293  # Convert to unix timestamps
294  t0 = unix_timestamp(t0)
295  t1 = unix_timestamp(t1)
296 
297  # Instantiate total
298  potTotal = 0.0
299 
300  for (time, value) in spills:
301  if ( (time >= t0) & (time <= t1) & (value > GOOD_SPILL_POT) ):
302  potTotal += value
303 
304  # Value from database is in units of 1E+12. Convert to 1E+18
305  return potTotal * (1.0E12 / 1.0E18)
306 
307 
308 #.......................................................................
def pot_between(t0, t1, spills)
def unix_timestamp(input)
def POTPopulateData.run_startstop (   runs)
Determine the first and final runs of the day
  
  Args:
  runs : A Python object with run information

  Returns:
  (runInitial, runFinal) : (int, int)

Definition at line 255 of file POTPopulateData.py.

Referenced by main().

255 def run_startstop(runs):
256  """ Determine the first and final runs of the day
257 
258  Args:
259  runs : A Python object with run information
260 
261  Returns:
262  (runInitial, runFinal) : (int, int)
263  """
264 
265  # Assign initial values
266  runInitial = runs[0]['run']
267  runFinal = runs[0]['run']
268 
269  # Loop through runs
270  for run in runs:
271  if (run['run'] < runInitial):
272  runInitial = run['run']
273  if (run['run'] > runFinal):
274  runFinal = run['run']
275 
276  # Return initial and final run numbers
277  return (runInitial, runFinal)
278 
279 
280 #.......................................................................
def run_startstop(runs)
def POTPopulateData.unix_timestamp (   input)
Convert a datetime to a unix timestamp
  
  Args:
  input : A datetime object

  Returns:
  difference : A unix timestamp

Definition at line 45 of file POTPopulateData.py.

Referenced by get_spills(), and pot_between().

45 def unix_timestamp(input):
46  """ Convert a datetime to a unix timestamp
47 
48  Args:
49  input : A datetime object
50 
51  Returns:
52  difference : A unix timestamp
53  """
54 
55  t0 = datetime(year = 1970, month = 1, day = 1)
56  difference = input - t0
57 
58  # Return unix timestamp
59  return difference.total_seconds()
60 
61 
62 #.......................................................................
def unix_timestamp(input)

Variable Documentation

float POTPopulateData.GOOD_SPILL_POT = 0.05

Definition at line 39 of file POTPopulateData.py.

string POTPopulateData.JSON_FILE_PATH = '/nusoft/app/web/htdoc/nova/datacheck/nearline/'

Definition at line 40 of file POTPopulateData.py.