MCenterSpills.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #..................................................
4 # INCLUDES
5 #..................................................
6 import sys, os
7 import urllib2, csv
8 import getopt
9 from datetime import date, datetime, timedelta
10 import psycopg2
11 import psycopg2.extras
12 
13 # Change to output directory if you want to specify where the root file goes
14 BASE_DIR = os.getcwd()
15 filename = 'MCenterSpills.root'
16 
17 DETECTOR = 'testbeam'
18 DB_CURSOR = None
19 DB_QUERY_TIME = timedelta(0)
20 
21 #..................................................
22 def usage():
23  """ Print out usage.
24  """
25  print "Usage:"
26  print "python MCenterSpills.py -s YYYY-mm-dd -e YYYY-mm-dd -n #"
27  print ""
28  print "Where:"
29  print "-s : start date"
30  print "-e : end date"
31  print "-n : number of days"
32  print ""
33  print "You can specify any 2 of the above 3 options. The third will be calculated for you."
34  print "There is no default, you must supply options"
35  print ""
36  print "Creates root file with date, first and last run, number of recorded spills, and recorded POT [counts] for each day."
37 
38 #..................................................
39 def options():
40  """ Parse the command line options
41  Returns:
42  t1 (datetime) : First day to report
43  t2 (datetime) : Last day to report
44  n (int) : Number of days to report
45  """
46  date_start = None
47  date_end = None
48  ndays = None
49 
50  global DETECTOR
51 
52  try:
53  (opts, args) = getopt.getopt(sys.argv[1:],\
54  "hs:e:n:",\
55  ["help",\
56  "start=",\
57  "end=",\
58  "ndays="])
59  except getopt.GetoptError as err:
60  print str(err)
61  sys.exit(2)
62  for opt, arg in opts:
63  if opt in ("-h", "--help"):
64  print "help opt"
65  usage()
66  exit(0)
67  elif opt in ("-s", "--start"):
68  date_start = arg
69  elif opt in ("-e", "--end"):
70  date_end = arg
71  elif opt in ("-n", "--ndays"):
72  ndays = int(arg)
73  else:
74  assert False, "unknown option"
75 
76 
77  if (date_start!=None):
78  t0 = datetime.strptime(date_start,'%Y-%m-%d')
79  if (date_end!=None):
80  t1 = datetime.strptime(date_end,'%Y-%m-%d')
81 
82  # missing start date - use end date and ndays to construct
83  if ((date_start==None)&(date_end!=None)&(ndays!=None)):
84  t0 = t1
85  t0 -= timedelta(days=(ndays-1))
86 
87  # missing end date - use start date and ndays to construct
88  if ((date_start!=None)&(date_end==None)&(ndays!=None)):
89  t1 = t0
90  t1 += timedelta(days=ndays)
91 
92  # missing ndays - use start and end dates to construct
93  if ((date_start!=None)&(date_end!=None)&(ndays==None)):
94  n = (t1-t0)+timedelta(days=1)
95  ndays = int(n.days)
96 
97  if ((date_start!=None)&(date_end!=None)&(ndays!=None)):
98  #
99  # Have everything we need, but are they consistent?
100  #
101  d = (t1-t0)+timedelta(days=1)
102  n = int(d.days)
103  if (ndays != n):
104  print 'Inconsistent time range'
105  exit(2)
106 
107  return (t0,t1,ndays)
108 
109 #..................................................
110 
111 def runs_by_day(day0):
112  """ Get a complete list of run numbers for a specified day
113 
114  Any run that starts or stops on the specified date is contaned in
115  the list
116 
117  Args:
118  day0: A datetime object for the requested day
119 
120  Returns:
121  An list of SQL results for run,tstart,tstop ordered by start time
122  """
123  day1 = day0+timedelta(days=1)
124  return runs_for_period(day0, day1)
125 
126 #..................................................
127 
128 def runs_for_period(day0, day1):
129  """ Get all runs that start or stop within a particular date range
130 
131  Args:
132  day0 : A datetime object for first day
133  day1 : A datetime object for the last day
134  """
135  sql0 = "select run,tstart,tstop from "+DETECTOR+".runs";
136  cut1 = " where partition=1";
137  cut2 = " tstop is not null"
138  cut3 = "tstart >= '{}'".format(day0.strftime("%Y-%m-%dT%H:%M:%S"))
139  cut4 = "tstart < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
140 
141  cut5 = "tstop >= '{}'".format(day0.strftime("%Y-%m-%dT%H:%M:%S"))
142  cut6 = "tstop < '{}'".format(day1.strftime("%Y-%m-%dT%H:%M:%S"))
143 
144  sql1 = "order by tstart asc"
145 
146  SQL = \
147  sql0 + \
148  cut1 + " and " + \
149  cut2 + " and " + \
150  "((" + cut3+" and "+cut4+") or ("+cut5+" and "+cut6+")) " + \
151  sql1
152 
153 
154  DB_CURSOR.execute(SQL);
155  runs = DB_CURSOR.fetchall();
156 
157  return runs
158 
159 #..................................................
160 
161 def run_minmax(runs):
162  """ Find the highest and lowest run number in a list
163 
164  Args:
165  runs: A list of SQL results (run,tstart,tstop)
166 
167  Returns:
168  (runlo,runhi), the lowest and highest run numbers in the list
169  """
170  runmin = runs[0]['run']
171  runmax = runs[0]['run']
172  for r in runs:
173  if (r['run']<runmin):
174  runmin = r['run']
175  if (r['run']>runmax):
176  runmax = r['run']
177  return (runmin,runmax)
178 
179 #..................................................
180 
181 def mc7_spills_between(t1, t2):
182  """
183  Return spill information (unix time and POT) for a datetime range
184 
185  Args:
186  t1 (datetime): Start time
187  t2 (datetime): End time
188 
189  Returns:
190  List of (time,pot) tuples
191  """
192  global DB_QUERY_TIME
193 
194  spills = [];
195 
196  tf1 = in_unix(t1)
197  tf2 = in_unix(t2)
198  webaddress = 'http://ifb-data.fnal.gov:8099'\
199  '/ifbeam/data/data?'\
200  'v=F:MC7SC1&e=e,36&t0={}&t1={}&&f=csv'.\
201  format(tf1,tf2)
202 
203  dd = None
204  #
205  # Make three tries and then bail...
206  #
207  t0 = datetime.utcnow()
208  try:
209  dd = urllib2.urlopen(webaddress,None,600)
210  except:
211  try:
212  dd = urllib2.urlopen(webaddress,None,600)
213  except:
214  try:
215  dd = urllib2.urlopen(webaddress,None,600)
216  except:
217  return spills
218  t1 = datetime.utcnow()
219 
220  DB_QUERY_TIME += (t1-t0)
221 
222  if (dd==None):
223  return spills
224 
225  csvdata = csv.reader(dd)
226 
227  for row in csvdata:
228  if (row.count('F:MC7SC1')>0):
229  if (row[4]!='null'):
230  spills.append( (float(row[2])/1000., float(row[4])) )
231 
232  return spills
233 
234 #..................................................
235 
236 def compute_pot(day,runs,spills):
237  """ Tally the protons on target delivered and recorded in a day
238 
239  Args:
240  day : datetime of day of interest
241  runs (sql) : SQL set of runs (run, tstart, tend)
242  spills (list[(time,pot)]) : Complete list of spills for time period
243 
244  Returns:
245  (pot_recorded,pot_delivered) (float,float)
246  """
247  daystart = day
248  dayend = day + timedelta(days=1)
249 
250  pot_delivered = pot_between(daystart,dayend,spills)
251 
252  pot_recorded = 0.0
253  spills_recorded = 0
254  for r in runs:
255  t1 = r['tstart']
256  t2 = r['tstop']
257  #
258  # Handle end points. Runs that carried into this day and runs
259  # that extedned beyond the end of this day should be pinned to
260  # the start and end of the day.
261  #
262  if (t1<daystart):
263  t1 = daystart
264  if (t2>dayend):
265  t2 = dayend
266 
267  pot_recorded += pot_between(t1,t2,spills)
268  spills_recorded += spills_between(t1,t2,spills)
269 
270  return (pot_recorded,pot_delivered,spills_recorded)
271 
272 #..................................................
273 
274 def pot_between(t1,t2,spills):
275  """ Compute the POT recorded between two time stamps.
276 
277  Args:
278  t1 (datetime) : Start time
279  t2 (datetime) : End time
280  spills ( list[(time,pot)]) : Complete list of spills for time period
281 
282  Returns:
283  pot (float) : Scintillator counts
284  """
285 
286  tf1 = in_unix(t1)
287  tf2 = in_unix(t2)
288 
289  sum = 0.0
290  for (t,q) in spills:
291  if (t>=tf1)&(t<=tf2):
292  sum += q
293 
294  return sum
295 
296 #..................................................
297 
298 def spills_between(t1,t2,spills):
299  """ Compute the POT recorded between two time stamps.
300 
301  Args:
302  t1 (datetime) : Start time
303  t2 (datetime) : End time
304  spills ( list[(time,pot)]) : Complete list of spills for time period
305 
306  Returns:
307  pot (float) : scintillator counts
308  """
309 
310  tf1 = in_unix(t1)
311  tf2 = in_unix(t2)
312 
313  nspill = 0
314  for (t,q) in spills:
315  if (t>=tf1)&(t<=tf2):
316  nspill += 1
317 
318  return nspill
319 
320 #..................................................
321 
322 def in_unix(input):
323  """ Convert a datetime to UNIX time
324 
325  Args:
326  input: A datetime object
327 
328  Returns:
329  A floating points number of seconds since Jan. 1, 1970 (UNIX standard)
330 
331  """
332  start = datetime(year=1970,month=1,day=1,hour=0,minute=0,second=0)
333  diff = input - start
334  return diff.total_seconds()
335 
336 #..................................................
337 
338 def main():
339 
340  (t0, t1, ndays) = options()
341 
342  # setup PyRoot and other root stuff
343  # Yes, these make more sense at the top of the file,
344  # but if you import ROOT before parsing the arguments,
345  # it takes over sys.argv and tries printing its own help
346  # function instead of the one in this file.
347  from ROOT import TFile, TTree
348  from ROOT import gROOT, AddressOf
349 
350  file = TFile(BASE_DIR+"/"+filename,'RECREATE')
351 
352  gROOT.ProcessLine("struct BeamStruct {Double_t Date; Int_t spills_rec; Double_t pot_rec; Int_t run_low; Int_t run_high;};")
353  from ROOT import BeamStruct
354  beamStruct = BeamStruct()
355  tree = TTree('BeamMetrics','POTtree')
356  tree.Branch('Date',AddressOf(beamStruct,'Date'),'Date/D')
357  tree.Branch('spills_rec',AddressOf(beamStruct,'spills_rec'),'spills_rec/I')
358  tree.Branch('pot_rec', AddressOf(beamStruct,'pot_rec'),'pot_rec/D')
359  tree.Branch('run_low',AddressOf(beamStruct,'run_low'),'run_low/I')
360  tree.Branch('run_high',AddressOf(beamStruct,'run_high'),'run_high/I')
361 
362 
363  # DB connection info for run info
364  DB_NAME = os.environ['NOVADBNAME'] # 'nova_prod'
365  DB_HOST = os.environ['NOVADBHOST'] # 'ifdbrep.fnal.gov'
366  DB_USER = os.environ['NOVADBUSER'] # 'nova_reader'
367  DB_PASS = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
368  DB_PORT = os.environ['NOVANEARDAQDBPORT'] # '5434'
369 
370  try:
371  conn = psycopg2.connect(\
372  "dbname=%s host=%s user=%s password=%s port=%s" % \
373  (DB_NAME, DB_HOST, DB_USER, DB_PASS, DB_PORT))
374  except:
375  print "I am unable to connect to the database"
376 
377  global DB_CURSOR
378  DB_CURSOR = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
379 
380 
381  date0 = date( year=t0.year, month=t0.month, day=t0.day)
382  datetime0 = datetime(year=t0.year, month=t0.month, day=t0.day)
383 
384 
385  # Loop over days to get spill info
386  for i in range(0,ndays):
387  datei = date0 + timedelta(days=i)
388  datetimei = datetime0 + timedelta(days=i)
389  datetimef = datetimei + timedelta(days=1)
390  # get unix timestamp for day - add 1 second to ensure binning is correct
391  unixi = (datetimei - datetime(1970,1,1)).total_seconds()+1
392 
393  #
394  # Get the complete list of runs for the day
395  #
396  runs = runs_by_day(datei)
397 
398  #
399  # Get the complete list of spills for the day
400  #
401  spills = mc7_spills_between(datetimei, datetimef)
402 
403  r1 = 0
404  r2 = 0
405  if (runs):
406  (r1,r2) = run_minmax(runs);
407  #
408  # Compute recorded and delivered POT
409  #
410  (pot_rec,pot_del,num_spill_rec) = compute_pot(datetimei, runs, spills)
411 
412 
413  print datei,",",\
414  unixi,",",\
415  r1,",",\
416  r2,",",\
417  len(spills),",",\
418  num_spill_rec,",",\
419  pot_del,",",\
420  pot_rec;
421 
422  beamStruct.Date = unixi
423  beamStruct.spills_rec = num_spill_rec
424  beamStruct.pot_rec = pot_rec
425  beamStruct.run_low = r1
426  beamStruct.run_high = r2
427  tree.Fill()
428 
429  file.Write()
430  file.Close()
431 
432 #..................................................
433 
434 if __name__ == "__main__":
435  main()
def pot_between(t1, t2, spills)
def in_unix(input)
::xsd::cxx::tree::date< char, simple_type > date
Definition: Database.h:186
def runs_by_day(day0)
def compute_pot(day, runs, spills)
def runs_for_period(day0, day1)
def run_minmax(runs)
def mc7_spills_between(t1, t2)
def spills_between(t1, t2, spills)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
exit(0)
def main()
MAIN FUNCTION.