getGoodRuns4SAM.py
Go to the documentation of this file.
1 # Script to copy the nova good runs table to sam
2 # The script also computes the number of good, and
3 # good analysis diblocks and writes those to the sam
4 # db.
5 
6 from ConDB import ConDB
7 from datetime import datetime, timedelta
8 from timelib import text2datetime, epoch
9 import sys, getopt, os
10 import psycopg2, psycopg2.extras
11 from operator import itemgetter
12 import csv
13 
14 ##################################################################
15 
16 # count the number of 4 or more consecutive ones
17 def countAnaBlocks(diblock_mask_binary):
18  anadb = 0
19  counter = 0
20  # first get rid of the '0b' that binary strings start with
21  bit_mask = diblock_mask_binary[2:]
22  # split into substrings at every 0
23  bit_mask_burst = bit_mask.split('0')
24  # should be left with substrings containing only 1's
25  # add up if substring length is 4 or more
26  for s in bit_mask_burst:
27  if len(s) > 3:
28  anadb += len(s)
29  return anadb
30 
31 
32 ##################################################################
33 
34 # Function to consolidate the db information to validity ranges
35 def consolidate(run_rows):
36  run_rows.sort()
37 
38  n = len(run_rows)
39  print "Read {} rows from the NOvA good runs database".format(n)
40  i=0
41 
42  prev_isGood = not(run_rows[0][3])
43  prev_diblock_mask = -1
44 
45  ret_rows = []
46 
47  for i in range(n):
48 
49  isGood = run_rows[i][3]
50  diblock_mask = run_rows[i][4]
51 
52  if ((prev_diblock_mask != diblock_mask) or (prev_isGood != isGood) or
53  (i == n-1)) :
54  ret_rows.append(run_rows[i])
55  prev_isGood = isGood
56  prev_diblock_mask = diblock_mask
57 
58 
59  print "Consisting of {} validity ranges.".format(len(ret_rows))
60  return ret_rows
61 
62 ##################################################################
63 
64 # Function to process data fetched from the db for fardet
65 
66 def processFarDetData(data, detname, tag):
67 
68  li = list(data)
69 
70  prev_diblock_mask = -1
71  if li == []:
72  print "\nNOvA database returned no information.\n"
73  exit(1)
74 
75  # set the variable that keep account of what the previous subrun's
76  # isgood flag was to the compliment of the flag for the first subrun
77  prev_isGood = ( li[0][2][0] > 1 )
78 
79  run_rows = []
80  n=0
81 
82  # parse information from the good runs database and put it
83  # into a list of ntuples that can later be sorted by run
84  for c, tv, values in li:
85 
86  run = int(epoch(tv))
87  isGood=True
88  if (values[1] > 0):
89  isGood = False
90 
91  diblock_mask = int(values[0])
92  ngood_cont_diblock = values[2]
93 
94  #convert to binary
95  diblock_mask_binary = bin( diblock_mask)
96  #count the number of 1's in the binary diblock mask
97  ngood_tot_diblock = diblock_mask_binary.count('1')
98 
99  ngood_ana_diblock = countAnaBlocks(diblock_mask_binary)
100 
101  tup = (run,
102  detname,tag,isGood,
103  diblock_mask,
104  ngood_cont_diblock,
105  ngood_ana_diblock,
106  ngood_tot_diblock)
107  run_rows.append(tup)
108 
109  return consolidate(run_rows)
110 
111 ##################################################################
112 
113 # Function to process data fetched from the db for neardet
114 def processNearDetData(data, detname, tag):
115  li = list(data)
116  if li == []:
117  print "\nNOvA database returned no information.\n"
118  exit(1)
119  # set the variable that keep account of what the previous subrun's
120  # isgood flag was to the compliment of the flag for the first subrun
121 
122  run_rows = []
123 
124  for c, tv, values in li:
125 
126  run = int(epoch(tv))
127  isGood=(values[0] < 1)
128 
129  # set FD specific metrics to 0
130  diblock_mask = 0
131  ngood_cont_diblock = 0
132  ngood_tot_diblock = 0
133  ngood_ana_diblock = 0
134 
135  tup = (run,
136  detname,tag,isGood,
137  diblock_mask,
138  ngood_cont_diblock,
139  ngood_ana_diblock,
140  ngood_tot_diblock)
141  run_rows.append(tup)
142 
143  return consolidate(run_rows)
144 
145 
146 
147 ##################################################################
148 
149 # main function, where we connect to the db, and read/write info
150 # from/to it
151 if __name__ == '__main__':
152 
153  Usage = """
154 python getGoodRuns4SAM.py [options]
155 options:
156  -T <tag> [REQUIRED]
157  -D <fardet|neardet> [REQUIRED]
158  --test [optional]
159 """
160 
161  DB_PWD = open(os.environ['NOVADBPWDFILE'], 'r').readlines()[0].strip()
162  data_type = 'data'
163  tag = None
164  det = None
165  test = False
166  dbcon = []
167  dbcon.append("host=ifdbprod.fnal.gov port=5433 dbname=nova_prod ")
168  t0 = text2datetime("0")
169  t1 = text2datetime("100000000000")
170  channel_range = None
171  username = "nova_reader"
172 
173  # Parse the arguments passed
174  opts, args = getopt.getopt(sys.argv[1:], 'T:D:', ["test"])
175 
176  if len(args) > 0 :
177  if args[0] == 'help':
178  print Usage
179  sys.exit(0)
180 
181  for opt, val in opts:
182  if opt == '-T': tag = val
183  elif opt == '-D': det = val
184  elif opt == '--test': test = True
185  else:
186  print "Unknown argument."
187  print Usage
188  sys.exit(0)
189 
190  if not tag or not det:
191  print "Tag and Detector are required arguments!"
192  sys.exit(1)
193 
194  # Let's try to gather the information we need to connect to
195  # the NOvA database
196  dbcon.append("user=%s" % (username,))
197  dbcon.append("password=%s" % (DB_PWD))
198 
199  dbcon = ' '.join(dbcon)
200  if det == 'fardet':
201  tname = det + ".dq_subrun_info"
202  columns = ["dbmask","flag","ngooddb"]
203  elif det == 'neardet':
204  tname = det + ".dq_subrun_info_nd"
205  columns = ["flag", "miprate"]
206  else:
207  print "Invalid detector name. Valid options are fardet or neardet"
208  exit(1)
209 
210  db = ConDB(dbcon)
211  table = db.table(tname, columns)
212 
213  # If we have gotten here, we have all we need.
214  # Let's try to connect to the dB.
215 
216  if not test:
217  print "Establishing connection to SAM database.."
218 
219  try:
220  conn = psycopg2.connect("dbname=sam_nova_prd host=sampgsdb01.fnal.gov "\
221  "port=5433")
222  cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
223  except:
224  print "Unable to connect to the SAM database"
225  exit(1)
226 
227  print "Success!"
228 
229  print "Retrieving information from the nova good runs database..."
230 
231  data = table.getDataIntervalIter(t0, t1, data_type=data_type, tag=tag,
232  channel_range = channel_range)
233 
234  # Parse the information based on which detector we are working with
235  # store the result in run_rows.
236 
237  if det == "fardet":
238  run_rows = processFarDetData(data, det, tag)
239  else:
240  run_rows = processNearDetData(data, det, tag)
241 
242  print "Proceeding to write to sam database..."
243 
244  fout = open("./goodRuns4SAM.csv","w")
245 
246  # loop through all entries in run_row
247  # Form run ranges over which the information in a row is valid.
248  # The run range's lower bound is the current row's run and the
249  # upper bound is next row's run.
250  for idx, row in enumerate(run_rows):
251 
252  if idx >= len(run_rows)-1:
253  break
254  thisrun = row[0]
255  # get the run number from the next row
256  nextrun = run_rows[ (idx+1)][0]
257 
258  # making a range out of these two run numbers
259  # the first run is included in this range, while the second run
260  # is excluded from this range, hence the [)
261  run_range = psycopg2.extras.NumericRange(thisrun, nextrun, '[)')
262 
263  if not test:
264  cur.execute('INSERT INTO nova.good_runs_r (run_number,detector,tag,good,diblock_mask,ngood_cont_diblock,ngood_ana_diblock,ngood_tot_diblock) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)',
265  (run_range,
266  row[1],
267  row[2],
268  row[3],
269  row[4],
270  row[5],
271  row[6],
272  row[7]))
273 
274 
275  line = '{0},{1},{2},{3},{4},{5},{6},{7}\n'.format(run_range,row[1],
276  row[2],row[3],row[4],
277  row[5],row[6],row[7])
278  fout.write(line)
279 
280 
281  if not test:
282  # phew, done!
283  print """Successfully wrote data to sam db.
284  To confirm, login (psql -h cspgsdev.fnal.gov -d samtest -p 5432)
285  and check the information is there"""
286 
287  # make the db changes persist
288  conn.commit()
289 
290  # goodbye!
291  cur.close()
292  conn.close()
293 
294  else:
295  print """Test finished. SAM has not been updated. Check the newly created goodRuns4SAM.csv file for consistency.
296  If it looks OK, run this command again without the --test flag to push to the SAM database"""
297 
def consolidate(run_rows)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
float bin[41]
Definition: plottest35.C:14
procfile open("FD_BRL_v0.txt")
def processFarDetData(data, detname, tag)
exit(0)
def countAnaBlocks(diblock_mask_binary)
def processNearDetData(data, detname, tag)