update_sam_good_runs_metadata.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import os
4 import json
5 import samweb_client
6 import sys
7 
8 
9 #cuts = ' and (data_tier raw or data_tier artdaq) and (online.stream 0 or online.stream 2)'
10 cuts = ' and data_tier raw and online.stream 0'
11 
12 
13 def percentStr(x, y):
14  return str(.1*int((1000*x+.5)/y))+'%'
15 
16 def bulkMark(samweb, files, flag):
17  assert flag == 'true' or flag == 'false'
18 
19  if len(files) == 0: return
20 
21  # Undocumented bulk metadata update. Private mail from Robert I
22 
23  mdlist = [ {'file_name': f, 'dq.isgoodrun': flag} for f in files ]
24 
25  samweb.putURL('/files',
26  params = {'continue_on_error': 1},
27  data = json.dumps(mdlist),
28  content_type = 'application/json',
29  secure = True,
30  role='*')
31 
32 
33 if len(sys.argv) != 4:
34  print 'Usage: update_sam_good_runs_metadata.py [neardet|fardet] GRL.txt BRL.txt'
35  sys.exit(1)
36 
37 det = sys.argv[1]
38 assert det == 'neardet' or det == 'fardet'
39 
40 goodname = sys.argv[2]
41 badname = sys.argv[3]
42 
43 print 'Reading', goodname
44 fgood = open(goodname, 'r')
45 
46 good = set()
47 
48 for line in fgood:
49  fields = line.split('\t') # ' '
50  assert len(fields) == 2 #6
51 
52  run = int(fields[0])
53  subrun = int(fields[1])
54  assert subrun >= 0 and subrun <= 63
55 
56  good.update([(run, subrun)])
57 
58 print len(good), 'good subruns in input file'
59 
60 
61 print 'Reading', badname
62 fbad = open(badname, 'r')
63 
64 bad = set()
65 
66 for line in fbad:
67  fields = line.split('\t') # ' '
68  assert len(fields) == 2 #6
69 
70  run = int(fields[0])
71  subrun = int(fields[1])
72  assert subrun >= 0 and subrun <= 63
73 
74  bad.update([(run, subrun)])
75 
76 
77 print len(bad), 'bad subruns in input file'
78 
79 print
80 
81 print 'Do you have all the necessary tickets? You\'ll be sad in 20 minutes or so if you don\'t.'
82 
83 print
84 
85 samweb = samweb_client.SAMWebClient(experiment='nova')
86 
87 print 'Getting list of bad-runs files from SAM. This will take some time'
88 dbBad = samweb.listFiles('online.detector '+det+' and dq.isgoodrun false' + cuts)
89 
90 print 'Found', len(dbBad), 'bad-runs files'
91 
92 toMarkGood = []
93 
94 for fname in dbBad:
95  try:
96  fields = fname.split('_')
97  run = int(fields[1][1:])
98  subrun = int(fields[2][1:])
99  except:
100  print "Couldn't parse filename", fname
101  continue
102 
103  if (run, subrun) in good:
104  toMarkGood += [fname]
105  else:
106  # Presumably bad, make sure
107  if (run, subrun) not in bad:
108  print 'Warning:', fname, 'not in either good or bad list'
109 
110 print 'Need to flip goodruns field in', len(toMarkGood), 'of them ('+percentStr(len(toMarkGood), len(dbBad))+')'
111 
112 print
113 
114 print 'Getting list of good-runs files from SAM. This will take some time'
115 dbGood = samweb.listFiles('online.detector '+det+' and dq.isgoodrun true' + cuts)
116 
117 print 'Found', len(dbGood), 'good-runs files'
118 
119 toMarkBad = []
120 
121 for fname in dbGood:
122  try:
123  fields = fname.split('_')
124  run = int(fields[1][1:])
125  subrun = int(fields[2][1:])
126  except:
127  print "Couldn't parse filename", fname
128  continue
129 
130  if (run, subrun) in bad:
131  toMarkBad += [fname]
132  else:
133  # Presumably good, make sure
134  if (run, subrun) not in good:
135  print 'Warning:', fname, 'not in good or bad list'
136 
137 print 'Need to flip goodruns field in', len(toMarkBad), 'of them ('+percentStr(len(toMarkBad), len(dbGood))+')'
138 
139 print
140 
141 stride = 100
142 
143 for i in range(0, len(toMarkGood), stride):
144  print 'Marking good:', i, '/', len(toMarkGood)
145  bulkMark(samweb, toMarkGood[i:i+stride], 'true')
146 
147 for i in range(0, len(toMarkBad), stride):
148  print 'Marking bad:', i, '/', len(toMarkBad)
149  bulkMark(samweb, toMarkBad[i:i+stride], 'false')
150 
151 #for fname in toMarkGood:
152 # print 'Marking', fname, 'good'
153 # samweb.modifyFileMetadata(fname, {"dq.isgoodrun": "true"})
154 
155 #for fname in toMarkBad:
156 # print 'Marking', fname, 'bad'
157 # samweb.modifyFileMetadata(fname, {"dq.isgoodrun": "false"})
procfile open("FD_BRL_v0.txt")