make_goodruns_defs.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 from __future__ import print_function
3 from builtins import str
4 from builtins import range
5 import numpy as np
6 #------------------------------------------------------------#
7 #
8 # usage: python make_goodruns_defs.py <definition_name>
9 #
10 #
11 # help: lasquith@fnal.gov, scalvez@fnal.gov
12 #------------------------------------------------------------#
13 
14 import sys, shlex, subprocess, math
15 from subprocess import Popen, PIPE
16 from datetime import datetime
17 
18 #------------------------------------------------------------#
19 # the number of subruns to let sam handle in a single definition.
20 # could be optimised further for timing.
21 #------------------------------------------------------------#
22 splitLen = 350
23 startTime = datetime.now()
24 subrun_list=[]
25 
26 GRL=sys.argv[1]
27 subruns = open(GRL, "r")
28 
29 #32399 00 1551220313 1551220479 1995243297701888.0 0
30 for s in subruns:
31 
32  run = s.split(" ")[0]
33 
34  #if(np.int64(run) < 34574):
35  if(np.int64(run) < 35869):
36  continue
37 
38  subrun = s.split(" ")[1]
39  subrun_list.append(run+'.'+subrun)
40 
41 
42 #------------------------------------------------------------#
43 # divide the list of subruns into chunks of length splitLen
44 # if there are <3 subruns in last chunk, add them to the previous chunk
45 # make a list of lists, fcl_chunks[ chunk1,...,chunk<nfiles/splitLen>]
46 #------------------------------------------------------------#
47 chunks=[]
48 counter=0
49 print("Found %d subruns in %s" % (len(subrun_list), GRL))
50 for lines in range(0, len(subrun_list), splitLen):
51 
52  chunk = subrun_list[lines:lines+splitLen]
53 
54  # bail if we've over chunked
55  if(len(chunk)==0):
56  break
57 
58  # if <3 files in the last chunk, put them in the previous chunk
59  if(len(chunk)>2):
60  chunks.append(chunk)
61  elif(counter>0):
62  chunks[counter-1] += chunk
63  break
64  else:
65  print("There are less than 100 subruns, something is wrong")
66  exit(0)
67  counter+=1
68 # finished loop over fcls in input def
69 nsubdefs = len(chunks)
70 
71 
72 #------------------------------------------------------------#
73 # loop over chunks to construct a command to make defintions
74 # each chunk has its own temp definition
75 # each definition is an AND of cosmics def and run,subrun numbers
76 #------------------------------------------------------------#
77 atime=datetime.now()
78 snapshot_ids=[]
79 temp_defs=[]
80 cc=0
81 
82 run_subrub_cycle=[]
83 duplicates=0
84 
85 for chunk in chunks:
86 
87  subset=str(cc).zfill(4) #000,001,...
88  # the definition name for this subset of cosmics
89  subdef="temp_grl_s"+subset
90  temp_defs.append(subdef)
91  cmd = ""
92  # for each subrun in this chunk
93  fc=0
94  for subrun in chunk:
95 
96  sd = subdef+' \" (run_number '+subrun
97 
98  if fc==0:
99  cmd +='samweb create-definition '+sd+' '
100  fc += 1
101  elif fc % (len(chunk)-1) ==0:
102  cmd +=' or run_number '+subrun
103  fc+=1
104  else:
105  cmd +=' or run_number '+subrun
106  fc+=1
107  # finished loop over list of grls for this chunk
108  cmd += ' )\" '
109  cc+=1
110 
111  #------------------------------------------------------------#
112  # issue command to make chunk definition
113  # wait() is needed here because communicate() is not used
114  # take a snapshot of the chunk definition
115  # append chunk snapshot id to a list of them
116  #------------------------------------------------------------#
117 
118  #print "cmd: ",cmd
119  p2 = Popen(shlex.split(cmd))
120  p2.wait()
121 
122  cmd = 'samweb take-snapshot '+subdef
123  p3 = Popen(shlex.split(cmd), stdout=PIPE,stderr=PIPE)
124  out3,err3=p3.communicate()
125  snapshot_ids.append(str(out3))
126 
127 # finished loop over all chunks
128 #exit(0)
129 
130 # ------------------------------------------------------------#
131 # loop over chunk snapshot ids to construct full definition
132 # ------------------------------------------------------------#
133 cmd4 = ""
134 count_ids=0
135 #FULL_DEF_NAME="hack_isgood_fd_period10abc"
136 FULL_DEF_NAME="hack_isgood_fd_period10d"
137 for sid in snapshot_ids:
138  if(count_ids==0):
139  cmd4 += 'samweb create-definition '+FULL_DEF_NAME+' \"snapshot_id '+sid+' or'
140  elif(count_ids==len(snapshot_ids)-1):
141  cmd4 += ' snapshot_id '+sid+'\"'
142  else:
143  cmd4 += ' snapshot_id '+sid+' or'
144  count_ids+=1
145 
146 # finished loop over all snapshot_ids
147 
148 # ------------------------------------------------------------#
149 # issue command to construct full definition
150 # take a snapshot of full definition
151 # issue command to make snapshot definition
152 # ------------------------------------------------------------#
153 print(">>>>> Making full definition from subdefinition snapshots")
154 p4 = Popen( shlex.split(cmd4) )
155 p4.wait()
156 print(">>>>>> Taking a snapshot of %s" % (FULL_DEF_NAME))
157 
158 cmd5='samweb take-snapshot '+FULL_DEF_NAME
159 p5 = Popen( shlex.split(cmd5),stdout=PIPE,stderr=PIPE )
160 out5,err5=p5.communicate()
161 
162 cmd6='samweb create-definition '+FULL_DEF_NAME+'_snapshot'+out5+' \"snapshot_id '+out5+'\"'
163 p6 = Popen( shlex.split(cmd6) )
164 p6.wait()
165 
166 
167 # ------------------------------------------------------------#
168 # delete the temporary subdef snapshots and the full cosmics def
169 # this is so that rerunning doesn't pick up old ones
170 # ------------------------------------------------------------#
171 for td in temp_defs:
172  print("deleting temp definition ",td)
173  cmd7='samweb delete-definition '+td
174  p7 = Popen( shlex.split(cmd7) )
175  p7.wait()
176 
177 print("deleting full definition ",FULL_DEF_NAME)
178 cmd8='samweb delete-definition '+FULL_DEF_NAME
179 p8 = Popen( shlex.split(cmd8) )
180 p8.wait()
181 
182 
183 
184 print("FIN.")
185 print("time taken: ",(datetime.now() - startTime))
186 
if(dump)
bool print
procfile open("FD_BRL_v0.txt")
exit(0)