DeclareReco.py
Go to the documentation of this file.
1 # This is a first pass at defining metadata for old NDOS reconstructed files.
2 
3 import os
4 import sys
5 import subprocess
6 
7 # Globval Variables
8 VERBOSE = False
9 DEV = False
10 TEST = False
11 SAMFAIL = False
12 FFILE = ""
13 MFILE = ""
14 MDDIR = "/nova/ana/users/janzirn/MDTmp/"
15 
16 def showHelp():
17  print("Usage:")
18  print("DeclareReco.py [options] file\n")
19  print("Declares listed file to SAM and adds their location to SAM.\n")
20  print("Options:")
21  print("-v,--verbose Verbose")
22  print("-t,--test Test run (do not issue SAM commands)")
23  print("-d,--dev Use the development SAM db")
24 
25 def getMetadata(file):
26  global MFILE
27 
28  size = os.path.getsize(file)
29  chkSum = subprocess.check_output("samweb file-checksum "+file,stderr=subprocess.STDOUT,shell=True)
30  fname=os.path.basename(file)
31 
32  # Figure out release
33  # This needs work and will be hardcoded for the first couple of backups
34  #relstartid = fname.find("S1")
35  #relendid = fname.find("_",relstartid)
36  #release = fname[relstartid:relendid]
37  release = "S12.02.14"
38 
39  # Figure out run number
40  # The run number is never the first thing in a name
41  runstartid = fname.find("r",fname.find("_"))
42  runendid = fname.find("_",runstartid)
43  fullRunNum = fname[runstartid+1:runendid]
44  runNum = fullRunNum.lstrip('0')
45  #TODO Check that it's actually a valid run number, cast to int make sure it works
46 
47 
48  # Figuge out subrun number
49  srunstartid = fname.find("s",runendid)
50  srunendid = fname.find("_",srunstartid)
51  fullSubRun = fname[srunstartid+1:srunendid]
52  subRun = fullSubRun.lstrip('0')
53 
54  # Not going to work for subrun 0, also
55  # check that we have a valid subrun number
56  try:
57  int(fullSubRun)
58  if subRun == '':
59  subRun = '0'
60  except ValueError:
61  print fullSubRun, " is not a valid subrun number"
62  sys.exit(1)
63 
64  # Figure out version number
65  verstartid = fname.find("v")
66  if int(verstartid) < 0:
67  version = "1"
68  else:
69  verendid = fname.find("_",verstartid)
70  version = fname[verstartid:verendid].lstrip('0')
71 
72  # Make sure it's a valid version, if not set it to 1
73  try:
74  int(version)
75  except ValueError:
76  version = "1"
77 
78  parent = "ndos_r"+fullRunNum+"_s"+fullSubRun+"_t01.raw"
79  parentMD = subprocess.check_output("samweb -e nova get-metadata "+parent,
80  stderr=subprocess.STDOUT,shell=True)
81 
82  # Parse the output of the parent meta data file and get the run/subrun times
83  sMD = parentMD.split()
84 
85  try:
86  runStartTime = sMD[sMD.index("Online.RunStartTime:")+1]
87  except ValueError:
88  runStartTime = ''
89 
90  try:
91  runEndTime = sMD[sMD.index("Online.RunEndTime:")+1]
92  except ValueError:
93  runEndTime = ''
94 
95  try:
96  srunStartTime = sMD[sMD.index("Online.SubRunStartTime:")+1]
97  except ValueError:
98  srunStartTime = ''
99 
100  try:
101  srunEndTime = sMD[sMD.index("Online.SubRunEndTime:")+1]
102  except ValueError:
103  srunEndTime = ''
104 
105  # JSON format of Metadata
106  jD = "{"
107  jD+= '\n\t"file_name": "'+fname+'",'
108  jD+= '\n\t"file_type": "importedDetector",'
109  jD+= '\n\t"file_size": '+str(size)+','
110  jD+= '\n\t"crc": '+str(chkSum)+','
111  jD+= '\n\t"file_format": "root",'
112  jD+= '\n\t"data_stream": "1",'
113  jD+= '\n\t"data_tier": "reconstructed",'
114  jD+= '\n\t"Nova.DetectorID": "ndos",'
115  jD+= '\n\t"Nova.HornConfig": "none",'
116  jD+= '\n\t"Nova.HornPolarity": "none",'
117  jD+= '\n\t"Nova.SubVersion": "'+version+'",'
118  jD+= '\n\t"Nova.Label": "preMeta",'
119  jD+= '\n\t"Reconstructed.base_release": "'+release+'",'
120 
121  # The development database has different value types
122  # and different values for run_type from the normal database
123  if DEV:
124  jD+= '\n\t"Online.RunStartTime": "'+runStartTime+'",'
125  jD+= '\n\t"Online.RunEndTime": "'+runEndTime+'",'
126  jD+= '\n\t"Online.SubRunStartTime": "'+srunStartTime+'",'
127  jD+= '\n\t"Online.SubRunEndTime": "'+srunEndTime+'",'
128  jD+= '\n\t"Online.Subrun": "'+subRun+'",'
129  jD+= '\n\t"Online.RunNumber": "'+runNum+'",'
130  jD+= '\n\t"runs": [ ['+str(runNum)+',"physics"] ]'
131  else:
132  jD+= '\n\t"Online.RunStartTime": '+runStartTime+','
133  jD+= '\n\t"Online.RunEndTime": '+runEndTime+','
134  jD+= '\n\t"Online.SubRunStartTime": '+srunStartTime+','
135  jD+= '\n\t"Online.SubRunEndTime": '+srunEndTime+','
136  jD+= '\n\t"Online.Subrun": '+subRun+','
137  jD+= '\n\t"Online.RunNumber": '+runNum+','
138  jD+= '\n\t"parents": [{"file_name": "'+parent+'"}],'
139  jD+= '\n\t"runs": [ ['+str(runNum)+',"commissioning"] ]'
140  jD+= '\n}'
141 
142  if VERBOSE:
143  print jD
144 
145  # Define the output file and write the data to it
146  mdfile = open(MDDIR+os.path.basename(file)+'.md','w')
147  mdfile.write(jD)
148  mdfile.close()
149 
150  # Set the global variable so we can use it in main
151  MFILE = MDDIR+os.path.basename(file)+'.md'
152 
153 # Parse arguments and set global variables accordingly
154 def getArgs(argv):
155  global VERBOSE
156  global TEST
157  global DEV
158  global FFILE
159  for opt in argv:
160  if opt == '-h':
161  showHelp()
162  sys.exit()
163  else:
164  if opt in ("-v","--verbose"):
165  VERBOSE = True
166  if opt in ("-t","--test"):
167  TEST = True
168  if opt in ("-d","--dev"):
169  DEV = True
170  if os.path.exists(opt):
171  if FFILE == '':
172  FFILE = opt
173  else:
174  showHelp()
175  sys.exit(2)
176  if FFILE == '':
177  showHelp()
178  sys.exit(3)
179 
180 # For debug reasons
182  print "VERBOSE=", VERBOSE
183  print "DEV=", DEV
184  print "TEST=", TEST
185  print "FFILE=", FFILE
186  print "MFILE=", MFILE
187  print "MDDIR=", MDDIR
188  print "SAMFAIL=", SAMFAIL
189  sys.exit()
190 
191 # Delete the meta data file
192 def cleanUp(file):
193  if os.path.isfile(file):
194  os.remove(file)
195  else:
196  print file, "Is not a valid Meta Data file"
197  sys.exit(3)
198 
199 # Actually declare the metadata to SAM
200 def declareMetadata(file):
201  global SAMFAIL
202  if DEV:
203  devOpt='--dev'
204  else:
205  devOpt=''
206 
207  if VERBOSE:
208  print "samweb declare-file "+devOpt+" -e nova "+file
209 
210  try:
211  subprocess.check_call("samweb declare-file "+devOpt+" -e nova "+file,stderr=subprocess.STDOUT,shell=True)
212  except subprocess.CalledProcessError:
213  SAMFAIL = True
214  return
215 
216  if VERBOSE:
217  print "samweb add-file-location "+devOpt+" -e nova "+os.path.basename(FFILE)+" novadata:"+os.path.dirname(FFILE)
218 
219  try:
220  subprocess.check_call("samweb add-file-location "+devOpt+" -e nova "+os.path.basename(FFILE)+" novadata:"+os.path.dirname(FFILE)
221  ,stderr=subprocess.STDOUT,shell=True)
222  except subprocess.CalledProcessError:
223  SAMFAIL = True
224  return
225 
226 # The meat and potatoes
227 def main(argv):
228  getArgs(argv)
229  #printGlobvar()
230  getMetadata(FFILE)
231 
232  if not TEST:
233  declareMetadata(MFILE)
234 
235  cleanUp(MFILE)
236 
237  if SAMFAIL:
238  sys.exit(1)
239  else:
240  sys.exit()
241 
242 if __name__ == "__main__":
243  main(sys.argv[1:])
244 
def cleanUp(file)
Definition: DeclareReco.py:192
def getMetadata(file)
Definition: DeclareReco.py:25
bool print
def printGlobvar()
Definition: DeclareReco.py:181
def main(argv)
Definition: DeclareReco.py:227
procfile open("FD_BRL_v0.txt")
def declareMetadata(file)
Definition: DeclareReco.py:200
def showHelp()
Definition: DeclareReco.py:16
def getArgs(argv)
Definition: DeclareReco.py:154