runNovaSAM.py
Go to the documentation of this file.
1 #!/bin/env python
2 
3 import os, sys
4 import shutil
5 import samweb_client, ifdh
6 import pprint
7 import argparse
8 import subprocess
9 import re
10 import md5
11 import json
12 import resource
13 import string
14 import datetime
15 from string import atoi
16 from samweb_client.utility import fileEnstoreChecksum
17 from samweb_client.exceptions import *
18 
19 import MetadataUtils
20 import NovaGridUtils as NGU
21 
22 ## Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing --help
23 argvCopy = sys.argv[:] # Make a copy
24 sys.argv = sys.argv[:1] # Replace it with just the first argument, i.e. script name
25 import ROOT
26 # The weird thing is that you have to do something with ROOT in order to make it parse args. Ok, we would have done this anyway.
27 ROOT.gErrorIgnoreLevel=3000 # Stop ROOT from complaining about not having dictionaries, 3000 is below kError but above kWarning.
28 sys.argv = argvCopy # Restore sys.argv. Thanks Wim, that's a real "solution"!
29 
30 #files
31 _skip_pattern = re.compile(r"^.*(RootOutput).*\.root", re.IGNORECASE)
32 
34  #memLimit = 3.9*1024**3
35  memLimit = 3.9*1000**3
36  print "Old virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)
37  print "Limiting the virtual memory of the nova job to 3.9 GB"
38  resource.setrlimit(resource.RLIMIT_AS, (memLimit, memLimit))
39  print "New virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)
40 
41 def makeDirSafely(dir):
42  if "/pnfs" != dir[0:5]:
43  makeDirIfNeeded(dir)
44  return
45 
46  print "runNovaSAM is making a directory with IFDH"
47  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
48  try:
49  print "Checking if directory ", dir, "exists"
50  dh.ls(dir, 1, "")
51  dh.chmod(774,dir,"")
52 
53  except (RuntimeError, IOError) as e:
54  try:
55  print "It doesn't - make directory", dir
56  dh.mkdir(dir)
57  except:
58  print "Tried to make directory and couldn't. Perhaps it already exists?"
59 
60 
61 def makeDirIfNeeded(dir):
62  os.umask(002) # (rw-rw-r--) for files and (rwxrwxr-x) for directories.
63  if not os.path.isdir(dir):
64  print "runNovaSAM is making a directory: ", dir
65  try:
66  os.mkdir(dir)
67  except:
68  print "Couldn't make the directory... some other job perhaps did, or permissions did not allow "
69  if not os.path.isdir(dir):
70  raise Exception("Failed to make directory + " + dir )
71 
72 
73 def getOutDir(pathname, dest, hashDirs=False, runDirs=False, runNum=0):
74  dirs = [dest]
75  if hashDirs:
76  head, tail = os.path.split(pathname)
77  hash = md5.new(tail)
78  dirs += list(hash.hexdigest()[:3])
79  if runDirs:
80  head, tail = os.path.split(pathname)
81  runStr = str(runNum)
82  multiRunDir = runStr[:3].zfill(6)
83  makeDirSafely(os.path.join(dest, multiRunDir))
84  makeDirSafely(os.path.join(dest, multiRunDir, runStr))
85  dirs += [multiRunDir, runStr]
86  return os.path.join(*dirs)
87 
88 def checkAndMoveFiles(inFile, outputs, noCleanup=False):
89  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
90  inFileBase = os.path.basename(inFile)
91  baseDir = "."
92  # If declaring files have to do non-decafs first, since they're the parents
93  # of the decafs. If not, it doesn't hurt to do it in that order anyway.
94  for decafPass in [False, True]:
95  for root, dirs, filenames in os.walk(baseDir):
96  if root == baseDir:
97  for file in filenames:
98  if file.endswith('decaf.root') != decafPass: continue
99  # If file not in outputs, then delete.
100  if (file.endswith (".root") or file.endswith(".h5")) and file != inFileBase:
101  fileWPath = os.path.join(root, file)
102  # If the file isn't in my outputs then delete it.
103  if file not in outputs:
104  if not noCleanup:
105  # Do not copy over, just delete
106  print "File", fileWPath, " is not among requested outputs, removing"
107  os.remove(fileWPath)
108  continue
109  # Now that I know that this file is in my output list, lets check it is valid...
110  # First, if it is a root file.
111  if file.endswith (".root"):
112  print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
113  rootFile = ROOT.TFile(fileWPath)
114  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
115  #do not copy over, just delete
116  print "File", fileWPath, "is Zombie - remove it"
117  os.remove(fileWPath)
118  else:
119  newFilePath = os.path.join(root, "results", file)
120  print "New file name is %s" %newFilePath
121  os.renames(fileWPath, newFilePath)
122  rootFile.Close()
123  # Next, if it is a h5 file.
124  elif file.endswith(".h5"):
125  # Only import h5py if have h5 files.
126  import h5py
127  print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
128  # If a valid HDF5 files.
129  if h5py.is_hdf5(fileWPath):
130  newFilePath = os.path.join(root, "results", file)
131  os.renames(fileWPath, newFilePath)
132  print "Moved ", fileWPath, "to ", newFilePath
133  # If not a valid HDF5 file.
134  else:
135  print "File", fileWPath, "is Zombie - remove it"
136  os.remove(fileWPath)
137  return
138 
139 #def h5MetaHack(inFile):
140 # print "Doing a hack to get HDF5 metadata...Will take CAF metadata and change subtly."
141 # ### Make my client and get my CAF metadata
142 # samweb = samweb_client.SAMWebClient(experiment='nova')
143 # md = MetadataUtils.createMetadata( inFile.replace('.h5caf.h5', '.caf.root') )
144 # ### Change some parameters.
145 # md['file_name'] = inFile
146 # md['data_tier'] = unicode('h5')
147 # md['file_size'] = os.path.getsize( inFile )
148 # ### return md
149 # return md
150 
151 def declareFile(fileWPath):
152  """Checks file for a TKey of RootFileDB. If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM"""
153  samweb = samweb_client.SAMWebClient(experiment='nova')
154  rootFile = ROOT.TFile(fileWPath)
155  filename = os.path.basename(fileWPath)
156  print filename
157  olddir = os.getcwd()
158  os.chdir(os.path.dirname(fileWPath))
159  if rootFile.FindKey("RootFileDB") or filename.endswith("caf.root") or filename.endswith(".h5"):
160  md = MetadataUtils.createMetadata(filename)
161  # Check that md exists, and then declare!
162  if md == None:
163  print "No metadata found!"
164  else:
165  # If I have a transpose file want to add plane (and cell) number.
166  if "cell" in filename:
167  try:
168  plane = re.search('^.*outplane(\d*).', filename).group(1)
169  cell = re.search('^.*cell(\d*).', filename).group(1)
170  print "Plane number:", plane, "Cell number:", cell, ". Is a transpose file"
171  md['Calibration.PlaneNumber'] = plane
172  md['Calibration.CellNumber'] = cell
173  except:
174  print "No cell number found - could be a plane mode transpose file"
175 
176  elif "outplane" in filename:
177  print filename
178  try:
179  plane = re.search('^.*outplane(\d*).', filename).group(1)
180  print "Plane number:", plane, ". Is a transpose file"
181  md['Calibration.PlaneNumber'] = plane
182  except:
183  print "No plane number found - not a transpose file"
184  ### Make sure that the txtfile is in the parent list.
185  if args.txtfiledef:
186  md['parents'].append({u'file_name':unicode(inFile)})
187 
188  # Print out the metadata before trying to declare
189  pprint.pprint(md)
190  print ''
191  print "Declaring", fileWPath, "to SAM"
192  try:
193  samweb.declareFile(md)
194  #samweb.validateFileMetadata(md=md)
195  except Exception as inst:
196  #print fileWPath, "already exists in SAM"
197  print inst
198  else:
199  print fileWPath, "does not contain RootFileDB, do not try to declare"
200  rootFile.Close()
201  os.chdir(olddir)
202  return
203 
204 
205 def declareLogFile(logName, logFile, rootName):
206  # This is what NovaFTS/plugins/nova_log_metadata.py does
207  md = {
208  'file_name': os.path.basename(logName),
209  'file_size': os.path.getsize(logFile),
210  'data_tier': 'log',
211  'file_format': 'log',
212  'file_type': 'unknown', # maybe 'nonPhysicsGeneric'?
213  'parents': [{'file_name': os.path.basename(rootName)}]
214  }
215  print "16) declareLogFile(logName, logFile, rootName):"
216  print "Declaring", logName, "to SAM"
217  try:
218  samweb = samweb_client.SAMWebClient(experiment='nova')
219  samweb.declareFile(md)
220  except Exception as inst:
221  print inst
222 
223 
224 def fileExists(outPath, dh):
225  try:
226  # Check to see if you can ls the file.
227  # This works in ifdh 1_7_0 and newer
228  # If it exists, dh.ls() returns a tuple with one entry, converts to True
229  # If it doesn't, dh.ls() returns an empty tuple, converts to False
230  return bool(dh.ls(outPath, 1, ""))
231  except (RuntimeError, IOError) as e:
232  # Some versions of ifdh throw an exception when the file was not found
233  # But that means it is not there.
234  return False
235 
236 def listFiles(outPath, dh):
237  try:
238  return dh.ls(outPath, 1, "")
239  except:
240  return "Exception while trying to ls, OutPath =",outPath
241 
242 
244  """For every ROOT file, try to extract its metadata into a matching .json
245  file in the same directory"""
246  baseDir = "./results"
247  # If using --txtfiledef, want to pass multiple files at a time to sam_meta_dumper.
248  # This is because it simply takes too long to call it for each indiv file
249  # when there are 800+ outputs. 5 s each -> ~1 hour!
250  TransposeList = ""
251  # Declare the samweb client within this function
252  samweb = samweb_client.SAMWebClient(experiment='nova')
253  # Loop through directories to search for files to make json files for.
254  for root, dirs, filenames in os.walk(baseDir):
255  if root == baseDir:
256  # Push the h5 files to the front of the list so that the CAF files remain
257  # available to hack in the metadata.
258  for ifile in range(len(filenames)):
259  if("h5" in filenames[ifile]):
260  filenames = [filenames[ifile]] + filenames[:ifile] + filenames[ifile+1:]
261  for file in filenames:
262  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
263  skip_match = _skip_pattern.match(file)
264  if skip_match == None:
265  # Set some quick and useful variables.
266  olddir = os.getcwd()
267  fileWPath = os.path.join(root, file)
268  # If a transpose file want to make the json subtly differently.
269  if args.txtfiledef and "outplane" in file:
270  # This works for the cellmode of the transposer as well, due to the filename including outplane and cell.
271  print "Adding %s to TransposeList" %file
272  TransposeList += "%s " %file
273  continue
274  # Which extractor am I using?
275  extractor = 'sam_metadata_dumper'
276  if file.endswith('caf.root'):
277  extractor = 'extractCAFMetadata'
278  elif file.endswith('.h5'):
279  extractor = 'extractHDF5Metadata'
280  try:
281  # sam_metadata_dumper doesn't apply basename() to filename. https://cdcvs.fnal.gov/redmine/issues/8987
282  os.chdir(os.path.dirname(fileWPath))
283  meta = subprocess.check_output([extractor, os.path.basename(fileWPath)])
284  if file.endswith (".root"):
285 
286  jsonf = open(file.replace('.root', '.json'), 'w')
287  jsonf.write(meta)
288  jsonf.close()
289  print "Made metadata for %s" %file
290  elif file.endswith(".h5"):
291  print "\nNow to make my json file for my h5...\n"
292  jsonf = open(file.replace('.h5caf.h5', '.h5caf.json'), 'w')
293  jsonf.write(meta)
294  jsonf.close()
295  print "Made metadata for %s" %file
296  else:
297  print "I'm not sure what file extension you have..."
298  except:
299  print "Error extracting metadata from file."
300  finally:
301  os.chdir(olddir)
302  # Make the Transpose json files.
303  # Again same argument, outplane is already in the filenames for cell mode
304  if args.txtfiledef and "outplane" in file:
305  olddir = os.getcwd()
306  os.chdir(baseDir)
307  MakeTransposeJson( TransposeList )
308  os.chdir(olddir)
309 
310 def MakeTransposeJson( TransposeList ):
311  """Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
312  This takes in a list of files, and makes appropriate .json files in the same directory"""
313  # If using --txtfiledef, I can now pass my file list to sam_meta_dumper.
314  print "Is MakeTransposeJson called without a txt def, if you see this then yes."
315  MetaListFile="AllMetaJson.txt"
316  meta_cmd="sam_metadata_dumper -s " + TransposeList + " > " + MetaListFile
317  os.system(meta_cmd)
318  # Now want to open the file and split by "}," character.
319  MetaFile = open( MetaListFile )
320  MetaLines = MetaFile.read().split(" },")
321  # Loop through lines, and appropriately separate out json files.
322  for i in range(0,len( MetaLines ) ):
323  meta=MetaLines[i]
324  # Figure out file name...this is assuming that file name is always the first entry...
325  StName=re.search('"(.+?).root"', meta ).group(1)
326  filename=StName+".json" # Effecitvely replacing .root with .json
327  # If transpose file add PlaneNumber and CellNumber if run in cell mode
328  if "cell" in filename:
329  try:
330  plane = re.search('^.*outplane(\d*).', filename).group(1)
331  cell = re.search('^.*cell(\d*).', filename).group(1)
332  print "Plane number:", plane, "Cell number:", cell, ". Is a transpose file"
333  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.CellNumber": "%s",\n "calibration.base_release"')%(plane, cell)
334  except:
335  print "No cell number found - could be a plane mode transpose file"
336 
337  elif "outplane" in filename:
338  try:
339  plane = re.search('^.*outplane(\d*).', filename).group(1)
340  print "Plane number:", plane, ". Is a transpose file"
341  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.base_release"') %plane
342  except:
343  print "Error extracting plane number from transpose file."
344 
345  ### Make sure that the txtfile is in the parent list.
346  meta['parents'].append({u'file_name':unicode(inFile)})
347 
348  # Now open the json file
349  fout=open(filename,'w')
350  # Want to make sure that the json starts with '{'
351  if meta[0] not in "{":
352  meta = meta[:0] + '{\n' + meta[1:]
353  # Want to make sure that the json ends with a double '}'
354  if i < len(MetaLines)-1:
355  meta += "}\n}\n"
356  # Write and close the json file
357  fout.write(meta)
358  fout.close()
359 
360 
361 def copyOutFiles(dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False):
362  """Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
363  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
364  baseDir = "./results"
365  declareFiles = declareLogs
366  for root, dirs, filenames in os.walk(baseDir):
367  if root == baseDir:
368  for file in filenames:
369  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
370  fileWPath = os.path.join(root, file)
371  outDir = getOutDir(file, dest, hashDirs, runDirs, runNum)
372  skip_match = _skip_pattern.match(file)
373  if skip_match == None:
374  outPath = os.path.join(outDir, file)
375 
376  # note: this will fail if the file already exists
377  returnValue = dh.cp(["-D", fileWPath, outDir])
378  if returnValue != 0:
379  print >> sys.stderr, "Copy out failed for file:", fileWPath
380  print >> sys.stderr, "Skipping it."
381  else:
382  if declareFiles:
383  declareFile(fileWPath)
384  ###################
385  # Declare the file's location to SAM if we have the declareLocation option on
386  if declareLocation==True :
387  loc = string.replace(outDir, 's3://','s3:/')
388  print "Declaring location %s for file %s\n" % (loc,file)
389  sam = samweb_client.SAMWebClient('nova')
390  ret=sam.addFileLocation(file, loc)
391  if ret.status_code != 200 :
392  print " SAMWEB Unable to declare file location (%s, %s) status code %s" %(file, loc, ret.status_code)
393  if fileWPath.endswith (".root"):
394  jsonPath = fileWPath.replace('.root', '.json')
395  elif fileWPath.endswith (".h5"):
396  jsonPath = fileWPath + '.json'
397  if os.path.isfile(jsonPath):
398  returnValue = dh.cp(['-D', jsonPath, outDir])
399  if returnValue != 0:
400  print >> sys.stderr, "Copy out failed for file: " + jsonPath
401  print >> sys.stderr, "Skipping it."
402 
403  for ext in ['.bz2', '']:
404  if os.path.isfile('log.txt'+ext):
405  if file.endswith (".root"):
406  logName = file.replace('.root', '.log'+ext)
407  elif file.endswith (".h5"):
408  logName = file + '.log'+ext
409  returnValue = dh.cp(['log.txt'+ext, outDir+'/'+logName])
410 
411  if returnValue != 0:
412  print >> sys.stderr, "Copy out failed for file: " + logName
413  print >> sys.stderr, "Skipping it."
414  if declareLogs:
415  declareLogFile(logName, 'log.txt'+ext, file)
416 
417  # Remove the copied-out log so it's not in
418  # the way for new log creation.
419  os.remove('log.txt'+ext)
420 
421  break
422 
423  else:
424  print "It does exist, not copying."
425  if not noCleanup:
426  print "Removing", fileWPath
427  os.remove(fileWPath)
428 
429  if fileWPath.endswith(".root"):
430  jsonPath = fileWPath.replace('.root', '.json')
431  elif fileWPath.endswith(".h5"):
432  jsonPath = fileWPath + ".json"
433  if os.path.isfile(jsonPath):
434  print 'Removing', jsonPath
435  os.remove(jsonPath)
436  return
437 
438 def makeDeCAF(script, fname, special):
439  trimname = fname[:-5] # cut off .root from end
440  trimidx = trimname.rindex('.')+1 # find last period before .root
441  decaf_tier = trimname[trimidx:-3]+'de'+trimname[-3:] # properly insert 'de'
442  oname = '{0}_{1}.{2}.root'.format(trimname, special, decaf_tier)
443  novaSource = os.getenv("SRT_PUBLIC_CONTEXT", "undefined")
444  if(novaSource == "undefined"):
445  novaSource = os.getenv("NOVASOFT_DIR", "undefined")
446  if(novaSource == "undefined"):
447  NGU.fail("Unable to locate NOvA source code")
448  else:
449  novaSource = os.getenv("NOVASOFT_DIR") + "/source"
450 
451  os.system('cafe -bq '+novaSource+'/CAFAna/'+script+' '+fname+' '+oname+' 2>&1')
452  return oname
453 
454 def resolveFclPath(fcl):
455  # Check if we have an absolute path name, return it if so
456  if fcl[0] == "/":
457  return fcl
458 
459  # Otherwise, we need to do some searching
460  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
461  for path in fclPaths:
462  # ensure there is always a trailing "/" on the path
463  path += "/"
464  if os.path.isfile(path + fcl):
465  return path + fcl
466 
467  # If we haven't found it, we have a problem.
468  raise IOError(sys.argv[0] + ": config file "+ fcl+" not found in FHICL_FILE_PATH")
469 
470 
471 if __name__=='__main__':
472 
473  parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')
474  parser.add_argument('inFile', help='The input file to run over', type=str)
475  parser.add_argument('--config', '-c', help='FHiCL file to use as configuration for nova executable', type=str)
476  parser.add_argument('--outTier', help="""
477  Data tier of the output file, multiple allowed, formatted as
478  <name_in_fcl_outputs>:<data_tier>.' Optionally, if a second colon is
479  included, the third argument will be treated as an additional naming string,
480  allowing multiple outputs with the same data_tier but unique file names.
481  Example: out1:reco:shifted leads to <file_id>_shifted.reco.root
482  """, type=str, action='append')
483  parser.add_argument('--cafTier', help="""Module label for CAF output,
484  multiple allowed. Format as <cafmaker_module_label>:<data_tier>.
485  Optionally, if a second colon is
486  included, the third argument will be treated as an additional naming string,
487  allowing multiple outputs with the same data_tier but unique file names.
488  Example: cafmaker:caf:shifted leads to <file_id>_shifted.caf.root
489  """, type=str, action='append')
490  parser.add_argument('--flatTier', help="""Module label for FlatCAF output,
491  multiple allowed. Format as <flatmaker_module_label>:<data_tier>.
492  Optionally, if a second colon is
493  included, the third argument will be treated as an additional naming string,
494  allowing multiple outputs with the same data_tier but unique file names.
495  Example: flatmaker:flatcaf:shifted leads to <file_id>_shifted.flatcaf.root
496  """, type=str, action='append')
497  parser.add_argument('--histTier', help='File identifier string for TFileService output, only one allowed. Supply as --histTier <id> for output_name.<id>.root, where output_name is assembled based on the input file.', type=str)
498  parser.add_argument('--h5Tier', help="""Module label for H5 output,
499  multiple allowed. Format as <h5maker_module_label>:<data_tier>.
500  Optionally, if a second colon is
501  included, the third argument will be treated as an additional naming string,
502  allowing multiple outputs with the same data_tier but unique file names.
503  Example: h5maker:h5:shifted leads to <file_id>_shifted.h5
504  """, type=str, action='append')
505  parser.add_argument('--outputNumuDeCAF', help='Make standard numu decafs for all CAF files produced', action='store_true')
506  parser.add_argument('--outputNueDeCAF', help='Make standard nue decafs for all CAF files produced', action='store_true')
507  parser.add_argument('--outputNumuOrNueDeCAF', help='Make standard numu or nue decafs for all CAF files produced', action='store_true')
508  parser.add_argument('--outputNusDeCAF', help='Make standard nus decafs for all CAF files produced', action='store_true')
509  parser.add_argument('--outputValidationDeCAF', help='Make validation (nue_or_numu_or_nus) decafs for all CAF files produced during the job', action='store_true')
510  parser.add_argument('--cosmicsPolarity', help='.', type=str)
511  parser.add_argument('--npass', help='.', type=str)
512  parser.add_argument('--skim', help='Specify skimming name.', type=str)
513  parser.add_argument('--systematic', help='Flag as systematic variation (append to file name and metadata parameters).', type=str)
514  parser.add_argument('--specialName', help='Additional name to add before data tier in output.', type=str)
515  parser.add_argument('--genietune', help='Specify the GENIE tune (append to file name and metadata parameters).', type=str)
516  parser.add_argument('--NPPFX', help='Number of PPFX universes.', type=str)
517  parser.add_argument('-n', help='Number of events to run over', type=int)
518  parser.add_argument('--copyOut', help='Use the built in copy out mechanism', action='store_true')
519  parser.add_argument('--dest', '-d', help='Output file destination for --copyOut functionality.', type=str)
520  parser.add_argument('--hashDirs', help='Use hash directory structure in destination directory.', action='store_true')
521  parser.add_argument('--runDirs', help='Use run directory structure in destination directory, 000XYZ/XYZUW for run number XYZUW.', action='store_true')
522  parser.add_argument('--autoDropbox', help='Use automatic dropox location', default=False, action='store_true')
523  parser.add_argument('--jsonMetadata', help='Create JSON files with metadata corresponding to each output file, and copy them to the same destinations', action='store_true')
524  parser.add_argument('--declareFiles', help='Declare files with metadata on worker node', action='store_true')
525  parser.add_argument('--declareLocations', help='Declare the file output locations to SAM during the copy back of the files', action='store_true')
526  parser.add_argument('--logs', help='Return .log files corresponding to every output', action='store_true')
527  parser.add_argument('--zipLogs', help='Format logs as .bz2 files. Implies --logs', action='store_true')
528  parser.add_argument('--noCleanup', help='Skip working directory cleanup step, good for interactive debugging or custom copy-out.', action='store_true')
529  parser.add_argument('--gdb', help='Run nova executable under gdb, print full stack trace, then quit gdb.', action='store_true')
530  parser.add_argument('--lemBalance', help='Choose lem server based on (CLUSTER+PROCESS)%%2 to balance load', action='store_true')
531  parser.add_argument('--lemServer', help='Specify lem server', type=str)
532  parser.add_argument('--txtfiledef', help='Use if the input definition is made up of text files, each containing a list of file names',default=False, action='store_true')
533  parser.add_argument('--precopyscript', help='Execute script PRECOPYSCRIPT within runNovaSAM.py, after running the nova -c command.', type=str, action='append')
534  args = parser.parse_args()
535 
536  # Sanity check for output
537  if args.copyOut:
538  if not (args.outTier or args.cafTier or args.flatTier or args.histTier or args.h5Tier):
539  raise Exception("Copy-out requested with --copyOut, but no outputs specified. Nothing will happen with output, aborting.")
540  if not (args.dest or "DEST" in os.environ):
541  raise Exception("Copy-out requested with --copyOut, but no output directory specified. Use --dest or $DEST.")
542 
543  # No longer set VMem limit -- causes problems on some OSG sites
544  #setVMemLimit()
545 
546  samweb = samweb_client.SAMWebClient(experiment='nova')
547 
548  if "SRT_BASE_RELEASE" in os.environ:
549  release = os.environ["SRT_BASE_RELEASE"]
550  elif "NOVASOFT_VERSION" in os.environ:
551  release = os.environ["NOVASOFT_VERSION"]
552  else:
553  print "No release set!"
554  exit(1)
555 
556  inFile = args.inFile
557  inFileBase = os.path.basename(inFile)
558 
559  # Which file do I want to use to get the metadata?
560  if not args.txtfiledef:
561  # Normally my infile.
562  metadata = samweb.getMetadata(inFileBase)
563  fileMetaDataMgr = MetadataUtils.metaDataMgr(inFile, metadata, release, args.systematic, args.skim, args.cosmicsPolarity, args.npass, args.specialName)
564  else:
565  # However, if using a txtfile def, want to use the first file in the txt file.
566  with open( inFile ) as f:
567  PassFile = f.readline().strip()
568  print "Looking at ", PassFile
569  #metadata_cmd = "ifdh_fetch %s" %PassFile
570  #os.system(metadata_cmd)
571  metadata = samweb.getMetadata(PassFile)
572  fileMetaDataMgr = MetadataUtils.metaDataMgr(PassFile, metadata, release, args.systematic, args.skim, args.cosmicsPolarity, args.npass, args.specialName)
573  #os.environ["NPASS"]=args.npass
574  fclFile = args.config
575  # Get a copy of the fcl file and bring it here
576  # First step, find the fcl
577  fclPath = resolveFclPath(fclFile)
578  print "Found fcl file here: ", fclPath
579  # Next step, make a temporary name for the fcl in the local directory and copy it here
580  tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
581  print "Creating local copy : ", tmpFclName
582  shutil.copy(fclPath, tmpFclName)
583 
584  isRunningMetadataModule=False
585  # NOTE:
586  # fhicl-expand (and other FCL tools) don't allow you
587  # to give the absolute path to a FCL. Instead FCLs have to
588  # live in $FHICL_FILE_PATH. $FHICL_FILE_PATH always begins
589  # with './:', i.e., search current dir first.
590  # so just make sure you don't cd() away from this dir
591  # between when the FCL is copied to this dir above and the check below.
592  isRunningMetadataModule=True
593  try:
594  subprocess.check_call("fhicl-expand %s | grep -q ^physics.analyzers.metadata.params." % tmpFclName, shell=True)
595  except subprocess.CalledProcessError:
596  isRunningMetadataModule=False
597 
598  # Open the fcl file so that we can append output filenames to it
599  fclFileObj = open(tmpFclName, 'a')
600  print " Open the fcl file so that we can append output filenames to it ::::::::::::::::::::::::::: fclFileObj=", fclFileObj
601  doMeta = True
602  if not (args.outTier or args.cafTier or args.flatTier or args.h5Tier):
603  doMeta = False
604 
605  # Start setting up the nova command, add SAM parameters
606  cmdList = []
607  cmdList.append('nova')
608  cmdList.append('-c')
609  cmdList.append(tmpFclName)
610  if doMeta:
611  cmdList.append('--sam-application-family=nova')
612  cmdList.append('--sam-application-version=' + release)
613  if not fileMetaDataMgr.isSam4Users():
614  cmdList.append('--sam-file-type=' + fileMetaDataMgr.fileType)
615 
616  if isRunningMetadataModule:
617  if not fileMetaDataMgr.isSam4Users():
618  if args.npass != None:
619  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.subversion", '"' + fileMetaDataMgr.subversion + '"')
620 
621  if fileMetaDataMgr.dataFlag == "sim" and fileMetaDataMgr.generator in MetadataUtils.neutrinoGenerators:
622  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.flux_version", '"' + fileMetaDataMgr.fluxVersion + '"')
623 
624  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.skim", '"' + fileMetaDataMgr.skim + '"')
625  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.systematic", '"' + fileMetaDataMgr.systematic + '"')
626  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.Special", '"' + fileMetaDataMgr.special + '"')
627 
628  if not args.outTier:
629  args.outTier = []
630  if not args.cafTier:
631  args.cafTier = []
632  if not args.flatTier:
633  args.flatTier = []
634  if not args.h5Tier:
635  args.h5Tier = []
636 
637  if not args.precopyscript:
638  args.precopyscript = []
639 
640  outList = [] # list of files to be supplied with -o
641  outputs = [] # list of files that will be moved to results directory, includes CAFs
642  # Loop over output tiers
643  for outTier in args.outTier:
644 
645  try:
646  output = outTier.split(":")[0]
647 
648  tier = outTier.split(":")[1]
649 
650  except:
651  raise ValueError("Output data tier: " + outTier + "not formatted corectly, should be <output_name>:<data_tier>")
652 
653  if "," in output:
654  if(re.search('cell',output)):
655  outP, outC = re.findall(',(.+?),', output)
656  outXp, outXc = [re.findall('(.+)-', outP)[0], re.findall('(.+)-', outC)[0]] # p stands for plane, c stands for cell
657  outYp, outYc = [re.findall('-(.+)', outP)[0], re.findall('-(.+)', outC)[0]]
658  for i in range( int(outXp), int(outYp)+1 ):
659  for j in range( int(outXc), int(outYc)+1):
660  NewOMod = re.search('^(.+?),', output).group(1) + `i` + re.search('cell',output).group() + `j`
661  cmdList.append('--sam-data-tier=' + ":".join([NewOMod, tier]))
662  if not fileMetaDataMgr.isSam4Users():
663  if fileMetaDataMgr.dataFlag == "data":
664  cmdList.append('--sam-stream-name=' + NewOMod + ':' + str(fileMetaDataMgr.stream))
665  else:
666  outX = re.search(',(.+?)-', output).group(1)
667  outY = re.search('-(.+?),', output).group(1)
668  for i in range( int(outX), int(outY)+1 ):
669  NewOMod = re.search('^(.+?),', output).group(1)+`i`
670  cmdList.append('--sam-data-tier=' + ":".join([NewOMod, tier]))
671  if not fileMetaDataMgr.isSam4Users():
672  if fileMetaDataMgr.dataFlag == "data":
673  cmdList.append('--sam-stream-name=' + NewOMod + ':' + str(fileMetaDataMgr.stream))
674  else:
675  cmdList.append('--sam-data-tier=' + ":".join([output, tier]))
676  if not fileMetaDataMgr.isSam4Users():
677  if fileMetaDataMgr.dataFlag == "data":
678  cmdList.append('--sam-stream-name=' +output + ':' + str(fileMetaDataMgr.stream))
679 
680  outNameTemp = fileMetaDataMgr.getOutputFileName(tier)
681  if args.txtfiledef:
682  FirstRun = re.search('FirstRun-(.+?)_LastRun', os.path.basename(inFile)).group(1).zfill(8)
683  LastRun = re.search('LastRun-(.+?)_TotFiles', os.path.basename(inFile)).group(1).zfill(8)
684  Index=outNameTemp.find("_r0")
685  outNameTemp=outNameTemp[:Index]+"_r"+FirstRun+"_r"+LastRun+outNameTemp[Index+14:]
686  outName = os.path.basename(outNameTemp)
687 
688  if "," in output:
689  if(re.search('cell',output)):
690  outP, outC = re.findall(',(.+?),', output)
691  outXp, outXc = [re.findall('(.+)-', outP)[0], re.findall('(.+)-', outC)[0]] # p stands for plane, c stands for cell
692  outYp, outYc = [re.findall('-(.+)', outP)[0], re.findall('-(.+)', outC)[0]]
693  for i in range( int(outXp), int(outYp)+1 ):
694  for j in range( int(outXc), int(outYc)+1):
695  NewOMod = re.search('^(.+?),', output).group(1)+`i` + re.search('cell',output).group() + `j`
696  tier = outTier.split(":")[1]
697  NewOName = outName.replace(str("."+tier), str("-"+NewOMod+"."+tier))
698  fclFileObj.write("\noutputs." + NewOMod + '.fileName: "'+ NewOName + '"\n')
699  outList.append(NewOName)
700  outputs.append(NewOName)
701  else:
702  print "Running in Plane Mode"
703  outX = re.search(',(.+?)-', output).group(1)
704  outY = re.search('-(.+?),', output).group(1)
705  for i in range( int(outX), int(outY)+1 ):
706  NewOMod = re.search('^(.+?),', output).group(1)+`i`
707  tier = outTier.split(":")[1]
708  NewOName = outName.replace(str("."+tier), str("-"+NewOMod+"."+tier))
709  fclFileObj.write("\noutputs." + NewOMod + '.fileName: "'+ NewOName + '"\n')
710  outList.append(NewOName)
711  outputs.append(NewOName)
712  else:
713  print "Output file name: ", outName, " for tier ", tier, " and output ", output
714  fclFileObj.write("\noutputs." + output + '.fileName: "'+ outName + '"\n')
715  outList.append(outName)
716  outputs.append(outName)
717 
718  for cafTier in args.cafTier:
719  try:
720  cafLabel = cafTier.split(":")[0]
721  tier = cafTier.split(":")[1]
722  except:
723  raise ValueError("Output data tier: " + cafTier + "not formatted corectly, should be <output_name>:<data_tier>")
724 
725  cafName = fileMetaDataMgr.getOutputFileName(tier)
726  print "Adding CAF: ", cafLabel, tier, cafName
727 
728  fclFileObj.write("\nphysics.producers." + cafLabel + '.CAFFilename: "' + cafName + '" \n')
729  fclFileObj.write("physics.producers." + cafLabel + '.DataTier: "' + tier + '" \n')
730  outputs.append(cafName)
731 
732 
733  for flatTier in args.flatTier:
734  try:
735  flatLabel = flatTier.split(":")[0]
736  tier = flatTier.split(":")[1]
737  except:
738  raise ValueError("Output data tier: " + flatTier + "not formatted corectly, should be <output_name>:<data_tier>")
739 
740  flatName = fileMetaDataMgr.getOutputFileName(tier)
741  print "Adding FlatCAF: ", flatLabel, tier, flatName
742 
743  fclFileObj.write("\nphysics.producers." + flatLabel + '.OutputName: "' + flatName + '" \n')
744  fclFileObj.write("physics.producers." + flatLabel + '.DataTier: "' + tier + '" \n')
745  outputs.append(flatName)
746 
747 
748  for h5Tier in args.h5Tier:
749  try:
750  h5Label = h5Tier.split(":")[0]
751  tier = h5Tier.split(":")[1]
752  except:
753  raise ValueError("Output data tier: " + h5Tier + "not formatted corectly, should be <output_name>:<data_tier>")
754 
755  h5Name = fileMetaDataMgr.getOutputFileName(tier)
756  print "Adding H5: ", h5Label, tier, h5Name
757 
758  fclFileObj.write("\nphysics.analyzers." + h5Label + '.H5Filename: "' + h5Name +'"' )
759  fclFileObj.write("\nphysics.analyzers." + h5Label + '.DataTier: "" \n')
760  outputs.append(h5Name+".h5")
761 
762  if args.lemBalance and (atoi(os.environ["PROCESS"])+atoi(os.environ["CLUSTER"]))%2==0:
763  fclFileObj.write("physics.producers.lem.WebSettings.Host: \"lem2.hep.caltech.edu\"\n")
764  elif args.lemServer:
765  fclFileObj.write("physics.producers.lem.WebSettings.Host: \"%s\"\n" % args.lemServer)
766 
767 
768  if args.histTier:
769  try:
770  tier = str(args.histTier)
771  except:
772  raise Exception("Histogram identifier supplied by --histTier could not be converted to a string.")
773 
774  histName = fileMetaDataMgr.getOutputFileName(tier)
775 
776  outputs.append(histName)
777  fclFileObj.write("\nservices.TFileService.fileName: " + '"' + histName + '"\n')
778 
779  fclFileObj.close()
780 
781  print "Config: "
782  for line in open(tmpFclName):
783  print line.strip()
784 
785  if args.n != None:
786  cmdList.append('-n')
787  cmdList.append(str(args.n))
788 
789  if args.txtfiledef:
790  print "\nI have a text file definition, InFile is {}".format(inFile)
791  ### Are we streaming the files via xrootd?
792  #txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"%inFile
793  #os.system(txtcmd)
794  #with open("xrootd_inFile.txt") as f:
795  # for line in f:
796  # print line.strip()
797  # cmdList.append( line.strip() )
798  #print ""
799  ### Are we going to copy the files?
800  olddir = os.getcwd()
801  os.system("mkdir InFiles")
802  allFiles = 0.
803  failFiles = 0.
804  with open(inFile) as f:
805  os.chdir("InFiles")
806  for line in f:
807  allFiles += 1
808  copyfile = "InFiles/%s" %line.strip()
809  print "Now copying",line.strip(),"to ",copyfile
810  ifdhcmd = "ifdh cp -D `samweb2xrootd %s` ." %line.strip()
811  print datetime.datetime.now()
812  ret = os.system( ifdhcmd )
813  if ret == 0:
814  cmdList.append( copyfile )
815  else:
816  failFiles += 1
817  print("Copy in success ratio: " + str((allFiles-failFiles)/allFiles))
818  os.chdir(olddir)
819  else:
820  cmdList.append(inFile)
821 
822  if args.gdb:
823  gdbArgs = ["gdb", "-return-child-result", "--ex", "run", "--ex", "bt", "full", "--ex", "q", "--args"]
824  cmdList = gdbArgs + cmdList
825 
826  cmd = ' '.join(cmdList)
827 
828  print 'Running:', cmd
829  sys.stdout.flush() # flush the stdout buffer before running the nova executable, cleans up output.
830 
831  if args.logs or args.zipLogs:
832  with open('log.txt', 'w') as logfile:
833  sys.stderr.write('\nnova command runs here. stderr redirected to stdout\n\n')
834  retCode = subprocess.call(cmdList, stdout=logfile, stderr=subprocess.STDOUT)
835  # Print all the output to the screen as well so that regular condor
836  # logs include it too.
837  with open('log.txt', 'r') as logfile:
838  for line in logfile:
839  print line,
840 
841  if args.zipLogs:
842  os.system('bzip2 -f log.txt')
843  else:
844  retCode = subprocess.call(cmdList)
845 
846  ### If using a txtfiledef make sure to clean up the InputFile List....
847  if args.txtfiledef:
848  os.system("rm -rf InFiles")
849 
850  if retCode != 0:
851  print "Want to copy back the logs for this job somehwere...."
852  else:
853 
854  # If the initial job finished successfully, I may want to run some post art scripts within runNovaSAM.
855  # Loop over any intermediate scripts
856  #for pcscript in args.precopyscript:
857  # print "Now I want to run %s...Just going to run it straight off the command line..." %pcscript
858  # print "If you need to something more intelligent you'll have to add some extra flags here!"
859  # os.system(pcscript)
860 
861  if args.outputNumuDeCAF or args.outputNueDeCAF or args.outputNumuOrNueDeCAF or args.outputNusDeCAF or args.outputValidationDeCAF:
862  decafdir = os.listdir(".")
863  for fname in decafdir:
864  if fname.endswith("caf.root"):
865  if args.outputNumuDeCAF:
866  outputs.append(makeDeCAF('numu/FirstAnalysis/reduce_numu_fa.C',fname,'numu_contain'))
867  if args.outputNueDeCAF:
868  outputs.append(makeDeCAF('nue/reduce_nue_sa.C',fname,'nue_contain'))
869  if args.outputNumuOrNueDeCAF:
870  outputs.append(makeDeCAF('nue/reduce_nue_or_numu_sa.C',fname,'nue_or_numu_contain'))
871  if args.outputNusDeCAF:
872  outputs.append(makeDeCAF('nus/reduce_nus.C',fname,'nus_contain'))
873  if args.outputValidationDeCAF:
874  outputs.append(makeDeCAF('nus/reduce_nue_or_numu_or_nus.C',fname,'nue_or_numu_or_nus_contain'))
875  print "\nAt the start of check and move."
876  checkAndMoveFiles(inFile, outputs, args.noCleanup)
877 
878  if args.copyOut:
879  if args.dest:
880  dest = args.dest
881  elif "DEST" in os.environ:
882  dest = os.environ["DEST"]
883  else:
884  raise Exception("Copy out requested with --copyOut, but no destination supplied. Use --dest or $DEST")
885 
886  if args.autoDropbox:
887  dest=NGU.get_prod_dropbox()
888  print "Getting automatic dropbox location",dest
889 
890  print "\nMake JSONs"
891  if args.jsonMetadata:
893  print "\nMade JSONs, now to copy files."
894  copyOutFiles(dest, args.hashDirs, args.runDirs, fileMetaDataMgr.runNum, args.noCleanup, args.declareLocations, args.declareFiles)
895  print "Copied files."
896  else:
897  #job didn't succeed, remove output files if they exist
898  for file in outList:
899  try:
900  os.remove("./" + file)
901  except OSError:
902  pass
903 
904  #clean up section
905  if not args.noCleanup:
906  os.remove(tmpFclName)
907  os.remove(inFile)
908  dirList = os.listdir(".")
909  for file in dirList:
910  skip_match = _skip_pattern.match(file)
911  if skip_match != None:
912  print file, "contains RootOutput*.root: clean up"
913  os.remove("./" + file)
914 
915  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
916  dh.cleanup()
917 
918  exit(retCode)
def makeMetadataJSONs()
Definition: runNovaSAM.py:243
void split(double tt, double *fr)
def copyOutFiles(hashDirs=False)
Definition: runNovaSAM.py:149
def makeDirSafely(dir)
Definition: runNovaSAM.py:41
def checkAndMoveFiles(inFile, declareFiles)
Definition: runNovaSAM.py:105
def resolveFclPath(fcl)
Definition: runNovaSAM.py:188
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM.py:92
if(dump)
bool print
def MakeTransposeJson(TransposeList)
Definition: runNovaSAM.py:310
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
def createMetadata(inFile)
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
def addMetadataToFCL(fclFile, parName, parValue)
exit(0)
def listFiles(outPath, dh)
Definition: runNovaSAM.py:236
def setVMemLimit()
Definition: runNovaSAM.py:33
def fileExists(outPath, dh)
Definition: runNovaSAM.py:224
def makeDirIfNeeded(dir)
Definition: runNovaSAM.py:61
def declareFile(fileWPath)
Definition: runNovaSAM.py:127
def declareLogFile(logName, logFile, rootName)
Definition: runNovaSAM.py:205
def makeDeCAF(script, fname, special)
Definition: runNovaSAM.py:438