Functions | Variables
runNovaSAM Namespace Reference


def unCamelCase (s)
def createMetadata (inFile)
def getOutDir (pathname, hashDirs=False)
def checkAndMoveFiles (inFile, declareFiles)
def declareFile (fileWPath)
def copyOutFiles (hashDirs=False)
def getOutputFileName (detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
def resolveFclPath (fcl)
def setVMemLimit ()
def makeDirSafely (dir)
def makeDirIfNeeded (dir)
def getOutDir (pathname, dest, hashDirs=False, runDirs=False, runNum=0)
def checkAndMoveFiles (inFile, outputs, noCleanup=False)
def declareLogFile (logName, logFile, rootName)
def fileExists (outPath, dh)
def listFiles (outPath, dh)
def makeMetadataJSONs ()
def MakeTransposeJson (TransposeList)
def copyOutFiles (dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False)
def makeDeCAF (script, fname, special)


 detectors = dict()
 _lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
 _skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
 parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')
 args = parser.parse_args()
 inFile = args.inFile
 samweb = samweb_client.SAMWebClient(experiment='nova')
 inFileBase = os.path.basename(inFile)
 metadata = samweb.getMetadata(inFileBase)
 runNum = int(metadata["runs"][0][0])
 subNum = int(metadata["runs"][0][1])
 run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")
 run_match = run_pattern.match(inFileBase)
 stream = int(metadata["data_stream"])
string streamEntry = 't{0:02d}'
 stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)
 stream_match = stream_pattern.match(inFileBase)
 detector = metadata["online.detector"].lower()
 detId = metadata["nova.detectorid"].lower()
 release = os.environ["SRT_BASE_RELEASE"]
 file_type = metadata["file_type"]
string dataFlag = "data"
 subversion = os.environ["NPASS"]
 nevt = None
 fclFile = args.config
 fclPath = resolveFclPath(fclFile)
 tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
 fclFileObj = open(tmpFclName, 'a')
list cmdList = ['nova']
list outList = []
 output = outTier.split(":")[0]
 tier = outTier.split(":")[1]
 outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 cafLabel = cafTier.split(":")[0]
 cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
string cmd = ' '
 retCode =
 dirList = os.listdir(".")
 skip_match = _skip_pattern.match(file)
 dh = ifdh.ifdh("")
 Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help. More...
 Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files? More...
 If using a txtfiledef make sure to clean up the InputFile List.... More...

Function Documentation

def runNovaSAM.checkAndMoveFiles (   inFile,
  noCleanup = False 
Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 88 of file

References checkAndMoveFiles(), MetadataUtils.createMetadata(), declareFile(), and

88 def checkAndMoveFiles(inFile, outputs, noCleanup=False):
89  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
90  inFileBase = os.path.basename(inFile)
91  baseDir = "."
92  # If declaring files have to do non-decafs first, since they're the parents
93  # of the decafs. If not, it doesn't hurt to do it in that order anyway.
94  for decafPass in [False, True]:
95  for root, dirs, filenames in os.walk(baseDir):
96  if root == baseDir:
97  for file in filenames:
98  if file.endswith('decaf.root') != decafPass: continue
99  # If file not in outputs, then delete.
100  if (file.endswith (".root") or file.endswith(".h5")) and file != inFileBase:
101  fileWPath = os.path.join(root, file)
102  # If the file isn't in my outputs then delete it.
103  if file not in outputs:
104  if not noCleanup:
105  # Do not copy over, just delete
106  print "File", fileWPath, " is not among requested outputs, removing"
107  os.remove(fileWPath)
108  continue
109  # Now that I know that this file is in my output list, lets check it is valid...
110  # First, if it is a root file.
111  if file.endswith (".root"):
112  print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
113  rootFile = ROOT.TFile(fileWPath)
114  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
115  #do not copy over, just delete
116  print "File", fileWPath, "is Zombie - remove it"
117  os.remove(fileWPath)
118  else:
119  newFilePath = os.path.join(root, "results", file)
120  print "New file name is %s" %newFilePath
121  os.renames(fileWPath, newFilePath)
122  rootFile.Close()
123  # Next, if it is a h5 file.
124  elif file.endswith(".h5"):
125  # Only import h5py if have h5 files.
126  import h5py
127  print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
128  # If a valid HDF5 files.
129  if h5py.is_hdf5(fileWPath):
130  newFilePath = os.path.join(root, "results", file)
131  os.renames(fileWPath, newFilePath)
132  print "Moved ", fileWPath, "to ", newFilePath
133  # If not a valid HDF5 file.
134  else:
135  print "File", fileWPath, "is Zombie - remove it"
136  os.remove(fileWPath)
137  return
139 #def h5MetaHack(inFile):
140 # print "Doing a hack to get HDF5 metadata...Will take CAF metadata and change subtly."
141 # ### Make my client and get my CAF metadata
142 # samweb = samweb_client.SAMWebClient(experiment='nova')
143 # md = MetadataUtils.createMetadata( inFile.replace('.h5caf.h5', '.caf.root') )
144 # ### Change some parameters.
145 # md['file_name'] = inFile
146 # md['data_tier'] = unicode('h5')
147 # md['file_size'] = os.path.getsize( inFile )
148 # ### return md
149 # return md
def checkAndMoveFiles(inFile, declareFiles)
def runNovaSAM.checkAndMoveFiles (   inFile,
Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 105 of file

References declareFile().

Referenced by checkAndMoveFiles().

105 def checkAndMoveFiles(inFile, declareFiles):
106  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
107  inFileBase = os.path.basename(inFile)
108  baseDir = "."
109  for root, dirs, filenames in os.walk(baseDir):
110  if root == baseDir:
111  for file in filenames:
112  if file.endswith (".root") and file != inFileBase:
113  fileWPath = os.path.join(root, file)
114  rootFile = ROOT.TFile(fileWPath)
115  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
116  #do not copy over, just delete
117  print "File", fileWPath, "is Zombie - remove it"
118  os.remove(fileWPath)
119  else:
120  if declareFiles == True:
121  declareFile(fileWPath)
122  newFilePath = os.path.join(root, "results", file)
123  os.renames(fileWPath, newFilePath)
124  rootFile.Close()
125  return
def checkAndMoveFiles(inFile, declareFiles)
def declareFile(fileWPath)
def runNovaSAM.copyOutFiles (   hashDirs = False)
Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 149 of file

References getOutDir().

Referenced by copyOutFiles().

149 def copyOutFiles(hashDirs=False):
150  """Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
151  print "Copying out files"
152  dh = ifdh.ifdh("")
153  baseDir = "./results"
154  for root, dirs, filenames in os.walk(baseDir):
155  if root == baseDir:
156  for file in filenames:
157  if file.endswith (".root") and file != inFileBase:
158  fileWPath = os.path.join(root, file)
159  dest = getOutDir(file, hashDirs)
160  skip_match = _skip_pattern.match(file)
161  if skip_match == None:
162  try:
163  print "Checking if", dest + "/" + file, "exists"
164 + "/" + file, 1, "")
165  except RuntimeError:
166  print "It doesn't - copy", fileWPath, "to", dest
167  dh.cp(["-D", fileWPath, dest])
168  print "Removing", fileWPath
169  os.remove(fileWPath)
170  dh.cleanup()
171  return
def copyOutFiles(hashDirs=False)
def getOutDir(pathname, hashDirs=False)
def runNovaSAM.copyOutFiles (   dest,
  hashDirs = False,
  runDirs = False,
  runNum = 0,
  noCleanup = False,
  declareLocation = False,
  declareLogs = False 
Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 361 of file

References copyOutFiles(), declareFile(), declareLogFile(), and getOutDir().

361 def copyOutFiles(dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False):
362  """Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
363  dh = ifdh.ifdh("")
364  baseDir = "./results"
365  declareFiles = declareLogs
366  for root, dirs, filenames in os.walk(baseDir):
367  if root == baseDir:
368  for file in filenames:
369  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
370  fileWPath = os.path.join(root, file)
371  outDir = getOutDir(file, dest, hashDirs, runDirs, runNum)
372  skip_match = _skip_pattern.match(file)
373  if skip_match == None:
374  outPath = os.path.join(outDir, file)
376  # note: this will fail if the file already exists
377  returnValue = dh.cp(["-D", fileWPath, outDir])
378  if returnValue != 0:
379  print >> sys.stderr, "Copy out failed for file:", fileWPath
380  print >> sys.stderr, "Skipping it."
381  else:
382  if declareFiles:
383  declareFile(fileWPath)
384  ###################
385  # Declare the file's location to SAM if we have the declareLocation option on
386  if declareLocation==True :
387  loc = string.replace(outDir, 's3://','s3:/')
388  print "Declaring location %s for file %s\n" % (loc,file)
389  sam = samweb_client.SAMWebClient('nova')
390  ret=sam.addFileLocation(file, loc)
391  if ret.status_code != 200 :
392  print " SAMWEB Unable to declare file location (%s, %s) status code %s" %(file, loc, ret.status_code)
393  if fileWPath.endswith (".root"):
394  jsonPath = fileWPath.replace('.root', '.json')
395  elif fileWPath.endswith (".h5"):
396  jsonPath = fileWPath + '.json'
397  if os.path.isfile(jsonPath):
398  returnValue = dh.cp(['-D', jsonPath, outDir])
399  if returnValue != 0:
400  print >> sys.stderr, "Copy out failed for file: " + jsonPath
401  print >> sys.stderr, "Skipping it."
403  for ext in ['.bz2', '']:
404  if os.path.isfile('log.txt'+ext):
405  if file.endswith (".root"):
406  logName = file.replace('.root', '.log'+ext)
407  elif file.endswith (".h5"):
408  logName = file + '.log'+ext
409  returnValue = dh.cp(['log.txt'+ext, outDir+'/'+logName])
411  if returnValue != 0:
412  print >> sys.stderr, "Copy out failed for file: " + logName
413  print >> sys.stderr, "Skipping it."
414  if declareLogs:
415  declareLogFile(logName, 'log.txt'+ext, file)
417  # Remove the copied-out log so it's not in
418  # the way for new log creation.
419  os.remove('log.txt'+ext)
421  break
423  else:
424  print "It does exist, not copying."
425  if not noCleanup:
426  print "Removing", fileWPath
427  os.remove(fileWPath)
429  if fileWPath.endswith(".root"):
430  jsonPath = fileWPath.replace('.root', '.json')
431  elif fileWPath.endswith(".h5"):
432  jsonPath = fileWPath + ".json"
433  if os.path.isfile(jsonPath):
434  print 'Removing', jsonPath
435  os.remove(jsonPath)
436  return
def copyOutFiles(hashDirs=False)
def getOutDir(pathname, hashDirs=False)
def declareFile(fileWPath)
def declareLogFile(logName, logFile, rootName)
def runNovaSAM.createMetadata (   inFile)

Definition at line 36 of file

References samweb_client.utility.fileEnstoreChecksum(), site_stats_from_log.get, str, and unCamelCase().

Referenced by declareFile().

36 def createMetadata(inFile):
37  filename = os.path.basename(inFile)
38  filesize = os.path.getsize(inFile)
40  try:
41  data = subprocess.check_output(["sam_metadata_dumper", str(inFile)])
42  except subprocess.CalledProcessError:
43  print "sam_metadata_dumper failed!"
44  return None
46  md = {'file_name': filename, 'group': 'nova', 'file_size': filesize, 'file_format': 'root'}
48  # An ad-hoc parser for the output
49  mode = 'new'
50  for line in data.split('\n'):
51  m = _lineregex.match(line)
52  if mode == 'json':
53  if not m and not line.startswith('-'):
54  data += line
55  else:
56  md[key] = json.loads(data)
57  mode = 'new'
59  if mode == 'new':
60  if m:
61  key =
62  value =
64  if '.' not in key:
65  key = unCamelCase(key)
67  # ART does not appear to set the application name
68  # but does provide a process name
69  if key == 'process_name' and md.get('application',[]).get('name') is None:
70  key = 'application_name'
71  elif key == 'stream_name':
72  key = 'data_stream'
74  if key in ('start_date','end_date'):
75  value = long(value)
77  if key == 'runs' or key == 'parents':
78  mode = 'json'
79  data = value
80  elif key.startswith('application_'):
81  # special handling for applications
82  if 'application' not in md:
83  md['application'] = {}
84  md['application'][key[12:]] = value
85  else:
86  md[key] = value
88  md['crc'] = fileEnstoreChecksum(inFile)
90  return md
def unCamelCase(s)
def fileEnstoreChecksum(path)
def createMetadata(inFile)
def runNovaSAM.declareFile (   fileWPath)
Checks file for a TKey of RootFileDB.  If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM

Definition at line 127 of file

References createMetadata().

Referenced by checkAndMoveFiles(), and copyOutFiles().

127 def declareFile(fileWPath):
128  """Checks file for a TKey of RootFileDB. If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM"""
129  samweb = samweb_client.SAMWebClient(experiment='nova')
130  rootFile = ROOT.TFile(fileWPath)
131  if rootFile.FindKey("RootFileDB"):
132  print "Found key (RootFileDB): constructing metadata for", fileWPath
133  md = createMetadata(fileWPath)
134  if (md == None):
135  print "No metadata found!"
136  else:
137  pprint.pprint(md)
138  print ''
139  print "Declaring", fileWPath, "to SAM"
140  try:
141  samweb.declareFile(md)
142  except:
143  print fileWPath, "already exists in SAM"
144  else:
145  print fileWPath, "does not contain RootFileDB, do not try to declare"
146  rootFile.Close()
147  return
def createMetadata(inFile)
def declareFile(fileWPath)
def runNovaSAM.declareLogFile (   logName,

Definition at line 205 of file

Referenced by copyOutFiles().

205 def declareLogFile(logName, logFile, rootName):
206  # This is what NovaFTS/plugins/ does
207  md = {
208  'file_name': os.path.basename(logName),
209  'file_size': os.path.getsize(logFile),
210  'data_tier': 'log',
211  'file_format': 'log',
212  'file_type': 'unknown', # maybe 'nonPhysicsGeneric'?
213  'parents': [{'file_name': os.path.basename(rootName)}]
214  }
215  print "16) declareLogFile(logName, logFile, rootName):"
216  print "Declaring", logName, "to SAM"
217  try:
218  samweb = samweb_client.SAMWebClient(experiment='nova')
219  samweb.declareFile(md)
220  except Exception as inst:
221  print inst
def declareLogFile(logName, logFile, rootName)
def runNovaSAM.fileExists (   outPath,

Definition at line 224 of file

224 def fileExists(outPath, dh):
225  try:
226  # Check to see if you can ls the file.
227  # This works in ifdh 1_7_0 and newer
228  # If it exists, returns a tuple with one entry, converts to True
229  # If it doesn't, returns an empty tuple, converts to False
230  return bool(, 1, ""))
231  except (RuntimeError, IOError) as e:
232  # Some versions of ifdh throw an exception when the file was not found
233  # But that means it is not there.
234  return False
def fileExists(outPath, dh)
def runNovaSAM.getOutDir (   pathname,
  hashDirs = False,
  runDirs = False,
  runNum = 0 

Definition at line 73 of file

References getOutDir(), parse_dependency_file_t.list, makeDirSafely(), and str.

73 def getOutDir(pathname, dest, hashDirs=False, runDirs=False, runNum=0):
74  dirs = [dest]
75  if hashDirs:
76  head, tail = os.path.split(pathname)
77  hash =
78  dirs += list(hash.hexdigest()[:3])
79  if runDirs:
80  head, tail = os.path.split(pathname)
81  runStr = str(runNum)
82  multiRunDir = runStr[:3].zfill(6)
83  makeDirSafely(os.path.join(dest, multiRunDir))
84  makeDirSafely(os.path.join(dest, multiRunDir, runStr))
85  dirs += [multiRunDir, runStr]
86  return os.path.join(*dirs)
def makeDirSafely(dir)
def getOutDir(pathname, hashDirs=False)
def runNovaSAM.getOutDir (   pathname,
  hashDirs = False 

Definition at line 92 of file

References exit(), and parse_dependency_file_t.list.

Referenced by copyOutFiles(), and getOutDir().

92 def getOutDir(pathname, hashDirs=False):
93  if "DEST" in os.environ:
94  dest = os.environ["DEST"]
95  else:
96  print "No destination set!"
97  exit(1)
98  dirs = [dest]
99  if hashDirs:
100  head, tail = os.path.split(pathname)
101  hash =
102  dirs += list(hash.hexdigest()[:3])
103  return os.path.join(*dirs)
def getOutDir(pathname, hashDirs=False)
def runNovaSAM.getOutputFileName (   detector,

Definition at line 173 of file

References exit(), and novadaq::HexUtils.format().

173 def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt):
174  if dataFlag == "sim":
175  pattern = re.compile(r"^.*?_(.*?)_\d+.*_v\d+_(.*?)(\.|\_)sim\..+")
176  match = pattern.match(inFileBase)
177  if match != None:
178  simLabel = match.groups()[0]
179  uniq = match.groups()[1]
180  outName = '{0}_{1}_{2}_r{3:08d}_s{4:02d}_{5}_v{6}_{7}_{8}.{9}.root'.format(detector, simLabel, nevt, runNum, subNum, release, subversion, uniq, dataFlag, tier)
181  else:
182  print "regex couldn't find the sim label and the uniqueifier"
183  exit(1)
184  else:
185  outName = '{0}_r{1:08d}_s{2:02d}_{3}_{4}_v{5}_{6}.{7}.root'.format(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier)
186  return outName
def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def runNovaSAM.listFiles (   outPath,

Definition at line 236 of file

236 def listFiles(outPath, dh):
237  try:
238  return, 1, "")
239  except:
240  return "Exception while trying to ls, OutPath =",outPath
def listFiles(outPath, dh)
def runNovaSAM.makeDeCAF (   script,

Definition at line 438 of file

References novadaq::HexUtils.format(), if(), and resolveFclPath().

438 def makeDeCAF(script, fname, special):
439  trimname = fname[:-5] # cut off .root from end
440  trimidx = trimname.rindex('.')+1 # find last period before .root
441  decaf_tier = trimname[trimidx:-3]+'de'+trimname[-3:] # properly insert 'de'
442  oname = '{0}_{1}.{2}.root'.format(trimname, special, decaf_tier)
443  novaSource = os.getenv("SRT_PUBLIC_CONTEXT", "undefined")
444  if(novaSource == "undefined"):
445  novaSource = os.getenv("NOVASOFT_DIR", "undefined")
446  if(novaSource == "undefined"):
447"Unable to locate NOvA source code")
448  else:
449  novaSource = os.getenv("NOVASOFT_DIR") + "/source"
451  os.system('cafe -bq '+novaSource+'/CAFAna/'+script+' '+fname+' '+oname+' 2>&1')
452  return oname
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def makeDeCAF(script, fname, special)
def runNovaSAM.makeDirIfNeeded (   dir)

Definition at line 61 of file

Referenced by makeDirSafely().

61 def makeDirIfNeeded(dir):
62  os.umask(002) # (rw-rw-r--) for files and (rwxrwxr-x) for directories.
63  if not os.path.isdir(dir):
64  print "runNovaSAM is making a directory: ", dir
65  try:
66  os.mkdir(dir)
67  except:
68  print "Couldn't make the directory... some other job perhaps did, or permissions did not allow "
69  if not os.path.isdir(dir):
70  raise Exception("Failed to make directory + " + dir )
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
def makeDirIfNeeded(dir)
def runNovaSAM.makeDirSafely (   dir)

Definition at line 41 of file

References makeDirIfNeeded().

Referenced by getOutDir().

41 def makeDirSafely(dir):
42  if "/pnfs" != dir[0:5]:
43  makeDirIfNeeded(dir)
44  return
46  print "runNovaSAM is making a directory with IFDH"
47  dh = ifdh.ifdh("")
48  try:
49  print "Checking if directory ", dir, "exists"
50, 1, "")
51  dh.chmod(774,dir,"")
53  except (RuntimeError, IOError) as e:
54  try:
55  print "It doesn't - make directory", dir
56  dh.mkdir(dir)
57  except:
58  print "Tried to make directory and couldn't. Perhaps it already exists?"
def makeDirSafely(dir)
def makeDirIfNeeded(dir)
def runNovaSAM.makeMetadataJSONs ( )
For every ROOT file, try to extract its metadata into a matching .json
file in the same directory

Definition at line 243 of file

References if(), MakeTransposeJson(), open(), and PandAna.Demos.demo1.range.

244  """For every ROOT file, try to extract its metadata into a matching .json
245  file in the same directory"""
246  baseDir = "./results"
247  # If using --txtfiledef, want to pass multiple files at a time to sam_meta_dumper.
248  # This is because it simply takes too long to call it for each indiv file
249  # when there are 800+ outputs. 5 s each -> ~1 hour!
250  TransposeList = ""
251  # Declare the samweb client within this function
252  samweb = samweb_client.SAMWebClient(experiment='nova')
253  # Loop through directories to search for files to make json files for.
254  for root, dirs, filenames in os.walk(baseDir):
255  if root == baseDir:
256  # Push the h5 files to the front of the list so that the CAF files remain
257  # available to hack in the metadata.
258  for ifile in range(len(filenames)):
259  if("h5" in filenames[ifile]):
260  filenames = [filenames[ifile]] + filenames[:ifile] + filenames[ifile+1:]
261  for file in filenames:
262  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
263  skip_match = _skip_pattern.match(file)
264  if skip_match == None:
265  # Set some quick and useful variables.
266  olddir = os.getcwd()
267  fileWPath = os.path.join(root, file)
268  # If a transpose file want to make the json subtly differently.
269  if args.txtfiledef and "outplane" in file:
270  # This works for the cellmode of the transposer as well, due to the filename including outplane and cell.
271  print "Adding %s to TransposeList" %file
272  TransposeList += "%s " %file
273  continue
274  # Which extractor am I using?
275  extractor = 'sam_metadata_dumper'
276  if file.endswith('caf.root'):
277  extractor = 'extractCAFMetadata'
278  elif file.endswith('.h5'):
279  extractor = 'extractHDF5Metadata'
280  try:
281  # sam_metadata_dumper doesn't apply basename() to filename.
282  os.chdir(os.path.dirname(fileWPath))
283  meta = subprocess.check_output([extractor, os.path.basename(fileWPath)])
284  if file.endswith (".root"):
286  jsonf = open(file.replace('.root', '.json'), 'w')
287  jsonf.write(meta)
288  jsonf.close()
289  print "Made metadata for %s" %file
290  elif file.endswith(".h5"):
291  print "\nNow to make my json file for my h5...\n"
292  jsonf = open(file.replace('.h5caf.h5', '.h5caf.json'), 'w')
293  jsonf.write(meta)
294  jsonf.close()
295  print "Made metadata for %s" %file
296  else:
297  print "I'm not sure what file extension you have..."
298  except:
299  print "Error extracting metadata from file."
300  finally:
301  os.chdir(olddir)
302  # Make the Transpose json files.
303  # Again same argument, outplane is already in the filenames for cell mode
304  if args.txtfiledef and "outplane" in file:
305  olddir = os.getcwd()
306  os.chdir(baseDir)
307  MakeTransposeJson( TransposeList )
308  os.chdir(olddir)
def makeMetadataJSONs()
def MakeTransposeJson(TransposeList)
procfile open("FD_BRL_v0.txt")
def runNovaSAM.MakeTransposeJson (   TransposeList)
Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
This takes in a list of files, and makes appropriate .json files in the same directory

Definition at line 310 of file

References, open(), PandAna.Demos.demo1.range, and split().

Referenced by makeMetadataJSONs().

310 def MakeTransposeJson( TransposeList ):
311  """Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
312  This takes in a list of files, and makes appropriate .json files in the same directory"""
313  # If using --txtfiledef, I can now pass my file list to sam_meta_dumper.
314  print "Is MakeTransposeJson called without a txt def, if you see this then yes."
315  MetaListFile="AllMetaJson.txt"
316  meta_cmd="sam_metadata_dumper -s " + TransposeList + " > " + MetaListFile
317  os.system(meta_cmd)
318  # Now want to open the file and split by "}," character.
319  MetaFile = open( MetaListFile )
320  MetaLines =" },")
321  # Loop through lines, and appropriately separate out json files.
322  for i in range(0,len( MetaLines ) ):
323  meta=MetaLines[i]
324  # Figure out file name...this is assuming that file name is always the first entry...
325'"(.+?).root"', meta ).group(1)
326  filename=StName+".json" # Effecitvely replacing .root with .json
327  # If transpose file add PlaneNumber and CellNumber if run in cell mode
328  if "cell" in filename:
329  try:
330  plane ='^.*outplane(\d*).', filename).group(1)
331  cell ='^.*cell(\d*).', filename).group(1)
332  print "Plane number:", plane, "Cell number:", cell, ". Is a transpose file"
333  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.CellNumber": "%s",\n "calibration.base_release"')%(plane, cell)
334  except:
335  print "No cell number found - could be a plane mode transpose file"
337  elif "outplane" in filename:
338  try:
339  plane ='^.*outplane(\d*).', filename).group(1)
340  print "Plane number:", plane, ". Is a transpose file"
341  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.base_release"') %plane
342  except:
343  print "Error extracting plane number from transpose file."
345  ### Make sure that the txtfile is in the parent list.
346  meta['parents'].append({u'file_name':unicode(inFile)})
348  # Now open the json file
349  fout=open(filename,'w')
350  # Want to make sure that the json starts with '{'
351  if meta[0] not in "{":
352  meta = meta[:0] + '{\n' + meta[1:]
353  # Want to make sure that the json ends with a double '}'
354  if i < len(MetaLines)-1:
355  meta += "}\n}\n"
356  # Write and close the json file
357  fout.write(meta)
358  fout.close()
void split(double tt, double *fr)
def MakeTransposeJson(TransposeList)
procfile open("FD_BRL_v0.txt")
def runNovaSAM.resolveFclPath (   fcl)

Definition at line 188 of file

Referenced by makeDeCAF().

188 def resolveFclPath(fcl):
189  # Check if we have an absolute path name, return it if so
190  if fcl[0] == "/":
191  return fcl
193  # Otherwise, we need to do some searching
194  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
195  for path in fclPaths:
196  if os.path.isfile(path + fcl):
197  return path + fcl
199  # If we haven't found it, we have a problem.
200  raise IOError(sys.argv[0] + ": config file "+ fcl+" not found in FHICL_FILE_PATH")
void split(double tt, double *fr)
def resolveFclPath(fcl)
def runNovaSAM.setVMemLimit ( )

Definition at line 33 of file

34  #memLimit = 3.9*1024**3
35  memLimit = 3.9*1000**3
36  print "Old virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)
37  print "Limiting the virtual memory of the nova job to 3.9 GB"
38  resource.setrlimit(resource.RLIMIT_AS, (memLimit, memLimit))
39  print "New virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)
def setVMemLimit()
def runNovaSAM.unCamelCase (   s)
Convert CamelCase to camel_case 

Definition at line 29 of file

Referenced by createMetadata().

29 def unCamelCase(s):
30  """ Convert CamelCase to camel_case """
31  # \B matches an empty string which is NOT at the beginning of a word
32  # so requiring this means no _ will be inserted at the start
33  return re.sub( r'\B([A-Z])', r'_\1', s).lower()
def unCamelCase(s)

Variable Documentation

runNovaSAM._lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')

Definition at line 24 of file

runNovaSAM._skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)

Definition at line 27 of file


Definition at line 207 of file


Definition at line 802 of file

runNovaSAM.args = parser.parse_args()

Definition at line 215 of file


Definition at line 24 of file


Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help.

Definition at line 23 of file

runNovaSAM.cafLabel = cafTier.split(":")[0]

Definition at line 362 of file

Definition at line 366 of file


Definition at line 342 of file

string runNovaSAM.cmd = ' '

Definition at line 383 of file

list runNovaSAM.cmdList = ['nova']

Definition at line 330 of file


Definition at line 808 of file

string runNovaSAM.dataFlag = "data"

Definition at line 297 of file


If using a txtfiledef make sure to clean up the InputFile List....

Definition at line 862 of file


Definition at line 522 of file


Definition at line 473 of file


Definition at line 880 of file

runNovaSAM.detector = metadata["online.detector"].lower()

Definition at line 272 of file

Referenced by ApplyOscillations(), daq2raw::DAQHit2Raw.beginRun(), BestFit(), BuildMetricsTree(), cmf::DataVarVals.DataVars(), DCSPlotMaker(), cmf::MetaData.DetectorString(), novaddt::utils::DetectorUtils.DetectorUtils(), daqchannelmap::NearDetDAQChannelMap.encodeDChan(), daqchannelmap::FarDetDAQChannelMap.encodeDChan(), daqchannelmap::TestBeamDAQChannelMap.encodeDChan(), daqchannelmap::DAQChannelMapOLD.encodeDChan(), daqchannelmap::NearDetDAQChannelMap.encodeLChan(), daqchannelmap::FarDetDAQChannelMap.encodeLChan(), daqchannelmap::TestBeamDAQChannelMap.encodeLChan(), meta::Metadata.FillMetadata(), daqchannelmap::DAQChannelMapOLD.getBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getBlockType(), GetCosmics(), daqchannelmap::DAQChannelMapOLD.getDCMLinkOffline(), daqchannelmap::DAQChannelMapOLD.getDCMOffline(), daqchannelmap::DAQChannelMapOLD.getDiBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockTypeOffline(), daqchannelmap::DAQChannelMapOLD.getFirstDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLastDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getLocalPlaneInBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockViewOffline(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInXPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInYPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfXDCM(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfYDCM(), daqchannelmap::DAQChannelMapOLD.getOnlineCellOffline(), daqchannelmap::DAQChannelMapOLD.getPixelOffline(), daqchannelmap::DAQChannelMapOLD.getPlaneViewOffline(), GetSystsFromFile(), daqchannelmap::DAQChannelMapBaseOLD.getTotalNumberOfDiblocks(), GoodRuns(), daqchannelmap::DAQChannelMapBaseOLD.initNumberOfDCMsInDiblockDetector(), nova::dbi::RunHistory.LoadDAQRunHistory(), main(), make_fc_nus_surfs_nersc_2019(), MakeCovarSim(), MakeSurface(), MakeSurfaceJoint(), MakeSurfaceLLTest(), MakeSurfaceNoNDOsc(), NewNearlinePlots(), NuMu2019_BasicPIDPlots_FD(), NuMu2019_BasicPIDPlots_ND(), NuMu2019_BasicPIDPlots_Spectrum(), NuMu2020_BasicPIDPlots_FD(), NuMu2020_BasicPIDPlots_ND(), NuMu2020_BasicPIDPlots_Spectrum(), rawdata::RawBeamlineTrigger.RawBeamlineTrigger(), rawdata::RawBeamlineWC.RawBeamlineWC(), fnex::ReadCAFMakeEventList.ReadCAFTree(), om::HistoTable.ReadFile(), rawdata::RawBeamlineTrigger.SetDetectorID(), rawdata::RawBeamlineWC.SetDetectorID(), ana.TAHadEND(), and rawdata::RawBeamlineWC.~RawBeamlineWC().

runNovaSAM.detectors = dict()
runNovaSAM.detId = metadata["nova.detectorid"].lower()
runNovaSAM.dh = ifdh.ifdh("")
runNovaSAM.dirList = os.listdir(".")

Definition at line 406 of file


Definition at line 601 of file


Definition at line 546 of file


Definition at line 803 of file


Definition at line 522 of file

runNovaSAM.fclFile = args.config

Definition at line 315 of file

runNovaSAM.fclFileObj = open(tmpFclName, 'a')

Definition at line 327 of file

runNovaSAM.fclPath = resolveFclPath(fclFile)

Definition at line 319 of file

runNovaSAM.file_type = metadata["file_type"]

Definition at line 291 of file


Definition at line 563 of file


Definition at line 682 of file


Definition at line 735 of file


Definition at line 740 of file


Definition at line 633 of file


Definition at line 823 of file


Definition at line 750 of file


Definition at line 755 of file


Definition at line 635 of file

Definition at line 205 of file


Definition at line 774 of file


Definition at line 810 of file

runNovaSAM.inFile = args.inFile

Definition at line 217 of file

runNovaSAM.inFileBase = os.path.basename(inFile)

Definition at line 221 of file


Definition at line 584 of file


Definition at line 683 of file


Definition at line 834 of file

Referenced by main(), and nl::timing::LogFile.ProcessLog().

dictionary runNovaSAM.metadata = samweb.getMetadata(inFileBase)
runNovaSAM.nevt = None

Definition at line 311 of file


Definition at line 660 of file


Definition at line 697 of file


Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files?

Definition at line 800 of file

Referenced by numusand::NumuSandFxs.getScatt(), cosrej::CosRejFxs.getScatt(), and ndreco::NDRecoFxs.getTrackScatt().


Definition at line 655 of file

list runNovaSAM.outList = []

Definition at line 344 of file

Referenced by fnex::EventListManipulator.CreatePoissonFakeData().


Definition at line 680 of file


Definition at line 655 of file

runNovaSAM.output = outTier.split(":")[0]

Definition at line 348 of file


Definition at line 641 of file


Definition at line 340 of file


Definition at line 666 of file


Definition at line 656 of file


Definition at line 656 of file


Definition at line 667 of file


Definition at line 657 of file


Definition at line 657 of file

runNovaSAM.parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')

Definition at line 204 of file


Definition at line 567 of file


Definition at line 638 of file

runNovaSAM.release = os.environ["SRT_BASE_RELEASE"]

Definition at line 285 of file

Referenced by AnalyzeSysts(), BestFit(), bin_composition_pie_chart(), CalcRWithSystsNus17(), CompareBinningSchemes(), CompareMissingLeptons(), ConsolidateSystMaker(), containment_optimization_plots(), CopyPred(), CVNCuts(), demo_cut_optimization(), demo_trivial_xsec_analysis(), draw_michel_plots(), DrawCovMx(), DrawLLComparisonContours(), drawPlot(), drawSensitivity(), drawSystsShiftingNDdata(), drawSystsShiftingNDdata_updatedAna(), fake_future_data(), FCContour(), FCCorrectSlice(), fd_plot(), fill_col(), FillHists(), GeniePredictionToRoot(), get_univ_chisq(), GetBiases(), getContProf(), getContProf_Sensitivity(), getHists_FNEX(), GetMatrices(), GetMockData(), GetNueCosmics2017(), GetNueCosmics2018(), GetNueCosmics2019(), ana.GetNueCosmics2020(), GetNueCosmicsFuture(), GetNueData2017(), GetNueData2018(), GetNueData2019(), ana.GetNueData2020(), GetNuePrediction2017(), GetNuePrediction2018(), GetNuePrediction2019(), ana.GetNuePrediction2020(), GetNuePredictionFuture(), ana.GetNumuData2020(), GetNumuPredictionsFuture(), getSensitivity(), GetSurf(), GetSystsFromFile(), getTimePeakPlots(), LoadCovMx(), LoadData(), LoadExtrapPrediction(), ana::CountingExperiment.LoadFrom(), ana::TrivialBkgdEstimator.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::FDPredictionSterile.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::NDPredictionSterile.LoadFrom(), ana::SingleNucAnalysis.LoadFrom(), ana::FitInAnaBinsBkgdEstimator.LoadFrom(), ana::NumuCC2p2hAnalysis.LoadFrom(), ana::NumuCC2p2hBkgdEstimator.LoadFrom(), ana::nueccinc::NueCCIncMRECorrection.LoadFrom(), ana::MultiExperiment.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::PredictionCombinePeriods.LoadFrom(), ana::PredictionAddRock.LoadFrom(), ana::PredictionExtendToPeripheral.LoadFrom(), ana::PredictionScaleComp.LoadFrom(), ana::CutOptimization.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::CrossSectionAnalysis.LoadFrom(), ana::Multiverse.LoadFrom(), ana::SpectrumComponents.LoadFrom(), ana::NumuCCIncAnalysis.LoadFrom(), ana::nueccinc::NueCCIncCrossSectionAnalysis.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::SystMaker.LoadFrom(), ana::nueccinc::NueCCIncEnhancedSamples.LoadFrom(), ana::SystematicsMaker.LoadFrom(), ana::PredictionInterp.LoadFromBody(), LoadMaps(), ana.LoadMaps(), LoadPPFX(), LoadPrediction(), LoadSystsFromFile(), Make2DPlot(), make_extrap_figure(), make_extrap_figure_hists(), make_fc_nus_surfs_nersc_2019(), make_files_for_decomp_plots(), make_muonid_opt(), make_nominal_xs(), Make_NuMuCC_Inc_XS(), make_nus17_fc_surfs(), make_nus_fc_surfs(), make_starplots(), make_surfprof(), make_vertex_optimiz(), make_xs(), make_xs_1D(), makeEnergyEstimator(), makeFlatWeight(), MakeISysts(), makeMatrixElementSurface(), MakeNumuCovMx(), MakePeriCutPlot(), MakePlot(), MakePlots(), MakeSurface(), MakeSurfaceBinningStudy(), MakeSurfaceLLTest(), MakeSurfaceNoNDOsc(), MakeSystsPDF(), MakeTable(), mec_tuning_fitter_2020(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), mre_blessed(), mre_comp_split(), muonid_optimization_plots(), nue_decomp_scales_for_make_decomp(), nuebar_signif(), NumuExtrap(), PeripheralCuts(), plot_3NDvsFD(), plot_3NDvsFD_FHC(), plot_3NDvsFD_RHC(), plot_BEN(), plot_contprof(), plot_datamc_ND_numu(), plot_datamc_ND_numu_REW(), plot_datamcpred(), plot_datapredictions(), plot_joint_fit_2020_contours(), plot_nd_data_mc(), plot_ND_numu_NOMvsREW(), plot_NDvsFD_REW(), plot_NDvsFD_weights(), plot_NDvsFD_weights_FHC(), plot_NDvsFD_weights_RHC(), plot_nuCrystalBall(), plot_nue_filesyst_pred(), plot_nue_xsec_pred(), plot_nueFD_Signal_REWvsNOM_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_RHC(), plot_nueFD_Signal_REWvsNOM_RHC(), plot_nuTree(), plot_pi0_xcheck(), plot_pid(), plot_predictions(), plot_rationoosc(), plot_resolution(), plot_shifts(), plot_syst_contours(), plot_time(), plot_uncertainty(), plotContProf(), plotDataPred(), plotDataPred_SplitCanvas(), PlotNus17PredSystsData(), PlotNus18Sideband(), PlotNusSensAna01(), PlotPionPlots(), plots(), Plotting_DataAndPrediction(), pred_err(), PredictCosmic(), ana::PredictionExtrapSum.PredictionExtrapSum(), ana::PredictionSyst3Flavor2020.PredictionSyst3Flavor2020(), ana::PredictionSystJoint2018.PredictionSystJoint2018(), ana::PredictionSystJointDemo.PredictionSystJointDemo(), ana::PredictionSystNue2017.PredictionSystNue2017(), ana::PredictionSystNumu2017.PredictionSystNumu2017(), preselection_cutflow(), print_tables(), saveContours_addExpt(), saveContours_complete(), saveContours_oscpar(), saveContours_simple(), saveContours_systs(), selection_story_plots(), SetBestFitParams(), SetPrediction(), signal_count(), starPlot(), syst_plot_test(), syst_table_fit(), syst_table_fit_new(), test_nue2017Prediction(), test_nue2018_fitter(), test_prediction_interp(), TestPred(), ana::Bayesian1DMarginal.ToTH1(), Toy_analyses(), UnfoldInOut(), and xsec_extrap_plots().


Definition at line 812 of file

Referenced by ana::OscillatableSpectrum._Oscillated(), ana::PredictionExtrap._PredictComponent(), ana::PredictionExtrapSum._PredictComponent(), ana::PredictionExtendToPeripheral._PredictComponent(), ana::PredictionAddRock._PredictComponentSyst(), ana::PredictionInterp._PredictComponentSyst(), ana::CutOptimization.AbsUncertainty(), ana::CutOptimization.AbsUncertaintySquared(), ana::Seed.ActiveFitVars(), ana::SeedList.ActiveFitVars(), ana::SystShifts.ActiveSysts(), ana.AddErrorInQuadrature(), AddHistograms(), ana.AddToSummaryV(), ana::Hist.Adopt(), ana::Hist.AdoptSparse(), ana::Hist.AdoptStan(), rawfileparser::RawFileParser.AdvanceToConfigEnd(), rawfileparser::RawFileParser.AdvanceToConfigStart_file(), rawfileparser::RawFileParser.AdvanceToConfigStart_mem(), rawfileparser::RawFileParser.AdvanceToFirstEvent_file(), rawfileparser::RawFileParser.AdvanceToFirstEvent_mem(), rawfileparser::RawFileParser.AdvanceToNextEvent_file(), rawfileparser::RawFileParser.AdvanceToNextEvent_mem(), rawfileparser::RawFileParser.AdvanceToPreviousEvent_mem(), rawfileparser::RawFileParser.AdvanceToRunHeader_file(), rawfileparser::RawFileParser.AdvanceToRunHeader_mem(), rawfileparser::RawFileParser.AdvanceToRunTail_file(), rawfileparser::RawFileParser.AdvanceToRunTail_mem(), dt::Chain.AllChunks(), dt::Chain.AllHits(), ana.AnaTypeToString(), dt::Chain.AsChunkMap(), ana::TrivialBkgdEstimator.Background(), calib::BetheBlochTables.BetheBlochTables(), stan::services::optimize.bfgs(), rawfileparser::RawFileParser.BuildEventIndex_file(), rawfileparser::RawFileParser.BuildEventIndex_mem(), mcmc.BuildExperiments(), ana::StanFitter.BuildInitContext(), ana::GenericSystematicDef< SRType >.BuildSpectrumDown(), ana::GenericSystematicDef< SRType >.BuildSpectrumUp(), caldp::AttenProfilesMap.Channels(), ana::WildcardSource.CheckedWildcard(), ana::SolarConstraints.ChiSq(), ana::MultiExperiment.ChiSq(), rawfileparser::RawFileParser.close(), stan::math.columns_dot_product(), stan::math.columns_dot_self(), CombineHistograms(), ana::PredictionExtrapSum.ComponentCC(), ana::PredictionExtrapSum.ComponentNC(), ana::PredictionExtrapSum.ComponentNCAnti(), ana::PredictionExtrapSum.ComponentNCTotal(), ana::ThreadPool::TaskQueue.consume(), convert(), convert_mcnp_txt_to_root(), ConvertNormalBasis(), osc::OscCalcGeneral.Copy(), osc::_OscCalcPMNSOpt< T >.Copy(), osc::OscCalcPMNSOptEigen.Copy(), ana::Multiverse.CopyAndTransform(), gibuu::GiBUURegen.CopyGenieEvent(), ana.CopyUpDownSpectrum(), dt::ViewMerger.CountUnexplainedOnLine(), ana.DataMCComparison(), ana.DataMCComparisonAreaNormalized(), ana.DataMCComparisonComponents(), novarwgt.DecodeGeneratorVersion(), caf.DecodeGeneratorVersion(), DecorrelateFD(), DecorrelateND(), ana.DefaultOscCalc(), ana.DefaultOscCalcIH(), ana.DefaultSterileCalc(), stan::math.dot_product(), lem::Potential.Downsampled(), lem::MatchableEvent.Downsampled(), ana::CovMxSurface.DrawContours(), ana.DrawQuantLabel(), ana::Prod4LoadersBase.ECAFTypeFlatToTier(), ana::Prod5LoadersBase.ECAFTypeFlatToTier(), ana.EigenMatrixXdFromTMatrixD(), novarwgt.EncodeGeneratorVersion(), EnergyBin(), ErrorBand(), ErrorsToHist(), tf::HoughScore.Eval(), NovaMCEmptyEvent.eventTimestamp(), rb::Cluster.Exclude(), gibuu::GiBUURegen.ExpandLibraryPath(), ana::IFitter.ExpandSeeds(), mcmc.ExptPtrs(), ana.factorial(), ana::Spectrum.FakeData(), dt::ViewMerger.FakeThirdDimension(), FillPCAContainers(), caf.FillProngVars(), calib::ThroughgoingSelection.filter(), calib::RockMuonStopperSelection.filter(), calib::StopperSelection.filter(), novaddt::MoonShadow.filter(), EliminateBeamSpills.filter(), hough::LiteTH2.FindBinX(), hv::LiteTH2.FindBinX(), hough::LiteTH2.FindBinY(), hv::LiteTH2.FindBinY(), ana.FindCurveCrossings(), geo::GeometryBase.FindGDMLFile(), caf::CAFMaker.FindManyPStrict(), lem.FindMatches(), ana.FindPackageDir(), trk::KalmanTrackAna.FindVisibleProngs(), cheat::RecoCheckAna.FindVisibleProngs(), fuzz::FuzzyKValidate.FindVisibleProngs3D(), fuzz::FuzzyKVertex.FindVisibleProngs3D(), calib::AttenFit.FitQuality(), calib::AttenuationFit.FitQuality(), ana::PredictionInterp.FitRatios(), lem::MatchableEvent.Flipped(), ana::covmx::CovarianceMatrix.ForcePosDef(), ana::Progress.FormatTime(), nuesand::FillNueSandbox.FracModalHits(), ana::Hist.FromDirectory(), lem::dec::Forest.FromFile(), ana::SpectrumLoader.FromSAMProject(), lem::PIDExtraVars.FromString(), lem::PIDDetails.FromString(), ana::FCBin.FromTree(), lem::EventSummary.FromTree(), GenerateFutureData(), caf.GenieReweightTable(), ana2019::fakedata.Get2019Prediction(), ana.get3FlavorAna2020AllCPTSysts(), ana.get3FlavorAna2020AllSysts(), ana.get3FlavorAna2020LargeXsecSysts(), ana.getAllAna2017Systs(), ana.getAllAna2018Systs(), ana.getAllCPTSysts(), caldp::AttenHistsMap.GetAllHistsByPlane(), ana.getAllNumuProd4FDHistDef(), ana.getAllNumuProd4NDHistDef(), ana.getAllNumuProd5FDHistDef(), ana.getAllNumuProd5FDMultiHistDef(), ana.getAllNumuProd5FDNuTruthHistDef(), ana.getAllNumuProd5NDHistDef(), ana.getAllNumuProd5NDMultiHistDef(), ana.getAllNumuProd5NDNuTruthHistDef(), ana.getAllNusAna01FDHistDefs(), ana.getAllNusAna01NDHistDefs(), caldp::AttenProfilesMap.GetAllProfilesByPlaneAndCell(), GetAllSamples(), ana::CutOptimization.GetAllSystDefs(), GetAllSysts(), ana.getAllXsecNuTruthSysts_2017(), ana.getAna2017LargeXsecSysts(), ana.getAna2018LargeXsecSysts(), ana.getAna2018NueOnly(), ana.getAna2018NumuOnly(), ana.getAna2020NueOnly(), ana.getAna2020NumuOnly(), cosrej::CosRejFxs.getAngle(), numusand::NumuSandFxs.getAngle(), GetBaseSystNames(), GetBG(), ana.GetBG(), GetBiasErrorLine(), ana::covmx::CovarianceMatrix.GetBinnings(), ana.GetCAFMetadata(), ana::covmx.GetComponents(), rawfileparser::RawFileParser.getConfigBlock(), GetConstraints(), ana::CovMxSurface.GetContours(), GetCorrelations(), ana.GetCorrelations(), GetCosmics(), ana::covmx::CovarianceMatrix.GetCovMxRelative(), evd::GeometryDrawer.GetDCMBoxes(), calib::IBetheBloch.GetdEdxGraph(), calib::EnergyLossVsDistance.GetEnergyLossGraph(), rawfileparser::RawFileParser.getEvent(), gibuu::GiBUURegen.GetEvent(), ana::LikelihoodCovMxExperiment.GetExpectedSpectrum(), GetExtrapolationDefs(), GetFakeData(), ana.GetFDMCComponents(), rawfileparser::RawFileParser.getFirstEvent(), GetFNRatio(), GetGaussianProfile(), GetGaussianSurface(), geo::CellGeo.GetGeoMatrix(), ana::ISurface.GetGraphs(), GetHist(), GetJointFitSystematicList(), ana.GetJointFitSystematicList(), GetKeySystNames(), GetKeySysts(), GetMatrices(), GetMockData(), calib::IBetheBloch.GetMPVGraph(), ana::covmx::CovarianceMatrix.GetNBins(), GetNC(), ana.GetNC(), GetNCFDSamples(), GetNCNDSamples(), ana::covmx::CovarianceMatrix.GetNComponents(), GetNCSamples(), ana.GetNDComponents(), ana.GetNDCompsFromDecomp(), ana.GetNDDecompsFromDecomp(), ana.GetNDMCComponents(), rawfileparser::RawFileParser.getNextEvent(), ana.getNue2018PredFile(), ana.getNue2018PredVar(), GetNueNDComponents(), GetNueNDComponentsMC(), GetNuePredPath(), ana.getNumu2018PredFile(), ana.getNumu2018PredVar(), GetNumuFDSamples(), GetNumuNDSamples(), GetNumuSamples(), GetNus18Binning(), GetNus18Constraints(), GetNus18FitVars(), GetNus18SeedValues(), ana.getNusAna2020AllSysts(), ana.getNusAna2020LargeXsecSysts(), getPadHists(), osc::OscCalcSterileBeam.GetParamsHash(), osc::_NoOscillations< T >.GetParamsHash(), osc::OscCalcSterile.GetParamsHash(), osc::OscCalcPMNSOptEigen.GetParamsHash(), osc::OscCalcCPT.GetParamsHash(), osc::_IOscCalcAdjustable< T >.GetParamsHashDefault(), osc::OscCalcPMNS_CPT.GetParamsHashDefaultBar(), GetPlotOptions(), GetPOTCombination(), GetPOTComboTitle(), rawfileparser::RawFileParser.getPreviousEvent(), ana::T2KToyExperiment.GetReweightedSignal(), rawfileparser::RawFileParser.getRunHeader(), rawfileparser::RawFileParser.getRunTail(), GetSampleFromOptString(), GetSamplesFromOptString(), SystGroupHelper.GetShifts(), GetShifts2017(), ana.GetShifts2018(), ana.GetShifts2020(), ana.GetSpectrum(), GetSystConcats(), GetSystematics2017(), ana.GetSystematics2018(), ana.GetSystematics2020(), ana::covmx::Sample.GetSystShifts(), calib::EnergyLossVsDistance.GetTF1(), GetTotFNRatio(), nuesand::FillNueSandbox.GetTrackAngle(), ana::FitDelta13InPiUnitsSterile.GetValue(), ana::FitDelta14InPiUnitsSterile.GetValue(), ana::FitDelta24InPiUnitsSterile.GetValue(), lem::LEMWorker.GetWork(), rawfileparser::RawFileParser.GotoEvent_file(), HistNumuOpt(), dt::Chunk.HitsOnLine(), cheat::BackTracker.HitsToParticle(), cheat::BackTracker.HitToFLSHit(), tf::TimingFitAlg.HoughFitPts(), ana::Binning.IDMap(), ImportDataGraph(), InitCutDefs(), InitHistDefs(), stan::optimization::BFGSMinimizer< ModelAdaptor< M >, QNUpdateType, Scalar, DimAtCompile >.initialize(), InitSpectra(), geo::LiveGeometry.InMuonCatcher(), InputPath(), util.ipow(), ana.JoinGraphs(), JointGraphs(), ana.kMinPngNhit(), ana::SystShifts.LatexName(), stan::services::optimize.lbfgs(), load(), load_libs_muonid(), load_libs_nuebarccinc(), load_libs_nueccinc(), load_libs_numucc2p2h(), load_libs_numucc_0Pi(), load_libs_numuccinc(), LoadCosmic(), mcmc.LoadFakeData(), LoadFakeData(), ana.LoadFileList(), ana::PredictionXSecTuning.LoadFrom(), ana::CountingExperiment.LoadFrom(), ana::SolarConstraints.LoadFrom(), ana::PredictionNoOsc.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::CheatDecomp.LoadFrom(), ana::ModularExtrap.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::PredictionExtrap.LoadFrom(), ana::Binning.LoadFrom(), ana::CovMxManager.LoadFrom(), ana::NCDecomp.LoadFrom(), ana::NDExtrapBeamComponent.LoadFrom(), ana::CovMxSurface.LoadFrom(), ana::NueDecomp.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::NDExtrapComponent.LoadFrom(), ana::NumuDecomp.LoadFrom(), ana::PredictionSystJoint2018.LoadFrom(), ana::PredictionSystJointDemo.LoadFrom(), ana::NDOscillatableSpectrum.LoadFrom(), ana::PredictionNoExtrap.LoadFrom(), ana::FDExtrap.LoadFrom(), ana::NDExtrap.LoadFrom(), ana::ProportionalDecomp.LoadFrom(), ana::FakeDecomp.LoadFrom(), ana::TrivialExtrap.LoadFrom(), ana::ReweightableSpectrum.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::Multiverse.LoadFrom(), ana::PredictionInterp.LoadFrom(), ana::TwoSampleDecomp.LoadFrom(), ana::SystShifts.LoadFrom(), ana::SpectrumComponents.LoadFrom(), ana::covmx::CovarianceMatrix.LoadFrom(), ana::OscillatableSpectrum.LoadFrom(), ana::FluxDecomp.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::BENDecomp.LoadFrom(), ana::PredictionSystNue2017.LoadFrom(), ana::PredictionSystNumu2017.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::Spectrum.LoadFrom(), ana.LoadFrom< osc::IOscCalcAdjustable >(), ana::FCCollection.LoadFromFile(), ana::FCSurface.LoadFromFile(), genie::GSimFiles.LoadFromFile(), genie::XSecSplineList.LoadFromXml(), LoadPrediction(), ana.LoadPredictions(), ana.LoadRealData(), ana::SAMQuerySource.LocateSAMFiles(), ana::MultiExperiment.LogLikelihood(), ana::GaussianPriorSystShifts.LogPrior(), calib::AttenFit.lowessFit(), calib::AttenuationFit.lowessFit(), ana::GradientDescent.Magnitude(), MakeAverageTrueEnergySurface(), dt::View.MakeChunkCombos(), ana.MakeCosOutOfTimeList(), MakeCutFlowSpectra(), MakeDataReleaseFileContour(), MakeDataReleaseFileSlice(), MakeKin2DSpectra(), MakeKinSpectra(), MakeName(), ana.MakeNumiUnblindList(), MakeRatio(), MakeRatios(), MakeSample(), MakeSpectra(), MakeTitle(), ana.MakeUnblindList(), MakeUnblindList(), hough::MultiHough2P.MapToTH2F(), dt::ViewMerger.MatchVerticals(), osc.MatrixExp(), caldp::TCTrack.MaxCell(), rb::Cluster.MaxCell(), nuesand::FillNueSandbox.MaxHits(), caldp::TCTrack.MaxPath(), caldp::TCTrack.MaxPlane(), rb::Cluster.MaxPlane(), caldp::TCTrack.MaxTNS(), rb::Cluster.MaxTNS(), lem::Match.MaybeFlip(), dt::Cand.MaybeFlip(), cheat::BackTracker.MCTruthToParticles(), calib::AttenFit.MeanProfile(), calib::AttenuationFit.MeanProfile(), ana::ResolutionScan.MeanScan(), rb::Cluster.MeanTNS(), calib::AttenFit.MedianProfile(), calib::AttenuationFit.MedianProfile(), caldp::TCTrack.MinCell(), rb::Cluster.MinCell(), caldp::TCTrack.MinPath(), caldp::TCTrack.MinPlane(), rb::Cluster.MinPlane(), caldp::TCTrack.MinTNS(), rb::Cluster.MinTNS(), lem.MMapFileAtAddress(), ana::Spectrum.MockData(), MostConservative(), MyDefaultOscCalc(), dt::Cand.NHitPlanes(), dt::Chain.NHitPlanes(), ana::RecordMultiSink.NSinks(), rb::Cluster.OfflineChans(), ana::Ratio.operator*(), ana::Multiverse.operator*(), ana::Spectrum.operator*(), ana::ReweightableSpectrum.operator+(), ana::OscillatableSpectrum.operator+(), ana::Spectrum.operator+(), ana::ReweightableSpectrum.operator-(), ana::OscillatableSpectrum.operator-(), ana::Spectrum.operator-(), ana::Ratio.operator/(), ana::Multiverse.operator/(), ana::Spectrum.operator/(), osc::_IOscCalc< T >.P(), lem::FindMatchesAlg.PackageMatches(), PackMatrix(), PaintReachCanvas(), parse_as_vector(), ParseOption(), ParseOptions(), ana::SystShifts.Penalty(), PIDFileTag(), PIDLongName(), plot_ratios(), ana::NumuCCIncAnalysis.PlotHelper(), PlotMCComponentsErrorBand(), dt::DiscreteTracker.PossibleNextSegs(), ana::FDPredictionSterile.Predict(), ana::NDPredictionSterile.Predict(), ana::PredictionScaleComp.Predict(), ana::covmx::CovarianceMatrix.Predict(), ana::PredictionSterile.PredictComponent(), ana::FDPredictionSterile.PredictComponent(), ana::NDPredictionSterile.PredictComponent(), ana::PredictionNoOsc.PredictComponent(), PredictCosmic(), ana::PredictionInterp.PredictionInterp(), Print(), ana.Profile(), ProfileAxes(), calib::BetheBlochFit.ProfileMPV(), lem::dec::Cut.Prune(), caf.PtrVecToVec(), cheat::BackTracker.PtrVecToVecRawPtr(), stan::math.quad_form_sym(), ana::MCMCSamples.QuantileLL(), lem::dec::Forest.RandomOrthoMatrix(), RebinToShortAxis(), dt::DiscreteTracker.RecoDiscrete(), ana.RefineSeeds(), RemoveBins(), ana.RemoveBins(), ana.replaceAll(), ana::ResolutionScan.RMSScan(), ana::PredictionAddRock.RockComponent(), stan::math.rows_dot_product(), stan::math.rows_dot_self(), ana::T2KToyExperiment.SetPOTAnti(), ana::FitSinSq2Theta13.SetValue(), ana::SummedSyst.Shift(), ShiftedCosmics(), ana::SystShifts.ShortName(), ana::Registry< T >.ShortNameToPtr(), murem::MuonRemove.SliceSelection(), ana.sort_chisq_tot(), SortSystHists(), SplitHistograms(), ana.SqrtProfile(), ana.SqrtSlice(), stan::math::LDLT_factor< var, R, C >.success(), mcmc.SupportedSysts(), ana.systsQ3Q0(), TEST(), slid::DedxDistribution.TH1DToTH1F(), TH1ToTH2(), calib::ThresholdCorrMap.ThresholdCorrAt(), ana.TMatrixDFromEigenMatrixXd(), stan::math.to_fvar(), dt::Chain.ToDebugTracks(), ana.ToEigen(), ana::CutOptimization.ToHist(), ana::BayesianMarginal.ToHistogram(), ana::Multiverse.ToSpectrum(), lem::PIDExtraVars.ToString(), ValidCalibPDF.ToString(), rb::Cluster.TotalADC(), caldp::LiteProfile.TotalEntries(), rb::Cluster.TotalGeV(), caldp::LiteProfile.TotalHits(), rb::Prong.TotalLength(), rb::Cluster.TotalPE(), rb::Cluster.TotalWeight(), ana::NDOscCurve.ToTH1(), ana::OscCurve.ToTH1(), ana::Ratio.ToTH1(), ana::Hist.ToTH1(), ana::Spectrum.ToTH1(), ana::Ratio.ToTH2(), ana::ReweightableSpectrum.ToTH2(), ana::ISurface.ToTH2(), ana::Spectrum.ToTH2(), hough::LiteTH2.ToTH2F(), hv::LiteTH2.ToTH2F(), ana.ToTH2Helper(), ana::Spectrum.ToTH3(), ana.ToTH3Helper(), ana.ToTHStack(), ana::MCMCSample.ToTMap(), dt::Cand.ToTrack(), ana::CutOptimization.ToUpDownHist(), murem::TrackCleanUpAlg.TrackEinMIP(), lem::dec::Forest.Train(), lem::dec.TransformEvent(), calib::AttenCache.TranslateChannelMC(), ncid::MakeNCCosRej.TransMomFraction(), ncid::NCNNKeras.TransMomFraction(), cosrej::MakeNueCosRej.TransMomFraction(), calib::AttenFit.TruncatedMeanProfile(), calib::AttenuationFit.TruncatedMeanProfile(), ana::UnfoldSVD.Truth(), ana::UnfoldTikhonov.Truth(), UnpackMatrix(), cheat::BackTracker.VecPtrToVecRawPtr(), cheat::BackTracker.VecToVecRawPtr(), ana::Nus17SystFromHist.WeightFor(), ana::NusSystFromHist.WeightFor(), ana::NuISyst.WeightFor(), ana::Nus18SystFromHist.WeightFor(), ana::SpectrumLoaderBase.WildcardOrSAMQuery(), stan::optimization.WolfeLineSearch(), ana::Hist.Zero(), ana::Hist.ZeroSparse(), dt::ViewMerger.ZipVerticalTracks(), rb::Track.ZipWith(), ana::ProfilerSupport.~ProfilerSupport(), and calib::ThresholdCorrMap.~ThresholdCorrMap().

runNovaSAM.retCode =
runNovaSAM.run_match = run_pattern.match(inFileBase)

Definition at line 238 of file

runNovaSAM.run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")

Definition at line 237 of file

runNovaSAM.runNum = int(metadata["runs"][0][0])

Definition at line 234 of file

Referenced by calib::Calibrator.postBeginRun().

runNovaSAM.samweb = samweb_client.SAMWebClient(experiment='nova')

Definition at line 219 of file

Definition at line 594 of file

runNovaSAM.skip_match = _skip_pattern.match(file)

Definition at line 408 of file


Definition at line 834 of file


Definition at line 834 of file


Definition at line 207 of file

Referenced by createMetadata(), and getOutDir(). = int(metadata["data_stream"])

Definition at line 251 of file

Referenced by genie::Algorithm.AllowReconfig(), genie::RSHelicityAmpl.Amp20Plus(), genie::ProcessInfo.AsString(), ClassImp(), genie::geometry::FidPolyhedron.clear(), nova::database::Row.Col(), genie::GVldContext.Emax(), genie::RunOpt.EnableBareXSecPreCalc(), novaddt::HitSubtract.endJob(), Expression.Eval(), genie::geometry::FidCylinder.FidCylinder(), genie::geometry::FidSphere.FidSphere(), genie::Spline.GetAsTSpline(), nutools::dbi::Table.GetConnectionTimeout(), rvp::RecVarPID.getMethodNameFromXML(), xnue::XnuePID.getMethodNameFromXML(), rvp::RecVarPID.getVarNamesFromXML(), xnue::XnuePID.getVarNamesFromXML(), genie::PDF.Gluon(), genie::Kinematics.HadSystP4(), genie::Target.HitNucP4(), genie::geometry::PathSegment.IsTrimmedEmpty(), genie::geometry::PlaneParam.IsValid(), genie::AlgId.Key(), genie::AxialFormFactor.Model(), genie::ELFormFactors.Model(), genie::QELFormFactors.Model(), genie::DISStructureFunc.Model(), genie::CacheBranchNtp.Ntuple(), genie::TuneId.OnlyConfiguration(), genie::geometry.operator<(), genie::utils.operator<<(), genie::exceptions.operator<<(), genie::geometry.operator<<(), operator<<(), genie.operator<<(), nova::database.operator<<(), nutools::dbi.operator<<(), nova::dbi.operator<<(), genie::flux.operator<<(), genie::Interaction.PhaseSpacePtr(), genie::exceptions::EVGThreadException.ReturnStep(), genie::GHepParticle.SetLastDaughter(), genie::RegistryItem< T >.SetLocal(), genie::XclsTag.SetNNeutrons(), nutools::dbi::Row.SetVldTimeEnd(), nova::dbi::Row.SetVldTimeEnd(), genie::exceptions::INukeException.ShowReason(), genie::exceptions::NievesQELException.ShowReason(), genie::CacheBranchFx.Spl(), genie::InitialState.TgtPtr(), nova::database::Column.Update(), nutools::dbi::Column.Update(), nova::dbi::Column.Update(), genie::GHepRecord.Vertex(), genie::GEVGDriver.XSecSumSpline(), genie::geometry::FidShape.~FidShape(), genie::flux::GJPARCNuFluxPassThroughInfo.~GJPARCNuFluxPassThroughInfo(), genie::flux::GNuMIFluxPassThroughInfo.~GNuMIFluxPassThroughInfo(), genie::flux::GSimpleNtpAux.~GSimpleNtpAux(), genie::flux::GSimpleNtpEntry.~GSimpleNtpEntry(), genie::flux::GSimpleNtpNuMI.~GSimpleNtpNuMI(), genie::exceptions::InteractionException.~InteractionException(), and genie::utils::T2KEvGenMetaData.~T2KEvGenMetaData().

runNovaSAM.stream_match = stream_pattern.match(inFileBase)

Definition at line 263 of file

runNovaSAM.stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)

Definition at line 262 of file

string runNovaSAM.streamEntry = 't{0:02d}'

Definition at line 252 of file

runNovaSAM.subNum = int(metadata["runs"][0][1])

Definition at line 235 of file

int runNovaSAM.subversion = os.environ["NPASS"]

Definition at line 305 of file

runNovaSAM.tier = outTier.split(":")[1]
runNovaSAM.tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")

Definition at line 322 of file


Definition at line 205 of file