Functions | Variables
runNovaSAM Namespace Reference

Functions

def unCamelCase (s)
 
def createMetadata (inFile)
 
def getOutDir (pathname, hashDirs=False)
 
def checkAndMoveFiles (inFile, declareFiles)
 
def declareFile (fileWPath)
 
def copyOutFiles (hashDirs=False)
 
def getOutputFileName (detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
def resolveFclPath (fcl)
 
def make_temp_fcl (fclFile, inFileBase)
 
def setVMemLimit ()
 
def makeDirSafely (dir)
 
def makeDirIfNeeded (dir)
 
def getOutDir (pathname, dest, hashDirs=False, runDirs=False, runNum=0)
 
def checkAndMoveFiles (inFile, outputs, noCleanup=False, copyOnly=False)
 
def ModifyParentMetadata (inParentFile, inFile)
 
def declareLogFile (logName, logFile, rootName)
 
def fileExists (outPath, dh)
 
def listFiles (outPath, dh)
 
def makeMetadataJSONs ()
 
def MakeTransposeJson (TransposeList)
 
def copyOutFiles (dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False)
 
def makeDeCAF (script, fname, special)
 

Variables

 gErrorIgnoreLevel
 
 detectors = dict()
 
 _lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
 
 _skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
 
 parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')
 
 help
 
 type
 
 str
 
 action
 
 args = parser.parse_args()
 
 inFile = args.inFile
 
 samweb = samweb_client.SAMWebClient(experiment='nova')
 
 inFileBase = os.path.basename(inFile)
 
 metadata = samweb.getMetadata(inFileBase)
 
 runNum = int(metadata["runs"][0][0])
 
 subNum = int(metadata["runs"][0][1])
 
 run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")
 
 run_match = run_pattern.match(inFileBase)
 
 stream = int(metadata["data_stream"])
 
string streamEntry = 't{0:02d}'
 
 stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)
 
 stream_match = stream_pattern.match(inFileBase)
 
 detector = metadata["online.detector"].lower()
 
 detId = metadata["nova.detectorid"].lower()
 
 release = os.environ["SRT_BASE_RELEASE"]
 
 file_type = metadata["file_type"]
 
string dataFlag = "data"
 
 subversion = os.environ["NPASS"]
 
 nevt = None
 
 fclFile = args.config
 
 fclPath = resolveFclPath(fclFile)
 
 tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
 
 fclFileObj = open(tmpFclName, 'a')
 
list cmdList = ['nova']
 
 outTier
 
 cafTier
 
list outList = []
 
 output = outTier.split(":")[0]
 
 tier = outTier.split(":")[1]
 
 outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
 cafLabel = cafTier.split(":")[0]
 
 cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
string cmd = ' '
 
 retCode = subprocess.call(cmdList)
 
 dirList = os.listdir(".")
 
 skip_match = _skip_pattern.match(file)
 
 dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
 
 argvCopy
 Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help. More...
 
 argv
 
 description
 
 default
 
 False
 
 experiment
 
 fileMetaDataMgr
 
 PassFile
 
 doMeta
 
 flatTier
 
 h5Tier
 
 precopyscript
 
 outputs
 
 outP
 
 outC
 
 outXp
 
 outXc
 
 outYp
 
 outYc
 
 NewOMod
 
 outX
 
 outY
 
 outNameTemp
 
 FirstRun
 
 LastRun
 
 Index
 
 NewOName
 
 flatLabel
 
 flatName
 
 h5Label
 
 h5Name
 
 histName
 
 olddir
 Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files? More...
 
 allFiles
 
 failFiles
 
 copyfile
 
 ifdhcmd
 
 ret
 
 gdbArgs
 
 stdout
 
 logfile
 
 stderr
 
 line
 
 end
 
 dest
 If using a txtfiledef make sure to clean up the InputFile List.... More...
 
 noCleanup
 
 True
 
 copyOnly
 
 lastdir
 
 tmpdir
 
 tmpSecondFclName
 
 inPID
 
 _first
 
 _second
 
 _outputs
 
 _bad
 
 _good
 
 decafdir
 

Function Documentation

def runNovaSAM.checkAndMoveFiles (   inFile,
  declareFiles 
)
Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 105 of file runNovaSAM.py.

References declareFile().

Referenced by checkAndMoveFiles().

105 def checkAndMoveFiles(inFile, declareFiles):
106  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
107  inFileBase = os.path.basename(inFile)
108  baseDir = "."
109  for root, dirs, filenames in os.walk(baseDir):
110  if root == baseDir:
111  for file in filenames:
112  if file.endswith (".root") and file != inFileBase:
113  fileWPath = os.path.join(root, file)
114  rootFile = ROOT.TFile(fileWPath)
115  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
116  #do not copy over, just delete
117  print "File", fileWPath, "is Zombie - remove it"
118  os.remove(fileWPath)
119  else:
120  if declareFiles == True:
121  declareFile(fileWPath)
122  newFilePath = os.path.join(root, "results", file)
123  os.renames(fileWPath, newFilePath)
124  rootFile.Close()
125  return
126 
def checkAndMoveFiles(inFile, declareFiles)
Definition: runNovaSAM.py:105
def declareFile(fileWPath)
Definition: runNovaSAM.py:127
def runNovaSAM.checkAndMoveFiles (   inFile,
  outputs,
  noCleanup = False,
  copyOnly = False 
)

Definition at line 130 of file runNovaSAM.py.

References checkAndMoveFiles(), makeDirIfNeeded(), and print.

130 def checkAndMoveFiles(inFile, outputs, noCleanup=False, copyOnly=False):
131  makeDirIfNeeded('./copyback')
132  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the copyback subdirectory for copy out ease"""
133  inFileBase = os.path.basename(inFile)
134  baseDir = "."
135 
136  print('Looking for requested outputs: ')
137  for o in outputs: print(o)
138 
139  # If declaring files have to do non-decafs first, since they're the parents
140  # of the decafs. If not, it doesn't hurt to do it in that order anyway.
141  for secondPass in [False, True]:
142  for root, dirs, filenames in os.walk(baseDir):
143  if root == baseDir:
144  for file in filenames:
145  if file.endswith('decaf.root') != secondPass: continue
146  # If file not in outputs, then delete.
147  if (file.endswith (".root") or file.endswith(".h5")) and file != inFileBase:
148  fileWPath = os.path.join(root, file)
149  # If the file isn't in my outputs then delete it.
150  if file not in outputs:
151  if not noCleanup:
152  # Do not copy over, just delete
153  print("File", fileWPath, " is not among requested outputs, removing")
154  os.remove(fileWPath)
155  continue
156  # Now that I know that this file is in my output list, lets check it is valid...
157  # First, if it is a root file.
158  if file.endswith (".root"):
159  print("In checkAndMoveFiles, fileWPath is %s" %fileWPath)
160  rootFile = ROOT.TFile(fileWPath)
161  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
162  #do not copy over, just delete
163  print("File", fileWPath, "is Zombie - remove it")
164  os.remove(fileWPath)
165  else:
166  newFilePath = os.path.join(root, "copyback", file)
167  print("New file name is %s" %newFilePath)
168  if copyOnly:
169  shutil.copyfile(fileWPath, newFilePath)
170  else:
171  shutil.move(fileWPath, newFilePath)
172  rootFile.Close()
173  # Next, if it is a h5 file.
174  elif file.endswith(".h5"):
175  # Only import h5py if have h5 files.
176  import h5py
177  print("In checkAndMoveFiles, fileWPath is %s" %fileWPath)
178  # If a valid HDF5 files.
179  if h5py.is_hdf5(fileWPath):
180  newFilePath = os.path.join(root, "copyback", file)
181  print("New file name is %s" %newFilePath)
182  if copyOnly:
183  shutil.copyfile(fileWPath, newFilePath)
184  else:
185  shutil.move(fileWPath, newFilePath)
186 
187  # If not a valid HDF5 file.
188  else:
189  print("File", fileWPath, "is Zombie - remove it")
190  os.remove(fileWPath)
191  return
192 
def checkAndMoveFiles(inFile, declareFiles)
Definition: runNovaSAM.py:105
bool print
def makeDirIfNeeded(dir)
Definition: runNovaSAM.py:103
def runNovaSAM.copyOutFiles (   hashDirs = False)
Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 149 of file runNovaSAM.py.

References getOutDir().

Referenced by copyOutFiles().

149 def copyOutFiles(hashDirs=False):
150  """Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
151  print "Copying out files"
152  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
153  baseDir = "./results"
154  for root, dirs, filenames in os.walk(baseDir):
155  if root == baseDir:
156  for file in filenames:
157  if file.endswith (".root") and file != inFileBase:
158  fileWPath = os.path.join(root, file)
159  dest = getOutDir(file, hashDirs)
160  skip_match = _skip_pattern.match(file)
161  if skip_match == None:
162  try:
163  print "Checking if", dest + "/" + file, "exists"
164  dh.ls(dest + "/" + file, 1, "")
165  except RuntimeError:
166  print "It doesn't - copy", fileWPath, "to", dest
167  dh.cp(["-D", fileWPath, dest])
168  print "Removing", fileWPath
169  os.remove(fileWPath)
170  dh.cleanup()
171  return
172 
def copyOutFiles(hashDirs=False)
Definition: runNovaSAM.py:149
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM.py:92
def runNovaSAM.copyOutFiles (   dest,
  hashDirs = False,
  runDirs = False,
  runNum = 0,
  noCleanup = False,
  declareLocation = False,
  declareLogs = False 
)
Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 457 of file runNovaSAM.py.

References copyOutFiles(), declareFile(), declareLogFile(), fileExists(), getOutDir(), and print.

457 def copyOutFiles(dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False):
458  """Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
459  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
460  baseDir = "./copyback"
461  declareFiles = declareLogs
462  for root, dirs, filenames in os.walk(baseDir):
463  if root == baseDir:
464 
465  # copy out root files before h5 files
466  ordered_files = [s for s in filenames if ".root" in s]
467  for s in filenames:
468  if ".h5" in s:
469  ordered_files.append(s)
470 
471  for file in ordered_files:
472  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
473  fileWPath = os.path.join(root, file)
474  outDir = getOutDir(file, dest, hashDirs, runDirs, runNum)
475 
476  skip_match = _skip_pattern.match(file)
477  if skip_match == None:
478  outPath = os.path.join(outDir, file)
479 
480  if fileExists(outPath, dh):
481  print('copyOutFiles: ', outPath, 'already moved. Skipping')
482  else:
483  # note: this will fail if the file already exists
484  returnValue = dh.cp(["-D", fileWPath, outDir])
485  if returnValue != 0:
486  print("Copy out failed for file:", fileWPath, file=sys.stderr)
487  print("Skipping it.", file=sys.stderr)
488  else:
489  if declareFiles:
490  declareFile(fileWPath)
491  ###################
492  # Declare the file's location to SAM if we have the declareLocation option on
493  if declareLocation==True :
494  loc = string.replace(outDir, 's3://','s3:/')
495  print("Declaring location %s for file %s\n" % (loc,file))
496  sam = samweb_client.SAMWebClient('nova')
497  ret=sam.addFileLocation(file, loc)
498  if ret.status_code != 200 :
499  print(" SAMWEB Unable to declare file location (%s, %s) status code %s" %(file, loc, ret.status_code))
500  if fileWPath.endswith (".root"):
501  jsonPath = fileWPath.replace('.root', '.json')
502  elif fileWPath.endswith (".h5"):
503  jsonPath = fileWPath[:-3] + '.json'
504  if os.path.isfile(jsonPath):
505  if fileExists(os.path.join(outDir, os.path.basename(jsonPath)), dh):
506  print('copyOutFiles: ', os.path.join(outDir, os.path.basename(jsonPath)), 'already moved. Skipping')
507  else:
508  returnValue = dh.cp(['-D', jsonPath, outDir])
509  if returnValue != 0:
510  print("Copy out failed for file: " + jsonPath, file=sys.stderr)
511  print("Skipping it.", file=sys.stderr)
512  else:
513  print('JSON not found %s' % jsonPath)
514 
515  for ext in ['.bz2', '']:
516  if os.path.isfile('log.txt'+ext):
517  if file.endswith (".root"):
518  logName = file.replace('.root', '.log'+ext)
519  elif file.endswith (".h5"):
520  logName = file + '.log'+ext
521  if fileExists(os.path.join(outDir,logName), dh):
522  print('copyOutFiles: ', os.path.join(outDir, logName), 'already moved. Skipping')
523  else:
524  returnValue = dh.cp(['log.txt'+ext, os.path.join(outDir, logName)])
525 
526  if returnValue != 0:
527  print("Copy out failed for file: " + logName, file=sys.stderr)
528  print("Skipping it.", file=sys.stderr)
529 
530  if declareLogs:
531  declareLogFile(logName, 'log.txt'+ext, file)
532 
533  # Remove the copied-out log so it's not in
534  # the way for new log creation.
535  os.remove('log.txt'+ext)
536 
537  break
538 
539  else:
540  print("It does exist, not copying.")
541  if not noCleanup:
542  print("Removing", fileWPath)
543  os.remove(fileWPath)
544 
545  if fileWPath.endswith(".root"):
546  jsonPath = fileWPath.replace('.root', '.json')
547  elif fileWPath.endswith(".h5"):
548  jsonPath = fileWPath + ".json"
549  if os.path.isfile(jsonPath):
550  print('Removing', jsonPath)
551  os.remove(jsonPath)
552  return
553 
def copyOutFiles(hashDirs=False)
Definition: runNovaSAM.py:149
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM.py:92
bool print
def fileExists(outPath, dh)
Definition: runNovaSAM.py:308
def declareFile(fileWPath)
Definition: runNovaSAM.py:127
def declareLogFile(logName, logFile, rootName)
Definition: runNovaSAM.py:289
def runNovaSAM.createMetadata (   inFile)

Definition at line 36 of file runNovaSAM.py.

References samweb_client.utility.fileEnstoreChecksum(), site_stats_from_log.get, str, and unCamelCase().

Referenced by declareFile().

36 def createMetadata(inFile):
37  filename = os.path.basename(inFile)
38  filesize = os.path.getsize(inFile)
39 
40  try:
41  data = subprocess.check_output(["sam_metadata_dumper", str(inFile)])
42  except subprocess.CalledProcessError:
43  print "sam_metadata_dumper failed!"
44  return None
45 
46  md = {'file_name': filename, 'group': 'nova', 'file_size': filesize, 'file_format': 'root'}
47 
48  # An ad-hoc parser for the output
49  mode = 'new'
50  for line in data.split('\n'):
51  m = _lineregex.match(line)
52  if mode == 'json':
53  if not m and not line.startswith('-'):
54  data += line
55  else:
56  md[key] = json.loads(data)
57  mode = 'new'
58 
59  if mode == 'new':
60  if m:
61  key = m.group(1)
62  value = m.group(2)
63 
64  if '.' not in key:
65  key = unCamelCase(key)
66 
67  # ART does not appear to set the application name
68  # but does provide a process name
69  if key == 'process_name' and md.get('application',[]).get('name') is None:
70  key = 'application_name'
71  elif key == 'stream_name':
72  key = 'data_stream'
73 
74  if key in ('start_date','end_date'):
75  value = long(value)
76 
77  if key == 'runs' or key == 'parents':
78  mode = 'json'
79  data = value
80  elif key.startswith('application_'):
81  # special handling for applications
82  if 'application' not in md:
83  md['application'] = {}
84  md['application'][key[12:]] = value
85  else:
86  md[key] = value
87 
88  md['crc'] = fileEnstoreChecksum(inFile)
89 
90  return md
91 
def unCamelCase(s)
Definition: runNovaSAM.py:29
def fileEnstoreChecksum(path)
Definition: utility.py:50
def createMetadata(inFile)
Definition: runNovaSAM.py:36
def runNovaSAM.declareFile (   fileWPath)
Checks file for a TKey of RootFileDB.  If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM

Definition at line 127 of file runNovaSAM.py.

References createMetadata().

Referenced by checkAndMoveFiles(), copyOutFiles(), and ModifyParentMetadata().

127 def declareFile(fileWPath):
128  """Checks file for a TKey of RootFileDB. If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM"""
129  samweb = samweb_client.SAMWebClient(experiment='nova')
130  rootFile = ROOT.TFile(fileWPath)
131  if rootFile.FindKey("RootFileDB"):
132  print "Found key (RootFileDB): constructing metadata for", fileWPath
133  md = createMetadata(fileWPath)
134  if (md == None):
135  print "No metadata found!"
136  else:
137  pprint.pprint(md)
138  print ''
139  print "Declaring", fileWPath, "to SAM"
140  try:
141  samweb.declareFile(md)
142  except:
143  print fileWPath, "already exists in SAM"
144  else:
145  print fileWPath, "does not contain RootFileDB, do not try to declare"
146  rootFile.Close()
147  return
148 
def createMetadata(inFile)
Definition: runNovaSAM.py:36
def declareFile(fileWPath)
Definition: runNovaSAM.py:127
def runNovaSAM.declareLogFile (   logName,
  logFile,
  rootName 
)

Definition at line 289 of file runNovaSAM.py.

References print.

Referenced by copyOutFiles().

289 def declareLogFile(logName, logFile, rootName):
290  # This is what NovaFTS/plugins/nova_log_metadata.py does
291  md = {
292  'file_name': os.path.basename(logName),
293  'file_size': os.path.getsize(logFile),
294  'data_tier': 'log',
295  'file_format': 'log',
296  'file_type': 'unknown', # maybe 'nonPhysicsGeneric'?
297  'parents': [{'file_name': os.path.basename(rootName)}]
298  }
299  print("16) declareLogFile(logName, logFile, rootName):")
300  print("Declaring", logName, "to SAM")
301  try:
302  samweb = samweb_client.SAMWebClient(experiment='nova')
303  samweb.declareFile(md)
304  except Exception as inst:
305  print(inst)
306 
307 
bool print
def declareLogFile(logName, logFile, rootName)
Definition: runNovaSAM.py:289
def runNovaSAM.fileExists (   outPath,
  dh 
)

Definition at line 308 of file runNovaSAM.py.

Referenced by copyOutFiles().

308 def fileExists(outPath, dh):
309  try:
310  # Check to see if you can ls the file.
311  # This works in ifdh 1_7_0 and newer
312  # If it exists, dh.ls() returns a tuple with one entry, converts to True
313  # If it doesn't, dh.ls() returns an empty tuple, converts to False
314  return bool(dh.ls(outPath, 1, ""))
315  except (RuntimeError, IOError) as e:
316  # Some versions of ifdh throw an exception when the file was not found
317  # But that means it is not there.
318  return False
319 
def fileExists(outPath, dh)
Definition: runNovaSAM.py:308
def runNovaSAM.getOutDir (   pathname,
  hashDirs = False 
)

Definition at line 92 of file runNovaSAM.py.

References exit(), and parse_dependency_file_t.list.

Referenced by copyOutFiles(), and getOutDir().

92 def getOutDir(pathname, hashDirs=False):
93  if "DEST" in os.environ:
94  dest = os.environ["DEST"]
95  else:
96  print "No destination set!"
97  exit(1)
98  dirs = [dest]
99  if hashDirs:
100  head, tail = os.path.split(pathname)
101  hash = md5.new(tail)
102  dirs += list(hash.hexdigest()[:3])
103  return os.path.join(*dirs)
104 
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM.py:92
exit(0)
def runNovaSAM.getOutDir (   pathname,
  dest,
  hashDirs = False,
  runDirs = False,
  runNum = 0 
)

Definition at line 115 of file runNovaSAM.py.

References getOutDir(), parse_dependency_file_t.list, makeDirSafely(), and str.

115 def getOutDir(pathname, dest, hashDirs=False, runDirs=False, runNum=0):
116  dirs = [dest]
117  if hashDirs:
118  head, tail = os.path.split(pathname)
119  hash = hashlib.md5(tail.encode())
120  dirs += list(hash.hexdigest()[:3])
121  if runDirs:
122  head, tail = os.path.split(pathname)
123  runStr = str(runNum)
124  multiRunDir = runStr[:3].zfill(6)
125  makeDirSafely(os.path.join(dest, multiRunDir))
126  makeDirSafely(os.path.join(dest, multiRunDir, runStr))
127  dirs += [multiRunDir, runStr]
128  return os.path.join(*dirs)
129 
def makeDirSafely(dir)
Definition: runNovaSAM.py:83
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM.py:92
def runNovaSAM.getOutputFileName (   detector,
  runNum,
  subNum,
  streamEntry,
  release,
  subversion,
  dataFlag,
  tier,
  nevt 
)

Definition at line 173 of file runNovaSAM.py.

References exit(), and novadaq::HexUtils.format().

173 def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt):
174  if dataFlag == "sim":
175  pattern = re.compile(r"^.*?_(.*?)_\d+.*_v\d+_(.*?)(\.|\_)sim\..+")
176  match = pattern.match(inFileBase)
177  if match != None:
178  simLabel = match.groups()[0]
179  uniq = match.groups()[1]
180  outName = '{0}_{1}_{2}_r{3:08d}_s{4:02d}_{5}_v{6}_{7}_{8}.{9}.root'.format(detector, simLabel, nevt, runNum, subNum, release, subversion, uniq, dataFlag, tier)
181  else:
182  print "regex couldn't find the sim label and the uniqueifier"
183  exit(1)
184  else:
185  outName = '{0}_r{1:08d}_s{2:02d}_{3}_{4}_v{5}_{6}.{7}.root'.format(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier)
186  return outName
187 
def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
Definition: runNovaSAM.py:173
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
exit(0)
def runNovaSAM.listFiles (   outPath,
  dh 
)

Definition at line 320 of file runNovaSAM.py.

320 def listFiles(outPath, dh):
321  try:
322  return dh.ls(outPath, 1, "")
323  except:
324  return "Exception while trying to ls, OutPath =",outPath
325 
326 
def listFiles(outPath, dh)
Definition: runNovaSAM.py:320
def runNovaSAM.make_temp_fcl (   fclFile,
  inFileBase 
)

Definition at line 37 of file runNovaSAM.py.

References MetadataUtils.addMetadataToFCL(), open(), print, and resolveFclPath().

37 def make_temp_fcl(fclFile, inFileBase):
38  # make a temporary copy of the fcl file
39  fclPath = resolveFclPath(fclFile)
40  print("Found fcl file here: ", fclPath)
41  tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
42  print("Creating local copy : ", tmpFclName)
43  shutil.copy(fclPath, tmpFclName)
44 
45  # determine if we're using the metadata module
46  # add parameters that describe this job
47  # NOTE:
48  # fhicl-expand (and other FCL tools) don't allow you
49  # to give the absolute path to a FCL. Instead FCLs have to
50  # live in $FHICL_FILE_PATH. $FHICL_FILE_PATH always begins
51  # with './:', i.e., search current dir first.
52  # so just make sure you don't cd() away from this dir
53  # between when the FCL is copied to this dir above and the check below.
54  isRunningMetadataModule=True
55  try:
56  subprocess.check_call("fhicl-expand %s | grep -q ^physics.analyzers.metadata.params." % tmpFclName, shell=True)
57  except subprocess.CalledProcessError:
58  isRunningMetadataModule=False
59 
60  if isRunningMetadataModule:
61  with open(tmpFclName, 'a') as fclFileObj:
62  if not fileMetaDataMgr.isSam4Users():
63  if args.npass != None:
64  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.subversion", '"' + fileMetaDataMgr.subversion + '"')
65 
66  if fileMetaDataMgr.dataFlag == "sim" and fileMetaDataMgr.generator in MetadataUtils.neutrinoGenerators:
67  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.flux_version", '"' + fileMetaDataMgr.fluxVersion + '"')
68 
69  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.skim", '"' + fileMetaDataMgr.skim + '"')
70  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.systematic", '"' + fileMetaDataMgr.systematic + '"')
71  MetadataUtils.addMetadataToFCL(fclFileObj, "NOVA.Special", '"' + fileMetaDataMgr.special + '"')
72 
73  return tmpFclName
74 
def resolveFclPath(fcl)
Definition: runNovaSAM.py:188
bool print
procfile open("FD_BRL_v0.txt")
def addMetadataToFCL(fclFile, parName, parValue)
def make_temp_fcl(fclFile, inFileBase)
Definition: runNovaSAM.py:37
def runNovaSAM.makeDeCAF (   script,
  fname,
  special 
)

Definition at line 554 of file runNovaSAM.py.

References novadaq::HexUtils.format(), if(), and resolveFclPath().

554 def makeDeCAF(script, fname, special):
555  trimname = fname[:-5] # cut off .root from end
556  trimidx = trimname.rindex('.')+1 # find last period before .root
557  decaf_tier = trimname[trimidx:-3]+'de'+trimname[-3:] # properly insert 'de'
558  oname = '{0}_{1}.{2}.root'.format(trimname, special, decaf_tier)
559  novaSource = os.getenv("SRT_PUBLIC_CONTEXT", "undefined")
560  if(novaSource == "undefined"):
561  novaSource = os.getenv("NOVASOFT_DIR", "undefined")
562  if(novaSource == "undefined"):
563  NGU.fail("Unable to locate NOvA source code")
564  else:
565  novaSource = os.getenv("NOVASOFT_DIR") + "/source"
566 
567  os.system('cafe -bq '+novaSource+'/CAFAna/'+script+' '+fname+' '+oname+' 2>&1')
568  return oname
569 
if(dump)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def makeDeCAF(script, fname, special)
Definition: runNovaSAM.py:554
def runNovaSAM.makeDirIfNeeded (   dir)

Definition at line 103 of file runNovaSAM.py.

References print.

Referenced by checkAndMoveFiles(), and makeDirSafely().

104  os.umask(0o02) # (rw-rw-r--) for files and (rwxrwxr-x) for directories.
105  if not os.path.isdir(dir):
106  print("runNovaSAM is making a directory: ", dir)
107  try:
108  os.mkdir(dir)
109  except:
110  print("Couldn't make the directory... some other job perhaps did, or permissions did not allow ")
111  if not os.path.isdir(dir):
112  raise Exception("Failed to make directory + " + dir )
113 
114 
bool print
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
def makeDirIfNeeded(dir)
Definition: runNovaSAM.py:103
def runNovaSAM.makeDirSafely (   dir)

Definition at line 83 of file runNovaSAM.py.

References makeDirIfNeeded(), and print.

Referenced by getOutDir().

83 def makeDirSafely(dir):
84  if "/pnfs" != dir[0:5]:
85  makeDirIfNeeded(dir)
86  return
87 
88  print("runNovaSAM is making a directory with IFDH")
89  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
90  try:
91  print("Checking if directory ", dir, "exists")
92  dh.ls(dir, 1, "")
93  dh.chmod(774,dir,"")
94 
95  except (RuntimeError, IOError) as e:
96  try:
97  print("It doesn't - make directory", dir)
98  dh.mkdir(dir)
99  except:
100  print("Tried to make directory and couldn't. Perhaps it already exists?")
101 
102 
def makeDirSafely(dir)
Definition: runNovaSAM.py:83
bool print
def makeDirIfNeeded(dir)
Definition: runNovaSAM.py:103
def runNovaSAM.makeMetadataJSONs ( )
For every ROOT file, try to extract its metadata into a matching .json
file in the same directory

Definition at line 327 of file runNovaSAM.py.

References if(), MakeTransposeJson(), ModifyParentMetadata(), open(), print, and PandAna.Demos.demo1.range.

328  """For every ROOT file, try to extract its metadata into a matching .json
329  file in the same directory"""
330  baseDir = "./copyback"
331  # If using --txtfiledef, want to pass multiple files at a time to sam_meta_dumper.
332  # This is because it simply takes too long to call it for each indiv file
333  # when there are 800+ outputs. 5 s each -> ~1 hour!
334  TransposeList = ""
335  # Declare the samweb client within this function
336  samweb = samweb_client.SAMWebClient(experiment='nova')
337  # Loop through directories to search for files to make json files for.
338  for root, dirs, filenames in os.walk(baseDir):
339  if root == baseDir:
340  # Push the h5 files to the front of the list so that the CAF files remain
341  # available to hack in the metadata.
342  for ifile in range(len(filenames)):
343  if("h5" in filenames[ifile]):
344  filenames = [filenames[ifile]] + filenames[:ifile] + filenames[ifile+1:]
345  for file in filenames:
346  if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
347  skip_match = _skip_pattern.match(file)
348  if skip_match == None:
349  # Set some quick and useful variables.
350  olddir = os.getcwd()
351  fileWPath = os.path.join(root, file)
352  # If a transpose file want to make the json subtly differently.
353  if args.txtfiledef and "outplane" in file:
354  # This works for the cellmode of the transposer as well, due to the filename including outplane and cell.
355  print("Adding %s to TransposeList" %file)
356  TransposeList += "%s " %file
357  continue
358  # Which extractor am I using?
359  extractor = 'sam_metadata_dumper'
360  if file.endswith('caf.root'):
361  extractor = 'extractCAFMetadata'
362  elif file.endswith('.h5'):
363  extractor = 'extractHDF5Metadata'
364  try:
365  # if we're running an analyzer, then there's no outTier, no cafTier,
366  # and we just make the metadata from the parent file
367  meta=''
368  if args.useParentMetadata:
369  # ensure we're in the directory where the parent file lives
370  os.chdir(os.path.dirname(inFile))
371  md = ModifyParentMetadata(os.path.basename(inFile),file)
372  # and now switch back to copyback dir to make sure we can copy back json
373  os.chdir(olddir)
374  os.chdir(os.path.dirname(fileWPath))
375  md['file_size'] = os.path.getsize(file)
376  # meta is currently a list, convert to json format
377  meta = json.dumps(md, separators = (', ', ': '))
378  else:
379  # sam_metadata_dumper doesn't apply basename() to filename. https://cdcvs.fnal.gov/redmine/issues/8987
380  os.chdir(os.path.dirname(fileWPath))
381  meta = subprocess.check_output([extractor, os.path.basename(fileWPath)])
382  if file.endswith (".root"):
383  jsonf = open(file.replace('.root', '.json'), 'wb')
384  jsonf.write(meta)
385  jsonf.close()
386  print("Made metadata for %s" %file)
387  elif file.endswith(".h5"):
388  print("\nNow to make my json file for my h5...\n")
389  jsonf = open(file.replace('.h5caf.h5', '.h5caf.json'), 'wb')
390  jsonf.write(meta)
391  jsonf.close()
392  print("Made metadata for %s" %file)
393  else:
394  print("I'm not sure what file extension you have...")
395  except:
396  print("Error extracting metadata from file.")
397  finally:
398  os.chdir(olddir)
399  # Make the Transpose json files.
400  # Again same argument, outplane is already in the filenames for cell mode
401  if args.txtfiledef and "outplane" in file:
402  olddir = os.getcwd()
403  os.chdir(baseDir)
404  MakeTransposeJson( TransposeList )
405  os.chdir(olddir)
406 
def makeMetadataJSONs()
Definition: runNovaSAM.py:327
if(dump)
bool print
def MakeTransposeJson(TransposeList)
Definition: runNovaSAM.py:407
procfile open("FD_BRL_v0.txt")
def ModifyParentMetadata(inParentFile, inFile)
Definition: runNovaSAM.py:193
def runNovaSAM.MakeTransposeJson (   TransposeList)
Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
This takes in a list of files, and makes appropriate .json files in the same directory

Definition at line 407 of file runNovaSAM.py.

References append(), hadd_reco_validation.group, open(), print, PandAna.Demos.demo1.range, split(), and str.

Referenced by makeMetadataJSONs().

407 def MakeTransposeJson( TransposeList ):
408  """Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
409  This takes in a list of files, and makes appropriate .json files in the same directory"""
410  # If using --txtfiledef, I can now pass my file list to sam_meta_dumper.
411  print("Is MakeTransposeJson called without a txt def, if you see this then yes.")
412  MetaListFile="AllMetaJson.txt"
413  meta_cmd="sam_metadata_dumper -s " + TransposeList + " > " + MetaListFile
414  os.system(meta_cmd)
415  # Now want to open the file and split by "}," character.
416  MetaFile = open( MetaListFile )
417  MetaLines = MetaFile.read().split(" },")
418  # Loop through lines, and appropriately separate out json files.
419  for i in range(0,len( MetaLines ) ):
420  meta=MetaLines[i]
421  # Figure out file name...this is assuming that file name is always the first entry...
422  StName=re.search('"(.+?).root"', meta ).group(1)
423  filename=StName+".json" # Effecitvely replacing .root with .json
424  # If transpose file add PlaneNumber and CellNumber if run in cell mode
425  if "cell" in filename:
426  try:
427  plane = re.search('^.*outplane(\d*).', filename).group(1)
428  cell = re.search('^.*cell(\d*).', filename).group(1)
429  print("Plane number:", plane, "Cell number:", cell, ". Is a transpose file")
430  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.CellNumber": "%s",\n "calibration.base_release"')%(plane, cell)
431  except:
432  print("No cell number found - could be a plane mode transpose file")
433 
434  elif "outplane" in filename:
435  try:
436  plane = re.search('^.*outplane(\d*).', filename).group(1)
437  print("Plane number:", plane, ". Is a transpose file")
438  meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n "calibration.base_release"') %plane
439  except:
440  print("Error extracting plane number from transpose file.")
441 
442  ### Make sure that the txtfile is in the parent list.
443  meta['parents'].append({u'file_name':str(inFile)})
444 
445  # Now open the json file
446  fout=open(filename,'w')
447  # Want to make sure that the json starts with '{'
448  if meta[0] not in "{":
449  meta = meta[:0] + '{\n' + meta[1:]
450  # Want to make sure that the json ends with a double '}'
451  if i < len(MetaLines)-1:
452  meta += "}\n}\n"
453  # Write and close the json file
454  fout.write(meta)
455  fout.close()
456 
void split(double tt, double *fr)
void append()
Definition: append.C:24
bool print
def MakeTransposeJson(TransposeList)
Definition: runNovaSAM.py:407
procfile open("FD_BRL_v0.txt")
def runNovaSAM.ModifyParentMetadata (   inParentFile,
  inFile 
)

Definition at line 193 of file runNovaSAM.py.

References append(), MetadataUtils.createMetadata(), declareFile(), hadd_reco_validation.group, print, and str.

Referenced by makeMetadataJSONs().

193 def ModifyParentMetadata(inParentFile, inFile):
194  print("Using parent metadata and modifying, using", inParentFile, "for", inFile)
195  samweb = samweb_client.SAMWebClient(experiment='nova')
196  md = MetadataUtils.createMetadata(inParentFile)
197  md['file_name'] = inFile
198  md['data_tier'] = args.histTier
199  md['file_format'] = "root"
200  parent_list=[]
201  for filename in os.listdir('.'):
202  print("found parent file",filename)
203  parent_dict = {'file_name': filename}
204  parent_list.append(parent_dict)
205 
206  print("parent list:")
207  print(parent_list)
208  parent_dumps = json.dumps(parent_list, separators = (', ', ': '))
209  print(parent_dumps)
210  md['parents'] = parent_list
211  print(md)
212  return md
213 
214 #def h5MetaHack(inFile):
215 # print "Doing a hack to get HDF5 metadata...Will take CAF metadata and change subtly."
216 # ### Make my client and get my CAF metadata
217 # samweb = samweb_client.SAMWebClient(experiment='nova')
218 # md = MetadataUtils.createMetadata( inFile.replace('.h5caf.h5', '.caf.root') )
219 # ### Change some parameters.
220 # md['file_name'] = inFile
221 # md['data_tier'] = unicode('h5')
222 # md['file_size'] = os.path.getsize( inFile )
223 # return md
224 
bool print
def createMetadata(inFile)
def ModifyParentMetadata(inParentFile, inFile)
Definition: runNovaSAM.py:193
def runNovaSAM.resolveFclPath (   fcl)

Definition at line 188 of file runNovaSAM.py.

Referenced by make_temp_fcl(), and makeDeCAF().

188 def resolveFclPath(fcl):
189  # Check if we have an absolute path name, return it if so
190  if fcl[0] == "/":
191  return fcl
192 
193  # Otherwise, we need to do some searching
194  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
195  for path in fclPaths:
196  if os.path.isfile(path + fcl):
197  return path + fcl
198 
199  # If we haven't found it, we have a problem.
200  raise IOError(sys.argv[0] + ": config file "+ fcl+" not found in FHICL_FILE_PATH")
201 
202 
void split(double tt, double *fr)
def resolveFclPath(fcl)
Definition: runNovaSAM.py:188
def runNovaSAM.setVMemLimit ( )

Definition at line 75 of file runNovaSAM.py.

References print.

76  #memLimit = 3.9*1024**3
77  memLimit = 3.9*1000**3
78  print("Old virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS))
79  print("Limiting the virtual memory of the nova job to 3.9 GB")
80  resource.setrlimit(resource.RLIMIT_AS, (memLimit, memLimit))
81  print("New virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS))
82 
bool print
def setVMemLimit()
Definition: runNovaSAM.py:75
def runNovaSAM.unCamelCase (   s)
Convert CamelCase to camel_case 

Definition at line 29 of file runNovaSAM.py.

Referenced by createMetadata().

29 def unCamelCase(s):
30  """ Convert CamelCase to camel_case """
31  # \B matches an empty string which is NOT at the beginning of a word
32  # so requiring this means no _ will be inserted at the start
33  return re.sub( r'\B([A-Z])', r'_\1', s).lower()
34 
35 
def unCamelCase(s)
Definition: runNovaSAM.py:29

Variable Documentation

runNovaSAM._bad
private

Definition at line 1033 of file runNovaSAM.py.

runNovaSAM._first
private

Definition at line 1030 of file runNovaSAM.py.

runNovaSAM._good
private

Definition at line 1034 of file runNovaSAM.py.

runNovaSAM._lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
private

Definition at line 24 of file runNovaSAM.py.

runNovaSAM._outputs
private

Definition at line 1032 of file runNovaSAM.py.

runNovaSAM._second
private

Definition at line 1031 of file runNovaSAM.py.

runNovaSAM._skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
private

Definition at line 27 of file runNovaSAM.py.

runNovaSAM.action

Definition at line 207 of file runNovaSAM.py.

runNovaSAM.allFiles

Definition at line 895 of file runNovaSAM.py.

runNovaSAM.args = parser.parse_args()

Definition at line 215 of file runNovaSAM.py.

runNovaSAM.argv

Definition at line 28 of file runNovaSAM.py.

runNovaSAM.argvCopy

Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help.

Definition at line 27 of file runNovaSAM.py.

runNovaSAM.cafLabel = cafTier.split(":")[0]

Definition at line 362 of file runNovaSAM.py.

Definition at line 366 of file runNovaSAM.py.

runNovaSAM.cafTier

Definition at line 342 of file runNovaSAM.py.

string runNovaSAM.cmd = ' '

Definition at line 383 of file runNovaSAM.py.

list runNovaSAM.cmdList = ['nova']

Definition at line 330 of file runNovaSAM.py.

runNovaSAM.copyfile

Definition at line 901 of file runNovaSAM.py.

runNovaSAM.copyOnly

Definition at line 965 of file runNovaSAM.py.

string runNovaSAM.dataFlag = "data"

Definition at line 297 of file runNovaSAM.py.

runNovaSAM.decafdir

Definition at line 1061 of file runNovaSAM.py.

runNovaSAM.default

Definition at line 638 of file runNovaSAM.py.

runNovaSAM.description

Definition at line 589 of file runNovaSAM.py.

runNovaSAM.dest

If using a txtfiledef make sure to clean up the InputFile List....

Definition at line 949 of file runNovaSAM.py.

runNovaSAM.detector = metadata["online.detector"].lower()

Definition at line 272 of file runNovaSAM.py.

Referenced by ApplyOscillations(), daq2raw::DAQHit2Raw.beginRun(), BuildMetricsTree(), cmf::DataVarVals.DataVars(), DCSPlotMaker(), cmf::MetaData.DetectorString(), novaddt::utils::DetectorUtils.DetectorUtils(), daqchannelmap::NearDetDAQChannelMap.encodeDChan(), daqchannelmap::FarDetDAQChannelMap.encodeDChan(), daqchannelmap::TestBeamDAQChannelMap.encodeDChan(), daqchannelmap::DAQChannelMapOLD.encodeDChan(), daqchannelmap::NearDetDAQChannelMap.encodeLChan(), daqchannelmap::FarDetDAQChannelMap.encodeLChan(), daqchannelmap::TestBeamDAQChannelMap.encodeLChan(), meta::Metadata.FillMetadata(), daqchannelmap::DAQChannelMapOLD.getBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getBlockType(), GetCosmics(), daqchannelmap::DAQChannelMapOLD.getDCMLinkOffline(), daqchannelmap::DAQChannelMapOLD.getDCMOffline(), daqchannelmap::DAQChannelMapOLD.getDiBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockTypeOffline(), daqchannelmap::DAQChannelMapOLD.getFirstDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLastDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getLocalPlaneInBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockViewOffline(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInXPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInYPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfXDCM(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfYDCM(), daqchannelmap::DAQChannelMapOLD.getOnlineCellOffline(), daqchannelmap::DAQChannelMapOLD.getPixelOffline(), daqchannelmap::DAQChannelMapOLD.getPlaneViewOffline(), GetSystsFromFile(), daqchannelmap::DAQChannelMapBaseOLD.getTotalNumberOfDiblocks(), GoodRuns(), daqchannelmap::DAQChannelMapBaseOLD.initNumberOfDCMsInDiblockDetector(), nova::dbi::RunHistory.LoadDAQRunHistory(), main(), make_fc_nus_surfs_nersc_2019(), MakeCovarSim(), MakeSurface(), NewNearlinePlots(), NuMu2019_BasicPIDPlots_FD(), NuMu2019_BasicPIDPlots_ND(), NuMu2019_BasicPIDPlots_Spectrum(), NuMu2020_BasicPIDPlots_FD(), NuMu2020_BasicPIDPlots_ND(), NuMu2020_BasicPIDPlots_Spectrum(), rawdata::RawBeamlineTrigger.RawBeamlineTrigger(), rawdata::RawBeamlineWC.RawBeamlineWC(), om::HistoTable.ReadFile(), rawdata::RawBeamlineTrigger.SetDetectorID(), rawdata::RawBeamlineWC.SetDetectorID(), ana.TAHadEND(), and rawdata::RawBeamlineWC.YPulses().

runNovaSAM.detectors = dict()
runNovaSAM.detId = metadata["nova.detectorid"].lower()
runNovaSAM.dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
runNovaSAM.dirList = os.listdir(".")

Definition at line 406 of file runNovaSAM.py.

runNovaSAM.doMeta

Definition at line 706 of file runNovaSAM.py.

runNovaSAM.end

Definition at line 932 of file runNovaSAM.py.

runNovaSAM.experiment

Definition at line 669 of file runNovaSAM.py.

runNovaSAM.failFiles

Definition at line 896 of file runNovaSAM.py.

runNovaSAM.False

Definition at line 638 of file runNovaSAM.py.

runNovaSAM.fclFile = args.config

Definition at line 315 of file runNovaSAM.py.

runNovaSAM.fclFileObj = open(tmpFclName, 'a')

Definition at line 327 of file runNovaSAM.py.

runNovaSAM.fclPath = resolveFclPath(fclFile)

Definition at line 319 of file runNovaSAM.py.

runNovaSAM.file_type = metadata["file_type"]

Definition at line 291 of file runNovaSAM.py.

runNovaSAM.fileMetaDataMgr

Definition at line 687 of file runNovaSAM.py.

runNovaSAM.FirstRun

Definition at line 775 of file runNovaSAM.py.

runNovaSAM.flatLabel

Definition at line 828 of file runNovaSAM.py.

runNovaSAM.flatName

Definition at line 833 of file runNovaSAM.py.

runNovaSAM.flatTier

Definition at line 726 of file runNovaSAM.py.

runNovaSAM.gdbArgs

Definition at line 916 of file runNovaSAM.py.

runNovaSAM.gErrorIgnoreLevel
runNovaSAM.h5Label

Definition at line 843 of file runNovaSAM.py.

runNovaSAM.h5Name

Definition at line 848 of file runNovaSAM.py.

runNovaSAM.h5Tier

Definition at line 728 of file runNovaSAM.py.

runNovaSAM.help

Definition at line 205 of file runNovaSAM.py.

runNovaSAM.histName

Definition at line 865 of file runNovaSAM.py.

runNovaSAM.ifdhcmd

Definition at line 903 of file runNovaSAM.py.

runNovaSAM.Index
runNovaSAM.inFile = args.inFile

Definition at line 217 of file runNovaSAM.py.

runNovaSAM.inFileBase = os.path.basename(inFile)

Definition at line 221 of file runNovaSAM.py.

runNovaSAM.inPID

Definition at line 998 of file runNovaSAM.py.

runNovaSAM.lastdir

Definition at line 976 of file runNovaSAM.py.

runNovaSAM.LastRun

Definition at line 776 of file runNovaSAM.py.

runNovaSAM.line

Definition at line 932 of file runNovaSAM.py.

runNovaSAM.logfile

Definition at line 927 of file runNovaSAM.py.

Referenced by main(), and nl::timing::LogFile.ProcessLog().

dictionary runNovaSAM.metadata = samweb.getMetadata(inFileBase)
runNovaSAM.nevt = None

Definition at line 311 of file runNovaSAM.py.

runNovaSAM.NewOMod

Definition at line 753 of file runNovaSAM.py.

runNovaSAM.NewOName

Definition at line 790 of file runNovaSAM.py.

runNovaSAM.noCleanup

Definition at line 965 of file runNovaSAM.py.

runNovaSAM.olddir

Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files?

Definition at line 893 of file runNovaSAM.py.

Referenced by numusand::NumuSandFxs.getScatt(), cosrej::CosRejFxs.getScatt(), and ndreco::NDRecoFxs.getTrackScatt().

runNovaSAM.outC

Definition at line 748 of file runNovaSAM.py.

list runNovaSAM.outList = []

Definition at line 344 of file runNovaSAM.py.

runNovaSAM.outNameTemp

Definition at line 773 of file runNovaSAM.py.

runNovaSAM.outP

Definition at line 748 of file runNovaSAM.py.

runNovaSAM.output = outTier.split(":")[0]

Definition at line 348 of file runNovaSAM.py.

runNovaSAM.outputs

Definition at line 734 of file runNovaSAM.py.

runNovaSAM.outTier

Definition at line 340 of file runNovaSAM.py.

runNovaSAM.outX

Definition at line 759 of file runNovaSAM.py.

runNovaSAM.outXc

Definition at line 749 of file runNovaSAM.py.

runNovaSAM.outXp

Definition at line 749 of file runNovaSAM.py.

runNovaSAM.outY

Definition at line 760 of file runNovaSAM.py.

runNovaSAM.outYc

Definition at line 750 of file runNovaSAM.py.

runNovaSAM.outYp

Definition at line 750 of file runNovaSAM.py.

runNovaSAM.parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')

Definition at line 204 of file runNovaSAM.py.

runNovaSAM.PassFile

Definition at line 691 of file runNovaSAM.py.

runNovaSAM.precopyscript

Definition at line 731 of file runNovaSAM.py.

runNovaSAM.release = os.environ["SRT_BASE_RELEASE"]

Definition at line 285 of file runNovaSAM.py.

Referenced by AnalyzeSysts(), bin_composition_pie_chart(), CAFCMFContours(), CalcRWithSystsNus17(), CompareBinningSchemes(), CompareMissingLeptons(), ConsolidateSystMaker(), containment_optimization_plots(), CopyPred(), CVNCuts(), demo_cut_optimization(), demo_trivial_xsec_analysis(), draw_michel_plots(), DrawContours(), DrawCovMx(), DrawLLComparisonContours(), drawPlot(), drawSensitivity(), drawSystsShiftingNDdata(), drawSystsShiftingNDdata_updatedAna(), example_plot(), fake_future_data(), FCContour(), FCCorrectSlice(), fd_plot(), fill_col(), FillHists(), GeniePredictionToRoot(), get_univ_chisq(), GetBiases(), getContProf(), getContProf_Sensitivity(), getHists_FNEX(), GetMatrices(), GetMockData(), GetNueCosmics2017(), GetNueCosmics2018(), GetNueCosmics2019(), ana.GetNueCosmics2020(), GetNueCosmicsFuture(), GetNueData2017(), GetNueData2018(), GetNueData2019(), ana.GetNueData2020(), GetNuePrediction2017(), GetNuePrediction2018(), GetNuePrediction2019(), ana.GetNuePrediction2020(), GetNuePredictionFuture(), ana.GetNumuData2020(), GetNumuPredictionsFuture(), getSensitivity(), GetSurf(), GetSystsFromFile(), getTimePeakPlots(), LoadCovMx(), LoadData(), LoadExtrapPrediction(), ana::CountingExperiment.LoadFrom(), ana::TrivialBkgdEstimator.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::FDPredictionSterile.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::NDPredictionSterile.LoadFrom(), ana::SingleNucAnalysis.LoadFrom(), ana::FitInAnaBinsBkgdEstimator.LoadFrom(), ana::NumuCC2p2hAnalysis.LoadFrom(), ana::NumuCC2p2hBkgdEstimator.LoadFrom(), ana::nueccinc::NueCCIncMRECorrection.LoadFrom(), ana::MultiExperiment.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::PredictionCombinePeriods.LoadFrom(), ana::PredictionAddRock.LoadFrom(), ana::PredictionExtendToPeripheral.LoadFrom(), ana::PredictionScaleComp.LoadFrom(), ana::CutOptimization.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::CrossSectionAnalysis.LoadFrom(), ana::Multiverse.LoadFrom(), ana::NumuCCIncAnalysis.LoadFrom(), ana::nueccinc::NueCCIncCrossSectionAnalysis.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::SystMaker.LoadFrom(), ana::nueccinc::NueCCIncEnhancedSamples.LoadFrom(), ana::SystematicsMaker.LoadFrom(), ana::PredictionInterp.LoadFromBody(), LoadMaps(), ana.LoadMaps(), LoadPPFX(), LoadPrediction(), ana.LoadSystsFromFile(), Make2DPlot(), make_extrap_figure(), make_extrap_figure_hists(), make_fc_nus_surfs_nersc_2019(), make_files_for_decomp_plots(), make_muonid_opt(), make_nominal_xs(), Make_NuMuCC_Inc_XS(), make_nus17_fc_surfs(), make_nus_fc_surfs(), make_starplots(), make_surfprof(), make_vertex_optimiz(), make_xs(), make_xs_1D(), makeEnergyEstimator(), makeFlatWeight(), MakeISysts(), makeMatrixElementSurface(), MakeNumuCovMx(), MakePeriCutPlot(), MakePlot(), MakePlots(), MakeSurface(), MakeSurfaceBinningStudy(), MakeSystsPDF(), MakeTable(), mec_nux_tester_2020(), mec_tuning_fitter_2020(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), mre_blessed(), mre_comp_split(), muonid_optimization_plots(), nue_decomp_scales_for_make_decomp(), nuebar_signif(), NumuExtrap(), PeripheralCuts(), plot_3NDvsFD(), plot_3NDvsFD_FHC(), plot_3NDvsFD_RHC(), plot_BEN(), plot_contprof(), plot_datamc_ND_numu(), plot_datamc_ND_numu_REW(), plot_datamcpred(), plot_datapredictions(), plot_joint_fit_2020_contours(), plot_ND_numu_NOMvsREW(), plot_NDvsFD_REW(), plot_NDvsFD_weights(), plot_NDvsFD_weights_FHC(), plot_NDvsFD_weights_RHC(), plot_nuCrystalBall(), plot_nue_filesyst_pred(), plot_nue_xsec_pred(), plot_nueFD_Signal_REWvsNOM_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_RHC(), plot_nueFD_Signal_REWvsNOM_RHC(), plot_nuTree(), plot_pi0_xcheck(), plot_pid(), plot_predictions(), plot_rationoosc(), plot_resolution(), plot_shifts(), plot_syst_contours(), plot_time(), plot_uncertainty(), plotContProf(), plotDataPred(), plotDataPred_SplitCanvas(), PlotNus17PredSystsData(), PlotNus18Sideband(), PlotNusSensAna01(), PlotPionPlots(), plots(), Plotting_DataAndPrediction(), pred_err(), PredictCosmic(), ana::PredictionExtrapSum.PredictionExtrapSum(), ana::PredictionSyst3Flavor2020.PredictionSyst3Flavor2020(), ana::PredictionSystJoint2018.PredictionSystJoint2018(), ana::PredictionSystJointDemo.PredictionSystJointDemo(), ana::PredictionSystNue2017.PredictionSystNue2017(), ana::PredictionSystNumu2017.PredictionSystNumu2017(), preselection_cutflow(), print_tables(), saveContours_addExpt(), saveContours_complete(), saveContours_oscpar(), saveContours_simple(), saveContours_systs(), selection_story_plots(), SetBestFitParams(), SetCosmic(), SetPrediction(), signal_count(), starPlot(), syst_plot_test(), syst_table_fit(), syst_table_fit_new(), test_nue2017Prediction(), test_nue2018_fitter(), test_prediction_interp(), TestPred(), ana::Bayesian1DMarginal.ToTH1(), Toy_analyses(), UnfoldInOut(), vertex_fiducial_optimization_plots(), and xsec_extrap_plots().

runNovaSAM.ret

Definition at line 905 of file runNovaSAM.py.

Referenced by ana::OscillatableSpectrum._Oscillated(), ana::PredictionExtrap._PredictComponent(), ana::PredictionExtrapSum._PredictComponent(), ana::PredictionExtendToPeripheral._PredictComponent(), ana::PredictionAddRock._PredictComponentSyst(), ana::PredictionInterp._PredictComponentSyst(), ana::CutOptimization.AbsUncertainty(), ana::CutOptimization.AbsUncertaintySquared(), ana::Seed.ActiveFitVars(), ana::SeedList.ActiveFitVars(), ana::SystShifts.ActiveSysts(), ana.AddErrorInQuadrature(), AddHistograms(), ana.AddToSummaryV(), ana::Hist.Adopt(), ana::Hist.AdoptSparse(), ana::Hist.AdoptStan(), rawfileparser::RawFileParser.AdvanceToConfigEnd(), rawfileparser::RawFileParser.AdvanceToConfigStart_file(), rawfileparser::RawFileParser.AdvanceToConfigStart_mem(), rawfileparser::RawFileParser.AdvanceToFirstEvent_file(), rawfileparser::RawFileParser.AdvanceToFirstEvent_mem(), rawfileparser::RawFileParser.AdvanceToNextEvent_file(), rawfileparser::RawFileParser.AdvanceToNextEvent_mem(), rawfileparser::RawFileParser.AdvanceToPreviousEvent_mem(), rawfileparser::RawFileParser.AdvanceToRunHeader_file(), rawfileparser::RawFileParser.AdvanceToRunHeader_mem(), rawfileparser::RawFileParser.AdvanceToRunTail_file(), rawfileparser::RawFileParser.AdvanceToRunTail_mem(), dt::Chain.AllChunks(), dt::Chain.AllHits(), ana.AnaTypeToString(), dt::Chain.AsChunkMap(), ana::Spectrum.AsimovData(), ana::TrivialBkgdEstimator.Background(), calib::BetheBlochTables.BetheBlochTables(), rawfileparser::RawFileParser.BuildEventIndex_file(), rawfileparser::RawFileParser.BuildEventIndex_mem(), mcmc.BuildExperiments(), ana::StanFitter.BuildInitContext(), ana::GenericSystematicDef< SRType >.BuildSpectrumDown(), ana::GenericSystematicDef< SRType >.BuildSpectrumUp(), caldp::AttenProfilesMap.Channels(), ana::WildcardSource.CheckedWildcard(), ana::SolarConstraints.ChiSq(), ana::MultiExperiment.ChiSq(), rawfileparser::RawFileParser.close(), CombineHistograms(), ana::PredictionExtrapSum.ComponentCC(), ana::PredictionExtrapSum.ComponentNC(), ana::PredictionExtrapSum.ComponentNCAnti(), ana::PredictionExtrapSum.ComponentNCTotal(), ana::ThreadPool::TaskQueue.consume(), convert(), convert_mcnp_txt_to_root(), ConvertNormalBasis(), osc::OscCalcGeneral.Copy(), osc::_OscCalcPMNSOpt< T >.Copy(), osc::OscCalcPMNSOptEigen.Copy(), ana::Multiverse.CopyAndTransform(), gibuu::GiBUURegen.CopyGenieEvent(), ana.CopyUpDownSpectrum(), dt::ViewMerger.CountUnexplainedOnLine(), ana.DataMCComparison(), ana.DataMCComparisonAreaNormalized(), ana.DataMCComparisonComponents(), novarwgt.DecodeGeneratorVersion(), caf.DecodeGeneratorVersion(), DecorrelateFD(), DecorrelateND(), ana.DefaultOscCalc(), ana.DefaultOscCalcIH(), ana.DefaultSterileCalc(), ana::MarginalLikelihood.DoDerivative(), ana::MarginalLikelihood.DoEval(), lem::Potential.Downsampled(), lem::MatchableEvent.Downsampled(), ana.DrawQuantLabel(), ana::Prod5LoadersBase.ECAFTypeFlatToTier(), ana::Prod4LoadersBase.ECAFTypeFlatToTier(), ana.EigenMatrixXdFromTMatrixD(), art::thread_safe_registry_via_id< K, M >.emplace(), art::thread_safe_registry_via_id< K, M >.empty(), novarwgt.EncodeGeneratorVersion(), EnergyBin(), ErrorBand(), ErrorsToHist(), tf::HoughScore.Eval(), NovaMCEmptyEvent.eventTimestamp(), rb::Cluster.Exclude(), gibuu::GiBUURegen.ExpandLibraryPath(), ana::IFitter.ExpandSeeds(), mcmc.ExptPtrs(), ana.factorial(), dt::ViewMerger.FakeThirdDimension(), FillPCAContainers(), caf.FillProngVars(), calib::ThroughgoingSelection.filter(), calib::RockMuonStopperSelection.filter(), calib::StopperSelection.filter(), novaddt::MoonShadow.filter(), EliminateBeamSpills.filter(), hough::LiteTH2.FindBinX(), hv::LiteTH2.FindBinX(), hough::LiteTH2.FindBinY(), hv::LiteTH2.FindBinY(), ana.FindCurveCrossings(), geo::GeometryBase.FindGDMLFile(), caf::CAFMaker.FindManyPStrict(), lem.FindMatches(), ana.FindPackageDir(), trk::KalmanTrackAna.FindVisibleProngs(), cheat::RecoCheckAna.FindVisibleProngs(), fuzz::FuzzyKValidate.FindVisibleProngs3D(), fuzz::FuzzyKVertex.FindVisibleProngs3D(), calib::AttenFit.FitQuality(), calib::AttenuationFit.FitQuality(), ana::PredictionInterp.FitRatios(), lem::MatchableEvent.Flipped(), ana::Progress.FormatTime(), nuesand::FillNueSandbox.FracModalHits(), ana::Hist.FromDirectory(), lem::dec::Forest.FromFile(), ana::SpectrumLoader.FromSAMProject(), lem::PIDExtraVars.FromString(), lem::PIDDetails.FromString(), ana::FCBin.FromTree(), lem::EventSummary.FromTree(), GenerateFutureData(), caf.GenieReweightTable(), art::thread_safe_registry_via_id< K, M >.get(), ana2019::fakedata.Get2019Prediction(), ana.get3FlavorAna2020AllCPTSysts(), ana.get3FlavorAna2020AllSysts(), ana.get3FlavorAna2020LargeXsecSysts(), ana.getAllAna2017Systs(), ana.getAllAna2018Systs(), ana.getAllCPTSysts(), ana.getAllFluxSysts_2020(), caldp::AttenHistsMap.GetAllHistsByPlane(), ana.getAllNumuProd4FDHistDef(), ana.getAllNumuProd4NDHistDef(), ana.getAllNumuProd5FDHistDef(), ana.getAllNumuProd5FDMultiHistDef(), ana.getAllNumuProd5FDNuTruthHistDef(), ana.getAllNumuProd5NDHistDef(), ana.getAllNumuProd5NDMultiHistDef(), ana.getAllNumuProd5NDNuTruthHistDef(), ana.getAllNusAna01FDHistDefs(), ana.getAllNusAna01NDHistDefs(), caldp::AttenProfilesMap.GetAllProfilesByPlaneAndCell(), GetAllSamples(), ana::CutOptimization.GetAllSystDefs(), ana.getAllXsecNuTruthSysts_2017(), ana.getAna2017LargeXsecSysts(), ana.getAna2018LargeXsecSysts(), ana.getAna2018NueOnly(), ana.getAna2018NumuOnly(), ana.getAna2020NueOnly(), ana.getAna2020NumuOnly(), cosrej::CosRejFxs.getAngle(), numusand::NumuSandFxs.getAngle(), GetBG(), ana.GetBG(), GetBiasErrorLine(), ana::covmx::CovarianceMatrix.GetBinnings(), ana.GetCAFMetadata(), ana::covmx.GetComponents(), rawfileparser::RawFileParser.getConfigBlock(), GetConstraints(), GetCorrelations(), ana.GetCorrelations(), GetCosmics(), ana::covmx::CovarianceMatrix.GetCovMxRelative(), evd::GeometryDrawer.GetDCMBoxes(), calib::IBetheBloch.GetdEdxGraph(), calib::EnergyLossVsDistance.GetEnergyLossGraph(), rawfileparser::RawFileParser.getEvent(), gibuu::GiBUURegen.GetEvent(), GetExtrapolationDefs(), GetFakeData(), ana.GetFDMCComponents(), rawfileparser::RawFileParser.getFirstEvent(), GetFNRatio(), GetGaussianProfile(), GetGaussianSurface(), geo::CellGeo.GetGeoMatrix(), ana::ISurface.GetGraphs(), GetHist(), GetJointFitSystematicList(), ana.GetJointFitSystematicList(), ana.GetKeySystNames(), ana.GetKeySysts(), GetMatrices(), ana.GetMECWeights(), GetMockData(), calib::IBetheBloch.GetMPVGraph(), ana::covmx::CovarianceMatrix.GetNBins(), GetNC(), ana.GetNC(), GetNCFDSamples(), GetNCNDSamples(), ana::covmx::CovarianceMatrix.GetNComponents(), GetNCSamples(), ana.GetNDComponents(), ana.GetNDCompsFromDecomp(), ana.GetNDDecompsFromDecomp(), ana.GetNDMCComponents(), rawfileparser::RawFileParser.getNextEvent(), ana.getNue2018PredFile(), ana.getNue2018PredVar(), GetNueNDComponents(), GetNueNDComponentsMC(), GetNuePredPath(), ana.getNumu2018PredFile(), ana.getNumu2018PredVar(), GetNumuFDSamples(), GetNumuNDSamples(), GetNumuSamples(), GetNus18Binning(), GetNus18Constraints(), GetNus18FitVars(), GetNus18SeedValues(), ana.getNusAna2020AllSysts(), getPadHists(), osc::OscCalcSterileBeam.GetParamsHash(), osc::OscCalcSterile.GetParamsHash(), osc::_NoOscillations< T >.GetParamsHash(), osc::_OscCalcDMP< T >.GetParamsHash(), osc::OscCalcPMNSOptEigen.GetParamsHash(), osc::OscCalcCPT.GetParamsHash(), osc::_IOscCalcAdjustable< T >.GetParamsHashDefault(), osc::OscCalcPMNS_CPT.GetParamsHashDefaultBar(), GetPlotOptions(), GetPOTCombination(), GetPOTComboTitle(), rawfileparser::RawFileParser.getPreviousEvent(), ana::T2KToyExperiment.GetReweightedSignal(), rawfileparser::RawFileParser.getRunHeader(), rawfileparser::RawFileParser.getRunTail(), GetSampleFromOptString(), GetSamplesFromOptString(), SystGroupHelper.GetShifts(), GetShifts2017(), ana.GetShifts2018(), ana.GetShifts2020(), ana.getSomeXsecSysts(), ana.GetSpectrum(), GetSystConcats(), GetSystematics2017(), ana.GetSystematics2018(), ana.GetSystematics2020(), GetSystGroup(), ana::covmx::Sample.GetSystShifts(), calib::EnergyLossVsDistance.GetTF1(), GetTotFNRatio(), nuesand::FillNueSandbox.GetTrackAngle(), ana::FitDelta13InPiUnitsSterile.GetValue(), ana::FitDelta14InPiUnitsSterile.GetValue(), lem::LEMWorker.GetWork(), rawfileparser::RawFileParser.GotoEvent_file(), HistNumuOpt(), dt::Chunk.HitsOnLine(), cheat::BackTracker.HitsToParticle(), cheat::BackTracker.HitToFLSHit(), tf::TimingFitAlg.HoughFitPts(), ImportDataGraph(), InitCutDefs(), InitHistDefs(), InitSpectra(), geo::LiveGeometry.InMuonCatcher(), ana.InputPath(), util.ipow(), art::Hash< I >.isValid(), ana.JoinGraphs(), JointGraphs(), ana.kKalmanMinMax_byPdg(), ana.kMinPngNhit(), ana.kProngContainment_coord(), ana::SystShifts.LatexName(), load(), load_libs_muonid(), load_libs_nuebarccinc(), load_libs_nueccinc(), load_libs_numubarccpi0(), load_libs_numucc2p2h(), load_libs_numucc_0Pi(), load_libs_numuccinc(), LoadCosmic(), mcmc.LoadFakeData(), LoadFakeData(), ana.LoadFileList(), ana::PredictionXSecTuning.LoadFrom(), ana::CountingExperiment.LoadFrom(), ana::SolarConstraints.LoadFrom(), ana::PredictionNoOsc.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::CheatDecomp.LoadFrom(), ana::ModularExtrap.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::PredictionExtrap.LoadFrom(), ana::Binning.LoadFrom(), ana::CovMxManager.LoadFrom(), ana::NCDecomp.LoadFrom(), ana::NDExtrapBeamComponent.LoadFrom(), ana::NueDecomp.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::NDExtrapComponent.LoadFrom(), ana::NumuDecomp.LoadFrom(), ana::PredictionSystJoint2018.LoadFrom(), ana::PredictionSystJointDemo.LoadFrom(), ana::NDOscillatableSpectrum.LoadFrom(), ana::PredictionNoExtrap.LoadFrom(), ana::FDExtrap.LoadFrom(), ana::NDExtrap.LoadFrom(), ana::ProportionalDecomp.LoadFrom(), ana::FakeDecomp.LoadFrom(), ana::TrivialExtrap.LoadFrom(), ana::ReweightableSpectrum.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::PredictionInterp.LoadFrom(), ana::Multiverse.LoadFrom(), ana::TwoSampleDecomp.LoadFrom(), ana::SystShifts.LoadFrom(), ana::SpectrumComponents.LoadFrom(), ana::covmx::CovarianceMatrix.LoadFrom(), ana::OscillatableSpectrum.LoadFrom(), ana::FluxDecomp.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::BENDecomp.LoadFrom(), ana::PredictionSystNue2017.LoadFrom(), ana::PredictionSystNumu2017.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::Spectrum.LoadFrom(), ana.LoadFrom< osc::IOscCalcAdjustable >(), ana::FCCollection.LoadFromFile(), ana::FCSurface.LoadFromFile(), genie::GSimFiles.LoadFromFile(), ana::FrequentistSurface.LoadFromMulti(), genie::XSecSplineList.LoadFromXml(), LoadPrediction(), ana.LoadPredictions(), ana.LoadRealData(), mcmc.LoadSystsCreateDummyMECSysts(), ana::SAMQuerySource.LocateSAMFiles(), ana::MultiExperiment.LogLikelihood(), ana::GaussianPriorSystShifts.LogPrior(), ana::ExponentialPriorSystShifts.LogPrior(), calib::AttenFit.lowessFit(), calib::AttenuationFit.lowessFit(), ana::GradientDescent.Magnitude(), cet::sqlite::ConnectionFactory.make_connection(), MakeAverageTrueEnergySurface(), dt::View.MakeChunkCombos(), ana.MakeCosOutOfTimeList(), MakeCutFlowSpectra(), MakeDataReleaseFileContour(), MakeDataReleaseFileSlice(), MakeKin2DSpectra(), MakeKinSpectra(), MakeName(), ana.MakeNumiUnblindList(), MakeRatio(), MakeRatios(), MakeSample(), MakeSpectra(), MakeTitle(), ana.MakeUnblindList(), MakeUnblindList(), hough::MultiHough2P.MapToTH2F(), dt::ViewMerger.MatchVerticals(), osc.MatrixExp(), caldp::TCTrack.MaxCell(), rb::Cluster.MaxCell(), nuesand::FillNueSandbox.MaxHits(), caldp::TCTrack.MaxPath(), caldp::TCTrack.MaxPlane(), rb::Cluster.MaxPlane(), caldp::TCTrack.MaxTNS(), rb::Cluster.MaxTNS(), lem::Match.MaybeFlip(), dt::Cand.MaybeFlip(), cheat::BackTracker.MCTruthToParticles(), calib::AttenFit.MeanProfile(), calib::AttenuationFit.MeanProfile(), ana::ResolutionScan.MeanScan(), rb::Cluster.MeanTNS(), calib::AttenFit.MedianProfile(), calib::AttenuationFit.MedianProfile(), caldp::TCTrack.MinCell(), rb::Cluster.MinCell(), caldp::TCTrack.MinPath(), caldp::TCTrack.MinPlane(), rb::Cluster.MinPlane(), caldp::TCTrack.MinTNS(), rb::Cluster.MinTNS(), lem.MMapFileAtAddress(), ana::Spectrum.MockData(), MostConservative(), MyDefaultOscCalc(), dt::Cand.NHitPlanes(), dt::Chain.NHitPlanes(), ana::RecordMultiSink.NSinks(), rb::Cluster.OfflineChans(), ana::Ratio.operator*(), ana::Multiverse.operator*(), ana::Spectrum.operator*(), ana::ReweightableSpectrum.operator+(), ana::OscillatableSpectrum.operator+(), ana::Spectrum.operator+(), ana::ReweightableSpectrum.operator-(), ana::OscillatableSpectrum.operator-(), ana::Spectrum.operator-(), ana::Ratio.operator/(), ana::Multiverse.operator/(), ana::Spectrum.operator/(), ana::_preview::_RecordSource< T >.operator[](), osc::_IOscCalc< T >.P(), lem::FindMatchesAlg.PackageMatches(), PackMatrix(), PaintReachCanvas(), parse_as_vector(), ParseOption(), ParseOptions(), ana::SystShifts.Penalty(), PIDFileTag(), PIDLongName(), plot_ratios(), ana::NumuCCIncAnalysis.PlotHelper(), PlotMCComponentsErrorBand(), dt::DiscreteTracker.PossibleNextSegs(), ana::FDPredictionSterile.Predict(), ana::NDPredictionSterile.Predict(), ana::PredictionScaleComp.Predict(), ana::covmx::CovarianceMatrix.Predict(), ana::NDPredictionNoOsc.Predict(), ana::PredictionSterile.PredictComponent(), ana::FDPredictionSterile.PredictComponent(), ana::NDPredictionSterile.PredictComponent(), ana::PredictionNoOsc.PredictComponent(), ana::NDPredictionNoOsc.PredictComponent(), PredictCosmic(), ana::PredictionInterp.PredictionInterp(), art::TrivialFileDelivery.prependFileDesignation(), Print(), ana.Profile(), ProfileAxes(), calib::BetheBlochFit.ProfileMPV(), lem::dec::Cut.Prune(), caf.PtrVecToVec(), cheat::BackTracker.PtrVecToVecRawPtr(), ana::MCMCSamples.QuantileLL(), lem::dec::Forest.RandomOrthoMatrix(), ana.RatioPlots(), RebinToShortAxis(), dt::DiscreteTracker.RecoDiscrete(), ana.RefineSeeds(), RemoveBins(), ana.RemoveBins(), ana.replaceAll(), ana::ResolutionScan.RMSScan(), ana::PredictionAddRock.RockComponent(), ana.samplenames(), ana::T2KToyExperiment.SetPOTAnti(), ana::FitSinSq2Theta13.SetValue(), ana::SummedSyst.Shift(), ShiftedCosmics(), ana::SystShifts.ShortName(), ana::Registry< T >.ShortNameToPtr(), murem::MuonRemove.SliceSelection(), ana.sort_chisq_tot(), SortSystHists(), SplitHistograms(), ana.SqrtProfile(), ana.SqrtSlice(), caf.StripSubscripts(), flat::Flat< std::vector< T > >.SubLengthName(), flat::Flat< T[N]>.SubLengthName(), mcmc.SupportedSysts(), ana.systnames(), ana.systs_params_double_gauss_extra(), ana.systsQ3Q0(), slid::DedxDistribution.TH1DToTH1F(), TH1ToTH2(), calib::ThresholdCorrMap.ThresholdCorrAt(), ana.TMatrixDFromEigenMatrixXd(), dt::Chain.ToDebugTracks(), ana.ToEigen(), ana.ToEigenSterile(), ana::CutOptimization.ToHist(), ana::BayesianMarginal.ToHistogram(), ana::Multiverse.ToSpectrum(), lem::PIDExtraVars.ToString(), ValidCalibPDF.ToString(), rb::Cluster.TotalADC(), caldp::LiteProfile.TotalEntries(), rb::Cluster.TotalGeV(), caldp::LiteProfile.TotalHits(), rb::Prong.TotalLength(), rb::Cluster.TotalPE(), rb::Cluster.TotalWeight(), ana::NDOscCurve.ToTH1(), ana::OscCurve.ToTH1(), ana::Ratio.ToTH1(), ana::Hist.ToTH1(), ana::Spectrum.ToTH1(), ana::Ratio.ToTH2(), ana::ReweightableSpectrum.ToTH2(), ana::ISurface.ToTH2(), ana::Spectrum.ToTH2(), hough::LiteTH2.ToTH2F(), hv::LiteTH2.ToTH2F(), ana.ToTH2Helper(), ana::Spectrum.ToTH3(), ana.ToTH3Helper(), ana.ToTHStack(), ana::MCMCSample.ToTMap(), dt::Cand.ToTrack(), ana.ToUpDownHist(), ana::CutOptimization.ToUpDownHist(), ana::MCMCSamples.Trace(), murem::TrackCleanUpAlg.TrackEinMIP(), lem::dec::Forest.Train(), lem::dec.TransformEvent(), calib::AttenCache.TranslateChannelMC(), ncid::MakeNCCosRej.TransMomFraction(), ncid::NCNNKeras.TransMomFraction(), cosrej::MakeNueCosRej.TransMomFraction(), calib::AttenFit.TruncatedMeanProfile(), calib::AttenuationFit.TruncatedMeanProfile(), ana::UnfoldSVD.Truth(), ana::UnfoldTikhonov.Truth(), UnpackMatrix(), cheat::BackTracker.VecPtrToVecRawPtr(), cheat::BackTracker.VecToVecRawPtr(), ana::Nus17SystFromHist.WeightFor(), ana::NusSystFromHist.WeightFor(), ana::NuISyst.WeightFor(), ana::Nus18SystFromHist.WeightFor(), ana::SpectrumLoaderBase.WildcardOrSAMQuery(), ana::Hist.Zero(), ana::Hist.ZeroSparse(), dt::ViewMerger.ZipVerticalTracks(), rb::Track.ZipWith(), ana::ProfilerSupport.~ProfilerSupport(), and calib::ThresholdCorrMap.~ThresholdCorrMap().

runNovaSAM.retCode = subprocess.call(cmdList)
runNovaSAM.run_match = run_pattern.match(inFileBase)

Definition at line 238 of file runNovaSAM.py.

runNovaSAM.run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")

Definition at line 237 of file runNovaSAM.py.

runNovaSAM.runNum = int(metadata["runs"][0][0])

Definition at line 234 of file runNovaSAM.py.

Referenced by calib::Calibrator.postBeginRun().

runNovaSAM.samweb = samweb_client.SAMWebClient(experiment='nova')

Definition at line 219 of file runNovaSAM.py.

runNovaSAM.skip_match = _skip_pattern.match(file)

Definition at line 408 of file runNovaSAM.py.

runNovaSAM.stderr

Definition at line 927 of file runNovaSAM.py.

runNovaSAM.stdout

Definition at line 927 of file runNovaSAM.py.

runNovaSAM.str

Definition at line 207 of file runNovaSAM.py.

Referenced by createMetadata(), getOutDir(), MakeTransposeJson(), and ModifyParentMetadata().

runNovaSAM.stream = int(metadata["data_stream"])

Definition at line 251 of file runNovaSAM.py.

Referenced by genie::Algorithm.AllowReconfig(), genie::RSHelicityAmpl.Amp20Plus(), genie::ProcessInfo.AsString(), ClassImp(), genie::geometry::FidPolyhedron.clear(), nova::database::Row.Col(), genie::GVldContext.Emax(), genie::RunOpt.EnableBareXSecPreCalc(), novaddt::HitSubtract.endJob(), Expression.Eval(), genie::geometry::FidCylinder.FidCylinder(), genie::geometry::FidSphere.FidSphere(), genie::Spline.GetAsTSpline(), nutools::dbi::Table.GetConnectionTimeout(), rvp::RecVarPID.getMethodNameFromXML(), xnue::XnuePID.getMethodNameFromXML(), rvp::RecVarPID.getVarNamesFromXML(), xnue::XnuePID.getVarNamesFromXML(), genie::PDF.Gluon(), genie::Kinematics.HadSystP4(), genie::Target.HitNucP4(), genie::geometry::PathSegment.IsTrimmedEmpty(), genie::geometry::PlaneParam.IsValid(), genie::AlgId.Key(), genie::AxialFormFactor.Model(), genie::ELFormFactors.Model(), genie::QELFormFactors.Model(), genie::DISStructureFunc.Model(), genie::CacheBranchNtp.Ntuple(), genie::TuneId.OnlyConfiguration(), genie::geometry.operator<(), genie::utils.operator<<(), genie::exceptions.operator<<(), genie::geometry.operator<<(), operator<<(), genie.operator<<(), nova::database.operator<<(), nutools::dbi.operator<<(), nova::dbi.operator<<(), genie::flux.operator<<(), genie::Interaction.PhaseSpacePtr(), genie::exceptions::EVGThreadException.ReturnStep(), genie::GHepParticle.SetLastDaughter(), genie::RegistryItem< T >.SetLocal(), genie::XclsTag.SetNNeutrons(), nutools::dbi::Row.SetVldTimeEnd(), nova::dbi::Row.SetVldTimeEnd(), genie::exceptions::INukeException.ShowReason(), genie::exceptions::NievesQELException.ShowReason(), genie::CacheBranchFx.Spl(), genie::InitialState.TgtPtr(), nova::database::Column.Update(), nutools::dbi::Column.Update(), nova::dbi::Column.Update(), genie::GHepRecord.Vertex(), genie::GEVGDriver.XSecSumSpline(), genie::geometry::FidShape.~FidShape(), genie::flux::GJPARCNuFluxPassThroughInfo.~GJPARCNuFluxPassThroughInfo(), genie::flux::GNuMIFluxPassThroughInfo.~GNuMIFluxPassThroughInfo(), genie::flux::GSimpleNtpAux.~GSimpleNtpAux(), genie::flux::GSimpleNtpEntry.~GSimpleNtpEntry(), genie::flux::GSimpleNtpNuMI.~GSimpleNtpNuMI(), genie::exceptions::InteractionException.~InteractionException(), and genie::utils::T2KEvGenMetaData.~T2KEvGenMetaData().

runNovaSAM.stream_match = stream_pattern.match(inFileBase)

Definition at line 263 of file runNovaSAM.py.

runNovaSAM.stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)

Definition at line 262 of file runNovaSAM.py.

string runNovaSAM.streamEntry = 't{0:02d}'

Definition at line 252 of file runNovaSAM.py.

runNovaSAM.subNum = int(metadata["runs"][0][1])

Definition at line 235 of file runNovaSAM.py.

int runNovaSAM.subversion = os.environ["NPASS"]

Definition at line 305 of file runNovaSAM.py.

runNovaSAM.tier = outTier.split(":")[1]
runNovaSAM.tmpdir

Definition at line 977 of file runNovaSAM.py.

runNovaSAM.tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")

Definition at line 322 of file runNovaSAM.py.

runNovaSAM.tmpSecondFclName

Definition at line 981 of file runNovaSAM.py.

runNovaSAM.True

Definition at line 965 of file runNovaSAM.py.

runNovaSAM.type

Definition at line 205 of file runNovaSAM.py.