runNovaSAM Namespace Reference

Functions
def	unCamelCase (s)
def	createMetadata (inFile)
def	getOutDir (pathname, hashDirs=False)
def	checkAndMoveFiles (inFile, declareFiles)
def	declareFile (fileWPath)
def	copyOutFiles (hashDirs=False)
def	getOutputFileName (detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
def	resolveFclPath (fcl)
def	setVMemLimit ()
def	makeDirSafely (dir)
def	makeDirIfNeeded (dir)
def	getOutDir (pathname, dest, hashDirs=False, runDirs=False, runNum=0)
def	checkAndMoveFiles (inFile, outputs, noCleanup=False)
def	declareLogFile (logName, logFile, rootName)
def	fileExists (outPath, dh)
def	listFiles (outPath, dh)
def	makeMetadataJSONs ()
def	MakeTransposeJson (TransposeList)
def	copyOutFiles (dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False)
def	makeDeCAF (script, fname, special)

Variables
	gErrorIgnoreLevel
	detectors = dict()
	_lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
	_skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
	parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')
	help
	type
	str
	action
	args = parser.parse_args()
	inFile = args.inFile
	samweb = samweb_client.SAMWebClient(experiment='nova')
	inFileBase = os.path.basename(inFile)
	metadata = samweb.getMetadata(inFileBase)
	runNum = int(metadata["runs"][0][0])
	subNum = int(metadata["runs"][0][1])
	run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")
	run_match = run_pattern.match(inFileBase)
	stream = int(metadata["data_stream"])
string	streamEntry = 't{0:02d}'
	stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)
	stream_match = stream_pattern.match(inFileBase)
	detector = metadata["online.detector"].lower()
	detId = metadata["nova.detectorid"].lower()
	release = os.environ["SRT_BASE_RELEASE"]
	file_type = metadata["file_type"]
string	dataFlag = "data"
	subversion = os.environ["NPASS"]
	nevt = None
	fclFile = args.config
	fclPath = resolveFclPath(fclFile)
	tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
	fclFileObj = open(tmpFclName, 'a')
list	cmdList = ['nova']
	outTier
	cafTier
list	outList = []
	output = outTier.split(":")[0]
	tier = outTier.split(":")[1]
	outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
	cafLabel = cafTier.split(":")[0]
	cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
string	cmd = ' '
	retCode = subprocess.call(cmdList)
	dirList = os.listdir(".")
	skip_match = _skip_pattern.match(file)
	dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
	argvCopy
	Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help. More...
	argv
	description
	default
	False
	experiment
	fileMetaDataMgr
	PassFile
	isRunningMetadataModule
	shell
	doMeta
	flatTier
	h5Tier
	precopyscript
	outputs
	outP
	outC
	outXp
	outXc
	outYp
	outYc
	NewOMod
	outX
	outY
	outNameTemp
	FirstRun
	LastRun
	Index
	NewOName
	flatLabel
	flatName
	h5Label
	h5Name
	histName
	olddir
	Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files? More...
	allFiles
	failFiles
	copyfile
	ifdhcmd
	ret
	gdbArgs
	stdout
	logfile
	stderr
	decafdir
	If using a txtfiledef make sure to clean up the InputFile List.... More...
	dest

Function Documentation

def runNovaSAM.checkAndMoveFiles	(		inFile,
			outputs,
			noCleanup = False
	)

Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 88 of file runNovaSAM.py.

References checkAndMoveFiles(), MetadataUtils.createMetadata(), declareFile(), and hadd_reco_validation.group.

def checkAndMoveFiles(inFile, outputs, noCleanup=False):
    """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
    inFileBase = os.path.basename(inFile)
    baseDir = "."
    # If declaring files have to do non-decafs first, since they're the parents
    # of the decafs. If not, it doesn't hurt to do it in that order anyway.
    for decafPass in [False, True]:
        for root, dirs, filenames in os.walk(baseDir):
            if root == baseDir:
                for file in filenames:
                    if file.endswith('decaf.root') != decafPass: continue
                    # If file not in outputs, then delete.
                    if (file.endswith (".root") or file.endswith(".h5")) and file != inFileBase:
                        fileWPath = os.path.join(root, file)
                        # If the file isn't in my outputs then delete it.
                        if file not in outputs:
                            if not noCleanup:
                                # Do not copy over, just delete
                                print "File", fileWPath, " is not among requested outputs, removing"
                                os.remove(fileWPath)
                            continue
                        # Now that I know that this file is in my output list, lets check it is valid...
                        # First, if it is a root file.
                        if file.endswith (".root"):
                            print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
                            rootFile = ROOT.TFile(fileWPath)
                            if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
                                #do not copy over, just delete
                                print "File", fileWPath, "is Zombie - remove it"
                                os.remove(fileWPath)
                            else:
                                newFilePath = os.path.join(root, "results", file)
                                print "New file name is %s" %newFilePath
                                os.renames(fileWPath, newFilePath)
                            rootFile.Close()
                        # Next, if it is a h5 file.
                        elif file.endswith(".h5"):
                            # Only import h5py if have h5 files.
                            import h5py
                            print "In checkAndMoveFiles, fileWPath is %s" %fileWPath
                            # If a valid HDF5 files.
                            if h5py.is_hdf5(fileWPath):
                                newFilePath = os.path.join(root, "results", file)
                                os.renames(fileWPath, newFilePath)
                                print "Moved ", fileWPath, "to ", newFilePath
                                # If not a valid HDF5 file.
                            else:
                                print "File", fileWPath, "is Zombie - remove it"
                                os.remove(fileWPath)
    return

#def h5MetaHack(inFile):
#    print "Doing a hack to get HDF5 metadata...Will take CAF metadata and change subtly."
#    ### Make my client and get my CAF metadata
#    samweb = samweb_client.SAMWebClient(experiment='nova')
#    md = MetadataUtils.createMetadata( inFile.replace('.h5caf.h5', '.caf.root') )
#    ### Change some parameters.
#    md['file_name'] = inFile
#    md['data_tier'] = unicode('h5')
#    md['file_size'] = os.path.getsize( inFile )
#    ### return md
#    return md

def runNovaSAM.checkAndMoveFiles	(		inFile,
			declareFiles
	)

Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 105 of file runNovaSAM.py.

References declareFile().

Referenced by checkAndMoveFiles().

def checkAndMoveFiles(inFile, declareFiles):
    """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
    inFileBase = os.path.basename(inFile)
    baseDir = "."
    for root, dirs, filenames in os.walk(baseDir):
        if root == baseDir:
            for file in filenames:
                if file.endswith (".root") and file != inFileBase:
                    fileWPath = os.path.join(root, file)
                    rootFile = ROOT.TFile(fileWPath)
                    if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
                        #do not copy over, just delete 
                        print "File", fileWPath, "is Zombie - remove it"
                        os.remove(fileWPath)
                    else:
                        if declareFiles == True:
                            declareFile(fileWPath)
                        newFilePath = os.path.join(root, "results", file)
                        os.renames(fileWPath, newFilePath)
                    rootFile.Close()
    return

def runNovaSAM.copyOutFiles

(

hashDirs = False

)

Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 149 of file runNovaSAM.py.

References getOutDir().

Referenced by copyOutFiles().

def copyOutFiles(hashDirs=False):
    """Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
    print "Copying out files"
    dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
    baseDir = "./results"
    for root, dirs, filenames in os.walk(baseDir):
        if root == baseDir:
            for file in filenames:
                if file.endswith (".root") and file != inFileBase:
                    fileWPath = os.path.join(root, file)
                    dest = getOutDir(file, hashDirs)
                    skip_match = _skip_pattern.match(file)
                    if skip_match == None:
                        try:
                            print "Checking if", dest + "/" + file, "exists"
                            dh.ls(dest + "/" + file, 1, "")
                        except RuntimeError:
                            print "It doesn't - copy", fileWPath, "to", dest
                            dh.cp(["-D", fileWPath, dest])
                    print "Removing", fileWPath
                    os.remove(fileWPath)
    dh.cleanup()
    return

def runNovaSAM.copyOutFiles	(		dest,
			hashDirs = False,
			runDirs = False,
			runNum = 0,
			noCleanup = False,
			declareLocation = False,
			declareLogs = False
	)

Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 361 of file runNovaSAM.py.

References copyOutFiles(), declareFile(), declareLogFile(), and getOutDir().

def copyOutFiles(dest, hashDirs=False, runDirs=False, runNum=0, noCleanup=False, declareLocation=False, declareLogs=False):
    """Builtin facility to copy out art files. This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
    dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
    baseDir = "./results"
    declareFiles = declareLogs
    for root, dirs, filenames in os.walk(baseDir):
        if root == baseDir:
            for file in filenames:
                if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
                    fileWPath = os.path.join(root, file)
                    outDir = getOutDir(file, dest, hashDirs, runDirs, runNum)
                    skip_match = _skip_pattern.match(file)
                    if skip_match == None:
                        outPath = os.path.join(outDir, file)

                        # note: this will fail if the file already exists
                        returnValue = dh.cp(["-D", fileWPath, outDir])
                        if returnValue != 0:
                            print >> sys.stderr, "Copy out failed for file:", fileWPath
                            print >> sys.stderr, "Skipping it."
                        else:
                            if declareFiles:
                                declareFile(fileWPath)
                        ###################
                        # Declare the file's location to SAM if we have the declareLocation option on
                        if declareLocation==True :
                            loc = string.replace(outDir, 's3://','s3:/')
                            print "Declaring location %s for file %s\n" % (loc,file)
                            sam = samweb_client.SAMWebClient('nova')
                            ret=sam.addFileLocation(file, loc)
                            if ret.status_code != 200 :
                                print "  SAMWEB Unable to declare file location (%s, %s) status code %s" %(file, loc, ret.status_code)
                        if fileWPath.endswith (".root"):
                            jsonPath = fileWPath.replace('.root', '.json')
                        elif fileWPath.endswith (".h5"):
                            jsonPath = fileWPath + '.json'
                        if os.path.isfile(jsonPath):
                            returnValue = dh.cp(['-D', jsonPath, outDir])
                            if returnValue != 0:
                                print >> sys.stderr, "Copy out failed for file: " + jsonPath
                                print >> sys.stderr, "Skipping it."

                        for ext in ['.bz2', '']:
                            if os.path.isfile('log.txt'+ext):
                                if file.endswith (".root"):
                                    logName = file.replace('.root', '.log'+ext)
                                elif file.endswith (".h5"):
                                    logName = file + '.log'+ext
                                returnValue = dh.cp(['log.txt'+ext, outDir+'/'+logName])

                                if returnValue != 0:
                                    print >> sys.stderr, "Copy out failed for file: " + logName
                                    print >> sys.stderr, "Skipping it."
                                if declareLogs:
                                    declareLogFile(logName, 'log.txt'+ext, file)

                                # Remove the copied-out log so it's not in
                                # the way for new log creation.
                                os.remove('log.txt'+ext)

                                break

                        else:
                          print "It does exist, not copying."
                    if not noCleanup:
                      print "Removing", fileWPath
                      os.remove(fileWPath)

                      if fileWPath.endswith(".root"):
                          jsonPath = fileWPath.replace('.root', '.json')
                      elif fileWPath.endswith(".h5"):
                          jsonPath = fileWPath + ".json"
                      if os.path.isfile(jsonPath):
                          print 'Removing', jsonPath
                          os.remove(jsonPath)
    return

def runNovaSAM.createMetadata

(

inFile

)

Definition at line 36 of file runNovaSAM.py.

References samweb_client.utility.fileEnstoreChecksum(), site_stats_from_log.get, str, and unCamelCase().

Referenced by declareFile().

def createMetadata(inFile):
    filename = os.path.basename(inFile)
    filesize = os.path.getsize(inFile)
    
    try:
        data = subprocess.check_output(["sam_metadata_dumper", str(inFile)])
    except subprocess.CalledProcessError:
        print "sam_metadata_dumper failed!"
        return None

    md = {'file_name': filename, 'group': 'nova', 'file_size': filesize, 'file_format': 'root'}

    # An ad-hoc parser for the output
    mode = 'new'
    for line in data.split('\n'):
        m = _lineregex.match(line)
        if mode == 'json':
            if not m and not line.startswith('-'):
                data += line
            else:
                md[key] = json.loads(data)
                mode = 'new'

        if mode == 'new':
            if m:
                key = m.group(1)
                value = m.group(2)

                if '.' not in key:
                    key = unCamelCase(key)
                    
                    # ART does not appear to set the application name
                    # but does provide a process name
                if key == 'process_name' and md.get('application',[]).get('name') is None:
                    key = 'application_name'
                elif key == 'stream_name':
                    key = 'data_stream'
                    
                if key in ('start_date','end_date'):
                    value = long(value)
                    
                if key == 'runs' or key == 'parents':
                    mode = 'json'
                    data = value
                elif key.startswith('application_'):
                    # special handling for applications
                    if 'application' not in md:
                        md['application'] = {}
                    md['application'][key[12:]] = value
                else:
                    md[key] = value
                    
    md['crc'] = fileEnstoreChecksum(inFile)
    
    return md

def runNovaSAM.declareFile

(

fileWPath

)

Checks file for a TKey of RootFileDB.  If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM

Definition at line 127 of file runNovaSAM.py.

References createMetadata().

Referenced by checkAndMoveFiles(), and copyOutFiles().

def declareFile(fileWPath):
    """Checks file for a TKey of RootFileDB.  If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM"""
    samweb = samweb_client.SAMWebClient(experiment='nova')
    rootFile = ROOT.TFile(fileWPath)
    if rootFile.FindKey("RootFileDB"):
        print "Found key (RootFileDB): constructing metadata for", fileWPath
        md = createMetadata(fileWPath)
        if (md == None):
            print "No metadata found!"
        else:
            pprint.pprint(md)
            print ''
            print "Declaring", fileWPath, "to SAM"
            try:
                samweb.declareFile(md)
            except:
                print fileWPath, "already exists in SAM"
    else:
        print fileWPath, "does not contain RootFileDB, do not try to declare"
    rootFile.Close()
    return

def runNovaSAM.declareLogFile	(		logName,
			logFile,
			rootName
	)

Definition at line 205 of file runNovaSAM.py.

Referenced by copyOutFiles().

def declareLogFile(logName, logFile, rootName):
    # This is what NovaFTS/plugins/nova_log_metadata.py does
    md = {
        'file_name': os.path.basename(logName),
        'file_size': os.path.getsize(logFile),
        'data_tier': 'log',
        'file_format': 'log',
        'file_type': 'unknown', # maybe 'nonPhysicsGeneric'?
        'parents': [{'file_name': os.path.basename(rootName)}]
        }
    print "16) declareLogFile(logName, logFile, rootName):"
    print "Declaring", logName, "to SAM"
    try:
        samweb = samweb_client.SAMWebClient(experiment='nova')
        samweb.declareFile(md)
    except Exception as inst:
        print inst

def runNovaSAM.fileExists	(		outPath,
			dh
	)

Definition at line 224 of file runNovaSAM.py.

def fileExists(outPath, dh):
  try:
    # Check to see if you can ls the file.
    # This works in ifdh 1_7_0 and newer
    # If it exists, dh.ls() returns a tuple with one entry, converts to True
    # If it doesn't, dh.ls() returns an empty tuple, converts to False
    return bool(dh.ls(outPath, 1, ""))
  except (RuntimeError, IOError) as e:
    # Some versions of ifdh throw an exception when the file was not found
    # But that means it is not there.
    return False

def runNovaSAM.getOutDir	(		pathname,
			dest,
			hashDirs = False,
			runDirs = False,
			runNum = 0
	)

Definition at line 73 of file runNovaSAM.py.

References getOutDir(), parse_dependency_file_t.list, makeDirSafely(), and str.

def getOutDir(pathname, dest, hashDirs=False, runDirs=False, runNum=0):
    dirs = [dest]
    if hashDirs:
      head, tail = os.path.split(pathname)
      hash = md5.new(tail)
      dirs += list(hash.hexdigest()[:3])
    if runDirs:
      head, tail = os.path.split(pathname)
      runStr = str(runNum)
      multiRunDir = runStr[:3].zfill(6)
      makeDirSafely(os.path.join(dest, multiRunDir))
      makeDirSafely(os.path.join(dest, multiRunDir, runStr))
      dirs += [multiRunDir, runStr]
    return os.path.join(*dirs)

def runNovaSAM.getOutDir	(		pathname,
			hashDirs = False
	)

Definition at line 92 of file runNovaSAM.py.

References exit(), and parse_dependency_file_t.list.

Referenced by copyOutFiles(), and getOutDir().

def getOutDir(pathname, hashDirs=False):
    if "DEST" in os.environ:
        dest = os.environ["DEST"]
    else:
        print "No destination set!"
        exit(1)
    dirs = [dest]
    if hashDirs:
      head, tail = os.path.split(pathname)
      hash = md5.new(tail)
      dirs += list(hash.hexdigest()[:3])
    return os.path.join(*dirs)

def runNovaSAM.getOutputFileName	(		detector,
			runNum,
			subNum,
			streamEntry,
			release,
			subversion,
			dataFlag,
			tier,
			nevt
	)

Definition at line 173 of file runNovaSAM.py.

References exit(), and novadaq::HexUtils.format().

def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt):
  if dataFlag == "sim":
    pattern = re.compile(r"^.*?_(.*?)_\d+.*_v\d+_(.*?)(\.|\_)sim\..+")
    match = pattern.match(inFileBase)
    if match != None:
      simLabel = match.groups()[0]
      uniq     = match.groups()[1]
      outName = '{0}_{1}_{2}_r{3:08d}_s{4:02d}_{5}_v{6}_{7}_{8}.{9}.root'.format(detector, simLabel, nevt, runNum, subNum, release, subversion, uniq, dataFlag, tier)
    else:
      print "regex couldn't find the sim label and the uniqueifier"
      exit(1)
  else:
    outName = '{0}_r{1:08d}_s{2:02d}_{3}_{4}_v{5}_{6}.{7}.root'.format(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier)
  return outName
  

def runNovaSAM.listFiles	(		outPath,
			dh
	)

Definition at line 236 of file runNovaSAM.py.

def listFiles(outPath, dh):
    try:
        return dh.ls(outPath, 1, "")
    except:
        return "Exception while trying to ls, OutPath =",outPath

def runNovaSAM.makeDeCAF	(		script,
			fname,
			special
	)

Definition at line 438 of file runNovaSAM.py.

References novadaq::HexUtils.format(), if(), and resolveFclPath().

def makeDeCAF(script, fname, special):
    trimname = fname[:-5] # cut off .root from end
    trimidx = trimname.rindex('.')+1 # find last period before .root
    decaf_tier = trimname[trimidx:-3]+'de'+trimname[-3:] # properly insert 'de'
    oname = '{0}_{1}.{2}.root'.format(trimname, special, decaf_tier)
    novaSource = os.getenv("SRT_PUBLIC_CONTEXT", "undefined")
    if(novaSource == "undefined"):
        novaSource = os.getenv("NOVASOFT_DIR", "undefined")
        if(novaSource == "undefined"):
           NGU.fail("Unable to locate NOvA source code")
        else:
           novaSource = os.getenv("NOVASOFT_DIR") + "/source"

    os.system('cafe -bq '+novaSource+'/CAFAna/'+script+' '+fname+' '+oname+' 2>&1')
    return oname

def runNovaSAM.makeDirIfNeeded

(

dir

)

Definition at line 61 of file runNovaSAM.py.

Referenced by makeDirSafely().

def makeDirIfNeeded(dir):
  os.umask(002)  #  (rw-rw-r--) for files and (rwxrwxr-x) for directories.
  if not os.path.isdir(dir):
    print "runNovaSAM is making a directory: ", dir
    try:
      os.mkdir(dir)
    except:
      print "Couldn't make the directory... some other job perhaps did, or permissions did not allow "
  if not os.path.isdir(dir):
    raise Exception("Failed to make directory + "  + dir )

def runNovaSAM.makeDirSafely

(

dir

)

Definition at line 41 of file runNovaSAM.py.

References makeDirIfNeeded().

Referenced by getOutDir().

def makeDirSafely(dir):
  if "/pnfs" != dir[0:5]:
      makeDirIfNeeded(dir)
      return

  print "runNovaSAM is making a directory with IFDH"
  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
  try:
    print "Checking if directory ", dir, "exists"
    dh.ls(dir, 1, "")
    dh.chmod(774,dir,"")

  except (RuntimeError, IOError) as e:
    try:
      print "It doesn't - make directory", dir
      dh.mkdir(dir)
    except:
      print "Tried to make directory and couldn't.  Perhaps it already exists?"

def runNovaSAM.makeMetadataJSONs

(

)

For every ROOT file, try to extract its metadata into a matching .json
file in the same directory

Definition at line 243 of file runNovaSAM.py.

References if(), MakeTransposeJson(), open(), and PandAna.Demos.demo1.range.

def makeMetadataJSONs():
    """For every ROOT file, try to extract its metadata into a matching .json
    file in the same directory"""
    baseDir = "./results"
    # If using --txtfiledef, want to pass multiple files at a time to sam_meta_dumper.
    # This is because it simply takes too long to call it for each indiv file
    #  when there are 800+ outputs. 5 s each -> ~1 hour!
    TransposeList = ""
    # Declare the samweb client within this function
    samweb = samweb_client.SAMWebClient(experiment='nova')
    # Loop through directories to search for files to make json files for.
    for root, dirs, filenames in os.walk(baseDir):
        if root == baseDir:
            # Push the h5 files to the front of the list so that the CAF files remain
            # available to hack in the metadata.
            for ifile in range(len(filenames)):
                if("h5" in filenames[ifile]):
                    filenames = [filenames[ifile]] + filenames[:ifile] + filenames[ifile+1:]
            for file in filenames:
                if (file.endswith (".root") or file.endswith(".h5") ) and file != inFileBase:
                    skip_match = _skip_pattern.match(file)
                    if skip_match == None:
                        # Set some quick and useful variables.
                        olddir = os.getcwd()
                        fileWPath = os.path.join(root, file)
                        # If a transpose file want to make the json subtly differently.
                        if args.txtfiledef and "outplane" in file:
                            # This works for the cellmode of the transposer as well, due to the filename including outplane and cell.
                            print "Adding %s to TransposeList" %file
                            TransposeList += "%s " %file
                            continue
                        # Which extractor am I using?
                        extractor = 'sam_metadata_dumper'
                        if file.endswith('caf.root'):
                            extractor = 'extractCAFMetadata'
                        elif file.endswith('.h5'):
                            extractor = 'extractHDF5Metadata'
                        try:
                            # sam_metadata_dumper doesn't apply basename() to filename. https://cdcvs.fnal.gov/redmine/issues/8987
                            os.chdir(os.path.dirname(fileWPath))
                            meta = subprocess.check_output([extractor, os.path.basename(fileWPath)])
                            if file.endswith (".root"):
                                
                                jsonf = open(file.replace('.root', '.json'), 'w')
                                jsonf.write(meta)
                                jsonf.close()
                                print "Made metadata for %s" %file
                            elif file.endswith(".h5"):
                                print "\nNow to make my json file for my h5...\n"
                                jsonf = open(file.replace('.h5caf.h5', '.h5caf.json'), 'w')
                                jsonf.write(meta)
                                jsonf.close()
                                print "Made metadata for %s" %file
                            else:
                                print "I'm not sure what file extension you have..."
                        except:
                            print "Error extracting metadata from file."
                        finally:
                            os.chdir(olddir)
    # Make the Transpose json files.
    # Again same argument, outplane is already in the filenames for cell mode
    if args.txtfiledef and "outplane" in file:
        olddir = os.getcwd()
        os.chdir(baseDir)
        MakeTransposeJson( TransposeList  )
        os.chdir(olddir)

def runNovaSAM.MakeTransposeJson

(

TransposeList

)

Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
This takes in a list of files, and makes appropriate .json files in the same directory

Definition at line 310 of file runNovaSAM.py.

References hadd_reco_validation.group, open(), PandAna.Demos.demo1.range, and split().

Referenced by makeMetadataJSONs().

def MakeTransposeJson( TransposeList ):
    """Transpose files need some extra tweaking when making .json files, largely because there are so many of them.
    This takes in a list of files, and makes appropriate .json files in the same directory"""
    # If using --txtfiledef, I can now pass my file list to sam_meta_dumper.
    print "Is MakeTransposeJson called without a txt def, if you see this then yes."
    MetaListFile="AllMetaJson.txt"
    meta_cmd="sam_metadata_dumper -s " + TransposeList + " > " + MetaListFile
    os.system(meta_cmd)
    # Now want to open the file and split by "}," character.
    MetaFile  = open( MetaListFile )
    MetaLines = MetaFile.read().split(" },")
    # Loop through lines, and appropriately separate out json files.
    for i in range(0,len( MetaLines ) ):
        meta=MetaLines[i]
        # Figure out file name...this is assuming that file name is always the first entry...
        StName=re.search('"(.+?).root"', meta ).group(1)
        filename=StName+".json"  # Effecitvely replacing .root with .json
        # If transpose file add PlaneNumber and CellNumber if run in cell mode
        if "cell" in filename:
            try:
                plane = re.search('^.*outplane(\d*).', filename).group(1)
                cell = re.search('^.*cell(\d*).', filename).group(1)
                print "Plane number:", plane, "Cell number:", cell, ". Is a transpose file"
                meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n    "calibration.CellNumber": "%s",\n    "calibration.base_release"')%(plane, cell)
            except:
                print "No cell number found - could be a plane mode transpose file"

        elif "outplane" in filename:
            try:
                plane = re.search('^.*outplane(\d*).', filename).group(1)
                print "Plane number:", plane, ". Is a transpose file"
                meta = meta.replace('"calibration.base_release"', '"calibration.PlaneNumber": "%s",\n    "calibration.base_release"') %plane
            except:
                print "Error extracting plane number from transpose file."

        ### Make sure that the txtfile is in the parent list.
        meta['parents'].append({u'file_name':unicode(inFile)})

        # Now open the json file
        fout=open(filename,'w')
        # Want to make sure that the json starts with '{'
        if meta[0] not in "{":
            meta = meta[:0] + '{\n' + meta[1:]
        # Want to make sure that the json ends with a double '}'
        if i < len(MetaLines)-1:
            meta += "}\n}\n"
        # Write and close the json file
        fout.write(meta)
        fout.close()

def runNovaSAM.resolveFclPath

(

fcl

)

Definition at line 188 of file runNovaSAM.py.

Referenced by makeDeCAF().

def resolveFclPath(fcl):
   # Check if we have an absolute path name, return it if so
  if fcl[0] == "/":
    return fcl 

  # Otherwise, we need to do some searching 
  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
  for path in fclPaths:
    if os.path.isfile(path + fcl):
      return path + fcl
      
  # If we haven't found it, we have a problem.
  raise IOError(sys.argv[0] + ": config  file "+ fcl+" not found in FHICL_FILE_PATH")
    

def runNovaSAM.setVMemLimit

(

)

Definition at line 33 of file runNovaSAM.py.

def setVMemLimit():
    #memLimit = 3.9*1024**3
    memLimit = 3.9*1000**3
    print "Old virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)
    print "Limiting the virtual memory of the nova job to 3.9 GB"
    resource.setrlimit(resource.RLIMIT_AS, (memLimit, memLimit))
    print "New virtual memory limit:", resource.getrlimit(resource.RLIMIT_AS)

def runNovaSAM.unCamelCase

(

s

)

Convert CamelCase to camel_case 

Definition at line 29 of file runNovaSAM.py.

Referenced by createMetadata().

def unCamelCase(s):
    """ Convert CamelCase to camel_case """
    # \B matches an empty string which is NOT at the beginning of a word
    # so requiring this means no _ will be inserted at the start
    return re.sub( r'\B([A-Z])', r'_\1', s).lower()
                

Variable Documentation

runNovaSAM._lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')

private

Definition at line 24 of file runNovaSAM.py.

runNovaSAM._skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)

private

Definition at line 27 of file runNovaSAM.py.

runNovaSAM.action

Definition at line 207 of file runNovaSAM.py.

Referenced by CondenseRunList(), art::Worker.doWork(), plotTime(), art::EndPathExecutor.process(), art::Schedule.process(), art::EventProcessor.process_(), and HEPREP::HepRepWriter.~HepRepWriter().

runNovaSAM.allFiles

Definition at line 802 of file runNovaSAM.py.

runNovaSAM.args = parser.parse_args()

Definition at line 215 of file runNovaSAM.py.

Referenced by art.add_opt(), ana::ThreadPool.AddMemberTask(), art::ServiceDirector.addSystemService(), ana::ThreadPool.AddTask(), cet::PluginFactory.call(), lem::SortedMatchList.emplace(), shims::map< Key, T, Compare, Allocator >.emplace(), art::RangeSet.emplace_range(), lem::LEMWorker.GetWork(), lem.GetWorkThreadFunc(), art::RPManager.invoke(), art::GlobalSignal< SRTYPE, ResultType(Args...)>.invoke(), art::LocalSignal< STYPE, ResultType(Args...)>.invoke(), art::TFileDirectory.make(), art::gfactory.make_group(), art::TFileDirectory.makeAndRegister(), cet::BasicPluginFactory.makePlugin(), art::detail.makeWatchFunc(), stan::lang::addition_expr3.operator()(), stan::lang::subtraction_expr3.operator()(), stan::lang::binary_op_expr.operator()(), stan::lang::exponentiation_expr.operator()(), stan::lang::multiplication_expr.operator()(), stan::lang::division_expr.operator()(), stan::lang::modulus_expr.operator()(), stan::lang::left_division_expr.operator()(), stan::lang::elt_multiplication_expr.operator()(), stan::lang::elt_division_expr.operator()(), stan::lang::negate_expr.operator()(), stan::lang::logical_negate_expr.operator()(), stan::lang::transpose_expr.operator()(), lem::LEMClient.produce(), lem::LEMWorkerOutput.ReturnResults(), lem.ReturnResultsThreadFunc(), ana.SymmMxInverse(), TEST(), lem::dec.thread_func(), lem::dec::Forest.Train(), and ana.VarFromNOvARwgt().

runNovaSAM.argv

Definition at line 24 of file runNovaSAM.py.

runNovaSAM.argvCopy

Do a little bit of trickery with sys.argv to stop ROOT from gobbling it up and killing –help.

Definition at line 23 of file runNovaSAM.py.

runNovaSAM.cafLabel = cafTier.split(":")[0]

Definition at line 362 of file runNovaSAM.py.

runNovaSAM.cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)

Definition at line 366 of file runNovaSAM.py.

runNovaSAM.cafTier

Definition at line 342 of file runNovaSAM.py.

string runNovaSAM.cmd = ' '

Definition at line 383 of file runNovaSAM.py.

Referenced by DispatcherCMD.addCommand(), nova::dbi::Util.CheckConnection(), nutools::dbi::Util.CheckConnection(), dbutils::Krb5.checkTicket(), nova::database::Table.CheckValidXMLDef(), nova::dbi::Table.CheckValidXMLDef(), configure(), create_test_file(), nova::database::Table.CreateFromDB(), DispatcherCMD.DispatcherCMD(), nova::database::Table.ExistsInDB(), nova::dbi::Table.ExistsInDB(), nutools::dbi::Table.ExistsInDB(), genie::masterclass::GNuMcMainFrame.FileOpen(), dbutils::gui::DCSConfigEditorDisplay.generateRecipeScripts(), nova::database::Table.GetCol(), nova::dbi::Table.GetCol(), nutools::dbi::Table.GetCol(), nutools::dbi::Table.GetColsFromDB(), nova::database::Table.GetConnection(), nutools::dbi::Table.GetConnection(), nova::dbi::Table.GetConnection(), nova::database::Table.GetCurrSeqVal(), nova::dbi::Table.GetCurrSeqVal(), nutools::dbi::Table.GetCurrSeqVal(), rvp::RecVarPID.getMethodNameFromXML(), xnue::XnuePID.getMethodNameFromXML(), nova::database::Table.GetRowByColSet(), dbutils::Krb5.getTicket(), rvp::RecVarPID.getVarNamesFromXML(), xnue::XnuePID.getVarNamesFromXML(), caf::CAFMaker.InitializeOutfile(), joint_fit_2017_fc_submit(), dbutils::RunHistory.Load(), nova::database::Table.LoadFromDB(), nova::dbi::Table.LoadFromDB(), nutools::dbi::Table.LoadFromDB(), OnMonPlotMaker(), PrintSyntax(), daq2raw::NOvASocketInputDriver.readFile(), jmshower::RecoJMShower.RecoShowers(), request_event(), request_xsec(), DispatcherCMD.setCommandDefault(), dbutils::daqappmgr::ApplicationType.setRshCommandForStart(), dbutils::daqappmgr::ApplicationType.setRshCommandForStop(), dbutils::daqappmgr::ApplicationType.setStartCommand(), dbutils::daqappmgr::ApplicationType.setStopCommand(), nova::database::Table.SetValiditySQL(), nova::dbi::Table.SetValiditySQL(), nutools::dbi::Table.SetValiditySQL(), stanc_helper(), nutools::dbi::Table.Table(), nova::dbi::Table.Table(), nova::database::Table.WriteToDB(), nova::dbi::Table.WriteToDB(), and nutools::dbi::Table.WriteToDB().

list runNovaSAM.cmdList = ['nova']

Definition at line 330 of file runNovaSAM.py.

runNovaSAM.copyfile

Definition at line 808 of file runNovaSAM.py.

string runNovaSAM.dataFlag = "data"

Definition at line 297 of file runNovaSAM.py.

runNovaSAM.decafdir

If using a txtfiledef make sure to clean up the InputFile List....

Definition at line 862 of file runNovaSAM.py.

runNovaSAM.default

Definition at line 522 of file runNovaSAM.py.

runNovaSAM.description

Definition at line 473 of file runNovaSAM.py.

runNovaSAM.dest

Definition at line 880 of file runNovaSAM.py.

runNovaSAM.detector = metadata["online.detector"].lower()

Definition at line 272 of file runNovaSAM.py.

Referenced by daq2raw::DAQHit2Raw.beginRun(), BuildMetricsTree(), DCSPlotMaker(), cmf::MetaData.DetectorString(), novaddt::utils::DetectorUtils.DetectorUtils(), daqchannelmap::NearDetDAQChannelMap.encodeDChan(), daqchannelmap::FarDetDAQChannelMap.encodeDChan(), daqchannelmap::TestBeamDAQChannelMap.encodeDChan(), daqchannelmap::DAQChannelMapOLD.encodeDChan(), daqchannelmap::NearDetDAQChannelMap.encodeLChan(), daqchannelmap::FarDetDAQChannelMap.encodeLChan(), daqchannelmap::TestBeamDAQChannelMap.encodeLChan(), meta::Metadata.FillMetadata(), daqchannelmap::DAQChannelMapOLD.getBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getBlockType(), GetCosmics(), daqchannelmap::DAQChannelMapOLD.getDCMLinkOffline(), daqchannelmap::DAQChannelMapOLD.getDCMOffline(), daqchannelmap::DAQChannelMapOLD.getDiBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockType(), daqchannelmap::DAQChannelMapOLD.getDiBlockTypeOffline(), daqchannelmap::DAQChannelMapOLD.getFirstDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLastDCMInViewOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInBlockOffline(), daqchannelmap::DAQChannelMapBaseOLD.getLocalPlaneInBlockOrDiblock(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockOffline(), daqchannelmap::DAQChannelMapOLD.getLocalPlaneInDiblockViewOffline(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInXPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfChannelsInYPlane(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesBeforeDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInBlock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfPlanesInDiblock(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfXDCM(), daqchannelmap::DAQChannelMapBaseOLD.getNumberOfYDCM(), daqchannelmap::DAQChannelMapOLD.getOnlineCellOffline(), daqchannelmap::DAQChannelMapOLD.getPixelOffline(), daqchannelmap::DAQChannelMapOLD.getPlaneViewOffline(), GetSystsFromFile(), daqchannelmap::DAQChannelMapBaseOLD.getTotalNumberOfDiblocks(), GoodRuns(), daqchannelmap::DAQChannelMapBaseOLD.initNumberOfDCMsInDiblockDetector(), nova::dbi::RunHistory.LoadDAQRunHistory(), main(), make_fc_nus_surfs_nersc_2019(), MakeCovarSim(), MakeSurface(), NewNearlinePlots(), NuMu2019_BasicPIDPlots_FD(), NuMu2019_BasicPIDPlots_ND(), NuMu2019_BasicPIDPlots_Spectrum(), NuMu2020_BasicPIDPlots_FD(), NuMu2020_BasicPIDPlots_ND(), NuMu2020_BasicPIDPlots_Spectrum(), rawdata::RawBeamlineTrigger.RawBeamlineTrigger(), rawdata::RawBeamlineWC.RawBeamlineWC(), fnex::ReadCAFMakeEventList.ReadCAFTree(), om::HistoTable.ReadFile(), rawdata::RawBeamlineTrigger.SetDetectorID(), rawdata::RawBeamlineWC.SetDetectorID(), ana.TAHadEND(), and rawdata::RawBeamlineWC.~RawBeamlineWC().

runNovaSAM.detectors = dict()

Definition at line 18 of file runNovaSAM.py.

Referenced by fnex::Correlations.ApplyCorrelations(), cmf::CovarianceMatrixMaker.beginJob(), cmf::CovarianceBinUtility.CheckBinToEnergyMapping(), cmf::CalibSystHistMaker.CreateHistograms(), eventListToTextFile(), cmf::CovarianceMatrixDecorrelator.GetComponentNameAndOffsets(), cmf::CalibSyst.LoadCalibSystRatios(), fnex::CorrSpec_NoExtrapNuMu.MakeCorrectedHistograms(), fnex::CorrSpec_PropDecomp.MakeCorrectedHistograms(), fnex::CorrSpec_SimpleExtrap.MakeCorrectedHistograms(), fnex::CorrSpec_SimpleNuMuExtrap.MakeCorrectedHistograms(), fnex::CorrSpec_MichelDecomp.MakeCorrectedHistograms(), fnex::CorrSpec_NoExtrapNuE.MakeCorrectedHistograms(), fnex::CorrSpec_BENDecomp.MakeCorrectedHistograms(), cmf::CovarianceFitHelper.MakeDataMCCanv(), cmf::EventListManipulator.reconfigure(), fnex::CorrSpec_NoExtrapNuMu.UpdateCorrectedHistograms(), fnex::CorrSpec_MichelDecomp.UpdateCorrectedHistograms(), fnex::CorrSpec_SimpleNuMuExtrap.UpdateCorrectedHistograms(), fnex::CorrSpec_NoExtrapNuE.UpdateCorrectedHistograms(), fnex::CorrSpec_PropDecomp.UpdateCorrectedHistograms(), fnex::CorrSpec_SimpleExtrap.UpdateCorrectedHistograms(), and fnex::CorrSpec_BENDecomp.UpdateCorrectedHistograms().

runNovaSAM.detId = metadata["nova.detectorid"].lower()

Definition at line 274 of file runNovaSAM.py.

Referenced by align::SplitTracks.analyze(), slid::SPIDBuilder.beginRun(), slid::LIDTraining.beginRun(), chaninfo::FindSwappedChannels.beginSubRun(), LSTME::SliceVars.getCoarseTiming(), nova::dbi::RunHistory.GetConfiguration(), LSTME::SliceVars.getLowGain(), daqchannelmap::DAQChannelMapBaseOLD.getPlaneType(), daqchannelmap::DAQChannelMapBaseOLD.getTotalNumberOfPlanes(), skim::EvaluatorNumuCCpi.Initialize(), skim::EvaluatorNue.Initialize(), skim::EvaluatorNumu.Initialize(), cmf::ShifterAndWeighter.InitShiftsAndWeightsToUse(), nova::dbi::RunHistory.IsDiBlockFullyInstrumented(), nova::dbi::RunHistory.LoadAPDSettings(), nova::dbi::RunHistory.LoadDAQRunHistory(), cmap::CMap.LoadExceptions(), nova::dbi::RunHistory.LoadFEBMasks(), nova::dbi::RunHistory.LoadGoodDiBlockMask(), nova::dbi::RunHistory.LoadHardwareInfo(), chaninfo::BadChanList.LoadNewData(), nova::dbi::RunHistory.LoadPedestals(), nova::dbi::RunHistory.LoadPixelMasks(), chaninfo::BadChanList.NBadInSubRun(), calib::FiberCalibration.produce(), calib::TimingCalibration.produce(), slid::LIDBuilder.produce(), cmf::RandomUniverses.reconfigure(), LSTME::FillLSTME.selectModel(), and cmap::dataprov::CMap.SetStdMap().

runNovaSAM.dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")

Definition at line 415 of file runNovaSAM.py.

Referenced by DCSPlotMaker(), and art::detail::IPRHelper< ProdA, ProdB, Data, DATACOLL, EVENT >.operator()().

runNovaSAM.dirList = os.listdir(".")

Definition at line 406 of file runNovaSAM.py.

runNovaSAM.doMeta

Definition at line 601 of file runNovaSAM.py.

runNovaSAM.experiment

Definition at line 546 of file runNovaSAM.py.

runNovaSAM.failFiles

Definition at line 803 of file runNovaSAM.py.

runNovaSAM.False

Definition at line 522 of file runNovaSAM.py.

runNovaSAM.fclFile = args.config

Definition at line 315 of file runNovaSAM.py.

runNovaSAM.fclFileObj = open(tmpFclName, 'a')

Definition at line 327 of file runNovaSAM.py.

runNovaSAM.fclPath = resolveFclPath(fclFile)

Definition at line 319 of file runNovaSAM.py.

runNovaSAM.file_type = metadata["file_type"]

Definition at line 291 of file runNovaSAM.py.

runNovaSAM.fileMetaDataMgr

Definition at line 563 of file runNovaSAM.py.

runNovaSAM.FirstRun

Definition at line 682 of file runNovaSAM.py.

runNovaSAM.flatLabel

Definition at line 735 of file runNovaSAM.py.

runNovaSAM.flatName

Definition at line 740 of file runNovaSAM.py.

runNovaSAM.flatTier

Definition at line 633 of file runNovaSAM.py.

runNovaSAM.gdbArgs

Definition at line 823 of file runNovaSAM.py.

runNovaSAM.gErrorIgnoreLevel

Definition at line 16 of file runNovaSAM.py.

Referenced by countEvents(), ana::MinuitFitter.FitHelperSeeded(), KinematicsPlots(), LoadExtraOptions(), main(), ndrockmacro(), neutKEsyst_plot(), plot_prediction_systs(), util.SaveObj(), and ana::UnfoldTikhonov.Truth().

runNovaSAM.h5Label

Definition at line 750 of file runNovaSAM.py.

runNovaSAM.h5Name

Definition at line 755 of file runNovaSAM.py.

runNovaSAM.h5Tier

Definition at line 635 of file runNovaSAM.py.

runNovaSAM.help

Definition at line 205 of file runNovaSAM.py.

Referenced by Analyse_GetEfficiency(), Analyse_GetEfficiency_UseNEntries(), Analyse_OverlayStudies(), GetCommandLineArgs(), makeCanvas(), Plotting_OverlayStudies(), and scheduleCuts().

runNovaSAM.histName

Definition at line 774 of file runNovaSAM.py.

runNovaSAM.ifdhcmd

Definition at line 810 of file runNovaSAM.py.

runNovaSAM.Index

Definition at line 684 of file runNovaSAM.py.

Referenced by boost::python::detail::slice_helper< Container, DerivedPolicies, ProxyHandler, Data, Index >.base_delete_slice(), boost::python::detail::no_proxy_helper< Container, DerivedPolicies, ContainerElement, Index >.base_erase_indexes(), boost::python::detail::proxy_helper< Container, DerivedPolicies, ContainerElement, Index >.base_erase_indexes(), boost::python::detail::proxy_helper< Container, DerivedPolicies, ContainerElement, Index >.base_get_item_(), boost::python::detail::slice_helper< Container, DerivedPolicies, ProxyHandler, Data, Index >.base_get_slice(), boost::python::detail::slice_helper< Container, DerivedPolicies, ProxyHandler, Data, Index >.base_get_slice_data(), boost::python::detail::slice_helper< Container, DerivedPolicies, ProxyHandler, Data, Index >.base_set_slice(), Eigen::NumTraits< stan::math::var >.digits10(), boost::python::detail::container_element< Container, Index, Policies >.get_container(), boost::python::detail::container_element< Container, Index, Policies >.get_links(), Eigen::internal::general_matrix_vector_product< Index, stan::math::var, RowMajor, ConjugateLhs, stan::math::var, ConjugateRhs >.run(), and Eigen::internal::general_matrix_matrix_product< Index, stan::math::var, LhsStorageOrder, ConjugateLhs, stan::math::var, RhsStorageOrder, ConjugateRhs, ColMajor >.run().

runNovaSAM.inFile = args.inFile

Definition at line 217 of file runNovaSAM.py.

runNovaSAM.inFileBase = os.path.basename(inFile)

Definition at line 221 of file runNovaSAM.py.

runNovaSAM.isRunningMetadataModule

Definition at line 584 of file runNovaSAM.py.

runNovaSAM.LastRun

Definition at line 683 of file runNovaSAM.py.

runNovaSAM.logfile

Definition at line 834 of file runNovaSAM.py.

Referenced by main(), and nl::timing::LogFile.ProcessLog().

dictionary runNovaSAM.metadata = samweb.getMetadata(inFileBase)

Definition at line 223 of file runNovaSAM.py.

Referenced by braggPeak(), checkBeam(), ConvertToGRooTracker(), flat::FlatMaker.endJob(), hdf5::HDF5Maker.endJob(), caf::CAFMaker.endJob(), fragmentAngularDistribution(), fragmentAngularDistributionGM(), fragmentAngularDistributionHistogram(), fragmentEnergy(), fragmentEnergyDistributionDifferentAngles(), fragmentYields(), fragmentYieldsPlot(), main(), ndrockmacro(), caf::CAFMaker.produce(), TEST_F(), and daq2raw::BeamlineUnpack.UnpackTrigger().

runNovaSAM.nevt = None

Definition at line 311 of file runNovaSAM.py.

runNovaSAM.NewOMod

Definition at line 660 of file runNovaSAM.py.

runNovaSAM.NewOName

Definition at line 697 of file runNovaSAM.py.

runNovaSAM.olddir

Are we streaming the files via xrootd? txtcmd="cat %s | xargs -n1 samweb2xrootd > xrootd_inFile.txt"inFile os.system(txtcmd) with open("xrootd_inFile.txt") as f: for line in f: print line.strip() cmdList.append( line.strip() ) print "" Are we going to copy the files?

Definition at line 800 of file runNovaSAM.py.

Referenced by numusand::NumuSandFxs.getScatt(), and cosrej::CosRejFxs.getScatt().

runNovaSAM.outC

Definition at line 655 of file runNovaSAM.py.

list runNovaSAM.outList = []

Definition at line 344 of file runNovaSAM.py.

Referenced by cmf::EventListManipulator.CreatePoissonFakeData(), and fnex::EventListManipulator.CreatePoissonFakeData().

runNovaSAM.outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)

Definition at line 354 of file runNovaSAM.py.

Referenced by eventListToTextFile(), mre_example_test(), plot_datapredictions(), plot_predictions(), plot_rationoosc(), plotDataPred(), and plotDataPred_SplitCanvas().

runNovaSAM.outNameTemp

Definition at line 680 of file runNovaSAM.py.

runNovaSAM.outP

Definition at line 655 of file runNovaSAM.py.

runNovaSAM.output = outTier.split(":")[0]

Definition at line 348 of file runNovaSAM.py.

Referenced by sim::ParticleNavigator.Add(), stan::math.bernoulli_rng(), stan::math.beta_rng(), stan::math.binomial_rng(), stan::math.cauchy_rng(), caf::Proxy< caf::SRCVNResult >.CheckEquals(), caf::Proxy< caf::SRCVNFeatures >.CheckEquals(), stan::math.chi_square_rng(), ana.ComparisonTable(), ana.ComparisonTableNbins(), ana.ComparisonTableOneNbins(), confusionMatrix(), confusionMatrixProng(), containment_optimization(), ConvertToGHad(), ConvertToGTracker(), ConvertToGXML(), rsim::PulseShaper.CreateTrace(), DataMCLoad(), demo_cut_optimization(), stan::math.double_exponential_rng(), cmf::EventListToTextFile.endJob(), stan::math.exp_mod_normal_rng(), stan::math.exponential_rng(), fiducial_optimization(), ana::nueccinc::NueCCIncTemplateFitter.FillCovarianceMatrix(), ana::NumuCCIncPionTemplateFitter.FillCovarianceMatrixExtra(), nueccinc_test::NueCCIncGlobalFitter.FillCovarianceMatrixFull(), nueccinc_test::NueCCIncGlobalFitter.FillCovarianceMatrixReduced(), stan::math.frechet_rng(), stan::math.gamma_rng(), stan::math.gumbel_rng(), heatMap(), stan::math.inv_chi_square_rng(), stan::math.inv_gamma_rng(), stan::math.logistic_rng(), stan::math.lognormal_rng(), make_files_for_decomp_plots(), make_pair(), MakeMap(), ana.MakeSummaryV(), ana.MakeTextListFile(), model_to_cpp(), stan::math.multi_normal_cholesky_rng(), stan::math.multi_normal_rng(), stan::math.multi_student_t_rng(), muonid_optimization(), nc_bkgd_by_interaction_mode(), stan::math.neg_binomial_2_log_rng(), stan::math.neg_binomial_2_rng(), stan::math.neg_binomial_rng(), stan::math.normal_rng(), cvn::TrainingData.NSimpleOutput(), sim.operator<<(), simb.operator<<(), caf::Proxy< caf::SRCVNResult >.operator=(), caf::Proxy< caf::SRCVNFeatures >.operator=(), INukeNucleonCorr.OutputFiles(), stan::math.pareto_rng(), stan::math.pareto_type_2_rng(), sim::ParticleHistory.ParticleNavigator(), pidPlot(), pidPlotProng(), genie::utils::hadxs::berger.PionNucleonXSec(), PlotResolution(), stan::math.poisson_log_rng(), stan::math.poisson_rng(), preselection_cutflow(), ana::CSVMaker.printHeader(), ana.PrintLatexFigure(), ana.PrintOscilationParameters(), ana.PrintRawEventCounts(), ana::CSVMaker.printRow(), ana.PrintTableHeader(), prod4_pid(), cvneventtf::CVNEventTF.produce(), cvntf::CVNProngEvaluatorTF.produce(), prongcvn_optimization(), stan::math.rayleigh_rng(), SavePrediction(), stan::math.scaled_inv_chi_square_rng(), signal_count(), stan::math.skew_normal_rng(), stan::math.student_t_rng(), TEST(), stan::math.uniform_rng(), stan::math.von_mises_rng(), stan::math.weibull_rng(), and wrong_sign_uncertainty().

runNovaSAM.outputs

Definition at line 641 of file runNovaSAM.py.

runNovaSAM.outTier

Definition at line 340 of file runNovaSAM.py.

runNovaSAM.outX

Definition at line 666 of file runNovaSAM.py.

runNovaSAM.outXc

Definition at line 656 of file runNovaSAM.py.

runNovaSAM.outXp

Definition at line 656 of file runNovaSAM.py.

runNovaSAM.outY

Definition at line 667 of file runNovaSAM.py.

runNovaSAM.outYc

Definition at line 657 of file runNovaSAM.py.

runNovaSAM.outYp

Definition at line 657 of file runNovaSAM.py.

runNovaSAM.parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')

Definition at line 204 of file runNovaSAM.py.

Referenced by nova::dbi::dBTable.deserialize(), nova::database::dBTable.deserialize(), GetCommandLineArgs(), main(), and genie::RunOpt.ReadFromCommandLine().

runNovaSAM.PassFile

Definition at line 567 of file runNovaSAM.py.

runNovaSAM.precopyscript

Definition at line 638 of file runNovaSAM.py.

runNovaSAM.release = os.environ["SRT_BASE_RELEASE"]

Definition at line 285 of file runNovaSAM.py.

Referenced by AnalyzeNus18Systs(), bin_composition_pie_chart(), CalcRWithSystsNus17(), calculateComponentsNumu(), calculateWrongSignNue(), calculateWrongSignNumuQ1(), calculateWrongSignNumuQ2(), calculateWrongSignNumuQ3(), calculateWrongSignNumuQ4(), ConsolidateSystMaker(), ana::CovMxManager.CovMxManager(), CVNCuts(), demo_cut_optimization(), demo_trivial_xsec_analysis(), draw_michel_plots(), DrawExtrapSurface(), drawPlot(), drawSensitivity(), drawSystsShiftingNDdata(), drawSystsShiftingNDdata_updatedAna(), fake_future_data(), FCContour(), FCCorrectSlice(), fd_plot(), fill_col(), FillHists(), futureSig_reach_singlePOTcombo_syst(), genie_contours(), GeniePredictionToRoot(), GetBiases(), getContProf(), getContProf_Sensitivity(), getHists_FNEX(), GetMatrices(), GetMockData(), GetNueCosmics2017(), GetNueCosmics2018(), GetNueCosmics2019(), ana.GetNueCosmics2020(), GetNueCosmicsFuture(), GetNueData2017(), GetNueData2018(), GetNueData2019(), ana.GetNueData2020(), GetNuePrediction2017(), GetNuePrediction2018(), GetNuePrediction2019(), ana.GetNuePrediction2020(), GetNuePredictionFuture(), GetNumuData2017(), ana.GetNumuData2020(), GetNumuPredictionsFuture(), getSensitivity(), GetSurf(), GetSystsFromFile(), getTimePeakPlots(), ana::InteractionSpectra.InteractionSpectra(), LoadCovMx(), LoadData(), LoadExtrapPrediction(), ana::CountingExperiment.LoadFrom(), ana::TrivialBkgdEstimator.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::TrivialSignalEstimator.LoadFrom(), ana::FDPredictionSterile.LoadFrom(), ana::NDPredictionSterile.LoadFrom(), ana::SingleNucAnalysis.LoadFrom(), ana::FitInAnaBinsBkgdEstimator.LoadFrom(), ana::NumuCC2p2hAnalysis.LoadFrom(), ana::NumuCC2p2hBkgdEstimator.LoadFrom(), ana::MultiExperiment.LoadFrom(), ana::nueccinc::NueCCIncMRECorrection.LoadFrom(), ana::PredictionCombinePeriods.LoadFrom(), ana::PredictionAddRock.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::CrossSectionSpectra.LoadFrom(), ana::PredictionExtendToPeripheral.LoadFrom(), ana::PredictionScaleComp.LoadFrom(), ana::CutOptimization.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::CrossSectionAnalysis.LoadFrom(), ana::Multiverse.LoadFrom(), ana::NumuCCIncAnalysis.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::nueccinc::NueCCIncCrossSectionAnalysis.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::NusSystMaker.LoadFrom(), ana::nueccinc::NueCCIncEnhancedSamples.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::NusSystematicsMaker.LoadFrom(), ana::PredictionInterp.LoadFromBody(), ana.LoadFromUpDownSpectra(), LoadMaps(), ana.LoadMaps(), LoadPPFX(), LoadPrediction(), Make2DPlot(), make_extrap_figure(), make_extrap_figure_hists(), make_fc_nus_surfs_nersc_2019(), make_files_for_decomp_plots(), make_muonid_opt(), make_nominal_xs(), Make_NuMuCC_Inc_XS(), make_nus17_fc_surfs(), make_nus_fc_surfs(), make_pdf(), make_starplots(), make_surfprof(), make_vertex_optimiz(), make_xs(), make_xs_1D(), makeEnergyEstimator(), makeFlatWeight(), MakeISysts(), makeMatrixElementSurface(), MakePeriCutPlot(), MakePlot(), MakeSurface(), MakeTable(), mec_tuning_fitter_2020(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), mre_blessed(), mre_comp_split(), muonid_optimization_plots(), nue_decomp_scales(), nue_decomp_scales_for_make_decomp(), nuebar_signif(), NumuExtrap(), PeripheralCuts(), plot_3NDvsFD(), plot_3NDvsFD_FHC(), plot_3NDvsFD_RHC(), plot_BEN(), plot_contprof(), plot_datamc_ND_numu(), plot_datamc_ND_numu_REW(), plot_datamcpred(), plot_datapredictions(), plot_joint_fit_2020_contours(), plot_nd_data_mc(), plot_ND_numu_NOMvsREW(), plot_NDvsFD_REW(), plot_NDvsFD_weights(), plot_NDvsFD_weights_FHC(), plot_NDvsFD_weights_RHC(), plot_nominal_spectra(), plot_nuCrystalBall(), plot_nue_filesyst_pred(), plot_nue_xsec_pred(), plot_nueFD_Signal_REWvsNOM_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_FHC(), plot_nueFD_Signal_REWvsNOM_pTExtrap_RHC(), plot_nueFD_Signal_REWvsNOM_RHC(), plot_nuTree(), plot_pi0_xcheck(), plot_pid(), plot_predictions(), plot_rationoosc(), plot_recoE_numu(), plot_resolution(), plot_shifts(), plot_syst_contours(), plot_time(), plot_uncertainty(), plotContProf(), plotDataPred(), plotDataPred_SplitCanvas(), PlotNus17PredSystsData(), PlotNus18Sideband(), PlotNusSensAna01(), PlotPionPlots(), plots(), Plotting_CompareMultipleContours(), Plotting_DataAndPrediction(), pred_err(), PredictCosmic(), ana::PredictionExtrapSum.PredictionExtrapSum(), ana::PredictionSyst3Flavor2020.PredictionSyst3Flavor2020(), ana::PredictionSystJoint2018.PredictionSystJoint2018(), ana::PredictionSystJointDemo.PredictionSystJointDemo(), ana::PredictionSystNue2017.PredictionSystNue2017(), ana::PredictionSystNumu2017.PredictionSystNumu2017(), preselection_cutflow(), print_tables(), saveContours_addExpt(), saveContours_complete(), saveContours_oscpar(), saveContours_simple(), saveContours_systs(), selection_story_plots(), SetBestFitParams(), signal_count(), starPlot(), syst_plot_test(), syst_table_fit(), syst_table_fit_new(), test_nue2017Prediction(), test_nue2018_fitter(), test_prediction_interp(), test_xsecanalysis(), TestPred(), Toy_analyses(), UnfoldInOut(), and xsec_extrap_plots().

runNovaSAM.ret

Definition at line 812 of file runNovaSAM.py.

Referenced by ana::PredictionExtrap._PredictComponent(), ana::PredictionInterp._PredictComponentSyst(), ana::CutOptimization.AbsUncertainty(), ana::CutOptimization.AbsUncertaintySquared(), ana::Seed.ActiveFitVars(), ana::SeedList.ActiveFitVars(), ana::SystShifts.ActiveSysts(), ana.AddErrorInQuadrature(), AddHistograms(), ana.AddToSummaryV(), rawfileparser::RawFileParser.AdvanceToConfigEnd(), rawfileparser::RawFileParser.AdvanceToConfigStart_file(), rawfileparser::RawFileParser.AdvanceToConfigStart_mem(), rawfileparser::RawFileParser.AdvanceToFirstEvent_file(), rawfileparser::RawFileParser.AdvanceToFirstEvent_mem(), rawfileparser::RawFileParser.AdvanceToNextEvent_file(), rawfileparser::RawFileParser.AdvanceToNextEvent_mem(), rawfileparser::RawFileParser.AdvanceToPreviousEvent_mem(), rawfileparser::RawFileParser.AdvanceToRunHeader_file(), rawfileparser::RawFileParser.AdvanceToRunHeader_mem(), rawfileparser::RawFileParser.AdvanceToRunTail_file(), rawfileparser::RawFileParser.AdvanceToRunTail_mem(), dt::Chain.AllChunks(), dt::Chain.AllHits(), ana.AnaTypeToString(), dt::Chain.AsChunkMap(), ana::TrivialBkgdEstimator.Background(), calib::BetheBlochTables.BetheBlochTables(), stan::services::optimize.bfgs(), rawfileparser::RawFileParser.BuildEventIndex_file(), rawfileparser::RawFileParser.BuildEventIndex_mem(), ana::StanFitter.BuildInitContext(), ana::GenericSystematicDef< SRType >.BuildSpectrumDown(), ana::GenericSystematicDef< SRType >.BuildSpectrumUp(), caldp::AttenProfilesMap.Channels(), ana::SolarConstraints.ChiSq(), ana::MultiExperiment.ChiSq(), rawfileparser::RawFileParser.close(), stan::math.columns_dot_product(), stan::math.columns_dot_self(), CombineHistograms(), ana::PredictionExtrapSum.ComponentCC(), ana::PredictionExtrapSum.ComponentNC(), ana::PredictionExtrapSum.ComponentNCAnti(), ana::PredictionExtrapSum.ComponentNCTotal(), convert(), convert_mcnp_txt_to_root(), ConvertNormalBasis(), osc::OscCalculatorGeneral.Copy(), osc::_OscCalculatorPMNSOpt< T >.Copy(), ana::HistCache.Copy(), osc::OscCalculatorPMNSOptEigen.Copy(), ana::Multiverse.CopyAndTransform(), gibuu::GiBUURegen.CopyGenieEvent(), ana.CopyUpDownSpectrum(), dt::ViewMerger.CountUnexplainedOnLine(), ana.DataMCComparison(), ana.DataMCComparisonAreaNormalized(), ana.DataMCComparisonComponents(), novarwgt.DecodeGeneratorVersion(), caf.DecodeGeneratorVersion(), ana.DefaultOscCalc(), ana.DefaultOscCalcIH(), ana.DefaultSterileCalc(), ana::T2KToyExperiment.Derivative(), stan::math.dot_product(), lem::Potential.Downsampled(), lem::MatchableEvent.Downsampled(), ana::CovMxSurface.DrawContours(), ana.DrawQuantLabel(), ana::Prod4LoadersBase.ECAFTypeFlatToTier(), ana::Prod5LoadersBase.ECAFTypeFlatToTier(), novarwgt.EncodeGeneratorVersion(), EnergyBin(), ErrorBand(), ErrorsToHist(), tf::HoughScore.Eval(), NovaMCEmptyEvent.eventTimestamp(), rb::Cluster.Exclude(), gibuu::GiBUURegen.ExpandLibraryPath(), ana::IFitter.ExpandSeeds(), ana.factorial(), ana::Spectrum.FakeData(), dt::ViewMerger.FakeThirdDimension(), FillPCAContainers(), caf.FillProngVars(), calib::ThroughgoingSelection.filter(), calib::RockMuonStopperSelection.filter(), calib::StopperSelection.filter(), novaddt::MoonShadow.filter(), EliminateBeamSpills.filter(), hough::LiteTH2.FindBinX(), hv::LiteTH2.FindBinX(), hough::LiteTH2.FindBinY(), hv::LiteTH2.FindBinY(), ana.FindCurveCrossings(), geo::GeometryBase.FindGDMLFile(), caf::CAFMaker.FindManyPStrict(), lem.FindMatches(), ana.FindPackageDir(), trk::KalmanTrackAna.FindVisibleProngs(), cheat::RecoCheckAna.FindVisibleProngs(), fuzz::FuzzyKValidate.FindVisibleProngs3D(), fuzz::FuzzyKVertex.FindVisibleProngs3D(), calib::AttenFit.FitQuality(), calib::AttenuationFit.FitQuality(), ana::PredictionInterp.FitRatios(), lem::MatchableEvent.Flipped(), ana::Progress.FormatTime(), nuesand::FillNueSandbox.FracModalHits(), lem::dec::Forest.FromFile(), ana::SpectrumLoader.FromSAMProject(), lem::PIDExtraVars.FromString(), lem::PIDDetails.FromString(), ana::FCBin.FromTree(), lem::EventSummary.FromTree(), GenerateFutureData(), caf.GenieReweightTable(), ana2019::fakedata.Get2019Prediction(), ana.get3FlavorAna2020AllCPTSysts(), ana.get3FlavorAna2020AllSysts(), ana.get3FlavorAna2020LargeXsecSysts(), ana.getAllAna2017Systs(), ana.getAllAna2018Systs(), ana.getAllCPTSysts(), caldp::AttenHistsMap.GetAllHistsByPlane(), ana.getAllNumuProd4FDHistDef(), ana.getAllNumuProd4NDHistDef(), ana.getAllNumuProd5FDHistDef(), ana.getAllNumuProd5FDMultiHistDef(), ana.getAllNumuProd5FDNuTruthHistDef(), ana.getAllNumuProd5NDHistDef(), ana.getAllNumuProd5NDMultiHistDef(), ana.getAllNumuProd5NDNuTruthHistDef(), ana.getAllNusAna01FDHistDefs(), ana.getAllNusAna01NDHistDefs(), caldp::AttenProfilesMap.GetAllProfilesByPlaneAndCell(), ana::CutOptimization.GetAllSystDefs(), ana.getAllXsecNuTruthSysts_2017(), ana.getAna2017LargeXsecSysts(), ana.getAna2018LargeXsecSysts(), ana.getAna2018NueOnly(), ana.getAna2018NumuOnly(), ana.getAna2020NueOnly(), ana.getAna2020NumuOnly(), cosrej::CosRejFxs.getAngle(), numusand::NumuSandFxs.getAngle(), GetBG(), ana.GetBG(), GetBiasErrorLine(), ana.GetCAFMetadata(), rawfileparser::RawFileParser.getConfigBlock(), GetConstraints(), ana::CovMxSurface.GetContours(), GetCorrelations(), ana.GetCorrelations(), GetCosmics(), ana::CovarianceMatrix.GetCovMxAbsoluteWithStats(), ana::CovarianceMatrix.GetCovMxRelative(), ana::CovarianceMatrix.GetCovMxRelativeWithStats(), evd::GeometryDrawer.GetDCMBoxes(), calib::IBetheBloch.GetdEdxGraph(), calib::EnergyLossVsDistance.GetEnergyLossGraph(), rawfileparser::RawFileParser.getEvent(), gibuu::GiBUURegen.GetEvent(), GetExtrapolationDefs(), ana.GetFDMCComponents(), rawfileparser::RawFileParser.getFirstEvent(), GetFNRatio(), GetGaussianProfile(), GetGaussianSurface(), geo::CellGeo.GetGeoMatrix(), ana::ISurface.GetGraphs(), GetHist(), ana::SystConcat.GetIndividualSystNames(), GetJointFitSystematicList(), ana.GetJointFitSystematicList(), GetMatrices(), calib::IBetheBloch.GetMPVGraph(), GetNC(), ana.GetNC(), ana.GetNDComponents(), ana.GetNDCompsFromDecomp(), ana.GetNDDecompsFromDecomp(), ana.GetNDMCComponents(), rawfileparser::RawFileParser.getNextEvent(), ana.getNue2018PredFile(), ana.getNue2018PredVar(), GetNueNDComponents(), GetNueNDComponentsMC(), GetNuePredPath(), ana.getNumu2018PredFile(), ana.getNumu2018PredVar(), GetNus18Binning(), GetNus18Constraints(), GetNus18FitVars(), GetNus18SeedValues(), getPadHists(), osc::OscCalculatorSterileBeam.GetParamsHash(), osc::_NoOscillations< T >.GetParamsHash(), osc::OscCalculatorSterile.GetParamsHash(), osc::OscCalculatorPMNSOptEigen.GetParamsHash(), osc::OscCalculatorCPT.GetParamsHash(), osc::_IOscCalculatorAdjustable< T >.GetParamsHashDefault(), osc::OscCalculatorPMNS_CPT.GetParamsHashDefaultBar(), GetPlotOptions(), GetPOTCombination(), GetPOTComboTitle(), rawfileparser::RawFileParser.getPreviousEvent(), ana::T2KToyExperiment.GetReweightedSignal(), rawfileparser::RawFileParser.getRunHeader(), rawfileparser::RawFileParser.getRunTail(), ana::SystConcat.GetShifts(), SystGroupHelper.GetShifts(), GetShifts2017(), ana.GetShifts2018(), ana.GetShifts2020(), GetSystConcats(), GetSystematics2017(), ana.GetSystematics2018(), ana.GetSystematics2020(), ana::SystConcat.GetSysts(), calib::EnergyLossVsDistance.GetTF1(), GetTotFNRatio(), nuesand::FillNueSandbox.GetTrackAngle(), ana::FitDeltaInPiUnits.GetValue(), ana::FitDelta13InPiUnitsSterile.GetValue(), ana::FitDelta14InPiUnitsSterile.GetValue(), ana::FitDelta24InPiUnitsSterile.GetValue(), lem::LEMWorker.GetWork(), rawfileparser::RawFileParser.GotoEvent_file(), ana::MinuitFitter.Gradient(), HistNumuOpt(), dt::Chunk.HitsOnLine(), cheat::BackTracker.HitsToParticle(), cheat::BackTracker.HitToFLSHit(), tf::TimingFitAlg.HoughFitPts(), ana::Binning.IDMap(), ImportDataGraph(), InitCutDefs(), InitHistDefs(), stan::optimization::BFGSMinimizer< ModelAdaptor< M >, QNUpdateType, Scalar, DimAtCompile >.initialize(), InitSpectra(), geo::LiveGeometry.InMuonCatcher(), util.ipow(), ana.JoinGraphs(), JointGraphs(), ana.kMinPngNhit(), ana::SystShifts.LatexName(), stan::services::optimize.lbfgs(), load(), load_libs_muonid(), load_libs_nuebarccinc(), load_libs_nueccinc(), load_libs_numucc2p2h(), load_libs_numucc_0Pi(), load_libs_numuccinc(), LoadCosmic(), ana.LoadFileList(), ana::PredictionXSecTuning.LoadFrom(), ana::CountingExperiment.LoadFrom(), ana::PredictionNoOsc.LoadFrom(), ana::PredictionSterile.LoadFrom(), ana::SolarConstraints.LoadFrom(), ana::CheatDecomp.LoadFrom(), ana::Binning.LoadFrom(), ana::NCDecomp.LoadFrom(), ana::NDExtrapBeamComponent.LoadFrom(), ana::CovMxSurface.LoadFrom(), ana::NueDecomp.LoadFrom(), ana::NDOscillatableSpectrum.LoadFrom(), ana::NDExtrapComponent.LoadFrom(), ana::SingleSampleExperiment.LoadFrom(), ana::NumuDecomp.LoadFrom(), ana::PredictionSystJoint2018.LoadFrom(), ana::PredictionSystJointDemo.LoadFrom(), ana::CovMxManager.LoadFrom(), ana::FDExtrap.LoadFrom(), ana::NDExtrap.LoadFrom(), ana::ProportionalDecomp.LoadFrom(), ana::TrivialExtrap.LoadFrom(), ana::FakeDecomp.LoadFrom(), ana::TrivialCrossSectionAnalysis.LoadFrom(), ana::Multiverse.LoadFrom(), ana::SystShifts.LoadFrom(), ana::TwoSampleDecomp.LoadFrom(), ana::SpectrumComponents.LoadFrom(), ana::PredictionInterp.LoadFrom(), ana::CovarianceMatrix.LoadFrom(), ana::FluxDecomp.LoadFrom(), ana::ModularExtrapSterile.LoadFrom(), ana::MichelDecomp.LoadFrom(), ana::ReweightableSpectrum.LoadFrom(), ana::OscillatableSpectrum.LoadFrom(), ana::BENDecomp.LoadFrom(), ana::PredictionSystNue2017.LoadFrom(), ana::PredictionSystNumu2017.LoadFrom(), ana::TruthReweight.LoadFrom(), ana::DataMCPair.LoadFrom(), ana::RecoReweight.LoadFrom(), ana::ModularExtrap.LoadFrom(), ana::FluxReweight.LoadFrom(), ana::Spectrum.LoadFrom(), ana::NusISyst.LoadFrom(), ana.LoadFrom< osc::IOscCalculatorAdjustable >(), ana::FCCollection.LoadFromFile(), ana::FCSurface.LoadFromFile(), genie::GSimFiles.LoadFromFile(), genie::XSecSplineList.LoadFromXml(), LoadPrediction(), ana.LoadPredictions(), ana.LoadRealData(), ana::SAMQuerySource.LocateSAMFiles(), ana.LogLikelihoodDerivative(), ana::GaussianPriorSystShifts.LogPrior(), calib::AttenFit.lowessFit(), calib::AttenuationFit.lowessFit(), ana::GradientDescent.Magnitude(), MakeAverageTrueEnergySurface(), dt::View.MakeChunkCombos(), ana.MakeCosOutOfTimeList(), MakeCutFlowSpectra(), MakeDataReleaseFileContour(), MakeDataReleaseFileSlice(), ana::NumuAnalysis.MakeExperiment(), MakeKin2DSpectra(), MakeKinSpectra(), MakeName(), ana.MakeNumiUnblindList(), MakeRatio(), MakeRatios(), MakeSample(), MakeSpectra(), MakeTitle(), ana.MakeUnblindList(), MakeUnblindList(), hough::MultiHough2P.MapToTH2F(), dt::ViewMerger.MatchVerticals(), osc.MatrixExp(), caldp::TCTrack.MaxCell(), rb::Cluster.MaxCell(), nuesand::FillNueSandbox.MaxHits(), caldp::TCTrack.MaxPath(), caldp::TCTrack.MaxPlane(), rb::Cluster.MaxPlane(), caldp::TCTrack.MaxTNS(), rb::Cluster.MaxTNS(), lem::Match.MaybeFlip(), dt::Cand.MaybeFlip(), cheat::BackTracker.MCTruthToParticles(), calib::AttenFit.MeanProfile(), calib::AttenuationFit.MeanProfile(), ana::ResolutionScan.MeanScan(), rb::Cluster.MeanTNS(), calib::AttenFit.MedianProfile(), calib::AttenuationFit.MedianProfile(), caldp::TCTrack.MinCell(), rb::Cluster.MinCell(), caldp::TCTrack.MinPath(), caldp::TCTrack.MinPlane(), rb::Cluster.MinPlane(), caldp::TCTrack.MinTNS(), rb::Cluster.MinTNS(), lem.MMapFileAtAddress(), ana::Spectrum.MockData(), MostConservative(), MyDefaultOscCalc(), ana::HistCache.New(), ana::HistCache.NewF(), ana::HistCache.NewTH2D(), ana::HistCache.NewTH2F(), dt::Cand.NHitPlanes(), dt::Chain.NHitPlanes(), rb::Cluster.OfflineChans(), ana::Ratio.operator*(), ana::Multiverse.operator*(), ana::Spectrum.operator*(), ana::SpectrumStan.operator+(), ana::ReweightableSpectrum.operator+(), ana::OscillatableSpectrum.operator+(), ana::Spectrum.operator+(), ana::SpectrumStan.operator-(), ana::OscillatableSpectrum.operator-(), ana::Spectrum.operator-(), ana::Ratio.operator/(), ana::Multiverse.operator/(), ana::Spectrum.operator/(), ana::OscillatableSpectrum.Oscillated(), osc::_IOscCalculator< T >.P(), lem::FindMatchesAlg.PackageMatches(), PaintReachCanvas(), parse_as_vector(), ana::SystShifts.Penalty(), PIDFileTag(), PIDLongName(), plot_ratios(), ana::NumuCCIncAnalysis.PlotHelper(), PlotMCComponentsErrorBand(), dt::DiscreteTracker.PossibleNextSegs(), ana::FDPredictionSterile.Predict(), ana::NDPredictionSterile.Predict(), ana::PredictionScaleComp.Predict(), ana::PredictionSterile.PredictComponent(), ana::FDPredictionSterile.PredictComponent(), ana::PredictionExtrapSum.PredictComponent(), ana::NDPredictionSterile.PredictComponent(), ana::PredictionNoOsc.PredictComponent(), ana::PredictionExtendToPeripheral.PredictComponent(), ana::PredictionAddRock.PredictComponentSyst(), PredictCosmic(), ana::PredictionInterp.PredictionInterp(), Print(), ana::GaussianPriorSystShifts.Prior(), ana.Profile(), ProfileAxes(), calib::BetheBlochFit.ProfileMPV(), lem::dec::Cut.Prune(), caf.PtrVecToVec(), cheat::BackTracker.PtrVecToVecRawPtr(), stan::math.quad_form_sym(), ana::MCMCSamples.QuantileLL(), lem::dec::Forest.RandomOrthoMatrix(), RebinToShortAxis(), dt::DiscreteTracker.RecoDiscrete(), ana.RefineSeeds(), RemoveBins(), ana.RemoveBins(), ana.replaceAll(), ana::ResolutionScan.RMSScan(), ana::PredictionAddRock.RockComponent(), stan::math.rows_dot_product(), stan::math.rows_dot_self(), ana::SummedSyst.Shift(), ShiftedCosmics(), ana::SystShifts.ShortName(), ana::Registry< T >.ShortNameToPtr(), murem::MuonRemove.SliceSelection(), SortSystHists(), SplitHistograms(), ana.SqrtProfile(), ana.SqrtSlice(), stan::math::LDLT_factor< var, R, C >.success(), ana.systsQ3Q0(), TEST(), slid::DedxDistribution.TH1DToTH1F(), TH1ToTH2(), calib::ThresholdCorrMap.ThresholdCorrAt(), stan::math.to_fvar(), ana::SpectrumStan.ToBins(), dt::Chain.ToDebugTracks(), ana::CutOptimization.ToHist(), ana::BayesianMarginal.ToHistogram(), ana::Multiverse.ToSpectrum(), lem::PIDExtraVars.ToString(), ValidCalibPDF.ToString(), rb::Cluster.TotalADC(), caldp::LiteProfile.TotalEntries(), rb::Cluster.TotalGeV(), caldp::LiteProfile.TotalHits(), rb::Prong.TotalLength(), rb::Cluster.TotalPE(), ana::SpectrumLoaderBase::IDMap< T, U >.TotalSize(), rb::Cluster.TotalWeight(), ana::OscCurve.ToTH1(), ana::Ratio.ToTH1(), ana::Bayesian1DMarginal.ToTH1(), ana::Spectrum.ToTH1(), ana::Ratio.ToTH2(), ana::ISurface.ToTH2(), ana::ReweightableSpectrum.ToTH2(), ana::Spectrum.ToTH2(), hough::LiteTH2.ToTH2F(), hv::LiteTH2.ToTH2F(), ana.ToTH2Helper(), ana::Spectrum.ToTH3(), ana.ToTH3Helper(), ana.ToTHStack(), dt::Cand.ToTrack(), ana::CutOptimization.ToUpDownHist(), murem::TrackCleanUpAlg.TrackEinMIP(), lem::dec::Forest.Train(), lem::dec.TransformEvent(), calib::AttenCache.TranslateChannelMC(), ncid::MakeNCCosRej.TransMomFraction(), ncid::NCNNKeras.TransMomFraction(), cosrej::MakeNueCosRej.TransMomFraction(), calib::AttenFit.TruncatedMeanProfile(), calib::AttenuationFit.TruncatedMeanProfile(), ana::UnfoldSVD.Truth(), ana::UnfoldTikhonov.Truth(), cheat::BackTracker.VecPtrToVecRawPtr(), cheat::BackTracker.VecToVecRawPtr(), ana::Nus17SystFromHist.WeightFor(), ana::NusSystFromHist.WeightFor(), ana::Nus18SystFromHist.WeightFor(), ana::NusISyst.WeightFor(), ana::WildcardSource.WildcardOrLs(), ana::SpectrumLoaderBase.WildcardOrSAMQuery(), stan::optimization.WolfeLineSearch(), dt::ViewMerger.ZipVerticalTracks(), rb::Track.ZipWith(), ana::ProfilerSupport.~ProfilerSupport(), and calib::ThresholdCorrMap.~ThresholdCorrMap().

runNovaSAM.retCode = subprocess.call(cmdList)

Definition at line 387 of file runNovaSAM.py.

Referenced by dbutils::gui::BasicEditorDisplay._confirmSave(), dbutils::gui::BasicEditorDisplay._getUserResponseToGlobalCopyFailure(), dbutils::gui::BasicEditorDisplay.closeEvent(), dbutils::gui::DCSConfigEditorDisplay.generateRecipeScripts(), dbutils::gui::DCSConfigEditorDisplay.loadHVAndTemps(), dbutils::gui::BasicEditorDisplay.loadNewDataset(), dbutils::gui::DAQConfigEditorDisplay.loadThresholdsAndMasks(), dbutils::gui::BasicEditorDisplay.reloadCurrentDataset(), stan::optimization::BFGSMinimizer< ModelAdaptor< M >, QNUpdateType, Scalar, DimAtCompile >.step(), and stan::optimization.WolfeLineSearch().

runNovaSAM.run_match = run_pattern.match(inFileBase)

Definition at line 238 of file runNovaSAM.py.

runNovaSAM.run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")

Definition at line 237 of file runNovaSAM.py.

runNovaSAM.runNum = int(metadata["runs"][0][0])

Definition at line 234 of file runNovaSAM.py.

Referenced by calib::Calibrator.postBeginRun().

runNovaSAM.samweb = samweb_client.SAMWebClient(experiment='nova')

Definition at line 219 of file runNovaSAM.py.

runNovaSAM.shell

Definition at line 594 of file runNovaSAM.py.

runNovaSAM.skip_match = _skip_pattern.match(file)

Definition at line 408 of file runNovaSAM.py.

runNovaSAM.stderr

Definition at line 834 of file runNovaSAM.py.

runNovaSAM.stdout

Definition at line 834 of file runNovaSAM.py.

runNovaSAM.str

Definition at line 207 of file runNovaSAM.py.

Referenced by site_stats_from_log.SiteStats.__repr__(), file_size_ana.Node.__repr__(), log_trawl.Process.__repr__(), file_size_ana.Node.__str__(), log_trawl.Process.__str__(), nova_art_metadata._createMetadata(), samweb_client.http_client_requests.RequestsHTTPClient._doURL(), samweb_client.http_client_urllib2.URLLib2HTTPClient._doURL(), samweb_client.http_client.SAMWebHTTPClient._logger(), samweb_client._exceptions._Exceptions._makeHTTPError(), fabricate._results_handler(), make_static_page.AddFile(), samweb_client.admin.addValue(), canMan.ProgressTracker.advance(), POTGetData.append_json(), fabricate.args_to_list(), g4zmq.beamOn(), samweb_client.utility.calculateChecksum(), run_hadd.checkFileCount(), checkCalibrationCSV.NovaCalibTest.checkRunCoverage(), MetadataUtils.cleanup_metadata(), demo.App.cmd_beamOn(), Lesson1.App.cmd_beamOn(), moon_position_table_new3.computeDayBoundaryMoon(), createMetadata(), MetadataUtils.createMetadata(), ProductionTestTools.crontabToGantt(), POTGetData.database_connect(), fhicl::detail.decode(), log_trawl.Project.decomposeModes(), demoFitSlices(), gen_hdf5record.deps_emitted(), POTUptimeData.do_detector_mass(), bp.DownloadFiles(), plot_validation_datamc.Spectrum.Draw(), generic.tools.ImageMaker.ImageMaker.draw(), plot_validation_datamc_2018.Spectrum.Draw(), canMan.drawOverlay(), stan::lang.dump_program_line(), fabricate.Builder.echo_command(), log_trawl.Process.errPath(), escapeQuotes(), cmf::EventListManipulator.ExtractEventsFromFile(), generic.controllers.MakeCanvases.HistogramExtractor.ExtractHistsFromDirectory(), fake_future_data(), PlotUtilityNew.file_list(), samweb_client.utility.fileChecksum(), PlotUtilityNew.find_files(), bp.FindFiles(), regression.generate_name(), POTPopulateData.get_active_chans(), POTGetData.get_active_chans(), POTGetData.get_runs(), POTSpillRate.get_spills(), POTGetData.get_spills(), lem_server.get_tag(), genie::utils::xml.GetAttribute(), ana::ISurface.GetBestFitY(), TB_WatchdogFx.getDBRunsHist(), WatchdogFx.getDBRunsHist(), helpers.GetDoc(), bp_docdb.DocDB.GetDocIDsByCategory(), bp_docdb.DocDB.GetDocument(), TB_WatchdogFx.Tier.getHist(), WatchdogFx.Tier.getHist(), meta::MetadataManager.GetKeys(), TB_WatchdogFx.getMaxMinRunNumDB(), WatchdogFx.getMaxMinRunNumDB(), DeclareReco.getMetadata(), GetNumuPtQuant(), getOutDir(), GetPtNQuants(), TB_WatchdogFx.getSAMRunsHist(), WatchdogFx.getSAMRunsHist(), ana::SpectrumLoaderBase.Gone(), lem_server.handle_backlog(), lem_server.handle_ids(), om::HeaderBar.HandleTimer(), log_trawl.Project.inspectProcs(), cmf.InteractionTypeKey(), joint_fit_2017_slices(), joint_fit_2018_slices(), joint_fit_2019_bestfit(), joint_fit_2019_slices(), joint_fit_future_bestfit_univ(), PandAna.core.core.load_tree(), ana::MCMCSamples.LoadFrom(), nova_caf_metadata.loads_recursive(), ana::SAMQuerySource.LocationsForSAMQuery(), PandAna.utils.sadd_pandana.main(), DCS_db_parser.main(), pot-for-runs.main(), POTUptimeData.main(), POTPopulateData.main(), POTGetData.main(), om::PlotOptions.MakeLabelText(), PlotDataQA.MakePlots(), samweb_client.projects.makeProcessUrl(), cmf::StarPlot.MakeStarPlot(), fnex::StarPlot.MakeStarPlot(), nova_uptime.nactive_chans(), canMan.newCan(), nova_t2k_comparison(), POTSpillRate.num_spills_between(), MCenterSpills.options(), POTUptimeData.options(), POTPopulateData.options(), POTGetData.options(), nova_uptime.options(), log_trawl.Process.outPath(), Expression.Parse(), evd::PlotView.Parse(), update_sam_good_runs_metadata.percentStr(), plot_joint_fit_2020_slices(), plot_shifts(), Tier.Tier.prepareAndExecuteProcess(), file_size_ana.prettyPrint(), daqdataformats::VERSION_NAMESPACE::RawConfigurationSystemID.print(), daqdataformats::RawDAQData.print(), SAMView.SamView.print_summary(), uptime_graphic.print_table(), log_trawl.Project.printModes(), bp.ProcessImages(), PlotUtilityNew.pull_data(), nova::dbi::Table.RemoveValidityRange(), run_joint_fit_2020_bestfit(), run_joint_fit_2020_slices(), BatchLog.BatchLog.run_time(), lemmon_tools.runStatusCheck(), ViewHTML.samDefinitionDescriptionsModal(), samweb_client.http_client.SAMWebHTTPClient.set_timezone(), ana::Progress.SetProgress(), SliceLegend(), lem_server.status_text(), cmf.StringToDetectorType(), StripSpecialChar(), submit_concat_project.Submit(), stan::mcmc.TEST(), TEST(), TEST_F(), plot_timing_data.time_format(), file_size_ana.toJSON(), daqdataformats::RawMilliSlice.tprintf(), daqdataformats::VERSION_NAMESPACE::RawDataBlock.tprintf(), genie::utils::xml.TrimSpaces(), Tutorial2019Fit(), Tutorial2019FitSlices(), gen_hdf5record.variables_inc_bases(), gen_hdf5record.write_hd5record_h(), POTUptimeData.write_json(), POTGetData.write_json(), ana.WriteCAFMetadata(), lem.writeMemoryCallback(), genie::Spline.YCanBeNegative(), genie::flux::GDk2NuFluxXMLHelper.~GDk2NuFluxXMLHelper(), and genie::flux::GNuMIFluxXMLHelper.~GNuMIFluxXMLHelper().

runNovaSAM.stream = int(metadata["data_stream"])

Definition at line 251 of file runNovaSAM.py.

Referenced by genie::Algorithm.AllowReconfig(), genie::RSHelicityAmpl.Amp20Plus(), genie::ProcessInfo.AsString(), ClassImp(), genie::geometry::FidPolyhedron.clear(), nova::database::Row.Col(), genie::GVldContext.Emax(), genie::RunOpt.EnableBareXSecPreCalc(), novaddt::HitSubtract.endJob(), Expression.Eval(), genie::geometry::FidCylinder.FidCylinder(), genie::geometry::FidSphere.FidSphere(), genie::Spline.GetAsTSpline(), nutools::dbi::Table.GetConnectionTimeout(), rvp::RecVarPID.getMethodNameFromXML(), xnue::XnuePID.getMethodNameFromXML(), rvp::RecVarPID.getVarNamesFromXML(), xnue::XnuePID.getVarNamesFromXML(), genie::PDF.Gluon(), genie::Kinematics.HadSystP4(), genie::Target.HitNucP4(), genie::geometry::PathSegment.IsTrimmedEmpty(), genie::geometry::PlaneParam.IsValid(), genie::AlgId.Key(), genie::AxialFormFactor.Model(), genie::ELFormFactors.Model(), genie::QELFormFactors.Model(), genie::DISStructureFunc.Model(), genie::CacheBranchNtp.Ntuple(), genie::TuneId.OnlyConfiguration(), genie::geometry.operator<(), genie::utils.operator<<(), genie::exceptions.operator<<(), genie::geometry.operator<<(), operator<<(), genie.operator<<(), nova::database.operator<<(), nutools::dbi.operator<<(), nova::dbi.operator<<(), genie::flux.operator<<(), genie::Interaction.PhaseSpacePtr(), genie::exceptions::EVGThreadException.ReturnStep(), genie::GHepParticle.SetLastDaughter(), genie::RegistryItem< T >.SetLocal(), genie::XclsTag.SetNNeutrons(), nutools::dbi::Row.SetVldTimeEnd(), nova::dbi::Row.SetVldTimeEnd(), genie::exceptions::INukeException.ShowReason(), genie::exceptions::NievesQELException.ShowReason(), genie::CacheBranchFx.Spl(), genie::InitialState.TgtPtr(), nova::database::Column.Update(), nutools::dbi::Column.Update(), nova::dbi::Column.Update(), genie::GHepRecord.Vertex(), genie::GEVGDriver.XSecSumSpline(), genie::geometry::FidShape.~FidShape(), genie::flux::GJPARCNuFluxPassThroughInfo.~GJPARCNuFluxPassThroughInfo(), genie::flux::GNuMIFluxPassThroughInfo.~GNuMIFluxPassThroughInfo(), genie::flux::GSimpleNtpAux.~GSimpleNtpAux(), genie::flux::GSimpleNtpEntry.~GSimpleNtpEntry(), genie::flux::GSimpleNtpNuMI.~GSimpleNtpNuMI(), genie::exceptions::InteractionException.~InteractionException(), and genie::utils::T2KEvGenMetaData.~T2KEvGenMetaData().

runNovaSAM.stream_match = stream_pattern.match(inFileBase)

Definition at line 263 of file runNovaSAM.py.

runNovaSAM.stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)

Definition at line 262 of file runNovaSAM.py.

string runNovaSAM.streamEntry = 't{0:02d}'

Definition at line 252 of file runNovaSAM.py.

runNovaSAM.subNum = int(metadata["runs"][0][1])

Definition at line 235 of file runNovaSAM.py.

int runNovaSAM.subversion = os.environ["NPASS"]

Definition at line 305 of file runNovaSAM.py.

runNovaSAM.tier = outTier.split(":")[1]

Definition at line 349 of file runNovaSAM.py.

Referenced by ana::Prod5AbsCalibLoaders.Prod5AbsCalibLoaders(), ana::Prod5CalibDriftLoaders.Prod5CalibDriftLoaders(), ana::Prod5CalibShapeLoaders.Prod5CalibShapeLoaders(), ana::Prod5CherenkovLoaders.Prod5CherenkovLoaders(), ana::Prod5LightLevelLoaders.Prod5LightLevelLoaders(), and ana::FileReducer.UpdateMetadata().

runNovaSAM.tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")

Definition at line 322 of file runNovaSAM.py.

runNovaSAM.type

Definition at line 205 of file runNovaSAM.py.