Inheritance diagram for PandAna.core.core.loader:

Public Member Functions
def	__init__ (self, filesource, stride=1, offset=0, limit=None, spillcuts=None, index=None)

def	getSource (self)

def	sum_POT (self)

def	add_spectrum (self, spec)

def	reset_index (self)

def	__setitem__ (self, key, df)

def	__getitem__ (self, key)

def	setupGo (self)

def	getFile (self)

def	setFile (self, f)

def	closeFile (self)

def	readData (self)

def	fillSpectra (self)

def	Go (self)

def	cleanup (self)

Public Attributes
	gone

	interactive

	histdefs

	index

	dflist

	summing

	openfile

	concat_time

Private Attributes
	_files

	_tables

	_spillcuts

	_POT

	_potspecnocut

	_potspeccut

	_POTBase

	_filegen

Detailed Description

Definition at line 378 of file core.py.

Constructor & Destructor Documentation

def PandAna.core.core.loader.__init__	(	self,
		filesource,
		stride = `1`,
		offset = `0`,
		limit = `None`,
		spillcuts = `None`,
		index = `None`
	)

Definition at line 379 of file core.py.

     def __init__(self, filesource, stride = 1, offset = 0, limit = None, spillcuts=None, index=None):
 
         self._files = sourcewrapper(filesource, stride, offset, limit)
         
         # _tables stores the entire dataset read from file
         # index key holds the global index range to be accessed from the dataset by a cut/var
         self._tables = {'indices':0}
         self.gone = False
         self.interactive = False
         self.histdefs = []
         self.index=index
         self.dflist = {}
         self._spillcuts = spillcuts
 
         # add extra spectra to keep track of exposure
         self._POT = 0
         self.sum_POT()
 

Member Function Documentation

def PandAna.core.core.loader.__getitem__	(	self,
		key
	)

Definition at line 447 of file core.py.

References PandAna.core.core.spectrum._tables, PandAna.core.core.loader._tables, novaddt::Boundary.index, minos::NuMCEvent.index, minos::NuEvent.index, PandAna.core.core.loader.index, print, and PandAna.core.core.loader.summing.

     def __getitem__(self, key):
         if not self.summing:
             if type(key)==str and key.startswith('spill'):
                 key = 'rec.'+key
         # actually build the cache before Go()
         if type(key) == str and not key in self._tables: 
             # Pick up the right index
             index = KL if key.startswith('rec') else KLN if key.startswith('neutrino') else KLS
             if self.index and key.startswith('rec'):
               index = self.index
             self[key] = dfproxy(columns=index)
         # assume key is a filtered index range after a cut 
         if type(key) is not str:
             self._tables['indices'] = key
             return self
         # no filtering 
         if self._tables['indices'] is 0:
             return self._tables[key]
         # use global index to slice dataframe requested
         elif self._tables[key].dropna().empty:
             # sometimes there's no data available in the file, allow it but warn
             print("Warning! No data read for %s" % key)
             return self._tables[key]
         else:
             if self._tables[key].index.intersection(self._tables['indices']).empty:
               return dfproxy(columns=self._tables[key].index.names)
             else:
               dfslice = self._tables[key].loc[self._tables['indices']]
               return dfslice
 

def PandAna.core.core.loader.__setitem__	(	self,
		key,
		df
	)

Definition at line 439 of file core.py.

References PandAna.core.core.spectrum._tables, PandAna.core.core.loader._tables, novaddt::Boundary.index, minos::NuMCEvent.index, minos::NuEvent.index, and PandAna.core.core.loader.index.

     def __setitem__(self, key, df):
         # set multiindex for recTree data
         index = KL if key.startswith('rec') else KLN if key.startswith('neutrino') else KLS
         if self.index and key.startswith('rec'):
           index = self.index
         df.set_index(index, inplace=True)
         self._tables[key] = df
 

def PandAna.core.core.loader.add_spectrum	(	self,
		spec
	)

Definition at line 428 of file core.py.

References PandAna.core.core.loader._spillcuts, PandAna.core.core.loader.histdefs, and PandAna.core.core.loader.summing.

     def add_spectrum(self, spec):
         if not spec in self.histdefs:
             if self._spillcuts is not None and not self.summing:
                 spec._cutfcn = spec._cutfcn & self._spillcuts
 
             self.histdefs.append(spec)
 

def PandAna.core.core.loader.cleanup ( self )

Definition at line 569 of file core.py.

References PandAna.core.core.spectrum._tables, PandAna.core.core.loader._tables, and PandAna.core.core.loader.histdefs.

Referenced by PandAna.core.core.loader.Go().

     def cleanup(self):
         # free up some memory
         self._tables = {'indices':0}
         # remove associations with spectra
         self.histdefs = []
 
 # Different loaders end up starting their own SAM projects, even for the exact same queries. 
 # This doesn't guarantee that they'll run over the files in the same order. 
 # Coupled with the fact that the projects can be shared over different grid jobs, 
 # this can result in unexpected behaviour if the macro expects them to share the same data downstream. 
 # This class allows the user to use a single project over multiple loaders 

def PandAna.core.core.loader.closeFile ( self )

Definition at line 491 of file core.py.

Referenced by PandAna.core.core.loader.Go(), and PandAna.core.core.associate.Go().

     def closeFile(self):
         self.openfile.close()
 

def PandAna.core.core.loader.fillSpectra ( self )

Definition at line 522 of file core.py.

Referenced by PandAna.core.core.loader.Go().

     def fillSpectra(self):
         self.concat_time = time.time()
         for key in self.dflist:
             # set index for all dataframes
             self[key] = pd.concat(self.dflist[key])
             # sort index
             self._tables[key].sort_index(inplace=True)
         self.dflist = {}
 
         # Compute POT and then fill spectra
         self.sum_POT()
 
         # Let's not refill these
         self.histdefs.remove(self._potspecnocut)
         if self._potspeccut:
             self.histdefs.remove(self._potspeccut)
 
         spec_idx = 0
         #  spec_progbar = ProgressBar(len(self.histdefs))
         print("Filling %s spectra\n" % len(self.histdefs))
         for spec in self.histdefs:
             spec_idx += 1
             #  spec_progbar.update(spec_idx)
             spec.fill()
         

def PandAna.core.core.loader.getFile ( self )

Definition at line 485 of file core.py.

References PandAna.core.core.loader._filegen.

Referenced by ProjMan.Consumer.__iter__(), PandAna.core.core.loader.Go(), and PandAna.core.core.associate.Go().

     def getFile(self):
         return self._filegen()
 

def PandAna.core.core.loader.getSource ( self )

Definition at line 397 of file core.py.

References PandAna.core.core.loader._files.

     def getSource(self):
         return self._files
 

def PandAna.core.core.loader.Go ( self )

Definition at line 547 of file core.py.

     def Go(self):
         t0 = time.time()
         self.setupGo()
         file_idx = 0
         #  file_progbar = ProgressBar(self._filegen.nFiles())
         while True:
           try:
             fname = self.getFile()
             self.setFile(h5py.File(fname, 'r'))
             self.readData()
             self.closeFile()
        
             file_idx += 1
             #  file_progbar.update(file_idx)
           except StopIteration:
             break
         
         self.fillSpectra()
         # cleanup
         self.cleanup()
         print("\nTotal time : %s sec\n" % (time.time() - t0))
 

def PandAna.core.core.loader.readData ( self )

Definition at line 494 of file core.py.

References PandAna.core.core.spectrum._tables, PandAna.core.core.loader._tables, append(), PandAna.core.core.loader.dflist, parse_dependency_file_t.list, and print.

Referenced by PandAna.core.core.loader.Go().

     def readData(self):
         for key in self._tables:
             if key is 'indices':
                 continue
             if not key in self.dflist:
                 self.dflist[key] = []
             # branches from cache
             if not key in self.openfile.keys():
               print("Group %s doesn't exist!" % key)
               sys.exit(2)
             group = self.openfile.get(key)
             values = {}
             # leaves from cache
             keycache = self._tables[key]._proxycols
             for k in keycache:
                 try:
                   dataset = group.get(k)[()]
                 except TypeError:
                   # better error message
                   print("Dataset %s for group %s doesn't exist!" % (k, group))
                   sys.exit(2)
                 if dataset.shape[1] == 1:
                     dataset = dataset.flatten()
                 else:
                     dataset = list(dataset)
                 values[k] = dataset
             self.dflist[key].append(pd.DataFrame(values))
 

def PandAna.core.core.loader.reset_index ( self )

Definition at line 435 of file core.py.

References PandAna.core.core.spectrum._tables, and PandAna.core.core.loader._tables.

     def reset_index(self):
         # reset after each spectrum fill
         self._tables['indices'] = 0
 

def PandAna.core.core.loader.setFile	(	self,
		f
	)

Definition at line 488 of file core.py.

Referenced by PandAna.core.core.loader.Go(), and PandAna.core.core.associate.Go().

     def setFile(self, f):
         self.openfile = f
     

def PandAna.core.core.loader.setupGo ( self )

Definition at line 477 of file core.py.

References PandAna.core.core.loader.gone.

Referenced by PandAna.core.core.loader.Go(), and PandAna.core.core.associate.Go().

     def setupGo(self):
         if self.gone: 
             return
         self.gone = True
         self._filegen = self._files()
 
         print("Reading data from %s files : \n" % self._filegen.nFiles())
 

def PandAna.core.core.loader.sum_POT ( self )

Definition at line 400 of file core.py.

     def sum_POT(self):
         self.summing = True
 
         # If not gone, construct spectra
         if not self.gone:
             self._potspecnocut = spectrum(self, lambda tables: tables['spill']['spillpot']>0, \
                                               lambda tables: tables['spill']['spillpot'])
 
             if self._spillcuts:
                 self._potspeccut = spectrum(self, self._spillcuts, \
                                                 lambda tables: tables['spill']['spillpot'])
             else:
                 self._potspeccut = None
 
         # If gone, fill spectra and compute POT
         else:
             self._potspecnocut.fill()
             self._POTBase = self._potspecnocut.df().sum()
             if self._potspeccut:
                 self._potspeccut.fill()
                 self._POT = self._potspeccut.df().sum()
             else:
                 # Use base pot if not using spill cuts
                 self._POT = self._POTBase
             frac = 100*self._POT/self._POTBase
             print('Found {:0.5E} POT passing spillcuts from {:0.5E} POT ({:0.1f}%).'.format(self._POT, self._POTBase, frac))
         self.summing = False
 