Public Member Functions
def	__init__ (self, config_dict, release="development", local=False, verbose=False, user=False)

def	run (self)

def	get_dataset (self)

def	setup_sam_view (self)

def	get_sam_status (self, sam)

def	get_expected_sam_output_files (self, sam)

def	get_sam_merge_status (self)

def	run_sam_merge (self)

Public Attributes
	local

	verbose

	user

	name

	version

	script

	dataset

	label

	release

	status

	sam_merge_status

	jobsub_command

	command

	lines

	good_submission

	run_time

	script_dataset

	sam_view

	n_files

	files_per_job

	n_jobs

	root_file

Detailed Description

Definition at line 9 of file Component.py.

Constructor & Destructor Documentation

def Component.Component.__init__	(	self,
		config_dict,
		release = `"development"`,
		local = `False`,
		verbose = `False`,
		user = `False`
	)

Definition at line 11 of file Component.py.

     def __init__(self, config_dict, release="development", local=False, verbose=False, user=False):
         self.local    = local
         self.verbose  = verbose
         self.user     = user
         # compulsory augments
         self.name     = config_dict["name"]
         self.version  = int(config_dict["version"])
         self.script   = config_dict["script"]
         self.dataset  = config_dict["dataset"]
         # optional with false defaults
         optional_attributes = ["datasets","local_dataset","local_datasets","sam","directories"]
         for attr in optional_attributes:
             if attr in config_dict.keys():
                 setattr(self, attr, config_dict[attr])
             else:
                 setattr(self, attr, False)
         # optional with non-false defaults
         if "label" in config_dict.keys():
             self.label = config_dict["label"]
         else:
             self.label = self.dataset
 
         if "release" in config_dict.keys():
             self.release = config_dict["release"]
         else:
             self.release = release
 
         string_attributes = ["n_jobs","files_per_job"]
         for attr in string_attributes:
             if attr in config_dict.keys():
                 setattr(self, attr, config_dict[attr])
             else:
                 setattr(self, attr, "")
 
         self.status   = "initialised"
         self.sam_merge_status = "not run"
 
         if (self.local or self.sam):
             self.jobsub_command = "sh %s"%(os.environ["NOVAANAVALID_DIR"])
         else:
             self.jobsub_command = "jobsub_submit -G nova --resource-provides=usage_model=DEDICATED,OPPORTUNISTIC --expected-lifetime=3600s --OS=SL6 file://%s"%(os.environ["NOVAANAVALID_DIR"])
 
         self.get_dataset()
 

Member Function Documentation

def Component.Component.get_dataset ( self )

Definition at line 76 of file Component.py.

     def get_dataset(self):
         if   self.local_datasets: self.script_dataset = " ".join(self.local_datasets)
         elif self.datasets:       self.script_dataset = " ".join(self.datasets)
         elif self.local_dataset:  self.script_dataset = self.local_dataset
         else:                     self.script_dataset = self.dataset
 

def Component.Component.get_expected_sam_output_files	(	self,
		sam
	)

Definition at line 94 of file Component.py.

     def get_expected_sam_output_files(self, sam):
         # work out the number of files we're expecting
         if (self.files_per_job == ""):
             n_files = False
         elif (self.files_per_job != "") and (self.n_jobs == ""):
             # in this case the number of n_files = files_per_job * n_jobs
             # as n_jobs isn't set here it is determined at run time:
             # the default way to do this is as N_TOTAL_FILES / 100
             total_files = sam.countFiles(dimensions="snapshot_id %s"%self.sam_view.sam_summary["snapshot_id"])
             self.n_jobs = int(total_files / 100)
             n_files = self.n_jobs * self.files_per_job
         else:
             n_files = self.n_jobs * self.files_per_job
         return n_files
 

def Component.Component.get_sam_merge_status ( self )

Definition at line 109 of file Component.py.

References ControllerView.ControllerView.sam, and Component.Component.sam_merge_status.

     def get_sam_merge_status(self):
         if (not self.sam):
             return "not applicable"
         if (not hasattr(self, "sam_merge_status")):
             self.sam_merge_status = "not run"
         return self.sam_merge_status
 

def Component.Component.get_sam_status	(	self,
		sam
	)

Definition at line 87 of file Component.py.

References Component.Component.setup_sam_view().

     def get_sam_status(self, sam):
         if (not hasattr(self, "sam_view")):
             self.setup_sam_view()
         
         self.n_files = self.get_expected_sam_output_files(sam)
         return self.sam_view.get_status(sam, n_files=self.n_files)
 

def Component.Component.run ( self )

Definition at line 55 of file Component.py.

References Component.Component.status, evgen::hepevtpart.status, and fabricate.StraceRunner.status.

     def run(self):
         self.status = "run"
         
         self.command = "%s/%s %s %s %s %s %s %s %s"%(self.jobsub_command, self.script, self.user, self.release, self.script_dataset, self.output_dir, self.name, self.n_jobs, self.files_per_job)
         if self.verbose: print "comp :     command: %s"%self.command
         
         process = subprocess.Popen(self.command.split(" "), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         
         output = process.stdout
         self.lines = output.readlines()
         self.good_submission = False
         for line in self.lines:
             if self.verbose: print line.strip()
             if "JobsubJobId of first job" in line: self.good_submission = True
         
         if (not self.good_submission):
             print "comp : ERROR in submission"
             self.status = "submission error"
 
         self.run_time = datetime.datetime.now()
 

def Component.Component.run_sam_merge ( self )

Definition at line 116 of file Component.py.

References SAMView.SamView.output_dir, PageMaker.PageMaker.output_dir, Controller.Controller.output_dir, generic.controllers.MakePages.PageMaker.output_dir, and Component.Component.sam_merge_status.

     def run_sam_merge(self):
         if hasattr(self, "sam_merge_status") and (self.sam_merge_status != "not run"):
             if (self.sam_merge_status != "complete"):
                 print "comp : SAM merge has already run, status: %s"%self.sam_merge_status
             return
 
         self.sam_merge_status = "running"
         output_files = glob.glob("%s/*.root"%self.output_dir)
         if (not len(output_files)):
             self.sam_merge_status = "no files"
             return
         self.root_file = "%s/hadd_%s_%s.root"%(self.output_dir, self.name, self.version)
         cmd = 'python %s/core/utils/hadd_caf.py -i %s/ -w * -o %s -s'%(os.environ["NOVAANAVALID_DIR"], self.output_dir, self.root_file)
         
         process = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         lines = []
         while True:
             line = process.stdout.readline()
             if line != "": lines.append(line)
             if line == '' and process.poll() != None: break
             # time.sleep(0.1)
         
         if (process.returncode != 0):
             print "comp : --- Exception caught while executing sam merge, return code: %i"%process.returncode
             print "comp :     command run: %s"%cmd
             for line in lines:
                 print line.strip()
             self.sam_merge_status = "failed: return code = %i"%process.returncode
 
         elif (not os.path.exists(self.root_file)):
             self.sam_merge_status = "failed: no output file"
         else:
             self.sam_merge_status = "complete"
 
 

def Component.Component.setup_sam_view ( self )

Definition at line 82 of file Component.py.

References Component.Component.lines, Component.Component.verbose, samweb_client.http_client_requests.RequestsHTTPClient.verbose, samweb_client.http_client.SAMWebHTTPClient.verbose, evg::MCTruthToDk2NuHackItr.verbose, and samweb_client.client.SAMWebClient.verbose.

Referenced by Component.Component.get_sam_status().

     def setup_sam_view(self):
         sam_url = tools.getSAMURL(self.lines)
         if self.verbose: print "comp : found sam URL: %s"%sam_url
         self.sam_view = sam_view.SamView(sam_url, self.output_dir, verbose=self.verbose)
 

Member Data Documentation

Component.Component.command

Definition at line 58 of file Component.py.

Component.Component.dataset

Definition at line 19 of file Component.py.

Component.Component.files_per_job

Definition at line 96 of file Component.py.

Component.Component.good_submission

Definition at line 65 of file Component.py.

Component.Component.jobsub_command

Definition at line 49 of file Component.py.

Component.Component.label

Definition at line 29 of file Component.py.

Component.Component.lines

Definition at line 64 of file Component.py.

Referenced by Component.Component.setup_sam_view().

Component.Component.local

Definition at line 12 of file Component.py.

Component.Component.n_files

Definition at line 91 of file Component.py.

Component.Component.n_jobs

Definition at line 98 of file Component.py.

Component.Component.name

Definition at line 16 of file Component.py.

Referenced by mcscore.MCParticle.__init__(), Definition.Definition.__repr__(), Chain.Chain.__repr__(), Chain.Chain.AddTier(), python.g4viscp.G4Scene.create_scene(), python3.g4viscp.G4Scene.create_scene(), DefinitionData.DefinitionData.fetchData(), std_candles.StdCandlesPlotter.IsMC(), mcscore.MCParticle.printout(), run_unfold_ppfx_multiprocess.Consumer.run(), run_unfold_systs_multiprocess.Consumer.run(), run_unfold_genie_multiprocess.Consumer.run(), PageMaker.PageMaker.setNavBarActive(), python3.g4viscp.G4Scene.update_scene(), python.g4viscp.G4Scene.update_scene(), and PageMaker.PageMaker.write().

Component.Component.release

Definition at line 34 of file Component.py.

Referenced by MetadataUtils.metaDataMgr.getOutputFileName().

Component.Component.root_file

Definition at line 127 of file Component.py.

Component.Component.run_time

Definition at line 74 of file Component.py.

Component.Component.sam_merge_status

Definition at line 46 of file Component.py.

Referenced by Component.Component.get_sam_merge_status(), and Component.Component.run_sam_merge().

Component.Component.sam_view

Definition at line 85 of file Component.py.

Component.Component.script

Definition at line 18 of file Component.py.

Component.Component.script_dataset

Definition at line 77 of file Component.py.

Component.Component.status

Definition at line 45 of file Component.py.

Referenced by Component.Component.run().

Component.Component.user

Definition at line 14 of file Component.py.

Referenced by ProjMan.Consumer.establish().

Component.Component.verbose

Definition at line 13 of file Component.py.

Referenced by PageMaker.PageMaker.__init__(), ControllerView.ControllerView.check_sam_status(), ControllerView.ControllerView.component_status(), View.ControllerView.component_view(), ControllerView.ControllerView.component_view(), Canvas.Canvas.draw(), generic.tools.ImageMaker.ImageMaker.draw(), ControllerView.ControllerView.draw_component_plots(), Controller.Controller.make_output_directory(), Component.Component.setup_sam_view(), PageMaker.PageMaker.write(), and ViewTimingDB.ViewTimingDB.write_csv().

Component.Component.version

Definition at line 17 of file Component.py.

The documentation for this class was generated from the following file:

/cvmfs/nova-development.opensciencegrid.org/novasoft/releases/N20-09-23/Validation/analysis/core/components/Component.py

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation