submit_nova_art Namespace Reference

Functions
def	check_env (vname)
def	check_fcl (tag, fcl)
def	remove_comments (src)
def	find_file (paths, filename)
def	find_file_in_list (filepath, pathlist)
def	build_jobsub_cmd ()

Variables
	user
	sam_user
	sam_station
	cvmfs_distro_base
	novasoft_cvmfs
	build_location_arguments
	setup_location
	jobsub_opts
	run_nova_sam_cmd
	run_nova_sam_opts
	export_to_run_nova_sam
	art_sam_wrap_cmd
	Setup art_sam_wrap #. More...
	art_sam_wrap_opts
	export_to_art_sam_wrap
	usage_models
	input_files
	veryearly_scripts
	early_scripts
	source_scripts
	pre_scripts
	inter_scripts
	post_scripts
	prog
	preliminary_parser
	Allow args to be passed in as a plain text file. More...
	description
	help
	number of jobs More...
	type
	str
	action
	pre_args
	unknown
	argv
	index
	fullpath
	text
	newargs
	parser
	add_help
	required_args
	required options More...
	required
	debugging_args
	debugging More...
	default
	metavar
	job_control_args
	job control More...
	int
	sing_group
	choices
	False
	novasoft_args
	software control More...
	testrel_gp
	output_args
	output control More...
	environment_args
	environment configuration More...
	support_args
	support options More...
	args
	Process and check command line arguments #. More...
	timestamp
	recommended_sites
	istarball
	use_recommended_sites
	test_njobs
	test_nevts
	test_dest
	mode
	test_expected_lifetime
	test_dynamic_lifetime
	test_files_per_job
	njobs
	nevts
	Setup runNovaSAM #. More...
	dest
	expected_lifetime
	dynamic_lifetime
	files_per_job
	declareFiles
	declareLocations
	autoDropbox
	poms
	offsite
	jobname
	defname
	snapshot_id
	print_jobsub
	printenv
	test
	tag
	srt_qual
	maxopt
	maxopt_opt
	fcl
	mcgen
	is_production_arg
	hashDirs
	role
	samweb
	Setup jobsub #. More...
	experiment
	num_project_files
	dimensions
	file
	njobs specified, but not files/job. More...
	awsfilepath
	singularity
	resource_opt
	site_opt
	disk_opt
	mem_opt
	cpu_opt
	kill_opt
	dummy
	allowed_lifetimes
	ngu_current
	shell
	cvmfs_rev
	group
	allowed_groups
	subgroup
	allowed_subgroups
	logs
	passThru
	va
	out_tiers
	outnum
	mcouttier
	copyback
	caf_tiers
	flat_tiers
	hist_tier
	h5_tiers
	script_path
	project_name
	start sam project More...
	start_project
	start_proj_command
	Start the SAM project #. More...
	start_proj_retval
	job_cmd
	scriptcount
	poms_campaign_id
	version
	dataset
	poms_task_id
	command_executed
	jobsub_cmd
	Actually launch the jobs #. More...

Function Documentation

def submit_nova_art.build_jobsub_cmd

(

)

Definition at line 144 of file submit_nova_art.py.

References print.

def build_jobsub_cmd():
    # Start with jobsub_submit and its options
    jobsub_cmd = "jobsub_submit \\\n"
    for opt in jobsub_opts:
        jobsub_cmd += "    " + opt + " \\\n"
    for export in export_to_art_sam_wrap:
        jobsub_cmd += "    -e " + export
    jobsub_cmd += " \\\n"

    # Add art_sam_wrap wrapper script and its options
    if args.testrel:
        if args.reuse_tarball:
            jobsub_cmd += "    --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
        else:
            jobsub_cmd += "    --tar_file_name tardir://" + args.testrel +" \\\n"
    elif args.user_tarball:
        if not os.path.isfile(args.user_tarball):
            print("Tarball filename passed to --user_tarball does not exit:", args.user_tarball)
            sys.exit(5)
        jobsub_cmd += "    --tar_file_name dropbox://" + args.user_tarball + " \\\n"

    jobsub_cmd += "    file://" + art_sam_wrap_cmd + " \\\n"
    for opt in art_sam_wrap_opts:
        jobsub_cmd += "      " + opt + " \\\n"
    for export in export_to_run_nova_sam :
        jobsub_cmd += "      --export " + export + " \\\n"



    # Now set the main program and its options
    if not mcgen:
        jobsub_cmd += "      -X " + run_nova_sam_cmd + " \\\n"
        for opt in run_nova_sam_opts:
            jobsub_cmd += "        " + opt + " \\\n"
    else:
        jobsub_cmd += "      -X nova \\\n"
        mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
        if nevts>0 :
            mcgen_opts += ["-n %d" % nevts]
        for opt in mcgen_opts:
            jobsub_cmd += "        " + opt + " \\\n"

    jobsub_cmd = jobsub_cmd[:-2]
    return jobsub_cmd

def submit_nova_art.check_env

(

vname

)

Definition at line 68 of file submit_nova_art.py.

References NovaGridUtils.fail().

def check_env(vname):
    value=os.getenv(vname)
    if None == value or "" == value:
        fail("Environment variable %s not defined" %vname)

def submit_nova_art.check_fcl	(		tag,
			fcl
	)

Definition at line 73 of file submit_nova_art.py.

References NovaGridUtils.fail(), print, and split().

def check_fcl(tag,fcl):
    if "/" in fcl[:1] and os.path.isfile(fcl):
        return fcl # hey I'm ok!
    # Loop over dirs in FHICL_FILE_PATH
    fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
    for path in fclPaths:
        fullPath = os.path.join(path, fcl)
        if os.path.isfile(fullPath):
            return fcl # hey you're ok!
    # Check if they are copying the fhicl file to the worker node.
    for input_file in input_files:
        if os.path.basename(input_file) == fcl:
            return fcl # Passing fhicl as argument, all good.
        elif os.path.basename(input_file) == os.path.basename(fcl):
            print("")
            print("The fhicl will be copied to $PWD on the worker node.")
            print("You specified some sort of file path which isn't needed. Fixing it for you :)")
            print(fcl + " --> " + os.path.basename(fcl))
            print("")
            return os.path.basename(fcl) # User incorrectly left file path there.

    fail("fcl file %s does not exist" %fcl)

def submit_nova_art.find_file	(		paths,
			filename
	)

Definition at line 128 of file submit_nova_art.py.

References NovaGridUtils.fail().

def find_file(paths, filename):
    if os.path.isfile(filename):
        return filename
    for path in paths:
        for root, dirs, files in os.walk(os.path.expandvars(path)):
            if filename in files:
                return os.path.join(root, filename)
    fail("Cannot find file "+filename)

def submit_nova_art.find_file_in_list	(		filepath,
			pathlist
	)

Definition at line 137 of file submit_nova_art.py.

def find_file_in_list(filepath, pathlist):
    for testpath in pathlist:
        if os.path.basename(filepath) == os.path.basename(testpath):
            return True
    return False

def submit_nova_art.remove_comments

(

src

)

This reads tokens using tokenize.generate_tokens and recombines them
using tokenize.untokenize, and skipping comment/docstring tokens in between

Definition at line 97 of file submit_nova_art.py.

References art.decode().

def remove_comments(src):
    """
    This reads tokens using tokenize.generate_tokens and recombines them
    using tokenize.untokenize, and skipping comment/docstring tokens in between
    """
    f = io.StringIO(src.encode().decode())
    class SkipException(Exception): pass
    processed_tokens = []
    last_token = None
    # go thru all the tokens and try to skip comments and docstrings
    for tok in tokenize.generate_tokens(f.readline):
        t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok

        try:
            if t_type == tokenize.COMMENT:
                raise SkipException()

            elif t_type == tokenize.STRING:

                if last_token is None or last_token[0] in [tokenize.INDENT]:
                    pass

        except SkipException:
            pass
        else:
            processed_tokens.append(tok)

        last_token = tok

    return tokenize.untokenize(processed_tokens)

Variable Documentation

submit_nova_art.action

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.add_help

Definition at line 233 of file submit_nova_art.py.

submit_nova_art.allowed_groups

Definition at line 1060 of file submit_nova_art.py.

submit_nova_art.allowed_lifetimes

Definition at line 1023 of file submit_nova_art.py.

submit_nova_art.allowed_subgroups

Definition at line 1071 of file submit_nova_art.py.

submit_nova_art.args

Process and check command line arguments #.

Definition at line 688 of file submit_nova_art.py.

submit_nova_art.argv

Definition at line 214 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_cmd

Setup art_sam_wrap #.

Definition at line 39 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_opts

Definition at line 40 of file submit_nova_art.py.

submit_nova_art.autoDropbox

Definition at line 751 of file submit_nova_art.py.

submit_nova_art.awsfilepath

Definition at line 936 of file submit_nova_art.py.

submit_nova_art.build_location_arguments

Definition at line 30 of file submit_nova_art.py.

submit_nova_art.caf_tiers

Definition at line 1177 of file submit_nova_art.py.

submit_nova_art.choices

Definition at line 381 of file submit_nova_art.py.

submit_nova_art.command_executed

Definition at line 1379 of file submit_nova_art.py.

submit_nova_art.copyback

Definition at line 1169 of file submit_nova_art.py.

submit_nova_art.cpu_opt

Definition at line 987 of file submit_nova_art.py.

submit_nova_art.cvmfs_distro_base

Definition at line 28 of file submit_nova_art.py.

submit_nova_art.cvmfs_rev

Definition at line 1043 of file submit_nova_art.py.

submit_nova_art.dataset

Definition at line 1373 of file submit_nova_art.py.

submit_nova_art.debugging_args

debugging

Definition at line 264 of file submit_nova_art.py.

submit_nova_art.declareFiles

Definition at line 745 of file submit_nova_art.py.

submit_nova_art.declareLocations

Definition at line 748 of file submit_nova_art.py.

submit_nova_art.default

Definition at line 268 of file submit_nova_art.py.

submit_nova_art.defname

Definition at line 761 of file submit_nova_art.py.

submit_nova_art.description

Definition at line 204 of file submit_nova_art.py.

submit_nova_art.dest

Definition at line 736 of file submit_nova_art.py.

submit_nova_art.dimensions

Definition at line 890 of file submit_nova_art.py.

submit_nova_art.disk_opt

Definition at line 979 of file submit_nova_art.py.

submit_nova_art.dummy

Definition at line 1020 of file submit_nova_art.py.

submit_nova_art.dynamic_lifetime

Definition at line 740 of file submit_nova_art.py.

submit_nova_art.early_scripts

Definition at line 62 of file submit_nova_art.py.

submit_nova_art.environment_args

environment configuration

Definition at line 635 of file submit_nova_art.py.

submit_nova_art.expected_lifetime

Definition at line 738 of file submit_nova_art.py.

submit_nova_art.experiment

Definition at line 886 of file submit_nova_art.py.

submit_nova_art.export_to_art_sam_wrap

Definition at line 41 of file submit_nova_art.py.

submit_nova_art.export_to_run_nova_sam

Definition at line 37 of file submit_nova_art.py.

submit_nova_art.False

Definition at line 426 of file submit_nova_art.py.

submit_nova_art.fcl

Definition at line 818 of file submit_nova_art.py.

submit_nova_art.file

njobs specified, but not files/job.

Just set njobs and don't force limits on files per jobs

Definition at line 915 of file submit_nova_art.py.

submit_nova_art.files_per_job

Definition at line 742 of file submit_nova_art.py.

submit_nova_art.flat_tiers

Definition at line 1182 of file submit_nova_art.py.

submit_nova_art.fullpath

Definition at line 221 of file submit_nova_art.py.

Referenced by geo::GeometryBase.getCoordinateTransformation().

submit_nova_art.group

Definition at line 1058 of file submit_nova_art.py.

submit_nova_art.h5_tiers

Definition at line 1191 of file submit_nova_art.py.

submit_nova_art.hashDirs

Definition at line 851 of file submit_nova_art.py.

submit_nova_art.help

number of jobs

general job control

Definition at line 207 of file submit_nova_art.py.

submit_nova_art.hist_tier

Definition at line 1187 of file submit_nova_art.py.

submit_nova_art.index

Definition at line 218 of file submit_nova_art.py.

submit_nova_art.input_files

Definition at line 59 of file submit_nova_art.py.

submit_nova_art.int

Definition at line 304 of file submit_nova_art.py.

submit_nova_art.inter_scripts

Definition at line 65 of file submit_nova_art.py.

submit_nova_art.is_production_arg

Definition at line 836 of file submit_nova_art.py.

submit_nova_art.istarball

Definition at line 693 of file submit_nova_art.py.

submit_nova_art.job_cmd

Definition at line 1283 of file submit_nova_art.py.

submit_nova_art.job_control_args

job control

Definition at line 300 of file submit_nova_art.py.

submit_nova_art.jobname

Definition at line 760 of file submit_nova_art.py.

submit_nova_art.jobsub_cmd

Actually launch the jobs #.

Definition at line 1391 of file submit_nova_art.py.

submit_nova_art.jobsub_opts

Definition at line 33 of file submit_nova_art.py.

submit_nova_art.kill_opt

Definition at line 1005 of file submit_nova_art.py.

submit_nova_art.logs

Definition at line 1109 of file submit_nova_art.py.

submit_nova_art.maxopt

Definition at line 782 of file submit_nova_art.py.

submit_nova_art.maxopt_opt

Definition at line 783 of file submit_nova_art.py.

submit_nova_art.mcgen

Definition at line 820 of file submit_nova_art.py.

Referenced by main(), NeutronOscGenerator(), and NucleonDecayGenerator().

submit_nova_art.mcouttier

Definition at line 1167 of file submit_nova_art.py.

submit_nova_art.mem_opt

Definition at line 983 of file submit_nova_art.py.

submit_nova_art.metavar

Definition at line 295 of file submit_nova_art.py.

submit_nova_art.mode

Definition at line 723 of file submit_nova_art.py.

Referenced by jmshower::JMShower.ANN(), ana::Bayesian1DMarginal.Bayesian1DMarginal(), ana::BayesianSurface.BayesianSurface(), ana.best_spline_find_mode(), caf::Proxy< caf::SRNeutrino >.CheckEquals(), flat::Flat< caf::SRNeutrino >.Clear(), DCSPlotMaker(), efficiency(), efficiencySA(), flat::Flat< caf::SRNeutrino >.Fill(), GausFit(), genie::NNBarOscPrimaryVtxGenerator.GenerateDecayProducts(), GetCommandLineArgs(), MCGIDI_quantitiesLookupModes.getCrossSectionMode(), ana::NuWROSyst.InitializeHistograms(), ana::BayesianMarginal.LoadInto(), supernova::SnovaGen.MakeMCTruth(), dt::ViewMerger.MatchTracks(), meanWeight_macro(), meanWeight_plot(), ModeRestrict(), mrbrem_get_initial_spectra(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), NuSCalculateCorr(), caf::Proxy< caf::SRNeutrino >.operator=(), cet::ostream_handle.ostream_handle(), plot_muonid_dist(), plot_nd_spectra_2018(), pots(), genie::GHepRecord.ProbePosition(), slidt.SaveHists(), ana::BayesianMarginal.SaveTo(), evgen::GENIENeutronOscGen.selectAnnihilationMode(), SelectAnnihilationMode(), jmshower::JMShower.SetANN(), MCGIDI_quantitiesLookupModes.setCrossSectionMode(), evd::SliceNavigator.SetFullEventMode(), genie::GHepRecord.TargetNucleusPosition(), genie::utils::intranuke.ThreeBodyKinematics(), genie::utils::intranuke2018.ThreeBodyKinematics(), uptime(), useBDTG(), and jmshower::JMShower.~JMShower().

submit_nova_art.nevts

Setup runNovaSAM #.

Definition at line 734 of file submit_nova_art.py.

Referenced by ndrockmacro(), and plot_xsecs().

submit_nova_art.newargs

Definition at line 226 of file submit_nova_art.py.

submit_nova_art.ngu_current

Definition at line 1033 of file submit_nova_art.py.

submit_nova_art.njobs

Definition at line 732 of file submit_nova_art.py.

submit_nova_art.novasoft_args

software control

Definition at line 459 of file submit_nova_art.py.

submit_nova_art.novasoft_cvmfs

Definition at line 29 of file submit_nova_art.py.

submit_nova_art.num_project_files

Definition at line 888 of file submit_nova_art.py.

submit_nova_art.offsite

Definition at line 758 of file submit_nova_art.py.

submit_nova_art.out_tiers

Definition at line 1164 of file submit_nova_art.py.

submit_nova_art.outnum

Definition at line 1167 of file submit_nova_art.py.

submit_nova_art.output_args

output control

Definition at line 511 of file submit_nova_art.py.

submit_nova_art.parser

Definition at line 233 of file submit_nova_art.py.

submit_nova_art.passThru

Definition at line 1116 of file submit_nova_art.py.

submit_nova_art.poms

Definition at line 754 of file submit_nova_art.py.

submit_nova_art.poms_campaign_id

Definition at line 1368 of file submit_nova_art.py.

submit_nova_art.poms_task_id

Definition at line 1376 of file submit_nova_art.py.

submit_nova_art.post_scripts

Definition at line 66 of file submit_nova_art.py.

submit_nova_art.pre_args

Definition at line 211 of file submit_nova_art.py.

submit_nova_art.pre_scripts

Definition at line 64 of file submit_nova_art.py.

submit_nova_art.preliminary_parser

Allow args to be passed in as a plain text file.

We make a preliminary parser get these arguments out for two reasons: 1) Maintain standard -h, –help functionality 2) Avoid necessity required arguments in initial parsing, allow them to be missing, but find them in the file.

Definition at line 204 of file submit_nova_art.py.

submit_nova_art.print_jobsub

Definition at line 763 of file submit_nova_art.py.

submit_nova_art.printenv

Definition at line 767 of file submit_nova_art.py.

submit_nova_art.prog

Definition at line 194 of file submit_nova_art.py.

submit_nova_art.project_name

start sam project

Definition at line 1241 of file submit_nova_art.py.

submit_nova_art.recommended_sites

Definition at line 692 of file submit_nova_art.py.

submit_nova_art.required

Definition at line 239 of file submit_nova_art.py.

submit_nova_art.required_args

required options

Definition at line 236 of file submit_nova_art.py.

submit_nova_art.resource_opt

Definition at line 955 of file submit_nova_art.py.

submit_nova_art.role

Definition at line 856 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_cmd

Definition at line 35 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_opts

Definition at line 36 of file submit_nova_art.py.

submit_nova_art.sam_station

Definition at line 25 of file submit_nova_art.py.

submit_nova_art.sam_user

Definition at line 24 of file submit_nova_art.py.

submit_nova_art.samweb

Setup jobsub #.

both njobs and files per job are specified. Just pass the settings through to jobsub and art_sam_wrap files/job specified, but not njobs. Calculate njobs on the fly get files in dataset

Definition at line 886 of file submit_nova_art.py.

submit_nova_art.script_path

Definition at line 1231 of file submit_nova_art.py.

submit_nova_art.scriptcount

Definition at line 1284 of file submit_nova_art.py.

submit_nova_art.setup_location

Definition at line 31 of file submit_nova_art.py.

submit_nova_art.shell

Definition at line 1033 of file submit_nova_art.py.

submit_nova_art.sing_group

Definition at line 371 of file submit_nova_art.py.

submit_nova_art.singularity

Definition at line 945 of file submit_nova_art.py.

submit_nova_art.site_opt

Definition at line 959 of file submit_nova_art.py.

submit_nova_art.snapshot_id

Definition at line 762 of file submit_nova_art.py.

submit_nova_art.source_scripts

Definition at line 63 of file submit_nova_art.py.

submit_nova_art.srt_qual

Definition at line 781 of file submit_nova_art.py.

submit_nova_art.start_proj_command

Start the SAM project #.

Definition at line 1263 of file submit_nova_art.py.

submit_nova_art.start_proj_retval

Definition at line 1273 of file submit_nova_art.py.

submit_nova_art.start_project

Definition at line 1244 of file submit_nova_art.py.

submit_nova_art.str

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.subgroup

Definition at line 1069 of file submit_nova_art.py.

submit_nova_art.support_args

support options

Definition at line 671 of file submit_nova_art.py.

submit_nova_art.tag

Definition at line 779 of file submit_nova_art.py.

submit_nova_art.test

Definition at line 769 of file submit_nova_art.py.

submit_nova_art.test_dest

Definition at line 720 of file submit_nova_art.py.

submit_nova_art.test_dynamic_lifetime

Definition at line 726 of file submit_nova_art.py.

submit_nova_art.test_expected_lifetime

Definition at line 725 of file submit_nova_art.py.

submit_nova_art.test_files_per_job

Definition at line 727 of file submit_nova_art.py.

submit_nova_art.test_nevts

Definition at line 719 of file submit_nova_art.py.

submit_nova_art.test_njobs

Definition at line 718 of file submit_nova_art.py.

submit_nova_art.testrel_gp

Definition at line 464 of file submit_nova_art.py.

submit_nova_art.text

Definition at line 224 of file submit_nova_art.py.

submit_nova_art.timestamp

Definition at line 689 of file submit_nova_art.py.

Referenced by daq2raw::BeamlineEvent.AddTimestamp(), beamlinereco::CFDHitFinder< double >.CFDHitFinder(), mcdatadaq::DAQSimulator.fillMicroSlices(), dbutils::daqconfig::NamedConfigUtils.getAvailableGlobalConfigs(), dbutils::daqconfig::NamedConfigUtils.getAvailableSubsystemConfigs(), getEventList(), mcdatadaq::DAQSimulator.getNanoSliceOrder(), dbutils::daqconfig::NamedConfigUtils.getRecentGlobalConfigs(), getTimePeakTotal(), daqdataformats::VERSION_NAMESPACE::RawSummaryDroppedMicroblock.init(), daqdataformats::VERSION_NAMESPACE::RawSummaryDCMDataPoint.init(), daqdataformats::VERSION_NAMESPACE::RawTriggerTimingMarker.init(), daqdataformats::VERSION_NAMESPACE::RawTriggerTime.init(), beamlinereco::LEHitFinder< double >.LEHitFinder(), daq2raw::BeamlineRawInputDriver.makeSpill(), OnMonPlotMaker(), beamlinereco::LEHitReco.produce(), beamlinereco::CFDHitReco.produce(), art::EmptyEvent.readEvent_(), gov::fnal::cd::rms::base::RmsMessage.RmsMessage(), brb::ToF.SetTimestamp(), rawdata::RawBeamlineTrigger.SetTimestamp(), rawdata::RawBeamlineDigit.SetTimestamp(), rawdata::RawBeamlineWC.SetTimestamp(), tbrecoplots(), daq2raw::BeamlineUnpack.UnpackDigits(), daq2raw::BeamlineUnpack.UnpackTrigger(), daq2raw::BeamlineUnpack.UnpackWC(), rawdata::RawBeamlineWC.YPulses(), rawdata::RawBeamlineDigit.~RawBeamlineDigit(), and rawdata::RawBeamlineTrigger.~RawBeamlineTrigger().

submit_nova_art.type

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.unknown

Definition at line 211 of file submit_nova_art.py.

submit_nova_art.usage_models

Definition at line 57 of file submit_nova_art.py.

submit_nova_art.use_recommended_sites

Definition at line 694 of file submit_nova_art.py.

submit_nova_art.user

Definition at line 21 of file submit_nova_art.py.

submit_nova_art.va

Definition at line 1118 of file submit_nova_art.py.

Referenced by ana.CombineExposures(), ana::MultiVar2DFunc.operator()(), ana::Var2DFunc< T >.operator()(), ana::Var3DFunc< T >.operator()(), SetDrawOptions(), test_fitvarsproduct(), and ana.ValHelper().

submit_nova_art.version

Definition at line 1372 of file submit_nova_art.py.

submit_nova_art.veryearly_scripts

Definition at line 61 of file submit_nova_art.py.