submit_nova_art Namespace Reference

Functions
def	check_env (vname)
def	check_fcl (tag, fcl)
def	remove_comments (src)
def	find_file (paths, filename)
def	find_file_in_list (filepath, pathlist)
def	build_jobsub_cmd ()

Variables
	user
	sam_user
	sam_station
	recommended_sites
	cvmfs_distro_base
	novasoft_cvmfs
	build_location_arguments
	setup_location
	jobsub_opts
	run_nova_sam_cmd
	run_nova_sam_opts
	export_to_run_nova_sam
	art_sam_wrap_cmd
	Setup art_sam_wrap #. More...
	art_sam_wrap_opts
	export_to_art_sam_wrap
	usage_models
	njobs specified, but not files/job. More...
	input_files
	veryearly_scripts
	early_scripts
	source_scripts
	pre_scripts
	inter_scripts
	post_scripts
	prog
	preliminary_parser
	Allow args to be passed in as a plain text file. More...
	description
	help
	number of jobs More...
	type
	str
	action
	pre_args
	unknown
	argv
	index
	fullpath
	text
	newargs
	parser
	add_help
	required_args
	required options More...
	required
	debugging_args
	debugging More...
	default
	metavar
	job_control_args
	job control More...
	int
	False
	novasoft_args
	software control More...
	testrel_gp
	output_args
	output control More...
	environment_args
	environment configuration More...
	support_args
	support options More...
	args
	Process and check command line arguments #. More...
	timestamp
	test_njobs
	test_nevts
	test_dest
	mode
	test_expected_lifetime
	test_dynamic_lifetime
	test_files_per_job
	njobs
	nevts
	Setup runNovaSAM #. More...
	dest
	expected_lifetime
	dynamic_lifetime
	files_per_job
	declareFiles
	declareLocations
	autoDropbox
	poms
	offsite
	jobname
	defname
	snapshot_id
	print_jobsub
	printenv
	test
	tag
	srt_qual
	maxopt
	maxopt_opt
	fcl
	mcgen
	is_production_arg
	hashDirs
	role
	samweb
	Setup jobsub #. More...
	experiment
	num_project_files
	dimensions
	awsfilepath
	resource_opt
	site_opt
	allowed_os
	os_opt
	disk_opt
	mem_opt
	cpu_opt
	kill_opt
	dummy
	allowed_lifetimes
	cvmfs_rev
	group
	allowed_groups
	subgroup
	allowed_subgroups
	logs
	passThru
	va
	out_tiers
	outnum
	mcouttier
	copyback
	caf_tiers
	flat_tiers
	hist_tier
	h5_tiers
	script_path
	project_name
	start sam project More...
	start_project
	start_proj_command
	Start the SAM project #. More...
	start_proj_retval
	job_cmd
	scriptcount
	poms_campaign_id
	version
	dataset
	poms_task_id
	command_executed
	jobsub_cmd
	Actually launch the jobs #. More...

Function Documentation

def submit_nova_art.build_jobsub_cmd

(

)

Definition at line 162 of file submit_nova_art.py.

def build_jobsub_cmd():
    # Start with jobsub_submit and its options
    jobsub_cmd = "jobsub_submit \\\n"
    for opt in jobsub_opts:
        jobsub_cmd += "    " + opt + " \\\n"
    for export in export_to_art_sam_wrap:
        jobsub_cmd += "    -e " + export
    jobsub_cmd += " \\\n"

    # Add art_sam_wrap wrapper script and its options
    if args.testrel:
        if args.reuse_tarball:
            jobsub_cmd += "    --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
        else:
            jobsub_cmd += "    --tar_file_name tardir://" + args.testrel +" \\\n"
    elif args.user_tarball:
        if not os.path.isfile(args.user_tarball):
            print "Tarball filename passed to --user_tarball does not exit:", args.user_tarball
            sys.exit(5)
        jobsub_cmd += "    --tar_file_name dropbox://" + args.user_tarball + " \\\n"

    jobsub_cmd += "    file://" + art_sam_wrap_cmd + " \\\n"
    for opt in art_sam_wrap_opts:
        jobsub_cmd += "      " + opt + " \\\n"
    for export in export_to_run_nova_sam :
        jobsub_cmd += "      --export " + export + " \\\n"



    # Now set the main program and its options
    if not mcgen:
        jobsub_cmd += "      -X " + run_nova_sam_cmd + " \\\n"
        for opt in run_nova_sam_opts:
            jobsub_cmd += "        " + opt + " \\\n"
    else:
        jobsub_cmd += "      -X nova \\\n"
        mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
        if nevts>0 :
            mcgen_opts += ["-n %d" % nevts]
        for opt in mcgen_opts:
            jobsub_cmd += "        " + opt + " \\\n"

    jobsub_cmd = jobsub_cmd[:-2]
    return jobsub_cmd

def submit_nova_art.check_env

(

vname

)

Definition at line 86 of file submit_nova_art.py.

References common_tools.fail().

def check_env(vname):
    value=os.getenv(vname)
    if None == value or "" == value:
        fail("Environment variable %s not defined" %vname)

def submit_nova_art.check_fcl	(		tag,
			fcl
	)

Definition at line 91 of file submit_nova_art.py.

References common_tools.fail(), and split().

def check_fcl(tag,fcl):
    if "/" in fcl[:1] and os.path.isfile(fcl):
        return fcl # hey I'm ok!
    # Loop over dirs in FHICL_FILE_PATH
    fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
    for path in fclPaths:
        fullPath = os.path.join(path, fcl)
        if os.path.isfile(fullPath):
            return fcl # hey you're ok!
    # Check if they are copying the fhicl file to the worker node.
    for input_file in input_files:
        if os.path.basename(input_file) == fcl:
            return fcl # Passing fhicl as argument, all good.
        elif os.path.basename(input_file) == os.path.basename(fcl):
            print ""
            print "The fhicl will be copied to $PWD on the worker node."
            print "You specified some sort of file path which isn't needed. Fixing it for you :)"
            print fcl + " --> " + os.path.basename(fcl)
            print ""
            return os.path.basename(fcl) # User incorrectly left file path there.

    fail("fcl file %s does not exist" %fcl)

def submit_nova_art.find_file	(		paths,
			filename
	)

Definition at line 146 of file submit_nova_art.py.

References common_tools.fail().

def find_file(paths, filename):
    if os.path.isfile(filename):
        return filename
    for path in paths:
        for root, dirs, files in os.walk(os.path.expandvars(path)):
            if filename in files:
                return os.path.join(root, filename)
    fail("Cannot find file "+filename)

def submit_nova_art.find_file_in_list	(		filepath,
			pathlist
	)

Definition at line 155 of file submit_nova_art.py.

def find_file_in_list(filepath, pathlist):
    for testpath in pathlist:
        if os.path.basename(filepath) == os.path.basename(testpath):
            return True
    return False

def submit_nova_art.remove_comments

(

src

)

This reads tokens using tokenize.generate_tokens and recombines them
using tokenize.untokenize, and skipping comment/docstring tokens in between

Definition at line 115 of file submit_nova_art.py.

def remove_comments(src):
    """
    This reads tokens using tokenize.generate_tokens and recombines them
    using tokenize.untokenize, and skipping comment/docstring tokens in between
    """
    f = cStringIO.StringIO(src)
    class SkipException(Exception): pass
    processed_tokens = []
    last_token = None
    # go thru all the tokens and try to skip comments and docstrings
    for tok in tokenize.generate_tokens(f.readline):
        t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok

        try:
            if t_type == tokenize.COMMENT:
                raise SkipException()

            elif t_type == tokenize.STRING:

                if last_token is None or last_token[0] in [tokenize.INDENT]:
                    pass

        except SkipException:
            pass
        else:
            processed_tokens.append(tok)

        last_token = tok

    return tokenize.untokenize(processed_tokens)

Variable Documentation

submit_nova_art.action

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.add_help

Definition at line 251 of file submit_nova_art.py.

submit_nova_art.allowed_groups

Definition at line 1039 of file submit_nova_art.py.

submit_nova_art.allowed_lifetimes

Definition at line 1013 of file submit_nova_art.py.

submit_nova_art.allowed_os

Definition at line 960 of file submit_nova_art.py.

submit_nova_art.allowed_subgroups

Definition at line 1050 of file submit_nova_art.py.

submit_nova_art.args

Process and check command line arguments #.

Definition at line 687 of file submit_nova_art.py.

submit_nova_art.argv

Definition at line 232 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_cmd

Setup art_sam_wrap #.

Definition at line 57 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_opts

Definition at line 58 of file submit_nova_art.py.

submit_nova_art.autoDropbox

Definition at line 736 of file submit_nova_art.py.

submit_nova_art.awsfilepath

Definition at line 921 of file submit_nova_art.py.

submit_nova_art.build_location_arguments

Definition at line 48 of file submit_nova_art.py.

submit_nova_art.caf_tiers

Definition at line 1149 of file submit_nova_art.py.

submit_nova_art.command_executed

Definition at line 1351 of file submit_nova_art.py.

submit_nova_art.copyback

Definition at line 1141 of file submit_nova_art.py.

submit_nova_art.cpu_opt

Definition at line 977 of file submit_nova_art.py.

submit_nova_art.cvmfs_distro_base

Definition at line 46 of file submit_nova_art.py.

submit_nova_art.cvmfs_rev

Definition at line 1023 of file submit_nova_art.py.

submit_nova_art.dataset

Definition at line 1345 of file submit_nova_art.py.

submit_nova_art.debugging_args

debugging

Definition at line 283 of file submit_nova_art.py.

submit_nova_art.declareFiles

Definition at line 730 of file submit_nova_art.py.

submit_nova_art.declareLocations

Definition at line 733 of file submit_nova_art.py.

submit_nova_art.default

Definition at line 287 of file submit_nova_art.py.

submit_nova_art.defname

Definition at line 746 of file submit_nova_art.py.

submit_nova_art.description

Definition at line 222 of file submit_nova_art.py.

submit_nova_art.dest

Definition at line 721 of file submit_nova_art.py.

submit_nova_art.dimensions

Definition at line 875 of file submit_nova_art.py.

submit_nova_art.disk_opt

Definition at line 969 of file submit_nova_art.py.

submit_nova_art.dummy

Definition at line 1010 of file submit_nova_art.py.

submit_nova_art.dynamic_lifetime

Definition at line 725 of file submit_nova_art.py.

submit_nova_art.early_scripts

Definition at line 80 of file submit_nova_art.py.

submit_nova_art.environment_args

environment configuration

Definition at line 635 of file submit_nova_art.py.

submit_nova_art.expected_lifetime

Definition at line 723 of file submit_nova_art.py.

submit_nova_art.experiment

Definition at line 871 of file submit_nova_art.py.

submit_nova_art.export_to_art_sam_wrap

Definition at line 59 of file submit_nova_art.py.

submit_nova_art.export_to_run_nova_sam

Definition at line 55 of file submit_nova_art.py.

submit_nova_art.False

Definition at line 428 of file submit_nova_art.py.

submit_nova_art.fcl

Definition at line 803 of file submit_nova_art.py.

submit_nova_art.files_per_job

Definition at line 727 of file submit_nova_art.py.

submit_nova_art.flat_tiers

Definition at line 1154 of file submit_nova_art.py.

submit_nova_art.fullpath

Definition at line 239 of file submit_nova_art.py.

Referenced by geo::GeometryBase.getCoordinateTransformation().

submit_nova_art.group

Definition at line 1037 of file submit_nova_art.py.

submit_nova_art.h5_tiers

Definition at line 1163 of file submit_nova_art.py.

submit_nova_art.hashDirs

Definition at line 836 of file submit_nova_art.py.

submit_nova_art.help

number of jobs

general job control

Definition at line 225 of file submit_nova_art.py.

submit_nova_art.hist_tier

Definition at line 1159 of file submit_nova_art.py.

submit_nova_art.index

Definition at line 236 of file submit_nova_art.py.

submit_nova_art.input_files

Definition at line 77 of file submit_nova_art.py.

submit_nova_art.int

Definition at line 323 of file submit_nova_art.py.

submit_nova_art.inter_scripts

Definition at line 83 of file submit_nova_art.py.

submit_nova_art.is_production_arg

Definition at line 821 of file submit_nova_art.py.

submit_nova_art.job_cmd

Definition at line 1255 of file submit_nova_art.py.

submit_nova_art.job_control_args

job control

Definition at line 319 of file submit_nova_art.py.

submit_nova_art.jobname

Definition at line 745 of file submit_nova_art.py.

submit_nova_art.jobsub_cmd

Actually launch the jobs #.

Definition at line 1365 of file submit_nova_art.py.

submit_nova_art.jobsub_opts

Definition at line 51 of file submit_nova_art.py.

submit_nova_art.kill_opt

Definition at line 995 of file submit_nova_art.py.

submit_nova_art.logs

Definition at line 1088 of file submit_nova_art.py.

submit_nova_art.maxopt

Definition at line 767 of file submit_nova_art.py.

submit_nova_art.maxopt_opt

Definition at line 768 of file submit_nova_art.py.

submit_nova_art.mcgen

Definition at line 805 of file submit_nova_art.py.

Referenced by main(), NeutronOscGenerator(), and NucleonDecayGenerator().

submit_nova_art.mcouttier

Definition at line 1139 of file submit_nova_art.py.

submit_nova_art.mem_opt

Definition at line 973 of file submit_nova_art.py.

submit_nova_art.metavar

Definition at line 314 of file submit_nova_art.py.

submit_nova_art.mode

Definition at line 708 of file submit_nova_art.py.

Referenced by jmshower::JMShower.ANN(), ana::Bayesian1DMarginal.Bayesian1DMarginal(), ana::BayesianSurface.BayesianSurface(), ana.best_spline_find_mode(), caf::Proxy< caf::SRNeutrino >.CheckEquals(), DCSPlotMaker(), efficiency(), efficiencySA(), flat::FlatQepid.Fill(), flat::FlatQepid.FlatQepid(), GausFit(), genie::NNBarOscPrimaryVtxGenerator.GenerateDecayProducts(), cmf::ShifterAndWeighter.GeniePCASystWeight(), fnex::ShifterAndWeighter.GeniePCASystWeight(), GetCommandLineArgs(), MCGIDI_quantitiesLookupModes.getCrossSectionMode(), ana::NuWROSyst.InitializeHistograms(), ana::BayesianMarginal.LoadInto(), supernova::SnovaGen.MakeMCTruth(), dt::ViewMerger.MatchTracks(), ModeRestrict(), mrbrem_get_initial_spectra(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), NuSCalculateCorr(), caf::Proxy< caf::SRNeutrino >.operator=(), cet::ostream_handle.ostream_handle(), plot_nd_spectra_2018(), pots(), genie::GHepRecord.ProbePosition(), slidt.SaveHists(), ana::BayesianMarginal.SaveTo(), evgen::GENIENeutronOscGen.selectAnnihilationMode(), SelectAnnihilationMode(), jmshower::JMShower.SetANN(), MCGIDI_quantitiesLookupModes.setCrossSectionMode(), evd::SliceNavigator.SetFullEventMode(), genie::GHepRecord.TargetNucleusPosition(), genie::utils::intranuke.ThreeBodyKinematics(), genie::utils::intranuke2018.ThreeBodyKinematics(), uptime(), useBDTG(), and jmshower::JMShower.~JMShower().

submit_nova_art.nevts

Setup runNovaSAM #.

Definition at line 719 of file submit_nova_art.py.

Referenced by ndrockmacro(), and plot_xsecs().

submit_nova_art.newargs

Definition at line 244 of file submit_nova_art.py.

submit_nova_art.njobs

Definition at line 717 of file submit_nova_art.py.

submit_nova_art.novasoft_args

software control

Definition at line 465 of file submit_nova_art.py.

submit_nova_art.novasoft_cvmfs

Definition at line 47 of file submit_nova_art.py.

submit_nova_art.num_project_files

Definition at line 873 of file submit_nova_art.py.

submit_nova_art.offsite

Definition at line 743 of file submit_nova_art.py.

submit_nova_art.os_opt

Definition at line 965 of file submit_nova_art.py.

submit_nova_art.out_tiers

Definition at line 1136 of file submit_nova_art.py.

submit_nova_art.outnum

Definition at line 1139 of file submit_nova_art.py.

submit_nova_art.output_args

output control

Definition at line 517 of file submit_nova_art.py.

submit_nova_art.parser

Definition at line 251 of file submit_nova_art.py.

submit_nova_art.passThru

Definition at line 1095 of file submit_nova_art.py.

submit_nova_art.poms

Definition at line 739 of file submit_nova_art.py.

submit_nova_art.poms_campaign_id

Definition at line 1340 of file submit_nova_art.py.

submit_nova_art.poms_task_id

Definition at line 1348 of file submit_nova_art.py.

submit_nova_art.post_scripts

Definition at line 84 of file submit_nova_art.py.

submit_nova_art.pre_args

Definition at line 229 of file submit_nova_art.py.

submit_nova_art.pre_scripts

Definition at line 82 of file submit_nova_art.py.

submit_nova_art.preliminary_parser

Allow args to be passed in as a plain text file.

We make a preliminary parser get these arguments out for two reasons: 1) Maintain standard -h, –help functionality 2) Avoid necessity required arguments in initial parsing, allow them to be missing, but find them in the file.

Definition at line 222 of file submit_nova_art.py.

submit_nova_art.print_jobsub

Definition at line 748 of file submit_nova_art.py.

submit_nova_art.printenv

Definition at line 752 of file submit_nova_art.py.

submit_nova_art.prog

Definition at line 212 of file submit_nova_art.py.

submit_nova_art.project_name

start sam project

Definition at line 1213 of file submit_nova_art.py.

submit_nova_art.recommended_sites

Definition at line 21 of file submit_nova_art.py.

submit_nova_art.required

Definition at line 257 of file submit_nova_art.py.

submit_nova_art.required_args

required options

Definition at line 254 of file submit_nova_art.py.

submit_nova_art.resource_opt

Definition at line 936 of file submit_nova_art.py.

submit_nova_art.role

Definition at line 841 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_cmd

Definition at line 53 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_opts

Definition at line 54 of file submit_nova_art.py.

submit_nova_art.sam_station

Definition at line 18 of file submit_nova_art.py.

submit_nova_art.sam_user

Definition at line 17 of file submit_nova_art.py.

submit_nova_art.samweb

Setup jobsub #.

both njobs and files per job are specified. Just pass the settings through to jobsub and art_sam_wrap files/job specified, but not njobs. Calculate njobs on the fly get files in dataset

Definition at line 871 of file submit_nova_art.py.

submit_nova_art.script_path

Definition at line 1203 of file submit_nova_art.py.

submit_nova_art.scriptcount

Definition at line 1256 of file submit_nova_art.py.

submit_nova_art.setup_location

Definition at line 49 of file submit_nova_art.py.

submit_nova_art.site_opt

Definition at line 940 of file submit_nova_art.py.

submit_nova_art.snapshot_id

Definition at line 747 of file submit_nova_art.py.

submit_nova_art.source_scripts

Definition at line 81 of file submit_nova_art.py.

submit_nova_art.srt_qual

Definition at line 766 of file submit_nova_art.py.

submit_nova_art.start_proj_command

Start the SAM project #.

Definition at line 1235 of file submit_nova_art.py.

submit_nova_art.start_proj_retval

Definition at line 1245 of file submit_nova_art.py.

submit_nova_art.start_project

Definition at line 1216 of file submit_nova_art.py.

submit_nova_art.str

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.subgroup

Definition at line 1048 of file submit_nova_art.py.

submit_nova_art.support_args

support options

Definition at line 670 of file submit_nova_art.py.

submit_nova_art.tag

Definition at line 764 of file submit_nova_art.py.

submit_nova_art.test

Definition at line 754 of file submit_nova_art.py.

submit_nova_art.test_dest

Definition at line 705 of file submit_nova_art.py.

submit_nova_art.test_dynamic_lifetime

Definition at line 711 of file submit_nova_art.py.

submit_nova_art.test_expected_lifetime

Definition at line 710 of file submit_nova_art.py.

submit_nova_art.test_files_per_job

Definition at line 712 of file submit_nova_art.py.

submit_nova_art.test_nevts

Definition at line 704 of file submit_nova_art.py.

submit_nova_art.test_njobs

Definition at line 703 of file submit_nova_art.py.

submit_nova_art.testrel_gp

Definition at line 470 of file submit_nova_art.py.

submit_nova_art.text

Definition at line 242 of file submit_nova_art.py.

submit_nova_art.timestamp

Definition at line 688 of file submit_nova_art.py.

Referenced by daq2raw::BeamlineEvent.AddTimestamp(), beamlinereco::CFDHitFinder< double >.CFDHitFinder(), mcdatadaq::DAQSimulator.fillMicroSlices(), stan::test::performance.get_git_date(), dbutils::daqconfig::NamedConfigUtils.getAvailableGlobalConfigs(), dbutils::daqconfig::NamedConfigUtils.getAvailableSubsystemConfigs(), getEventList(), mcdatadaq::DAQSimulator.getNanoSliceOrder(), dbutils::daqconfig::NamedConfigUtils.getRecentGlobalConfigs(), daqdataformats::VERSION_NAMESPACE::RawSummaryDroppedMicroblock.init(), daqdataformats::VERSION_NAMESPACE::RawSummaryDCMDataPoint.init(), daqdataformats::VERSION_NAMESPACE::RawTriggerTimingMarker.init(), daqdataformats::VERSION_NAMESPACE::RawTriggerTime.init(), beamlinereco::LEHitFinder< double >.LEHitFinder(), daq2raw::BeamlineRawInputDriver.makeSpill(), OnMonPlotMaker(), beamlinereco::LEHitReco.produce(), beamlinereco::CFDHitReco.produce(), gov::fnal::cd::rms::base::RmsMessage.RmsMessage(), brb::ToF.SetTimestamp(), rawdata::RawBeamlineTrigger.SetTimestamp(), rawdata::RawBeamlineDigit.SetTimestamp(), rawdata::RawBeamlineWC.SetTimestamp(), daq2raw::BeamlineUnpack.UnpackDigits(), daq2raw::BeamlineUnpack.UnpackTrigger(), daq2raw::BeamlineUnpack.UnpackWC(), rawdata::RawBeamlineDigit.~RawBeamlineDigit(), rawdata::RawBeamlineTrigger.~RawBeamlineTrigger(), and rawdata::RawBeamlineWC.~RawBeamlineWC().

submit_nova_art.type

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.unknown

Definition at line 229 of file submit_nova_art.py.

submit_nova_art.usage_models

njobs specified, but not files/job.

Just set njobs and don't force limits on files per jobs

Definition at line 75 of file submit_nova_art.py.

submit_nova_art.user

Definition at line 14 of file submit_nova_art.py.

submit_nova_art.va

Definition at line 1097 of file submit_nova_art.py.

Referenced by ana.CombineExposures(), ana::MultiVar2DFunc.operator()(), ana::Var2DFunc< T >.operator()(), ana::Var3DFunc< T >.operator()(), SetDrawOptions(), test_fitvarsproduct(), and ana.ValHelper().

submit_nova_art.version

Definition at line 1344 of file submit_nova_art.py.

submit_nova_art.veryearly_scripts

Definition at line 79 of file submit_nova_art.py.