Functions | Variables
submit_nova_art Namespace Reference

Functions

def check_env (vname)
 
def check_fcl (tag, fcl)
 
def remove_comments (src)
 
def find_file (paths, filename)
 
def find_file_in_list (filepath, pathlist)
 
def build_jobsub_cmd ()
 

Variables

 user
 
 sam_user
 
 sam_station
 
 recommended_sites
 
 cvmfs_distro_base
 
 novasoft_cvmfs
 
 build_location_arguments
 
 setup_location
 
 jobsub_opts
 
 run_nova_sam_cmd
 
 run_nova_sam_opts
 
 export_to_run_nova_sam
 
 art_sam_wrap_cmd
 Setup art_sam_wrap #. More...
 
 art_sam_wrap_opts
 
 export_to_art_sam_wrap
 
 usage_models
 njobs specified, but not files/job. More...
 
 input_files
 
 veryearly_scripts
 
 early_scripts
 
 source_scripts
 
 pre_scripts
 
 inter_scripts
 
 post_scripts
 
 prog
 
 preliminary_parser
 Allow args to be passed in as a plain text file. More...
 
 description
 
 help
 number of jobs More...
 
 type
 
 str
 
 action
 
 pre_args
 
 unknown
 
 argv
 
 index
 
 fullpath
 
 text
 
 newargs
 
 parser
 
 add_help
 
 required_args
 required options More...
 
 required
 
 debugging_args
 debugging More...
 
 default
 
 metavar
 
 job_control_args
 job control More...
 
 int
 
 False
 
 novasoft_args
 software control More...
 
 testrel_gp
 
 output_args
 output control More...
 
 environment_args
 environment configuration More...
 
 support_args
 support options More...
 
 args
 Process and check command line arguments #. More...
 
 timestamp
 
 test_njobs
 
 test_nevts
 
 test_dest
 
 mode
 
 test_expected_lifetime
 
 test_dynamic_lifetime
 
 test_files_per_job
 
 njobs
 
 nevts
 Setup runNovaSAM #. More...
 
 dest
 
 expected_lifetime
 
 dynamic_lifetime
 
 files_per_job
 
 declareFiles
 
 declareLocations
 
 autoDropbox
 
 poms
 
 offsite
 
 jobname
 
 defname
 
 snapshot_id
 
 print_jobsub
 
 printenv
 
 test
 
 tag
 
 srt_qual
 
 maxopt
 
 maxopt_opt
 
 fcl
 
 mcgen
 
 is_production_arg
 
 hashDirs
 
 role
 
 samweb
 Setup jobsub #. More...
 
 experiment
 
 num_project_files
 
 dimensions
 
 awsfilepath
 
 resource_opt
 
 site_opt
 
 allowed_os
 
 os_opt
 
 disk_opt
 
 mem_opt
 
 cpu_opt
 
 kill_opt
 
 dummy
 
 allowed_lifetimes
 
 cvmfs_rev
 
 group
 
 allowed_groups
 
 subgroup
 
 allowed_subgroups
 
 logs
 
 passThru
 
 va
 
 out_tiers
 
 outnum
 
 mcouttier
 
 copyback
 
 caf_tiers
 
 flat_tiers
 
 hist_tier
 
 h5_tiers
 
 script_path
 
 project_name
 start sam project More...
 
 start_project
 
 start_proj_command
 Start the SAM project #. More...
 
 start_proj_retval
 
 job_cmd
 
 scriptcount
 
 poms_campaign_id
 
 version
 
 dataset
 
 poms_task_id
 
 command_executed
 
 jobsub_cmd
 Actually launch the jobs #. More...
 

Function Documentation

def submit_nova_art.build_jobsub_cmd ( )

Definition at line 162 of file submit_nova_art.py.

163  # Start with jobsub_submit and its options
164  jobsub_cmd = "jobsub_submit \\\n"
165  for opt in jobsub_opts:
166  jobsub_cmd += " " + opt + " \\\n"
167  for export in export_to_art_sam_wrap:
168  jobsub_cmd += " -e " + export
169  jobsub_cmd += " \\\n"
170 
171  # Add art_sam_wrap wrapper script and its options
172  if args.testrel:
173  if args.reuse_tarball:
174  jobsub_cmd += " --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
175  else:
176  jobsub_cmd += " --tar_file_name tardir://" + args.testrel +" \\\n"
177  elif args.user_tarball:
178  if not os.path.isfile(args.user_tarball):
179  print "Tarball filename passed to --user_tarball does not exit:", args.user_tarball
180  sys.exit(5)
181  jobsub_cmd += " --tar_file_name dropbox://" + args.user_tarball + " \\\n"
182 
183  jobsub_cmd += " file://" + art_sam_wrap_cmd + " \\\n"
184  for opt in art_sam_wrap_opts:
185  jobsub_cmd += " " + opt + " \\\n"
186  for export in export_to_run_nova_sam :
187  jobsub_cmd += " --export " + export + " \\\n"
188 
189 
190 
191  # Now set the main program and its options
192  if not mcgen:
193  jobsub_cmd += " -X " + run_nova_sam_cmd + " \\\n"
194  for opt in run_nova_sam_opts:
195  jobsub_cmd += " " + opt + " \\\n"
196  else:
197  jobsub_cmd += " -X nova \\\n"
198  mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
199  if nevts>0 :
200  mcgen_opts += ["-n %d" % nevts]
201  for opt in mcgen_opts:
202  jobsub_cmd += " " + opt + " \\\n"
203 
204  jobsub_cmd = jobsub_cmd[:-2]
205  return jobsub_cmd
206 
207 
def submit_nova_art.check_env (   vname)

Definition at line 86 of file submit_nova_art.py.

References common_tools.fail().

86 def check_env(vname):
87  value=os.getenv(vname)
88  if None == value or "" == value:
89  fail("Environment variable %s not defined" %vname)
90 
def fail(msg)
print a failure message, from: https://cdcvs.fnal.gov/redmine/projects/novaart/repository/entry/trunk...
Definition: common_tools.py:7
def check_env(vname)
def submit_nova_art.check_fcl (   tag,
  fcl 
)

Definition at line 91 of file submit_nova_art.py.

References common_tools.fail(), and split().

91 def check_fcl(tag,fcl):
92  if "/" in fcl[:1] and os.path.isfile(fcl):
93  return fcl # hey I'm ok!
94  # Loop over dirs in FHICL_FILE_PATH
95  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
96  for path in fclPaths:
97  fullPath = os.path.join(path, fcl)
98  if os.path.isfile(fullPath):
99  return fcl # hey you're ok!
100  # Check if they are copying the fhicl file to the worker node.
101  for input_file in input_files:
102  if os.path.basename(input_file) == fcl:
103  return fcl # Passing fhicl as argument, all good.
104  elif os.path.basename(input_file) == os.path.basename(fcl):
105  print ""
106  print "The fhicl will be copied to $PWD on the worker node."
107  print "You specified some sort of file path which isn't needed. Fixing it for you :)"
108  print fcl + " --> " + os.path.basename(fcl)
109  print ""
110  return os.path.basename(fcl) # User incorrectly left file path there.
111 
112  fail("fcl file %s does not exist" %fcl)
113 
114 
void split(double tt, double *fr)
def fail(msg)
print a failure message, from: https://cdcvs.fnal.gov/redmine/projects/novaart/repository/entry/trunk...
Definition: common_tools.py:7
def check_fcl(tag, fcl)
def submit_nova_art.find_file (   paths,
  filename 
)

Definition at line 146 of file submit_nova_art.py.

References common_tools.fail().

146 def find_file(paths, filename):
147  if os.path.isfile(filename):
148  return filename
149  for path in paths:
150  for root, dirs, files in os.walk(os.path.expandvars(path)):
151  if filename in files:
152  return os.path.join(root, filename)
153  fail("Cannot find file "+filename)
154 
def find_file(paths, filename)
def fail(msg)
print a failure message, from: https://cdcvs.fnal.gov/redmine/projects/novaart/repository/entry/trunk...
Definition: common_tools.py:7
def submit_nova_art.find_file_in_list (   filepath,
  pathlist 
)

Definition at line 155 of file submit_nova_art.py.

155 def find_file_in_list(filepath, pathlist):
156  for testpath in pathlist:
157  if os.path.basename(filepath) == os.path.basename(testpath):
158  return True
159  return False
160 
161 
def find_file_in_list(filepath, pathlist)
def submit_nova_art.remove_comments (   src)
This reads tokens using tokenize.generate_tokens and recombines them
using tokenize.untokenize, and skipping comment/docstring tokens in between

Definition at line 115 of file submit_nova_art.py.

116  """
117  This reads tokens using tokenize.generate_tokens and recombines them
118  using tokenize.untokenize, and skipping comment/docstring tokens in between
119  """
120  f = cStringIO.StringIO(src)
121  class SkipException(Exception): pass
122  processed_tokens = []
123  last_token = None
124  # go thru all the tokens and try to skip comments and docstrings
125  for tok in tokenize.generate_tokens(f.readline):
126  t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok
127 
128  try:
129  if t_type == tokenize.COMMENT:
130  raise SkipException()
131 
132  elif t_type == tokenize.STRING:
133 
134  if last_token is None or last_token[0] in [tokenize.INDENT]:
135  pass
136 
137  except SkipException:
138  pass
139  else:
140  processed_tokens.append(tok)
141 
142  last_token = tok
143 
144  return tokenize.untokenize(processed_tokens)
145 
def remove_comments(src)
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66

Variable Documentation

submit_nova_art.action

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.add_help

Definition at line 251 of file submit_nova_art.py.

submit_nova_art.allowed_groups

Definition at line 1039 of file submit_nova_art.py.

submit_nova_art.allowed_lifetimes

Definition at line 1013 of file submit_nova_art.py.

submit_nova_art.allowed_os

Definition at line 960 of file submit_nova_art.py.

submit_nova_art.allowed_subgroups

Definition at line 1050 of file submit_nova_art.py.

submit_nova_art.args

Process and check command line arguments #.

Definition at line 687 of file submit_nova_art.py.

submit_nova_art.argv

Definition at line 232 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_cmd

Setup art_sam_wrap #.

Definition at line 57 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_opts

Definition at line 58 of file submit_nova_art.py.

submit_nova_art.autoDropbox

Definition at line 736 of file submit_nova_art.py.

submit_nova_art.awsfilepath

Definition at line 921 of file submit_nova_art.py.

submit_nova_art.build_location_arguments

Definition at line 48 of file submit_nova_art.py.

submit_nova_art.caf_tiers

Definition at line 1149 of file submit_nova_art.py.

submit_nova_art.command_executed

Definition at line 1351 of file submit_nova_art.py.

submit_nova_art.copyback

Definition at line 1141 of file submit_nova_art.py.

submit_nova_art.cpu_opt

Definition at line 977 of file submit_nova_art.py.

submit_nova_art.cvmfs_distro_base

Definition at line 46 of file submit_nova_art.py.

submit_nova_art.cvmfs_rev

Definition at line 1023 of file submit_nova_art.py.

submit_nova_art.dataset

Definition at line 1345 of file submit_nova_art.py.

submit_nova_art.debugging_args

debugging

Definition at line 283 of file submit_nova_art.py.

submit_nova_art.declareFiles

Definition at line 730 of file submit_nova_art.py.

submit_nova_art.declareLocations

Definition at line 733 of file submit_nova_art.py.

submit_nova_art.default

Definition at line 287 of file submit_nova_art.py.

submit_nova_art.defname

Definition at line 746 of file submit_nova_art.py.

submit_nova_art.description

Definition at line 222 of file submit_nova_art.py.

submit_nova_art.dest

Definition at line 721 of file submit_nova_art.py.

submit_nova_art.dimensions

Definition at line 875 of file submit_nova_art.py.

submit_nova_art.disk_opt

Definition at line 969 of file submit_nova_art.py.

submit_nova_art.dummy

Definition at line 1010 of file submit_nova_art.py.

submit_nova_art.dynamic_lifetime

Definition at line 725 of file submit_nova_art.py.

submit_nova_art.early_scripts

Definition at line 80 of file submit_nova_art.py.

submit_nova_art.environment_args

environment configuration

Definition at line 635 of file submit_nova_art.py.

submit_nova_art.expected_lifetime

Definition at line 723 of file submit_nova_art.py.

submit_nova_art.experiment

Definition at line 871 of file submit_nova_art.py.

submit_nova_art.export_to_art_sam_wrap

Definition at line 59 of file submit_nova_art.py.

submit_nova_art.export_to_run_nova_sam

Definition at line 55 of file submit_nova_art.py.

submit_nova_art.False

Definition at line 428 of file submit_nova_art.py.

submit_nova_art.fcl

Definition at line 803 of file submit_nova_art.py.

submit_nova_art.files_per_job

Definition at line 727 of file submit_nova_art.py.

submit_nova_art.flat_tiers

Definition at line 1154 of file submit_nova_art.py.

submit_nova_art.fullpath

Definition at line 239 of file submit_nova_art.py.

Referenced by geo::GeometryBase.getCoordinateTransformation().

submit_nova_art.group

Definition at line 1037 of file submit_nova_art.py.

submit_nova_art.h5_tiers

Definition at line 1163 of file submit_nova_art.py.

submit_nova_art.hashDirs

Definition at line 836 of file submit_nova_art.py.

submit_nova_art.help

number of jobs

general job control

Definition at line 225 of file submit_nova_art.py.

submit_nova_art.hist_tier

Definition at line 1159 of file submit_nova_art.py.

submit_nova_art.index

Definition at line 236 of file submit_nova_art.py.

submit_nova_art.input_files

Definition at line 77 of file submit_nova_art.py.

submit_nova_art.int

Definition at line 323 of file submit_nova_art.py.

submit_nova_art.inter_scripts

Definition at line 83 of file submit_nova_art.py.

submit_nova_art.is_production_arg

Definition at line 821 of file submit_nova_art.py.

submit_nova_art.job_cmd

Definition at line 1255 of file submit_nova_art.py.

submit_nova_art.job_control_args

job control

Definition at line 319 of file submit_nova_art.py.

submit_nova_art.jobname

Definition at line 745 of file submit_nova_art.py.

submit_nova_art.jobsub_cmd

Actually launch the jobs #.

Definition at line 1365 of file submit_nova_art.py.

submit_nova_art.jobsub_opts

Definition at line 51 of file submit_nova_art.py.

submit_nova_art.kill_opt

Definition at line 995 of file submit_nova_art.py.

submit_nova_art.logs

Definition at line 1088 of file submit_nova_art.py.

submit_nova_art.maxopt

Definition at line 767 of file submit_nova_art.py.

submit_nova_art.maxopt_opt

Definition at line 768 of file submit_nova_art.py.

submit_nova_art.mcgen

Definition at line 805 of file submit_nova_art.py.

Referenced by main(), NeutronOscGenerator(), and NucleonDecayGenerator().

submit_nova_art.mcouttier

Definition at line 1139 of file submit_nova_art.py.

submit_nova_art.mem_opt

Definition at line 973 of file submit_nova_art.py.

submit_nova_art.metavar

Definition at line 314 of file submit_nova_art.py.

submit_nova_art.mode

Definition at line 708 of file submit_nova_art.py.

Referenced by jmshower::JMShower.ANN(), ana::Bayesian1DMarginal.Bayesian1DMarginal(), ana::BayesianSurface.BayesianSurface(), ana.best_spline_find_mode(), caf::Proxy< caf::SRNeutrino >.CheckEquals(), DCSPlotMaker(), efficiency(), efficiencySA(), flat::FlatQepid.Fill(), flat::FlatQepid.FlatQepid(), GausFit(), genie::NNBarOscPrimaryVtxGenerator.GenerateDecayProducts(), cmf::ShifterAndWeighter.GeniePCASystWeight(), fnex::ShifterAndWeighter.GeniePCASystWeight(), GetCommandLineArgs(), MCGIDI_quantitiesLookupModes.getCrossSectionMode(), ana::HistogramTypePolicy.HistogramTypePolicy(), ana::NuWROSyst.InitializeHistograms(), ana::BayesianMarginal.LoadInto(), supernova::SnovaGen.MakeMCTruth(), dt::ViewMerger.MatchTracks(), ModeRestrict(), mrbrem_get_initial_spectra(), mrbrem_get_reweighted_spectra(), mrbrem_plots(), caf::Proxy< caf::SRNeutrino >.operator=(), cet::ostream_handle.ostream_handle(), plot_nd_spectra_2018(), pots(), ana::PredictionInterp.PredictionInterp(), genie::GHepRecord.ProbePosition(), slidt.SaveHists(), ana::BayesianMarginal.SaveTo(), evgen::GENIENeutronOscGen.selectAnnihilationMode(), SelectAnnihilationMode(), jmshower::JMShower.SetANN(), MCGIDI_quantitiesLookupModes.setCrossSectionMode(), evd::SliceNavigator.SetFullEventMode(), genie::GHepRecord.TargetNucleusPosition(), genie::utils::intranuke.ThreeBodyKinematics(), genie::utils::intranuke2018.ThreeBodyKinematics(), uptime(), useBDTG(), and jmshower::JMShower.~JMShower().

submit_nova_art.nevts

Setup runNovaSAM #.

Definition at line 719 of file submit_nova_art.py.

Referenced by ndrockmacro(), and plot_xsecs().

submit_nova_art.newargs

Definition at line 244 of file submit_nova_art.py.

submit_nova_art.njobs

Definition at line 717 of file submit_nova_art.py.

submit_nova_art.novasoft_args

software control

Definition at line 465 of file submit_nova_art.py.

submit_nova_art.novasoft_cvmfs

Definition at line 47 of file submit_nova_art.py.

submit_nova_art.num_project_files

Definition at line 873 of file submit_nova_art.py.

submit_nova_art.offsite

Definition at line 743 of file submit_nova_art.py.

submit_nova_art.os_opt

Definition at line 965 of file submit_nova_art.py.

submit_nova_art.out_tiers

Definition at line 1136 of file submit_nova_art.py.

submit_nova_art.outnum

Definition at line 1139 of file submit_nova_art.py.

submit_nova_art.output_args

output control

Definition at line 517 of file submit_nova_art.py.

submit_nova_art.parser

Definition at line 251 of file submit_nova_art.py.

submit_nova_art.passThru

Definition at line 1095 of file submit_nova_art.py.

submit_nova_art.poms

Definition at line 739 of file submit_nova_art.py.

submit_nova_art.poms_campaign_id

Definition at line 1340 of file submit_nova_art.py.

submit_nova_art.poms_task_id

Definition at line 1348 of file submit_nova_art.py.

submit_nova_art.post_scripts

Definition at line 84 of file submit_nova_art.py.

submit_nova_art.pre_args

Definition at line 229 of file submit_nova_art.py.

submit_nova_art.pre_scripts

Definition at line 82 of file submit_nova_art.py.

submit_nova_art.preliminary_parser

Allow args to be passed in as a plain text file.

We make a preliminary parser get these arguments out for two reasons: 1) Maintain standard -h, –help functionality 2) Avoid necessity required arguments in initial parsing, allow them to be missing, but find them in the file.

Definition at line 222 of file submit_nova_art.py.

submit_nova_art.print_jobsub

Definition at line 748 of file submit_nova_art.py.

submit_nova_art.printenv

Definition at line 752 of file submit_nova_art.py.

submit_nova_art.prog

Definition at line 212 of file submit_nova_art.py.

submit_nova_art.project_name

start sam project

Definition at line 1213 of file submit_nova_art.py.

submit_nova_art.recommended_sites

Definition at line 21 of file submit_nova_art.py.

submit_nova_art.required

Definition at line 257 of file submit_nova_art.py.

submit_nova_art.required_args

required options

Definition at line 254 of file submit_nova_art.py.

submit_nova_art.resource_opt

Definition at line 936 of file submit_nova_art.py.

submit_nova_art.role

Definition at line 841 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_cmd

Definition at line 53 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_opts

Definition at line 54 of file submit_nova_art.py.

submit_nova_art.sam_station

Definition at line 18 of file submit_nova_art.py.

submit_nova_art.sam_user

Definition at line 17 of file submit_nova_art.py.

submit_nova_art.samweb

Setup jobsub #.

both njobs and files per job are specified. Just pass the settings through to jobsub and art_sam_wrap files/job specified, but not njobs. Calculate njobs on the fly get files in dataset

Definition at line 871 of file submit_nova_art.py.

submit_nova_art.script_path

Definition at line 1203 of file submit_nova_art.py.

submit_nova_art.scriptcount

Definition at line 1256 of file submit_nova_art.py.

submit_nova_art.setup_location

Definition at line 49 of file submit_nova_art.py.

submit_nova_art.site_opt

Definition at line 940 of file submit_nova_art.py.

submit_nova_art.snapshot_id

Definition at line 747 of file submit_nova_art.py.

submit_nova_art.source_scripts

Definition at line 81 of file submit_nova_art.py.

submit_nova_art.srt_qual

Definition at line 766 of file submit_nova_art.py.

submit_nova_art.start_proj_command

Start the SAM project #.

Definition at line 1235 of file submit_nova_art.py.

submit_nova_art.start_proj_retval

Definition at line 1245 of file submit_nova_art.py.

submit_nova_art.start_project

Definition at line 1216 of file submit_nova_art.py.

submit_nova_art.str

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.subgroup

Definition at line 1048 of file submit_nova_art.py.

submit_nova_art.support_args

support options

Definition at line 670 of file submit_nova_art.py.

submit_nova_art.tag

Definition at line 764 of file submit_nova_art.py.

submit_nova_art.test

Definition at line 754 of file submit_nova_art.py.

submit_nova_art.test_dest

Definition at line 705 of file submit_nova_art.py.

submit_nova_art.test_dynamic_lifetime

Definition at line 711 of file submit_nova_art.py.

submit_nova_art.test_expected_lifetime

Definition at line 710 of file submit_nova_art.py.

submit_nova_art.test_files_per_job

Definition at line 712 of file submit_nova_art.py.

submit_nova_art.test_nevts

Definition at line 704 of file submit_nova_art.py.

submit_nova_art.test_njobs

Definition at line 703 of file submit_nova_art.py.

submit_nova_art.testrel_gp

Definition at line 470 of file submit_nova_art.py.

submit_nova_art.text

Definition at line 242 of file submit_nova_art.py.

submit_nova_art.timestamp
submit_nova_art.type

Definition at line 228 of file submit_nova_art.py.

submit_nova_art.unknown

Definition at line 229 of file submit_nova_art.py.

submit_nova_art.usage_models

njobs specified, but not files/job.

Just set njobs and don't force limits on files per jobs

Definition at line 75 of file submit_nova_art.py.

submit_nova_art.user

Definition at line 14 of file submit_nova_art.py.

submit_nova_art.va
submit_nova_art.version

Definition at line 1344 of file submit_nova_art.py.

submit_nova_art.veryearly_scripts

Definition at line 79 of file submit_nova_art.py.