Functions | Variables
submit_nova_art Namespace Reference

Functions

def check_env (vname)
 
def check_fcl (tag, fcl)
 
def remove_comments (src)
 
def find_file (paths, filename)
 
def find_file_in_list (filepath, pathlist)
 
def build_jobsub_cmd ()
 

Variables

 user
 
 sam_user
 
 sam_station
 
 cvmfs_distro_base
 
 novasoft_cvmfs
 
 build_location_arguments
 
 setup_location
 
 jobsub_opts
 
 run_nova_sam_cmd
 
 run_nova_sam_opts
 
 export_to_run_nova_sam
 
 art_sam_wrap_cmd
 Setup art_sam_wrap #. More...
 
 art_sam_wrap_opts
 
 export_to_art_sam_wrap
 
 usage_models
 
 input_files
 
 veryearly_scripts
 
 early_scripts
 
 source_scripts
 
 pre_scripts
 
 inter_scripts
 
 post_scripts
 
 prog
 
 preliminary_parser
 Allow args to be passed in as a plain text file. More...
 
 description
 
 help
 number of jobs More...
 
 type
 
 str
 
 action
 
 pre_args
 
 unknown
 
 argv
 
 index
 
 fullpath
 
 text
 
 newargs
 
 parser
 
 add_help
 
 required_args
 required options More...
 
 required
 
 debugging_args
 debugging More...
 
 default
 
 metavar
 
 job_control_args
 job control More...
 
 int
 
 sing_group
 
 choices
 
 False
 
 novasoft_args
 software control More...
 
 testrel_gp
 
 output_args
 output control More...
 
 environment_args
 environment configuration More...
 
 support_args
 support options More...
 
 args
 Process and check command line arguments #. More...
 
 timestamp
 
 recommended_sites
 
 istarball
 
 use_recommended_sites
 
 test_njobs
 
 test_nevts
 
 test_dest
 
 mode
 
 test_expected_lifetime
 
 test_dynamic_lifetime
 
 test_files_per_job
 
 njobs
 
 nevts
 Setup runNovaSAM #. More...
 
 dest
 
 expected_lifetime
 
 dynamic_lifetime
 
 files_per_job
 
 declareFiles
 
 declareLocations
 
 autoDropbox
 
 poms
 
 offsite
 
 jobname
 
 defname
 
 snapshot_id
 
 print_jobsub
 
 printenv
 
 test
 
 tag
 
 srt_qual
 
 maxopt
 
 maxopt_opt
 
 fcl
 
 mcgen
 
 is_production_arg
 
 hashDirs
 
 role
 
 samweb
 Setup jobsub #. More...
 
 experiment
 
 num_project_files
 
 dimensions
 
 file
 njobs specified, but not files/job. More...
 
 awsfilepath
 
 singularity
 
 resource_opt
 
 site_opt
 
 disk_opt
 
 mem_opt
 
 cpu_opt
 
 kill_opt
 
 dummy
 
 allowed_lifetimes
 
 ngu_current
 
 shell
 
 cvmfs_rev
 
 group
 
 allowed_groups
 
 subgroup
 
 allowed_subgroups
 
 logs
 
 passThru
 
 va
 
 out_tiers
 
 outnum
 
 mcouttier
 
 copyback
 
 caf_tiers
 
 flat_tiers
 
 hist_tier
 
 h5_tiers
 
 script_path
 
 project_name
 start sam project More...
 
 start_project
 
 start_proj_command
 Start the SAM project #. More...
 
 start_proj_retval
 
 job_cmd
 
 scriptcount
 
 poms_campaign_id
 
 version
 
 dataset
 
 poms_task_id
 
 command_executed
 
 jobsub_cmd
 Actually launch the jobs #. More...
 

Function Documentation

def submit_nova_art.build_jobsub_cmd ( )

Definition at line 144 of file submit_nova_art.py.

References print.

145  # Start with jobsub_submit and its options
146  jobsub_cmd = "jobsub_submit \\\n"
147  for opt in jobsub_opts:
148  jobsub_cmd += " " + opt + " \\\n"
149  for export in export_to_art_sam_wrap:
150  jobsub_cmd += " -e " + export
151  jobsub_cmd += " \\\n"
152 
153  # Add art_sam_wrap wrapper script and its options
154  if args.testrel:
155  if args.reuse_tarball:
156  jobsub_cmd += " --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
157  else:
158  jobsub_cmd += " --tar_file_name tardir://" + args.testrel +" \\\n"
159  elif args.user_tarball:
160  if not os.path.isfile(args.user_tarball):
161  print("Tarball filename passed to --user_tarball does not exit:", args.user_tarball)
162  sys.exit(5)
163  jobsub_cmd += " --tar_file_name dropbox://" + args.user_tarball + " \\\n"
164 
165  jobsub_cmd += " file://" + art_sam_wrap_cmd + " \\\n"
166  for opt in art_sam_wrap_opts:
167  jobsub_cmd += " " + opt + " \\\n"
168  for export in export_to_run_nova_sam :
169  jobsub_cmd += " --export " + export + " \\\n"
170 
171 
172 
173  # Now set the main program and its options
174  if not mcgen:
175  jobsub_cmd += " -X " + run_nova_sam_cmd + " \\\n"
176  for opt in run_nova_sam_opts:
177  jobsub_cmd += " " + opt + " \\\n"
178  else:
179  jobsub_cmd += " -X nova \\\n"
180  mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
181  if nevts>0 :
182  mcgen_opts += ["-n %d" % nevts]
183  for opt in mcgen_opts:
184  jobsub_cmd += " " + opt + " \\\n"
185 
186  jobsub_cmd = jobsub_cmd[:-2]
187  return jobsub_cmd
188 
189 
bool print
def submit_nova_art.check_env (   vname)

Definition at line 68 of file submit_nova_art.py.

References NovaGridUtils.fail().

68 def check_env(vname):
69  value=os.getenv(vname)
70  if None == value or "" == value:
71  fail("Environment variable %s not defined" %vname)
72 
def fail(msg)
def check_env(vname)
def submit_nova_art.check_fcl (   tag,
  fcl 
)

Definition at line 73 of file submit_nova_art.py.

References NovaGridUtils.fail(), print, and split().

73 def check_fcl(tag,fcl):
74  if "/" in fcl[:1] and os.path.isfile(fcl):
75  return fcl # hey I'm ok!
76  # Loop over dirs in FHICL_FILE_PATH
77  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
78  for path in fclPaths:
79  fullPath = os.path.join(path, fcl)
80  if os.path.isfile(fullPath):
81  return fcl # hey you're ok!
82  # Check if they are copying the fhicl file to the worker node.
83  for input_file in input_files:
84  if os.path.basename(input_file) == fcl:
85  return fcl # Passing fhicl as argument, all good.
86  elif os.path.basename(input_file) == os.path.basename(fcl):
87  print("")
88  print("The fhicl will be copied to $PWD on the worker node.")
89  print("You specified some sort of file path which isn't needed. Fixing it for you :)")
90  print(fcl + " --> " + os.path.basename(fcl))
91  print("")
92  return os.path.basename(fcl) # User incorrectly left file path there.
93 
94  fail("fcl file %s does not exist" %fcl)
95 
96 
void split(double tt, double *fr)
def fail(msg)
bool print
def check_fcl(tag, fcl)
def submit_nova_art.find_file (   paths,
  filename 
)

Definition at line 128 of file submit_nova_art.py.

References NovaGridUtils.fail().

128 def find_file(paths, filename):
129  if os.path.isfile(filename):
130  return filename
131  for path in paths:
132  for root, dirs, files in os.walk(os.path.expandvars(path)):
133  if filename in files:
134  return os.path.join(root, filename)
135  fail("Cannot find file "+filename)
136 
def fail(msg)
def find_file(paths, filename)
def submit_nova_art.find_file_in_list (   filepath,
  pathlist 
)

Definition at line 137 of file submit_nova_art.py.

137 def find_file_in_list(filepath, pathlist):
138  for testpath in pathlist:
139  if os.path.basename(filepath) == os.path.basename(testpath):
140  return True
141  return False
142 
143 
def find_file_in_list(filepath, pathlist)
def submit_nova_art.remove_comments (   src)
This reads tokens using tokenize.generate_tokens and recombines them
using tokenize.untokenize, and skipping comment/docstring tokens in between

Definition at line 97 of file submit_nova_art.py.

References art.decode().

97 def remove_comments(src):
98  """
99  This reads tokens using tokenize.generate_tokens and recombines them
100  using tokenize.untokenize, and skipping comment/docstring tokens in between
101  """
102  f = io.StringIO(src.encode().decode())
103  class SkipException(Exception): pass
104  processed_tokens = []
105  last_token = None
106  # go thru all the tokens and try to skip comments and docstrings
107  for tok in tokenize.generate_tokens(f.readline):
108  t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok
109 
110  try:
111  if t_type == tokenize.COMMENT:
112  raise SkipException()
113 
114  elif t_type == tokenize.STRING:
115 
116  if last_token is None or last_token[0] in [tokenize.INDENT]:
117  pass
118 
119  except SkipException:
120  pass
121  else:
122  processed_tokens.append(tok)
123 
124  last_token = tok
125 
126  return tokenize.untokenize(processed_tokens)
127 
def remove_comments(src)
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
void decode(std::any const &, InputTag &)

Variable Documentation

submit_nova_art.action

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.add_help

Definition at line 233 of file submit_nova_art.py.

submit_nova_art.allowed_groups

Definition at line 1060 of file submit_nova_art.py.

submit_nova_art.allowed_lifetimes

Definition at line 1023 of file submit_nova_art.py.

submit_nova_art.allowed_subgroups

Definition at line 1071 of file submit_nova_art.py.

submit_nova_art.args

Process and check command line arguments #.

Definition at line 688 of file submit_nova_art.py.

submit_nova_art.argv

Definition at line 214 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_cmd

Setup art_sam_wrap #.

Definition at line 39 of file submit_nova_art.py.

submit_nova_art.art_sam_wrap_opts

Definition at line 40 of file submit_nova_art.py.

submit_nova_art.autoDropbox

Definition at line 751 of file submit_nova_art.py.

submit_nova_art.awsfilepath

Definition at line 936 of file submit_nova_art.py.

submit_nova_art.build_location_arguments

Definition at line 30 of file submit_nova_art.py.

submit_nova_art.caf_tiers

Definition at line 1177 of file submit_nova_art.py.

submit_nova_art.choices

Definition at line 381 of file submit_nova_art.py.

submit_nova_art.command_executed

Definition at line 1379 of file submit_nova_art.py.

submit_nova_art.copyback

Definition at line 1169 of file submit_nova_art.py.

submit_nova_art.cpu_opt

Definition at line 987 of file submit_nova_art.py.

submit_nova_art.cvmfs_distro_base

Definition at line 28 of file submit_nova_art.py.

submit_nova_art.cvmfs_rev

Definition at line 1043 of file submit_nova_art.py.

submit_nova_art.dataset

Definition at line 1373 of file submit_nova_art.py.

submit_nova_art.debugging_args

debugging

Definition at line 264 of file submit_nova_art.py.

submit_nova_art.declareFiles

Definition at line 745 of file submit_nova_art.py.

submit_nova_art.declareLocations

Definition at line 748 of file submit_nova_art.py.

submit_nova_art.default

Definition at line 268 of file submit_nova_art.py.

submit_nova_art.defname

Definition at line 761 of file submit_nova_art.py.

submit_nova_art.description

Definition at line 204 of file submit_nova_art.py.

submit_nova_art.dest

Definition at line 736 of file submit_nova_art.py.

submit_nova_art.dimensions

Definition at line 890 of file submit_nova_art.py.

submit_nova_art.disk_opt

Definition at line 979 of file submit_nova_art.py.

submit_nova_art.dummy

Definition at line 1020 of file submit_nova_art.py.

submit_nova_art.dynamic_lifetime

Definition at line 740 of file submit_nova_art.py.

submit_nova_art.early_scripts

Definition at line 62 of file submit_nova_art.py.

submit_nova_art.environment_args

environment configuration

Definition at line 635 of file submit_nova_art.py.

submit_nova_art.expected_lifetime

Definition at line 738 of file submit_nova_art.py.

submit_nova_art.experiment

Definition at line 886 of file submit_nova_art.py.

submit_nova_art.export_to_art_sam_wrap

Definition at line 41 of file submit_nova_art.py.

submit_nova_art.export_to_run_nova_sam

Definition at line 37 of file submit_nova_art.py.

submit_nova_art.False

Definition at line 426 of file submit_nova_art.py.

submit_nova_art.fcl

Definition at line 818 of file submit_nova_art.py.

submit_nova_art.file

njobs specified, but not files/job.

Just set njobs and don't force limits on files per jobs

Definition at line 915 of file submit_nova_art.py.

submit_nova_art.files_per_job

Definition at line 742 of file submit_nova_art.py.

submit_nova_art.flat_tiers

Definition at line 1182 of file submit_nova_art.py.

submit_nova_art.fullpath

Definition at line 221 of file submit_nova_art.py.

Referenced by geo::GeometryBase.getCoordinateTransformation().

submit_nova_art.group

Definition at line 1058 of file submit_nova_art.py.

submit_nova_art.h5_tiers

Definition at line 1191 of file submit_nova_art.py.

submit_nova_art.hashDirs

Definition at line 851 of file submit_nova_art.py.

submit_nova_art.help

number of jobs

general job control

Definition at line 207 of file submit_nova_art.py.

submit_nova_art.hist_tier

Definition at line 1187 of file submit_nova_art.py.

submit_nova_art.index

Definition at line 218 of file submit_nova_art.py.

submit_nova_art.input_files

Definition at line 59 of file submit_nova_art.py.

submit_nova_art.int

Definition at line 304 of file submit_nova_art.py.

submit_nova_art.inter_scripts

Definition at line 65 of file submit_nova_art.py.

submit_nova_art.is_production_arg

Definition at line 836 of file submit_nova_art.py.

submit_nova_art.istarball

Definition at line 693 of file submit_nova_art.py.

submit_nova_art.job_cmd

Definition at line 1283 of file submit_nova_art.py.

submit_nova_art.job_control_args

job control

Definition at line 300 of file submit_nova_art.py.

submit_nova_art.jobname

Definition at line 760 of file submit_nova_art.py.

submit_nova_art.jobsub_cmd

Actually launch the jobs #.

Definition at line 1391 of file submit_nova_art.py.

submit_nova_art.jobsub_opts

Definition at line 33 of file submit_nova_art.py.

submit_nova_art.kill_opt

Definition at line 1005 of file submit_nova_art.py.

submit_nova_art.logs

Definition at line 1109 of file submit_nova_art.py.

submit_nova_art.maxopt

Definition at line 782 of file submit_nova_art.py.

submit_nova_art.maxopt_opt

Definition at line 783 of file submit_nova_art.py.

submit_nova_art.mcgen

Definition at line 820 of file submit_nova_art.py.

Referenced by main(), NeutronOscGenerator(), and NucleonDecayGenerator().

submit_nova_art.mcouttier

Definition at line 1167 of file submit_nova_art.py.

submit_nova_art.mem_opt

Definition at line 983 of file submit_nova_art.py.

submit_nova_art.metavar

Definition at line 295 of file submit_nova_art.py.

submit_nova_art.mode
submit_nova_art.nevts

Setup runNovaSAM #.

Definition at line 734 of file submit_nova_art.py.

Referenced by ndrockmacro(), and plot_xsecs().

submit_nova_art.newargs

Definition at line 226 of file submit_nova_art.py.

submit_nova_art.ngu_current

Definition at line 1033 of file submit_nova_art.py.

submit_nova_art.njobs

Definition at line 732 of file submit_nova_art.py.

submit_nova_art.novasoft_args

software control

Definition at line 459 of file submit_nova_art.py.

submit_nova_art.novasoft_cvmfs

Definition at line 29 of file submit_nova_art.py.

submit_nova_art.num_project_files

Definition at line 888 of file submit_nova_art.py.

submit_nova_art.offsite

Definition at line 758 of file submit_nova_art.py.

submit_nova_art.out_tiers

Definition at line 1164 of file submit_nova_art.py.

submit_nova_art.outnum

Definition at line 1167 of file submit_nova_art.py.

submit_nova_art.output_args

output control

Definition at line 511 of file submit_nova_art.py.

submit_nova_art.parser

Definition at line 233 of file submit_nova_art.py.

submit_nova_art.passThru

Definition at line 1116 of file submit_nova_art.py.

submit_nova_art.poms

Definition at line 754 of file submit_nova_art.py.

submit_nova_art.poms_campaign_id

Definition at line 1368 of file submit_nova_art.py.

submit_nova_art.poms_task_id

Definition at line 1376 of file submit_nova_art.py.

submit_nova_art.post_scripts

Definition at line 66 of file submit_nova_art.py.

submit_nova_art.pre_args

Definition at line 211 of file submit_nova_art.py.

submit_nova_art.pre_scripts

Definition at line 64 of file submit_nova_art.py.

submit_nova_art.preliminary_parser

Allow args to be passed in as a plain text file.

We make a preliminary parser get these arguments out for two reasons: 1) Maintain standard -h, –help functionality 2) Avoid necessity required arguments in initial parsing, allow them to be missing, but find them in the file.

Definition at line 204 of file submit_nova_art.py.

submit_nova_art.print_jobsub

Definition at line 763 of file submit_nova_art.py.

submit_nova_art.printenv

Definition at line 767 of file submit_nova_art.py.

submit_nova_art.prog

Definition at line 194 of file submit_nova_art.py.

submit_nova_art.project_name

start sam project

Definition at line 1241 of file submit_nova_art.py.

submit_nova_art.recommended_sites

Definition at line 692 of file submit_nova_art.py.

submit_nova_art.required

Definition at line 239 of file submit_nova_art.py.

submit_nova_art.required_args

required options

Definition at line 236 of file submit_nova_art.py.

submit_nova_art.resource_opt

Definition at line 955 of file submit_nova_art.py.

submit_nova_art.role

Definition at line 856 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_cmd

Definition at line 35 of file submit_nova_art.py.

submit_nova_art.run_nova_sam_opts

Definition at line 36 of file submit_nova_art.py.

submit_nova_art.sam_station

Definition at line 25 of file submit_nova_art.py.

submit_nova_art.sam_user

Definition at line 24 of file submit_nova_art.py.

submit_nova_art.samweb

Setup jobsub #.

both njobs and files per job are specified. Just pass the settings through to jobsub and art_sam_wrap files/job specified, but not njobs. Calculate njobs on the fly get files in dataset

Definition at line 886 of file submit_nova_art.py.

submit_nova_art.script_path

Definition at line 1231 of file submit_nova_art.py.

submit_nova_art.scriptcount

Definition at line 1284 of file submit_nova_art.py.

submit_nova_art.setup_location

Definition at line 31 of file submit_nova_art.py.

submit_nova_art.shell

Definition at line 1033 of file submit_nova_art.py.

submit_nova_art.sing_group

Definition at line 371 of file submit_nova_art.py.

submit_nova_art.singularity

Definition at line 945 of file submit_nova_art.py.

submit_nova_art.site_opt

Definition at line 959 of file submit_nova_art.py.

submit_nova_art.snapshot_id

Definition at line 762 of file submit_nova_art.py.

submit_nova_art.source_scripts

Definition at line 63 of file submit_nova_art.py.

submit_nova_art.srt_qual

Definition at line 781 of file submit_nova_art.py.

submit_nova_art.start_proj_command

Start the SAM project #.

Definition at line 1263 of file submit_nova_art.py.

submit_nova_art.start_proj_retval

Definition at line 1273 of file submit_nova_art.py.

submit_nova_art.start_project

Definition at line 1244 of file submit_nova_art.py.

submit_nova_art.str

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.subgroup

Definition at line 1069 of file submit_nova_art.py.

submit_nova_art.support_args

support options

Definition at line 671 of file submit_nova_art.py.

submit_nova_art.tag

Definition at line 779 of file submit_nova_art.py.

submit_nova_art.test

Definition at line 769 of file submit_nova_art.py.

submit_nova_art.test_dest

Definition at line 720 of file submit_nova_art.py.

submit_nova_art.test_dynamic_lifetime

Definition at line 726 of file submit_nova_art.py.

submit_nova_art.test_expected_lifetime

Definition at line 725 of file submit_nova_art.py.

submit_nova_art.test_files_per_job

Definition at line 727 of file submit_nova_art.py.

submit_nova_art.test_nevts

Definition at line 719 of file submit_nova_art.py.

submit_nova_art.test_njobs

Definition at line 718 of file submit_nova_art.py.

submit_nova_art.testrel_gp

Definition at line 464 of file submit_nova_art.py.

submit_nova_art.text

Definition at line 224 of file submit_nova_art.py.

submit_nova_art.timestamp
submit_nova_art.type

Definition at line 210 of file submit_nova_art.py.

submit_nova_art.unknown

Definition at line 211 of file submit_nova_art.py.

submit_nova_art.usage_models

Definition at line 57 of file submit_nova_art.py.

submit_nova_art.use_recommended_sites

Definition at line 694 of file submit_nova_art.py.

submit_nova_art.user

Definition at line 21 of file submit_nova_art.py.

submit_nova_art.va
submit_nova_art.version

Definition at line 1372 of file submit_nova_art.py.

submit_nova_art.veryearly_scripts

Definition at line 61 of file submit_nova_art.py.