submit_nova_art.py
Go to the documentation of this file.
1 #!/bin/env python
2 
3 from __future__ import print_function
4 from __future__ import division
5 from future import standard_library
6 standard_library.install_aliases()
7 from builtins import str
8 from past.utils import old_div
9 import os, sys, stat, pwd, re
10 import argparse
11 import datetime
12 import samweb_client
13 import string
14 import tokenize
15 import io
16 import subprocess
17 import NovaGridUtils
18 from NovaGridUtils import *
19 import recommended_sites as rs
20 
21 user=os.getenv("USER")
22 os.environ["GRID_USER"]=user
23 
24 sam_user=user
25 sam_station=os.getenv("SAM_STATION")
26 os.system("echo SAM_STATION DEFINED AS $SAM_STATION")
27 
28 cvmfs_distro_base = "/cvmfs/nova.opensciencegrid.org"
29 novasoft_cvmfs = "%s/novasoft/slf6/novasoft" % cvmfs_distro_base
30 build_location_arguments = "" # Becomes more specific with --cvmfs
31 setup_location = ""
32 
33 jobsub_opts = []
34 
35 run_nova_sam_cmd="runNovaSAM.py"
36 run_nova_sam_opts= []
37 export_to_run_nova_sam = []
38 
39 art_sam_wrap_cmd="$NOVAGRIDUTILS_DIR/bin/art_sam_wrap.sh"
40 art_sam_wrap_opts= []
41 export_to_art_sam_wrap=[]
42 export_to_art_sam_wrap.append("SAM_PROJECT_NAME")
43 export_to_art_sam_wrap.append("SAM_STATION")
44 export_to_art_sam_wrap.append("IFDH_BASE_URI")
45 if "IFDH_DEBUG" in os.environ:
46  export_to_art_sam_wrap.append("IFDH_DEBUG")
47 if "G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION" in os.environ:
48  export_to_art_sam_wrap.append("G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION")
49 
50 export_to_art_sam_wrap.append("EXPERIMENT")
51 export_to_art_sam_wrap.append("GRID_USER")
52 
53 os.environ["CVMFS_DISTRO_BASE"]=cvmfs_distro_base
54 export_to_art_sam_wrap.append("CVMFS_DISTRO_BASE")
55 
56 #there must be a better name for this variable
57 usage_models=["DEDICATED"]
58 
59 input_files=[]
60 
61 veryearly_scripts=[]
62 early_scripts=[]
63 source_scripts=[]
64 pre_scripts=[]
65 inter_scripts=[]
66 post_scripts=[]
67 
68 def check_env(vname):
69  value=os.getenv(vname)
70  if None == value or "" == value:
71  fail("Environment variable %s not defined" %vname)
72 
73 def check_fcl(tag,fcl):
74  if "/" in fcl[:1] and os.path.isfile(fcl):
75  return fcl # hey I'm ok!
76  # Loop over dirs in FHICL_FILE_PATH
77  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
78  for path in fclPaths:
79  fullPath = os.path.join(path, fcl)
80  if os.path.isfile(fullPath):
81  return fcl # hey you're ok!
82  # Check if they are copying the fhicl file to the worker node.
83  for input_file in input_files:
84  if os.path.basename(input_file) == fcl:
85  return fcl # Passing fhicl as argument, all good.
86  elif os.path.basename(input_file) == os.path.basename(fcl):
87  print("")
88  print("The fhicl will be copied to $PWD on the worker node.")
89  print("You specified some sort of file path which isn't needed. Fixing it for you :)")
90  print(fcl + " --> " + os.path.basename(fcl))
91  print("")
92  return os.path.basename(fcl) # User incorrectly left file path there.
93 
94  fail("fcl file %s does not exist" %fcl)
95 
96 
97 def remove_comments(src):
98  """
99  This reads tokens using tokenize.generate_tokens and recombines them
100  using tokenize.untokenize, and skipping comment/docstring tokens in between
101  """
102  f = io.StringIO(src.encode().decode())
103  class SkipException(Exception): pass
104  processed_tokens = []
105  last_token = None
106  # go thru all the tokens and try to skip comments and docstrings
107  for tok in tokenize.generate_tokens(f.readline):
108  t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok
109 
110  try:
111  if t_type == tokenize.COMMENT:
112  raise SkipException()
113 
114  elif t_type == tokenize.STRING:
115 
116  if last_token is None or last_token[0] in [tokenize.INDENT]:
117  pass
118 
119  except SkipException:
120  pass
121  else:
122  processed_tokens.append(tok)
123 
124  last_token = tok
125 
126  return tokenize.untokenize(processed_tokens)
127 
128 def find_file(paths, filename):
129  if os.path.isfile(filename):
130  return filename
131  for path in paths:
132  for root, dirs, files in os.walk(os.path.expandvars(path)):
133  if filename in files:
134  return os.path.join(root, filename)
135  fail("Cannot find file "+filename)
136 
137 def find_file_in_list(filepath, pathlist):
138  for testpath in pathlist:
139  if os.path.basename(filepath) == os.path.basename(testpath):
140  return True
141  return False
142 
143 
145  # Start with jobsub_submit and its options
146  jobsub_cmd = "jobsub_submit \\\n"
147  for opt in jobsub_opts:
148  jobsub_cmd += " " + opt + " \\\n"
149  for export in export_to_art_sam_wrap:
150  jobsub_cmd += " -e " + export
151  jobsub_cmd += " \\\n"
152 
153  # Add art_sam_wrap wrapper script and its options
154  if args.testrel:
155  if args.reuse_tarball:
156  jobsub_cmd += " --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
157  else:
158  jobsub_cmd += " --tar_file_name tardir://" + args.testrel +" \\\n"
159  elif args.user_tarball:
160  if not os.path.isfile(args.user_tarball):
161  print("Tarball filename passed to --user_tarball does not exit:", args.user_tarball)
162  sys.exit(5)
163  jobsub_cmd += " --tar_file_name dropbox://" + args.user_tarball + " \\\n"
164 
165  jobsub_cmd += " file://" + art_sam_wrap_cmd + " \\\n"
166  for opt in art_sam_wrap_opts:
167  jobsub_cmd += " " + opt + " \\\n"
168  for export in export_to_run_nova_sam :
169  jobsub_cmd += " --export " + export + " \\\n"
170 
171 
172 
173  # Now set the main program and its options
174  if not mcgen:
175  jobsub_cmd += " -X " + run_nova_sam_cmd + " \\\n"
176  for opt in run_nova_sam_opts:
177  jobsub_cmd += " " + opt + " \\\n"
178  else:
179  jobsub_cmd += " -X nova \\\n"
180  mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
181  if nevts>0 :
182  mcgen_opts += ["-n %d" % nevts]
183  for opt in mcgen_opts:
184  jobsub_cmd += " " + opt + " \\\n"
185 
186  jobsub_cmd = jobsub_cmd[:-2]
187  return jobsub_cmd
188 
189 
190 #######################################################################################
191 
192 if __name__=='__main__':
193 
194  prog=os.path.basename(sys.argv[0])
195  NovaGridUtils.prog=prog
196 
197 
198  while "-f" in sys.argv or "--file" in sys.argv:
199  ### Allow args to be passed in as a plain text file.
200  ### We make a preliminary parser get these arguments out for two reasons:
201  ### 1) Maintain standard -h, --help functionality
202  ### 2) Avoid necessity required arguments in initial parsing,
203  ### allow them to be missing, but find them in the file.
204  preliminary_parser = argparse.ArgumentParser(prog=prog, description='Submit nova art job')
205 
206  preliminary_parser.add_argument('-f', '--file',
207  help="""Text file containing any arguments to this utility. Multiple allowed.
208  Arguments should look just like they would on the command line,
209  but the parsing of this file is whitespace insenstive.
210  Commented lines will be identified with the # character and removed. """, type=str, action='append')
211  pre_args, unknown = preliminary_parser.parse_known_args()
212 
213  # Remove pre_args from sys.argv so they are not processed again
214  sys.argv = [x for x in sys.argv if x not in [ "-f", "--file"]]
215 
216  if pre_args.file:
217  for filepath in pre_args.file:
218  index = sys.argv.index(filepath)
219  sys.argv.remove(filepath)
220  if os.path.isfile(filepath):
221  fullpath = filepath
222  else:
223  fullpath = find_file(["$NOVAGRIDUTILS_DIR/configs/"],filepath)
224  text = open(fullpath, 'r').read()
225  text = remove_comments(text) # Strip out commented lines
226  newargs = []
227  for line in text.splitlines():
228  # Insert arguments into list in order
229  # where the -f appeared
230  newargs += line.split()
231  sys.argv[index:index] = newargs
232 
233  parser = argparse.ArgumentParser(prog=prog, description='Submit nova art job', add_help=False)
234 
235  ###required options
236  required_args = parser.add_argument_group("Required arguments", "These arguments must be supplied.")
237 
238  required_args.add_argument('--jobname',
239  required=True,
240  help='Job name',
241  type=str)
242 
243  required_args.add_argument('--defname',
244  required=True,
245  help='SAM dataset definition to run over',
246  type=str)
247 
248  required_args.add_argument('--config', '-c',
249  required=True,
250  help='FHiCL file to use as configuration for nova executable. The path given should be relative to the $SRT_PRIVATE_CONTEXT of any test release you submit using',
251  type=str)
252 
253  required_args.add_argument("--tag",
254  required=True,
255  help="Tag of novasoft to use",
256  type=str)
257 
258  required_args.add_argument("--dest",
259  required=True,
260  help="Destination for output files",
261  type=str)
262 
263  ###debugging
264  debugging_args = parser.add_argument_group("Debugging options", "These optional arguments can help debug your submission.")
265 
266  debugging_args.add_argument('--print_jobsub',
267  help='Print jobsub command',
268  action='store_true',default=False)
269 
270  debugging_args.add_argument('--printenv',
271  help='Print environment variables',
272  action='store_true',default=False)
273 
274  debugging_args.add_argument('--test',
275  help='Do not actually do anything, just run tests and print jobsub cmd',
276  action='store_true',default=False)
277 
278  debugging_args.add_argument('--gdb',
279  help='Run nova executable under gdb, print full stack trace, then quit gdb.',
280  action='store_true',default=False)
281 
282  debugging_args.add_argument('--test_submission',
283  help='Override other arguments given to submit a test to the grid. It will run 1 job with 3 events and write the output to /pnfs/nova/scratch/users/<user>/test_jobs/<date>_<time>',
284  action='store_true',default=False)
285 
286  debugging_args.add_argument('--jobsub_server',
287  help='Submit using the specified jobsub server',
288  default="")
289 
290  debugging_args.add_argument('--test_queue',
291  help='Submit jobs to the test jobsub queue for higher starting priority. NB NOvA is limited to 10 jobs at a time on this queue.',
292  action='store_true', default=False)
293 
294  debugging_args.add_argument("--kill_after",
295  metavar="SEC",
296  help="If job is still running after this many seconds, kill in such a way that a log will be returned",
297  type=int)
298 
299  ###job control
300  job_control_args = parser.add_argument_group("Job control options", "These optional arguments help control where and how your jobs land.")
301  ###number of jobs
302  job_control_args.add_argument('--njobs',
303  help='Number of jobs to submit',
304  type=int, default = 0)
305 
306  job_control_args.add_argument('--maxConcurrent',
307  help='Run a maximum of N jobs simultaneously',
308  metavar='N',
309  type=int, default=0)
310 
311  job_control_args.add_argument('--files_per_job',
312  help='Number of files per job - if zero, calculate from number of jobs', metavar='N',
313  type=int, default = 0)
314 
315  job_control_args.add_argument('--nevts',
316  help='Number of events per file to process',
317  type=int, default = 0)
318 
319 
320  job_control_args.add_argument('--no_multifile',
321  help='Do not use art_sam_wrap.sh multifile mode, which is on by default',
322  action='store_true')
323 
324  job_control_args.add_argument('--txtfiledef',
325  help='Use if the input definition is made up of text files, each containing a list of file names',
326  action='store_true',default=False)
327 
328  ###general job control
329  job_control_args.add_argument('--opportunistic',
330  help='Run opportunistically on the fermigrid',
331  action='store_true',default=False)
332 
333  job_control_args.add_argument('--offsite',
334  help='Allow to run on offsite resources as well. Implies --opportunistic.',
335  action='store_true',default=False)
336 
337  job_control_args.add_argument('--offsite_only',
338  help='Allow to run solely on offsite resources.',
339  action='store_true',default=False)
340 
341  job_control_args.add_argument('--amazon',
342  help='Run at amazon.',
343  action='store_true',default=False)
344 
345  job_control_args.add_argument('--site',
346  help='Specify allowed offsite locations. Omit to allow running at any offsite location',
347  type=str,action='append')
348 
349  job_control_args.add_argument('--exclude_site',
350  help='Specify an offsite location to exclude.',
351  metavar='SITE',
352  type=str,action='append')
353 
354  job_control_args.add_argument('--recommended_sites',
355  help='Specify known working offsite locations.',
356  action='store_true',default=False)
357 
358  job_control_args.add_argument('--recommended_sites_rockgen',
359  help='Specify known working offiste locations, for rock generation.',
360  action='store_true',default=False)
361 
362  job_control_args.add_argument('--recommended_sites_generation',
363  help='Specify known working offiste locations, for GENIE/GEANT generation.',
364  action='store_true',default=False)
365 
366  job_control_args.add_argument('--autoDropbox',
367  help='Use automatic dropbox location based on site',
368  action='store_true',default=False)
369 
370  # --singularity and --os are mutually exclusive
371  sing_group = job_control_args.add_mutually_exclusive_group()
372  sing_group.add_argument('--singularity',
373  help='Location in CVMFS of a singularity container to launch the job into. ' + \
374  "If you're looking for a stock Scientific Linux image, try --os instead. " + \
375  "(This option is mutually exclusive with --os.)",
376  type=str,
377  default=None)
378  sing_group.add_argument("--os",
379  help="Run this job inside a Scientific Linux singularity image with given OS. " + \
380  "(This option is mutually exclusive with --singularity.)",
381  choices=NovaGridUtils.KNOWN_OS_CONTAINERS)
382 
383  job_control_args.add_argument('--disk',
384  help='Local disk space requirement for worker node in MB.',
385  type=int, default=10000)
386 
387  job_control_args.add_argument('--memory',
388  help='Local memory requirement for worker node in MB.',
389  type=int, default=1900)
390 
391  job_control_args.add_argument('--expected_lifetime',
392  help='Expected job lifetime (default is 10800s=3h). Valid values are an integer number of seconds or one of \"short\" (6h), \"medium\" (12h) or \"long\" (24h, jobsub default)', metavar='LIFETIME',
393  type=str, default="10800")
394 
395  job_control_args.add_argument('--dynamic_lifetime',
396  help="Dynamically determine whether a new file should be started based on glidein lifetime. Specify the maximum length expected for a single file to take to process in seconds.", metavar="LIFETIME",
397  type=str)
398 
399  job_control_args.add_argument('--cpu',
400  help="Request worker nodes that have at least NUMBER cpus",
401  type=int, default=1)
402 
403 
404  job_control_args.add_argument('--group', '-G',
405  help="Specify batch group GROUP -- mainly used to set job priority. At present, only supportable value is nova",
406  type=str, default="nova")
407 
408  job_control_args.add_argument('--subgroup',
409  help='Production subgroup',
410  type=str)
411 
412  job_control_args.add_argument("--role",
413  help="Specify role to run on the grid. Can be Analysis (default) or Production. This option is no longer supported",
414  type=str,default="Analysis")
415 
416  job_control_args.add_argument('--continue_project',
417  help="Don't start a new samweb project, instead continue this one.", metavar='PROJECT',
418  type=str,default="")
419 
420  job_control_args.add_argument("--snapshot_id",
421  help="Use this existing snapshot instead of creating a new one.", metavar="ID",
422  type=int,default=0)
423 
424  job_control_args.add_argument("--poms",
425  help="Start/continue a poms campaign and task for this submission",
426  default=False, action="store_true")
427 
428  job_control_args.add_argument("--mix",
429  help="Pass a mixing script to the job to pull in a files for job mixing.",
430  default="", type=str)
431 
432  job_control_args.add_argument("--mail_always",
433  help="Do you want an email whenever every jobs finishes?",
434  default=False, action="store_true")
435 
436  job_control_args.add_argument("--mail_on_error",
437  help="Do you want an email whenever a job fails on an error?",
438  default=False, action="store_true")
439 
440 # job_control_args.add_argument("--poms_definition",
441 # help="POMS definition to use",
442 # default="GenercicSubmitNOvAArt", type=str)
443 
444  job_control_args.add_argument('--user_priority',
445  help='Priority (integer) within a user\'s jobs (default = 0)',
446  type=int, default=0)
447 
448  job_control_args.add_argument('--jobfactory',
449  help='Use the specified JobFactoryType.',
450  default="")
451 
452  job_control_args.add_argument("--gpu",
453  help="Request a node with a GPU",
454  default=False, action="store_true")
455 
456  add_node_features_arg(job_control_args)
457 
458  ###software control
459  novasoft_args = parser.add_argument_group("NOvA software options", "These options control the novasoft setup.")
460  novasoft_args.add_argument('--maxopt',
461  help='Run in maxopt mode',
462  action='store_true',default=True)
463 
464  testrel_gp = novasoft_args.add_mutually_exclusive_group(required=False)
465  testrel_gp.add_argument("--testrel",
466  help="Use a test release at location TESTREL. It will be tarred up, and sent to the worker node.",
467  type=str)
468  testrel_gp.add_argument("--user_tarball",
469  help="Use existing test release tarball in specified location rather than having jobsub make one for you (conflicts with --testrel)",
470  type=str)
471 
472  novasoft_args.add_argument('--reuse_tarball',
473  help='Do you want to reuse a tarball that is already in resilient space? If using this option avoid trailing slash in --testrel option. (conflicts with --user_tarball)',
474  action='store_true',default=False)
475 
476  novasoft_args.add_argument('--cvmfs',
477  help='Does nothing (always true), but retained for compatibility: pull software from CVMFS.',
478  action='store_true')
479 
480 
481  novasoft_args.add_argument('--disable_cvmfs_version_matching',
482  help="Don't perform a CVMFS-is-up-to-date check on target nodes via Condor requirements. (For advanced debugging use.) ",
483  action="store_true",
484  default=False)
485 
486  novasoft_args.add_argument('--novasoftups',
487  help='Use the ups build of novasoft, must be used with source to setup.',
488  action='store_true')
489 
490  novasoft_args.add_argument('--ngu_test',
491  help='Setup the test version of NovaGridUtils in the grid jobs.',
492  action='store_true')
493 
494  novasoft_args.add_argument('--ngu_version',
495  help='Setup a specific NovaGridUtils version in the grid jobs.', metavar='VERSION',
496  type=str)
497 
498  novasoft_args.add_argument('--testrel_ngu',
499  help="Must be used with --testrel, with NGU checked out. After unpacking tarball will setup the local version of NGU you are using on the work.",
500  action='store_true')
501 
502  novasoft_args.add_argument('--lemBalance',
503  help='Choose lem server based on (CLUSTER+PROCESS)%%2 to balance load',
504  action='store_true', default=False)
505 
506  novasoft_args.add_argument('--lemServer',
507  help='Specify lem server',
508  type=str)
509 
510  ###output control
511  output_args = parser.add_argument_group("Output file options", "Note that you must specify either --copyOut or --copyOutScript.")
512 
513 
514  output_args.add_argument('--copyOutScript',
515  help='Use script COPYOUTSCRIPT to copy back your output',
516  type=str)
517 
518  output_args.add_argument('--copyOut',
519  help='Use the built in copy out mechanism. If used, you must specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier',
520  action='store_true')
521 
522  output_args.add_argument('--logs',
523  help='Return .log files corresponding to every output',
524  action='store_true')
525  output_args.add_argument('--zipLogs',
526  help='Format logs as .bz2 files. Implies --logs',
527  action='store_true')
528 
529  output_args.add_argument('--outTier',
530  help='Data tier of the output file, multiple allowed, formatted as <name_in_fcl_outputs>:<data_tier>',
531  type=str, action='append')
532 
533  output_args.add_argument('--cafTier',
534  help='Module label for CAF output, multiple allowed. Format as <cafmaker_module_label>:<data_tier>',
535  type=str, action='append')
536 
537  output_args.add_argument('--flatTier',
538  help='Module label for FlatCAF output, multiple allowed. Format as <flatmaker_module_label>:<data_tier>',
539  type=str, action='append')
540 
541  output_args.add_argument('--histTier',
542  help='File identifier string for TFileService output, only one allowed. Supply as --histTier <id> for output_name.<id>.root, where output_name is assembled based on the input file.',
543  type=str)
544 
545  output_args.add_argument('--h5Tier',
546  help='File identifier for h5 output, multiple allowed. Format as <hdf5maker_module>:<data_tier>',
547  type=str, action='append')
548  output_args.add_argument('--second_config',
549  help="""Second configuration fcl executed after main process.
550  nova is executed with art file that is output from the main process.
551  Files that get produced by this process that are named identically
552  to files produced by the main process and are among the requested outputs
553  are ignored and the file produced by the first process is returned""",
554  type=str)
555 
556  output_args.add_argument('--outputNumuDeCAF',
557  help='Make standard numu decafs for all CAF files produced during the job',
558  action='store_true',default=False)
559 
560  output_args.add_argument('--outputNueDeCAF',
561  help='Make standard nue decafs for all CAF files produced during the job',
562  action='store_true',default=False)
563 
564  output_args.add_argument('--outputNumuOrNueDeCAF',
565  help='Make standard nue or numu decafs for all CAF files produced during the job',
566  action='store_true',default=False)
567 
568  output_args.add_argument('--outputNusDeCAF',
569  help='Make standard nus decafs for all CAF files produced during the job',
570  action='store_true',default=False)
571 
572  output_args.add_argument('--outputValidationDeCAF',
573  help='Make validation (nue_or_numu_or_nus) decafs for all CAF files produced during the job',
574  action='store_true',default=False)
575 
576  output_args.add_argument('--cosmicsPolarity',
577  help='To specify a horn polarity for the cosmics output file name',
578  type=str)
579 
580  output_args.add_argument('--npass',
581  help='To specify npass (aka nova.subversion)',
582  type=str)
583 
584  output_args.add_argument('--skim',
585  help='To specify nova.skim (does not work with mc gen)',
586  type=str)
587 
588  output_args.add_argument('--systematic',
589  help='To specify nova.systematic (does not work with mc gen)', metavar='SYST',
590  type=str)
591 
592  output_args.add_argument('--specialName',
593  help='To specify nova.special name (does not work with mc gen)',
594  type=str)
595 
596  output_args.add_argument('--genietune',
597  help='To specify nova.genietune (does not work with mc gen)', metavar='TUNE',
598  type=str)
599 
600  output_args.add_argument('--NPPFX',
601  help='To specify number of PPFX universes',
602  type=str)
603 
604  output_args.add_argument('--hashDirs',
605  help='Use hash directory structure in destination directory.',
606  action='store_true')
607 
608  output_args.add_argument('--runDirs',
609  help='Use run directory structure in destination directory, 000XYZ/XYZUW for run number XYZUW.',
610  action='store_true')
611 
612  output_args.add_argument('--noCleanup',
613  help='Pass --noCleanup argument to runNovaSAM.py. Necessary when using a postscript for copyout.',
614  action='store_true')
615 
616  output_args.add_argument('--jsonMetadata', help='Create JSON files with metadata corresponding to each output file, and copy them to the same destinations', action='store_true')
617 
618  output_args.add_argument('--declareFiles',
619  help='Declare files with metadata on worker node',
620  action='store_true')
621 
622  output_args.add_argument('--production',
623  help='Submit production style jobs. Implies \"--role=Production --hashDirs --jsonMetadata --zipLogs\", and checks that other settings needed for production are specified',
624  action='store_true')
625 
626  output_args.add_argument('--calibration',
627  help='Submit calibration style jobs. Implies \"--role=Production\", and checks that other settings needed for calibration are specified',
628  action='store_true')
629 
630  output_args.add_argument('--declareLocations',
631  help='Declare the file output locations to SAM during the copy back of the files',
632  action='store_true')
633 
634  ###environment configuration
635  environment_args = parser.add_argument_group("Environment options", "These optional arguments allow control of the grid running environment.")
636 
637  environment_args.add_argument('--export',
638  help='Export variable EXPORT to art_sam_wrap.sh',
639  type=str, action='append')
640 
641  environment_args.add_argument('--veryearlyscript',
642  help='Source script EARLYSCRIPT before any environment setup or sourcing',
643  type=str, action='append')
644 
645  environment_args.add_argument('--source',
646  help='Source script SOURCE',
647  type=str, action='append')
648 
649  environment_args.add_argument('--earlyscript',
650  help='Execute script EARLYSCRIPT before any environment setup',
651  type=str, action='append')
652 
653  environment_args.add_argument('--prescript',
654  help='Execute script PRESCRIPT before executing runNovaSAM.py',
655  type=str, action='append')
656 
657  environment_args.add_argument('--precopyscript',
658  help='Execute script PRECOPYSCRIPT within runNovaSAM.py, after running the nova -c command.',
659  type=str, action='append')
660 
661  environment_args.add_argument('--postscript',
662  help='Execute script POSTSCRIPT after executing runNovaSAM.py',
663  type=str, action='append')
664 
665  environment_args.add_argument('--inputfile',
666  help='Copy this extra input file into job area before running executable',
667  type=str, action='append')
668 
669 
670  ###support options
671  support_args = parser.add_argument_group("Support options", "These optional arguments using this submission utility easier.")
672 
673  support_args.add_argument("-h", "--help", action="help", help="Show this help message and exit")
674 
675  support_args.add_argument('-f', '--file',
676  help="""Text file containing any arguments to this utility. Multiple allowed.
677  Arguments should look just like they would on the command line,
678  but the parsing of this file is whitespace insenstive.
679  Comments will be identified with the # character and removed. """, type=str, action='append')
680 
681 
682 
683 
684  ############################################
685  # Process and check command line arguments #
686  ############################################
687 
688  args = parser.parse_args()
689  timestamp=datetime.datetime.now().strftime("%Y%m%d_%H%M")
690 
691  # Get sites we're going to use
692  recommended_sites=[]
693  istarball = args.testrel or args.user_tarball
694  use_recommended_sites=False
695  if args.recommended_sites:
696  recommended_sites = rs.get_recommended_sites('default', istarball, args.offsite_only)
697  use_recommended_sites=True
698  if args.recommended_sites_rockgen:
699  recommended_sites = rs.get_recommended_sites('rockgen', istarball, args.offsite_only)
700  use_recommended_sites=True
701  if args.recommended_sites_generation:
702  recommended_sites = rs.get_recommended_sites('generation', istarball, args.offsite_only)
703  use_recommended_sites=True
704 
705  # Load POMS if we need to:
706  if args.poms:
707  try:
708  import poms_client
709  except ImportError:
710  print("POMS not setup. Run this and try again:")
711  print()
712  print(" setup poms_client")
713  print()
714  sys.exit(1)
715 
716  # Check for test submission. Has to be first to override other arguments
717  if args.test_submission:
718  test_njobs = 1
719  test_nevts = 3
720  test_dest = "/pnfs/nova/scratch/users/%s/test_jobs/%s" % (os.environ["USER"], timestamp)
721  if not os.path.exists(test_dest):
722  os.makedirs(test_dest)
723  mode = os.stat(test_dest).st_mode | stat.S_IXGRP | stat.S_IWGRP
724  os.chmod(test_dest, mode)
725  test_expected_lifetime = "0"
726  test_dynamic_lifetime = "500"
727  test_files_per_job = 1
728 
729  print("Running a test submission. Overwriting:")
730 
731  print(" njobs", args.njobs, "-->", test_njobs)
732  args.njobs = test_njobs
733  print(" nevts", args.nevts, "-->", test_nevts)
734  args.nevts = test_nevts
735  print(" dest", args.dest, "-->", test_dest)
736  args.dest = test_dest
737  print(" expected_lifetime", args.expected_lifetime, "-->", test_expected_lifetime)
738  args.expected_lifetime = test_expected_lifetime
739  print(" dynamic_lifetime", args.dynamic_lifetime, "-->", test_dynamic_lifetime)
740  args.dynamic_lifetime = test_dynamic_lifetime
741  print(" files_per_job", args.files_per_job, "-->", test_files_per_job)
742  args.files_per_job = test_files_per_job
743  if args.declareFiles:
744  print(" don't declareFiles")
745  args.declareFiles = False
746  if args.declareLocations:
747  print(" don't declareLocations")
748  args.declareLocations = False
749  if args.autoDropbox:
750  print(" don't use autoDropbox")
751  args.autoDropbox = False
752  if args.poms:
753  print(" don't use poms")
754  args.poms = False
755 
756  #print " use the test jobsub queue, so OnSite only."
757  #args.test_queue = True
758  args.offsite = False
759 
760  jobname=args.jobname
761  defname=args.defname
762  snapshot_id=args.snapshot_id
763  print_jobsub=args.print_jobsub
764 
765  if args.printenv :
766  print("Will print environment vars ")
767  printenv=True
768 
769  test=args.test
770  if test :
771  print_jobsub=True
772  print("")
773  warn("--test was specified, so all we do is run checks and print jobsub cmd.")
774 
775  check_env("SETUP_IFDH_ART")
776  check_env("SETUP_SAM_WEB_CLIENT")
777  check_env("SETUP_JOBSUB_CLIENT")
778  check_env("SAM_STATION")
779  tag=check_tag(args.tag)
780 
781  srt_qual="debug"
782  maxopt=args.maxopt
783  maxopt_opt=""
784  if maxopt:
785  maxopt_opt += "-b:maxopt"
786  srt_qual="maxopt"
787 
788  if args.reuse_tarball and not args.testrel:
789  fail("--reuse_tarball specified without --testrel??")
790 
791  if args.testrel:
792  check_dir(args.testrel)
793  if not os.path.isdir(args.testrel+'/lib/'+os.getenv('SRT_ARCH')+'-GCC-'+srt_qual):
794  fail(args.testrel+' has never been built '+srt_qual)
795 
796  if args.inputfile:
797  input_files += args.inputfile
798 
799  for input_file in input_files:
800  if not os.path.isfile(os.path.expandvars(input_file)):
801  fail("Input file %s does not exist!" % input_file)
802  if os.path.expandvars(input_file).startswith("/nova/"):
803  fail("Input file %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % input_file)
804  elif os.path.expandvars(input_file).startswith("/grid/"):
805  fail("Input file %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % input_file)
806 
807  if args.singularity and not os.path.exists(args.singularity):
808  fail("Requested singularity image cannot be found: %s" % args.singularity)
809 
810  if args.gpu and not args.singularity:
811  warn("Requested GPU, but did not request singularity. This is not likely to succeed.")
812 
813  if args.gpu and not args.offsite_only:
814  warn("GPUs are only available offsite, and you have not chosen --offsite_only")
815 
816 
817 
818  fcl=args.config
819 
820  mcgen = (fcl == "mcgen")
821 
822  if not mcgen:
823  fcl = check_fcl(tag,fcl)
824 
825  dest=args.dest
826  if not dest.startswith("s3://") :
827  check_dir(dest)
828 
829  if os.path.expandvars(dest).startswith("/nova/"):
830  fail("Destination directory %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % dest)
831  elif os.path.expandvars(dest).startswith("/grid/"):
832  fail("Destination directory %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % dest)
833 
834  export_to_run_nova_sam.append("DEST=%s"%dest)
835 
836  is_production_arg = args.production
837 
838  if "Production" in args.role and not is_production_arg:
839  fail("You specified --role=Production but not --production. This is no longer supported")
840 
841  if args.production:
842  setup_production(args)
843  elif args.calibration:
844  setup_calibration(args)
845  else:
846  setup_analysis(args)
847 
848  # Check for test submission. Has to be first to override other arguments
849  if args.test_submission:
850  print("Running a test submission, turning off hashDirs")
851  args.hashDirs = False
852 
853  if args.hashDirs and args.runDirs:
854  fail("Cannot specify both --hashDirs and --runDirs (note that hashDirs is implied by --production)")
855 
856  role=args.role
857 
858  njobs=args.njobs
859  files_per_job = args.files_per_job
860 
861  print("Definition name: %s" % defname)
862  if snapshot_id:
863  print(" with snapshot_id: %d" % snapshot_id)
864 
865 
866 
867 
868  ################
869  # Setup jobsub #
870  ################
871 
872  if args.jobsub_server:
873  jobsub_opts += ["--jobsub-server=%s"%args.jobsub_server]
874 
875  if files_per_job > 0 and njobs > 0 :
876  ## both njobs and files per job are specified. Just pass
877  ## the settings through to jobsub and art_sam_wrap
878  jobsub_opts += ["-N %d" %njobs]
879  art_sam_wrap_opts += ["--limit %d" % files_per_job]
880 
881  elif files_per_job > 0:
882  ##files/job specified, but not njobs. Calculate njobs
883  ## on the fly
884 
885  ##get files in dataset
886  samweb = samweb_client.SAMWebClient(experiment='nova')
887  if not snapshot_id:
888  num_project_files=samweb.countFiles(defname=defname)
889  else:
890  num_project_files=samweb.countFiles(dimensions="snapshot_id {0:d}".format(snapshot_id))
891  print("Definition file count %d" % num_project_files)
892 
893  njobs=(old_div(num_project_files, files_per_job)) +1
894  jobsub_opts += ["-N %d" %njobs]
895  art_sam_wrap_opts += ["--limit %d" % files_per_job]
896 
897  elif njobs > 0 :
898  ##njobs specified, but not files/job. Just set njobs
899  ## and don't force limits on files per jobs
900  jobsub_opts += ["-N %d" %njobs]
901 
902  else :
903  warn("Neither --njobs nor --files_per_job specified. Did you really want to do this? Sleeping for 5 seconds")
904  sleep(5)
905 
906  # allow a little bit of grace -- 5500 rather than 5000
907  if njobs > 5500:
908  print("""
909  Error: cannot submit more than 5000 jobs in one cluster.
910  Please break your submission into multiple batches of 5000 (or less) jobs,
911  and after submitting the first batch, use --continue_project with the project
912  that results from the first submission for the remaining batches.
913 
914  Please separate submissions by 5 minutes.
915  """, file=sys.stderr)
916  sys.exit(1)
917 
918  if args.maxConcurrent:
919  jobsub_opts += ["--maxConcurrent=%d" %args.maxConcurrent]
920 
921  if args.opportunistic or args.offsite:
922  usage_models.append("OPPORTUNISTIC")
923  if args.offsite :
924  usage_models.append("OFFSITE")
925  if args.offsite_only:
926  if args.offsite:
927  fail("Both --offsite and --offsite_only specified, these arguments conflict")
928 
929  if args.opportunistic:
930  fail("Both --opportunistic and --offsite_only specified, these arguments conflict")
931 
932  usage_models = ["OFFSITE"]
933 
934  if args.amazon :
935  usage_models=["AWS_HEPCLOUD"]
936  awsfilepath=os.path.expandvars("/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/$NOVAGRIDUTILS_VERSION/NULL/utils/aws_setup.sh")
937  source_scripts.append(awsfilepath)
938 
939  if args.autoDropbox:
940  run_nova_sam_opts += ["--autoDropbox"]
941 
942  # Singularity, offsite checks
943  if args.os:
944  # just redirect to the known container path
945  args.singularity = NovaGridUtils.KNOWN_OS_CONTAINERS[args.os]
946 
947  if args.singularity:
948  if not os.path.exists(args.singularity):
949  NovaGridUtils.fail("Requested singularity image cannot be found: %s" % args.singularity)
950 
951  # Check OS in off-site submissions
952  if (args.offsite or args.offsite_only) and not (args.singularity or args.os):
953  NovaGridUtils.fail("Running offsite, but OS version not specified!")
954 
955  resource_opt="--resource-provides=usage_model=" + ",".join( usage_models )
956  jobsub_opts += [resource_opt]
957 
958  if use_recommended_sites or args.site:
959  site_opt="--site="
960 
961  if use_recommended_sites:
962  for isite in recommended_sites:
963  site_opt += isite + ","
964  if args.site:
965  for isite in args.site:
966  if isite not in recommended_sites:
967  warn("Site "+isite+" is not known to work. Your jobs may fail at that site. Sleeping for 5 seconds")
968  sleep(5)
969  site_opt += isite + ","
970 
971  site_opt=site_opt[:-1]
972  jobsub_opts += [ site_opt ]
973 
974  if args.exclude_site:
975  for isite in args.exclude_site:
976  jobsub_opts += [ "--append_condor_requirements='(TARGET.GLIDEIN_Site\ isnt\ \\\"%s\\\")'" % isite ]
977 
978  if args.disk:
979  disk_opt="--disk=%sMB" % (args.disk)
980  jobsub_opts += [ disk_opt ]
981 
982  if args.memory:
983  mem_opt="--memory=%sMB" % (args.memory)
984  jobsub_opts += [ mem_opt ]
985 
986  if args.cpu:
987  cpu_opt="--cpu=%d" % (args.cpu)
988  jobsub_opts += [ cpu_opt ]
989 
990  if args.mail_always:
991  jobsub_opts += ["--mail_always"]
992  elif args.mail_on_error:
993  jobsub_opts += ["--mail_on_error"]
994  else:
995  jobsub_opts += ["--mail_never"]
996 
997  # The default jobsub_submit priority is 0
998  #production_priority_max = 100 # Reserved for keepup processing
999  if args.user_priority != 0 :
1000  #if args.production and args.user_priority >= production_priority_max :
1001  # fail( "Priority for production must be < %d" % production_priority_max )
1002  jobsub_opts += [ '-l "priority=%d"' % args.user_priority ]
1003 
1004  if args.kill_after:
1005  kill_opt="--self-destruct-timer %d" % (args.kill_after)
1006  art_sam_wrap_opts += [ kill_opt ]
1007 
1008  if args.dynamic_lifetime:
1009  # Check other arguments
1010  args.expected_lifetime = "0"
1011  if args.files_per_job > 1:
1012  warn("You have limited number of files per job to "+str(args.files_per_job)+" but this argument should not be necessary with dynamic_lifetime.")
1013  art_sam_wrap_opts += [ "--dynamic_lifetime " + args.dynamic_lifetime ]
1014  jobsub_opts += [ "--append_condor_requirements='(((TARGET.GLIDEIN_ToDie-CurrentTime)>%s)||isUndefined(TARGET.GLIDEIN_ToDie))'" % args.dynamic_lifetime]
1015 
1016  #expected lifetime can be an in (number of secs) or
1017  # one of a few strings, this should test for either
1018  # possibility
1019  try:
1020  dummy=int(args.expected_lifetime)
1021  jobsub_opts += ["--expected-lifetime=%ss" % (args.expected_lifetime)]
1022  except:
1023  allowed_lifetimes=["short","medium","long"]
1024  if args.expected_lifetime not in allowed_lifetimes:
1025  fail("Invalid expected_lifetime %s" % args.expected_lifetime)
1026  else:
1027  jobsub_opts += ["--expected-lifetime=%s" % (args.expected_lifetime)]
1028 
1029  # source stashcache.sh to set $PNFS_NOVA_STASH env varcorrectly for grid jobs
1030  # if we're using ngu_test, then the path to stascache doesn't work on the grid, so
1031  # just use the current tag
1032  if args.testrel_ngu:
1033  ngu_current = subprocess.check_output("ups list -c -KVERSION NovaGridUtils", shell=True).decode("utf-8").replace("\"","").strip()
1034  print("getting stashcache from /cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/"+ngu_current+"/NULL/bin/stashcache.sh")
1035  source_scripts.append(os.path.expandvars("/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/"+ngu_current+"/NULL/bin/stashcache.sh"))
1036  else:
1037  source_scripts.append(os.path.expandvars("$NOVAGRIDUTILS_DIR/bin/stashcache.sh"))
1038 
1039  # all software comes from CVMFS now since /nova/data is no longer mounted on Fermigrid
1040  build_location_arguments = "" # ":-e:%s/externals:/cvmfs/fermilab.opensciencegrid.org/products/common/db" % cvmfs_distro_base
1041  source_scripts.append( "/cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups.sh" )
1042  # assumes this job is being submitted on an up-to-date CVMFS install. best we can do...
1043  cvmfs_rev = subprocess.check_output(["attr", "-qg", "revision", "/cvmfs/nova.opensciencegrid.org"]).strip().decode("utf-8")
1044  if cvmfs_rev:
1045  art_sam_wrap_opts.append("--cvmfs-revision %s" % cvmfs_rev)
1046  if not args.disable_cvmfs_version_matching:
1047  # if NOvA CVMFS version is available, select on it; if not, just accept the slot anyway
1048  jobsub_opts += [ "--append_condor_requirements='ifThenElse(isUndefined(TARGET.HAS_CVMFS_nova_opensciencegrid_org)==FALSE,TARGET.CVMFS_nova_opensciencegrid_org_REVISION>=%s,TRUE)'" % cvmfs_rev ]
1049  if tag == "development" or tag[0] == 'N':
1050  # maybe we need to do -6 or -7 depending on what SRT_ARCH is, rather than just having them both ?
1051  build_location_arguments += ":-6:/cvmfs/nova-development.opensciencegrid.org/novasoft:-7:/cvmfs/nova-development.opensciencegrid.org/novasoft"
1052  else:
1053  build_location_arguments += ":-6:%(dir)s/novasoft/slf6/novasoft:-7:%(dir)s/novasoft/slf7/novasoft" % {"dir": cvmfs_distro_base}
1054 
1055  setup_location= "%s/novasoft/slf6/novasoft/setup/setup_nova.sh" % (cvmfs_distro_base)
1056 
1057 
1058  group=args.group
1059  #probably others are ok as well
1060  allowed_groups=['nova','fermilab']
1061  #allowed_groups=['nova_high_prio', 'nova_medium_prio', 'nova_low_prio', 'nova']
1062  if group not in allowed_groups :
1063  fail("The only valid args for --group are " + " ".join(allowed_group) )
1064  jobsub_opts += [ "-G %s" % group ]
1065 
1066  if args.test_queue:
1067  jobsub_opts += ["--subgroup test"]
1068  elif args.subgroup :
1069  subgroup = args.subgroup
1070  if is_production_arg :
1071  allowed_subgroups = [ "keepup_prio", "prod_high_prio", "prod_prio" ]
1072  else :
1073  fail( "Only production subgroups are available at this time and reqire production credentials" )
1074  #allowed_subgroups = [ "ana_prio" ]
1075 
1076  if subgroup in allowed_subgroups :
1077  jobsub_opts += [ "--subgroup %s" % subgroup ]
1078  else :
1079  fail( "Allowed subgroups: " + ", ".join( allowed_subgroups ) )
1080 
1081  jobsub_opts += ["--role=%s" %(role)]
1082 
1083 
1084  # Singularity
1085 
1086  if args.singularity:
1087  jobsub_opts += [ "--line='+SingularityImage=\\\"%s\\\"'" % args.singularity ]
1088  jobsub_opts += [ "--append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'"]
1089  if args.jobfactory:
1090  jobsub_opts += [ "--line='+JobFactoryType=\"%s\"'" % args.jobfactory ]
1091 
1092  # GPUs
1093  if args.gpu:
1094  jobsub_opts += [ "--line='+RequestGPUs=1'" ]
1095 
1096  if args.node_features:
1097  jobsub_opts += [ make_jobsub_node_features_arg(args.node_features) ]
1098 
1099 
1100 
1101  ####################
1102  # Setup runNovaSAM #
1103  ####################
1104 
1105  nevts=args.nevts
1106  if nevts > 0:
1107  run_nova_sam_opts += ["-n %d" % nevts ]
1108 
1109  if args.zipLogs: args.logs = True
1110 
1111  if args.lemBalance and args.lemServer:
1112  fail("Cannot specify both --lemServer and --lemBalance")
1113 
1114  # Toggled options accepted by runNovaSAM with exactly the same syntax as
1115  # our own options
1116  passThru = ['gdb', 'hashDirs', 'runDirs', 'noCleanup', 'jsonMetadata', 'copyOut', 'logs', 'zipLogs', 'outputNumuDeCAF', 'outputNueDeCAF', 'outputNumuOrNueDeCAF','outputNusDeCAF', 'outputValidationDeCAF', 'lemBalance']
1117 
1118  va = vars(args)
1119  for opt in passThru:
1120  if va[opt]:
1121  run_nova_sam_opts += ['--'+opt]
1122 
1123  # Could consider doing a similar loop for specialName, outTier, cafTier,
1124  # histTier, but gets a little complicated, and there are fewer of them to
1125  # begin with to make it worth the effort.
1126  if args.second_config:
1127  run_nova_sam_opts += ['--second_config ' + args.second_config]
1128 
1129  if args.lemServer:
1130  run_nova_sam_opts += ["--lemServer " + args.lemServer]
1131 
1132  if args.cosmicsPolarity:
1133  run_nova_sam_opts += ["--cosmicsPolarity " + args.cosmicsPolarity]
1134 
1135  if args.npass:
1136  run_nova_sam_opts += ["--npass " + args.npass]
1137 
1138  if args.skim :
1139  run_nova_sam_opts += ["--skim " + args.skim]
1140 
1141  if args.systematic :
1142  run_nova_sam_opts += ["--systematic " + args.systematic]
1143 
1144  if args.specialName :
1145  run_nova_sam_opts += ["--specialName " + args.specialName]
1146 
1147  if args.genietune :
1148  run_nova_sam_opts += ["--genietune " + args.genietune]
1149 
1150  if args.NPPFX :
1151  run_nova_sam_opts += ["--NPPFX " + args.NPPFX]
1152 
1153  if args.declareFiles:
1154  run_nova_sam_opts += ["--declareFiles"]
1155 
1156  if args.declareLocations:
1157  run_nova_sam_opts += ["--declareLocations"]
1158 
1159  # if only histTier provided, assume this means we're running an analyzer module,
1160  # and set to grab the metadata from the parent file
1161  if args.histTier and not (args.outTier or args.cafTier):
1162  run_nova_sam_opts += ["--useParentMetadata"]
1163 
1164  out_tiers=args.outTier
1165 
1166  if fcl == "mcgen" :
1167  outnum,mcouttier = out_tiers[0].split(":")
1168  if mcouttier == "artdaq" :
1169  copyback = '"*.daq.root"'
1170  else:
1171  copyback = '"*.'+ mcouttier +'.root"'
1172 
1173  if None != out_tiers :
1174  for tier in out_tiers :
1175  run_nova_sam_opts += ["--outTier " + tier]
1176 
1177  caf_tiers=args.cafTier
1178  if None != caf_tiers :
1179  for tier in caf_tiers :
1180  run_nova_sam_opts += ["--cafTier " + tier]
1181 
1182  flat_tiers=args.flatTier
1183  if None != flat_tiers :
1184  for tier in flat_tiers :
1185  run_nova_sam_opts += ["--flatTier " + tier]
1186 
1187  hist_tier=args.histTier
1188  if None != hist_tier :
1189  run_nova_sam_opts += ["--histTier " + hist_tier]
1190 
1191  h5_tiers=args.h5Tier
1192  if None != h5_tiers :
1193  for tier in h5_tiers :
1194  run_nova_sam_opts += ["--h5Tier " + tier]
1195 
1196  if args.copyOut and (None==hist_tier and None==caf_tiers and None==flat_tiers and None==out_tiers and None==h5_tiers):
1197  fail("You specified --copyOut but did not specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier")
1198 
1199  if not (args.copyOut or args.copyOutScript) :
1200  fail("Did not specify a method to copy back output (--copyOut or --copyOutScript)")
1201 
1202  if args.copyOut and args.copyOutScript :
1203  fail("The options --copyOut and --copyOutScript conflict")
1204 
1205  if args.export:
1206  export_to_art_sam_wrap += args.export
1207 
1208  if args.veryearlyscript:
1209  veryearly_scripts += args.veryearlyscript
1210 
1211  if args.source:
1212  source_scripts += args.source
1213 
1214  if args.earlyscript:
1215  early_scripts += args.earlyscript
1216 
1217  if args.prescript:
1218  pre_scripts += args.prescript
1219 
1220  if args.precopyscript:
1221  inter_scripts += args.precopyscript
1222 
1223  if args.postscript:
1224  post_scripts += args.postscript
1225 
1226  if args.copyOutScript:
1227  post_scripts.append(args.copyOutScript)
1228 
1229  for script in veryearly_scripts + early_scripts + source_scripts + pre_scripts + post_scripts + inter_scripts:
1230  if ":" in script:
1231  script_path = script.split(":")[0]
1232  else:
1233  script_path = script
1234 
1235  if not find_file_in_list(os.path.expandvars(script_path), input_files):
1236  if not find_file(os.environ["PATH"].split(os.pathsep), os.path.expandvars(script_path)):
1237  fail("Script %s does not exist!" % script_path)
1238 
1239  if not args.continue_project:
1240  ##start sam project
1241  project_name = user + "-" + jobname + "-" + timestamp
1242  if args.test_submission:
1243  project_name += "-testjobs"
1244  start_project = True
1245  else:
1246  project_name = args.continue_project
1247  start_project = False
1248 
1249  #sam_station=os.getenv("SAM_STATION")
1250 
1251  if args.novasoftups:
1252  art_sam_wrap_cmd="$NOVASOFT_FQ_DIR/bin/art_sam_wrap.sh"
1253 
1254 
1255 
1256 
1257 
1258  #########################
1259  # Start the SAM project #
1260  #########################
1261 
1262 
1263  start_proj_command ="samweb start-project "
1264  if not snapshot_id:
1265  start_proj_command+=" --defname=%s" %defname
1266  else:
1267  start_proj_command+=" --snapshot_id=%d" % snapshot_id
1268  start_proj_command+=" --group=nova"
1269  start_proj_command+=" --station=%s" % sam_station
1270 
1271  start_proj_command+= " %s" %project_name
1272  if start_project and not test:
1273  start_proj_retval=os.system(start_proj_command)
1274  print("start proj returned %d" % start_proj_retval)
1275  if start_proj_retval != 0:
1276  fail("Couldn't start project")
1277 
1278  print("Station monitor: http://samweb.fnal.gov:8480/station_monitor/nova/stations/" + sam_station +"/projects/" + project_name)
1279  os.putenv("SAM_PROJECT_NAME",project_name)
1280 
1281  check_make_dir("${CONDOR_EXEC}")
1282  # Ensure unique job command
1283  job_cmd=os.path.expandvars("${CONDOR_EXEC}/%s.sh" % project_name)
1284  scriptcount=1
1285  while os.path.exists(job_cmd):
1286  job_cmd=os.path.expandvars("${CONDOR_EXEC}/%s_%i.sh" % (project_name, scriptcount) )
1287  scriptcount += 1
1288 
1289 
1290 
1291 
1292  ######################
1293  # Setup art_sam_wrap #
1294  ######################
1295 
1296  if not test:
1297  #create symlink so that jobs get a better name
1298  os.symlink(os.path.expandvars(art_sam_wrap_cmd),job_cmd)
1299  art_sam_wrap_cmd=job_cmd
1300 
1301  sys.stdout.flush()
1302  sys.stderr.flush()
1303 
1304 
1305 
1306  if not args.no_multifile:
1307  art_sam_wrap_opts += ["--multifile"]
1308 
1309  if args.printenv:
1310  art_sam_wrap_opts += ["--printenv"]
1311 
1312  if args.txtfiledef:
1313  run_nova_sam_opts += ["--txtfiledef"]
1314  print("Passing --txtfiledef from submit_nova_art.py to runNovaSAM")
1315 
1316  if not mcgen:
1317  art_sam_wrap_opts += ["--config " + fcl]
1318 
1319  if mcgen:
1320  jobsub_opts += ['-l "+JobType="MC""']
1321  art_sam_wrap_opts += ["--getconfig"]
1322 
1323  if args.mix:
1324  art_sam_wrap_opts += ["--mix",args.mix]
1325 
1326  if not args.novasoftups :
1327  art_sam_wrap_opts += ["--source %s:-r:%s:%s%s" %(setup_location, tag, maxopt_opt, build_location_arguments)]
1328  if args.testrel or args.user_tarball:
1329  art_sam_wrap_opts += ["--setup_testrel"]
1330 
1331  if args.ngu_test:
1332  art_sam_wrap_opts += ["--source setup_test_product:NovaGridUtils"]
1333 
1334  if args.ngu_version:
1335  art_sam_wrap_opts += ["--source setup_product:NovaGridUtils:%s" %(args.ngu_version)]
1336 
1337  if args.testrel_ngu:
1338  art_sam_wrap_opts += ["--testrel_ngu"]
1339 
1340  for veryearly_script in veryearly_scripts:
1341  art_sam_wrap_opts += ["--veryearlyscript " + veryearly_script]
1342 
1343  for early_script in early_scripts:
1344  art_sam_wrap_opts += ["--earlyscript " + early_script]
1345 
1346  for source_script in source_scripts:
1347  art_sam_wrap_opts += ["--source " + source_script]
1348 
1349  for pre_script in pre_scripts:
1350  art_sam_wrap_opts += ["--prescript " + pre_script]
1351 
1352  for inter_script in inter_scripts:
1353  run_nova_sam_opts += ["--precopyscript " + inter_script]
1354 
1355  for post_script in post_scripts:
1356  art_sam_wrap_opts += ["--postscript " + post_script]
1357 
1358  for input_file in input_files:
1359  art_sam_wrap_opts += ["--inputfile " + input_file]
1360 
1361  if mcgen:
1362  art_sam_wrap_opts += ["--addoutput " + copyback]
1363  if args.hashDirs==True:
1364  art_sam_wrap_opts += ["--hash"]
1365  art_sam_wrap_opts += ["--dest " + dest]
1366 
1367  if args.poms and not test:
1368  poms_campaign_id = poms_client.register_poms_campaign(
1369  jobname,
1370  user = user,
1371  experiment = 'nova',
1372  version = tag,
1373  dataset = defname)
1374 # campaign_definition = args.poms_definition )
1375 
1376  poms_task_id = poms_client.get_task_id_for(
1377  poms_campaign_id,
1378  user = user,
1379  command_executed = build_jobsub_cmd() )
1380 
1381  export_to_art_sam_wrap += ["POMS_CAMPAIGN_ID={0}".format(poms_campaign_id),
1382  "POMS_TASK_ID={0}".format(poms_task_id)]
1383  print("POMS Campaign: https://pomsgpvm01.fnal.gov/poms/campaign_info?campaign_id={0}".format(poms_campaign_id))
1384 
1385 
1386 
1387  ############################
1388  # Actually launch the jobs #
1389  ############################
1390 
1391  jobsub_cmd = build_jobsub_cmd()
1392 
1393  if print_jobsub:
1394  print(jobsub_cmd)
1395  sys.stdout.flush()
1396  sys.stderr.flush()
1397 
1398 
1399  if not test:
1400  os.system(jobsub_cmd)
1401 
1402  if njobs > 1000:
1403  print()
1404  print("Please note: if you intend to submit any more jobs,")
1405  print(" please wait", old_div(njobs,1000), "minutes before your next submission")
1406  print(" so as to avoid overloading the jobsub server.")
void split(double tt, double *fr)
def setup_production(args)
def check_dir(output_dir, prefix='')
Definition: check_jobs.py:127
def fail(msg)
def find_file(paths, filename)
def remove_comments(src)
def setup_analysis(args)
def make_jobsub_node_features_arg(features)
bool print
def check_tag(tag)
const std::map< std::pair< std::string, std::string >, Variable > vars
def find_file_in_list(filepath, pathlist)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
void decode(std::any const &, InputTag &)
def check_fcl(tag, fcl)
def add_node_features_arg(parser)
def check_env(vname)
def warn(msg)
def setup_calibration(args)
def check_make_dir(dname)