3 from __future__
import print_function
4 from __future__
import division
5 from future
import standard_library
6 standard_library.install_aliases()
7 from builtins
import str
8 from past.utils
import old_div
9 import os, sys, stat, pwd, re
18 from NovaGridUtils
import *
19 import recommended_sites
as rs
21 user=os.getenv(
"USER")
22 os.environ[
"GRID_USER"]=user
25 sam_station=os.getenv(
"SAM_STATION")
26 os.system(
"echo SAM_STATION DEFINED AS $SAM_STATION")
28 cvmfs_distro_base =
"/cvmfs/nova.opensciencegrid.org" 29 novasoft_cvmfs =
"%s/novasoft/slf6/novasoft" % cvmfs_distro_base
30 build_location_arguments =
"" 35 run_nova_sam_cmd=
"runNovaSAM.py" 37 export_to_run_nova_sam = []
39 art_sam_wrap_cmd=
"$NOVAGRIDUTILS_DIR/bin/art_sam_wrap.sh" 41 export_to_art_sam_wrap=[]
42 export_to_art_sam_wrap.append(
"SAM_PROJECT_NAME")
43 export_to_art_sam_wrap.append(
"SAM_STATION")
44 export_to_art_sam_wrap.append(
"IFDH_BASE_URI")
45 if "IFDH_DEBUG" in os.environ:
46 export_to_art_sam_wrap.append(
"IFDH_DEBUG")
47 if "G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION" in os.environ:
48 export_to_art_sam_wrap.append(
"G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION")
50 export_to_art_sam_wrap.append(
"EXPERIMENT")
51 export_to_art_sam_wrap.append(
"GRID_USER")
53 os.environ[
"CVMFS_DISTRO_BASE"]=cvmfs_distro_base
54 export_to_art_sam_wrap.append(
"CVMFS_DISTRO_BASE")
57 usage_models=[
"DEDICATED"]
69 value=os.getenv(vname)
70 if None == value
or "" == value:
71 fail(
"Environment variable %s not defined" %vname)
74 if "/" in fcl[:1]
and os.path.isfile(fcl):
77 fclPaths = os.environ[
"FHICL_FILE_PATH"].
split(
":")
79 fullPath = os.path.join(path, fcl)
80 if os.path.isfile(fullPath):
83 for input_file
in input_files:
84 if os.path.basename(input_file) == fcl:
86 elif os.path.basename(input_file) == os.path.basename(fcl):
88 print(
"The fhicl will be copied to $PWD on the worker node.")
89 print(
"You specified some sort of file path which isn't needed. Fixing it for you :)")
90 print(fcl +
" --> " + os.path.basename(fcl))
92 return os.path.basename(fcl)
94 fail(
"fcl file %s does not exist" %fcl)
99 This reads tokens using tokenize.generate_tokens and recombines them 100 using tokenize.untokenize, and skipping comment/docstring tokens in between 102 f = io.StringIO(src.encode().
decode())
104 processed_tokens = []
107 for tok
in tokenize.generate_tokens(f.readline):
108 t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok
111 if t_type == tokenize.COMMENT:
112 raise SkipException()
114 elif t_type == tokenize.STRING:
116 if last_token
is None or last_token[0]
in [tokenize.INDENT]:
119 except SkipException:
122 processed_tokens.append(tok)
126 return tokenize.untokenize(processed_tokens)
129 if os.path.isfile(filename):
132 for root, dirs, files
in os.walk(os.path.expandvars(path)):
133 if filename
in files:
134 return os.path.join(root, filename)
135 fail(
"Cannot find file "+filename)
138 for testpath
in pathlist:
139 if os.path.basename(filepath) == os.path.basename(testpath):
146 jobsub_cmd =
"jobsub_submit \\\n" 147 for opt
in jobsub_opts:
148 jobsub_cmd +=
" " + opt +
" \\\n" 149 for export
in export_to_art_sam_wrap:
150 jobsub_cmd +=
" -e " + export
151 jobsub_cmd +=
" \\\n" 155 if args.reuse_tarball:
156 jobsub_cmd +=
" --tar_file_name dropbox://" + os.path.basename(args.testrel) +
".tar \\\n" 158 jobsub_cmd +=
" --tar_file_name tardir://" + args.testrel +
" \\\n" 159 elif args.user_tarball:
160 if not os.path.isfile(args.user_tarball):
161 print(
"Tarball filename passed to --user_tarball does not exit:", args.user_tarball)
163 jobsub_cmd +=
" --tar_file_name dropbox://" + args.user_tarball +
" \\\n" 165 jobsub_cmd +=
" file://" + art_sam_wrap_cmd +
" \\\n" 166 for opt
in art_sam_wrap_opts:
167 jobsub_cmd +=
" " + opt +
" \\\n" 168 for export
in export_to_run_nova_sam :
169 jobsub_cmd +=
" --export " + export +
" \\\n" 175 jobsub_cmd +=
" -X " + run_nova_sam_cmd +
" \\\n" 176 for opt
in run_nova_sam_opts:
177 jobsub_cmd +=
" " + opt +
" \\\n" 179 jobsub_cmd +=
" -X nova \\\n" 180 mcgen_opts = [
"--sam-file-type=importedSimulated",
"--sam-application-family=nova",
"--sam-data-tier=" + mcouttier,
"--sam-application-version=" + tag]
182 mcgen_opts += [
"-n %d" % nevts]
183 for opt
in mcgen_opts:
184 jobsub_cmd +=
" " + opt +
" \\\n" 186 jobsub_cmd = jobsub_cmd[:-2]
192 if __name__==
'__main__':
194 prog=os.path.basename(sys.argv[0])
195 NovaGridUtils.prog=prog
198 while "-f" in sys.argv
or "--file" in sys.argv:
204 preliminary_parser = argparse.ArgumentParser(prog=prog, description=
'Submit nova art job')
206 preliminary_parser.add_argument(
'-f',
'--file',
207 help=
"""Text file containing any arguments to this utility. Multiple allowed. 208 Arguments should look just like they would on the command line, 209 but the parsing of this file is whitespace insenstive. 210 Commented lines will be identified with the # character and removed. """, type=str, action=
'append')
211 pre_args, unknown = preliminary_parser.parse_known_args()
214 sys.argv = [x
for x
in sys.argv
if x
not in [
"-f",
"--file"]]
217 for filepath
in pre_args.file:
218 index = sys.argv.index(filepath)
219 sys.argv.remove(filepath)
220 if os.path.isfile(filepath):
223 fullpath =
find_file([
"$NOVAGRIDUTILS_DIR/configs/"],filepath)
224 text =
open(fullpath,
'r').read() 227 for line
in text.splitlines():
230 newargs += line.split()
231 sys.argv[index:index] = newargs
233 parser = argparse.ArgumentParser(prog=prog, description=
'Submit nova art job', add_help=
False)
236 required_args = parser.add_argument_group(
"Required arguments",
"These arguments must be supplied.")
238 required_args.add_argument(
'--jobname',
243 required_args.add_argument(
'--defname',
245 help=
'SAM dataset definition to run over',
248 required_args.add_argument(
'--config',
'-c',
250 help=
'FHiCL file to use as configuration for nova executable. The path given should be relative to the $SRT_PRIVATE_CONTEXT of any test release you submit using',
253 required_args.add_argument(
"--tag",
255 help=
"Tag of novasoft to use",
258 required_args.add_argument(
"--dest",
260 help=
"Destination for output files",
264 debugging_args = parser.add_argument_group(
"Debugging options",
"These optional arguments can help debug your submission.")
266 debugging_args.add_argument(
'--print_jobsub',
267 help=
'Print jobsub command',
268 action=
'store_true',default=
False)
270 debugging_args.add_argument(
'--printenv',
271 help=
'Print environment variables',
272 action=
'store_true',default=
False)
274 debugging_args.add_argument(
'--test',
275 help=
'Do not actually do anything, just run tests and print jobsub cmd',
276 action=
'store_true',default=
False)
278 debugging_args.add_argument(
'--gdb',
279 help=
'Run nova executable under gdb, print full stack trace, then quit gdb.',
280 action=
'store_true',default=
False)
282 debugging_args.add_argument(
'--test_submission',
283 help=
'Override other arguments given to submit a test to the grid. It will run 1 job with 3 events and write the output to /pnfs/nova/scratch/users/<user>/test_jobs/<date>_<time>',
284 action=
'store_true',default=
False)
286 debugging_args.add_argument(
'--jobsub_server',
287 help=
'Submit using the specified jobsub server',
290 debugging_args.add_argument(
'--test_queue',
291 help=
'Submit jobs to the test jobsub queue for higher starting priority. NB NOvA is limited to 10 jobs at a time on this queue.',
292 action=
'store_true', default=
False)
294 debugging_args.add_argument(
"--kill_after",
296 help=
"If job is still running after this many seconds, kill in such a way that a log will be returned",
300 job_control_args = parser.add_argument_group(
"Job control options",
"These optional arguments help control where and how your jobs land.")
302 job_control_args.add_argument(
'--njobs',
303 help=
'Number of jobs to submit',
304 type=int, default = 0)
306 job_control_args.add_argument(
'--maxConcurrent',
307 help=
'Run a maximum of N jobs simultaneously',
311 job_control_args.add_argument(
'--files_per_job',
312 help=
'Number of files per job - if zero, calculate from number of jobs', metavar=
'N',
313 type=int, default = 0)
315 job_control_args.add_argument(
'--nevts',
316 help=
'Number of events per file to process',
317 type=int, default = 0)
320 job_control_args.add_argument(
'--no_multifile',
321 help=
'Do not use art_sam_wrap.sh multifile mode, which is on by default',
324 job_control_args.add_argument(
'--txtfiledef',
325 help=
'Use if the input definition is made up of text files, each containing a list of file names',
326 action=
'store_true',default=
False)
329 job_control_args.add_argument(
'--opportunistic',
330 help=
'Run opportunistically on the fermigrid',
331 action=
'store_true',default=
False)
333 job_control_args.add_argument(
'--offsite',
334 help=
'Allow to run on offsite resources as well. Implies --opportunistic.',
335 action=
'store_true',default=
False)
337 job_control_args.add_argument(
'--offsite_only',
338 help=
'Allow to run solely on offsite resources.',
339 action=
'store_true',default=
False)
341 job_control_args.add_argument(
'--amazon',
342 help=
'Run at amazon.',
343 action=
'store_true',default=
False)
345 job_control_args.add_argument(
'--site',
346 help=
'Specify allowed offsite locations. Omit to allow running at any offsite location',
347 type=str,action=
'append')
349 job_control_args.add_argument(
'--exclude_site',
350 help=
'Specify an offsite location to exclude.',
352 type=str,action=
'append')
354 job_control_args.add_argument(
'--recommended_sites',
355 help=
'Specify known working offsite locations.',
356 action=
'store_true',default=
False)
358 job_control_args.add_argument(
'--recommended_sites_rockgen',
359 help=
'Specify known working offiste locations, for rock generation.',
360 action=
'store_true',default=
False)
362 job_control_args.add_argument(
'--recommended_sites_generation',
363 help=
'Specify known working offiste locations, for GENIE/GEANT generation.',
364 action=
'store_true',default=
False)
366 job_control_args.add_argument(
'--autoDropbox',
367 help=
'Use automatic dropbox location based on site',
368 action=
'store_true',default=
False)
371 sing_group = job_control_args.add_mutually_exclusive_group()
372 sing_group.add_argument(
'--singularity',
373 help=
'Location in CVMFS of a singularity container to launch the job into. ' + \
374 "If you're looking for a stock Scientific Linux image, try --os instead. " + \
375 "(This option is mutually exclusive with --os.)",
378 sing_group.add_argument(
"--os",
379 help=
"Run this job inside a Scientific Linux singularity image with given OS. " + \
380 "(This option is mutually exclusive with --singularity.)",
381 choices=NovaGridUtils.KNOWN_OS_CONTAINERS)
383 job_control_args.add_argument(
'--disk',
384 help=
'Local disk space requirement for worker node in MB.',
385 type=int, default=10000)
387 job_control_args.add_argument(
'--memory',
388 help=
'Local memory requirement for worker node in MB.',
389 type=int, default=1900)
391 job_control_args.add_argument(
'--expected_lifetime',
392 help=
'Expected job lifetime (default is 10800s=3h). Valid values are an integer number of seconds or one of \"short\" (6h), \"medium\" (12h) or \"long\" (24h, jobsub default)', metavar=
'LIFETIME',
393 type=str, default=
"10800")
395 job_control_args.add_argument(
'--dynamic_lifetime',
396 help=
"Dynamically determine whether a new file should be started based on glidein lifetime. Specify the maximum length expected for a single file to take to process in seconds.", metavar=
"LIFETIME",
399 job_control_args.add_argument(
'--cpu',
400 help=
"Request worker nodes that have at least NUMBER cpus",
404 job_control_args.add_argument(
'--group',
'-G',
405 help=
"Specify batch group GROUP -- mainly used to set job priority. At present, only supportable value is nova",
406 type=str, default=
"nova")
408 job_control_args.add_argument(
'--subgroup',
409 help=
'Production subgroup',
412 job_control_args.add_argument(
"--role",
413 help=
"Specify role to run on the grid. Can be Analysis (default) or Production. This option is no longer supported",
414 type=str,default=
"Analysis")
416 job_control_args.add_argument(
'--continue_project',
417 help=
"Don't start a new samweb project, instead continue this one.", metavar=
'PROJECT',
420 job_control_args.add_argument(
"--snapshot_id",
421 help=
"Use this existing snapshot instead of creating a new one.", metavar=
"ID",
424 job_control_args.add_argument(
"--poms",
425 help=
"Start/continue a poms campaign and task for this submission",
426 default=
False, action=
"store_true")
428 job_control_args.add_argument(
"--mix",
429 help=
"Pass a mixing script to the job to pull in a files for job mixing.",
430 default=
"", type=str)
432 job_control_args.add_argument(
"--mail_always",
433 help=
"Do you want an email whenever every jobs finishes?",
434 default=
False, action=
"store_true")
436 job_control_args.add_argument(
"--mail_on_error",
437 help=
"Do you want an email whenever a job fails on an error?",
438 default=
False, action=
"store_true")
444 job_control_args.add_argument(
'--user_priority',
445 help=
'Priority (integer) within a user\'s jobs (default = 0)',
448 job_control_args.add_argument(
'--jobfactory',
449 help=
'Use the specified JobFactoryType.',
452 job_control_args.add_argument(
"--gpu",
453 help=
"Request a node with a GPU",
454 default=
False, action=
"store_true")
459 novasoft_args = parser.add_argument_group(
"NOvA software options",
"These options control the novasoft setup.")
460 novasoft_args.add_argument(
'--maxopt',
461 help=
'Run in maxopt mode',
462 action=
'store_true',default=
True)
464 testrel_gp = novasoft_args.add_mutually_exclusive_group(required=
False)
465 testrel_gp.add_argument(
"--testrel",
466 help=
"Use a test release at location TESTREL. It will be tarred up, and sent to the worker node.",
468 testrel_gp.add_argument(
"--user_tarball",
469 help=
"Use existing test release tarball in specified location rather than having jobsub make one for you (conflicts with --testrel)",
472 novasoft_args.add_argument(
'--reuse_tarball',
473 help=
'Do you want to reuse a tarball that is already in resilient space? If using this option avoid trailing slash in --testrel option. (conflicts with --user_tarball)',
474 action=
'store_true',default=
False)
476 novasoft_args.add_argument(
'--cvmfs',
477 help=
'Does nothing (always true), but retained for compatibility: pull software from CVMFS.',
481 novasoft_args.add_argument(
'--disable_cvmfs_version_matching',
482 help=
"Don't perform a CVMFS-is-up-to-date check on target nodes via Condor requirements. (For advanced debugging use.) ",
486 novasoft_args.add_argument(
'--novasoftups',
487 help=
'Use the ups build of novasoft, must be used with source to setup.',
490 novasoft_args.add_argument(
'--ngu_test',
491 help=
'Setup the test version of NovaGridUtils in the grid jobs.',
494 novasoft_args.add_argument(
'--ngu_version',
495 help=
'Setup a specific NovaGridUtils version in the grid jobs.', metavar=
'VERSION',
498 novasoft_args.add_argument(
'--testrel_ngu',
499 help=
"Must be used with --testrel, with NGU checked out. After unpacking tarball will setup the local version of NGU you are using on the work.",
502 novasoft_args.add_argument(
'--lemBalance',
503 help=
'Choose lem server based on (CLUSTER+PROCESS)%%2 to balance load',
504 action=
'store_true', default=
False)
506 novasoft_args.add_argument(
'--lemServer',
507 help=
'Specify lem server',
511 output_args = parser.add_argument_group(
"Output file options",
"Note that you must specify either --copyOut or --copyOutScript.")
514 output_args.add_argument(
'--copyOutScript',
515 help=
'Use script COPYOUTSCRIPT to copy back your output',
518 output_args.add_argument(
'--copyOut',
519 help=
'Use the built in copy out mechanism. If used, you must specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier',
522 output_args.add_argument(
'--logs',
523 help=
'Return .log files corresponding to every output',
525 output_args.add_argument(
'--zipLogs',
526 help=
'Format logs as .bz2 files. Implies --logs',
529 output_args.add_argument(
'--outTier',
530 help=
'Data tier of the output file, multiple allowed, formatted as <name_in_fcl_outputs>:<data_tier>',
531 type=str, action=
'append')
533 output_args.add_argument(
'--cafTier',
534 help=
'Module label for CAF output, multiple allowed. Format as <cafmaker_module_label>:<data_tier>',
535 type=str, action=
'append')
537 output_args.add_argument(
'--flatTier',
538 help=
'Module label for FlatCAF output, multiple allowed. Format as <flatmaker_module_label>:<data_tier>',
539 type=str, action=
'append')
541 output_args.add_argument(
'--histTier',
542 help=
'File identifier string for TFileService output, only one allowed. Supply as --histTier <id> for output_name.<id>.root, where output_name is assembled based on the input file.',
545 output_args.add_argument(
'--h5Tier',
546 help=
'File identifier for h5 output, multiple allowed. Format as <hdf5maker_module>:<data_tier>',
547 type=str, action=
'append')
548 output_args.add_argument(
'--second_config',
549 help=
"""Second configuration fcl executed after main process. 550 nova is executed with art file that is output from the main process. 551 Files that get produced by this process that are named identically 552 to files produced by the main process and are among the requested outputs 553 are ignored and the file produced by the first process is returned""",
556 output_args.add_argument(
'--outputNumuDeCAF',
557 help=
'Make standard numu decafs for all CAF files produced during the job',
558 action=
'store_true',default=
False)
560 output_args.add_argument(
'--outputNueDeCAF',
561 help=
'Make standard nue decafs for all CAF files produced during the job',
562 action=
'store_true',default=
False)
564 output_args.add_argument(
'--outputNumuOrNueDeCAF',
565 help=
'Make standard nue or numu decafs for all CAF files produced during the job',
566 action=
'store_true',default=
False)
568 output_args.add_argument(
'--outputNusDeCAF',
569 help=
'Make standard nus decafs for all CAF files produced during the job',
570 action=
'store_true',default=
False)
572 output_args.add_argument(
'--outputValidationDeCAF',
573 help=
'Make validation (nue_or_numu_or_nus) decafs for all CAF files produced during the job',
574 action=
'store_true',default=
False)
576 output_args.add_argument(
'--cosmicsPolarity',
577 help=
'To specify a horn polarity for the cosmics output file name',
580 output_args.add_argument(
'--npass',
581 help=
'To specify npass (aka nova.subversion)',
584 output_args.add_argument(
'--skim',
585 help=
'To specify nova.skim (does not work with mc gen)',
588 output_args.add_argument(
'--systematic',
589 help=
'To specify nova.systematic (does not work with mc gen)', metavar=
'SYST',
592 output_args.add_argument(
'--specialName',
593 help=
'To specify nova.special name (does not work with mc gen)',
596 output_args.add_argument(
'--genietune',
597 help=
'To specify nova.genietune (does not work with mc gen)', metavar=
'TUNE',
600 output_args.add_argument(
'--NPPFX',
601 help=
'To specify number of PPFX universes',
604 output_args.add_argument(
'--hashDirs',
605 help=
'Use hash directory structure in destination directory.',
608 output_args.add_argument(
'--runDirs',
609 help=
'Use run directory structure in destination directory, 000XYZ/XYZUW for run number XYZUW.',
612 output_args.add_argument(
'--noCleanup',
613 help=
'Pass --noCleanup argument to runNovaSAM.py. Necessary when using a postscript for copyout.',
616 output_args.add_argument(
'--jsonMetadata', help=
'Create JSON files with metadata corresponding to each output file, and copy them to the same destinations', action=
'store_true')
618 output_args.add_argument(
'--declareFiles',
619 help=
'Declare files with metadata on worker node',
622 output_args.add_argument(
'--production',
623 help=
'Submit production style jobs. Implies \"--role=Production --hashDirs --jsonMetadata --zipLogs\", and checks that other settings needed for production are specified',
626 output_args.add_argument(
'--calibration',
627 help=
'Submit calibration style jobs. Implies \"--role=Production\", and checks that other settings needed for calibration are specified',
630 output_args.add_argument(
'--declareLocations',
631 help=
'Declare the file output locations to SAM during the copy back of the files',
635 environment_args = parser.add_argument_group(
"Environment options",
"These optional arguments allow control of the grid running environment.")
637 environment_args.add_argument(
'--export',
638 help=
'Export variable EXPORT to art_sam_wrap.sh',
639 type=str, action=
'append')
641 environment_args.add_argument(
'--veryearlyscript',
642 help=
'Source script EARLYSCRIPT before any environment setup or sourcing',
643 type=str, action=
'append')
645 environment_args.add_argument(
'--source',
646 help=
'Source script SOURCE',
647 type=str, action=
'append')
649 environment_args.add_argument(
'--earlyscript',
650 help=
'Execute script EARLYSCRIPT before any environment setup',
651 type=str, action=
'append')
653 environment_args.add_argument(
'--prescript',
654 help=
'Execute script PRESCRIPT before executing runNovaSAM.py',
655 type=str, action=
'append')
657 environment_args.add_argument(
'--precopyscript',
658 help=
'Execute script PRECOPYSCRIPT within runNovaSAM.py, after running the nova -c command.',
659 type=str, action=
'append')
661 environment_args.add_argument(
'--postscript',
662 help=
'Execute script POSTSCRIPT after executing runNovaSAM.py',
663 type=str, action=
'append')
665 environment_args.add_argument(
'--inputfile',
666 help=
'Copy this extra input file into job area before running executable',
667 type=str, action=
'append')
671 support_args = parser.add_argument_group(
"Support options",
"These optional arguments using this submission utility easier.")
673 support_args.add_argument(
"-h",
"--help", action=
"help", help=
"Show this help message and exit")
675 support_args.add_argument(
'-f',
'--file',
676 help=
"""Text file containing any arguments to this utility. Multiple allowed. 677 Arguments should look just like they would on the command line, 678 but the parsing of this file is whitespace insenstive. 679 Comments will be identified with the # character and removed. """, type=str, action=
'append')
688 args = parser.parse_args()
689 timestamp=datetime.datetime.now().strftime(
"%Y%m%d_%H%M")
693 istarball = args.testrel
or args.user_tarball
694 use_recommended_sites=
False 695 if args.recommended_sites:
696 recommended_sites = rs.get_recommended_sites(
'default', istarball, args.offsite_only)
697 use_recommended_sites=
True 698 if args.recommended_sites_rockgen:
699 recommended_sites = rs.get_recommended_sites(
'rockgen', istarball, args.offsite_only)
700 use_recommended_sites=
True 701 if args.recommended_sites_generation:
702 recommended_sites = rs.get_recommended_sites(
'generation', istarball, args.offsite_only)
703 use_recommended_sites=
True 710 print(
"POMS not setup. Run this and try again:")
712 print(
" setup poms_client")
717 if args.test_submission:
720 test_dest =
"/pnfs/nova/scratch/users/%s/test_jobs/%s" % (os.environ[
"USER"], timestamp)
721 if not os.path.exists(test_dest):
722 os.makedirs(test_dest)
723 mode = os.stat(test_dest).st_mode | stat.S_IXGRP | stat.S_IWGRP
724 os.chmod(test_dest, mode)
725 test_expected_lifetime =
"0" 726 test_dynamic_lifetime =
"500" 727 test_files_per_job = 1
729 print(
"Running a test submission. Overwriting:")
731 print(
" njobs", args.njobs,
"-->", test_njobs)
732 args.njobs = test_njobs
733 print(
" nevts", args.nevts,
"-->", test_nevts)
734 args.nevts = test_nevts
735 print(
" dest", args.dest,
"-->", test_dest)
736 args.dest = test_dest
737 print(
" expected_lifetime", args.expected_lifetime,
"-->", test_expected_lifetime)
738 args.expected_lifetime = test_expected_lifetime
739 print(
" dynamic_lifetime", args.dynamic_lifetime,
"-->", test_dynamic_lifetime)
740 args.dynamic_lifetime = test_dynamic_lifetime
741 print(
" files_per_job", args.files_per_job,
"-->", test_files_per_job)
742 args.files_per_job = test_files_per_job
743 if args.declareFiles:
744 print(
" don't declareFiles")
745 args.declareFiles =
False 746 if args.declareLocations:
747 print(
" don't declareLocations")
748 args.declareLocations =
False 750 print(
" don't use autoDropbox")
751 args.autoDropbox =
False 753 print(
" don't use poms")
762 snapshot_id=args.snapshot_id
763 print_jobsub=args.print_jobsub
766 print(
"Will print environment vars ")
773 warn(
"--test was specified, so all we do is run checks and print jobsub cmd.")
785 maxopt_opt +=
"-b:maxopt" 788 if args.reuse_tarball
and not args.testrel:
789 fail(
"--reuse_tarball specified without --testrel??")
793 if not os.path.isdir(args.testrel+
'/lib/'+os.getenv(
'SRT_ARCH')+
'-GCC-'+srt_qual):
794 fail(args.testrel+
' has never been built '+srt_qual)
797 input_files += args.inputfile
799 for input_file
in input_files:
800 if not os.path.isfile(os.path.expandvars(input_file)):
801 fail(
"Input file %s does not exist!" % input_file)
802 if os.path.expandvars(input_file).startswith(
"/nova/"):
803 fail(
"Input file %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % input_file)
804 elif os.path.expandvars(input_file).startswith(
"/grid/"):
805 fail(
"Input file %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % input_file)
807 if args.singularity
and not os.path.exists(args.singularity):
808 fail(
"Requested singularity image cannot be found: %s" % args.singularity)
810 if args.gpu
and not args.singularity:
811 warn(
"Requested GPU, but did not request singularity. This is not likely to succeed.")
813 if args.gpu
and not args.offsite_only:
814 warn(
"GPUs are only available offsite, and you have not chosen --offsite_only")
820 mcgen = (fcl ==
"mcgen")
826 if not dest.startswith(
"s3://") :
829 if os.path.expandvars(dest).startswith(
"/nova/"):
830 fail(
"Destination directory %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % dest)
831 elif os.path.expandvars(dest).startswith(
"/grid/"):
832 fail(
"Destination directory %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % dest)
834 export_to_run_nova_sam.append(
"DEST=%s"%dest)
836 is_production_arg = args.production
838 if "Production" in args.role
and not is_production_arg:
839 fail(
"You specified --role=Production but not --production. This is no longer supported")
843 elif args.calibration:
849 if args.test_submission:
850 print(
"Running a test submission, turning off hashDirs")
851 args.hashDirs =
False 853 if args.hashDirs
and args.runDirs:
854 fail(
"Cannot specify both --hashDirs and --runDirs (note that hashDirs is implied by --production)")
859 files_per_job = args.files_per_job
861 print(
"Definition name: %s" % defname)
863 print(
" with snapshot_id: %d" % snapshot_id)
872 if args.jobsub_server:
873 jobsub_opts += [
"--jobsub-server=%s"%args.jobsub_server]
875 if files_per_job > 0
and njobs > 0 :
878 jobsub_opts += [
"-N %d" %njobs]
879 art_sam_wrap_opts += [
"--limit %d" % files_per_job]
881 elif files_per_job > 0:
886 samweb = samweb_client.SAMWebClient(experiment=
'nova')
888 num_project_files=samweb.countFiles(defname=defname)
890 num_project_files=samweb.countFiles(dimensions=
"snapshot_id {0:d}".
format(snapshot_id))
891 print(
"Definition file count %d" % num_project_files)
893 njobs=(old_div(num_project_files, files_per_job)) +1
894 jobsub_opts += [
"-N %d" %njobs]
895 art_sam_wrap_opts += [
"--limit %d" % files_per_job]
900 jobsub_opts += [
"-N %d" %njobs]
903 warn(
"Neither --njobs nor --files_per_job specified. Did you really want to do this? Sleeping for 5 seconds")
909 Error: cannot submit more than 5000 jobs in one cluster. 910 Please break your submission into multiple batches of 5000 (or less) jobs, 911 and after submitting the first batch, use --continue_project with the project 912 that results from the first submission for the remaining batches. 914 Please separate submissions by 5 minutes. 915 """, file=sys.stderr)
918 if args.maxConcurrent:
919 jobsub_opts += [
"--maxConcurrent=%d" %args.maxConcurrent]
921 if args.opportunistic
or args.offsite:
922 usage_models.append(
"OPPORTUNISTIC")
924 usage_models.append(
"OFFSITE")
925 if args.offsite_only:
927 fail(
"Both --offsite and --offsite_only specified, these arguments conflict")
929 if args.opportunistic:
930 fail(
"Both --opportunistic and --offsite_only specified, these arguments conflict")
932 usage_models = [
"OFFSITE"]
935 usage_models=[
"AWS_HEPCLOUD"]
936 awsfilepath=os.path.expandvars(
"/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/$NOVAGRIDUTILS_VERSION/NULL/utils/aws_setup.sh")
937 source_scripts.append(awsfilepath)
940 run_nova_sam_opts += [
"--autoDropbox"]
945 args.singularity = NovaGridUtils.KNOWN_OS_CONTAINERS[args.os]
948 if not os.path.exists(args.singularity):
949 NovaGridUtils.fail(
"Requested singularity image cannot be found: %s" % args.singularity)
952 if (args.offsite
or args.offsite_only)
and not (args.singularity
or args.os):
955 resource_opt=
"--resource-provides=usage_model=" +
",".join( usage_models )
956 jobsub_opts += [resource_opt]
958 if use_recommended_sites
or args.site:
961 if use_recommended_sites:
962 for isite
in recommended_sites:
963 site_opt += isite +
"," 965 for isite
in args.site:
966 if isite
not in recommended_sites:
967 warn(
"Site "+isite+
" is not known to work. Your jobs may fail at that site. Sleeping for 5 seconds")
969 site_opt += isite +
"," 971 site_opt=site_opt[:-1]
972 jobsub_opts += [ site_opt ]
974 if args.exclude_site:
975 for isite
in args.exclude_site:
976 jobsub_opts += [
"--append_condor_requirements='(TARGET.GLIDEIN_Site\ isnt\ \\\"%s\\\")'" % isite ]
979 disk_opt=
"--disk=%sMB" % (args.disk)
980 jobsub_opts += [ disk_opt ]
983 mem_opt=
"--memory=%sMB" % (args.memory)
984 jobsub_opts += [ mem_opt ]
987 cpu_opt=
"--cpu=%d" % (args.cpu)
988 jobsub_opts += [ cpu_opt ]
991 jobsub_opts += [
"--mail_always"]
992 elif args.mail_on_error:
993 jobsub_opts += [
"--mail_on_error"]
995 jobsub_opts += [
"--mail_never"]
999 if args.user_priority != 0 :
1002 jobsub_opts += [
'-l "priority=%d"' % args.user_priority ]
1005 kill_opt=
"--self-destruct-timer %d" % (args.kill_after)
1006 art_sam_wrap_opts += [ kill_opt ]
1008 if args.dynamic_lifetime:
1010 args.expected_lifetime =
"0" 1011 if args.files_per_job > 1:
1012 warn(
"You have limited number of files per job to "+
str(args.files_per_job)+
" but this argument should not be necessary with dynamic_lifetime.")
1013 art_sam_wrap_opts += [
"--dynamic_lifetime " + args.dynamic_lifetime ]
1014 jobsub_opts += [
"--append_condor_requirements='(((TARGET.GLIDEIN_ToDie-CurrentTime)>%s)||isUndefined(TARGET.GLIDEIN_ToDie))'" % args.dynamic_lifetime]
1020 dummy=
int(args.expected_lifetime)
1021 jobsub_opts += [
"--expected-lifetime=%ss" % (args.expected_lifetime)]
1023 allowed_lifetimes=[
"short",
"medium",
"long"]
1024 if args.expected_lifetime
not in allowed_lifetimes:
1025 fail(
"Invalid expected_lifetime %s" % args.expected_lifetime)
1027 jobsub_opts += [
"--expected-lifetime=%s" % (args.expected_lifetime)]
1032 if args.testrel_ngu:
1033 ngu_current = subprocess.check_output(
"ups list -c -KVERSION NovaGridUtils", shell=
True).
decode(
"utf-8").replace(
"\"",
"").strip()
1034 print(
"getting stashcache from /cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/"+ngu_current+
"/NULL/bin/stashcache.sh")
1035 source_scripts.append(os.path.expandvars(
"/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/"+ngu_current+
"/NULL/bin/stashcache.sh"))
1037 source_scripts.append(os.path.expandvars(
"$NOVAGRIDUTILS_DIR/bin/stashcache.sh"))
1040 build_location_arguments =
"" 1041 source_scripts.append(
"/cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups.sh" )
1043 cvmfs_rev = subprocess.check_output([
"attr",
"-qg",
"revision",
"/cvmfs/nova.opensciencegrid.org"]).strip().
decode(
"utf-8")
1045 art_sam_wrap_opts.append(
"--cvmfs-revision %s" % cvmfs_rev)
1046 if not args.disable_cvmfs_version_matching:
1048 jobsub_opts += [
"--append_condor_requirements='ifThenElse(isUndefined(TARGET.HAS_CVMFS_nova_opensciencegrid_org)==FALSE,TARGET.CVMFS_nova_opensciencegrid_org_REVISION>=%s,TRUE)'" % cvmfs_rev ]
1049 if tag ==
"development" or tag[0] ==
'N':
1051 build_location_arguments +=
":-6:/cvmfs/nova-development.opensciencegrid.org/novasoft:-7:/cvmfs/nova-development.opensciencegrid.org/novasoft" 1053 build_location_arguments +=
":-6:%(dir)s/novasoft/slf6/novasoft:-7:%(dir)s/novasoft/slf7/novasoft" % {
"dir": cvmfs_distro_base}
1055 setup_location=
"%s/novasoft/slf6/novasoft/setup/setup_nova.sh" % (cvmfs_distro_base)
1060 allowed_groups=[
'nova',
'fermilab']
1062 if group
not in allowed_groups :
1063 fail(
"The only valid args for --group are " +
" ".join(allowed_group) )
1064 jobsub_opts += [
"-G %s" % group ]
1067 jobsub_opts += [
"--subgroup test"]
1068 elif args.subgroup :
1069 subgroup = args.subgroup
1070 if is_production_arg :
1071 allowed_subgroups = [
"keepup_prio",
"prod_high_prio",
"prod_prio" ]
1073 fail(
"Only production subgroups are available at this time and reqire production credentials" )
1076 if subgroup
in allowed_subgroups :
1077 jobsub_opts += [
"--subgroup %s" % subgroup ]
1079 fail(
"Allowed subgroups: " +
", ".join( allowed_subgroups ) )
1081 jobsub_opts += [
"--role=%s" %(role)]
1086 if args.singularity:
1087 jobsub_opts += [
"--line='+SingularityImage=\\\"%s\\\"'" % args.singularity ]
1088 jobsub_opts += [
"--append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'"]
1090 jobsub_opts += [
"--line='+JobFactoryType=\"%s\"'" % args.jobfactory ]
1094 jobsub_opts += [
"--line='+RequestGPUs=1'" ]
1096 if args.node_features:
1107 run_nova_sam_opts += [
"-n %d" % nevts ]
1109 if args.zipLogs: args.logs =
True 1111 if args.lemBalance
and args.lemServer:
1112 fail(
"Cannot specify both --lemServer and --lemBalance")
1116 passThru = [
'gdb',
'hashDirs',
'runDirs',
'noCleanup',
'jsonMetadata',
'copyOut',
'logs',
'zipLogs',
'outputNumuDeCAF',
'outputNueDeCAF',
'outputNumuOrNueDeCAF',
'outputNusDeCAF',
'outputValidationDeCAF',
'lemBalance']
1119 for opt
in passThru:
1121 run_nova_sam_opts += [
'--'+opt]
1126 if args.second_config:
1127 run_nova_sam_opts += [
'--second_config ' + args.second_config]
1130 run_nova_sam_opts += [
"--lemServer " + args.lemServer]
1132 if args.cosmicsPolarity:
1133 run_nova_sam_opts += [
"--cosmicsPolarity " + args.cosmicsPolarity]
1136 run_nova_sam_opts += [
"--npass " + args.npass]
1139 run_nova_sam_opts += [
"--skim " + args.skim]
1141 if args.systematic :
1142 run_nova_sam_opts += [
"--systematic " + args.systematic]
1144 if args.specialName :
1145 run_nova_sam_opts += [
"--specialName " + args.specialName]
1148 run_nova_sam_opts += [
"--genietune " + args.genietune]
1151 run_nova_sam_opts += [
"--NPPFX " + args.NPPFX]
1153 if args.declareFiles:
1154 run_nova_sam_opts += [
"--declareFiles"]
1156 if args.declareLocations:
1157 run_nova_sam_opts += [
"--declareLocations"]
1161 if args.histTier
and not (args.outTier
or args.cafTier):
1162 run_nova_sam_opts += [
"--useParentMetadata"]
1164 out_tiers=args.outTier
1167 outnum,mcouttier = out_tiers[0].
split(
":")
1168 if mcouttier ==
"artdaq" :
1169 copyback =
'"*.daq.root"' 1171 copyback =
'"*.'+ mcouttier +
'.root"' 1173 if None != out_tiers :
1174 for tier
in out_tiers :
1175 run_nova_sam_opts += [
"--outTier " + tier]
1177 caf_tiers=args.cafTier
1178 if None != caf_tiers :
1179 for tier
in caf_tiers :
1180 run_nova_sam_opts += [
"--cafTier " + tier]
1182 flat_tiers=args.flatTier
1183 if None != flat_tiers :
1184 for tier
in flat_tiers :
1185 run_nova_sam_opts += [
"--flatTier " + tier]
1187 hist_tier=args.histTier
1188 if None != hist_tier :
1189 run_nova_sam_opts += [
"--histTier " + hist_tier]
1191 h5_tiers=args.h5Tier
1192 if None != h5_tiers :
1193 for tier
in h5_tiers :
1194 run_nova_sam_opts += [
"--h5Tier " + tier]
1196 if args.copyOut
and (
None==hist_tier
and None==caf_tiers
and None==flat_tiers
and None==out_tiers
and None==h5_tiers):
1197 fail(
"You specified --copyOut but did not specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier")
1199 if not (args.copyOut
or args.copyOutScript) :
1200 fail(
"Did not specify a method to copy back output (--copyOut or --copyOutScript)")
1202 if args.copyOut
and args.copyOutScript :
1203 fail(
"The options --copyOut and --copyOutScript conflict")
1206 export_to_art_sam_wrap += args.export
1208 if args.veryearlyscript:
1209 veryearly_scripts += args.veryearlyscript
1212 source_scripts += args.source
1214 if args.earlyscript:
1215 early_scripts += args.earlyscript
1218 pre_scripts += args.prescript
1220 if args.precopyscript:
1221 inter_scripts += args.precopyscript
1224 post_scripts += args.postscript
1226 if args.copyOutScript:
1227 post_scripts.append(args.copyOutScript)
1229 for script
in veryearly_scripts + early_scripts + source_scripts + pre_scripts + post_scripts + inter_scripts:
1231 script_path = script.split(
":")[0]
1233 script_path = script
1236 if not find_file(os.environ[
"PATH"].
split(os.pathsep), os.path.expandvars(script_path)):
1237 fail(
"Script %s does not exist!" % script_path)
1239 if not args.continue_project:
1241 project_name = user +
"-" + jobname +
"-" + timestamp
1242 if args.test_submission:
1243 project_name +=
"-testjobs" 1244 start_project =
True 1246 project_name = args.continue_project
1247 start_project =
False 1251 if args.novasoftups:
1252 art_sam_wrap_cmd=
"$NOVASOFT_FQ_DIR/bin/art_sam_wrap.sh" 1263 start_proj_command =
"samweb start-project " 1265 start_proj_command+=
" --defname=%s" %defname
1267 start_proj_command+=
" --snapshot_id=%d" % snapshot_id
1268 start_proj_command+=
" --group=nova" 1269 start_proj_command+=
" --station=%s" % sam_station
1271 start_proj_command+=
" %s" %project_name
1272 if start_project
and not test:
1273 start_proj_retval=os.system(start_proj_command)
1274 print(
"start proj returned %d" % start_proj_retval)
1275 if start_proj_retval != 0:
1276 fail(
"Couldn't start project")
1278 print(
"Station monitor: http://samweb.fnal.gov:8480/station_monitor/nova/stations/" + sam_station +
"/projects/" + project_name)
1279 os.putenv(
"SAM_PROJECT_NAME",project_name)
1283 job_cmd=os.path.expandvars(
"${CONDOR_EXEC}/%s.sh" % project_name)
1285 while os.path.exists(job_cmd):
1286 job_cmd=os.path.expandvars(
"${CONDOR_EXEC}/%s_%i.sh" % (project_name, scriptcount) )
1298 os.symlink(os.path.expandvars(art_sam_wrap_cmd),job_cmd)
1299 art_sam_wrap_cmd=job_cmd
1306 if not args.no_multifile:
1307 art_sam_wrap_opts += [
"--multifile"]
1310 art_sam_wrap_opts += [
"--printenv"]
1313 run_nova_sam_opts += [
"--txtfiledef"]
1314 print(
"Passing --txtfiledef from submit_nova_art.py to runNovaSAM")
1317 art_sam_wrap_opts += [
"--config " + fcl]
1320 jobsub_opts += [
'-l "+JobType="MC""']
1321 art_sam_wrap_opts += [
"--getconfig"]
1324 art_sam_wrap_opts += [
"--mix",args.mix]
1326 if not args.novasoftups :
1327 art_sam_wrap_opts += [
"--source %s:-r:%s:%s%s" %(setup_location, tag, maxopt_opt, build_location_arguments)]
1328 if args.testrel
or args.user_tarball:
1329 art_sam_wrap_opts += [
"--setup_testrel"]
1332 art_sam_wrap_opts += [
"--source setup_test_product:NovaGridUtils"]
1334 if args.ngu_version:
1335 art_sam_wrap_opts += [
"--source setup_product:NovaGridUtils:%s" %(args.ngu_version)]
1337 if args.testrel_ngu:
1338 art_sam_wrap_opts += [
"--testrel_ngu"]
1340 for veryearly_script
in veryearly_scripts:
1341 art_sam_wrap_opts += [
"--veryearlyscript " + veryearly_script]
1343 for early_script
in early_scripts:
1344 art_sam_wrap_opts += [
"--earlyscript " + early_script]
1346 for source_script
in source_scripts:
1347 art_sam_wrap_opts += [
"--source " + source_script]
1349 for pre_script
in pre_scripts:
1350 art_sam_wrap_opts += [
"--prescript " + pre_script]
1352 for inter_script
in inter_scripts:
1353 run_nova_sam_opts += [
"--precopyscript " + inter_script]
1355 for post_script
in post_scripts:
1356 art_sam_wrap_opts += [
"--postscript " + post_script]
1358 for input_file
in input_files:
1359 art_sam_wrap_opts += [
"--inputfile " + input_file]
1362 art_sam_wrap_opts += [
"--addoutput " + copyback]
1363 if args.hashDirs==
True:
1364 art_sam_wrap_opts += [
"--hash"]
1365 art_sam_wrap_opts += [
"--dest " + dest]
1367 if args.poms
and not test:
1368 poms_campaign_id = poms_client.register_poms_campaign(
1371 experiment =
'nova',
1376 poms_task_id = poms_client.get_task_id_for(
1381 export_to_art_sam_wrap += [
"POMS_CAMPAIGN_ID={0}".
format(poms_campaign_id),
1382 "POMS_TASK_ID={0}".
format(poms_task_id)]
1383 print(
"POMS Campaign: https://pomsgpvm01.fnal.gov/poms/campaign_info?campaign_id={0}".
format(poms_campaign_id))
1400 os.system(jobsub_cmd)
1404 print(
"Please note: if you intend to submit any more jobs,")
1405 print(
" please wait", old_div(njobs,1000),
"minutes before your next submission")
1406 print(
" so as to avoid overloading the jobsub server.")
void split(double tt, double *fr)
def setup_production(args)
def check_dir(output_dir, prefix='')
def find_file(paths, filename)
def make_jobsub_node_features_arg(features)
def find_file_in_list(filepath, pathlist)
std::string format(const int32_t &value, const int &ndigits=8)
procfile open("FD_BRL_v0.txt")
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
void decode(std::any const &, InputTag &)
def add_node_features_arg(parser)
def setup_calibration(args)
def check_make_dir(dname)