3 import os, sys, stat, pwd, re
12 from NovaGridUtils
import *
14 user=os.getenv(
"USER")
15 os.environ[
"GRID_USER"]=user
18 sam_station=os.getenv(
"SAM_STATION")
19 os.system(
"echo SAM_STATION DEFINED AS $SAM_STATION")
21 recommended_sites=[
"BNL",
46 cvmfs_distro_base =
"/cvmfs/nova.opensciencegrid.org" 47 novasoft_cvmfs =
"%s/novasoft/slf6/novasoft" % cvmfs_distro_base
48 build_location_arguments =
"" 53 run_nova_sam_cmd=
"runNovaSAM.py" 55 export_to_run_nova_sam = []
57 art_sam_wrap_cmd=
"$NOVAGRIDUTILS_DIR/bin/art_sam_wrap.sh" 59 export_to_art_sam_wrap=[]
60 export_to_art_sam_wrap.append(
"SAM_PROJECT_NAME")
61 export_to_art_sam_wrap.append(
"SAM_STATION")
62 export_to_art_sam_wrap.append(
"IFDH_BASE_URI")
63 if "IFDH_DEBUG" in os.environ:
64 export_to_art_sam_wrap.append(
"IFDH_DEBUG")
65 if "G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION" in os.environ:
66 export_to_art_sam_wrap.append(
"G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION")
68 export_to_art_sam_wrap.append(
"EXPERIMENT")
69 export_to_art_sam_wrap.append(
"GRID_USER")
71 os.environ[
"CVMFS_DISTRO_BASE"]=cvmfs_distro_base
72 export_to_art_sam_wrap.append(
"CVMFS_DISTRO_BASE")
75 usage_models=[
"DEDICATED"]
87 value=os.getenv(vname)
88 if None == value
or "" == value:
89 fail(
"Environment variable %s not defined" %vname)
92 if "/" in fcl[:1]
and os.path.isfile(fcl):
95 fclPaths = os.environ[
"FHICL_FILE_PATH"].
split(
":")
97 fullPath = os.path.join(path, fcl)
98 if os.path.isfile(fullPath):
101 for input_file
in input_files:
102 if os.path.basename(input_file) == fcl:
104 elif os.path.basename(input_file) == os.path.basename(fcl):
106 print "The fhicl will be copied to $PWD on the worker node." 107 print "You specified some sort of file path which isn't needed. Fixing it for you :)" 108 print fcl +
" --> " + os.path.basename(fcl)
110 return os.path.basename(fcl)
112 fail(
"fcl file %s does not exist" %fcl)
117 This reads tokens using tokenize.generate_tokens and recombines them 118 using tokenize.untokenize, and skipping comment/docstring tokens in between 120 f = cStringIO.StringIO(src)
122 processed_tokens = []
125 for tok
in tokenize.generate_tokens(f.readline):
126 t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok
129 if t_type == tokenize.COMMENT:
130 raise SkipException()
132 elif t_type == tokenize.STRING:
134 if last_token
is None or last_token[0]
in [tokenize.INDENT]:
137 except SkipException:
140 processed_tokens.append(tok)
144 return tokenize.untokenize(processed_tokens)
147 if os.path.isfile(filename):
150 for root, dirs, files
in os.walk(os.path.expandvars(path)):
151 if filename
in files:
152 return os.path.join(root, filename)
153 fail(
"Cannot find file "+filename)
156 for testpath
in pathlist:
157 if os.path.basename(filepath) == os.path.basename(testpath):
164 jobsub_cmd =
"jobsub_submit \\\n" 165 for opt
in jobsub_opts:
166 jobsub_cmd +=
" " + opt +
" \\\n" 167 for export
in export_to_art_sam_wrap:
168 jobsub_cmd +=
" -e " + export
169 jobsub_cmd +=
" \\\n" 173 if args.reuse_tarball:
174 jobsub_cmd +=
" --tar_file_name dropbox://" + os.path.basename(args.testrel) +
".tar \\\n" 176 jobsub_cmd +=
" --tar_file_name tardir://" + args.testrel +
" \\\n" 177 elif args.user_tarball:
178 if not os.path.isfile(args.user_tarball):
179 print "Tarball filename passed to --user_tarball does not exit:", args.user_tarball
181 jobsub_cmd +=
" --tar_file_name dropbox://" + args.user_tarball +
" \\\n" 183 jobsub_cmd +=
" file://" + art_sam_wrap_cmd +
" \\\n" 184 for opt
in art_sam_wrap_opts:
185 jobsub_cmd +=
" " + opt +
" \\\n" 186 for export
in export_to_run_nova_sam :
187 jobsub_cmd +=
" --export " + export +
" \\\n" 193 jobsub_cmd +=
" -X " + run_nova_sam_cmd +
" \\\n" 194 for opt
in run_nova_sam_opts:
195 jobsub_cmd +=
" " + opt +
" \\\n" 197 jobsub_cmd +=
" -X nova \\\n" 198 mcgen_opts = [
"--sam-file-type=importedSimulated",
"--sam-application-family=nova",
"--sam-data-tier=" + mcouttier,
"--sam-application-version=" + tag]
200 mcgen_opts += [
"-n %d" % nevts]
201 for opt
in mcgen_opts:
202 jobsub_cmd +=
" " + opt +
" \\\n" 204 jobsub_cmd = jobsub_cmd[:-2]
210 if __name__==
'__main__':
212 prog=os.path.basename(sys.argv[0])
213 NovaGridUtils.prog=prog
216 while "-f" in sys.argv
or "--file" in sys.argv:
222 preliminary_parser = argparse.ArgumentParser(prog=prog, description=
'Submit nova art job')
224 preliminary_parser.add_argument(
'-f',
'--file',
225 help=
"""Text file containing any arguments to this utility. Multiple allowed. 226 Arguments should look just like they would on the command line, 227 but the parsing of this file is whitespace insenstive. 228 Commented lines will be identified with the # character and removed. """, type=str, action=
'append')
229 pre_args, unknown = preliminary_parser.parse_known_args()
232 sys.argv =
filter(
lambda x: x
not in [
"-f",
"--file"], sys.argv)
235 for filepath
in pre_args.file:
236 index = sys.argv.index(filepath)
237 sys.argv.remove(filepath)
238 if os.path.isfile(filepath):
241 fullpath =
find_file([
"$NOVAGRIDUTILS_DIR/configs/"],filepath)
242 text =
open(fullpath,
'r').read() 245 for line
in text.splitlines():
248 newargs += line.split()
249 sys.argv[index:index] = newargs
251 parser = argparse.ArgumentParser(prog=prog, description=
'Submit nova art job', add_help=
False)
254 required_args = parser.add_argument_group(
"Required arguments",
"These arguments must be supplied.")
256 required_args.add_argument(
'--jobname',
261 required_args.add_argument(
'--defname',
263 help=
'SAM dataset definition to run over',
266 required_args.add_argument(
'--config',
'-c',
268 help=
'FHiCL file to use as configuration for nova executable. The path given should be relative to the $SRT_PRIVATE_CONTEXT of any test release you submit using',
271 required_args.add_argument(
"--tag",
273 help=
"Tag of novasoft to use",
276 required_args.add_argument(
"--dest",
278 help=
"Destination for output files",
282 debugging_args = parser.add_argument_group(
"Debugging options",
"These optional arguments can help debug your submission.")
284 debugging_args.add_argument(
'--print_jobsub',
285 help=
'Print jobsub command',
286 action=
'store_true',default=
False)
288 debugging_args.add_argument(
'--printenv',
289 help=
'Print environment variables',
290 action=
'store_true',default=
False)
292 debugging_args.add_argument(
'--test',
293 help=
'Do not actually do anything, just run tests and print jobsub cmd',
294 action=
'store_true',default=
False)
296 debugging_args.add_argument(
'--gdb',
297 help=
'Run nova executable under gdb, print full stack trace, then quit gdb.',
298 action=
'store_true',default=
False)
300 debugging_args.add_argument(
'--test_submission',
301 help=
'Override other arguments given to submit a test to the grid. It will run 1 job with 3 events and write the output to /pnfs/nova/scratch/users/<user>/test_jobs/<date>_<time>',
302 action=
'store_true',default=
False)
304 debugging_args.add_argument(
'--jobsub_server',
305 help=
'Submit using the specified jobsub server',
308 debugging_args.add_argument(
'--test_queue',
309 help=
'Submit jobs to the test jobsub queue for higher starting priority. NB NOvA is limited to 10 jobs at a time on this queue.',
310 action=
'store_true', default=
False)
312 debugging_args.add_argument(
"--kill_after",
314 help=
"If job is still running after this many seconds, kill in such a way that a log will be returned",
318 job_control_args = parser.add_argument_group(
"Job control options",
"These optional arguments help control where and how your jobs land.")
320 job_control_args.add_argument(
'--njobs',
321 help=
'Number of jobs to submit',
322 type=int, default = 0)
324 job_control_args.add_argument(
'--maxConcurrent',
325 help=
'Run a maximum of N jobs simultaneously',
329 job_control_args.add_argument(
'--files_per_job',
330 help=
'Number of files per job - if zero, calculate from number of jobs', metavar=
'N',
331 type=int, default = 0)
333 job_control_args.add_argument(
'--nevts',
334 help=
'Number of events per file to process',
335 type=int, default = 0)
338 job_control_args.add_argument(
'--no_multifile',
339 help=
'Do not use art_sam_wrap.sh multifile mode, which is on by default',
342 job_control_args.add_argument(
'--txtfiledef',
343 help=
'Use if the input definition is made up of text files, each containing a list of file names',
344 action=
'store_true',default=
False)
347 job_control_args.add_argument(
'--opportunistic',
348 help=
'Run opportunistically on the fermigrid',
349 action=
'store_true',default=
False)
351 job_control_args.add_argument(
'--offsite',
352 help=
'Allow to run on offsite resources as well. Implies --opportunistic.',
353 action=
'store_true',default=
False)
355 job_control_args.add_argument(
'--offsite_only',
356 help=
'Allow to run solely on offsite resources.',
357 action=
'store_true',default=
False)
359 job_control_args.add_argument(
'--amazon',
360 help=
'Run at amazon.',
361 action=
'store_true',default=
False)
363 job_control_args.add_argument(
'--site',
364 help=
'Specify allowed offsite locations. Omit to allow running at any offsite location',
365 type=str,action=
'append')
367 job_control_args.add_argument(
'--exclude_site',
368 help=
'Specify an offsite location to exclude.',
370 type=str,action=
'append')
372 job_control_args.add_argument(
'--recommended_sites',
373 help=
'Specify known working offsite locations.',
374 action=
'store_true',default=
False)
376 job_control_args.add_argument(
'--autoDropbox',
377 help=
'Use automatic dropbox location based on site',
378 action=
'store_true',default=
False)
380 job_control_args.add_argument(
'--os',
381 help=
'Specify OS version of worker node',
382 type=str,action=
'append')
384 job_control_args.add_argument(
'--disk',
385 help=
'Local disk space requirement for worker node in MB.',
386 type=int, default=10000)
388 job_control_args.add_argument(
'--memory',
389 help=
'Local memory requirement for worker node in MB.',
390 type=int, default=1900)
392 job_control_args.add_argument(
'--expected_lifetime',
393 help=
'Expected job lifetime (default is 10800s=3h). Valid values are an integer number of seconds or one of \"short\" (6h), \"medium\" (12h) or \"long\" (24h, jobsub default)', metavar=
'LIFETIME',
394 type=str, default=
"10800")
396 job_control_args.add_argument(
'--dynamic_lifetime',
397 help=
"Dynamically determine whether a new file should be started based on glidein lifetime. Specify the maximum length expected for a single file to take to process in seconds.", metavar=
"LIFETIME",
400 job_control_args.add_argument(
'--cpu',
401 help=
"Request worker nodes that have at least NUMBER cpus",
405 job_control_args.add_argument(
'--group',
'-G',
406 help=
"Specify batch group GROUP -- mainly used to set job priority. At present, only supportable value is nova",
407 type=str, default=
"nova")
409 job_control_args.add_argument(
'--subgroup',
410 help=
'Production subgroup',
413 job_control_args.add_argument(
"--role",
414 help=
"Specify role to run on the grid. Can be Analysis (default) or Production. This option is no longer supported",
415 type=str,default=
"Analysis")
417 job_control_args.add_argument(
'--continue_project',
418 help=
"Don't start a new samweb project, instead continue this one.", metavar=
'PROJECT',
421 job_control_args.add_argument(
"--snapshot_id",
422 help=
"Use this existing snapshot instead of creating a new one.", metavar=
"ID",
425 job_control_args.add_argument(
"--poms",
426 help=
"Start/continue a poms campaign and task for this submission",
427 default=
False, action=
"store_true")
429 job_control_args.add_argument(
"--mix",
430 help=
"Pass a mixing script to the job to pull in a files for job mixing.",
431 default=
"", type=str)
433 job_control_args.add_argument(
"--mail_always",
434 help=
"Do you want an email whenever every jobs finishes?",
435 default=
False, action=
"store_true")
437 job_control_args.add_argument(
"--mail_on_error",
438 help=
"Do you want an email whenever a job fails on an error?",
439 default=
False, action=
"store_true")
445 job_control_args.add_argument(
'--user_priority',
446 help=
'Priority (integer) within a user\'s jobs (default = 0)',
449 job_control_args.add_argument(
'--singularity',
450 help=
'Location in CVMFS of a singularity container to launch the job into',
453 job_control_args.add_argument(
'--jobfactory',
454 help=
'Use the specified JobFactoryType.',
457 job_control_args.add_argument(
"--gpu",
458 help=
"Request a node with a GPU",
459 default=
False, action=
"store_true")
464 novasoft_args = parser.add_argument_group(
"NOvA software options",
"These options control the novasoft setup.")
465 novasoft_args.add_argument(
'--maxopt',
466 help=
'Run in maxopt mode',
467 action=
'store_true',default=
True)
469 testrel_gp = novasoft_args.add_mutually_exclusive_group(required=
False)
470 testrel_gp.add_argument(
"--testrel",
471 help=
"Use a test release at location TESTREL. It will be tarred up, and sent to the worker node.",
473 testrel_gp.add_argument(
"--user_tarball",
474 help=
"Use existing test release tarball in specified location rather than having jobsub make one for you (conflicts with --testrel)",
477 novasoft_args.add_argument(
'--reuse_tarball',
478 help=
'Do you want to reuse a tarball that is already in resilient space? If using this option avoid trailing slash in --testrel option. (conflicts with --user_tarball)',
479 action=
'store_true',default=
False)
481 novasoft_args.add_argument(
'--cvmfs',
482 help=
'Does nothing (always true), but retained for compatibility: pull software from CVMFS.',
486 novasoft_args.add_argument(
'--disable_cvmfs_version_matching',
487 help=
"Don't perform a CVMFS-is-up-to-date check on target nodes via Condor requirements. (For advanced debugging use.) ",
491 novasoft_args.add_argument(
'--novasoftups',
492 help=
'Use the ups build of novasoft, must be used with source to setup.',
495 novasoft_args.add_argument(
'--ngu_test',
496 help=
'Setup the test version of NovaGridUtils in the grid jobs.',
499 novasoft_args.add_argument(
'--ngu_version',
500 help=
'Setup a specific NovaGridUtils version in the grid jobs.', metavar=
'VERSION',
503 novasoft_args.add_argument(
'--testrel_ngu',
504 help=
"Must be used with --testrel, with NGU checked out. After unpacking tarball will setup the local version of NGU you are using on the work.",
507 novasoft_args.add_argument(
'--lemBalance',
508 help=
'Choose lem server based on (CLUSTER+PROCESS)%%2 to balance load',
509 action=
'store_true', default=
False)
511 novasoft_args.add_argument(
'--lemServer',
512 help=
'Specify lem server',
516 output_args = parser.add_argument_group(
"Output file options",
"Note that you must specify either --copyOut or --copyOutScript.")
519 output_args.add_argument(
'--copyOutScript',
520 help=
'Use script COPYOUTSCRIPT to copy back your output',
523 output_args.add_argument(
'--copyOut',
524 help=
'Use the built in copy out mechanism. If used, you must specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier',
527 output_args.add_argument(
'--logs',
528 help=
'Return .log files corresponding to every output',
530 output_args.add_argument(
'--zipLogs',
531 help=
'Format logs as .bz2 files. Implies --logs',
534 output_args.add_argument(
'--outTier',
535 help=
'Data tier of the output file, multiple allowed, formatted as <name_in_fcl_outputs>:<data_tier>',
536 type=str, action=
'append')
538 output_args.add_argument(
'--cafTier',
539 help=
'Module label for CAF output, multiple allowed. Format as <cafmaker_module_label>:<data_tier>',
540 type=str, action=
'append')
542 output_args.add_argument(
'--flatTier',
543 help=
'Module label for FlatCAF output, multiple allowed. Format as <flatmaker_module_label>:<data_tier>',
544 type=str, action=
'append')
546 output_args.add_argument(
'--histTier',
547 help=
'File identifier string for TFileService output, only one allowed. Supply as --histTier <id> for output_name.<id>.root, where output_name is assembled based on the input file.',
550 output_args.add_argument(
'--h5Tier',
551 help=
'File identifier for h5 output, multiple allowed. Format as <hdf5maker_module>:<data_tier>',
552 type=str, action=
'append')
553 output_args.add_argument(
'--second_config',
554 help=
"""Second configuration fcl executed after main process. 555 nova is executed with art file that is output from the main process. 556 Files that get produced by this process that are named identically 557 to files produced by the main process and are among the requested outputs 558 are ignored and the file produced by the first process is returned""",
561 output_args.add_argument(
'--outputNumuDeCAF',
562 help=
'Make standard numu decafs for all CAF files produced during the job',
563 action=
'store_true',default=
False)
565 output_args.add_argument(
'--outputNueDeCAF',
566 help=
'Make standard nue decafs for all CAF files produced during the job',
567 action=
'store_true',default=
False)
569 output_args.add_argument(
'--outputNumuOrNueDeCAF',
570 help=
'Make standard nue or numu decafs for all CAF files produced during the job',
571 action=
'store_true',default=
False)
573 output_args.add_argument(
'--outputNusDeCAF',
574 help=
'Make standard nus decafs for all CAF files produced during the job',
575 action=
'store_true',default=
False)
577 output_args.add_argument(
'--outputValidationDeCAF',
578 help=
'Make validation (nue_or_numu_or_nus) decafs for all CAF files produced during the job',
579 action=
'store_true',default=
False)
582 output_args.add_argument(
'--cosmicsPolarity',
583 help=
'To specify a horn polarity for the cosmics output file name',
586 output_args.add_argument(
'--npass',
587 help=
'To specify npass (aka nova.subversion)',
590 output_args.add_argument(
'--skim',
591 help=
'To specify nova.skim (does not work with mc gen)',
594 output_args.add_argument(
'--systematic',
595 help=
'To specify nova.systematic (does not work with mc gen)', metavar=
'SYST',
598 output_args.add_argument(
'--specialName',
599 help=
'To specify nova.special name (does not work with mc gen)',
602 output_args.add_argument(
'--genietune',
603 help=
'To specify nova.genietune (does not work with mc gen)', metavar=
'TUNE',
606 output_args.add_argument(
'--NPPFX',
607 help=
'To specify number of PPFX universes',
610 output_args.add_argument(
'--hashDirs',
611 help=
'Use hash directory structure in destination directory.',
614 output_args.add_argument(
'--runDirs',
615 help=
'Use run directory structure in destination directory, 000XYZ/XYZUW for run number XYZUW.',
618 output_args.add_argument(
'--noCleanup',
619 help=
'Pass --noCleanup argument to runNovaSAM.py. Necessary when using a postscript for copyout.',
622 output_args.add_argument(
'--jsonMetadata', help=
'Create JSON files with metadata corresponding to each output file, and copy them to the same destinations', action=
'store_true')
624 output_args.add_argument(
'--declareFiles',
625 help=
'Declare files with metadata on worker node',
628 output_args.add_argument(
'--production',
629 help=
'Submit production style jobs. Implies \"--role=Production --hashDirs --jsonMetadata --zipLogs\", and checks that other settings needed for production are specified',
632 output_args.add_argument(
'--calibration',
633 help=
'Submit calibration style jobs. Implies \"--role=Production\", and checks that other settings needed for calibration are specified',
636 output_args.add_argument(
'--declareLocations',
637 help=
'Declare the file output locations to SAM during the copy back of the files',
641 environment_args = parser.add_argument_group(
"Environment options",
"These optional arguments allow control of the grid running environment.")
643 environment_args.add_argument(
'--export',
644 help=
'Export variable EXPORT to art_sam_wrap.sh',
645 type=str, action=
'append')
647 environment_args.add_argument(
'--veryearlyscript',
648 help=
'Source script EARLYSCRIPT before any environment setup or sourcing',
649 type=str, action=
'append')
651 environment_args.add_argument(
'--source',
652 help=
'Source script SOURCE',
653 type=str, action=
'append')
655 environment_args.add_argument(
'--earlyscript',
656 help=
'Execute script EARLYSCRIPT before any environment setup',
657 type=str, action=
'append')
659 environment_args.add_argument(
'--prescript',
660 help=
'Execute script PRESCRIPT before executing runNovaSAM.py',
661 type=str, action=
'append')
663 environment_args.add_argument(
'--precopyscript',
664 help=
'Execute script PRECOPYSCRIPT within runNovaSAM.py, after running the nova -c command.',
665 type=str, action=
'append')
667 environment_args.add_argument(
'--postscript',
668 help=
'Execute script POSTSCRIPT after executing runNovaSAM.py',
669 type=str, action=
'append')
671 environment_args.add_argument(
'--inputfile',
672 help=
'Copy this extra input file into job area before running executable',
673 type=str, action=
'append')
677 support_args = parser.add_argument_group(
"Support options",
"These optional arguments using this submission utility easier.")
679 support_args.add_argument(
"-h",
"--help", action=
"help", help=
"Show this help message and exit")
681 support_args.add_argument(
'-f',
'--file',
682 help=
"""Text file containing any arguments to this utility. Multiple allowed. 683 Arguments should look just like they would on the command line, 684 but the parsing of this file is whitespace insenstive. 685 Comments will be identified with the # character and removed. """, type=str, action=
'append')
694 args = parser.parse_args()
695 timestamp=datetime.datetime.now().strftime(
"%Y%m%d_%H%M")
702 print "POMS not setup. Run this and try again:" 704 print " setup poms_client" 709 if args.test_submission:
712 test_dest =
"/pnfs/nova/scratch/users/%s/test_jobs/%s" % (os.environ[
"USER"], timestamp)
713 if not os.path.exists(test_dest):
714 os.makedirs(test_dest)
715 mode = os.stat(test_dest).st_mode | stat.S_IXGRP | stat.S_IWGRP
716 os.chmod(test_dest, mode)
717 test_expected_lifetime =
"0" 718 test_dynamic_lifetime =
"500" 719 test_files_per_job = 1
721 print "Running a test submission. Overwriting:" 723 print " njobs", args.njobs,
"-->", test_njobs
724 args.njobs = test_njobs
725 print " nevts", args.nevts,
"-->", test_nevts
726 args.nevts = test_nevts
727 print " dest", args.dest,
"-->", test_dest
728 args.dest = test_dest
729 print " expected_lifetime", args.expected_lifetime,
"-->", test_expected_lifetime
730 args.expected_lifetime = test_expected_lifetime
731 print " dynamic_lifetime", args.dynamic_lifetime,
"-->", test_dynamic_lifetime
732 args.dynamic_lifetime = test_dynamic_lifetime
733 print " files_per_job", args.files_per_job,
"-->", test_files_per_job
734 args.files_per_job = test_files_per_job
735 if args.declareFiles:
736 print " don't declareFiles" 737 args.declareFiles =
False 738 if args.declareLocations:
739 print " don't declareLocations" 740 args.declareLocations =
False 742 print " don't use autoDropbox" 743 args.autoDropbox =
False 745 print " don't use poms" 754 snapshot_id=args.snapshot_id
755 print_jobsub=args.print_jobsub
758 print "Will print environment vars " 765 warn(
"--test was specified, so all we do is run checks and print jobsub cmd.")
777 maxopt_opt +=
"-b:maxopt" 780 if args.reuse_tarball
and not args.testrel:
781 fail(
"--reuse_tarball specified without --testrel??")
785 if not os.path.isdir(args.testrel+
'/lib/'+os.getenv(
'SRT_ARCH')+
'-GCC-'+srt_qual):
786 fail(args.testrel+
' has never been built '+srt_qual)
789 input_files += args.inputfile
791 for input_file
in input_files:
792 if not os.path.isfile(os.path.expandvars(input_file)):
793 fail(
"Input file %s does not exist!" % input_file)
794 if os.path.expandvars(input_file).startswith(
"/nova/"):
795 fail(
"Input file %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % input_file)
796 elif os.path.expandvars(input_file).startswith(
"/grid/"):
797 fail(
"Input file %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % input_file)
799 if args.singularity
and not os.path.exists(args.singularity):
800 fail(
"Requested singularity image cannot be found: %s" % args.singularity)
802 if args.gpu
and not args.singularity:
803 warn(
"Requested GPU, but did not request singularity. This is not likely to succeed.")
805 if args.gpu
and not args.offsite_only:
806 warn(
"GPUs are only available offsite, and you have not chosen --offsite_only")
812 mcgen = (fcl ==
"mcgen")
818 if not dest.startswith(
"s3://") :
821 if os.path.expandvars(dest).startswith(
"/nova/"):
822 fail(
"Destination directory %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % dest)
823 elif os.path.expandvars(dest).startswith(
"/grid/"):
824 fail(
"Destination directory %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % dest)
826 export_to_run_nova_sam.append(
"DEST=%s"%dest)
828 is_production_arg = args.production
830 if "Production" in args.role
and not is_production_arg:
831 fail(
"You specified --role=Production but not --production. This is no longer supported")
835 elif args.calibration:
841 if args.test_submission:
842 print "Running a test submission, turning off hashDirs" 843 args.hashDirs =
False 845 if args.hashDirs
and args.runDirs:
846 fail(
"Cannot specify both --hashDirs and --runDirs (note that hashDirs is implied by --production)")
851 files_per_job = args.files_per_job
853 print "Definition name: %s" % defname
855 print " with snapshot_id: %d" % snapshot_id
864 if args.jobsub_server:
865 jobsub_opts += [
"--jobsub-server=%s"%args.jobsub_server]
867 if files_per_job > 0
and njobs > 0 :
870 jobsub_opts += [
"-N %d" %njobs]
871 art_sam_wrap_opts += [
"--limit %d" % files_per_job]
873 elif files_per_job > 0:
878 samweb = samweb_client.SAMWebClient(experiment=
'nova')
880 num_project_files=samweb.countFiles(defname=defname)
882 num_project_files=samweb.countFiles(dimensions=
"snapshot_id {0:d}".
format(snapshot_id))
883 print "Definition file count %d" % num_project_files
885 njobs=(num_project_files / files_per_job) +1
886 jobsub_opts += [
"-N %d" %njobs]
887 art_sam_wrap_opts += [
"--limit %d" % files_per_job]
892 jobsub_opts += [
"-N %d" %njobs]
895 warn(
"Neither --njobs nor --files_per_job specified. Did you really want to do this? Sleeping for 5 seconds")
900 print >> sys.stderr,
""" 901 Error: cannot submit more than 5000 jobs in one cluster. 902 Please break your submission into multiple batches of 5000 (or less) jobs, 903 and after submitting the first batch, use --continue_project with the project 904 that results from the first submission for the remaining batches. 906 Please separate submissions by 5 minutes. 910 if args.maxConcurrent:
911 jobsub_opts += [
"--maxConcurrent=%d" %args.maxConcurrent]
913 if args.opportunistic
or args.offsite:
914 usage_models.append(
"OPPORTUNISTIC")
916 usage_models.append(
"OFFSITE")
917 if args.offsite_only:
919 fail(
"Both --offsite and --offsite_only specified, these arguments conflict")
921 if args.opportunistic:
922 fail(
"Both --opportunistic and --offsite_only specified, these arguments conflict")
924 usage_models = [
"OFFSITE"]
927 usage_models=[
"AWS_HEPCLOUD"]
928 awsfilepath=os.path.expandvars(
"/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/$NOVAGRIDUTILS_VERSION/NULL/utils/aws_setup.sh")
929 source_scripts.append(awsfilepath)
932 run_nova_sam_opts += [
"--autoDropbox"]
935 if args.offsite
or args.offsite_only:
938 fail(
"Don't specify OS when submitting with --singularity")
941 fail(
"Running offsite, but OS version not specified!")
943 resource_opt=
"--resource-provides=usage_model=" + string.join(usage_models,
",")
944 jobsub_opts += [resource_opt]
946 if args.recommended_sites
or args.site:
949 if args.recommended_sites:
950 for isite
in recommended_sites:
951 site_opt += isite +
"," 953 for isite
in args.site:
954 if isite
not in recommended_sites:
955 warn(
"Site "+isite+
" is not known to work. Your jobs may fail at that site. Sleeping for 5 seconds")
957 site_opt += isite +
"," 959 site_opt=site_opt[:-1]
960 jobsub_opts += [ site_opt ]
962 if args.exclude_site:
963 for isite
in args.exclude_site:
964 jobsub_opts += [
"--append_condor_requirements='(TARGET.GLIDEIN_Site\ isnt\ \\\"%s\\\")'" % isite ]
969 if ios
not in allowed_os:
970 fail(
"Invalid OS %s" %ios)
972 os_opt=
"--OS=" + string.join(args.os,
",")
973 jobsub_opts += [ os_opt ]
976 disk_opt=
"--disk=%sMB" % (args.disk)
977 jobsub_opts += [ disk_opt ]
980 mem_opt=
"--memory=%sMB" % (args.memory)
981 jobsub_opts += [ mem_opt ]
984 cpu_opt=
"--cpu=%d" % (args.cpu)
985 jobsub_opts += [ cpu_opt ]
988 jobsub_opts += [
"--mail_always"]
989 elif args.mail_on_error:
990 jobsub_opts += [
"--mail_on_error"]
992 jobsub_opts += [
"--mail_never"]
996 if args.user_priority != 0 :
999 jobsub_opts += [
'-l "priority=%d"' % args.user_priority ]
1002 kill_opt=
"--self-destruct-timer %d" % (args.kill_after)
1003 art_sam_wrap_opts += [ kill_opt ]
1005 if args.dynamic_lifetime:
1007 args.expected_lifetime =
"0" 1008 if args.files_per_job > 1:
1009 warn(
"You have limited number of files per job to "+
str(args.files_per_job)+
" but this argument should not be necessary with dynamic_lifetime.")
1010 art_sam_wrap_opts += [
"--dynamic_lifetime " + args.dynamic_lifetime ]
1011 jobsub_opts += [
"--append_condor_requirements='(((TARGET.GLIDEIN_ToDie-CurrentTime)>%s)||isUndefined(TARGET.GLIDEIN_ToDie))'" % args.dynamic_lifetime]
1017 dummy=string.atoi(args.expected_lifetime)
1018 jobsub_opts += [
"--expected-lifetime=%ss" % (args.expected_lifetime)]
1020 allowed_lifetimes=[
"short",
"medium",
"long"]
1021 if args.expected_lifetime
not in allowed_lifetimes:
1022 fail(
"Invalid expected_lifetime %s" % args.expected_lifetime)
1024 jobsub_opts += [
"--expected-lifetime=%s" % (args.expected_lifetime)]
1027 build_location_arguments =
"" 1028 source_scripts.append(
"/cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups.sh" )
1030 cvmfs_rev = subprocess.check_output([
"attr",
"-qg",
"revision",
"/cvmfs/nova.opensciencegrid.org"]).strip()
1032 art_sam_wrap_opts.append(
"--cvmfs-revision %s" % cvmfs_rev)
1033 if not args.disable_cvmfs_version_matching:
1035 jobsub_opts += [
"--append_condor_requirements='ifThenElse(isUndefined(TARGET.HAS_CVMFS_nova_opensciencegrid_org)==FALSE,TARGET.CVMFS_nova_opensciencegrid_org_REVISION>=%s,TRUE)'" % cvmfs_rev ]
1036 if tag ==
"development" or tag[0] ==
'N':
1038 build_location_arguments +=
":-6:/cvmfs/nova-development.opensciencegrid.org/novasoft:-7:/cvmfs/nova-development.opensciencegrid.org/novasoft" 1040 build_location_arguments +=
":-6:%(dir)s/novasoft/slf6/novasoft:-7:%(dir)s/novasoft/slf7/novasoft" % {
"dir": cvmfs_distro_base}
1042 setup_location=
"%s/novasoft/slf6/novasoft/setup/setup_nova.sh" % (cvmfs_distro_base)
1047 allowed_groups=[
'nova',
'fermilab']
1049 if group
not in allowed_groups :
1050 fail(
"The only valid args for --group are " +
" ".join(allowed_group) )
1051 jobsub_opts += [
"-G %s" % group ]
1054 jobsub_opts += [
"--subgroup test"]
1055 elif args.subgroup :
1056 subgroup = args.subgroup
1057 if is_production_arg :
1058 allowed_subgroups = [
"keepup_prio",
"prod_high_prio",
"prod_prio" ]
1060 fail(
"Only production subgroups are available at this time and reqire production credentials" )
1063 if subgroup
in allowed_subgroups :
1064 jobsub_opts += [
"--subgroup %s" % subgroup ]
1066 fail(
"Allowed subgroups: " +
", ".join( allowed_subgroups ) )
1068 jobsub_opts += [
"--role=%s" %(role)]
1073 if args.singularity:
1074 jobsub_opts += [
"--line='+SingularityImage=\\\"%s\\\"'" % args.singularity ]
1075 jobsub_opts += [
"--append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'"]
1077 jobsub_opts += [
"--line='+JobFactoryType=\\\"%s\\\"'" % args.jobfactory ]
1081 jobsub_opts += [
"--line='+RequestGPUs=1'" ]
1083 if args.node_features:
1094 run_nova_sam_opts += [
"-n %d" % nevts ]
1096 if args.zipLogs: args.logs =
True 1098 if args.lemBalance
and args.lemServer:
1099 fail(
"Cannot specify both --lemServer and --lemBalance")
1103 passThru = [
'gdb',
'hashDirs',
'runDirs',
'noCleanup',
'jsonMetadata',
'copyOut',
'logs',
'zipLogs',
'outputNumuDeCAF',
'outputNueDeCAF',
'outputNumuOrNueDeCAF',
'outputNusDeCAF',
'outputValidationDeCAF',
'lemBalance']
1106 for opt
in passThru:
1108 run_nova_sam_opts += [
'--'+opt]
1113 if args.second_config:
1114 run_nova_sam_opts += [
'--second_config ' + args.second_config]
1117 run_nova_sam_opts += [
"--lemServer " + args.lemServer]
1119 if args.cosmicsPolarity:
1120 run_nova_sam_opts += [
"--cosmicsPolarity " + args.cosmicsPolarity]
1123 run_nova_sam_opts += [
"--npass " + args.npass]
1126 run_nova_sam_opts += [
"--skim " + args.skim]
1128 if args.systematic :
1129 run_nova_sam_opts += [
"--systematic " + args.systematic]
1131 if args.specialName :
1132 run_nova_sam_opts += [
"--specialName " + args.specialName]
1135 run_nova_sam_opts += [
"--genietune " + args.genietune]
1138 run_nova_sam_opts += [
"--NPPFX " + args.NPPFX]
1140 if args.declareFiles:
1141 run_nova_sam_opts += [
"--declareFiles"]
1143 if args.declareLocations:
1144 run_nova_sam_opts += [
"--declareLocations"]
1146 out_tiers=args.outTier
1149 outnum,mcouttier = out_tiers[0].
split(
":")
1150 if mcouttier ==
"artdaq" :
1151 copyback =
'"*.daq.root"' 1153 copyback =
'"*.'+ mcouttier +
'.root"' 1155 if None != out_tiers :
1156 for tier
in out_tiers :
1157 run_nova_sam_opts += [
"--outTier " + tier]
1159 caf_tiers=args.cafTier
1160 if None != caf_tiers :
1161 for tier
in caf_tiers :
1162 run_nova_sam_opts += [
"--cafTier " + tier]
1164 flat_tiers=args.flatTier
1165 if None != flat_tiers :
1166 for tier
in flat_tiers :
1167 run_nova_sam_opts += [
"--flatTier " + tier]
1169 hist_tier=args.histTier
1170 if None != hist_tier :
1171 run_nova_sam_opts += [
"--histTier " + hist_tier]
1173 h5_tiers=args.h5Tier
1174 if None != h5_tiers :
1175 for tier
in h5_tiers :
1176 run_nova_sam_opts += [
"--h5Tier " + tier]
1178 if args.copyOut
and (
None==hist_tier
and None==caf_tiers
and None==flat_tiers
and None==out_tiers
and None==h5_tiers):
1179 fail(
"You specified --copyOut but did not specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier")
1181 if not (args.copyOut
or args.copyOutScript) :
1182 fail(
"Did not specify a method to copy back output (--copyOut or --copyOutScript)")
1184 if args.copyOut
and args.copyOutScript :
1185 fail(
"The options --copyOut and --copyOutScript conflict")
1188 export_to_art_sam_wrap += args.export
1190 if args.veryearlyscript:
1191 veryearly_scripts += args.veryearlyscript
1194 source_scripts += args.source
1196 if args.earlyscript:
1197 early_scripts += args.earlyscript
1200 pre_scripts += args.prescript
1202 if args.precopyscript:
1203 inter_scripts += args.precopyscript
1206 post_scripts += args.postscript
1208 if args.copyOutScript:
1209 post_scripts.append(args.copyOutScript)
1211 for script
in veryearly_scripts + early_scripts + source_scripts + pre_scripts + post_scripts + inter_scripts:
1213 script_path = script.split(
":")[0]
1215 script_path = script
1218 if not find_file(os.environ[
"PATH"].
split(os.pathsep), os.path.expandvars(script_path)):
1219 fail(
"Script %s does not exist!" % script_path)
1221 if not args.continue_project:
1223 project_name = user +
"-" + jobname +
"-" + timestamp
1224 if args.test_submission:
1225 project_name +=
"-testjobs" 1226 start_project =
True 1228 project_name = args.continue_project
1229 start_project =
False 1233 if args.novasoftups:
1234 art_sam_wrap_cmd=
"$NOVASOFT_FQ_DIR/bin/art_sam_wrap.sh" 1245 start_proj_command =
"samweb start-project " 1247 start_proj_command+=
" --defname=%s" %defname
1249 start_proj_command+=
" --snapshot_id=%d" % snapshot_id
1250 start_proj_command+=
" --group=nova" 1251 start_proj_command+=
" --station=%s" % sam_station
1253 start_proj_command+=
" %s" %project_name
1254 if start_project
and not test:
1255 start_proj_retval=os.system(start_proj_command)
1256 print "start proj returned %d" % start_proj_retval
1257 if start_proj_retval != 0:
1258 fail(
"Couldn't start project")
1260 print "Station monitor: http://samweb.fnal.gov:8480/station_monitor/nova/stations/" + sam_station +
"/projects/" + project_name
1261 os.putenv(
"SAM_PROJECT_NAME",project_name)
1265 job_cmd=os.path.expandvars(
"${CONDOR_EXEC}/%s.sh" % project_name)
1267 while os.path.exists(job_cmd):
1268 job_cmd=os.path.expandvars(
"${CONDOR_EXEC}/%s_%i.sh" % (project_name, scriptcount) )
1280 os.symlink(os.path.expandvars(art_sam_wrap_cmd),job_cmd)
1281 art_sam_wrap_cmd=job_cmd
1288 if not args.no_multifile:
1289 art_sam_wrap_opts += [
"--multifile"]
1292 art_sam_wrap_opts += [
"--printenv"]
1295 run_nova_sam_opts += [
"--txtfiledef"]
1296 print "Passing --txtfiledef from submit_nova_art.py to runNovaSAM" 1299 art_sam_wrap_opts += [
"--config " + fcl]
1302 jobsub_opts += [
'-l "+JobType="MC""']
1303 art_sam_wrap_opts += [
"--getconfig"]
1306 art_sam_wrap_opts += [
"--mix",args.mix]
1308 if not args.novasoftups :
1309 art_sam_wrap_opts += [
"--source %s:-r:%s:%s%s" %(setup_location, tag, maxopt_opt, build_location_arguments)]
1310 if args.testrel
or args.user_tarball:
1311 art_sam_wrap_opts += [
"--setup_testrel"]
1314 art_sam_wrap_opts += [
"--source setup_test_product:NovaGridUtils"]
1316 if args.ngu_version:
1317 art_sam_wrap_opts += [
"--source setup_product:NovaGridUtils:%s" %(args.ngu_version)]
1319 if args.testrel_ngu:
1320 art_sam_wrap_opts += [
"--testrel_ngu"]
1322 for veryearly_script
in veryearly_scripts:
1323 art_sam_wrap_opts += [
"--veryearlyscript " + veryearly_script]
1325 for early_script
in early_scripts:
1326 art_sam_wrap_opts += [
"--earlyscript " + early_script]
1328 for source_script
in source_scripts:
1329 art_sam_wrap_opts += [
"--source " + source_script]
1331 for pre_script
in pre_scripts:
1332 art_sam_wrap_opts += [
"--prescript " + pre_script]
1334 for inter_script
in inter_scripts:
1335 run_nova_sam_opts += [
"--precopyscript " + inter_script]
1337 for post_script
in post_scripts:
1338 art_sam_wrap_opts += [
"--postscript " + post_script]
1340 for input_file
in input_files:
1341 art_sam_wrap_opts += [
"--inputfile " + input_file]
1344 art_sam_wrap_opts += [
"--addoutput " + copyback]
1345 if args.hashDirs==
True:
1346 art_sam_wrap_opts += [
"--hash"]
1347 art_sam_wrap_opts += [
"--dest " + dest]
1349 if args.poms
and not test:
1350 poms_campaign_id = poms_client.register_poms_campaign(
1353 experiment =
'nova',
1358 poms_task_id = poms_client.get_task_id_for(
1363 export_to_art_sam_wrap += [
"POMS_CAMPAIGN_ID={0}".
format(poms_campaign_id),
1364 "POMS_TASK_ID={0}".
format(poms_task_id)]
1365 print "POMS Campaign: https://pomsgpvm01.fnal.gov/poms/campaign_info?campaign_id={0}".
format(poms_campaign_id)
1382 os.system(jobsub_cmd)
1386 print "Please note: if you intend to submit any more jobs," 1387 print " please wait", njobs/1000,
"minutes before your next submission" 1388 print " so as to avoid overloading the jobsub server." void split(double tt, double *fr)
def setup_production(args)
def check_dir(output_dir, prefix='')
def find_file(paths, filename)
Module that kips a configurable number of events between each that it allows through. Note that this module really skips (N-1) events, it uses a simple modular division as its critera. This module will cut down the data sample to 1/N of its original size.
def make_jobsub_node_features_arg(features)
def find_file_in_list(filepath, pathlist)
std::string format(const int32_t &value, const int &ndigits=8)
procfile open("FD_BRL_v0.txt")
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
def add_node_features_arg(parser)
def setup_calibration(args)
def check_make_dir(dname)