submit_nova_art.py

Go to the documentation of this file.

#!/bin/env python

import os, sys, stat, pwd, re
import argparse
import datetime
import samweb_client
import string
import tokenize
import cStringIO
import subprocess
import NovaGridUtils
from NovaGridUtils import *

user=os.getenv("USER")
os.environ["GRID_USER"]=user

sam_user=user
sam_station=os.getenv("SAM_STATION")
os.system("echo SAM_STATION DEFINED AS $SAM_STATION")

recommended_sites=["BNL",
               "Caltech",
               "Clemson",
               "Cornell",
               "FZU",
#               "Harvard",
               "Hyak_CE",
               "Michigan",
#               "MIT",
               "MWT2",
               "Nebraska",
               "NotreDame",
               "Omaha",
               "OSC",
               "SMU_HPC",
               "SU-OG",
               "SU-ITS",
               "UChicago",
               "UCSD",
#               "TTU",
               "Wisconsin"]
# Not working now, may return: "Harvard", "MIT", "MWT2", "UChicago", "TTU"
# FIFE maintains a list, too. Find it here:
# https://cdcvs.fnal.gov/redmine/projects/fife/wiki/Information_about_job_submission_to_OSG_sites

cvmfs_distro_base = "/cvmfs/nova.opensciencegrid.org"
novasoft_cvmfs = "%s/novasoft/slf6/novasoft" % cvmfs_distro_base
build_location_arguments = ""  # Becomes more specific with --cvmfs
setup_location = ""

jobsub_opts = []

run_nova_sam_cmd="runNovaSAM.py"
run_nova_sam_opts= []
export_to_run_nova_sam = []

art_sam_wrap_cmd="$NOVAGRIDUTILS_DIR/bin/art_sam_wrap.sh"
art_sam_wrap_opts= []
export_to_art_sam_wrap=[]
export_to_art_sam_wrap.append("SAM_PROJECT_NAME")
export_to_art_sam_wrap.append("SAM_STATION")
export_to_art_sam_wrap.append("IFDH_BASE_URI")
if "IFDH_DEBUG" in os.environ:
    export_to_art_sam_wrap.append("IFDH_DEBUG")
if "G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION" in os.environ:
    export_to_art_sam_wrap.append("G4NEUTRONHP_USE_ONLY_PHOTONEVAPORATION")

export_to_art_sam_wrap.append("EXPERIMENT")
export_to_art_sam_wrap.append("GRID_USER")

os.environ["CVMFS_DISTRO_BASE"]=cvmfs_distro_base
export_to_art_sam_wrap.append("CVMFS_DISTRO_BASE")

#there must be a better name for this variable
usage_models=["DEDICATED"]

input_files=[]

veryearly_scripts=[]
early_scripts=[]
source_scripts=[]
pre_scripts=[]
inter_scripts=[]
post_scripts=[]

def check_env(vname):
    value=os.getenv(vname)
    if None == value or "" == value:
        fail("Environment variable %s not defined" %vname)

def check_fcl(tag,fcl):
    if "/" in fcl[:1] and os.path.isfile(fcl):
        return fcl # hey I'm ok!
    # Loop over dirs in FHICL_FILE_PATH
    fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
    for path in fclPaths:
        fullPath = os.path.join(path, fcl)
        if os.path.isfile(fullPath):
            return fcl # hey you're ok!
    # Check if they are copying the fhicl file to the worker node.
    for input_file in input_files:
        if os.path.basename(input_file) == fcl:
            return fcl # Passing fhicl as argument, all good.
        elif os.path.basename(input_file) == os.path.basename(fcl):
            print ""
            print "The fhicl will be copied to $PWD on the worker node."
            print "You specified some sort of file path which isn't needed. Fixing it for you :)"
            print fcl + " --> " + os.path.basename(fcl)
            print ""
            return os.path.basename(fcl) # User incorrectly left file path there.

    fail("fcl file %s does not exist" %fcl)


def remove_comments(src):
    """
    This reads tokens using tokenize.generate_tokens and recombines them
    using tokenize.untokenize, and skipping comment/docstring tokens in between
    """
    f = cStringIO.StringIO(src)
    class SkipException(Exception): pass
    processed_tokens = []
    last_token = None
    # go thru all the tokens and try to skip comments and docstrings
    for tok in tokenize.generate_tokens(f.readline):
        t_type, t_string, t_srow_scol, t_erow_ecol, t_line = tok

        try:
            if t_type == tokenize.COMMENT:
                raise SkipException()

            elif t_type == tokenize.STRING:

                if last_token is None or last_token[0] in [tokenize.INDENT]:
                    pass

        except SkipException:
            pass
        else:
            processed_tokens.append(tok)

        last_token = tok

    return tokenize.untokenize(processed_tokens)

def find_file(paths, filename):
    if os.path.isfile(filename):
        return filename
    for path in paths:
        for root, dirs, files in os.walk(os.path.expandvars(path)):
            if filename in files:
                return os.path.join(root, filename)
    fail("Cannot find file "+filename)

def find_file_in_list(filepath, pathlist):
    for testpath in pathlist:
        if os.path.basename(filepath) == os.path.basename(testpath):
            return True
    return False


def build_jobsub_cmd():
    # Start with jobsub_submit and its options
    jobsub_cmd = "jobsub_submit \\\n"
    for opt in jobsub_opts:
        jobsub_cmd += "    " + opt + " \\\n"
    for export in export_to_art_sam_wrap:
        jobsub_cmd += "    -e " + export
    jobsub_cmd += " \\\n"

    # Add art_sam_wrap wrapper script and its options
    if args.testrel:
        if args.reuse_tarball:
            jobsub_cmd += "    --tar_file_name dropbox://" + os.path.basename(args.testrel) +".tar \\\n"
        else:
            jobsub_cmd += "    --tar_file_name tardir://" + args.testrel +" \\\n"
    elif args.user_tarball:
        if not os.path.isfile(args.user_tarball):
            print "Tarball filename passed to --user_tarball does not exit:", args.user_tarball
            sys.exit(5)
        jobsub_cmd += "    --tar_file_name dropbox://" + args.user_tarball + " \\\n"

    jobsub_cmd += "    file://" + art_sam_wrap_cmd + " \\\n"
    for opt in art_sam_wrap_opts:
        jobsub_cmd += "      " + opt + " \\\n"
    for export in export_to_run_nova_sam :
        jobsub_cmd += "      --export " + export + " \\\n"



    # Now set the main program and its options
    if not mcgen:
        jobsub_cmd += "      -X " + run_nova_sam_cmd + " \\\n"
        for opt in run_nova_sam_opts:
            jobsub_cmd += "        " + opt + " \\\n"
    else:
        jobsub_cmd += "      -X nova \\\n"
        mcgen_opts = ["--sam-file-type=importedSimulated", "--sam-application-family=nova","--sam-data-tier=" + mcouttier,"--sam-application-version=" + tag]
        if nevts>0 :
            mcgen_opts += ["-n %d" % nevts]
        for opt in mcgen_opts:
            jobsub_cmd += "        " + opt + " \\\n"

    jobsub_cmd = jobsub_cmd[:-2]
    return jobsub_cmd


#######################################################################################

if __name__=='__main__':

    prog=os.path.basename(sys.argv[0])
    NovaGridUtils.prog=prog


    while "-f" in sys.argv or "--file" in sys.argv:
        ### Allow args to be passed in as a plain text file.
        ### We make a preliminary parser get these arguments out for two reasons:
        ###    1)  Maintain standard -h, --help functionality
        ###    2)  Avoid necessity required arguments in initial parsing,
        ###        allow them to be missing, but find them in the file.
        preliminary_parser = argparse.ArgumentParser(prog=prog, description='Submit nova art job')

        preliminary_parser.add_argument('-f', '--file',
                        help="""Text file containing any arguments to this utility.  Multiple allowed.
                        Arguments should look just like they would on the command line,
                        but the parsing of this file is whitespace insenstive.
                        Commented lines will be identified with the # character and removed. """, type=str, action='append')
        pre_args, unknown = preliminary_parser.parse_known_args()

        # Remove pre_args from sys.argv so they are not processed again
        sys.argv = filter(lambda x: x not in [ "-f", "--file"], sys.argv)

        if pre_args.file:
            for filepath in pre_args.file:
                index = sys.argv.index(filepath)
                sys.argv.remove(filepath)
                if os.path.isfile(filepath):
                    fullpath = filepath
                else:
                    fullpath = find_file(["$NOVAGRIDUTILS_DIR/configs/"],filepath)
                text = open(fullpath, 'r').read()
                text = remove_comments(text) # Strip out commented lines
                newargs = []
                for line in text.splitlines():
                    # Insert arguments into list in order
                    # where the -f appeared
                    newargs += line.split()
                sys.argv[index:index] = newargs

    parser = argparse.ArgumentParser(prog=prog, description='Submit nova art job', add_help=False)

    ###required options
    required_args = parser.add_argument_group("Required arguments", "These arguments must be supplied.")

    required_args.add_argument('--jobname',
                        required=True,
                        help='Job name',
                        type=str)

    required_args.add_argument('--defname',
                        required=True,
                        help='SAM dataset definition to run over',
                        type=str)

    required_args.add_argument('--config', '-c',
                        required=True,
                        help='FHiCL file to use as configuration for nova executable. The path given should be relative to the $SRT_PRIVATE_CONTEXT of any test release you submit using',
                        type=str)

    required_args.add_argument("--tag",
                        required=True,
                        help="Tag of novasoft to use",
                        type=str)

    required_args.add_argument("--dest",
                        required=True,
                        help="Destination for output files",
                        type=str)


    ###debugging
    debugging_args = parser.add_argument_group("Debugging options", "These optional arguments can help debug your submission.")

    debugging_args.add_argument('--print_jobsub',
                        help='Print jobsub command',
                        action='store_true',default=False)

    debugging_args.add_argument('--printenv',
                        help='Print environment variables',
                        action='store_true',default=False)

    debugging_args.add_argument('--test',
                        help='Do not actually do anything, just run tests and print jobsub cmd',
                        action='store_true',default=False)

    debugging_args.add_argument('--gdb',
                        help='Run nova executable under gdb, print full stack trace, then quit gdb.',
                        action='store_true',default=False)

    debugging_args.add_argument('--test_submission',
                        help='Override other arguments given to submit a test to the grid. It will run 1 job with 3 events and write the output to /pnfs/nova/scratch/users/<user>/test_jobs/<date>_<time>',
                        action='store_true',default=False)

    debugging_args.add_argument('--jobsub_server',
                        help='Submit using the specified jobsub server',
                        default="")

    debugging_args.add_argument('--test_queue',
                        help='Submit jobs to the test jobsub queue for higher starting priority. NB NOvA is limited to 10 jobs at a time on this queue.',
                        action='store_true', default=False)

    debugging_args.add_argument("--kill_after",
                                metavar="SEC",
                                help="If job is still running after this many seconds, kill in such a way that a log will be returned",
                                type=int)

    ###job control
    job_control_args = parser.add_argument_group("Job control options", "These optional arguments help control where and how your jobs land.")
    ###number of jobs
    job_control_args.add_argument('--njobs',
                        help='Number of jobs to submit',
                        type=int, default = 0)

    job_control_args.add_argument('--maxConcurrent',
                        help='Run a maximum of N jobs simultaneously',
                        metavar='N',
                        type=int, default=0)

    job_control_args.add_argument('--files_per_job',
                        help='Number of files per job - if zero, calculate from number of jobs', metavar='N',
                        type=int, default = 0)

    job_control_args.add_argument('--nevts',
                        help='Number of events per file to process',
                        type=int, default = 0)


    job_control_args.add_argument('--no_multifile',
                        help='Do not use art_sam_wrap.sh multifile mode, which is on by default',
                        action='store_true')

    job_control_args.add_argument('--txtfiledef',
                                  help='Use if the input definition is made up of text files, each containing a list of file names',
                                  action='store_true',default=False)

    ###general job control
    job_control_args.add_argument('--opportunistic',
                        help='Run opportunistically on the fermigrid',
                        action='store_true',default=False)

    job_control_args.add_argument('--offsite',
                        help='Allow to run on offsite resources as well.  Implies --opportunistic.',
                        action='store_true',default=False)

    job_control_args.add_argument('--offsite_only',
                        help='Allow to run solely on offsite resources.',
                        action='store_true',default=False)

    job_control_args.add_argument('--amazon',
                        help='Run at amazon.',
                        action='store_true',default=False)

    job_control_args.add_argument('--site',
                        help='Specify allowed offsite locations.  Omit to allow running at any offsite location',
                        type=str,action='append')

    job_control_args.add_argument('--exclude_site',
                        help='Specify an offsite location to exclude.',
                        metavar='SITE',
                        type=str,action='append')

    job_control_args.add_argument('--recommended_sites',
                        help='Specify known working offsite locations.',
                        action='store_true',default=False)

    job_control_args.add_argument('--autoDropbox',
                        help='Use automatic dropbox location based on site',
                        action='store_true',default=False)

    job_control_args.add_argument('--os',
                        help='Specify OS version of worker node',
                        type=str,action='append')

    job_control_args.add_argument('--disk',
                        help='Local disk space requirement for worker node in MB.',
                        type=int, default=10000)

    job_control_args.add_argument('--memory',
                        help='Local memory requirement for worker node in MB.',
                        type=int, default=1900)

    job_control_args.add_argument('--expected_lifetime',
                        help='Expected job lifetime (default is 10800s=3h).  Valid values are an integer number of seconds or one of \"short\" (6h), \"medium\" (12h) or \"long\" (24h, jobsub default)', metavar='LIFETIME',
                        type=str, default="10800")

    job_control_args.add_argument('--dynamic_lifetime',
                        help="Dynamically determine whether a new file should be started based on glidein lifetime. Specify the maximum length expected for a single file to take to process in seconds.", metavar="LIFETIME",
                        type=str)

    job_control_args.add_argument('--cpu',
                        help="Request worker nodes that have at least NUMBER cpus",
                        type=int, default=1)


    job_control_args.add_argument('--group', '-G',
                        help="Specify batch group GROUP -- mainly used to set job priority.  At present, only supportable value is nova",
                        type=str, default="nova")

    job_control_args.add_argument('--subgroup',
                        help='Production subgroup',
                        type=str)

    job_control_args.add_argument("--role",
                        help="Specify role to run on the grid.  Can be Analysis (default) or Production.  This option is no longer supported",
                        type=str,default="Analysis")

    job_control_args.add_argument('--continue_project',
                        help="Don't start a new samweb project, instead continue this one.", metavar='PROJECT',
                        type=str,default="")

    job_control_args.add_argument("--snapshot_id",
                        help="Use this existing snapshot instead of creating a new one.", metavar="ID",
                        type=int,default=0)

    job_control_args.add_argument("--poms",
                        help="Start/continue a poms campaign and task for this submission",
                        default=False, action="store_true")

    job_control_args.add_argument("--mix",
                        help="Pass a mixing script to the job to pull in a files for job mixing.",
                        default="", type=str)

    job_control_args.add_argument("--mail_always",
                        help="Do you want an email whenever every jobs finishes?",
                        default=False, action="store_true")

    job_control_args.add_argument("--mail_on_error",
                        help="Do you want an email whenever a job fails on an error?",
                        default=False, action="store_true")

#    job_control_args.add_argument("--poms_definition",
#                        help="POMS definition to use",
#                        default="GenercicSubmitNOvAArt", type=str)

    job_control_args.add_argument('--user_priority',
                        help='Priority (integer) within a user\'s jobs (default = 0)',
                        type=int, default=0)

    job_control_args.add_argument('--singularity',
                        help='Location in CVMFS of a singularity container to launch the job into',
                        type=str,default='')

    job_control_args.add_argument('--jobfactory',
                        help='Use the specified JobFactoryType.',
                        default="")

    job_control_args.add_argument("--gpu",
                        help="Request a node with a GPU",
                        default=False, action="store_true")

    add_node_features_arg(job_control_args)

    ###software control
    novasoft_args = parser.add_argument_group("NOvA software options", "These options control the novasoft setup.")
    novasoft_args.add_argument('--maxopt',
                        help='Run in maxopt mode',
                        action='store_true',default=True)

    testrel_gp = novasoft_args.add_mutually_exclusive_group(required=False)
    testrel_gp.add_argument("--testrel",
                            help="Use a test release at location TESTREL. It will be tarred up, and sent to the worker node.",
                            type=str)
    testrel_gp.add_argument("--user_tarball",
                            help="Use existing test release tarball in specified location rather than having jobsub make one for you (conflicts with --testrel)",
                            type=str)

    novasoft_args.add_argument('--reuse_tarball',
                               help='Do you want to reuse a tarball that is already in resilient space? If using this option avoid trailing slash in --testrel option. (conflicts with --user_tarball)',
                               action='store_true',default=False)

    novasoft_args.add_argument('--cvmfs',
                        help='Does nothing (always true), but retained for compatibility: pull software from CVMFS.',
                        action='store_true')


    novasoft_args.add_argument('--disable_cvmfs_version_matching',
                        help="Don't perform a CVMFS-is-up-to-date check on target nodes via Condor requirements. (For advanced debugging use.) ",
                        action="store_true",
                        default=False)

    novasoft_args.add_argument('--novasoftups',
                        help='Use the ups build of novasoft, must be used with source to setup.',
                        action='store_true')

    novasoft_args.add_argument('--ngu_test',
                        help='Setup the test version of NovaGridUtils in the grid jobs.',
                        action='store_true')

    novasoft_args.add_argument('--ngu_version',
                        help='Setup a specific NovaGridUtils version in the grid jobs.', metavar='VERSION',
                        type=str)

    novasoft_args.add_argument('--testrel_ngu',
                               help="Must be used with --testrel, with NGU checked out. After unpacking tarball will setup the local version of NGU you are using on the work.",
                               action='store_true')

    novasoft_args.add_argument('--lemBalance',
                               help='Choose lem server based on (CLUSTER+PROCESS)%%2 to balance load',
                               action='store_true', default=False)

    novasoft_args.add_argument('--lemServer',
                               help='Specify lem server',
                               type=str)

    ###output control
    output_args = parser.add_argument_group("Output file options", "Note that you must specify either --copyOut or --copyOutScript.")


    output_args.add_argument('--copyOutScript',
                        help='Use script COPYOUTSCRIPT to copy back your output',
                        type=str)

    output_args.add_argument('--copyOut',
                        help='Use the built in copy out mechanism.  If used, you must specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier',
                        action='store_true')

    output_args.add_argument('--logs',
                             help='Return .log files corresponding to every output',
                             action='store_true')
    output_args.add_argument('--zipLogs',
                             help='Format logs as .bz2 files. Implies --logs',
                             action='store_true')

    output_args.add_argument('--outTier',
                        help='Data tier of the output file, multiple allowed, formatted as <name_in_fcl_outputs>:<data_tier>',
                        type=str, action='append')

    output_args.add_argument('--cafTier',
                        help='Module label for CAF output, multiple allowed.  Format as <cafmaker_module_label>:<data_tier>',
                        type=str, action='append')

    output_args.add_argument('--flatTier',
                        help='Module label for FlatCAF output, multiple allowed.  Format as <flatmaker_module_label>:<data_tier>',
                        type=str, action='append')

    output_args.add_argument('--histTier',
                        help='File identifier string for TFileService output, only one allowed.  Supply as --histTier <id> for output_name.<id>.root, where output_name is assembled based on the input file.',
                        type=str)

    output_args.add_argument('--h5Tier',
                             help='File identifier for h5 output, multiple allowed. Format as <hdf5maker_module>:<data_tier>',
                             type=str, action='append')

    output_args.add_argument('--outputNumuDeCAF',
                             help='Make standard numu decafs for all CAF files produced during the job',
                             action='store_true',default=False)

    output_args.add_argument('--outputNueDeCAF',
                             help='Make standard nue decafs for all CAF files produced during the job',
                             action='store_true',default=False)

    output_args.add_argument('--outputNumuOrNueDeCAF',
                             help='Make standard nue or numu decafs for all CAF files produced during the job',
                             action='store_true',default=False)

    output_args.add_argument('--outputNusDeCAF',
                             help='Make standard nus decafs for all CAF files produced during the job',
                             action='store_true',default=False)

    output_args.add_argument('--outputValidationDeCAF',
                             help='Make validation (nue_or_numu_or_nus) decafs for all CAF files produced during the job',
                             action='store_true',default=False)


    output_args.add_argument('--cosmicsPolarity',
                           help='To specify a horn polarity for the cosmics output file name',
                           type=str)

    output_args.add_argument('--npass',
                           help='To specify npass (aka nova.subversion)',
                           type=str)

    output_args.add_argument('--skim',
                           help='To specify nova.skim (does not work with mc gen)',
                           type=str)

    output_args.add_argument('--systematic',
                           help='To specify nova.systematic (does not work with mc gen)', metavar='SYST',
                           type=str)

    output_args.add_argument('--specialName',
                           help='To specify nova.special name (does not work with mc gen)',
                           type=str)

    output_args.add_argument('--genietune',
                           help='To specify nova.genietune (does not work with mc gen)', metavar='TUNE',
                           type=str)

    output_args.add_argument('--NPPFX',
                           help='To specify number of PPFX universes',
                           type=str)

    output_args.add_argument('--hashDirs',
                        help='Use hash directory structure in destination directory.',
                        action='store_true')

    output_args.add_argument('--runDirs',
                        help='Use run directory structure in destination directory, 000XYZ/XYZUW for run number XYZUW.',
                        action='store_true')

    output_args.add_argument('--noCleanup',
                        help='Pass --noCleanup argument to runNovaSAM.py.  Necessary when using a postscript for copyout.',
                        action='store_true')

    output_args.add_argument('--jsonMetadata', help='Create JSON files with metadata corresponding to each output file, and copy them to the same destinations', action='store_true')

    output_args.add_argument('--declareFiles',
                        help='Declare files with metadata on worker node',
                        action='store_true')

    output_args.add_argument('--production',
                        help='Submit production style jobs.  Implies \"--role=Production --hashDirs --jsonMetadata --zipLogs\", and checks that other settings needed for production are specified',
                        action='store_true')

    output_args.add_argument('--calibration',
                        help='Submit calibration style jobs.  Implies \"--role=Production\", and checks that other settings needed for calibration are specified',
                        action='store_true')

    output_args.add_argument('--declareLocations',
                        help='Declare the file output locations to SAM during the copy back of the files',
                        action='store_true')

    ###environment  configuration
    environment_args = parser.add_argument_group("Environment options", "These optional arguments allow control of the grid running environment.")

    environment_args.add_argument('--export',
                        help='Export variable EXPORT to art_sam_wrap.sh',
                        type=str, action='append')

    environment_args.add_argument('--veryearlyscript',
                                  help='Source script EARLYSCRIPT before any environment setup or sourcing',
                                  type=str, action='append')

    environment_args.add_argument('--source',
                                  help='Source script SOURCE',
                                  type=str, action='append')

    environment_args.add_argument('--earlyscript',
                                  help='Execute script EARLYSCRIPT before any environment setup',
                                  type=str, action='append')

    environment_args.add_argument('--prescript',
                                  help='Execute script PRESCRIPT before executing runNovaSAM.py',
                                  type=str, action='append')

    environment_args.add_argument('--precopyscript',
                                  help='Execute script PRECOPYSCRIPT within runNovaSAM.py, after running the nova -c command.',
                                  type=str, action='append')

    environment_args.add_argument('--postscript',
                                  help='Execute script POSTSCRIPT after executing runNovaSAM.py',
                                  type=str, action='append')

    environment_args.add_argument('--inputfile',
                                  help='Copy this extra input file into job area before running executable',
                                  type=str, action='append')

    ###support options
    support_args = parser.add_argument_group("Support options", "These optional arguments using this submission utility easier.")

    support_args.add_argument("-h", "--help", action="help", help="Show this help message and exit")

    support_args.add_argument('-f', '--file',
                        help="""Text file containing any arguments to this utility.  Multiple allowed.
                        Arguments should look just like they would on the command line,
                        but the parsing of this file is whitespace insenstive.
                        Comments will be identified with the # character and removed. """, type=str, action='append')




    ############################################
    # Process and check command line arguments #
    ############################################

    args = parser.parse_args()
    timestamp=datetime.datetime.now().strftime("%Y%m%d_%H%M")

    # Load POMS if we need to:
    if args.poms:
        try:
            import poms_client
        except ImportError:
            print "POMS not setup. Run this and try again:"
            print
            print "    setup poms_client"
            print
            sys.exit(1)

    # Check for test submission. Has to be first to override other arguments
    if args.test_submission:
        test_njobs = 1
        test_nevts = 3
        test_dest = "/pnfs/nova/scratch/users/%s/test_jobs/%s" % (os.environ["USER"], timestamp)
        if not os.path.exists(test_dest):
            os.makedirs(test_dest)
            mode = os.stat(test_dest).st_mode | stat.S_IXGRP | stat.S_IWGRP
            os.chmod(test_dest, mode)
        test_expected_lifetime = "0"
        test_dynamic_lifetime = "500"
        test_files_per_job = 1

        print "Running a test submission. Overwriting:"

        print "  njobs", args.njobs, "-->", test_njobs
        args.njobs = test_njobs
        print "  nevts", args.nevts, "-->", test_nevts
        args.nevts = test_nevts
        print "  dest", args.dest, "-->", test_dest
        args.dest = test_dest
        print "  expected_lifetime", args.expected_lifetime, "-->", test_expected_lifetime
        args.expected_lifetime = test_expected_lifetime
        print "  dynamic_lifetime", args.dynamic_lifetime, "-->", test_dynamic_lifetime
        args.dynamic_lifetime = test_dynamic_lifetime
        print "  files_per_job", args.files_per_job, "-->", test_files_per_job
        args.files_per_job = test_files_per_job
        if args.declareFiles:
            print "  don't declareFiles"
            args.declareFiles = False
        if args.declareLocations:
            print "  don't declareLocations"
            args.declareLocations = False
        if args.autoDropbox:
            print "  don't use autoDropbox"
            args.autoDropbox = False
        if args.poms:
            print "  don't use poms"
            args.poms = False
        
        #print "  use the test jobsub queue, so OnSite only."
        #args.test_queue = True
        args.offsite = False

    jobname=args.jobname
    defname=args.defname
    snapshot_id=args.snapshot_id
    print_jobsub=args.print_jobsub

    if args.printenv :
        print "Will print environment vars "
        printenv=True

    test=args.test
    if test :
        print_jobsub=True
        print ""
        warn("--test was specified, so all we do is run checks and print jobsub cmd.")

    check_env("SETUP_IFDH_ART")
    check_env("SETUP_SAM_WEB_CLIENT")
    check_env("SETUP_JOBSUB_CLIENT")
    check_env("SAM_STATION")
    tag=check_tag(args.tag)

    srt_qual="debug"
    maxopt=args.maxopt
    maxopt_opt=""
    if maxopt:
        maxopt_opt += "-b:maxopt"
        srt_qual="maxopt"

    if args.reuse_tarball and not args.testrel:
        fail("--reuse_tarball specified without --testrel??")

    if args.testrel:
        check_dir(args.testrel)
        if not os.path.isdir(args.testrel+'/lib/'+os.getenv('SRT_ARCH')+'-GCC-'+srt_qual):
            fail(args.testrel+' has never been built '+srt_qual)

    if args.inputfile:
        input_files += args.inputfile

    for input_file in input_files:
        if not os.path.isfile(os.path.expandvars(input_file)):
            fail("Input file %s does not exist!" % input_file)
        if os.path.expandvars(input_file).startswith("/nova/"):
            fail("Input file %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % input_file)
        elif os.path.expandvars(input_file).startswith("/grid/"):
            fail("Input file %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % input_file)

    if args.singularity and not os.path.exists(args.singularity):
            fail("Requested singularity image cannot be found: %s" % args.singularity)

    if args.gpu and not args.singularity:
        warn("Requested GPU, but did not request singularity. This is not likely to succeed.")

    if args.gpu and not args.offsite_only:
        warn("GPUs are only available offsite, and you have not chosen --offsite_only")



    fcl=args.config

    mcgen = (fcl == "mcgen")

    if not mcgen:
        fcl = check_fcl(tag,fcl)

    dest=args.dest
    if not dest.startswith("s3://") :
        check_dir(dest)

    if os.path.expandvars(dest).startswith("/nova/"):
        fail("Destination directory %s cannot be on /nova/app or /nova/ana/ or /nova/data/ it must be in dCache /pnfs/nova/" % dest)
    elif os.path.expandvars(dest).startswith("/grid/"):
        fail("Destination directory %s cannot be on /grid/ it must be in dCache /pnfs/nova/" % dest)

    export_to_run_nova_sam.append("DEST=%s"%dest)

    is_production_arg   = args.production

    if "Production" in args.role and not is_production_arg:
        fail("You specified --role=Production but not --production.  This is no longer supported")

    if args.production:
        setup_production(args)
    elif args.calibration:
        setup_calibration(args)
    else:
        setup_analysis(args)

        # Check for test submission. Has to be first to override other arguments
    if args.test_submission:
        print "Running a test submission, turning off hashDirs"
        args.hashDirs = False

    if args.hashDirs and args.runDirs:
        fail("Cannot specify both --hashDirs and --runDirs (note that hashDirs is implied by --production)")

    role=args.role

    njobs=args.njobs
    files_per_job = args.files_per_job

    print "Definition name: %s" % defname
    if snapshot_id:
        print "   with snapshot_id: %d" % snapshot_id




    ################
    # Setup jobsub #
    ################

    if args.jobsub_server:
        jobsub_opts += ["--jobsub-server=%s"%args.jobsub_server]

    if files_per_job > 0 and njobs > 0 :
        ## both njobs and files per job are specified.  Just pass
        ## the settings through to jobsub and art_sam_wrap
        jobsub_opts += ["-N %d" %njobs]
        art_sam_wrap_opts += ["--limit %d" % files_per_job]

    elif files_per_job > 0:
        ##files/job specified, but not njobs.  Calculate njobs
        ## on the fly

        ##get files in dataset
        samweb = samweb_client.SAMWebClient(experiment='nova')
        if not snapshot_id:
            num_project_files=samweb.countFiles(defname=defname)
        else:
            num_project_files=samweb.countFiles(dimensions="snapshot_id {0:d}".format(snapshot_id))
        print "Definition file count %d" % num_project_files

        njobs=(num_project_files / files_per_job) +1
        jobsub_opts += ["-N %d" %njobs]
        art_sam_wrap_opts += ["--limit %d" % files_per_job]

    elif njobs > 0 :
        ##njobs specified, but not files/job.  Just set njobs
        ## and don't force limits on files per jobs
        jobsub_opts += ["-N %d" %njobs]

    else :
        warn("Neither --njobs nor --files_per_job specified.  Did you really want to do this? Sleeping for 5 seconds")
        sleep(5)

    # allow a little bit of grace -- 5500 rather than 5000
    if njobs > 5500:
      print >> sys.stderr, """
      Error: cannot submit more than 5000 jobs in one cluster.
      Please break your submission into multiple batches of 5000 (or less) jobs,
      and after submitting the first batch, use --continue_project with the project
      that results from the first submission for the remaining batches.

      Please separate submissions by 5 minutes.
      """
      sys.exit(1)

    if args.maxConcurrent:
        jobsub_opts += ["--maxConcurrent=%d" %args.maxConcurrent]

    if args.opportunistic or args.offsite:
        usage_models.append("OPPORTUNISTIC")
    if args.offsite :
        usage_models.append("OFFSITE")
    if args.offsite_only:
        if args.offsite:
            fail("Both --offsite and --offsite_only specified, these arguments conflict")

        if args.opportunistic:
            fail("Both --opportunistic and --offsite_only specified, these arguments conflict")

        usage_models = ["OFFSITE"]

    if args.amazon :
        usage_models=["AWS_HEPCLOUD"]
        awsfilepath=os.path.expandvars("/cvmfs/nova.opensciencegrid.org/externals/NovaGridUtils/$NOVAGRIDUTILS_VERSION/NULL/utils/aws_setup.sh")
        source_scripts.append(awsfilepath)

    if args.autoDropbox:
        run_nova_sam_opts += ["--autoDropbox"]

    # Check OS in off-site submissions
    if args.offsite or args.offsite_only:
        if args.singularity:
            if args.os:
                fail("Don't specify OS when submitting with --singularity")
        else:
            if not args.os:
                fail("Running offsite, but OS version not specified!")

    resource_opt="--resource-provides=usage_model=" + string.join(usage_models,",")
    jobsub_opts += [resource_opt]

    if args.recommended_sites or args.site:
        site_opt="--site="

        if args.recommended_sites:
            for isite in recommended_sites:
                site_opt += isite + ","
        if args.site:
            for isite in args.site:
                if isite not in recommended_sites:
                    warn("Site "+isite+" is not known to work. Your jobs may fail at that site. Sleeping for 5 seconds")
                    sleep(5)
                site_opt += isite + ","

        site_opt=site_opt[:-1]
        jobsub_opts += [ site_opt ]

    if args.exclude_site:
        for isite in args.exclude_site:
            jobsub_opts += [ "--append_condor_requirements='(TARGET.GLIDEIN_Site\ isnt\ \\\"%s\\\")'" % isite ]

    if args.os :
        allowed_os=["SL6"]
        for ios in args.os:
            if ios not in allowed_os:
                fail("Invalid OS %s" %ios)

        os_opt="--OS=" + string.join(args.os,",")
        jobsub_opts += [ os_opt ]

    if args.disk:
        disk_opt="--disk=%sMB" % (args.disk)
        jobsub_opts += [ disk_opt ]

    if args.memory:
        mem_opt="--memory=%sMB" % (args.memory)
        jobsub_opts += [ mem_opt ]

    if args.cpu:
        cpu_opt="--cpu=%d" % (args.cpu)
        jobsub_opts += [ cpu_opt ]

    if args.mail_always:
        jobsub_opts += ["--mail_always"]
    elif args.mail_on_error:
        jobsub_opts += ["--mail_on_error"]
    else:
        jobsub_opts += ["--mail_never"]

    # The default jobsub_submit priority is 0
    #production_priority_max = 100 # Reserved for keepup processing
    if args.user_priority != 0 :
        #if args.production and args.user_priority >= production_priority_max :
        #    fail( "Priority for production must be < %d" % production_priority_max )
        jobsub_opts += [ '-l "priority=%d"' % args.user_priority ]

    if args.kill_after:
        kill_opt="--self-destruct-timer %d" % (args.kill_after)
        art_sam_wrap_opts += [ kill_opt ]

    if args.dynamic_lifetime:
        # Check other arguments
        args.expected_lifetime = "0"
        if args.files_per_job > 1:
            warn("You have limited number of files per job to "+str(args.files_per_job)+" but this argument should not be necessary with dynamic_lifetime.")
        art_sam_wrap_opts += [ "--dynamic_lifetime " + args.dynamic_lifetime ]
        jobsub_opts       += [ "--append_condor_requirements='(((TARGET.GLIDEIN_ToDie-CurrentTime)>%s)||isUndefined(TARGET.GLIDEIN_ToDie))'" % args.dynamic_lifetime]

    #expected lifetime can be an in (number of secs) or
    # one of a few strings, this should test for either
    # possibility
    try:
        dummy=string.atoi(args.expected_lifetime)
        jobsub_opts += ["--expected-lifetime=%ss" % (args.expected_lifetime)]
    except:
        allowed_lifetimes=["short","medium","long"]
        if args.expected_lifetime not in allowed_lifetimes:
            fail("Invalid expected_lifetime %s" % args.expected_lifetime)
        else:
            jobsub_opts += ["--expected-lifetime=%s" % (args.expected_lifetime)]

    # all software comes from CVMFS now since /nova/data is no longer mounted on Fermigrid
    build_location_arguments = "" # ":-e:%s/externals:/cvmfs/fermilab.opensciencegrid.org/products/common/db" % cvmfs_distro_base
    source_scripts.append( "/cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups.sh" )
    # assumes this job is being submitted on an up-to-date CVMFS install.  best we can do...
    cvmfs_rev = subprocess.check_output(["attr", "-qg", "revision", "/cvmfs/nova.opensciencegrid.org"]).strip()
    if cvmfs_rev:
        art_sam_wrap_opts.append("--cvmfs-revision %s" % cvmfs_rev)
        if not args.disable_cvmfs_version_matching:
            # if NOvA CVMFS version is available, select on it; if not, just accept the slot anyway
            jobsub_opts += [ "--append_condor_requirements='ifThenElse(isUndefined(TARGET.HAS_CVMFS_nova_opensciencegrid_org)==FALSE,TARGET.CVMFS_nova_opensciencegrid_org_REVISION>=%s,TRUE)'" % cvmfs_rev ]
    if tag == "development" or tag[0] == 'N':
        build_location_arguments += ":-6:/cvmfs/nova-development.opensciencegrid.org/novasoft"
    else:
        build_location_arguments += ":-6:%(dir)s/novasoft/slf6/novasoft" % {"dir": cvmfs_distro_base}

    setup_location= "%s/novasoft/slf6/novasoft/setup/setup_nova.sh" % (cvmfs_distro_base)


    group=args.group
    #probably others are ok as well
    allowed_groups=['nova','fermilab']
    #allowed_groups=['nova_high_prio', 'nova_medium_prio', 'nova_low_prio', 'nova']
    if group not in allowed_groups :
        fail("The only valid args for --group are " + " ".join(allowed_group) )
    jobsub_opts += [ "-G %s" % group ]

    if args.test_queue:
        jobsub_opts += ["--subgroup test"]
    elif args.subgroup :
        subgroup = args.subgroup
        if is_production_arg :
            allowed_subgroups = [ "keepup_prio", "prod_high_prio", "prod_prio" ]
        else :
            fail( "Only production subgroups are available at this time and reqire production credentials" )
            #allowed_subgroups = [ "ana_prio" ]

        if subgroup in allowed_subgroups :
            jobsub_opts += [ "--subgroup %s" % subgroup ]
        else :
            fail( "Allowed subgroups: " + ", ".join( allowed_subgroups ) )

    jobsub_opts += ["--role=%s" %(role)]


    # Singularity

    if args.singularity:
        jobsub_opts += [ "--line='+SingularityImage=\\\"%s\\\"'" % args.singularity ]
        jobsub_opts += [ "--append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'"]
    if args.jobfactory:
        jobsub_opts += [ "--line='+JobFactoryType=\\\"%s\\\"'" % args.jobfactory ]

    # GPUs
    if args.gpu:
        jobsub_opts += [ "--line='+RequestGPUs=1'" ]

    if args.node_features:
        jobsub_opts += [ make_jobsub_node_features_arg(args.node_features) ]



    ####################
    # Setup runNovaSAM #
    ####################

    nevts=args.nevts
    if nevts > 0:
        run_nova_sam_opts += ["-n %d" % nevts ]

    if args.zipLogs: args.logs = True

    if args.lemBalance and args.lemServer:
        fail("Cannot specify both --lemServer and --lemBalance")

    # Toggled options accepted by runNovaSAM with exactly the same syntax as
    # our own options
    passThru = ['gdb', 'hashDirs', 'runDirs', 'noCleanup', 'jsonMetadata', 'copyOut', 'logs', 'zipLogs', 'outputNumuDeCAF', 'outputNueDeCAF', 'outputNumuOrNueDeCAF','outputNusDeCAF', 'outputValidationDeCAF', 'lemBalance']

    va = vars(args)
    for opt in passThru:
        if va[opt]:
            run_nova_sam_opts += ['--'+opt]

    # Could consider doing a similar loop for specialName, outTier, cafTier,
    # histTier, but gets a little complicated, and there are fewer of them to
    # begin with to make it worth the effort.

    if args.lemServer:
        run_nova_sam_opts += ["--lemServer " + args.lemServer]

    if args.cosmicsPolarity:
        run_nova_sam_opts += ["--cosmicsPolarity " + args.cosmicsPolarity]

    if args.npass:
        run_nova_sam_opts += ["--npass " + args.npass]

    if args.skim :
        run_nova_sam_opts += ["--skim " + args.skim]

    if args.systematic :
        run_nova_sam_opts += ["--systematic " + args.systematic]

    if args.specialName :
        run_nova_sam_opts += ["--specialName " + args.specialName]

    if args.genietune :
        run_nova_sam_opts += ["--genietune " + args.genietune]

    if args.NPPFX :
        run_nova_sam_opts += ["--NPPFX " + args.NPPFX]

    if args.declareFiles:
        run_nova_sam_opts += ["--declareFiles"]

    if args.declareLocations:
        run_nova_sam_opts += ["--declareLocations"]

    out_tiers=args.outTier

    if fcl == "mcgen" :
        outnum,mcouttier = out_tiers[0].split(":")
        if mcouttier == "artdaq" :
            copyback = '"*.daq.root"'
        else:
            copyback = '"*.'+ mcouttier +'.root"'

    if None != out_tiers :
        for tier in out_tiers :
            run_nova_sam_opts += ["--outTier " + tier]

    caf_tiers=args.cafTier
    if None != caf_tiers :
        for tier in caf_tiers :
            run_nova_sam_opts += ["--cafTier " + tier]

    flat_tiers=args.flatTier
    if None != flat_tiers :
        for tier in flat_tiers :
            run_nova_sam_opts += ["--flatTier " + tier]

    hist_tier=args.histTier
    if None != hist_tier :
        run_nova_sam_opts += ["--histTier " + hist_tier]

    h5_tiers=args.h5Tier
    if None != h5_tiers :
        for tier in h5_tiers :
            run_nova_sam_opts += ["--h5Tier " + tier]

    if args.copyOut and (None==hist_tier and None==caf_tiers and None==flat_tiers and None==out_tiers and None==h5_tiers):
        fail("You specified --copyOut but did not specify --outTier, --cafTier, --flatTier, --h5Tier or --histTier")

    if not (args.copyOut or args.copyOutScript) :
        fail("Did not specify a method to copy back output (--copyOut or --copyOutScript)")

    if args.copyOut and args.copyOutScript :
        fail("The options --copyOut and --copyOutScript conflict")

    if args.export:
        export_to_art_sam_wrap += args.export

    if args.veryearlyscript:
        veryearly_scripts += args.veryearlyscript

    if args.source:
        source_scripts += args.source

    if args.earlyscript:
        early_scripts += args.earlyscript

    if args.prescript:
        pre_scripts += args.prescript

    if args.precopyscript:
        inter_scripts += args.precopyscript

    if args.postscript:
        post_scripts += args.postscript

    if args.copyOutScript:
        post_scripts.append(args.copyOutScript)

    for script in veryearly_scripts + early_scripts + source_scripts + pre_scripts + post_scripts + inter_scripts:
        if ":" in script:
            script_path = script.split(":")[0]
        else:
            script_path = script

        if not find_file_in_list(os.path.expandvars(script_path), input_files):
            if not find_file(os.environ["PATH"].split(os.pathsep), os.path.expandvars(script_path)):
                fail("Script %s does not exist!" % script_path)

    if not args.continue_project:
        ##start sam project
        project_name = user + "-" + jobname + "-" + timestamp
        if args.test_submission:
            project_name += "-testjobs"
        start_project = True
    else:
        project_name = args.continue_project
        start_project = False

    #sam_station=os.getenv("SAM_STATION")

    if args.novasoftups:
        art_sam_wrap_cmd="$NOVASOFT_FQ_DIR/bin/art_sam_wrap.sh"





    #########################
    # Start the SAM project #
    #########################


    start_proj_command ="samweb start-project "
    if not snapshot_id:
        start_proj_command+=" --defname=%s" %defname
    else:
        start_proj_command+=" --snapshot_id=%d" % snapshot_id
    start_proj_command+=" --group=nova"
    start_proj_command+=" --station=%s" % sam_station

    start_proj_command+= " %s" %project_name
    if start_project and not test:
        start_proj_retval=os.system(start_proj_command)
        print "start proj returned %d" %  start_proj_retval
        if start_proj_retval != 0:
            fail("Couldn't start project")

    print  "Station monitor: http://samweb.fnal.gov:8480/station_monitor/nova/stations/" + sam_station +"/projects/"  + project_name
    os.putenv("SAM_PROJECT_NAME",project_name)

    check_make_dir("${CONDOR_EXEC}")
    # Ensure unique job command
    job_cmd=os.path.expandvars("${CONDOR_EXEC}/%s.sh" % project_name)
    scriptcount=1
    while os.path.exists(job_cmd):
        job_cmd=os.path.expandvars("${CONDOR_EXEC}/%s_%i.sh" % (project_name, scriptcount) )
        scriptcount += 1




    ######################
    # Setup art_sam_wrap #
    ######################

    if not test:
        #create symlink so that jobs get a better name
        os.symlink(os.path.expandvars(art_sam_wrap_cmd),job_cmd)
    art_sam_wrap_cmd=job_cmd

    sys.stdout.flush()
    sys.stderr.flush()



    if not args.no_multifile:
        art_sam_wrap_opts += ["--multifile"]

    if args.printenv:
        art_sam_wrap_opts += ["--printenv"]

    if args.txtfiledef:
        run_nova_sam_opts += ["--txtfiledef"]
        print "Passing --txtfiledef from submit_nova_art.py to runNovaSAM"

    if not mcgen:
        art_sam_wrap_opts += ["--config " + fcl]

    if mcgen:
        jobsub_opts += ['-l "+JobType="MC""']
        art_sam_wrap_opts += ["--getconfig"]

    if args.mix:
        art_sam_wrap_opts += ["--mix",args.mix]

    if not args.novasoftups :
        art_sam_wrap_opts += ["--source %s:-r:%s:%s%s" %(setup_location, tag, maxopt_opt, build_location_arguments)]
        if args.testrel or args.user_tarball:
            art_sam_wrap_opts += ["--setup_testrel"]

    if args.ngu_test:
        art_sam_wrap_opts += ["--source setup_test_product:NovaGridUtils"]

    if args.ngu_version:
        art_sam_wrap_opts += ["--source setup_product:NovaGridUtils:%s" %(args.ngu_version)]

    if args.testrel_ngu:
        art_sam_wrap_opts += ["--testrel_ngu"]

    for veryearly_script in veryearly_scripts:
        art_sam_wrap_opts += ["--veryearlyscript " + veryearly_script]

    for early_script in early_scripts:
        art_sam_wrap_opts += ["--earlyscript " + early_script]

    for source_script in source_scripts:
        art_sam_wrap_opts += ["--source " + source_script]

    for pre_script in pre_scripts:
        art_sam_wrap_opts += ["--prescript " + pre_script]

    for inter_script in inter_scripts:
        run_nova_sam_opts += ["--precopyscript " + inter_script]

    for post_script in post_scripts:
        art_sam_wrap_opts += ["--postscript " + post_script]

    for input_file in input_files:
        art_sam_wrap_opts += ["--inputfile " + input_file]

    if mcgen:
        art_sam_wrap_opts += ["--addoutput " + copyback]
        if args.hashDirs==True:
            art_sam_wrap_opts += ["--hash"]
        art_sam_wrap_opts += ["--dest " + dest]

    if args.poms and not test:
        poms_campaign_id = poms_client.register_poms_campaign(
                                       jobname,
                                       user = user,
                                       experiment = 'nova',
                                       version = tag,
                                       dataset = defname)
#                                      campaign_definition = args.poms_definition )

        poms_task_id = poms_client.get_task_id_for(
                                   poms_campaign_id,
                                   user = user,
                                   command_executed = build_jobsub_cmd() )

        export_to_art_sam_wrap += ["POMS_CAMPAIGN_ID={0}".format(poms_campaign_id),
                                   "POMS_TASK_ID={0}".format(poms_task_id)]
        print  "POMS Campaign: https://pomsgpvm01.fnal.gov/poms/campaign_info?campaign_id={0}".format(poms_campaign_id)





    ############################
    # Actually launch the jobs #
    ############################

    jobsub_cmd = build_jobsub_cmd()

    if print_jobsub:
        print jobsub_cmd
        sys.stdout.flush()
        sys.stderr.flush()


    if not test:
        os.system(jobsub_cmd)

    if njobs > 1000:
      print
      print "Please note: if you intend to submit any more jobs,"
      print "  please wait", njobs/1000, "minutes before your next submission"
      print "  so as to avoid overloading the jobsub server."