launch_batch_jobs.py

Go to the documentation of this file.

print "run  : --- Launch batch jobs"
########################### Imports
import subprocess
import os
import time
########################### Check that production validation is setup
assert "NOVAPRODVALID_DIR" in os.environ.keys(),"NOVAPRODVALID_DIR not set, try running setup_prod_valid.sh."
########################### Option parser
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-r", "--release",     help="which art release to use",          action="store", type=str, dest="release",  default=False)
parser.add_option("-d", "--dry_run",     help="dry run mode (don't run anything)", action="store_true",      dest="dry_run",  default=False)
parser.add_option("-b", "--debug",       help="run jobs in debug mode",            action="store_true",      dest="debug",    default=False)
parser.add_option("-s", "--short",       help="run in short mode",                 action="store_true",      dest="short",    default=False)
parser.add_option("-m", "--message",     help="add a message to info.pkl",         action="store", type=str, dest="message",  default="")
parser.add_option("-v", "--verbose",     help="turn on verbose mode",              action="store_true",      dest="verbose",  default=False)
(options, args) = parser.parse_args()
print "run  : --- Options"
print "run  :     release:      ",options.release
print "run  :     dry run mode: ",options.dry_run
print "run  :     debug mode:   ",options.debug
print "run  :     short mode:   ",options.short
print "run  :     message:      ",options.message
print "run  :     verbose mode: ",options.verbose
assert options.release, "No release specified, provide with -r"
########################### Make output directory
t_string = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
output_dir = "%s/production_testing/%s_%s/"%(os.environ["NOVAPRODVALID_GRID"], options.release, t_string)
print "run  :     output directory will be: %s"%output_dir
if not options.dry_run:
    print "run  :     making output directory"
    os.system("mkdir %s"%output_dir)
########################### clean up local directory
os.system("rm %s/testing/*.fcl"   %(os.environ["NOVAPRODVALID_DIR"]))
os.system("rm %s/testing/*.root"  %(os.environ["NOVAPRODVALID_DIR"]))
os.system("rm %s/testing/*.json"  %(os.environ["NOVAPRODVALID_DIR"]))
os.system("rm %s/testing/*.db"    %(os.environ["NOVAPRODVALID_DIR"]))
os.system("rm %s/testing/*.csv"   %(os.environ["NOVAPRODVALID_DIR"]))
os.system("rm %s/testing/output/*"%(os.environ["NOVAPRODVALID_DIR"]))
########################### collect latest working directory
command = "tar czf %s/../production_python.tgz %s/"%(os.environ["NOVAPRODVALID_DIR"], os.environ["NOVAPRODVALID_DIR"])
print "run  :     tar command: %s"%command
if not options.dry_run: os.system(command)
########################### Launch batch jobs
# jobsub_command = "jobsub -g %s/testing/scripts/batch-job.sh"%(os.environ["NOVAPRODVALID_DIR"]) # jobsub tools command
jobsub_command = "jobsub_submit -G nova --resource-provides=usage_model=DEDICATED,OPPORTUNISTIC --expected-lifetime=3500s --OS=SL6 --memory=3GB --disk=10GB file://%s/testing/scripts/batch-job.sh"%(os.environ["NOVAPRODVALID_DIR"])
########################### configure chains
chains = [] # format: [chain configuration to run, long number of events, short number of events]
chains.append(["FD_data_cosmics"     ,   2015,  200])
chains.append(["FD_data_NuMI"        ,    138,  138])
chains.append(["FD_cosmics"          ,    200,   20])
chains.append(["FD_genie_FHC_nonswap",   1000,   30])
chains.append(["FD_genie_FHC_swap"   ,   1000,   30])
chains.append(["FD_genie_FHC_tau"    ,   1000,   30])
chains.append(["FD_genie_RHC_nonswap",   1000,   30])
chains.append(["FD_genie_RHC_swap"   ,   1000,   30])
chains.append(["FD_genie_RHC_tau"    ,   1000,   30])
chains.append(["ND_data_cosmics"     ,   3599,  200])
chains.append(["ND_data_NuMI"        ,   1952,  200])
chains.append(["ND_cosmics"          , 250000, 2500])
chains.append(["ND_genie_FHC_nonswap",   2000,   30])
#chains.append(["ND_genie_RHC_nonswap",   2000,   30])
commands = []
i_evt = 1
if options.short: i_evt = 2
logs = []
compiler_flag = "maxopt"
if options.debug: compiler_flag = "debug"
for chain in chains:
    logs.append([])
    c = "%s %s %s %i %s %s"%(jobsub_command, options.release, chain[0], chain[i_evt], output_dir, compiler_flag)
    commands.append(c)
    print "run  :     command: %s"%c
    if not options.dry_run:
        good_submission = False
        process = subprocess.Popen(c.split(" "), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        output = process.stdout
        lines = output.readlines()
        for line in lines:
            print line.strip()
            # if "submitted" in line: good_submission = True
            if "JobsubJobId of first job" in line: good_submission = True
        if good_submission:
            print "run  : submission successful"
            for l in lines:
                if ".cmd" in l: print "saving link to log: %s"%l
                logs[-1].append(l.strip())
        else:
            assert False, "run  : ERROR in submission"

        time.sleep(2)
########################### pkl output
import cPickle
options.commands = commands
options.logs     = logs
options.time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
output_name = output_dir+"info.pkl"
if not options.dry_run:
    print "run  : Saved info as %s"%output_name
    cPickle.dump(options,open(output_name,"wb"))
########################### done
print "run  : done"