productionTest.py
Go to the documentation of this file.
1 #!/bin/env python
2 print "run : --- Production Test"
3 ########################### Imports
4 import os
5 import subprocess
6 import resource
7 import ProductionTestTools as tools
8 import TierConfigurations as TC
9 import Tier
10 import Chain
11 import ViewSTDOUT as view
12 import glob
13 import time
14 ########################### Check that production validation is setup
15 assert "NOVAPRODVALID_DIR" in os.environ.keys(),"NOVAPRODVALID_DIR not set, try running setup_prod_valid.sh."
16 ########################### Check that ART is setup
17 if ("SRT_BASE_RELEASE") not in os.environ:
18  print "Warning: SRT based ART is not setup. This was only tested with SRT, if you are using cmake this shouldn't be a problem."
19  #exit(1)
20 # ########################### Setup batch password
21 # os.system("export NOVADBPWDFILE=/grid/fermiapp/nova/novaart/novasvn/releases/S14-08-19/Database/config/nova_reader_pwd")
22 # os.environ["NOVADBPWDFILE"] = "/grid/fermiapp/nova/novaart/novasvn/releases/S14-08-19/Database/config/nova_reader_pwd"
23 # print "run : --- DB password file: "
24 # os.system("echo $NOVADBPWDFILE")
25 # os.system("cat $NOVADBPWDFILE")
26 ########################### Option parser
27 from optparse import OptionParser
28 parser = OptionParser()
29 parser.add_option("-n", "--event_limit", help="limit the number of events to process", action="store", type=int, dest="n_events", default=False)
30 parser.add_option("-d", "--dry_run", help="dry run mode (don't run anything)", action="store_true", dest="dry_run", default=False)
31 parser.add_option("-l", "--no_log_files",help="turn off sending job stdout to log files", action="store_false", dest="log_files",default=True)
32 parser.add_option("-c", "--clean", help="clean run (delete all local root files)", action="store_true", dest="clean", default=False)
33 parser.add_option("-t", "--tiers", help="which production tiers to run. The format is PRODUCTIONCHAIN<:START TIER:END TIER> where START and END TIERS are optional. e.g: './productionTest.py -t FD_data:pid:caf' or './productionTest.py -t FD_genie'. If you only include one tier e.g: '-t FD_cosmics:pid', then only this stage will be run. If you include a trailing colon e.g: 'ND_genie:reconstruction:' then all tiers from the specified tier to the end of the chain will be run. Requesting non-configured tiers, or tiers in the wrong order will not work.", \
34  action="store", type=str, dest="tiers", default="FD_data")
35 parser.add_option("-p", "--pickle", help="pickle the output", action="store_true", dest="pickle", default=True)
36 parser.add_option("-v", "--verbose", help="turn on verbose mode", action="store_true", dest="verbose", default=False)
37 (options, args) = parser.parse_args()
38 if options.verbose:
39  print "run : --- Options"
40  print "run : event limit: ",options.n_events
41  print "run : dry run mode: ",options.dry_run
42  print "run : write to logs:",options.log_files
43  print "run : clean run: ",options.clean
44  print "run : tiers: ",options.tiers
45  print "run : pickle: ",options.pickle
46  print "run : verbose mode: ",options.verbose
47  tools.printEnvironment()
48 ########################### Clean setup
49 if options.clean:
50  print "run : Clean running requested, removing all local *.root files"
51  file_list = glob.glob("*.root")
52  for f in file_list: os.remove(f)
53 ########################### Check that an output directory exists
54 if (not os.path.exists("./output")):
55  print "run : Creating output directory"
56  os.mkdir("./output")
57 ########################### Tier configurations
58 if options.verbose: print "run : --- Tiers"
59 assert(options.tiers), "Error, you must specify a tier with the -t option. run ./productionTest.py -h for more information."
60 chain, tiers = TC.parseTierConfiguration(options.tiers,options.verbose)
61 ########################### Run tiers
62 metrics = {}
63 for tier in tiers:
64  print "run : --- Running chain: %s, tier: %s"%(chain.name,tier.short_name)
65  if options.log_files: log_file = "output/%s_%s"%(chain.name,tier.short_name)
66  else: log_file = False
67  try:
68  metric = tier.runTier(options.dry_run,
69  options.n_events,
70  options.verbose,
71  log_file)
72  except:
73  print "run : ERROR, unhandled exception in run tier"
74  metric = False
75  if (metric == False) and not options.dry_run:
76  print "run : Tier failed"
77  break
78  else:
79  metrics[tier.short_name] = metric
80 
81  if options.verbose and not options.dry_run:
82  view.showMetric(metrics[tier.short_name])
83 ########################### Pickle
84 if options.pickle:
85  output_name = "output/%s.pkl"%options.tiers.replace(":","_")
86  print "run : Saving output chain as %s"%output_name
87  import cPickle
88  chain.env = os.environ
89  chain.time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
90  cPickle.dump(chain,open(output_name,"wb"),2)
91 ########################### Summary
92 if not options.dry_run:
93  view.showMetricsSummary(chain)
94 ########################### Done
95 print "run : --- Done\n"
procfile open("FD_BRL_v0.txt")
assert(nhit_max >=nhit_nbins)