raw2rootAnalyser.py
Go to the documentation of this file.
1 print "run : --- RAW 2 root analyser"
2 ########################### Imports
3 import samweb_client
4 sam = samweb_client.SAMWebClient(experiment="nova")
5 import cPickle
6 import ProductionTestTools as tools
7 import os
8 ########################## Check that production validation is setup
9 assert "NOVAPRODVALID_DIR" in os.environ.keys(),"NOVAPRODVALID_DIR not set, try running setup_prod_valid.sh."
10 ########################### Option parser
11 from optparse import OptionParser
12 parser = OptionParser()
13 parser.add_option("-s", "--sample", help="sample to study", action="store", type=str, dest="sample", default=False)
14 parser.add_option("-a", "--assume", help="assume children", action="store_true", dest="assume", default=False)
15 (options, args) = parser.parse_args()
16 print "run : --- Options"
17 print "run : study: ",options.sample
18 print "run : assume: ",options.assume
19 assert options.sample,"No sample provided (use -s)"
20 all_samples = ["neardet_cosmics", "fardet_cosmics", "neardet_numi", "fardet_numi","neardet_ddactivity"]
21 assert options.sample in all_samples, "Non-configured sample requested: %s, options are: %s"%(options.sample,repr(all_samples))
22 ########################### functions
23 def analyse(detector, stream, query=False, partition=False, write=False):
24  to_return = []
25  releases = []
26  runs = []
27  run_subrun_to_list_map = {}
28  raw_query = "data_tier raw and file_type importedDetector and online.detector %s"%detector
29  if type(stream) == int: raw_query +=" and online.stream %i"%stream
30  if type(partition) == int: raw_query += " and online.partition %i"%partition
31  if query: raw_query += " and %s"%query
32  print "run : using query: %s"%raw_query
33  raw_files = sam.listFiles(dimensions=raw_query)
34  raw_files.sort()
35  n_kids = 0
36  print "run : found %i raw files"%len(raw_files)
37  for i,raw_file in enumerate(raw_files):
38  if i%100 == 0: print "run : file: [%i/%i]"%(i,len(raw_files))
39  # if i > 20: break
40  run = tools.findRun(raw_file)
41  subrun = tools.findSubRun(raw_file)
42  if run not in runs: runs.append(run)
43  this_json = {"file_name":raw_file,"children":[],"run":run,"subrun":subrun}
44 
45  if not options.assume:
46  # explicitly check all the children of this file
47  kids = sam.getFileLineage("children",raw_file)
48  for kid in kids:
49  try:
50  metadata = sam.getMetadata(kid["file_name"])
51  except:
52  print "run : - exception with %s, %s"%(raw_file,kid["file_name"])
53  metadata = {}
54  # print metadata
55  if "DAQ2RawDigit.base_release" in metadata.keys():
56  release = metadata["DAQ2RawDigit.base_release"]
57  if release not in releases:
58  print "run : - new release found: %s"%release
59  releases.append(release)
60  this_json["children"].append({"file_name":kid["file_name"],"release":release})
61  n_kids +=1
62 
63  to_return.append(this_json)
64  run_subrun_to_list_map[(run,subrun)] = i
65  #### if we haven't explicitly found the children of a file, instead we assume a match based on run and subrun
66  if options.assume:
67  root_query = "data_tier artdaq and file_type importedDetector and online.detector %s"%detector
68  if type(stream) == int: root_query +=" and online.stream %i"%stream
69  if type(partition) == int: root_query += " and online.partition %i"%partition
70  if query: root_query += " and %s"%query
71  print "run : using query: %s"%root_query
72  root_files = sam.listFiles(dimensions=root_query)
73  root_files.sort()
74  print "run : found %i root files"%len(root_files)
75  for i,root_file in enumerate(root_files):
76  if i%100 == 0: print "run : file: [%i/%i]"%(i,len(root_files))
77  # if i > 20: break
78  try:
79  run = tools.findRun(root_file)
80  except:
81  print "run : - exception finding run for file: %s"%root_file
82  continue
83  subrun = tools.findSubRun(root_file)
84  if run not in runs:
85  print "run : - artdaq run found which doesn't match any raw run: %i"%run
86  continue
87  try:
88  release = tools.findRelease(root_file)
89  except:
90  print "run : - exception finding release for file: %s"%root_file
91  continue
92  if release not in releases:
93  print "run : - new release found: %s"%release
94  releases.append(release)
95  if (run,subrun) not in run_subrun_to_list_map.keys():
96  print "run : - artdaq run, subrun found which doesn't match any raw run: r%i s%i"%(run,subrun)
97  continue
98  this_json = to_return[run_subrun_to_list_map[(run,subrun)]]
99  this_json["children"].append({"file_name":root_file,"release":release})
100  n_kids +=1
101 
102  print "run : found %i kids, over %i runs, in %i releases"%(n_kids,len(runs),len(releases))
103  if write:
104  print "run : writing results as %s"%write
105  to_write = {}
106  to_write["raw_files"] = to_return
107  to_write["releases"] = releases
108  to_write["runs"] = runs
109  cPickle.dump(to_write,open(write,"wb"))
110  # return to_return, releases
111 ########################### Run
112 ################### ND cosmics
113 if options.sample == "neardet_cosmics":
114  print "run : --- ND cosmics"
115  analyse("neardet", 2, write="%s/raw2root_pkls/neardet_cosmics_raw2root.pkl"%os.environ["NOVAPRODVALID_DATA"])
116 ################### ND dd-activity
117 if options.sample == "neardet_ddactivity":
118  print "run : --- ND DD-activity"
119  analyse("neardet", "DDActivity1", query="( (file_name like %ddactivity1%) or (file_name like %DDActivity1%) )", write="%s/raw2root_pkls/neardet_activity_raw2root.pkl"%os.environ["NOVAPRODVALID_DATA"])
120 ################### ND numi
121 if options.sample == "neardet_numi":
122  print "run : --- ND NuMI"
123  analyse("neardet", 0, write="%s/raw2root_pkls/neardet_numi_raw2root.pkl"%os.environ["NOVAPRODVALID_DATA"])
124 ################### FD cosmics
125 if options.sample == "fardet_cosmics":
126  print "run : --- FD cosmics"
127  analyse("fardet", 2, partition=1, write="%s/raw2root_pkls/fardet_cosmics_raw2root.pkl"%os.environ["NOVAPRODVALID_DATA"])
128 ################### FD numi
129 if options.sample == "fardet_numi":
130  print "run : --- FD NuMI"
131  analyse("fardet", 0, partition=1, write="%s/raw2root_pkls/fardet_numi_raw2root.pkl"%os.environ["NOVAPRODVALID_DATA"])
procfile open("FD_BRL_v0.txt")
def analyse(detector, stream, query=False, partition=False, write=False)
functions