pclistAnalyser.py
Go to the documentation of this file.
1 print "run : --- PC-list analyser"
2 ########################### Imports
3 import samweb_client
4 sam = samweb_client.SAMWebClient(experiment="nova")
5 import cPickle
6 import ProductionTestTools as tools
7 import os
8 import json as JSON
9 ########################## Check that production validation is setup
10 assert "NOVAPRODVALID_DIR" in os.environ.keys(),"NOVAPRODVALID_DIR not set, try running setup_prod_valid.sh."
11 ########################### Option parser
12 from optparse import OptionParser
13 parser = OptionParser()
14 parser.add_option("-s", "--sample", help="sample to study", action="store", type=str, dest="sample", default=False)
15 (options, args) = parser.parse_args()
16 print "run : --- Options"
17 print "run : study: ",options.sample
18 assert options.sample,"No sample provided (use -s)"
19 all_samples = ["neardet_cosmics", "neardet_ddactivity", "fardet_cosmics"]
20 assert options.sample in all_samples, "Non-configured sample requested: %s, options are: %s"%(options.sample,repr(all_samples))
21 ########################### functions
22 def add_to_children(release,run,subrun,json):
23  for this_run in json:
24  if this_run["run"] != run: continue
25  for child in this_run["children"]:
26  if child["release"] != release: continue
27  child["subruns"] += 1
28  return
29  this_run["children"].append({"release":release,"subruns":1})
30  return
31 
32 def analyse(detector, stream, query=False, partition=False, write=False):
33  artdaq_releases = []
34  files = []
35  release_to_files_map = {}
36 
37  artdaq_query = "data_tier artdaq and file_type importedDetector and online.detector %s"%detector
38  if type(stream) == int: artdaq_query += " and online.stream %i"%stream
39  if type(partition) == int: artdaq_query += " and online.partition %i"%partition
40  if query: artdaq_query += " and %s"%query
41  print "run : using query: %s"%artdaq_query
42  artdaq_files = sam.listFiles(dimensions=artdaq_query)
43  artdaq_files.sort()
44  n_kids = 0
45  print "run : found %i artdaq files"%len(artdaq_files)
46  for i,artdaq_file in enumerate(artdaq_files):
47  if i%100 == 0: print "run : file: [%i/%i]"%(i,len(artdaq_files))
48  # if i>200: break
49  artdaq_release = tools.findRelease(artdaq_file)
50  if artdaq_release not in artdaq_releases:
51  artdaq_releases.append(artdaq_release)
52  files.append({"artdaq_release":artdaq_release,"releases":["artdaq"],"runs":[],"files":[]})
53  release_to_files_map[artdaq_release] = len(artdaq_releases)-1
54 
55  this_json = files[release_to_files_map[artdaq_release]]
56 
57  run = tools.findRun(artdaq_file)
58  subrun = tools.findSubRun(artdaq_file)
59  if run not in this_json["runs"]:
60  this_json["runs"].append(run)
61  this_json["files"].append( {"run":run,
62  "children": [{"release":"artdaq","subruns":0}]} )
63 
64  # add artdaq file to children
65  add_to_children("artdaq",run,subrun,this_json["files"])
66 
67  # explicitly check all the children of this file
68  kids = sam.getFileLineage("children",artdaq_file)
69  # print "run : has %i children"%len(kids)
70  for kid in kids:
71  if ".pclist." not in kid["file_name"]: continue
72  # print "run : pclist child:",kid["file_name"]
73  n_kids +=1
74  try:
75  release = tools.findRelease(kid["file_name"])
76  except:
77  print "run : - cannot deterimine release for file: ",file_name
78  continue
79  if release not in this_json["releases"]:
80  print "run : - new pclist release found: %s"%release
81  this_json["releases"].append(release)
82  add_to_children(release,run,subrun,this_json["files"])
83 
84  files[release_to_files_map[artdaq_release]] = this_json
85  # run_subrun_to_list_map[(run,subrun)] = i
86  # print artdaq_releases
87  # for i,f in enumerate(files):
88  # print "file[%i]"%i
89  # for key in f.keys():
90  # print " %s: %s"%(key,repr(f[key]))
91 
92  # print "run : found %i kids, over %i runs, in %i releases"%(n_kids,len(runs),len(releases))
93  if write:
94  print "run : writing results as %s"%write
95  to_write = {}
96  to_write["artdaq_releases"] = artdaq_releases
97  to_write["files"] = files
98  cPickle.dump(to_write,open(write,"wb"))
99  with open(write, 'w') as outfile:
100  JSON.dump(to_write, outfile)
101  # return to_return, releases
102 ########################### Run
103 ################### ND cosmics
104 if options.sample == "neardet_cosmics":
105  print "run : --- ND cosmics"
106  analyse("neardet", 2, write="%s/pclist_json/neardet_cosmics_pclist.json"%os.environ["NOVAPRODVALID_DATA"])
107 ################### ND activity
108 if options.sample == "neardet_ddactivity":
109  print "run : --- ND DD-activity"
110  analyse("neardet", "ddactivity", query="( (file_name like %ddactivity1%) or (file_name like %DDActivity1%) )", write="%s/pclist_json/neardet_ddactivity_pclist.json"%os.environ["NOVAPRODVALID_DATA"])
111 ################### FD cosmics
112 if options.sample == "fardet_cosmics":
113  print "run : --- FD cosmics"
114  analyse("fardet", 2, partition=1, write="%s/pclist_json/fardet_cosmics_pclist.json"%os.environ["NOVAPRODVALID_DATA"])
void append()
Definition: append.C:24
def add_to_children(release, run, subrun, json)
functions
def analyse(detector, stream, query=False, partition=False, write=False)
procfile open("FD_BRL_v0.txt")