9 print "\n\033[91mtools: --- ERROR" 17 print "\n\033[93mtool: --- WARNING" 25 if "JobsubJobId of first job" in l:
26 return_id = l.split(
":")[1].strip()
27 if return_id ==
False:
28 print "tools: getJobID failed. Given log:" 29 for l
in log:
print l.strip()
36 if "Station monitor will be " in l:
37 return_id = l.split(
"/")[-1].strip()
38 if return_id ==
False:
39 print "tools: getSAMURL failed. Given log:" 40 for l
in log:
print l.strip()
47 if "Station monitor will be " in l:
48 return_id = l.split(
"Station monitor will be ")[-1].strip()
49 if return_id ==
False:
50 print "tools: getStationMonitor failed. Given log:" 51 for l
in log:
print l.strip()
59 t = os.path.getmtime(filename)
60 return datetime.datetime.fromtimestamp(t)
63 def fetchLogs(job_id, run_datetime=False, min_caching_age_days=10, verbose=False):
70 if (
"SETUP_JOBSUB_CLIENT" not in os.environ):
71 print "tools: Jobsub client has not been setup. Cannot fetch logs without the tools to do this" 72 print "tools: Aborting\n" 75 if job_id ==
False:
return []
76 if (
not run_datetime): run_datetime = datetime.datetime.now()
78 log_dir =
"%s/jobsub_logs/%s"%(os.environ[
"NOVAANAVALID_GRID"],job_id)
79 if verbose:
print "tools: fetching logs to dir: %s"%log_dir
82 if os.path.exists(log_dir):
83 if verbose:
print "tools: - log directory exists" 85 now = datetime.datetime.now()
86 d_t = datetime.datetime.now() - run_datetime
87 if verbose:
print "tools: run datetime: %s, now: %s, days: %i"%(run_datetime,now,d_t.days)
88 if d_t.days < min_caching_age_days:
89 if verbose:
print "tools: re-fetching logs" 90 rm_command =
"rm %s/*"%log_dir
93 if verbose:
print "tools: using existing logs" 97 fetch_command =
"jobsub_fetchlog -G nova --job=%s --unzipdir=%s"%(job_id,log_dir)
98 if "novapro" in getpass.getuser(): fetch_command+=
" --role=Production" 100 process = subprocess.Popen(fetch_command.split(
" "), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
101 while process.poll() ==
None: time.sleep(0.1)
102 if process.returncode != 0:
103 print "tools : --- Exception caught while executing fetch log, return code: %i"%process.returncode
104 print "tools : command run: %s"%fetch_command
105 for line
in process.stdout.readlines():
109 file_list = glob.glob(
"%s/*"%log_dir)
114 if "glob" not in dir():
import glob
116 output_dir = component.output_dir
118 output_dir = output_dir.replace(os.environ[
"NOVAANAVALID_GRID"],os.environ[
"NOVAANAVALID_GRID_TEST"])
120 if hasattr(component,
"sam")
and component.sam:
121 file_name =
"%s/hadd_%s_%s.root"%(component.output_dir, component.name, component.version)
122 if os.path.exists(file_name):
123 root_files = [file_name]
127 root_files = glob.glob(
"%s/*.root"%output_dir)
129 print "tools: Found %i files"%len(root_files)
130 for root_file
in root_files:
131 print "tools: - %s"%root_file
135 def openPlots(root_file_name, skip_2d=False, directories=False, verbose=False):
136 from ROOT
import TFile
138 if verbose:
print "tools: Opening: %s"%root_file_name
139 open_root_file = TFile.Open( root_file_name )
140 assert open_root_file.IsOpen()
142 folder_list = open_root_file.GetListOfKeys()
144 if verbose:
print "tools: Found %i keys"%len(folder_list)
145 for folder
in folder_list:
146 folder_name = folder.GetName()
147 directory = open_root_file.GetDirectory(folder_name)
148 histogram_list = directory.GetListOfKeys()
149 histogram_list.sort()
150 for histogram
in histogram_list:
151 name = histogram.GetName()
152 master[
"%s_%s"%(folder_name,name)] = directory.Get(name)
153 assert master[
"%s_%s"%(folder_name,name)],
"tools: Failure opening file: %s, histogram: %s/%s"%(root_name,folder_name,name)
155 histogram_list = open_root_file.GetListOfKeys()
156 histogram_list.sort()
157 print "run : Found %i keys"%len(histogram_list)
158 for histogram
in histogram_list:
159 name = histogram.GetName()
160 if (skip_2d
and (
"_vs_" in name)):
continue 161 master[name] = open_root_file.Get(name)
162 assert master[name],
"tools: Failure opening file: %s, histogram: %s"%(root_name,name)
163 return master, open_root_file
168 if "/" in name: name = name.split(
"/")[-1]
169 if ".png" in name: name = name[:-4]
172 if len(name.split(
"-")) < 2:
return False 175 if category
not in naming_schema.categories.keys():
return False 178 if observable
not in naming_schema.observables.keys():
return False 181 if reco_level
not in naming_schema.reco_levels.keys():
return False 184 if true_level
not in naming_schema.true_levels.keys():
return False 190 if "/" in name: name = name.split(
"/")[-1]
191 if ".png" in name: name = name[:-4]
192 return name.split(
"-")[0]
196 if "/" in name: name = name.split(
"/")[-1]
197 if ".png" in name: name = name[:-4]
198 return name.split(
"-")[1]
203 if "/" in name: name = name.split(
"/")[-1]
204 if ".png" in name: name = name[:-4]
205 name = name.split(
"-")
211 if name[2]
in naming_schema.true_levels.keys():
return "" 212 if not (name[2]
in naming_schema.reco_levels.keys()):
213 fail(
"tools: unknown reco level found:%s (from: %s)"%(name[2],name))
217 if not (name[2]
in naming_schema.reco_levels.keys()):
218 fail(
"tools: unknown reco level found: %s (from: %s)"%(name[2],name))
226 if "/" in name: name = name.split(
"/")[-1]
227 if ".png" in name: name = name[:-4]
228 name = name.split(
"-")
234 if name[2]
in naming_schema.reco_levels.keys():
return "" 235 if not (name[2]
in naming_schema.true_levels.keys()):
236 fail(
"tools: unknown true level found: %s (from: %s)"%(name[2],name))
240 if not (name[3]
in naming_schema.true_levels.keys()):
241 fail(
"tools: unknown true level found: %s (from: %s)"%(name[3],name))
246 def mkdir(directory, verbose=False):
248 if verbose:
print "tools: mkdir: %s"%directory
249 if (
not os.path.exists(directory)):
250 os.system(
"mkdir -p %s"%directory)
254 from PlotStatistics
import PlotStatistics
255 from ROOT
import TH1F, TH1D, TGraph
257 if (type(plots[0])
in [TH1F, TH1D]):
259 stats.normalisations = [(
"area",
"area_norm_"),(
"none",
"")]
261 stats.normalisations = [(
"none",
"")]
263 stats.normalisations = []
264 for i,plot
in enumerate(plots):
265 if type(plot)
in [TGraph]:
266 stats.entries.append(-9)
267 stats.sum_of_weights.append(-9)
269 stats.entries .
append(plot.GetEntries())
270 stats.sum_of_weights.append(plot.GetSumOfWeights())
271 if type(plot)
in [TH1F,TH1D]:
272 stats.means .
append(plot.GetMean())
274 stats.modes .
append(plot.GetXaxis().GetBinCenter(plot.GetMaximumBin()))
276 stats.mean_diffs.append(0)
279 mean_diff = ((plot.GetMean() - stats.means[0]) / stats.means[0]) * 100.
282 stats.mean_diffs.append(mean_diff)
287 stats.mean_diffs.append(-9)
297 from array
import array
298 quantiles =
array(
"d",[(a+1.)/n_bins
for a
in range(n_bins)])
299 locations =
array(
"d",[0
for a
in range(n_bins)])
300 histogram.GetQuantiles(n_bins,locations,quantiles)
305 return directory.replace(
"/nusoft/app/web/htdoc/",
"http://nusoft.fnal.gov/")
308 def findMinMax(histograms,ignore_zero=False,include_errors=True,verbose=False):
311 for i,d
in enumerate(histograms):
313 start = d.GetXaxis().GetFirst()
314 for bin
in range(start, d.GetXaxis().GetLast()+1):
315 entries = d.GetBinContent(bin)
317 if include_errors: error = d.GetBinError(bin)
319 if (ignore_zero
and entries ==0):
continue 320 if ((entries)
and ((entries-error) < this_min)):
321 this_min = entries-error
322 if verbose:
print 'tools: new minium minimum %.2f, from %s, bin[%i]'%(this_min, name, bin)
323 if ((entries)
and ((entries+error) > this_max)):
324 this_max = entries+error
325 if verbose:
print 'tools: new maximum maximum %.2f, from %s, bin[%i]'%(this_max, name, bin)
326 if ignore_zero
and this_min == 0:
329 if this_min == 10e15:
333 return this_min, this_max
344 plot_type =
"%s-%s"%(category,observable)
345 if (plot_type
not in plot_types): plot_types.append(plot_type)
347 if (reco_level
not in reco_levels)
and (category
not in [
"meta"]): reco_levels.append(reco_level)
349 if (true_level
not in true_levels)
and (category
not in [
"meta"]): true_levels.append(true_level)
351 print "view : found %i plot types, %i reco-levels and %i true-levels"%(len(plot_types),len(reco_levels),len(true_levels))
352 return plot_types,reco_levels,true_levels
356 import nameing_schema
357 standard = set(naming_schema.standard_truth_components)
358 return set(levels) == standard
365 if observable
in observables:
continue 366 observables.append(observable)
375 plot_type =
"%s-%s"%(category,observable)
376 if (plot_type
not in plot_types): plot_types.append(plot_type)
382 print "vtool: --- find Max Skew" 384 max_skew = plots[0].GetSkewness()
385 for i,plot
in enumerate(plots):
386 this_skew = plot.GetSkewness()
387 if abs(this_skew) >
abs(max_skew):
390 print "vtool: [%i] %s: %f"%(i,plot.GetName(),this_skew)
392 if verbose:
print "vtool: Max: %f"%max_skew
398 print "vtool: --- find Max Asymmetry" 401 for i,plot
in enumerate(plots):
403 if abs(this_asym) >
abs(max_asym):
406 print "vtool: [%i] %s: %f"%(i,plot.GetName(),this_asym)
408 if verbose:
print "vtool: Max: %f"%max_asym
413 n_bins = plot.GetNbinsX()
414 low_int = plot.Integral(0,
int(n_bins/2))
415 high_int = plot.Integral(
int(n_bins/2), n_bins)
417 asym = (high_int - low_int) / (low_int + high_int)
418 except ZeroDivisionError:
430 tokens = filter_pattern.split(
":")
431 if len(tokens) == 1: tokens.append(
"*")
432 assert len(tokens) == 2,
"Filter list doesn't match expected pattern (*:*): %s"%filter_pattern
434 if (tokens[0] !=
"*")
and (tokens[0] != name):
return False 435 if (tokens[1] !=
"*")
and (tokens[1] != version):
return False
def observableName(name)
get a histogram observable name