2 models/Organizational.py: 3 Models related to plot organization used in the validation. 5 Original author: J. Wolcott <jwolcott@fnal.gov> 17 """ Central configuration for Validation run. """ 26 "plot_accept_filters": [],
27 "plot_reject_filters": [],
28 "static_files_url":
None,
32 "force_2d_comp":
False,
36 def FromYAML(yaml_fname):
39 def __init__(self, **kwargs):
41 for kw, val
in kwargs.iteritems():
44 self.folders = Configuration._ParseFolders(val)
45 elif kw
in Configuration.MANDATORY
or kw
in Configuration.DEFAULTS:
46 setattr(self, kw, val)
48 raise KeyError(
"Unknown configuration parameter '%s' with value: %s" % (kw, val))
51 for kw, val
in Configuration.DEFAULTS.iteritems():
52 if not hasattr(self, kw):
53 setattr(self, kw, val)
57 self.validation_dir = os.path.join(self.output_dir, self.validation_name)
60 def _ParseFolders(folder_args):
61 return [Folder.FromDict(folder_config)
for folder_config
in folder_args]
65 """ Ensure mandatory attributes were specified, consistency between arguments, etc. """ 67 missing_attrs = [k
for k
in Configuration.MANDATORY
if not hasattr(self, k)]
68 if len(missing_attrs) > 0:
69 raise ValueError(
"Configuration attributes %s are all mandatory (missing: %s)" % (Configuration.MANDATORY, missing_attrs))
71 if len(self.plot_accept_filters) > 0
and len(self.plot_reject_filters) > 0:
72 raise ValueError(
"Can't specify both 'plot_accept_filters' and 'plot_reject_filters' in Configuration")
75 """ Wrapper around plot identifier. Simplifies access to 'group' interface. 77 ID string as input should have the following format: 79 name_string<group1_string><group2_string>[possibly more <group_string>s] 81 where the <group_string>s can either be formatted as 87 {group=group name,cat=category name} 90 sample_plot{nocuts}{group=MC,cat=DIS} 92 Note that if you specify a category using the category-only syntax ('{category_name}'), 93 you must ensure it always occurs in the same sequence in any plots that are intended 94 to be grouped together (i.e., have the same name_string). 97 CATEGORY_PATTERN = re.compile(
r"(\{group=(?P<group>.+?),cat=(?P<group_cat>.+?)\}|\{(?P<cat>.+?)\})")
98 OVERLAY_STRING =
"all_overlay" 99 SUM_STRING =
"all_sum" 103 if isinstance(init, basestring):
106 elif hasattr(init,
"id_str"):
109 elif hasattr(init,
"__contains__")
and all(prop
in init
for prop
in (
"name",
"categories")):
110 id_str = init[
"name"]
111 groups_by_idx = {}
if "groups" not in init
else dict(zip(init[
"groups"].itervalues(), init[
"groups"].iterkeys()))
112 for idx, cat
in enumerate(init[
"categories"]):
113 if idx
in groups_by_idx:
114 id_str +=
"{group=%s,cat=%s}" % (groups_by_idx[idx], cat)
116 id_str +=
"{%s}" % cat
122 ( (
"groups" not in init
and len(self.
groups) == 0)
or \
123 (init[
"groups"] == self.
groups) ), \
124 "Didn't correctly reconstruct groups and categories for PlotID. Given: %s, %s; reconstructed: %s, %s" \
137 """ This ID 'contains' another if: 138 * they share the same 'name' 139 * this ID has at least as long a category list 140 * the categories match up to the end of the shorter list 141 * the group lists contain no incompatible category links. """ 143 if not all( hasattr(other, prop)
for prop
in (
"name",
"categories",
"groups") ):
146 if self.
name != other.name:
149 if len(self.
categories) > len(other.categories):
152 min_length =
min(len(self.
categories), len(other.categories))
153 if self.
categories[:min_length] != other.categories[:min_length]:
158 for key
in set(self.
groups) & set(other.groups):
159 if self.
groups[key] != other.groups[key]:
164 my_reverse_map = {v:k
for k, v
in self.groups.iteritems()}
165 other_reverse_map = {v:k
for k, v
in other.groups.iteritems()}
166 for key
in set(my_reverse_map) & set(other_reverse_map):
167 if my_reverse_map[key] != other_reverse_map[key]:
174 return hasattr(other,
"id_str")
and self.
id_str == other.id_str
177 return "'%s'" % self.
id_str 180 """ Try to understand the ID string. """ 191 splitted = self.id_str.split(
"{", 1)
193 if len(splitted) == 1:
196 remnants =
"{" + splitted[1]
197 for match
in PlotID.CATEGORY_PATTERN.finditer(remnants):
198 if all([match.group(k)
for k
in (
"group",
"group_cat")]):
199 self.categories.append(match.group(
"group_cat"))
201 elif match.group(
"cat"):
202 self.categories.append(match.group(
"cat"))
205 raise ValueError(
"Something bizarre happened in parsing PlotID. Matched string pieces: %s" % match.groupdict())
208 remnants = remnants.replace(match.group(0),
"", 1)
210 if len(remnants) > 0:
211 raise ValueError(
"Couldn't parse plot name. Full string: '%s'; unprocessed leftover text: '%s'" % (self.
id_str, remnants))
217 plot_id = plot_id
if isinstance(plot_id, PlotID)
else PlotID(plot_id)
218 object.__setattr__(self,
"plot_id", plot_id)
219 object.__setattr__(self,
"path", path)
220 object.__setattr__(self,
"data_set", data_set)
223 return hash((self.plot_id.id_str, self.data_set, self.path))
226 """ One Key 'contains' another if one of their PlotIDs contains the other 227 and their other attributes are the same. """ 229 if not all(hasattr(other, attr)
for attr
in (
"plot_id",
"data_set",
"path")):
232 return self.plot_id
in other.plot_id
and \
233 all(getattr(self, prop) == getattr(other, prop)
for prop
in (
"data_set",
"path"))
236 if not all(hasattr(other, attr)
for attr
in (
"plot_id",
"data_set",
"path")):
238 return (self.plot_id, self.data_set, self.path) == (other.plot_id, other.data_set, other.path)
242 return not (self == other)
245 return "%s|%s|%s" % (self.plot_id, self.data_set, self.path)
251 raise TypeError(
"%s is intended to be immutable" % self.__class__)
253 raise TypeError(
"%s is intended to be immutable" % self.__class__)
256 """ Collection of unique plot keys. """ 265 if not isinstance(obj, PlotCollectionKey):
266 raise TypeError(
"%s items must be of type %s, not %s" % (self.__class__.__name__, PlotCollectionKey, type(obj)))
279 if not isinstance(obj, PlotCollectionKey):
280 raise TypeError(
"%s items must be of type %s, not %s" % (self.__class__.__name__, PlotCollectionKey, type(obj)))
285 self._cache.discard(obj)
290 """ Iterate over the collections of plots with unique names """ 296 plot_name = work[0].plot_id.name
297 collection.add([work.pop(0)])
301 for plot_idx
in range(len(work)-1, 0, -1):
302 if work[plot_idx].plot_id.name == plot_name:
303 collection.add(work.pop(plot_idx))
308 """ Lookup collection for plots, keyed by PlotID """ 326 if not isinstance(key, PlotCollectionKey):
327 raise TypeError(
"%s keys must be of type %s, not %s" % (self.__class__.__name__, PlotCollectionKey, type(key)))
333 raise KeyError(
"%s has no such plot: %s" % (self.__class__.__name__, key))
342 if not isinstance(key, PlotCollectionKey):
343 raise TypeError(
"%s keys must be of type %s" % (self.__class__.__name__, PlotCollectionKey))
348 SERIALIZE_FILENAME =
"comparison_registry.pkl" 354 cPickle.dump(self,
open(os.path.join(base_path, ComparisonRegistry.SERIALIZE_FILENAME),
"w"))
358 obj = cPickle.load(
open(file_name))
359 if not isinstance(obj, ComparisonRegistry):
360 raise TypeError(
"Object loaded from alleged ComparisonRegistry file is actually a %s" % type(obj))
366 """ Collect summaries of plots drawn and where they were written in the same place """ 368 SERIALIZE_FILENAME =
"plot_registry.pkl" 378 assert isinstance(plot_key, PlotCollectionKey)
380 self.plots_by_dataset.setdefault(plot_key.data_set,
PlotLibrary())[plot_key] = plot_summary
386 collection = self.plots_by_dataset.setdefault(dataset,
PlotLibrary())
387 collection.update(plot_summaries)
390 cPickle.dump(self,
open(os.path.join(base_path, PlotRegistry.SERIALIZE_FILENAME),
"w"))
394 obj = cPickle.load(
open(file_name))
395 if not isinstance(obj, PlotRegistry):
396 raise TypeError(
"Object loaded from alleged PlotRegistry file is actually a %s" % type(obj))
400 """ Simple container for creating structure in plot collections. """ 427 assert "name" in config_dict,
"Folder configuration must specify 'name'" 429 for attr
in config_dict:
430 assert attr
in Folder.ALLOWED_ATTRS,
"Unrecognized attribute in folder configuration: '%s'" % attr
432 args = {attr: config_dict[attr]
for attr
in Folder.ALLOWED_ATTRS
if attr
in config_dict}
436 def __init__(self, name, plot_ids=[], subfolders=[]):
439 assert hasattr(plot_ids,
"__iter__"),
"Invalid configuration supplied for Folder plot entries: '%s'" % plot_ids
442 assert hasattr(subfolders,
"__iter__"),
"Invalid configuration supplied for Folder subfolders: '%s'" % subfolders
444 for subfolder
in subfolders:
445 if isinstance(subfolder, Folder):
446 self.subfolders.append(subfolder)
447 elif hasattr(subfolder,
"__contains__"):
448 self.subfolders.append(Folder.FromDict(subfolder))
def __contains__(self, obj)
def __setattr__(self, attr, val)
def __contains__(self, other)
def FromDict(config_dict)
def __contains__(self, other)
def Serialize(self, base_path)
def AddDatasetPlots(self, dataset, plot_summaries)
def __delitem__(self, key)
def __init__(self, name, plot_ids=[], subfolders=[])
def Deserialize(file_name)
def __delattr__(self, attr)
def __getitem__(self, key)
procfile open("FD_BRL_v0.txt")
static float min(const float a, const float b, const float c)
def Serialize(self, base_path)
def Deserialize(file_name)
def __setitem__(self, key, value)
def __contains__(self, key)
def AddComparisonPlot(self, plot_key, plot_summary)
def iter_plot_collections(self)
addons
char name[SIZE_OF_OBJ_NAME]
def AddPlot(self, plot_key, plot_summary)
def __init__(self, plot_id, data_set, path="")