MakePages.py
Go to the documentation of this file.
1 """
2  MakePages.py:
3  Construct HTML validation web pages.
4 
5  Original author: J. Wolcott <jwolcott@fnal.gov>
6  Date: September 2016
7 """
8 
9 import argparse
10 import os.path
11 
12 from controllers.ControllerBase import ControllerBase
13 from models.Organizational import ComparisonRegistry
14 from models.Organizational import Configuration
15 from models.Organizational import PlotRegistry
16 from models.Organizational import PlotLibrary
17 from tools import PathTools
18 from views import Pages
19 
20 class PageMaker(ControllerBase):
21  """ Control object that determines which Pages need to be made
22  for a given output location and writes the HTML returned by
23  Pages to the appropriate files. """
24 
25  def __init__(self, config):
26  super(PageMaker, self).__init__(config)
27 
28  self.comparison_dirname = PathTools.ComparisonSubdirName(config.histogram_topdirs)
29  self.validation_name = config.validation_name
30  self.output_dir = config.output_dir
31  self.static_url = config.static_files_url
32  self.base_url = PathTools.DirectoryToURL(self.output_dir)
33 
34  def MakeAllPages(self):
35  validation_dirs = PathTools.FindValidationSubdirs(self.output_dir)
36 
37  self.MakeFrontPage(validation_dirs)
38 
39  for validation_dir in validation_dirs:
40  # don't re-make pages that aren't part of this config.
41  # just leave them alone.
42  if os.path.basename(validation_dir) != self.validation_name:
43  continue
44 
45  try:
46  plot_registry = PlotRegistry.Deserialize(os.path.join(validation_dir, PlotRegistry.SERIALIZE_FILENAME))
47  except IOError:
48  print " ... subdir '%s' does not have a plot registry. Skipping." % os.path.basename(validation_dir)
49  continue
50  except TypeError as e:
51  print e.message
52  continue
53 
54  try:
55  comparison_registry = ComparisonRegistry.Deserialize(os.path.join(validation_dir, ComparisonRegistry.SERIALIZE_FILENAME))
56  except:
57  # this isn't required to proceed, so just use an empty one if one isn't found
58  comparison_registry = ComparisonRegistry()
59 
60 
61  self.MakeInstancePages(validation_dir, plot_registry, comparison_registry)
62 
63  def MakeFrontPage(self, validation_subdirs):
64  page = Pages.FrontPage(
65  top_url=self.base_url,
66  base_url=self.base_url,
67  static_url=self.static_url,
68  validation_subdirs=validation_subdirs
69  )
70  self.WritePage(page, "index.html")
71 
72  def MakeInstancePages(self, validation_dir, plot_registry, comparison_registry):
73  base_path = os.path.join(self.output_dir, validation_dir)
74  validation_name = os.path.basename(validation_dir)
75 
76  # first make the index page for this instance.
77  # need a list of the current (and any other) datasets
78  datasets = []
79  for d in os.listdir(base_path):
80  if not os.path.isdir(os.path.join(base_path, d)):
81  continue
82  if d == PathTools.DATASET_COMPARISON_DIRNAME:
83  continue
84 
85  datasets.append( Pages.SubdirInfo(d, is_current=d in plot_registry.plots_by_dataset) )
86 
87  datasets.sort()
88 
89  comparisons = []
90  try:
91  comparison_dir = os.path.join(base_path, PathTools.DATASET_COMPARISON_DIRNAME)
92  for d in os.listdir(comparison_dir):
93  if not os.path.isdir(os.path.join(comparison_dir, d)):
94  continue
95 
96  is_current = d == self.comparison_dirname
97  comparisons.append( Pages.SubdirInfo(d, is_current) )
98  except OSError: # comparison directory doesn't (yet) exist
99  pass
100 
101  comparisons.sort(key=lambda comparison: comparison.name) # gotta specify how to sort, otherwise they get sorted by object ID (== memory address)
102 
103  index_page = Pages.ValidationPageIndex(
104  top_url=self.base_url,
105  base_url="%s/%s" % (self.base_url, validation_name),
106  static_url=self.static_url,
107  validation_name=validation_name,
108  dataset_dirs=datasets,
109  comparison_dirs=comparisons
110  )
111  self.WritePage(index_page, os.path.join(validation_dir, "index.html"))
112 
113  # now descend into the directory and make plot pages
114  for dataset_dirinfo in datasets:
115  dataset = dataset_dirinfo.name
116 
117  # don't descend into directories that were found from previous runs, however.
118  if not dataset_dirinfo.is_current:
119  continue
120 
121  plot_tree = PageMaker.BuildPlotTree(plot_registry.plots_by_dataset[dataset], "%s/%s" % (base_path, dataset))
122 
123  for path, plots, subdirs in PageMaker.WalkPlotSubtree(plot_tree):
124  base_url = "/".join(["%s" % bit for bit in (self.base_url, validation_name, dataset)])
125  if path:
126  base_url += "/" + path
127  folder_page = Pages.PlotPage(
128  top_url=self.base_url,
129  base_url=base_url,
130  static_url=self.static_url,
131  plots=plots,
132  subdirs=subdirs,
133  )
134  self.WritePage(folder_page, os.path.join(validation_dir, dataset, path, "index.html"))
135 
136  # and finally, make the dataset comparison pages:
137  plot_tree = PageMaker.BuildPlotTree(plot_registry.dataset_comparisons, "%s/%s/%s" % (base_path, PathTools.DATASET_COMPARISON_DIRNAME, self.comparison_dirname))
138 
139  # ... comparison pages for the plots ...
140  for path, plots, subdirs in PageMaker.WalkPlotSubtree(plot_tree):
141  base_url = "/".join(["%s" % bit for bit in (self.base_url, validation_name, PathTools.DATASET_COMPARISON_DIRNAME, self.comparison_dirname)])
142  if path:
143  base_url += "/" + path
144  comparison_page = Pages.PlotPage(
145  top_url=self.base_url,
146  base_url=base_url,
147  static_url=self.static_url,
148  plots=plots,
149  subdirs=subdirs,
150  comparison_info=comparison_registry.comparisons,
151  )
152  self.WritePage(comparison_page, os.path.join(validation_dir, PathTools.DATASET_COMPARISON_DIRNAME, self.comparison_dirname, path, "index.html"))
153 
154  # ... and the table showing the statistics for all the plots on the same page.
155  if len(comparison_registry.comparisons) > 0:
156  comparison_stat_page = Pages.ComparisonTablePage(
157  top_url=self.base_url,
158  base_url=base_url,
159  static_url=self.static_url,
160  comparison_datasets=sorted(self.config.histogram_topdirs),
161  comparison_info=comparison_registry.comparisons
162  )
163  self.WritePage(comparison_stat_page, os.path.join(validation_dir, PathTools.DATASET_COMPARISON_DIRNAME, self.comparison_dirname, PathTools.STAT_PAGE_NAME + ".html"))
164 
165  @staticmethod
166  def BuildPlotTree(top_collection, base_path):
167  # build the tree.
168  # gotta do this in advance so that the pages get correct forward links and breadcrumbs
169  # (also want to group the 'same' plot together into a group)
170 
171  plot_tree = {}
172  for plot_key, info in top_collection.iteritems():
173  path = os.path.dirname(plot_key.path.replace(base_path, ""))
174 
175  collection = plot_tree
176  workpath = list(reversed(path.strip("/").split("/")))
177  while workpath:
178  bit = workpath.pop()
179  if not bit: # if the plot path was "", workpath will be [""], which means we get "" here
180  break
181 
182  # this is the last folder... remainder is plot
183  collection = collection.setdefault(bit, {})
184 
185  # the plots themselves are at key "" (empty string) inside that level's dictionary.
186  # this way the plots stick out from subdirectories
187  collection.setdefault("", {}).setdefault(plot_key.plot_id.name, PlotLibrary())[plot_key] = info
188 
189  return plot_tree
190 
191  @staticmethod
192  def WalkPlotSubtree(plot_collection, base_folder=""):
193  """ Recursively traverses a plot tree to pick out subcollections
194  of plots all at the same depth.
195 
196  Returns a 3-tuple: (folder path, plots, subdirectory names). """
197 
198  plots = []
199  subdirs = []
200 
201  for dirname, collection in plot_collection.iteritems():
202  if dirname == "":
203  plots = collection
204  else:
205  subdirs.append(dirname)
206  folder_path = "%s/%s" % (base_folder, dirname) if base_folder else dirname
207  for f, ps, ss in PageMaker.WalkPlotSubtree(collection, base_folder=folder_path):
208  yield f, ps, ss
209 
210  if len(plots) == len(subdirs) == 0:
211  raise StopIteration
212 
213  yield (base_folder, plots, subdirs)
214 
215 
216  def WritePage(self, page, file_path):
217  """ Write out the file with page's HTML in it.
218  file_path is assumed to be relative to the top directory. """
219 
220  with open( os.path.join(self.output_dir, file_path) , "w") as out_file:
221  out_file.write( page.GetHTML() )
222  print " Wrote page to file:", out_file.name
223 
224 # bootstrap, load config, go!
225 if __name__ == "__main__":
226  parser = argparse.ArgumentParser(description='Run validation.')
227  parser.add_argument('yaml_config', metavar='yaml_config',
228  help='YAML configuration as specified in README.txt')
229 
230  args = parser.parse_args()
231 
232  config = Configuration.FromYAML(args.yaml_config)
233 
234  page_maker = PageMaker(config)
235 
236  print "Building pages..."
237  page_maker.MakeAllPages()
238  print "... done. Bye."
void split(double tt, double *fr)
def WalkPlotSubtree(plot_collection, base_folder="")
Definition: MakePages.py:192
def WritePage(self, page, file_path)
Definition: MakePages.py:216
def BuildPlotTree(top_collection, base_path)
Definition: MakePages.py:166
def MakeFrontPage(self, validation_subdirs)
Definition: MakePages.py:63
procfile open("FD_BRL_v0.txt")
def MakeInstancePages(self, validation_dir, plot_registry, comparison_registry)
Definition: MakePages.py:72