gen_hdf5record.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 # This script is intended to be compatible with both python2 and python3
4 from __future__ import print_function
5 
6 from pygccxml import *
7 
8 import os
9 import sys
10 
11 # Types that we can assume are already defined, so don't form part of our
12 # dependency tree.
13 fundamental_types = ['int', 'float', 'double', 'bool', 'unsigned int',
14  'short', 'short int', 'short unsigned int',
15  'long', 'long unsigned int',
16  'long long int', 'unsigned char',
17  'size_t']
18 
19 # These are written for every table already
20 reserved = ['run', 'subrun', 'cycle', 'evt', 'subevt']
21 
22 enums = []
23 
24 def is_fundamental(type):
25  if type in fundamental_types: return True
26  if type in enums: return True
27  if type[:5] == 'caf::': return is_fundamental(type[5:])
28  return False
29 
30 
31 def translate_type(type):
32  if type in enums or type == 'bool': return 'unsigned short'
33  if type[:5] == 'caf::': return translate_type(type[5:])
34  return type
35 
36 
37 def is_vector(type):
38  return type[:12] == 'std::vector<'
39 
40 
41 def vector_contents(type):
42  assert is_vector(type)
43  return type[12:type.find(',')]
44 
45 
46 def is_array(type):
47  return '[' in type
48 
49 
50 def array_type(type):
51  assert is_array(type)
52  # There always seems to be a space before the dimensions
53  return type[:type.find('[')-1]
54 
55 
56 def array_size(type):
57  assert is_array(type)
58  return int(type[type.find('[')+1:-1])
59 
60 
62  assert not is_fundamental(type)
63  assert not is_vector(type)
64  assert not is_array(type)
65 
66  if type[:5] == 'caf::': return type_to_hdf5_type(type[5:])
67 
68  if type == 'StandardRecord': return 'HDF5Record'
69 
70  return 'HDF5'+type[2:]
71 
72 
73 def vetoed(type):
74  if type == 'std::string': return True
75  return False
76 
77 def base_class(klass):
78  assert len(klass.bases) < 2, 'Support for multiple base classes not implemented'
79  if len(klass.bases) == 1:
80  return klass.bases[0].related_class
81  return None
82 
83 # Recurse to find member variables including all base classes
85  vars = []
86  for v in list(klass.variables()):
87  if vetoed(str(v.decl_type)):
88  sys.stderr.write('Skipping '+str(v.decl_type)+' '+v.name+'\n')
89  else:
90  vars += [v]
91  base = base_class(klass)
92  if base: vars += variables_inc_bases(base)
93  return vars
94 
95 
96 if len(sys.argv) < 1 or len(sys.argv) > 3:
97  print('Usage: gen_hdf5record.py [/path/to/header/outputs/] [/path/to/cxx/outputs/]')
98  sys.exit(1)
99 
100 headerDir = os.getcwd()
101 if len(sys.argv) >= 2: headerDir = sys.argv[1]
102 cxxDir = os.getcwd()
103 if len(sys.argv) >= 3: cxxDir = sys.argv[2]
104 
105 # Locate the castxml executable
106 generator_path, generator_name = utils.find_xml_generator()
107 
108 # Figure out where our source files are
109 priv = os.environ['SRT_PRIVATE_CONTEXT'] if 'SRT_PRIVATE_CONTEXT' in os.environ else None
110 pub = os.environ['SRT_PUBLIC_CONTEXT'] if 'SRT_PUBLIC_CONTEXT' in os.environ else None
111 
112 # For cmake build
113 if not pub: pub = os.environ['NOVASOFT_DIR']
114 
115 context = priv if os.path.exists(priv+'/StandardRecord/') else pub
116 
117 path = []
118 if priv: path += [priv]
119 path += [pub]
120 path += [os.environ['ROOT_INC']]
121 
122 config = parser.xml_generator_configuration_t(
123  xml_generator_path=generator_path,
124  xml_generator=generator_name,
125  include_paths=path,
126  cflags='-std=c++1z -Wno-unknown-warning-option'#,
127 # start_with_declarations='caf::StandardRecord'
128  )
129 
130 decls = parser.parse([context+'/StandardRecord/StandardRecord.h'],
131  config)
132 
133 global_namespace = declarations.get_global_namespace(decls)
134 ns = global_namespace.namespace('caf')
135 
136 enums += [e.name for e in ns.enumerations()]
137 
138 # Keep track of which classes we've written out so far, for purposes of
139 # dependency tracking.
140 emitted = []
141 
142 
143 disclaimer = '''// This file was auto-generated by gen_hdf5record.py.
144 // DO NOT EDIT IT DIRECTLY.
145 // For documentation of the fields see the regular StandardRecord.h'''
146 
147 funcs = '''inline std::string Join(const std::string& a, const std::string& b)
148 {
149  if(a.empty()) return b;
150  return a+"."+b;
151 }
152 
153 typedef std::initializer_list<hep_hpc::hdf5::PropertyList> plist_init;
154 
155 /// Helper for make_cols() - specialized for regular fields
156 template<class T> hep_hpc::hdf5::Column<T>
157 make_col(const plist_init& cprop,
158  const std::string& name,
159  std::enable_if_t<!std::is_array<T>::value, void*> dummy = 0)
160 {
161  return hep_hpc::hdf5::make_scalar_column<T>(name, cprop);
162 }
163 
164 /// Helper for make_cols() - specialized for arrays
165 template<class T> hep_hpc::hdf5::Column<std::remove_extent_t<T>>
166 make_col(const plist_init& cprop,
167  const std::string& name,
168  std::enable_if_t<std::is_array<T>::value, void*> dummy = 0)
169 {
170  return hep_hpc::hdf5::make_column<std::remove_extent_t<T>>(name, std::extent<T>::value, cprop);
171 }
172 
173 /// Return a tuple of hdf5 columns, with the given names and types
174 template<class... Types, class... Names> static auto
175 make_cols(const plist_init& cprop,
176  Names... names)
177 {
178  static_assert(sizeof...(Names) == sizeof...(Types),
179  "Must specify same number of column types and column names");
180  return std::make_tuple(make_col<Types>(cprop, names)...);
181 }
182 
183 /// Wrap a basic datatype up as if it's a class with a single member
184 template<class W, class... Tidx> class Wrapped
185 {
186 public:
187  template<class... Cols> Wrapped(hid_t f, const std::string& name, const plist_init& cprop, Cols... cols)
188  : vals(f, name, make_cols<Tidx..., W>(cprop, cols..., "value"))
189  {
190  }
191 
192  void Fill(Tidx... idxs, W x) {vals.insert(idxs..., x);}
193 
194  hep_hpc::hdf5::Ntuple<Tidx..., W> vals;
195 };
196 '''
197 
198 # From this point on everything we print goes to HDF5Record.h. Remember to send
199 # messages for the user to stderr.
200 sys.stdout = open(headerDir+'/HDF5Record.h', 'w')
201 
202 print(disclaimer)
203 print()
204 print('#pragma once')
205 print()
206 print('#include "StandardRecord/StandardRecord.h"')
207 print()
208 print('#include "hep_hpc/hdf5/File.hpp"')
209 print('#include "hep_hpc/hdf5/Ntuple.hpp"')
210 print('#include "hep_hpc/hdf5/PropertyList.hpp"')
211 print('#include "hep_hpc/hdf5/make_column.hpp"')
212 print()
213 print('namespace hdf5')
214 print('{')
215 print()
216 print(funcs)
217 
218 debug = False
219 
220 def deps_emitted(klass):
221  pt = type_to_hdf5_type(klass.name)
222  base = base_class(klass)
223  if base: base = type_to_hdf5_type(base.name)
224 
225  if base and base not in emitted:
226  if debug: sys.stderr.write('Skipping '+pt+' because of '+base+'\n')
227  return False
228 
229  for v in variables_inc_bases(klass):
230  if is_fundamental(str(v.decl_type)):
231  continue
232  elif is_vector(str(v.decl_type)):
233  type = vector_contents(str(v.decl_type))
234  if is_fundamental(type): continue
235  type = type_to_hdf5_type(type)
236  elif is_array(str(v.decl_type)):
237  assert is_fundamental(array_type(str(v.decl_type)))
238  continue
239  elif type_to_hdf5_type(str(v.decl_type)) in ['HDF5Vector3D', 'HDF5LorentzVector']:
240  continue
241  else:
242  type = type_to_hdf5_type(str(v.decl_type))
243 
244  if type not in emitted:
245  if debug: sys.stderr.write('Skipping '+pt+' because of '+type+'\n')
246  return False
247 
248  return True
249 
250 
251 def write_hd5record_h(klass):
252  base = base_class(klass)
253  if base: base = type_to_hdf5_type(base.name)
254 
255  print('/// HDF5 encoding of \\ref', klass.name)
256 # if base:
257 # print 'class', pt+': public', base
258 # else:
259  print('template<class... Tidx> class', pt)
260 
261  print('{')
262  print('public:')
263  print(' template<class... Cols> '+pt+'(hid_t f, const std::string& name, const plist_init& cprop, Cols... cols);')
264  print()
265  print(' void Fill(Tidx... idxs, const caf::'+klass.name+'& sr);')
266  print()
267 
268  types = []
269  names = []
270  for v in variables_inc_bases(klass):
271  type = str(v.decl_type)
272 
273  if is_fundamental(type):
274  if v.name not in reserved:
275  types += [translate_type(type)]
276  names += [v.name]
277  elif is_vector(type):
278  inner_type = vector_contents(str(v.decl_type))
279  if not is_fundamental(inner_type):
280  print(' std::unique_ptr<'+type_to_hdf5_type(inner_type)+'<Tidx..., unsigned int>>', v.name+';')
281  else:
282  print(' std::unique_ptr<Wrapped<'+inner_type+', Tidx..., unsigned int>>', v.name+';')
283  elif is_array(type):
284  inner_type = array_type(type)
285  size = array_size(type)
286  assert is_fundamental(inner_type)
287  types += [inner_type]
288  names += [v.name]
289  elif type_to_hdf5_type(type) == 'HDF5Vector3D':
290  types += ['float', 'float', 'float']
291  names += [v.name+'.x', v.name+'.y', v.name+'.z']
292  elif type_to_hdf5_type(type) == 'HDF5LorentzVector':
293  types += ['float', 'float', 'float', 'float']
294  names += [v.name+'.E', v.name+'.px', v.name+'.py', v.name+'.pz']
295  else:
296  print(' std::unique_ptr<'+type_to_hdf5_type(str(v.decl_type))+'<Tidx...>>', v.name+';')
297 
298  if len(types) > 0:
299  print(' // '+', '.join(names))
300  print(' hep_hpc::hdf5::Ntuple<Tidx..., '+', '.join(types)+'> vals;')
301 
302  print('};');
303  print()
304 
305  if debug: sys.stderr.write('Wrote '+pt+'\n')
306 
307 
308 while True:
309  anyWritten = False
310  anySkipped = False
311 
312  for klass in ns.classes():
313  pt = type_to_hdf5_type(klass.name)
314  if pt in emitted: continue # Wrote this one already
315  if pt == 'HDF5Vector3D' or pt == 'HDF5LorentzVector': continue # no need, inlined where used
316 
317  if not deps_emitted(klass):
318  # Unmet dependencies, come back to it
319  anySkipped = True
320  continue
321 
322  write_hd5record_h(klass)
323 
324  anyWritten = True
325  emitted += [pt]
326  emitted += ['VectorProxy<'+pt+'>']
327 
328  if not anySkipped: break # We're done
329  if anyWritten: continue # Try for some more
330 
331  if not debug:
332  # Go round one more time to provide feedback
333  debug = True
334  else:
335  sys.stderr.write('Unable to meet all dependencies\n')
336  sys.exit(1)
337 
338 print('} // end namespace')
339 
340 
341 # And now we're writing to HDF5Record.cxx
342 sys.stdout = open(cxxDir+'/HDF5Record.cxx', 'w')
343 
344 print(disclaimer)
345 print()
346 print('#include "HDF5Maker/HDF5Record.h"')
347 print()
348 print('#include <tuple>')
349 print()
350 print('namespace hdf5\n{')
351 
352 # No need to specifically order the functions in the cxx file
353 for klass in ns.classes():
354  pt = type_to_hdf5_type(klass.name)
355  if pt == 'HDF5Vector3D' or pt == 'HDF5LorentzVector': continue # no need, inlined where used
356 
357  # Constructor
358  print('template<class... Tidx> template<class... Cols>',pt+'<Tidx...>::'+pt+'(hid_t f, const std::string& name, const plist_init& cprop, Cols... cols)')
359  # Initializer list
360  inits = []
361 # if base_class(klass): inits += [type_to_hdf5_type(base_class(klass).name)+'(tr, name)']
362  types = []
363  names = []
364  for v in variables_inc_bases(klass):
365  if is_vector(str(v.decl_type)):
366  inner_type = vector_contents(str(v.decl_type))
367  if is_fundamental(inner_type):
368  inits += [v.name + '(std::make_unique<Wrapped<'+(inner_type)+', Tidx..., unsigned int>>(f, Join(name, "'+v.name+'"), cprop, cols..., Join(name, "'+v.name+'")+"_idx"))']
369  else:
370  inits += [v.name + '(std::make_unique<'+type_to_hdf5_type(inner_type)+'<Tidx..., unsigned int>>(f, Join(name, "'+v.name+'"), cprop, cols..., Join(name, "'+v.name+'")+"_idx"))']
371  elif is_array(str(v.decl_type)):
372  inner_type = array_type(str(v.decl_type))
373  size = array_size(str(v.decl_type))
374  assert is_fundamental(inner_type)
375  types += [inner_type+'['+str(size)+']']
376  names += [v.name]
377  elif is_fundamental(str(v.decl_type)):
378  if v.name not in reserved:
379  types += [translate_type(str(v.decl_type))]
380  names += [v.name]
381  elif type_to_hdf5_type(str(v.decl_type)) == 'HDF5Vector3D':
382  types += ['float', 'float', 'float']
383  names += [v.name+'.x', v.name+'.y', v.name+'.z']
384  elif type_to_hdf5_type(str(v.decl_type)) == 'HDF5LorentzVector':
385  types += ['float', 'float', 'float', 'float']
386  names += [v.name+'.E', v.name+'.px', v.name+'.py', v.name+'.pz']
387  else:
388  inits += [v.name + '(std::make_unique<'+type_to_hdf5_type(str(v.decl_type))+'<Tidx...>>(f, Join(name, "'+v.name+'"), cprop, cols...))']
389 
390  if len(names) > 0:
391  inits += ['vals(f, name, make_cols<Tidx..., '+(', '.join(types))+'>(cprop, cols..., '+', '.join(['"'+n+'"' for n in names])+'))']
392 
393  if len(inits) > 0:
394  print(' : '+',\n '.join(inits))
395  print('{\n}\n')
396 
397  print('template<class... Tidx> void '+pt+'<Tidx...>::Fill(Tidx... idxs, const caf::'+klass.name+'& sr)')
398  print('{')
399  for v in variables_inc_bases(klass):
400  if is_vector(str(v.decl_type)):
401  inner_type = vector_contents(str(v.decl_type))
402  print(' for(unsigned int i = 0; i < sr.'+v.name+'.size(); ++i) '+v.name+'->Fill(idxs..., i, sr.'+v.name+'[i]);')
403  elif not is_array(str(v.decl_type)) and not is_fundamental(str(v.decl_type)) and type_to_hdf5_type(str(v.decl_type)) not in ['HDF5Vector3D', 'HDF5LorentzVector']:
404  print(' '+v.name+'->Fill(idxs..., sr.'+v.name+');')
405 
406  if len(names) > 0:
407  print(' vals.insert(idxs..., '+', '.join(['sr.'+n for n in names])+');')
408  print('}')
409  print()
410 
411 print('// Instantiate the ones we\'ll use')
412 print('template class HDF5Record<unsigned int, unsigned int, int, unsigned int, unsigned short>;')
413 print('template HDF5Record<unsigned int, unsigned int, int, unsigned int, unsigned short>::HDF5Record(hid_t f, const std::string& name, const plist_init& cprop, const char*, const char*, const char*, const char*, const char*);')
414 print()
415 print('template class HDF5Spill<unsigned int, unsigned int, unsigned int>;')
416 print('template HDF5Spill<unsigned int, unsigned int, unsigned int>::HDF5Spill(hid_t f, const std::string& name, const plist_init& cprop, const char*, const char*, const char*);')
417 print()
418 print('template class HDF5Neutrino<unsigned int, unsigned int, int, unsigned int>;')
419 print('template HDF5Neutrino<unsigned int, unsigned int, int, unsigned int>::HDF5Neutrino(hid_t f, const std::string& name, const plist_init& cprop, const char*, const char*, const char*, const char*);')
420 print()
421 print('} // namespace')
422 
423 
424 sys.stderr.write('Wrote HDF5Record.h and HDF5Record.cxx\n')
def base_class(klass)
def array_type(type)
def vetoed(type)
def is_vector(type)
def is_fundamental(type)
def type_to_hdf5_type(type)
def translate_type(type)
def vector_contents(type)
bool print
def array_size(type)
procfile open("FD_BRL_v0.txt")
def is_array(type)
def variables_inc_bases(klass)
def write_hd5record_h(klass)
def deps_emitted(klass)