Functions
build_tools Namespace Reference

Functions

def oname (build_dir, filename)
 
def merge_all (source_dir, build_dir)
 
def merge_jobs (source_dir, build_dir, prefix)
 
def merge_extraps (source_dir, build_dir)
 

Detailed Description

Useful functions for merging grid job output root files

...using `hadd_cafana`.

Function Documentation

def build_tools.merge_all (   source_dir,
  build_dir 
)
Correctly merge all output files in a single directory

...to end up with a fully merged file for all extrapolations, for a
single systematics group/directory.

Args:
    source_dir (string): Path to directory containing output root
        files
    build_dir (string): Path to build directory for merged root
        files

Definition at line 29 of file build_tools.py.

References novadaq::HexUtils.format(), merge_extraps(), and merge_jobs().

Referenced by build_template.compile(), and build_test.compile().

29 def merge_all(source_dir, build_dir):
30  """Correctly merge all output files in a single directory
31 
32  ...to end up with a fully merged file for all extrapolations, for a
33  single systematics group/directory.
34 
35  Args:
36  source_dir (string): Path to directory containing output root
37  files
38  build_dir (string): Path to build directory for merged root
39  files
40  """
41  print 'Merging files in: {:s}'.format(source_dir)
42 
43  # Merge no extrap first.
44  print 'Merging nxp'
45  merge_jobs(source_dir, build_dir, 'pred_nxp_')
46 
47  # Merge xp combo
48  print 'Merging xp_combo'
49  merge_jobs(source_dir, build_dir, 'pred_xp_combo_')
50 
51  # Merge xp prop
52  print 'Merging xp_prop'
53  merge_jobs(source_dir, build_dir, 'pred_xp_prop_')
54 
55  # Merge xp numu
56  print 'Merging xp_numu'
57  merge_jobs(source_dir, build_dir, 'pred_xp_numu_')
58 
59  # Merge xp numu
60  print 'Merging extraps'
61  merge_extraps(build_dir, build_dir)
62 
63 
def merge_jobs(source_dir, build_dir, prefix)
Definition: build_tools.py:64
def merge_extraps(source_dir, build_dir)
Definition: build_tools.py:91
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def merge_all(source_dir, build_dir)
Definition: build_tools.py:29
def build_tools.merge_extraps (   source_dir,
  build_dir 
)
Correctly merge over each extrapolation root file

...in a single directory, to end up with a merged file containing
all extrapolations for a single systematics group/directory.

Args:
    source_dir (string): Path to directory containing job output
        root files
    build_dir (string): Path to build directory for merged root
        file

Definition at line 91 of file build_tools.py.

References novadaq::HexUtils.format(), get_files(), and oname().

Referenced by merge_all().

91 def merge_extraps(source_dir, build_dir):
92  """Correctly merge over each extrapolation root file
93 
94  ...in a single directory, to end up with a merged file containing
95  all extrapolations for a single systematics group/directory.
96 
97  Args:
98  source_dir (string): Path to directory containing job output
99  root files
100  build_dir (string): Path to build directory for merged root
101  file
102  """
103  # Get list of root files in build_dir
104  root_files = get_files(source_dir, '.root')
105 
106  # Check if merged extraps already exist
107  existing_stems = []
108  for file in root_files:
109  if 'allxp' in file: # get stem for built targets
110  print "Found built allxp target: "
111  print file
112  parts = file.split('.')
113  existing_stem = parts[0]
114  existing_stem = existing_stem.replace('pred_allxp_', '')
115  existing_stems.append(existing_stem)
116 
117  files_to_merge = []
118  print 'Creating merge list'
119  for file in root_files:
120  merge_file = True
121  for existing_stem in existing_stems:
122  if existing_stem in file:
123  print 'Not merging file {:s}'.format(file)
124  merge_file = False
125  if merge_file:
126  print 'Merging file {:s}'.format(file)
127  files_to_merge.append(file)
128 
129  if not files_to_merge: # no files left to hadd
130  return
131 
132  # Get stem and handle exit if allxp target already exists
133  stem = ''
134  for file in files_to_merge:
135  parts = file.split('.') # split at .root to get rid of ext.
136  stem = parts[0]
137  # Now find stem by looping through extrap_strings
138  for extrap_string in extrap_strings:
139  prefix = 'pred_' + extrap_string + '_'
140  if prefix in file:
141  stem = stem.replace(prefix, '')
142  print "Found stem: "
143  print " --> {:s}".format(stem)
144 
145  # No full path in files_to_merge
146  inputs = [source_dir + '/' + file for file in files_to_merge]
147  filename = 'pred_allxp_' + stem + '.root'
148 
149  # Only merge if inputs exist:
150  if inputs:
151  run('hadd_cafana', oname(build_dir, filename), *inputs)
152  else:
153  print 'Found no files for {:s} --> skipping!'.format(stem)
154 
std::vector< std::string > get_files(std::string file_list)
def oname(build_dir, filename)
Definition: build_tools.py:13
def merge_extraps(source_dir, build_dir)
Definition: build_tools.py:91
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
Definition: run.py:1
def build_tools.merge_jobs (   source_dir,
  build_dir,
  prefix 
)
Correctly merge all job output root files in a single directory

...to end up with a merged file for a single extrapolations, for a
single systematics group/directory.

Args:
    source_dir (string): Path to directory containing job output
        root files
    build_dir (string): Path to build directory for merged root
        file

Definition at line 64 of file build_tools.py.

References check_jobs.check_dir(), novadaq::HexUtils.format(), and oname().

Referenced by merge_all().

64 def merge_jobs(source_dir, build_dir, prefix):
65  """Correctly merge all job output root files in a single directory
66 
67  ...to end up with a merged file for a single extrapolations, for a
68  single systematics group/directory.
69 
70  Args:
71  source_dir (string): Path to directory containing job output
72  root files
73  build_dir (string): Path to build directory for merged root
74  file
75  """
76  # Check source_dir
77  (total, log_files, complete_jobs, incomplete_jobs, root_files, stem,
78  successful_jobs, failed_jobs) = check_dir(source_dir, prefix)
79 
80  # No full path in root_files
81  inputs = [source_dir + '/' + file for file in root_files]
82  filename = prefix + stem + '.root'
83 
84  # Only merge if inputs exist:
85  if inputs:
86  run('hadd_cafana', oname(build_dir, filename), *inputs)
87  else:
88  print 'Found no files for {:s} --> skipping!'.format(prefix)
89 
90 
def check_dir(output_dir, prefix='')
Definition: check_jobs.py:127
def merge_jobs(source_dir, build_dir, prefix)
Definition: build_tools.py:64
def oname(build_dir, filename)
Definition: build_tools.py:13
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
Definition: run.py:1
def build_tools.oname (   build_dir,
  filename 
)
Define output file path

Function to help fabricate link inputs from source_dir with output
merged files it then creates in build_dir.

Args:
    build_dir (string): Path to build directory.
    filename (string): Output filename.

Returns:
    string: Full path to output file.

Definition at line 13 of file build_tools.py.

Referenced by merge_extraps(), and merge_jobs().

13 def oname(build_dir, filename):
14  """Define output file path
15 
16  Function to help fabricate link inputs from source_dir with output
17  merged files it then creates in build_dir.
18 
19  Args:
20  build_dir (string): Path to build directory.
21  filename (string): Output filename.
22 
23  Returns:
24  string: Full path to output file.
25  """
26  return os.path.join(build_dir, os.path.basename(filename))
27 
28 
def oname(build_dir, filename)
Definition: build_tools.py:13