check_jobs.py
Go to the documentation of this file.
1 from os import listdir
2 from os.path import isfile, join
3 
4 
5 def get_files(path, ext, prefix=""):
6  """Return a list of files in a directory.
7 
8  On error returns empty list.
9 
10  Args:
11  path (string): Path to directory containing files.
12  ext (string): File extension to filter list by.
13  prefix (string): Prefix to filename to filter list by.
14  (Actually the prefix string can be anywhere in the filename
15  including at the beginning).
16 
17  Returns:
18  list: Filtered list of filenames in a directory.
19  """
20  try:
21  if prefix:
22  return [file for file in listdir(path)
23  if isfile(join(path, file))
24  and prefix in file and ext in file]
25  else:
26  return [file for file in listdir(path)
27  if isfile(join(path, file)) and ext in file]
28  except OSError as error:
29  print "Handling runtime error"
30  print "Couldnt find directory at path {:s}.".format(path)
31  print error
32  print "Continuing..."
33  print ""
34  print ""
35  return []
36 
37 
38 def get_complete_jobs(log_files):
39  """Get a list of completed jobs.
40 
41  Jobs that finished running regardless of success/failure.
42 
43  Args:
44  log_files (list): list of log file filenames.
45 
46  Returns:
47  int: number of complete jobs.
48  dict: complete jobs, value is job index string e.g. '30_of_50' and key
49  is job index (int).
50  dict: incomplete jobs, value is job index string e.g. '31_of_50' and
51  key is job index (int).
52  """
53  total = 0
54  complete_jobs = {}
55  incomplete_jobs = {}
56 
57  # Make list of completed jobs
58  for file in log_files:
59  # split up filename into its constituent parts
60  parts = file.split(".")
61  counter = parts[4]
62 
63  # Extract idx and total from counter
64  idx = int(counter.split("_")[0])
65  total = int(counter.split("_")[2])
66  if counter in complete_jobs.values():
67  print "warning: potential duplicate for {0:s}".format(counter)
68  else:
69  complete_jobs[idx] = counter
70 
71  # Make a list of incomplete jobs
72  for idx in range(total):
73  counter = "%d_of_%d" % (idx+1, total)
74  if counter not in complete_jobs.values():
75  incomplete_jobs[idx+1] = counter
76 
77  return total, complete_jobs, incomplete_jobs
78 
79 
80 def get_successful_jobs(root_files, complete_jobs):
81  """Get a list of successful jobs
82 
83  Jobs that produced an output root file are considered succesful.
84 
85  Args:
86  root_files (list): list of root file paths
87  complete_jobs (dict): Dictionary indexed by job id and with job
88  strings e.g. "35_of_50" as values.
89 
90  Returns:
91  dict: Successful jobs, indexed by job id and with job strings
92  e.g. "35_of_50" as values.
93  dict: Failed jobs, indexed by job id and with job strings
94  e.g. "35_of_50" as values.
95 
96  Todo:
97  * Considers the jobs successful if at least one root file gets
98  produced so if a job produces more than one root file, e.g.
99  for different extrapolations, but not all the root files
100  are produced, this will still be counted as a success.
101  """
102  successful_jobs = {}
103  failed_jobs = {}
104 
105  # Make list of successful jobs
106  for file in root_files:
107  # split up filename into its constituent parts
108  parts = file.split(".")
109  counter = parts[1]
110 
111  # Extract idx and total from counter
112  idx = int(counter.split("_")[0])
113  total = int(counter.split("_")[2])
114  if counter not in successful_jobs.values():
115  successful_jobs[idx] = counter
116 
117  for counter in complete_jobs.values():
118  idx = int(counter.split("_")[0])
119  if (counter not in successful_jobs.values() and
120  counter not in failed_jobs.values()):
121  failed_jobs[idx] = counter
122 
123  return successful_jobs, failed_jobs
124 
125 
126 # Output directory to check
127 def check_dir(output_dir, prefix=''):
128  """ Check the output files in a directory.
129 
130  Checks all files in a Grid job output directory
131 
132  Args:
133  output_dir (string): Path to output directory to check.
134  prefix (string): Optional prefix to search for in output
135  filenames.
136 
137  Returns:
138  int: Total job outputs found.
139  list: List of log files.
140  dict: Complete jobs, indexed by job id and with job strings
141  e.g. "35_of_50" as values.
142  dict: Incomplete jobs, indexed by job id and with job strings
143  e.g. "35_of_50" as values.
144  list: Filtered list (by prefix) of root files.
145  string: Stem of root file names, ignoring prefix and ".root"
146  extension.
147  dict: Successful jobs, indexed by job id and with job strings
148  e.g. "35_of_50" as values.
149  dict: Failed jobs, indexed by job id and with job strings
150  e.g. "35_of_50" as values.
151  """
152  # Get list of log files
153  # Assumes *only* log files have ".txt" extension in output directory.
154  log_files = get_files(output_dir, ".txt")
155 
156  # Get (in)complete jobs from log file list.
157  total, complete_jobs, incomplete_jobs = get_complete_jobs(log_files)
158  # print total
159  # print complete_jobs
160  # print incomplete_jobs
161 
162  # Get list of output root files
163  root_files = get_files(output_dir, "_{:d}.root".format(total), prefix)
164 
165  # Get stem
166  try:
167  parts = root_files[0].split('.')
168  stem = parts[0].replace(prefix, '')
169  print "Stem: {:s}".format(stem)
170  except IndexError as e:
171  print 'Handling runtime exception {:s}'.format(e)
172  print '--> Cannot determine stem.'
173  stem = ''
174 
175  # Get succesful/failed jobs
176  successful_jobs, failed_jobs = get_successful_jobs(root_files,
177  complete_jobs)
178 
179  return (total, log_files, complete_jobs, incomplete_jobs,
180  root_files, stem, successful_jobs, failed_jobs)
181 
182 
183 def get_dir_status(output_dir, dir_summary={}):
184  """Get the status of a Grid job output directory.
185 
186  Args:
187  output_dir (string): Path to Grid job output directory to
188  check.
189  dir_summary (dict): Existing summary dictionary to append with
190  status of this directory.
191 
192  Returns:
193  bool: status of directory (True=all files exist, False=missing
194  files)
195  dict: Summary of directory status, including (in)complete and
196  succesful/failed jobs.
197  """
198  # Check directory
199  (total, log_files, complete_jobs, incomplete_jobs,
200  root_files, stem, successful_jobs, failed_jobs) = check_dir(output_dir)
201 
202  # Get success/failure status for directory
203  status = ((len(incomplete_jobs) == 0) and (len(failed_jobs) == 0))
204 
205  # Add to summary
206  dir_summary["total"] = total
207  dir_summary["output_dir"] = output_dir
208  dir_summary["complete"] = complete_jobs
209  dir_summary["incomplete"] = incomplete_jobs
210  dir_summary["succesful"] = successful_jobs
211  dir_summary["failed"] = failed_jobs
212  dir_summary["status"] = status
213 
214  return status, dir_summary
void split(double tt, double *fr)
def get_files(path, ext, prefix="")
Definition: check_jobs.py:5
def check_dir(output_dir, prefix='')
Definition: check_jobs.py:127
def get_successful_jobs(root_files, complete_jobs)
Definition: check_jobs.py:80
def get_complete_jobs(log_files)
Definition: check_jobs.py:38
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def get_dir_status(output_dir, dir_summary={})
Definition: check_jobs.py:183