GetFEBsFromWatchList.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 """
3  The following code takes in a .csv file from HWWatchList and outputs a txt
4  file with DCM, DB, and FEB printed (space-delimited)
5 
6  CONTACT: Justin Vasel (jvasel@indiana.edu)
7 """
8 
9 # SETTINGS
10 OUTPUT_DIR = './'
11 WATCHFILES_ARCHIVE_DIR = '/nusoft/app/web/htdoc/nova/datacheck/nearline/HWwatchFiles/Archive/'
12 
13 # INCLUDES
14 import csv
15 import sys
16 import itertools
17 import glob
18 import os
19 import collections
20 
21 # ARGUMENTS
22 if len(sys.argv) == 4:
23  detector = sys.argv[1]
24  nfiles = int(sys.argv[2])
25  output = sys.argv[3]
26 elif len(sys.argv) == 5:
27  detector = sys.argv[1]
28  nfiles = int(sys.argv[2])
29  startdate = sys.argv[3]
30  output = sys.argv[4]
31 else:
32  print("Arguments error!")
33 
34 
35 
36 ###############################################################################
37 # Adds FEBs from file to master list
39 
40  global startdate
41  # Get folder list and sort them by date created
42  files = glob.glob(WATCHFILES_ARCHIVE_DIR + detector + "/*")
43  files.sort(key=os.path.getmtime, reverse=True)
44 
45  fileList = []
46  dayOffset = 0;
47 
48  # If user specified a start date...
49  if 'startdate' in globals():
50  # Convert date string to object we can work with
51  date = map(int, startdate.split("-"))
52  year = date[0]
53  month = date[1]
54  day = date[2]
55 
56  while dayOffset <= 31:
57  startdate = "%s-%02d-%02d"%(year, month, day)
58 
59  # Figure out where in the list their date is
60  startIndex = [i for i,s in enumerate(files) if startdate in s]
61 
62  # If we found one folder in the list that matches the date...
63  if len(startIndex) == 1:
64  # Number of positions to rotate the list
65  nRotations = len(files) - startIndex[0]
66  # Recast `files` as a deque object
67  files = collections.deque(files)
68  files.rotate(nRotations)
69  break
70 
71  else:
72  dayOffset += 1
73  print("File for day %s-%02d-%02d not found. Reducing day by 1 and trying again..."%(year, month, day))
74  if day > 1:
75  day -= 1
76  if day == 1:
77  month -= 1
78  day = 31
79  else:
80  print("Found file for %s-%02d-%02d; using this instead of supplied date."%(year, month, day))
81  break
82 
83  for file in range(0, nfiles):
84  fileList.append(files[file])
85 
86  return fileList
87 
88 ###############################################################################
89 # Takes the CSV file and splits the header and data into separate
90 # elements
92 
93  # Instantiate both lists
94  fullList = []
95  headers = []
96 
97  # TODO: The following code gets the job done and is fairly robust, but could
98  # likely be made more efficient. Look into this.
99 
100  # Loop over rows from the CSV file
101  for row in reader:
102  # This IF lets us skip the first row, which only contains run numbers
103  if (len(row) > 2):
104  # Check to see if 'DCM' is an element of this row. If so, this row
105  # is the header, so we set a flag.
106  for position, item in enumerate(row):
107  if item == 'DB':
108  isHeader = 1
109  # If 'isHeader' flag is set, the header is equal to this row
110  if (isHeader):
111  headers = row
112  # reset the 'isHeader' flag to zero; there is only one header
113  isHeader = 0
114  else:
115  # All non-header rows should be appended to fullList
116  fullList.append(row)
117 
118  # Now that we've split up the header from the actual data, we can couple
119  # them back together, but in a more sensible way using object notation
120  data = []
121 
122  # Loop over each line in 'fullList'
123  for line in fullList:
124  # Instantiate a dictionary (key-value pairs)
125  dict = {}
126  # Loop over each item in the line
127  for item in range(len(line)):
128  # Set the dictionary key (from header) and value (from data)
129  dict[headers[item]] = line[item]
130  # Append the dictionary to the big list (each line has its own dict)
131  data.append(dict)
132 
133  # Return the header and the data as separate entities
134  return data
135 
136 ###############################################################################
137 # Adds FEBs from file to master list
138 def buildFEBList(data):
139  global listFromAllFiles
140  global issueThreshold
141  global dropoutThreshold
142 
143  for line in data:
144  if (float(line['issue rate (score)']) >= issueThreshold or float(line['dropout rate']) >= dropoutThreshold):
145  listFromAllFiles.append([int(line['DB']), int(line['DCM']), int(line['FEB'])])
146 
147  return listFromAllFiles
148 
149 
150 ###############################################################################
151 # Takes a master list of lists and removes duplicates from master list
152 def removeDuplicates(aList):
153  checked = []
154  for e in aList:
155  if e not in checked:
156  checked.append(e)
157  return checked
158 
159 
160 ###############################################################################
161 # Write the entries of interest to a file
162 def writeTextFile(aList):
163  global detector
164 
165  # Open the output text file for writing
166  textFile = open(output, 'w')
167 
168  # Loop over lines in the data (data[1] is actual data, data[0] is header)
169  for line in aList:
170  # Write the first three entries of each line (DB, DCM, FEB) to file
171  textFile.write('%d %d %d \n' % (line[0], line[1], line[2]))
172 
173  # Close the file after writing
174  textFile.close()
175 
176 
177 ###############################################################################
178 # MAIN FUNCTION
179 ###############################################################################
180 def main():
181  # These are the issue rate thresholds to make it onto the list
182  farDetIssueThreshold = 3
183  farDetDropoutThreshold = 1
184  nearDetIssueThreshold = 0
185  nearDetDropoutThreshold = 0
186 
187  global issueThreshold
188  global dropoutThreshold
189 
190  if (detector == "FarDet"):
191  issueThreshold = farDetIssueThreshold
192  dropoutThreshold = farDetDropoutThreshold
193  elif (detector == "NearDet"):
194  issueThreshold = nearDetIssueThreshold
195  dropoutThreshold = nearDetDropoutThreshold
196  else:
197  issueThreshold = 0
198  dropoutThreshold = 0
199 
200  global listFromAllFiles
201  listFromAllFiles = []
202 
203  # Get list of recent files
204  fileList = getRecentFilesList()
205 
206  for file in range(0, nfiles):
207 
208  thisFile = fileList[file] + "/" + detector + "HardwareWatchlist.csv"
209 
210  # Print info
211  print("Processing file: %s"%thisFile)
212 
213  # Open the file
214  f = open(thisFile, 'rt')
215 
216  # If successful print the rows
217  try:
218  reader = csv.reader(f)
219  data = convertCsvToPythonObject(reader)
220 
221  # Close the file when done
222  finally:
223  f.close()
224 
225  # Construct the list that will be written to file
226  listFromAllFiles = buildFEBList(data)
227 
228  # Remove duplicate entries from list
229  listFromAllFiles = removeDuplicates(listFromAllFiles)
230 
231  # Write the output text file
232  writeTextFile(listFromAllFiles)
233 
234  # Print info
235  print("Wrote %d lines to %s"%(len(listFromAllFiles), output))
236 
237 # Prevent execution on import
238 if __name__ == '__main__':
239  main()
240 
241 ###############################################################################
242 ###############################################################################
243 ###############################################################################
244 ###############################################################################
245 
def removeDuplicates(aList)
Takes a master list of lists and removes duplicates from master list.
def main()
MAIN FUNCTION.
def writeTextFile(aList)
Write the entries of interest to a file.
def convertCsvToPythonObject(reader)
Takes the CSV file and splits the header and data into separate elements.
bool print
procfile open("FD_BRL_v0.txt")
def buildFEBList(data)
Adds FEBs from file to master list.
def getRecentFilesList()
Adds FEBs from file to master list.