Functions | Variables
GetFEBsFromWatchList Namespace Reference

Functions

def getRecentFilesList ()
 Adds FEBs from file to master list. More...
 
def convertCsvToPythonObject (reader)
 Takes the CSV file and splits the header and data into separate elements. More...
 
def buildFEBList (data)
 Adds FEBs from file to master list. More...
 
def removeDuplicates (aList)
 Takes a master list of lists and removes duplicates from master list. More...
 
def writeTextFile (aList)
 Write the entries of interest to a file. More...
 
def main ()
 MAIN FUNCTION. More...
 

Variables

string OUTPUT_DIR = './'
 
string WATCHFILES_ARCHIVE_DIR = '/nusoft/app/web/htdoc/nova/datacheck/nearline/HWwatchFiles/Archive/'
 
 detector = sys.argv[1]
 
 nfiles = int(sys.argv[2])
 
 output = sys.argv[3]
 
 startdate = sys.argv[3]
 

Detailed Description

    The following code takes in a .csv file from HWWatchList and outputs a txt
    file with DCM, DB, and FEB printed (space-delimited)

    CONTACT: Justin Vasel (jvasel@indiana.edu)

Function Documentation

def GetFEBsFromWatchList.buildFEBList (   data)

Adds FEBs from file to master list.

Definition at line 138 of file GetFEBsFromWatchList.py.

References check_time_usage.float, and makeTrainCVSamples.int.

Referenced by main().

138 def buildFEBList(data):
139  global listFromAllFiles
140  global issueThreshold
141  global dropoutThreshold
142 
143  for line in data:
144  if (float(line['issue rate (score)']) >= issueThreshold or float(line['dropout rate']) >= dropoutThreshold):
145  listFromAllFiles.append([int(line['DB']), int(line['DCM']), int(line['FEB'])])
146 
147  return listFromAllFiles
148 
149 
def buildFEBList(data)
Adds FEBs from file to master list.
def GetFEBsFromWatchList.convertCsvToPythonObject (   reader)

Takes the CSV file and splits the header and data into separate elements.

Definition at line 91 of file GetFEBsFromWatchList.py.

References PandAna.Demos.demo1.range.

Referenced by main().

92 
93  # Instantiate both lists
94  fullList = []
95  headers = []
96 
97  # TODO: The following code gets the job done and is fairly robust, but could
98  # likely be made more efficient. Look into this.
99 
100  # Loop over rows from the CSV file
101  for row in reader:
102  # This IF lets us skip the first row, which only contains run numbers
103  if (len(row) > 2):
104  # Check to see if 'DCM' is an element of this row. If so, this row
105  # is the header, so we set a flag.
106  for position, item in enumerate(row):
107  if item == 'DB':
108  isHeader = 1
109  # If 'isHeader' flag is set, the header is equal to this row
110  if (isHeader):
111  headers = row
112  # reset the 'isHeader' flag to zero; there is only one header
113  isHeader = 0
114  else:
115  # All non-header rows should be appended to fullList
116  fullList.append(row)
117 
118  # Now that we've split up the header from the actual data, we can couple
119  # them back together, but in a more sensible way using object notation
120  data = []
121 
122  # Loop over each line in 'fullList'
123  for line in fullList:
124  # Instantiate a dictionary (key-value pairs)
125  dict = {}
126  # Loop over each item in the line
127  for item in range(len(line)):
128  # Set the dictionary key (from header) and value (from data)
129  dict[headers[item]] = line[item]
130  # Append the dictionary to the big list (each line has its own dict)
131  data.append(dict)
132 
133  # Return the header and the data as separate entities
134  return data
135 
def convertCsvToPythonObject(reader)
Takes the CSV file and splits the header and data into separate elements.
def GetFEBsFromWatchList.getRecentFilesList ( )

Adds FEBs from file to master list.

Definition at line 38 of file GetFEBsFromWatchList.py.

References print, and PandAna.Demos.demo1.range.

Referenced by main().

39 
40  global startdate
41  # Get folder list and sort them by date created
42  files = glob.glob(WATCHFILES_ARCHIVE_DIR + detector + "/*")
43  files.sort(key=os.path.getmtime, reverse=True)
44 
45  fileList = []
46  dayOffset = 0;
47 
48  # If user specified a start date...
49  if 'startdate' in globals():
50  # Convert date string to object we can work with
51  date = map(int, startdate.split("-"))
52  year = date[0]
53  month = date[1]
54  day = date[2]
55 
56  while dayOffset <= 31:
57  startdate = "%s-%02d-%02d"%(year, month, day)
58 
59  # Figure out where in the list their date is
60  startIndex = [i for i,s in enumerate(files) if startdate in s]
61 
62  # If we found one folder in the list that matches the date...
63  if len(startIndex) == 1:
64  # Number of positions to rotate the list
65  nRotations = len(files) - startIndex[0]
66  # Recast `files` as a deque object
67  files = collections.deque(files)
68  files.rotate(nRotations)
69  break
70 
71  else:
72  dayOffset += 1
73  print("File for day %s-%02d-%02d not found. Reducing day by 1 and trying again..."%(year, month, day))
74  if day > 1:
75  day -= 1
76  if day == 1:
77  month -= 1
78  day = 31
79  else:
80  print("Found file for %s-%02d-%02d; using this instead of supplied date."%(year, month, day))
81  break
82 
83  for file in range(0, nfiles):
84  fileList.append(files[file])
85 
86  return fileList
87 
bool print
def getRecentFilesList()
Adds FEBs from file to master list.
def GetFEBsFromWatchList.main ( )

MAIN FUNCTION.

1D plots hits vs w //

Definition at line 180 of file GetFEBsFromWatchList.py.

References buildFEBList(), convertCsvToPythonObject(), getRecentFilesList(), open(), print, PandAna.Demos.demo1.range, removeDuplicates(), and writeTextFile().

180 def main():
181  # These are the issue rate thresholds to make it onto the list
182  farDetIssueThreshold = 3
183  farDetDropoutThreshold = 1
184  nearDetIssueThreshold = 0
185  nearDetDropoutThreshold = 0
186 
187  global issueThreshold
188  global dropoutThreshold
189 
190  if (detector == "FarDet"):
191  issueThreshold = farDetIssueThreshold
192  dropoutThreshold = farDetDropoutThreshold
193  elif (detector == "NearDet"):
194  issueThreshold = nearDetIssueThreshold
195  dropoutThreshold = nearDetDropoutThreshold
196  else:
197  issueThreshold = 0
198  dropoutThreshold = 0
199 
200  global listFromAllFiles
201  listFromAllFiles = []
202 
203  # Get list of recent files
204  fileList = getRecentFilesList()
205 
206  for file in range(0, nfiles):
207 
208  thisFile = fileList[file] + "/" + detector + "HardwareWatchlist.csv"
209 
210  # Print info
211  print("Processing file: %s"%thisFile)
212 
213  # Open the file
214  f = open(thisFile, 'rt')
215 
216  # If successful print the rows
217  try:
218  reader = csv.reader(f)
219  data = convertCsvToPythonObject(reader)
220 
221  # Close the file when done
222  finally:
223  f.close()
224 
225  # Construct the list that will be written to file
226  listFromAllFiles = buildFEBList(data)
227 
228  # Remove duplicate entries from list
229  listFromAllFiles = removeDuplicates(listFromAllFiles)
230 
231  # Write the output text file
232  writeTextFile(listFromAllFiles)
233 
234  # Print info
235  print("Wrote %d lines to %s"%(len(listFromAllFiles), output))
236 
237 # Prevent execution on import
def removeDuplicates(aList)
Takes a master list of lists and removes duplicates from master list.
def main()
MAIN FUNCTION.
def writeTextFile(aList)
Write the entries of interest to a file.
def convertCsvToPythonObject(reader)
Takes the CSV file and splits the header and data into separate elements.
bool print
procfile open("FD_BRL_v0.txt")
def buildFEBList(data)
Adds FEBs from file to master list.
def getRecentFilesList()
Adds FEBs from file to master list.
def GetFEBsFromWatchList.removeDuplicates (   aList)

Takes a master list of lists and removes duplicates from master list.

Definition at line 152 of file GetFEBsFromWatchList.py.

Referenced by main().

152 def removeDuplicates(aList):
153  checked = []
154  for e in aList:
155  if e not in checked:
156  checked.append(e)
157  return checked
158 
159 
def removeDuplicates(aList)
Takes a master list of lists and removes duplicates from master list.
def GetFEBsFromWatchList.writeTextFile (   aList)

Write the entries of interest to a file.

Definition at line 162 of file GetFEBsFromWatchList.py.

References open().

Referenced by main().

162 def writeTextFile(aList):
163  global detector
164 
165  # Open the output text file for writing
166  textFile = open(output, 'w')
167 
168  # Loop over lines in the data (data[1] is actual data, data[0] is header)
169  for line in aList:
170  # Write the first three entries of each line (DB, DCM, FEB) to file
171  textFile.write('%d %d %d \n' % (line[0], line[1], line[2]))
172 
173  # Close the file after writing
174  textFile.close()
175 
176 
def writeTextFile(aList)
Write the entries of interest to a file.
procfile open("FD_BRL_v0.txt")

Variable Documentation

GetFEBsFromWatchList.detector = sys.argv[1]

Definition at line 23 of file GetFEBsFromWatchList.py.

GetFEBsFromWatchList.nfiles = int(sys.argv[2])
GetFEBsFromWatchList.output = sys.argv[3]

Definition at line 25 of file GetFEBsFromWatchList.py.

string GetFEBsFromWatchList.OUTPUT_DIR = './'

Definition at line 10 of file GetFEBsFromWatchList.py.

GetFEBsFromWatchList.startdate = sys.argv[3]

Definition at line 29 of file GetFEBsFromWatchList.py.

string GetFEBsFromWatchList.WATCHFILES_ARCHIVE_DIR = '/nusoft/app/web/htdoc/nova/datacheck/nearline/HWwatchFiles/Archive/'

Definition at line 11 of file GetFEBsFromWatchList.py.