NERDEvaluate.cxx
Go to the documentation of this file.
1 ////////////////////////////////////////////////////////////////////////////////////////////
2 /// \file NERDEvaluate.cxx
3 /// \brief Molds the pixel maps into the format expected then runs the tensorflow model
4 /// \author Micah Groh, grohmc@fnal.gov
5 ////////////////////////////////////////////////////////////////////////////////////////////
6 
8 
9 #include "boost/algorithm/string.hpp"
10 
11 namespace nerd
12 {
14  : fParams(params)
15  {
16  if(!fParams.UseFIFO()){
19 
20  load_meta();
21  load_anchors();
22  }
23  }
24 
26  {
27  //if(fTF) delete fTF;
28  }
29 
31  {
32  std::vector<unsigned char> obj_map = pm.PixelMapToObjVector(0);
33  std::vector<unsigned char> lab_map = pm.PixelMapToObjVector(1);
34 
35  int max = 0;
36  for( unsigned int i = 0; i < obj_map.size(); ++i ){
37  if( (int)obj_map[i] > max ) max = (int)obj_map[i];
38  }
39 
40  std::vector<std::vector<bool>> maskX(obj_map.size()/2,
41  std::vector<bool>(max, 0));
42  std::vector<std::vector<bool>> maskY(obj_map.size()/2,
43  std::vector<bool>(max, 0));
44  std::vector<std::vector<float>> labelX(max,
45  std::vector<float>(8,0));
46  std::vector<std::vector<float>> labelY(max,
47  std::vector<float>(8,0));
48 
49  for(unsigned int i = 0; i < obj_map.size(); ++i){
50  // Hack view switching while I think about
51  // how to invert GlobalToIndexSingle
52  if(obj_map[i] == 0) continue;
53  if(i < 8000) {
54  maskX[i][ obj_map[i]-1 ] = 1;
55  labelX[ obj_map[i]-1 ][ lab_map[i]-1 ] = 1;
56  }
57  else {
58  maskY[i-8000][ obj_map[i]-1 ] = 1;
59  labelY[ obj_map[i]-1 ][ lab_map[i]-1 ] = 1;
60  }
61  }
62 
63  return nerd::PixelPIDMaps(maskX, maskY, labelX, labelY);
64  }
65 
67  {
68  tensorflow::Tensor metaTensor(tensorflow::DT_FLOAT, {1,fMetaSize});
69  auto meta_map = metaTensor.tensor<float,2>();
70  for (int i = 0; i < fMetaSize; ++i) meta_map(0, i) = fMeta[i];
71 
72  fMetaTensor = metaTensor;
73  }
74 
76  {
77  // for 256x256 images
78  tensorflow::Tensor anchorTensor(tensorflow::DT_FLOAT, {1,fParams.NAnchors(),4});
79  auto anchors_map = anchorTensor.tensor<float, 3>();
80 
82  std::ifstream infile((fLibPath+fParams.AnchorName()).c_str());
83 
84  float a,b,c,d;
85  int i = 0; //line i
86  while (infile >> a >> b >> c >> d) {
87  std::vector<float> nums = {a,b,c,d};
88  for(unsigned int j = 0; j < nums.size(); ++j)
89  anchors_map(0, i, j ) = nums[j];
90  i++;
91  }//end while
92  infile.close();
93 
94  fAnchorTensor = anchorTensor;
95  }
96 
97  std::vector<tensorflow::Tensor> NERDEval::mold_input(cvn::PixelMap pm)
98  {
99  std::vector<unsigned char> pmslice = pm.PixelMapToVector(true);
100 
101  // reshape to 2x100x80
102  std::vector<std::vector<std::vector<unsigned char>>> shaped(2, std::vector<std::vector<unsigned char>>(100, std::vector<unsigned char>(80,0)));
103 
104  for(int i = 0; i < 2; ++i){
105  for(int j = 0; j < 100; ++j){
106  for(int k = 0; k < 80; ++k){
107  shaped[i][j][k] = pmslice[i*100*80+j*80+k];
108  }
109  }
110  }
111 
112  // Initialize the tensors
113  tensorflow::Tensor x_tensor(tensorflow::DT_FLOAT, {1,256,256,2});
114  tensorflow::Tensor y_tensor(tensorflow::DT_FLOAT, {1,256,256,2});
115  auto x_map = x_tensor.tensor<float,4>();
116  auto y_map = y_tensor.tensor<float,4>();
117 
118  // initialize to 0
119  for(int i = 0; i < 256; ++i){
120  for(int j = 0; j < 256; ++j){
121  for(int k = 0; k < 2; ++k){
122  x_map(0,i,j,k) = 0;
123  y_map(0,i,j,k) = 0;
124  }
125  }
126  }
127 
128  // Loop over each element
129  // Cut off downstream edge of pixel map
130 
131  // X-View
132  for(int iPlane = 0; iPlane < 85; ++iPlane){
133  for(int jCell = 0; jCell < 80; ++jCell){
134  // Compute the val
135  unsigned char xval = shaped[0][iPlane][jCell];
136  // Skip if there is no hit
137  if(xval == 0) continue;
138 
139  // We want to triple each hit
140  for(int k = 0; k < 3; ++k){
141  for(int l = 0; l < 3; ++l){
142  // Shift the cells by 8 to pad the sides to 256
143  x_map(0,3*iPlane+k,8+3*jCell+l,0) = xval;
144  x_map(0,3*iPlane+k,8+3*jCell+l,1) = 255;
145  }
146  }
147  }
148  }
149 
150  // Y-View
151  for(int iPlane = 0; iPlane < 85; ++iPlane){
152  for(int jCell = 0; jCell < 80; ++jCell){
153  unsigned char yval = shaped[1][iPlane][jCell];
154  if(yval == 0) continue;
155 
156  for(int k = 0; k < 3; ++k){
157  for(int l = 0; l < 3; ++l){
158  y_map(0,3*iPlane+k,8+3*jCell+l,0) = yval;
159  y_map(0,3*iPlane+k,8+3*jCell+l,1) = 255;
160  }
161  }
162  }
163  }
164 
165  return {x_tensor,y_tensor};
166  }
167 
169  {
170  static int i = 100;
171  return i++;
172  }
173 
174  std::pair<std::vector<std::vector<bool>>,std::vector<std::vector<float>>>
175  NERDEval::run_graph(tensorflow::Tensor input)
176  {
177  std::vector<std::string> outs = {"output_detections", "output_mrcnn_mask"};
178 
179  std::vector<tensorflow::Tensor> result = fTF->Predict(
180  {{"input_image",input},
181  {"input_image_meta",fMetaTensor},
182  {"input_anchors",fAnchorTensor}},
183  outs);
184 
185  auto detections = result[0].tensor<float, 3>();
186 
187  std::vector<cv::Mat> full_masks;
188  std::vector<std::vector<float>> scores;
189 
190  auto outputMasks = result[1].tensor<float,5>();
191 
192  const int
193  //size1 = result[1].shape().dim_size(1),
194  size2 = result[1].shape().dim_size(2),
195  size3 = result[1].shape().dim_size(3);
196  //size4 = result[1].shape().dim_size(4);
197 
198  for (int iMask = 0; iMask < 100; ++iMask) {
199  if(detections(0,iMask,0) < 0) break;
200 
201  int detectedClass = (int)detections(0, iMask, 4);
202 
203  float bbox[4] = {detections(0, iMask, 0),
204  detections(0, iMask, 1),
205  detections(0, iMask, 2),
206  detections(0, iMask, 3)};
207 
208  // change bbox from normalized coordinates to index coordinates.
209  denorm_bbox(bbox);
210 
211  // Construct the initial mask for the predicted class
212  const int maskHeight = bbox[2] - bbox[0];
213  const int maskWidth = bbox[3] - bbox[1];
214  if(maskHeight <= 0 || maskWidth <= 0) continue;
215 
216  cv::Mat detectedMask(cv::Size(size2,size3), CV_32FC1);
217  for(int i = 0; i < size2; ++i){
218  for(int j = 0; j < size3; ++j){
219  detectedMask.at<float>(i,j) = outputMasks(0,iMask,i,j,detectedClass);
220  }
221  }
222 
223  cv::Mat binaryFullMask = unmold_mask(detectedMask, maskHeight, maskWidth, bbox);
224 
225  full_masks.push_back(binaryFullMask);
226 
227  // Grab the scores to carry through the remainder of the algorithm.
228  std::vector<float> thisScores;
229  for(int i = 6; i < 6+fParams.TrainCategories(); ++i) thisScores.push_back(detections(0, iMask, i));
230  scores.push_back(thisScores);
231  } // iMask
232 
233  return remove_overlap(full_masks, scores);
234  }
235 
236  void NERDEval::denorm_bbox(float* bbox)
237  {
238  for(int i = 0; i < 4; ++i){
239  if(i == 0 || i == 2){
240  bbox[i] = (bbox[i] - fWindow[0]) / (fWindow[2] - fWindow[0]);
241  bbox[i] *= 254;
242  }
243  else{
244  bbox[i] = (bbox[i] - fWindow[1]) / (fWindow[3] - fWindow[1]);
245  bbox[i] *= 239;
246  }
247  bbox[i] = floor(bbox[i]+0.5);
248  }
249 
250  bbox[2]+=1;
251  bbox[3]+=1;
252  }
253 
254  cv::Mat NERDEval::unmold_mask(cv::Mat detectedMask, int height, int width, float bbox[4])
255  {
256  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_detmask.png", detectedMask*255 );
257 
258  // Scale mask to real shape
259  cv::Mat scaledMask(height, width, CV_32FC1);
260  cv::resize(detectedMask, scaledMask, scaledMask.size(), 0, 0);
261 
262  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_scaledmask.png", scaledMask*255 );
263 
264  // Copy mask to full image shape
265  cv::Mat fullMask(cv::Size(240,300), CV_32FC1, cv::Scalar(0));
266  scaledMask.copyTo(fullMask(cv::Rect(bbox[1], bbox[0], width, height)));
267 
268  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_fullmask.png", fullMask*255);
269 
270  // Rescale full mask to original pixel map size
271  cv::Mat scaledFullMask(cv::Size(80, 100), CV_32FC1);
272  cv::resize(fullMask, scaledFullMask, scaledFullMask.size(), 0, 0);
273 
274  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_scaledfullmask.png", scaledFullMask*255 );
275 
276  // Apply the score threshold
277  cv::Mat binaryFullMask(scaledFullMask.size(), CV_8UC1);
278  cv::threshold(scaledFullMask, binaryFullMask, fParams.MaskThreshold(), 1, cv::THRESH_BINARY);
279 
280  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_binaryfullmask.png", binaryFullMask*255 );
281 
282  return binaryFullMask;
283  }
284 
285  std::pair<std::vector<std::vector<bool>>,std::vector<std::vector<float>>>
286  NERDEval::remove_overlap(std::vector<cv::Mat> full_masks, std::vector<std::vector<float>> scores)
287  {
288  // Sometimes the network will find the same particle twice
289  // if they get different labels.
290  // Check this by IOU and remove the smaller
291  std::vector<int> good_masks_idx;
292  for(unsigned int i = 0; i < full_masks.size(); ++i){
293 
294  bool isBad = false;
295 
296  for(unsigned int j = 0; j < full_masks.size(); ++j){
297  if(j==i) continue;
298 
299  double maskIntersection = cv::sum( full_masks[i].mul(full_masks[j]) )[0];
300  double maskUnion = cv::sum( ((full_masks[i]+full_masks[j])>0)/255 )[0];
301  double IOU = maskIntersection / maskUnion;
302 
303  if(IOU <= fParams.IOUThreshold()) continue;
304 
305  double nhiti = cv::sum(full_masks[i])[0];
306  double nhitj = cv::sum(full_masks[j])[0];
307 
308  // TODO: What if they are equal?
309  if(nhiti > nhitj)
310  isBad=true;
311  }
312 
313  if(!isBad) good_masks_idx.push_back(i);
314  }
315 
316  // Remove the scores corresponding to bad masks
317  std::vector<std::vector<float>> kScores;
318  for(unsigned int k = 0; k < good_masks_idx.size(); ++k)
319  kScores.push_back(scores[ good_masks_idx[k] ]);
320 
321  // Flatten mask back to 1D pixel map coordinates
322  // TODO: Can this be generalized?
323  std::vector<std::vector<bool>> kMask;
324  for(int i = 0; i < 100; ++i){
325  for(int j = 0; j < 80; ++j){
326  std::vector<bool> temp;
327  for(unsigned int k = 0; k < good_masks_idx.size(); ++k)
328  temp.push_back(full_masks[ good_masks_idx[k] ].at<float>(i,j));
329  kMask.push_back(temp);
330  }
331  }
332 
333  return {kMask, kScores};
334  }
335 
337  {
338  // Open the output pipe
339  std::ofstream totf;
340  totf.open(fParams.PipeToTF(), std::ios::out);
341 
342  // Print the pixelmap to a txt file
343  std::vector<unsigned char> pmvec = pm.PixelMapToVector(true);
344  for(unsigned int i = 0; i < pmvec.size(); ++i)
345  totf << (int)pmvec[i] << " ";
346 
347  totf.close();
348 
349  // Open the pipe
350  std::ifstream fromtf;
351  fromtf.open(fParams.PipeFromTF());
352 
353  // Read results from the file
354 
355  // File is structured as
356  // X
357  // partid maskshape
358  // 28 rows of 28 mask values
359  // ...
360  // repeat for each partid
361  // Y
362  // ...
363 
364  bool isY = false;
365 
366  std::vector<cv::Mat> full_masksX;
367  std::vector<cv::Mat> full_masksY;
368  std::vector<std::vector<float>> scoresX;
369  std::vector<std::vector<float>> scoresY;
370 
372  while (std::getline(fromtf, line)){
373  if(line == "X") continue;
374 
375  if(line == "Y"){
376  isY = true;
377  continue;
378  }
379 
380  std::vector<std::string> infostring;
381  boost::split(infostring, line, boost::is_any_of(" "));
382  //unsigned int index = std::stoi(infostring[0]);
383  unsigned int maskshape = std::stoi(infostring[1]);;
384 
385  // Next line is the dections
386  std::getline(fromtf, line);
387 
388  std::vector<std::string> detectionsstring;
389  std::vector<float> detections;
390  boost::split(detectionsstring, line, boost::is_any_of(" "));
391 
392  // -1 to remove the trailing empty element
393  for(unsigned int i = 0; i < detectionsstring.size()-1; ++i)
394  detections.push_back( std::stof(detectionsstring[i]) );
395 
396  // The detections should have 13 values
397  // bbox {4}, class, max score, all scores {7}
398  if( detections.size() != 13 ) abort();
399 
400  float bbox[4] = {detections[0],
401  detections[1],
402  detections[2],
403  detections[3]};
404 
405  // change bbox from normalized coordinates to index coordinates.
406  denorm_bbox(bbox);
407 
408  // Construct the initial mask for the predicted class
409  const int maskHeight = bbox[2] - bbox[0];
410  const int maskWidth = bbox[3] - bbox[1];
411  if(maskHeight <= 0 || maskWidth <= 0) continue;
412 
413  cv::Mat detectedMask(cv::Size(maskshape, maskshape), CV_32FC1);
414 
415  // Remaining lines are all mask values
416  for(unsigned int i = 0; i < maskshape; ++i){
417  std::getline(fromtf, line);
418 
419  std::vector<std::string> maskstring;
420  boost::split(maskstring, line, boost::is_any_of(" "));
421 
422  if(maskstring.size() != maskshape+1) abort();
423 
424  for(unsigned int j = 0; j < maskshape; ++j)
425  detectedMask.at<float>(i,j) = stof(maskstring[j]);
426 
427  }
428 
429  cv::Mat binaryFullMask = unmold_mask(detectedMask, maskHeight, maskWidth, bbox);
430 
431  if(isY) full_masksY.push_back(binaryFullMask);
432  else full_masksX.push_back(binaryFullMask);
433 
434  // Grab the scores to carry through the remainder of the algorithm.
435  std::vector<float> thisScores;
436  for(int i = 6; i < 6+fParams.TrainCategories(); ++i) thisScores.push_back(detections[i]);
437 
438  if(isY) scoresY.push_back(thisScores);
439  else scoresX.push_back(thisScores);
440  }
441 
442  fromtf.close();
443 
444  auto xinfo = remove_overlap(full_masksX, scoresX);
445  auto yinfo = remove_overlap(full_masksY, scoresY);
446 
447  return PixelPIDMaps(xinfo.first, yinfo.first, xinfo.second, yinfo.second);
448  }
449 
451  {
452  // Skip the use of tensorflow GPU and use FIFO pipes instead
453  if(fParams.UseFIFO()){
454  PixelPIDMaps fifoout = predict_fifo(pm);
455  return fifoout;
456  }
457 
458  // Convert to format expected by the network
459  std::vector<tensorflow::Tensor> input_tensor = mold_input(pm);
460 
461  auto x_out = run_graph(input_tensor[0]);
462  auto y_out = run_graph(input_tensor[1]);
463 
464  PixelPIDMaps out = PixelPIDMaps(x_out.first, y_out.first, x_out.second, y_out.second);
465 
466  //bool same = fifoout == out;
467  //if(same) std::cout<<"Pixel Maps are identical"<<std::endl;
468 
469  return out;
470  }
471 }
void split(double tt, double *fr)
std::vector< unsigned char > PixelMapToObjVector(bool useLabels) const
Definition: PixelMap.cxx:417
static const unsigned short fMetaSize
Definition: NERDEvaluate.h:86
Atom< bool > UseFIFO
Definition: NERDEvaluate.h:33
PixelPIDMaps predict_fifo(cvn::PixelMap pm)
std::string EnvExpansion(const std::string &inString)
Function to expand environment variables.
Definition: EnvExpand.cxx:8
Atom< std::string > PipeToTF
Definition: NERDEvaluate.h:37
NERDEvalParams fParams
Definition: NERDEvaluate.h:63
Defines an enumeration for nerd classification.
std::vector< Tensor > Predict(std::vector< std::pair< std::string, Tensor >> inputs, std::vector< std::string > outputLabels)
Definition: TFHandler.cxx:64
Atom< std::string > ModelName
Definition: NERDEvaluate.h:41
Atom< int > CPUlimit
Definition: NERDEvaluate.h:35
const float fMeta[fMetaSize]
Definition: NERDEvaluate.h:87
Atom< int > TrainCategories
Definition: NERDEvaluate.h:45
cv::Mat unmold_mask(cv::Mat detectedMask, int height, int width, float bbox[4])
Molds the pixel maps into the format expected then runs the tensorflow model.
Atom< std::string > LibPath
Definition: NERDEvaluate.h:40
string infile
std::vector< unsigned char > PixelMapToVector(bool useGeV) const
Definition: PixelMap.cxx:449
std::pair< std::vector< std::vector< bool > >, std::vector< std::vector< float > > > run_graph(tensorflow::Tensor input)
const double a
Float_t d
Definition: plot.C:236
const double j
Definition: BetheBloch.cxx:29
tensorflow::Tensor fAnchorTensor
Definition: NERDEvaluate.h:68
Atom< float > MaskThreshold
Definition: NERDEvaluate.h:46
void denorm_bbox(float *bbox)
Atom< bool > DebugImages
Definition: NERDEvaluate.h:49
NERDEval(const NERDEvalParams &params)
tensorflow::TFHandler * fTF
Definition: NERDEvaluate.h:65
std::pair< std::vector< std::vector< bool > >, std::vector< std::vector< float > > > remove_overlap(std::vector< cv::Mat > full_masks, std::vector< std::vector< float >> scores)
Atom< std::string > PipeFromTF
Definition: NERDEvaluate.h:38
tensorflow::Tensor fMetaTensor
Definition: NERDEvaluate.h:67
Atom< int > NAnchors
Definition: NERDEvaluate.h:44
Atom< float > IOUThreshold
Definition: NERDEvaluate.h:47
PixelMap, basic input to CVN neural net.
Definition: PixelMap.h:23
const hit & b
Definition: hits.cxx:21
const float fWindow[4]
Definition: NERDEvaluate.h:83
std::vector< tensorflow::Tensor > mold_input(cvn::PixelMap pm)
PixelPIDMaps load_truth(cvn::PixelMap pm)
Double_t sum
Definition: plot.C:31
T max(sqlite3 *const db, std::string const &table_name, std::string const &column_name)
Definition: statistics.h:66
Atom< std::string > AnchorName
Definition: NERDEvaluate.h:42
Wrapper for Tensorflow which handles construction and prediction.
Definition: TFHandler.h:19
std::string to_string(ModuleType const mt)
Definition: ModuleType.h:34
PixelPIDMaps predict(cvn::PixelMap pm)
enum BeamMode string