NERDEvaluate.cxx
Go to the documentation of this file.
1 ////////////////////////////////////////////////////////////////////////////////////////////
2 /// \file NERDEvaluate.cxx
3 /// \brief Molds the pixel maps into the format expected then runs the tensorflow model
4 /// \author Micah Groh, grohmc@fnal.gov
5 ////////////////////////////////////////////////////////////////////////////////////////////
6 
8 
9 // Suppress a warning intrinsic to the opencv header
10 #pragma GCC diagnostic push
11 #pragma GCC diagnostic ignored "-Woverloaded-virtual"
12 #include <opencv2/opencv.hpp>
13 #pragma GCC diagnostic pop
14 
15 namespace nerd
16 {
18  : fParams(params)
19  {
20  std::cout<<"Loading model from "<<fParams.ModelName()<<std::endl;
22 
23  load_meta();
24  load_anchors();
25  }
26 
28  {
29  if(fTF) delete fTF;
30  }
31 
33  {
34  std::vector<unsigned char> obj_map = pm.PixelMapToObjVector(0);
35  std::vector<unsigned char> lab_map = pm.PixelMapToObjVector(1);
36 
37  int max = 0;
38  for( unsigned int i = 0; i < obj_map.size(); ++i ){
39  if( (int)obj_map[i] > max ) max = (int)obj_map[i];
40  }
41 
42  std::vector<std::vector<bool>> maskX(obj_map.size()/2,
43  std::vector<bool>(max, 0));
44  std::vector<std::vector<bool>> maskY(obj_map.size()/2,
45  std::vector<bool>(max, 0));
46  std::vector<std::vector<float>> labelX(max,
47  std::vector<float>(8,0));
48  std::vector<std::vector<float>> labelY(max,
49  std::vector<float>(8,0));
50 
51  for(unsigned int i = 0; i < obj_map.size(); ++i){
52  // Hack view switching while I think about
53  // how to invert GlobalToIndexSingle
54  if(obj_map[i] == 0) continue;
55  if(i < 8000) {
56  maskX[i][ obj_map[i]-1 ] = 1;
57  labelX[ obj_map[i]-1 ][ lab_map[i]-1 ] = 1;
58  }
59  else {
60  maskY[i-8000][ obj_map[i]-1 ] = 1;
61  labelY[ obj_map[i]-1 ][ lab_map[i]-1 ] = 1;
62  }
63  }
64 
65  return nerd::PixelPIDMaps(maskX, maskY, labelX, labelY);
66  }
67 
69  {
70  // { dummyid, original size {3}, new size {3}, window {4}, image scale, active ids {7} }
71  const int meta_length = 19;
72  float image_meta[meta_length] = {0, 255,240,2, 256,256,2, 0,8,255,248, 1, 0,0,0,0,0,0,0 };
73 
74  tensorflow::Tensor metaTensor(tensorflow::DT_FLOAT, {1,meta_length});
75  auto meta_map = metaTensor.tensor<float,2>();
76  for (int i = 0; i < meta_length; ++i) meta_map(0, i) = image_meta[i];
77 
78  fMetaTensor = metaTensor;
79  }
80 
82  {
83  // for 256x256 images
84  tensorflow::Tensor anchorTensor(tensorflow::DT_FLOAT, {1,fParams.NAnchors(),4});
85  auto anchors_map = anchorTensor.tensor<float, 3>();
86 
87  std::ifstream infile((fParams.LibPath()+fParams.AnchorName()).c_str());
88 
89  float a,b,c,d;
90  int i = 0; //line i
91  while (infile >> a >> b >> c >> d) {
92  std::vector<float> nums = {a,b,c,d};
93  for(unsigned int j = 0; j < nums.size(); ++j)
94  anchors_map(0, i, j ) = nums[j];
95  i++;
96  }//end while
97  infile.close();
98 
99  fAnchorTensor = anchorTensor;
100  }
101 
102  std::vector<tensorflow::Tensor> NERDEval::mold_input(cvn::PixelMap pm)
103  {
104  std::vector<unsigned char> pmslice = pm.PixelMapToVector(true);
105 
106  // reshape to 2x100x80
107  std::vector<std::vector<std::vector<unsigned char>>> shaped(2, std::vector<std::vector<unsigned char>>(100, std::vector<unsigned char>(80,0)));
108 
109  for(int i = 0; i < 2; ++i){
110  for(int j = 0; j < 100; ++j){
111  for(int k = 0; k < 80; ++k){
112  shaped[i][j][k] = pmslice[i*100*80+j*80+k];
113  }
114  }
115  }
116 
117  // Initialize the tensors
118  tensorflow::Tensor x_tensor(tensorflow::DT_FLOAT, {1,256,256,2});
119  tensorflow::Tensor y_tensor(tensorflow::DT_FLOAT, {1,256,256,2});
120  auto x_map = x_tensor.tensor<float,4>();
121  auto y_map = y_tensor.tensor<float,4>();
122 
123  // initialize to 0
124  for(int i = 0; i < 256; ++i){
125  for(int j = 0; j < 256; ++j){
126  for(int k = 0; k < 2; ++k){
127  x_map(0,i,j,k) = 0;
128  y_map(0,i,j,k) = 0;
129  }
130  }
131  }
132 
133  // Loop over each element
134  // Cut off downstream edge of pixel map
135 
136  // X-View
137  for(int iPlane = 0; iPlane < 85; ++iPlane){
138  for(int jCell = 0; jCell < 80; ++jCell){
139  // Compute the val
140  unsigned char xval = shaped[0][iPlane][jCell];
141  // Skip if there is no hit
142  if(xval == 0) continue;
143 
144  // We want to triple each hit
145  for(int k = 0; k < 3; ++k){
146  for(int l = 0; l < 3; ++l){
147  // Shift the cells by 8 to pad the sides to 256
148  x_map(0,3*iPlane+k,8+3*jCell+l,0) = xval;
149  x_map(0,3*iPlane+k,8+3*jCell+l,1) = 255;
150  }
151  }
152  }
153  }
154 
155  // Y-View
156  for(int iPlane = 0; iPlane < 85; ++iPlane){
157  for(int jCell = 0; jCell < 80; ++jCell){
158  unsigned char yval = shaped[1][iPlane][jCell];
159  if(yval == 0) continue;
160 
161  for(int k = 0; k < 3; ++k){
162  for(int l = 0; l < 3; ++l){
163  y_map(0,3*iPlane+k,8+3*jCell+l,0) = yval;
164  y_map(0,3*iPlane+k,8+3*jCell+l,1) = 255;
165  }
166  }
167  }
168  }
169 
170  return {x_tensor,y_tensor};
171  }
172 
174  {
175  static int i = 100;
176  return i++;
177  }
178 
179  std::pair<std::vector<std::vector<bool>>,std::vector<std::vector<float>>> NERDEval::run_graph(tensorflow::Tensor input)
180  {
181  std::vector<std::string> outs = {"output_detections", "output_mrcnn_mask"};
182  std::clock_t start = std::clock();
183  std::vector<tensorflow::Tensor> result = fTF->Predict(
184  {{"input_image",input},
185  {"input_image_meta",fMetaTensor},
186  {"input_anchors",fAnchorTensor}},
187  outs);
188  double duration = (std::clock() - start) / (double)CLOCKS_PER_SEC;
189  std::cout<<"Elapsed time: "<<duration<<std::endl;
190 
191  auto detections = result[0].tensor<float, 3>();
192 
193  std::vector<cv::Mat> full_masks;
194  std::vector<std::vector<float>> scores;
195 
196  auto outputMasks = result[1].tensor<float,5>();
197 
198  const int
199  //size1 = result[1].shape().dim_size(1),
200  size2 = result[1].shape().dim_size(2),
201  size3 = result[1].shape().dim_size(3);
202  //size4 = result[1].shape().dim_size(4);
203 
204  for (int iMask = 0; iMask < 100; ++iMask) {
205  if(detections(0,iMask,0) < 0) break;
206 
207  int detectedClass = (int)detections(0, iMask, 4);
208 
209  float bbox[4] = {detections(0, iMask, 0),
210  detections(0, iMask, 1),
211  detections(0, iMask, 2),
212  detections(0, iMask, 3)};
213 
214  for(int i = 0; i < 4; ++i){
215  if(i == 0 || i == 2){
216  bbox[i] = (bbox[i] - fWindow[0]) / (fWindow[2] - fWindow[0]);
217  bbox[i]*=254;
218  }
219  else{
220  bbox[i] = (bbox[i] - fWindow[1]) / (fWindow[3] - fWindow[1]);
221  bbox[i]*=239;
222  }
223  bbox[i] = floor(bbox[i]+0.5);
224  }
225 
226  bbox[2]+=1;
227  bbox[3]+=1;
228 
229  const int maskHeight = bbox[2] - bbox[0];
230  const int maskWidth = bbox[3] - bbox[1];
231  if(maskHeight <= 0 || maskWidth <= 0) continue;
232 
233  cv::Mat detectedMask(cv::Size(size2,size3), CV_32FC1);
234  for(int i = 0; i < size2; ++i){
235  for(int j = 0; j < size3; ++j){
236  detectedMask.at<float>(i,j) = outputMasks(0,iMask,i,j,detectedClass);
237  }
238  }
239 
240  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_detmask.png", detectedMask*255 );
241 
242  cv::Mat scaledMask(maskHeight, maskWidth, CV_32FC1);
243  cv::resize(detectedMask, scaledMask, scaledMask.size(), 0, 0);
244 
245  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_scaledmask.png", scaledMask*255 );
246 
247  cv::Mat fullMask(cv::Size(240,300), CV_32FC1, cv::Scalar(0));
248  scaledMask.copyTo(fullMask(cv::Rect(bbox[1],bbox[0],maskWidth,maskHeight)));
249 
250  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_fullmask.png", fullMask*255);
251 
252  cv::Mat scaledFullMask(cv::Size(80, 100), CV_32FC1);
253  cv::resize(fullMask, scaledFullMask, scaledFullMask.size(), 0, 0);
254 
255  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_scaledfullmask.png", scaledFullMask*255 );
256 
257  cv::Mat binaryFullMask(scaledFullMask.size(), CV_8UC1);
258  cv::threshold(scaledFullMask, binaryFullMask, fParams.MaskThreshold(), 1, cv::THRESH_BINARY);
259 
260  if(fParams.DebugImages()) cv::imwrite(std::to_string(getUniq())+"_binaryfullmask.png", binaryFullMask*255 );
261 
262  full_masks.push_back(binaryFullMask);
263 
264  std::vector<float> thisScores;
265  for(int i = 6; i < 6+fParams.TrainCategories(); ++i) thisScores.push_back(detections(0, iMask, i));
266  scores.push_back(thisScores);
267  } // iMask
268 
269  // Sometimes the network will find the same particle twice
270  // if they get different labels.
271  // Check this by IOU and remove the smaller
272  std::vector<int> good_masks_idx;
273  for(unsigned int i = 0; i < full_masks.size(); ++i){
274 
275  bool isBad = false;
276 
277  for(unsigned int j = 0; j < full_masks.size(); ++j){
278  if(j==i) continue;
279 
280  double maskIntersection = cv::sum( full_masks[i].mul(full_masks[j]) )[0];
281  double maskUnion = cv::sum( ((full_masks[i]+full_masks[j])>0)/255 )[0];
282  double IOU = maskIntersection / maskUnion;
283 
284  if(IOU <= fParams.IOUThreshold()) continue;
285 
286  double nhiti = cv::sum(full_masks[i])[0];
287  double nhitj = cv::sum(full_masks[j])[0];
288 
289  // TODO: What if they are equal?
290  if(nhiti > nhitj)
291  isBad=true;
292  }
293 
294  if(!isBad) good_masks_idx.push_back(i);
295  }
296 
297  // Remove the scores corresponding to bad masks
298  std::vector<std::vector<float>> kScores;
299  for(unsigned int k = 0; k < good_masks_idx.size(); ++k)
300  kScores.push_back(scores[ good_masks_idx[k] ]);
301 
302  // Flatten mask back to 1D pixel map coordinates
303  // TODO: Can this be generalized?
304  std::vector<std::vector<bool>> kMask;
305  for(int i = 0; i < 100; ++i){
306  for(int j = 0; j < 80; ++j){
307  std::vector<bool> temp;
308  for(unsigned int k = 0; k < good_masks_idx.size(); ++k)
309  temp.push_back(full_masks[ good_masks_idx[k] ].at<float>(i,j));
310  kMask.push_back(temp);
311  }
312  }
313 
314  return {kMask, kScores};
315  }
316 
318  {
319  std::cout<<"Molding Input"<<std::endl;
320  // Convert to format expected by the network
321  std::vector<tensorflow::Tensor> input_tensor = mold_input(pm);
322 
323  std::cout<<"Evaluating X View"<<std::endl;
324  auto x_out = run_graph(input_tensor[0]);
325  std::cout<<"Evaluating Y View"<<std::endl;
326  auto y_out = run_graph(input_tensor[1]);
327  std::cout<<"Finished Evaluation"<<std::endl;
328 
329  return PixelPIDMaps(x_out.first, y_out.first, x_out.second, y_out.second);
330  }
331 }
std::vector< unsigned char > PixelMapToObjVector(bool useLabels) const
Definition: PixelMap.cxx:417
NERDEvalParams fParams
Definition: NERDEvaluate.h:44
Defines an enumeration for nerd classification.
std::vector< Tensor > Predict(std::vector< std::pair< std::string, Tensor >> inputs, std::vector< std::string > outputLabels)
Definition: TFHandler.cxx:64
Atom< std::string > ModelName
Definition: NERDEvaluate.h:21
Atom< int > CPUlimit
Definition: NERDEvaluate.h:31
Atom< int > TrainCategories
Definition: NERDEvaluate.h:25
::xsd::cxx::tree::duration< char, simple_type > duration
Definition: Database.h:188
Molds the pixel maps into the format expected then runs the tensorflow model.
Atom< std::string > LibPath
Definition: NERDEvaluate.h:20
string infile
std::vector< unsigned char > PixelMapToVector(bool useGeV) const
Definition: PixelMap.cxx:449
std::pair< std::vector< std::vector< bool > >, std::vector< std::vector< float > > > run_graph(tensorflow::Tensor input)
const double a
void resize(T &x, std::vector< int > dims)
Definition: resize.hpp:41
Float_t d
Definition: plot.C:236
const double j
Definition: BetheBloch.cxx:29
tensorflow::Tensor fAnchorTensor
Definition: NERDEvaluate.h:49
Atom< float > MaskThreshold
Definition: NERDEvaluate.h:26
OStream cout
Definition: OStream.cxx:6
Atom< bool > DebugImages
Definition: NERDEvaluate.h:29
NERDEval(const NERDEvalParams &params)
tensorflow::TFHandler * fTF
Definition: NERDEvaluate.h:46
tensorflow::Tensor fMetaTensor
Definition: NERDEvaluate.h:48
Atom< int > NAnchors
Definition: NERDEvaluate.h:24
Atom< float > IOUThreshold
Definition: NERDEvaluate.h:27
PixelMap, basic input to CVN neural net.
Definition: PixelMap.h:23
fvar< T > floor(const fvar< T > &x)
Definition: floor.hpp:11
const hit & b
Definition: hits.cxx:21
std::string to_string(ModuleType mt)
Definition: ModuleType.h:32
const float fWindow[4]
Definition: NERDEvaluate.h:59
std::vector< tensorflow::Tensor > mold_input(cvn::PixelMap pm)
PixelPIDMaps load_truth(cvn::PixelMap pm)
Double_t sum
Definition: plot.C:31
T max(sqlite3 *const db, std::string const &table_name, std::string const &column_name)
Definition: statistics.h:68
Atom< std::string > AnchorName
Definition: NERDEvaluate.h:22
Wrapper for Tensorflow which handles construction and prediction.
Definition: TFHandler.h:19
PixelPIDMaps predict(cvn::PixelMap pm)