Metadata_module.cc
Go to the documentation of this file.
1 ////////////////////////////////////////////////////////////////////////
2 /// \brief Module to extract and fill file metadata
3 /// \file Metadata_module.cc
4 /// \author Adam Aurisano
5 ///
6 /// Generated at Fri Apr 12 17:18:04 2013 by Adam Aurisano using artmod
7 /// from art v1_02_06.
8 ////////////////////////////////////////////////////////////////////////
9 
10 #include <algorithm>
11 #include <cctype>
12 #include <cstdlib>
13 #include <iterator>
14 #include <map>
15 #include <memory>
16 #include <set>
17 #include <sstream>
18 #include <string>
19 #include <vector>
20 
21 #include "Geometry/Geometry.h"
22 #include "NovaDAQConventions/DAQConventions.h"
24 
33 #include "fhiclcpp/ParameterSet.h"
35 
36 #include "SummaryData/SpillData.h"
37 
38 #include <IFDH_service.h>
39 
40 
41 namespace meta {
42 
43  class Metadata : public art::EDAnalyzer {
44  public:
45  explicit Metadata(const fhicl::ParameterSet& p);
46  virtual ~Metadata();
47 
48  void beginRun(art::Run const & r);
49  void beginSubRun(art::SubRun const & sr);
50  void endRun(art::Run const & r);
51  void analyze(const art::Event& e);
52  void reconfigure(const fhicl::ParameterSet& p);
55 
56  private:
57  void GetInputMetadata (std::string inMetadata);
58  void FillMetadata();
59 
60  int fNEvts;
64 
66 
67  //horn polarity of the run
69  bool fisRHC;
70  bool fis0HC;
71 
72  std::set< int > fRuns;
73  std::set< std::pair<int, int> > fSubRuns;
74  std::vector< std::string > fParents;
75  };
76 }
77 
78 namespace meta {
80  EDAnalyzer (p),
81  fNEvts (0),
82 
83  fRunType (p.get< std::string >("runType", "physics")),
84  fUseSubruns (p.get< bool >("useSubruns", false) ),
85  fUseFhclParent(p.get< bool >("useFhclParent") ),
86 
87  fisRealData (false),
88  fHaveHornInfo (false),
89  fisRHC (false),
90  fis0HC (false)
91 
92  {
94 
95  fhicl::ParameterSet inPSet;
96  bool foundPSet = p.get_if_present<fhicl::ParameterSet>("params", inPSet);
97  if(foundPSet) manager.DoParsing(inPSet);
98 
99  //add a default value for Nova.Subversion
100  //Order of precedence
101  //1) NPASS environment variable
102  //2) fhicl file
103  //3) input file
104  //4) default value
105  manager.AddMetadata( "Nova.SubVersion", "1" );
106 
107  //push fcl (for simulation jobs into the parents vector
108  std::string fclName = manager.GetMetadata()["simulated.jobfcl"];
109  if(fclName != "") fParents.push_back(fclName);
110 
111  if(fUseFhclParent) {
113  std::string oldMetadata;
114  mf::LogInfo("Metadata") << "Parent File name is: " << fclName;
115  try {
116  oldMetadata = ifdhp->getMetadata(fclName);
117  }
118  catch(WebAPIException) {
119  mf::LogError("Metadata") << "Web API Exception: ";
120  oldMetadata = "";
121  }
122  mf::LogInfo("Metadata") << oldMetadata;
123  GetInputMetadata(oldMetadata);
124  }
125  }
126 
127  //.............................................................
129  {
130  }
131 
132  //.............................................................
134  {
135  }
136 
138  {
140 
141  std::istringstream metadataStream;
142  metadataStream.str(inMetadata);
143  while(metadataStream.good()){
145  std::getline(metadataStream, line);
146  size_t pos = line.find(":");
147  if(pos != std::string::npos){
148  key = line.substr(0, pos);
149  value = line.substr(pos+1);
150  size_t dot_pos = key.find(".");
151  //if the key contains a dot, this is experiment specific: copy over
152  if (dot_pos != std::string::npos) manager.AddMetadata(key, value);
153  }
154  }
155  return;
156  }
157 
158  //......................................................................
160  {
161  int run_num = r.run();
162  fRuns.insert( run_num );
163  return;
164  }
165 
166  //......................................................................
168  {
169  int run_num = sr.run();
170  int sub_num = sr.subRun();
171  fSubRuns.insert( std::make_pair( run_num, sub_num ) );
172  return;
173  }
174 
175  //.................................................................
177  {
178  //art::Timestamp evtTime = e.time();
179  //if (fNEvts == 0)
180  //{
181  // fStartTime = evtTime;
182  // fEndTime = evtTime;
183  //}
184  //else
185  //{
186  // if (evtTime < fStartTime) fStartTime = evtTime;
187  // if (evtTime > fEndTime) fEndTime = evtTime;
188  //}
189 
190  fisRealData = e.isRealData();
191  // Check for horn polarity:
192  // * for data, always. (this ensures that IFDB's values are always used
193  // even if erroneous metadata made it into a previous tier, which happens every once in a while.)
194  // * for MC, only if the horn current metadata isn't already set.
195  // (it should be for beam MC, where it's set at simulation stage,
196  // but occasionally the horn current metadata is used for other purposes
197  // in non-beam MC -- e.g., indicating which training of CVN was used
198  // on cosmic MC.)
199  if (fNEvts == 0)
200  {
201  bool checkHorn = false;
202  if (fisRealData)
203  checkHorn = true;
204  else
205  {
207  std::string hornCurr;
208  if (manager.GetMetadata().find("nova.hornpolarity") != manager.GetMetadata().end())
209  hornCurr = manager.GetMetadata().at("nova.hornpolarity");
210  checkHorn = hornCurr.empty();
211  }
212 
213  if (checkHorn)
214  {
216  e.getByLabel("ifdbspillinfo", spilldata);
217 
218  if (spilldata.isValid()) {
219  fHaveHornInfo = true;
220  if(spilldata->isRHC){
221  fisRHC = true;
222  }
223  else if(spilldata->is0HC){
224  fis0HC = true;
225  }
226  }
227  }
228  }
229 
230  ++fNEvts;
231  return;
232  }
233 
234  //....................................................................
236  {
237  //If we're forcing fcl parentage (like overlays)
238  //Don't get the metadata using the input file.
239  if(fUseFhclParent) return;
240 
241  std::string fileWPath = fb.fileName();
242  if (fileWPath == "") return;
244  size_t pos = fileWPath.rfind("/");
245  if (pos != std::string::npos) fileName = fileWPath.substr(pos+1);
246  else fileName = fileWPath;
247 
248  fParents.push_back( fileName );
249 
250  //get old metadata from input file
252  std::string oldMetadata;
253  try{
254  oldMetadata = ifdhp->getMetadata(fileName);
255  }
256  catch(WebAPIException){
257  oldMetadata = "";
258  }
259  GetInputMetadata(oldMetadata);
260 
261  return;
262  }
263 
264  //..................................................................
266  {
267 
268  //if(fUseFhclParent) return;
269  //fhicl file is already parsed - now add in additional metadata
271 
272  std::ostringstream value_stream;
273 
274  std::string detector("unknown");
276  if(geo->DetId() == novadaq::cnv::kFARDET) detector = "fd";
277  else if(geo->DetId() == novadaq::cnv::kNEARDET) detector = "nd";
278  else if(geo->DetId() == novadaq::cnv::kNDOS) detector = "ndos";
279  else if(geo->DetId() == novadaq::cnv::kTESTBEAM) detector = "tb";
280  manager.AddMetadata( "Nova.DetectorID", detector );
281 
282  //set release this stage of processing used (takes precedence over input files)
283  // NOVASOFT_VERSION is set for the ups build, but SRT_BASE_RELEASE is for SRT
284  char* novarelease = std::getenv("NOVASOFT_VERSION");
285  if( novarelease != nullptr)
286  manager.AddMetadata( "Nova.Release", novarelease);
287  else
288  manager.AddMetadata( "Nova.Release", std::getenv("SRT_BASE_RELEASE"));
289 
290  //value_stream.str("");
291  //value_stream << fStartTime.value();
292  //manager.AddMetadata( "start_time", value_stream.str() );
293  //value_stream.str("");
294  //value_stream << fEndTime.value();
295  //manager.AddMetadata( "end_time", value_stream.str() );
296 
297  //things in the high priority metadata have precedence over metadata from the fhicl file
298  //
299  // eg nova.subversion by default is set to 1
300  // if input files have a pass number, that supercedes the default
301  // if NPASS exists as an environment variable,
302  // that supercedes value in input files
303 
304  // Map from environment variable to metadata field name
305  const std::map<const char*, const char*> envMap =
306  {{"NPASS", "Nova.SubVersion"},
307  {"NOVAGRIDUTILS_VERSION", "Production.novagridutils_version"},
308  {"PROCESS", "Production.process"},
309  {"GRID_USER", "Production.grid_user"},
310  {"NOVAPRODUCTION_VERSION", "Production.novaproduction_version"},
311  {"SAM_PROJECT_NAME", "Production.sam_project_name"},
312  {"HOSTNAME", "Production.hostname"},
313  {"CLUSTER", "Production.cluster"},
314  };
315 
316  // Set the environment-derived parameters in the metadata
317  for(auto it: envMap){
318  const char* val = std::getenv(it.first);
319  if(val) manager.AddMetadataHighPrio(it.second, val);
320  }
321 
322  //Horn Polarity
323  if(fHaveHornInfo){
324  std::string polarity("fhc");
325  if(fisRHC) polarity = "rhc";
326  else if(fis0HC) polarity = "0hc";
327  manager.AddMetadataHighPrio( "Nova.HornPolarity", polarity ); // "high priority" so that it overrides the job FCL
328  }
329 
330 
331  //Add in event count
332  value_stream.str("");
333  value_stream << fNEvts;
334  manager.AddMetadata( "event_count", value_stream.str() );
335 
336  //Add in runs list
337  value_stream.str("");
338  if(fUseSubruns){
339  value_stream << "[";
340  for(auto it = fSubRuns.begin(); it != fSubRuns.end(); ++it){
341  if(it != fSubRuns.begin()) value_stream << ",\n";
342  value_stream << "[ " << it->first << " , "
343  << it->second << " , "
344  << "\"" << fRunType
345  << "\"" << " ]";
346  }
347  value_stream << "]";
348  }
349  else{
350  value_stream << "[";
351  for( auto it = fRuns.begin(); it != fRuns.end(); ++it){
352  if (it != fRuns.begin() ) value_stream << ",\n";
353  value_stream << "[ " << *it << " , "
354  << "\"" << fRunType << "\"" << " ]";
355  }
356  value_stream << "]";
357  }
358  manager.AddMetadata( "runs", value_stream.str() );
359 
360  //Add in parents
361  value_stream.str("");
362  value_stream << "[";
363  for(auto it = fParents.begin(); it != fParents.end(); ++it){
364  if(it != fParents.begin() ) value_stream << ",\n";
365  value_stream << "{ \"file_name\": \""
366  << *it << "\" }";
367  }
368  value_stream << "]";
369  manager.AddMetadata( "parents", value_stream.str() );
370  }
371 
372  //..................................................................
374  {
375  FillMetadata();
376  }
377 
379  {
380  //actually add the metadata to FileCatalogMetadata service here
381  //this is the only time we actually write it out
382 
383  FillMetadata();
384 
386 
388  for(auto& it : manager.GetMetadata()){
389  std::string key = it.first;
390  std::string value = it.second;
391 
392  //remove "jobfcl" from the metadata when committing it
393  size_t found = key.find("jobfcl");
394  if(found != std::string::npos) continue;
395 
396  md->addMetadata( key, value );
397  }
398  }
399 
401 }
bool isRHC
is the beam in antineutrino mode, aka RHC
Definition: SpillData.h:28
static MetadataManager & getInstance()
bool is0HC
horn off
Definition: SpillData.h:29
void GetInputMetadata(std::string inMetadata)
fileName
Definition: plotROC.py:78
set< int >::iterator it
SubRunNumber_t subRun() const
Definition: SubRun.h:44
void respondToOpenInputFile(art::FileBlock const &fb)
MaybeLogger_< ELseverityLevel::ELsev_info, false > LogInfo
void reconfigure(const fhicl::ParameterSet &p)
const char * p
Definition: xmltok.h:285
std::string const & fileName() const
Definition: FileBlock.h:38
std::map< std::string, std::string > & GetMetadata()
std::vector< std::string > fParents
std::pair< Spectrum *, CheatDecomp * > make_pair(SpectrumLoaderBase &loader_data, SpectrumLoaderBase &loader_mc, HistAxis *axis, Cut *cut, const SystShifts &shift, const Var &wei)
Definition: DataMCLoad.C:336
MaybeLogger_< ELseverityLevel::ELsev_error, false > LogError
bool isRealData() const
Definition: Event.h:83
DEFINE_ART_MODULE(TestTMapFile)
RunNumber_t run() const
Definition: Run.h:47
Definition: Run.h:31
bool isValid() const
Definition: Handle.h:189
void beginSubRun(art::SubRun const &sr)
Far Detector at Ash River, MN.
const XML_Char int const XML_Char * value
Definition: expat.h:331
Prototype Near Detector on the surface at FNAL.
std::string getenv(std::string const &name)
novadaq::cnv::DetId DetId() const
Prefer ds::DetectorService::DetId() instead.
Definition: GeometryBase.h:243
void analyze(const art::Event &e)
Near Detector in the NuMI cavern.
Class for parsing *.fcl files for their metadata information.
void AddMetadataHighPrio(std::string key, std::string value)
std::string fRunType
bool get_if_present(std::string const &key, T &value) const
Definition: ParameterSet.h:208
void AddMetadata(std::string key, std::string value)
EDAnalyzer(Table< Config > const &config)
Definition: EDAnalyzer.h:100
void beginRun(art::Run const &r)
::xsd::cxx::tree::string< char, simple_type > string
Definition: Database.h:154
void respondToCloseOutputFiles(art::FileBlock const &fb)
bool getByLabel(std::string const &label, std::string const &productInstanceName, Handle< PROD > &result) const
Definition: DataViewImpl.h:344
Metadata(const fhicl::ParameterSet &p)
void addMetadata(std::string const &key, std::string const &value)
TRandom3 r(0)
void DoParsing(fhicl::ParameterSet const &pset)
Float_t e
Definition: plot.C:35
Helper for AttenCurve.
Definition: Path.h:10
std::set< int > fRuns
Encapsulate the geometry of one entire detector (near, far, ndos)
RunNumber_t run() const
Definition: SubRun.h:49
static constexpr Double_t sr
Definition: Munits.h:164
std::set< std::pair< int, int > > fSubRuns
void endRun(art::Run const &r)