Metadata_module.cc

Go to the documentation of this file.

////////////////////////////////////////////////////////////////////////
/// \brief Module to extract and fill file metadata
/// \file  Metadata_module.cc
/// \author Adam Aurisano
///
/// Generated at Fri Apr 12 17:18:04 2013 by Adam Aurisano using artmod
/// from art v1_02_06.
////////////////////////////////////////////////////////////////////////

#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <iterator>
#include <map>
#include <memory>
#include <set>
#include <sstream>
#include <string>
#include <vector>

#include "Geometry/Geometry.h"
#include "NovaDAQConventions/DAQConventions.h"
#include "Metadata/MetadataManager.h"

#include "art/Framework/Core/EDAnalyzer.h"
#include "art/Framework/Core/ModuleMacros.h"
#include "art/Framework/Services/System/FileCatalogMetadata.h"
#include "art/Framework/Core/FileBlock.h"
#include "art/Framework/Principal/Run.h"
#include "art/Framework/Principal/SubRun.h"
#include "art/Framework/Principal/Event.h"
#include "canvas/Persistency/Provenance/Timestamp.h"
#include "fhiclcpp/ParameterSet.h"
#include "messagefacility/MessageLogger/MessageLogger.h"

#include "SummaryData/SpillData.h"

#include <IFDH_service.h>


namespace meta {
  
  class Metadata : public art::EDAnalyzer {
  public:
    explicit Metadata(const fhicl::ParameterSet& p);
    virtual ~Metadata();
    
    void beginRun(art::Run const & r);
    void beginSubRun(art::SubRun const & sr);
    void endRun(art::Run const & r);
    void analyze(const art::Event& e);
    void reconfigure(const fhicl::ParameterSet& p);
    void respondToOpenInputFile(art::FileBlock const & fb);
    void respondToCloseOutputFiles(art::FileBlock const & fb);
    
  private:
    void GetInputMetadata (std::string inMetadata);
    void FillMetadata();
    
    int         fNEvts;
    std::string fRunType;
    bool        fUseSubruns;
    bool        fUseFhclParent;

    bool        fisRealData;
    
    //horn polarity of the run
    bool        fHaveHornInfo;
    bool        fisRHC;
    bool        fis0HC;
    
    std::set< int >                 fRuns;
    std::set< std::pair<int, int> > fSubRuns;
    std::vector< std::string >      fParents;
  };
}

namespace meta {
  Metadata::Metadata(const fhicl::ParameterSet& p) : 
    EDAnalyzer    (p),
    fNEvts        (0),

    fRunType      (p.get< std::string >("runType", "physics")),
    fUseSubruns   (p.get< bool        >("useSubruns", false) ),
    fUseFhclParent(p.get< bool        >("useFhclParent")     ),

    fisRealData   (false),
    fHaveHornInfo (false),
    fisRHC        (false),
    fis0HC        (false)

  {
    MetadataManager& manager = MetadataManager::getInstance();
    
    fhicl::ParameterSet inPSet;
    bool foundPSet = p.get_if_present<fhicl::ParameterSet>("params", inPSet);
    if(foundPSet) manager.DoParsing(inPSet);

    //add a default value for Nova.Subversion
    //Order of precedence
    //1) NPASS environment variable
    //2) fhicl file
    //3) input file
    //4) default value
    manager.AddMetadata( "Nova.SubVersion", "1" );
    
    //push fcl (for simulation jobs into the parents vector
    std::string fclName = manager.GetMetadata()["simulated.jobfcl"];
    if(fclName != "") fParents.push_back(fclName);
    
    if(fUseFhclParent) {
      art::ServiceHandle<IFDH> ifdhp;
      std::string oldMetadata;
      mf::LogInfo("Metadata") << "Parent File name is: " << fclName;
      try {
        oldMetadata = ifdhp->getMetadata(fclName);
      }
      catch(WebAPIException) {
        mf::LogError("Metadata") << "Web API Exception: ";
        oldMetadata = "";
      }
      mf::LogInfo("Metadata") << oldMetadata;
      GetInputMetadata(oldMetadata);  
    }
  }
  
  //.............................................................
  Metadata::~Metadata()
  {
  }
  
  //.............................................................
  void Metadata::reconfigure(const fhicl::ParameterSet& pset)
  {
  }

  void Metadata::GetInputMetadata(std::string inMetadata)
  {
    MetadataManager& manager = MetadataManager::getInstance();
    
    std::istringstream metadataStream;
    metadataStream.str(inMetadata);
    while(metadataStream.good()){
      std::string line, key, value;
      std::getline(metadataStream, line);
      size_t pos = line.find(":");
      if(pos != std::string::npos){
        key = line.substr(0, pos);
        value = line.substr(pos+1);
        size_t dot_pos = key.find(".");
          //if the key contains a dot, this is experiment specific: copy over
        if (dot_pos != std::string::npos) manager.AddMetadata(key, value);
      }
    }  
    return;
  }
  
  //......................................................................
  void Metadata::beginRun(art::Run const & r)
  {
    int run_num = r.run();
    fRuns.insert( run_num );
    return;
  }
  
  //......................................................................
  void Metadata::beginSubRun(art::SubRun const & sr)
  {
    int run_num = sr.run();
    int sub_num = sr.subRun();
    fSubRuns.insert( std::make_pair( run_num, sub_num ) );
    return;
  }

  //.................................................................
  void Metadata::analyze(const art::Event & e)
  {
    //art::Timestamp evtTime = e.time();
    //if (fNEvts == 0)
    //{
    //  fStartTime = evtTime;
    //  fEndTime = evtTime;
    //}
    //else
    //{
    //  if (evtTime < fStartTime) fStartTime = evtTime;
    //  if (evtTime > fEndTime) fEndTime = evtTime;
    //}

    fisRealData = e.isRealData();
    // Check for horn polarity:
    //  * for data, always.  (this ensures that IFDB's values are always used
    //    even if erroneous metadata made it into a previous tier, which happens every once in a while.)
    //  * for MC, only if the horn current metadata isn't already set.
    //    (it should be for beam MC, where it's set at simulation stage,
    //     but occasionally the horn current metadata is used for other purposes
    //     in non-beam MC -- e.g., indicating which training of CVN was used
    //     on cosmic MC.)
    if (fNEvts == 0)
    {
      bool checkHorn = false;
      if (fisRealData)
        checkHorn = true;
      else
      {
        MetadataManager& manager = MetadataManager::getInstance();
        std::string hornCurr;
        if (manager.GetMetadata().find("nova.hornpolarity") != manager.GetMetadata().end())
          hornCurr = manager.GetMetadata().at("nova.hornpolarity");
        checkHorn = hornCurr.empty();
      }

      if (checkHorn)
      {
        art::Handle<sumdata::SpillData> spilldata;
        e.getByLabel("ifdbspillinfo", spilldata);
        
        if (spilldata.isValid()) {
          fHaveHornInfo = true;
          if(spilldata->isRHC){
            fisRHC = true;
          }
          else if(spilldata->is0HC){
            fis0HC = true;
          }
        }
      }
    }   

    ++fNEvts;
    return;
  }
  
  //....................................................................
  void Metadata::respondToOpenInputFile(art::FileBlock const & fb)
  {
    //If we're forcing fcl parentage (like overlays) 
    //Don't get the metadata using the input file.
    if(fUseFhclParent) return; 
    
    std::string fileWPath = fb.fileName();
    if (fileWPath == "") return;
    std::string fileName;
    size_t pos = fileWPath.rfind("/");
    if (pos != std::string::npos) fileName = fileWPath.substr(pos+1);
    else fileName = fileWPath;
    
    fParents.push_back( fileName );
    
    //get old metadata from input file
    art::ServiceHandle<IFDH> ifdhp;
    std::string oldMetadata;
    try{
      oldMetadata = ifdhp->getMetadata(fileName);
    }
    catch(WebAPIException){
      oldMetadata = "";
    }
    GetInputMetadata(oldMetadata);
    
    return;
  }
  
  //..................................................................
  void Metadata::FillMetadata()
  {
    
    //if(fUseFhclParent) return;
    //fhicl file is already parsed - now add in additional metadata 
    MetadataManager& manager = MetadataManager::getInstance();

    std::ostringstream value_stream;
    
    std::string detector("unknown");
    art::ServiceHandle<geo::Geometry>  geo;
    if(geo->DetId() == novadaq::cnv::kFARDET)        detector = "fd";
    else if(geo->DetId() == novadaq::cnv::kNEARDET)  detector = "nd";
    else if(geo->DetId() == novadaq::cnv::kNDOS)     detector = "ndos";
    else if(geo->DetId() == novadaq::cnv::kTESTBEAM) detector = "tb";
    manager.AddMetadata( "Nova.DetectorID", detector );
    
    //set release this stage of processing used (takes precedence over input files)
    // NOVASOFT_VERSION is set for the ups build, but SRT_BASE_RELEASE is for SRT
    char* novarelease = std::getenv("NOVASOFT_VERSION");
    if( novarelease != nullptr)
      manager.AddMetadata( "Nova.Release",   novarelease);
    else
      manager.AddMetadata( "Nova.Release",   std::getenv("SRT_BASE_RELEASE"));
    
    //value_stream.str("");
    //value_stream << fStartTime.value();
    //manager.AddMetadata( "start_time", value_stream.str() );
    //value_stream.str("");
    //value_stream << fEndTime.value();
    //manager.AddMetadata( "end_time", value_stream.str() );

    //things in the high priority metadata have precedence over metadata from the fhicl file
    //
    // eg nova.subversion by default is set to 1
    // if input files have a pass number, that supercedes the default
    // if NPASS exists as an environment variable,
    // that supercedes value in input files

    // Map from environment variable to metadata field name
    const std::map<const char*, const char*> envMap =
      {{"NPASS",                  "Nova.SubVersion"},
       {"NOVAGRIDUTILS_VERSION",  "Production.novagridutils_version"},
       {"PROCESS",                "Production.process"},
       {"GRID_USER",              "Production.grid_user"},
       {"NOVAPRODUCTION_VERSION", "Production.novaproduction_version"},
       {"SAM_PROJECT_NAME",       "Production.sam_project_name"},
       {"HOSTNAME",               "Production.hostname"},
       {"CLUSTER",                "Production.cluster"},
      };

    // Set the environment-derived parameters in the metadata
    for(auto it: envMap){
      const char* val = std::getenv(it.first);
      if(val) manager.AddMetadataHighPrio(it.second, val);
    }

    //Horn Polarity
    if(fHaveHornInfo){
      std::string polarity("fhc");
      if(fisRHC)       polarity = "rhc";
      else if(fis0HC)  polarity = "0hc";
      manager.AddMetadataHighPrio( "Nova.HornPolarity", polarity );  // "high priority" so that it overrides the job FCL
    }

    
    //Add in event count
    value_stream.str("");
    value_stream << fNEvts;
    manager.AddMetadata( "event_count", value_stream.str() );
    
    //Add in runs list
    value_stream.str("");  
    if(fUseSubruns){
      value_stream << "[";
      for(auto it = fSubRuns.begin(); it != fSubRuns.end(); ++it){
        if(it != fSubRuns.begin()) value_stream << ",\n";
        value_stream << "[ " << it->first << " , " 
                     << it->second << " , " 
                     << "\"" << fRunType 
                     << "\"" << " ]"; 
      }
      value_stream << "]";
    }
    else{
      value_stream << "[";
      for( auto it = fRuns.begin(); it != fRuns.end(); ++it){
        if (it != fRuns.begin() ) value_stream << ",\n";
        value_stream << "[ " << *it << " , " 
                     << "\"" << fRunType << "\"" << " ]"; 
      }
      value_stream << "]";
    }
    manager.AddMetadata( "runs", value_stream.str() );
    
    //Add in parents
    value_stream.str("");
    value_stream << "[";
    for(auto it = fParents.begin(); it != fParents.end(); ++it){
      if(it != fParents.begin() ) value_stream << ",\n";
      value_stream << "{ \"file_name\": \"" 
                   << *it << "\" }";
    }
    value_stream << "]";
    manager.AddMetadata( "parents", value_stream.str() );
  }
  
  //..................................................................
  void Metadata::endRun(art::Run const & r)
  {
    FillMetadata();
  }
  
  void Metadata::respondToCloseOutputFiles(art::FileBlock const & fb)
  {
    //actually add the metadata to FileCatalogMetadata service here
    //this is the only time we actually write it out
    
    FillMetadata();
    
    MetadataManager& manager = MetadataManager::getInstance();
    
    art::ServiceHandle<art::FileCatalogMetadata> md;
    for(auto& it : manager.GetMetadata()){
      std::string key = it.first;
      std::string value = it.second;
      
      //remove "jobfcl" from the metadata when committing it     
      size_t found = key.find("jobfcl");
      if(found != std::string::npos) continue;
      
      md->addMetadata( key, value );      
    }
  }
  
  DEFINE_ART_MODULE(Metadata)
}