LEMMixer_module.cc

Go to the documentation of this file.

////////////////////////////////////////////////////////////////////////
// \file    LEMMixer_module.cc
// \author  Christopher Backhouse - bckhouse@caltech.edu
////////////////////////////////////////////////////////////////////////

#include "LEM/PIDDetails.h"

// Framework includes
#include "art/Framework/Core/FileBlock.h"
#include "art/Framework/Core/ModuleMacros.h"
#include "art/Framework/Modules/MixFilter.h"

#include "Metadata/MetadataManager.h"

#include "ifdh.h"

namespace lem
{
  class LEMMixer
  {
  public:
    explicit LEMMixer(const fhicl::ParameterSet& pset,
                      art::MixHelper& helper);

    size_t nSecondaries() {return 1;}

    /// Fetch the matching lempart file from SAM
    virtual void respondToOpenInputFile(const art::FileBlock& fb);

  protected:
    /// Tell ART what file to mix with
    std::string getSecondaryFilename();

    /// Locate the release string within the filename
    void FindRelease(const std::string& fname,
                     size_t& pos, size_t& len) const;

    /// Locate the version string within the filename
    void FindVersion(const std::string& fname,
                     size_t& pos, size_t& len) const;

    /// Rewrite filename to find the corresponding lempart file
    std::string LEMPartFor(std::string fname) const;

    /// Do the actual mixing
    bool MixPIDDetails(const std::vector<const std::vector<PIDDetails>*>& pids,
                       std::vector<PIDDetails>& out,
                       const art::PtrRemapper& rm);

    bool fFromSAM; ///< Are we fetching lempart files from SAM?
    std::string fLempartFilename; ///< If so, this is the file we got

    std::string fLempartRelease; ///< Override lempart release to look for
    std::string fLempartVersion; ///< Override lempart version to look for

    std::string fUserSAMTag;

    /// Detect re-entry into \ref getSecondaryFilename
    bool fProvidedSecondaryFilename;
  };

  //......................................................................
  LEMMixer::LEMMixer(const fhicl::ParameterSet& pset,
                     art::MixHelper& helper)
    : fFromSAM(pset.get<bool>("FetchLempartFromSAM")),
      fLempartRelease(pset.get<std::string>("LempartRelease")),
      fLempartVersion(pset.get<std::string>("LempartVersion")),
      fUserSAMTag(pset.get<std::string>("UserSAMTag")),
      fProvidedSecondaryFilename(false)
  {
    helper.declareMixOp(art::InputTag(pset.get<std::string>("PIDLabel"), ""),
                        &LEMMixer::MixPIDDetails, *this);

    // If we're getting files from SAM, register this so that ART knows what
    // file we fetched. Otherwise it comes from the fclx
    if(fFromSAM) helper.registerSecondaryFileNameProvider(std::bind(&LEMMixer::getSecondaryFilename, this));
  }

  //......................................................................
  std::string LEMMixer::getSecondaryFilename()
  {
    if(fProvidedSecondaryFilename){
      mf::LogError("LEMMixer") << "getSecondaryFilename() called a second time. Is the lempart file too short? Aborting";
      abort();
    }
    fProvidedSecondaryFilename = true;

    assert(!fLempartFilename.empty());
    mf::LogInfo("LEMMixer") << "Merging with " << fLempartFilename << " from SAM...";
    return fLempartFilename;
  }

  //......................................................................
  bool LEMMixer::MixPIDDetails(const std::vector<const std::vector<PIDDetails>*>& pids,
                               std::vector<PIDDetails>& out,
                               const art::PtrRemapper& rm)
  {
    assert(pids.size() == 1);

    if(!pids[0]){
      mf::LogError("LEMMixer") << "PIDDetails missing from lempart stream. Normally this means that there were never even any slices. Go check your upstream files...";
      return true;
    }

    out = *pids[0];

    return true;
  }

  //......................................................................
  void LEMMixer::FindRelease(const std::string& fname,
                             size_t& pos, size_t& len) const
  {
    // Could check for a "1", but then we have a 2020 bug. Assume nothing
    // else in the filename will ever be capital.
    pos = fname.find("S");
    // Could also be an FA or R release
    if(pos == std::string::npos) pos = fname.find("FA");
    if(pos == std::string::npos) pos = fname.find("R");
    // But it must be one of those
    assert(pos != std::string::npos);

    // The release finishes at the next underscore (don't know how many
    // characters because it could have a ".a" suffix etc.
    const size_t posend = fname.find("_", pos);
    assert(posend != std::string::npos);

    len = posend-pos;
  }

  //......................................................................
  void LEMMixer::FindVersion(const std::string& fname,
                             size_t& pos, size_t& len) const
  {
    // Hope there's no other fields that ever begin with a v
    pos = fname.find("_v");
    assert(pos != std::string::npos);
    ++pos;
    assert(pos != std::string::npos);
    size_t posend = fname.find("_", pos);
    // Make sure the field was exactly the right length, because of worries
    // about other fields beginning "v". This will break if we ever get to a
    // v10. I really hope that never happens...
    assert(posend == pos+2);

    len = posend-pos;
  }

  //......................................................................
  std::string LEMMixer::LEMPartFor(std::string fname) const
  {
    // Remove any path information from the input file to get the basename
    size_t pos = fname.rfind("/");
    if(pos != std::string::npos) fname = fname.substr(pos+1);

    mf::LogInfo("LEMMixer") << "Input file basename is " << fname << std::endl;

    // Convert the filename to the matching lempart file
    pos = fname.find("pidpart");
    assert(pos != std::string::npos);
    fname.replace(pos, 7, "lempart");

    if(!fLempartRelease.empty()){
      size_t len;
      FindRelease(fname, pos, len);
      fname = fname.substr(0, pos) + fLempartRelease + fname.substr(pos+len, std::string::npos);
    }

    if(!fLempartVersion.empty()){
      size_t len;
      FindVersion(fname, pos, len);
      fname = fname.substr(0, pos) + fLempartVersion + fname.substr(pos+len, std::string::npos);
    }

    if(!fUserSAMTag.empty()){
      // If using user SAM the input lempart file might have a UUID on
      // the front. Detect it by the pattern of hyphens and strip it
      // out.
      if(fname.size() > 37 &&
         fname[8] == '-' && fname[13] == '-' &&
         fname[18] == '-' && fname[23] == '-' && fname[36] == '-')
        fname = &fname.c_str()[37];

      std::vector<std::string> fnames;
      ifdh i;
      i.set_debug("0"); // shut up
      // User SAM files have a UUID on the front. Wildcard over that
      fnames = i.translateConstraints(TString::Format("file_name %%%s and dataset.tag %s", fname.c_str(), fUserSAMTag.c_str()).Data());

      // ifdh likes to return empty strings at the end...
      while(!fnames.empty() && fnames.back().empty()) fnames.pop_back();

      if(fnames.empty()){
        std::cout << "Error: no lemsum file found matching " << fname
                  << " and user tag " << fUserSAMTag << std::endl;
        abort();
      }
      if(fnames.size() > 1){
        std::cout << "Error: multiple lemsums found matching " << fname
                  << " and user tag " << fUserSAMTag << std::endl;
        for(const std::string& s: fnames) std::cout << s << std::endl;
        abort();
      }

      fname = fnames[0];
    }

    return fname;
  }

  //......................................................................
  void LEMMixer::respondToOpenInputFile(const art::FileBlock& fb)
  {
    if(!fFromSAM) return;

    fLempartFilename = LEMPartFor(fb.fileName());

    mf::LogInfo("LEMMixer") << "Fetching " << fLempartFilename << " from SAM...";

    // lempart file is small, safe to fetch into current directory
    const int status = system(("ifdh_fetch -e nova "+fLempartFilename).c_str());
    assert(status == 0);


    // Fill in some metadata, because it doesn't automatically inherit
    meta::MetadataManager& mm = meta::MetadataManager::getInstance();
    if(fLempartRelease.empty()){
      size_t pos, len;
      FindRelease(fb.fileName(), pos, len);
      mm.AddMetadata("lempart.base_release", fb.fileName().substr(pos, len));
    }
    else{
      mm.AddMetadata("lempart.base_release", fLempartRelease);
    }

    if(fLempartVersion.empty()){
      size_t pos, len;
      FindVersion(fb.fileName(), pos, len);
      mm.AddMetadata("lempart.version", fb.fileName().substr(pos, len));
    }
    else{
      mm.AddMetadata("lempart.version", fLempartVersion);
    }

    mm.AddMetadata("lempart.file_name", fLempartFilename);
  }

  DEFINE_ART_MODULE(art::MixFilter<LEMMixer>)

} //namespace lem
////////////////////////////////////////////////////////////////////////