Util.cxx

Go to the documentation of this file.

////////////////////////////////////////////////////////////////////////
/// \file    Util.cxx
/// \brief   Utilities for mmap
/// \author  Christopher Backhouse - bckhouse@caltech.edu
////////////////////////////////////////////////////////////////////////

#include "LEM/func/Util.h"

#include <sys/file.h>
#include <sys/mman.h>
#include <sys/stat.h>

#include <cassert>
#include <iostream>

namespace lem
{
  long MMapFileAtAddress(const std::string& fname, void* addr, bool touchAll,
                         const std::string& lockName)
  {
    // Touching all the memory ("preloading") means we can be sure that it's
    // all quickly available to us later. The first process to do this has to
    // fetch the whole file from disk. We don't want more than one process
    // doing that, it's counter-productive. But once that's happened once, it's
    // in disk cache, and all other processes should feel free to preload in
    // parallel. This locking scheme ensures that.

    FILE* lockMain = 0;
    FILE* lockDone = 0;
    if(touchAll && !lockName.empty()){
      // No two processes should be preloading from disk at once

      // Make sure not to block out other users from this lock
      chmod(lockName.c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
      chmod((lockName+".done").c_str(), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);

      lockMain = fopen(lockName.c_str(), "w");
      int ret = flock(fileno(lockMain), LOCK_EX);
      assert(ret == 0); // Blocking flock call, must have got it

      // We're now holding the loading lock. Either it's our responsibility to
      // do the preload, or someone else beat us to it but is now done, so we
      // can go ahead freely. Check the ".done" lock to see which.

      lockDone = fopen((lockName+".done").c_str(), "w");
      ret = flock(fileno(lockDone), LOCK_EX | LOCK_NB);
      // If we succesfully get this lock, that means that no running processes
      // have preloaded the file for us.
      if(ret == 0){
        // Drop it again, we were just testing
        fclose(lockDone);
        lockDone = 0;
      }
      else{
        // Otherwise, someone else has already done the preload. Don't hold the
        // rest of the processes up.
        fclose(lockMain);
        lockMain = 0;
      }
    }


    FILE* f = fopen(fname.c_str(), "r");
    assert(f);

    struct stat st;
    fstat(fileno(f), &st);
    const long fsize = st.st_size;

    void* ret = mmap(addr, fsize, PROT_READ, MAP_SHARED | MAP_FIXED, fileno(f), 0);
    fclose(f);
    assert(ret != (void*)-1);


    if(touchAll){
      // Touch all the memory, one byte on each page
      char tot = 0;
      for(long i = 0; i < fsize; i += 4096){
        if((i/4096)%(fsize/409600) == 0) std::cout << (100*i)/fsize << "%" << std::endl;
        tot += ((char*)addr)[i];
      }
      // Printing forces it not to get optimized out
      std::cout << "Fully loaded: " << int(tot) << std::endl;

      if(!lockName.empty()){
        lockDone = fopen((lockName+".done").c_str(), "w");
        // Mark that we succesfully preloaded the file. This indication lasts
        // until the end of this process.
        flock(fileno(lockDone), LOCK_SH);

        // Release the main lock if we still have it
        if(lockMain) fclose(lockMain);
      }
    }

    return fsize;
  }
}