DefinitionData.py

Go to the documentation of this file.

#!/bin/env python
import os, sys, shutil
import samweb_client
import pickle
from collections import OrderedDict
from time import time, ctime
import datetime

sam = samweb_client.SAMWebClient(experiment='nova')
defDataDir="${NOVAPRODVALID_DATA}/definitions/current"
defDataCache="${NOVAPRODVALID_DATA}/definitions/cache"

def definitionPickleName(defname, useCache=False):
    tmp=""
    if useCache:
        tmp=defDataCache + "/" + defname + ".pkl"
    else:
        tmp=defDataDir + "/" + defname + ".pkl"
    return os.path.expandvars(tmp) 

def dumpDefinitionData(defData,useCache=False):

    fname   =  definitionPickleName(defData.name,useCache)
    outFile = open(fname,'w')

    pickle.dump(defData,outFile)
    outFile.close()

def loadDefinitionData(defname):

    fname =  definitionPickleName(defname)
    inFile = open(fname,'r')

    data=pickle.load(inFile)

    inFile.close()

    return data

def cacheDefinitionData(defname):
    pklName   = definitionPickleName(defname)
    cacheName = definitionPickleName(defname,True)
    if os.path.isfile(pklName):
        shutil.copy(pklName,cacheName)
    else:
        print "ERROR couldn't find pkl file for", defname

class DefinitionData:

    def __init__(self,name):
        self.name=name
        self.description      = { "defname":name }
        self.summary          = { "file_count":0, "total_event_count":0, "total_file_size":0 }
        self.state            = "noData"
        self.queryTime        = 0
        self.queryTimeStamp   = 0

    def queryDate(self):
        return ctime(self.queryTimeStamp)

    def sameData(self,other):
        
        if self.description != other.description:
            return False

        if self.summary != other.summary:
            return False

    def fetchData(self,maxCacheLifetime):
        ## we might want to add a fallback in case dataset exists but query
        ## fails
        print "Fetching data for", self.name
        cacheName=definitionPickleName(self.name, useCache=True)
        cacheData=None
        if not os.path.isfile(cacheName):
            pass #no cache so don't even worry about it
        else:
            cacheData=pickle.load(open(cacheName,'r'))
            #if cache data is recent, just use it, just use it,
            #  and that's the end of the story, but if the cache
            #  data is old, don't rely on it.  But we do need to keep
            #  the cache around fo comparison purposes
            if (time() - cacheData.queryTimeStamp) <  maxCacheLifetime:
                self.__dict__ = cacheData.__dict__
                self.state="cached"
                self.queryTimeStamp = cacheData.queryTimeStamp
                return

        try:
            self.description = sam.descDefinitionDict(self.name)

            startTime=time()
            #try:
            #    self.snapshotID  = sam.takeSnapshot(self.name)
            #    snapshotQuery    = "dataset_def_name_newest_snapshot %s" % self.name
            #    self.summary     = sam.listFilesSummary(dimensions=snapshotQuery)
            #    self.state       = "good"
            #except:
            #    self.summary     = sam.listFilesSummary(defname=self.name)
            #    self.state       = "snapshot failed"
            self.summary     = sam.listFilesSummary(defname=self.name)
            self.state       = "good"

            self.queryTime = time() - startTime
            if self.summary["file_count"] == 0:
                self.summary["total_event_count"]=0
                self.summary["total_file_size"]=0

        except samweb_client._exceptions.DefinitionNotFound:
            print "ERROR, definition %s not found" % self.name
            self.state="bad definition name error"
        except samweb_client._exceptions.Timeout:
            print "ERROR, Timeout getting data for %s" % self.name
            self.state="timed out"
        except samweb_client._exceptions.SAMWebHTTPError as e:
            print "ERROR, HTTP: ", e
            self.state="http error"
        except IOError:
            print "IOError", e
            self.state="proxy error"
        except KeyboardInterrupt:
            print "Stopping because you say so."
            sys.exit(99)
        except:
            print "ERROR, unknown exception fetching data for %s" % self.name
            self.state="unknown error"
        # Store the time, even if there is an exception
        self.queryTimeStamp = time()

        #so here we have cache data that is stale (else we would have exited the function
        #   already), so we know we've fetched the data from SAM.   Check if that matches the cache.
        #   If not print a warning, delete cache, as it's clearly no good.   If they do match, dump to
        #   the cache as well.

        if cacheData != None:
            if not self.sameData(cacheData):
                os.remove(cacheName)
                sys.stderr.write("Warning:: cache data and current data do not match for %s\n" %self.name)
            else:
                dumpDefinitionData(self,useCache=True)

    def printData(self):
        for key in self.description.keys():
            print key, ":", self.description[key]
        for key in self.summary.keys():
            print key, ":", self.summary[key]