log_trawl Namespace Reference

Classes
class	FileViewer
class	Process
class	Project

Functions
def	checkIsFile (path)
def	setPermission (path, permission=PERMISSION_RW_R_R)
def	tail (path, window=20)
def	findLogInDirectory (logDir)
def	retreiveLogFromJobSubClient (jobId)
def	main (argv)

Variables
	PERMISSION_RW_R
	PERMISSION_RW_R_R
	states
	proj
	local
	banner

Function Documentation

def log_trawl.checkIsFile

(

path

)

Definition at line 14 of file log_trawl.py.

Referenced by log_trawl.FileViewer.__repr__(), log_trawl.Process.errLess(), log_trawl.Process.errPath(), log_trawl.Process.errTail(), log_trawl.Process.outLess(), log_trawl.Process.outPath(), log_trawl.Process.outTail(), and setPermission().

def checkIsFile(path):
  if os.path.isfile(path):
    return 1
  print "Warning, not a file: ", path
  return 0

def log_trawl.findLogInDirectory

(

logDir

)

Definition at line 343 of file log_trawl.py.

Referenced by retreiveLogFromJobSubClient().

def findLogInDirectory(logDir):
  logDirContents = os.listdir(logDir)
  for fileName in logDirContents:
    if fileName.endswith(".log"):
      return os.path.join(logDir,fileName)
  return None

def log_trawl.main

(

argv

)

Definition at line 399 of file log_trawl.py.

References retreiveLogFromJobSubClient().

def main(argv):
  parser = argparse.ArgumentParser(description="""
            Loop over the ".log" file and find the final state of each
            process, then find the .err and .out""")
  parser.add_argument('logFile', help="""
    The input file to run over, or a jobsub_client jobid.
    In the jobid case, the files are fetched to CONDOR_TMP if it is defined,
    or falls back to the current working directory if not defined.  If the log
    has already been fetcheded, it just finds it there.""", type=str)

  args = parser.parse_args()
  logPath = args.logFile

  if not os.path.isfile(logPath) and "@" in logPath and ".fnal.gov" in logPath:
    logPath = retreiveLogFromJobSubClient(logPath)


  proj = Project(logPath)
  return proj

def log_trawl.retreiveLogFromJobSubClient

(

jobId

)

Definition at line 352 of file log_trawl.py.

References findLogInDirectory().

Referenced by main().

def retreiveLogFromJobSubClient(jobId):
  print "JobSub Client Job ID supplied, finding logs."
  if "CONDOR_TMP" in os.environ and os.path.isdir(os.environ["CONDOR_TMP"]):
    print "Found $CONDOR_TMP, logs will be stored there. "
    fetchLocation = os.environ["CONDOR_TMP"]
  else:
    print "Failed to find $CONDOR_TMP, fetching logs to current directory."
    fetchLocation = "./"

  logDir = os.path.join(fetchLocation, jobId)

  if not os.path.isdir(logDir):
    os.mkdir(logDir)
  print "Directory:", logDir
  extantLogFile = findLogInDirectory(logDir)
  if extantLogFile:
    print "Log file previously fetched, location:", extantLogFile
    return extantLogFile

  else:
    print "Fetching log files..."
    os.system("jobsub_fetchlog --jobid " + jobId + " --unzipdir " + logDir)

  fetchedLogFile = findLogInDirectory(logDir)
  if fetchedLogFile:
    print "Log file location:", fetchedLogFile
    return fetchedLogFile
  else:
    raise Exception("Failed to filnd log file after fetching, " + \
                    "something went wrong.")

def log_trawl.setPermission	(		path,
			permission = PERMISSION_RW_R_R
	)

Definition at line 20 of file log_trawl.py.

References checkIsFile().

Referenced by log_trawl.Project.makeReadable().

def setPermission(path, permission=PERMISSION_RW_R_R):
  if checkIsFile(path):
    os.chmod(path, permission)

def log_trawl.tail	(		path,
			window = 20
	)

Definition at line 24 of file log_trawl.py.

References confusionMatrixTree.count, and open().

Referenced by log_trawl.Process.errTail(), daqdataformats::RawMilliSlice.getMilliSliceIndex(), daqdataformats.getTailSize(), log_trawl.Process.outTail(), daqdataformats::VERSION_NAMESPACE::RawConfigurationBlock.print(), stan::mcmc::chains< RNG >.quantile(), stan::mcmc::chains< RNG >.quantiles(), and genie::TuneId.Tail().

def tail(path, window=20):
    f = open(path, 'r')
    BUFSIZ = 1024
    f.seek(0, 2)
    bytes = f.tell()
    size = window
    block = -1
    data = []
    while size > 0 and bytes > 0:
        if (bytes - BUFSIZ > 0):
            # Seek back one whole BUFSIZ
            f.seek(block*BUFSIZ, 2)
            # read BUFFER
            data.append(f.read(BUFSIZ))
        else:
            # file too small, start from begining
            f.seek(0,0)
            # only read what was not read
            data.append(f.read(bytes))
        linesFound = data[-1].count('\n')
        size -= linesFound
        bytes -= BUFSIZ
        block -= 1
    return '\n'.join(''.join(data).splitlines()[-window:])

Variable Documentation

log_trawl.banner

Definition at line 423 of file log_trawl.py.

log_trawl.local

Definition at line 423 of file log_trawl.py.

log_trawl.PERMISSION_RW_R

Definition at line 10 of file log_trawl.py.

log_trawl.PERMISSION_RW_R_R

Definition at line 12 of file log_trawl.py.

log_trawl.proj

Definition at line 422 of file log_trawl.py.

log_trawl.states

Definition at line 163 of file log_trawl.py.

Referenced by make_cosmic_eff_table().