train_latest_snapshot.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 import os
4 import glob
5 import optparse
6 import md5
7 import time
8 
9 parser = optparse.OptionParser(description="""Resume caffe training given a
10  and a directory to find the latest
11  snapshot solver""",
12  add_help_option=True)
13 parser.add_option('--solver', type=str,
14  help="Path to solver prototxt.")
15 parser.add_option('--logDir', type=str, default="~/",
16  help="Prefix for training log.")
17 
18 
19 required = ["solver"]
20 
21 (args, options) = parser.parse_args(args=None, values=None)
22 
23 def findLatest(paths):
24  if len(paths) is 0:
25  return None
26  mtime = lambda path: os.stat(path).st_mtime
27  return next(reversed(sorted(paths, key=mtime)))
28 
29 
30 argDict = vars(args)
31 for key in required:
32  if argDict[key] is None:
33  raise Exception("Argument '%s' is required." % (key))
34 
35 snapshotDirLine = [line for line in open(args.solver).readlines()
36  if "snapshot_prefix" in line][0]
37 
38 prefix = snapshotDirLine.split(":")[1].strip().strip('"').strip("'")
39 
40 pattern = prefix + "*_iter_*.solverstate"
41 
42 snapshot = findLatest(glob.glob(pattern))
43 
44 print "Found recent snapshot:%s" % snapshot
45 
46 jobid = os.environ["PBS_JOBID"] if "PBS_JOBID" in os.environ \
47  else "interactive_%s" % md5.new(str(time.time())).hexdigest()[:5]
48 logName = "train_latest_snapshot_%s.out" % jobid
49 logPath = os.path.join(args.logDir, logName)
50 
51 
52 command = ["caffe.bin", "--gpu", "all", "train",
53  "--solver", args.solver, "--snapshot", snapshot , ">>", logPath,
54  "2>&1"]
55 
56 print "Starting training... \n%s\n" % " ".join(command)
57 
58 os.system(" ".join(command))
const std::map< std::pair< std::string, std::string >, Variable > vars
procfile open("FD_BRL_v0.txt")
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
void next()
Definition: show_event.C:84