Functions | Variables
cache_state Namespace Reference

Functions

def FilelistCacheCount (files, verbose_flag, METHOD="pnfs")
 

Variables

 BULK_QUERY_SIZE
 
 WEBDAV_HOST
 
 PNFS_DIR_PATTERN
 
 gp
 
 nargs
 
 default
 
 metavar
 
 help
 
 dest
 
 action
 
 False
 
 choices
 
 args
 
 stdout
 
 stderr
 
 METHOD
 
 filelist
 
 sam
 
 dimensions
 
 samlist
 
 defname
 
 cache_count
 
 miss_count
 
 n_files
 
 announce
 
 progbar
 
 files_to_check
 
 n
 
 loc
 
 thePath
 
 non_enstore
 
 total
 
 cache_frac_str
 
 miss_frac_str
 

Function Documentation

def cache_state.FilelistCacheCount (   files,
  verbose_flag,
  METHOD = "pnfs" 
)

Definition at line 18 of file cache_state.py.

References open().

18 def FilelistCacheCount(files, verbose_flag, METHOD="pnfs"):
19  bulk_query_list = []
20 
21  if len(files) > 1:
22  print "Checking %d files:" % len(files)
23  cached = 0
24  progbar = ProgressBar(len(files))
25  n = 0
26 
27  for f in files:
28  if METHOD == "webdav":
29  f = PNFS_DIR_PATTERN.sub(r"/pnfs/fnal.gov/usr/\1", f)
30  bulk_query_list.append(f)
31 
32  else:
33  path, filename = os.path.split(f)
34  stat_file="%s/.(get)(%s)(locality)"%(path,filename)
35  theStatFile=open(stat_file)
36  state=theStatFile.readline()
37  theStatFile.close()
38  if 'ONLINE' in state:
39  cached += 1
40  if verbose_flag:
41  print filename,"is cached"
42 
43  n += 1
44  progbar.Update(n)
45 
46  if len(bulk_query_list) > 0:
47  while len(bulk_query_list) > 0:
48  # it's probably possible to actually implement this using urllib2 natively,
49  # but I couldn't make it work very quickly
50 
51  params = {
52  "local_cert": "/tmp/x509up_u%d" % os.getuid(),
53  "host": WEBDAV_HOST,
54  }
55 
56 
57  cmd = """
58  curl -L --capath /etc/grid-security/certificates \
59  --cert %(local_cert)s \
60  --cacert %(local_cert)s \
61  --key %(local_cert)s \
62  -s -X PROPFIND -H Depth:0 \
63  --data '<?xml version="1.0" encoding="utf-8"?>
64  <D:propfind xmlns:D="DAV:">
65  <D:prop xmlns:R="http://www.dcache.org/2013/webdav"
66  xmlns:S="http://srm.lbl.gov/StorageResourceManager">
67  <S:FileLocality/>
68  </D:prop>
69  </D:propfind>' \
70  """ % params
71  for f in bulk_query_list[:BULK_QUERY_SIZE]:
72  cmd += " %s/%s" % (WEBDAV_HOST, f)
73 
74  out = subprocess.check_output(shlex.split(cmd))
75  for l in out.split("\n"):
76  if "ONLINE" in l:
77  cached += 1
78  if verbose_flag:
79  print l
80 
81  # NOTE: *not* n*BULK_QUERY_SIZE since we've already stripped off (n-1)*BULK_QUERY_SIZE in previous iterations
82  bulk_query_list = bulk_query_list[BULK_QUERY_SIZE:]
83  n += 1
84  if len(bulk_query_list) > 0:
85  progbar.Update( n*BULK_QUERY_SIZE )
86 
87  if len(bulk_query_list) > 0:
88  progbar.Finish()
89 
90  return cached
91 
92 
93 parser= argparse.ArgumentParser()
94 
procfile open("FD_BRL_v0.txt")
def FilelistCacheCount(files, verbose_flag, METHOD="pnfs")
Definition: cache_state.py:18

Variable Documentation

cache_state.action

Definition at line 110 of file cache_state.py.

cache_state.announce

Definition at line 164 of file cache_state.py.

cache_state.args

Definition at line 113 of file cache_state.py.

cache_state.BULK_QUERY_SIZE

Definition at line 13 of file cache_state.py.

cache_state.cache_count

Definition at line 157 of file cache_state.py.

cache_state.cache_frac_str

Definition at line 224 of file cache_state.py.

cache_state.choices

Definition at line 111 of file cache_state.py.

cache_state.default

Definition at line 98 of file cache_state.py.

cache_state.defname

Definition at line 146 of file cache_state.py.

cache_state.dest

Definition at line 104 of file cache_state.py.

cache_state.dimensions

Definition at line 138 of file cache_state.py.

cache_state.False

Definition at line 110 of file cache_state.py.

cache_state.filelist

Definition at line 126 of file cache_state.py.

cache_state.files_to_check

Definition at line 170 of file cache_state.py.

cache_state.gp

Definition at line 95 of file cache_state.py.

cache_state.help

Definition at line 100 of file cache_state.py.

cache_state.loc

Definition at line 177 of file cache_state.py.

cache_state.metavar

Definition at line 99 of file cache_state.py.

cache_state.METHOD

Definition at line 124 of file cache_state.py.

cache_state.miss_count

Definition at line 158 of file cache_state.py.

cache_state.miss_frac_str

Definition at line 225 of file cache_state.py.

cache_state.n

Definition at line 171 of file cache_state.py.

cache_state.n_files

Definition at line 160 of file cache_state.py.

cache_state.nargs

Definition at line 97 of file cache_state.py.

cache_state.non_enstore

Definition at line 216 of file cache_state.py.

cache_state.PNFS_DIR_PATTERN

Definition at line 15 of file cache_state.py.

cache_state.progbar

Definition at line 168 of file cache_state.py.

cache_state.sam

Definition at line 128 of file cache_state.py.

cache_state.samlist

Definition at line 144 of file cache_state.py.

cache_state.stderr

Definition at line 119 of file cache_state.py.

cache_state.stdout

Definition at line 119 of file cache_state.py.

cache_state.thePath

Definition at line 210 of file cache_state.py.

cache_state.total

Definition at line 223 of file cache_state.py.

cache_state.WEBDAV_HOST

Definition at line 14 of file cache_state.py.