Functions | Variables
cache_state Namespace Reference

Functions

def FilelistCacheCount (files, verbose_flag, METHOD="pnfs")
 

Variables

 BULK_QUERY_SIZE
 
 WEBDAV_HOST
 
 PNFS_DIR_PATTERN
 
 gp
 
 nargs
 
 default
 
 metavar
 
 help
 
 dest
 
 action
 
 False
 
 choices
 
 args
 
 stdout
 
 stderr
 
 METHOD
 
 filelist
 
 sam
 
 dataset_name
 
 end
 
 dimensions
 
 samlist
 
 defname
 
 cache_count
 
 miss_count
 
 n_files
 
 announce
 
 progbar
 
 files_to_check
 
 n
 
 loc
 
 f
 
 file
 
 thePath
 
 non_enstore
 
 total
 
 cache_frac_str
 
 miss_frac_str
 

Function Documentation

def cache_state.FilelistCacheCount (   files,
  verbose_flag,
  METHOD = "pnfs" 
)

Definition at line 21 of file cache_state.py.

References open(), and print.

21 def FilelistCacheCount(files, verbose_flag, METHOD="pnfs"):
22  bulk_query_list = []
23 
24  if len(files) > 1:
25  print("Checking %d files:" % len(files))
26  cached = 0
27  progbar = ProgressBar(len(files))
28  n = 0
29 
30  for f in files:
31  if METHOD == "webdav":
32  f = PNFS_DIR_PATTERN.sub(r"/pnfs/fnal.gov/usr/\1", f)
33  bulk_query_list.append(f)
34 
35  else:
36  path, filename = os.path.split(f)
37  stat_file="%s/.(get)(%s)(locality)"%(path,filename)
38  theStatFile=open(stat_file)
39  state=theStatFile.readline()
40  theStatFile.close()
41  if 'ONLINE' in state:
42  cached += 1
43  if verbose_flag:
44  print(filename,"is cached")
45 
46  n += 1
47  progbar.Update(n)
48 
49  if len(bulk_query_list) > 0:
50  while len(bulk_query_list) > 0:
51  # it's probably possible to actually implement this using urllib2 natively,
52  # but I couldn't make it work very quickly
53 
54  params = {
55  "local_cert": "/tmp/x509up_u%d" % os.getuid(),
56  "host": WEBDAV_HOST,
57  }
58 
59 
60  cmd = """
61  curl -L --capath /etc/grid-security/certificates \
62  --cert %(local_cert)s \
63  --cacert %(local_cert)s \
64  --key %(local_cert)s \
65  -s -X PROPFIND -H Depth:0 \
66  --data '<?xml version="1.0" encoding="utf-8"?>
67  <D:propfind xmlns:D="DAV:">
68  <D:prop xmlns:R="http://www.dcache.org/2013/webdav"
69  xmlns:S="http://srm.lbl.gov/StorageResourceManager">
70  <S:FileLocality/>
71  </D:prop>
72  </D:propfind>' \
73  """ % params
74  for f in bulk_query_list[:BULK_QUERY_SIZE]:
75  cmd += " %s/%s" % (WEBDAV_HOST, f)
76 
77  out = subprocess.check_output(shlex.split(cmd))
78  for l in out.split("\n"):
79  if "ONLINE" in l:
80  cached += 1
81  if verbose_flag:
82  print(l)
83 
84  # NOTE: *not* n*BULK_QUERY_SIZE since we've already stripped off (n-1)*BULK_QUERY_SIZE in previous iterations
85  bulk_query_list = bulk_query_list[BULK_QUERY_SIZE:]
86  n += 1
87  if len(bulk_query_list) > 0:
88  progbar.Update( n*BULK_QUERY_SIZE )
89 
90  if len(bulk_query_list) > 0:
91  progbar.Finish()
92 
93  return cached
94 
95 
96 parser= argparse.ArgumentParser()
97 
bool print
procfile open("FD_BRL_v0.txt")
def FilelistCacheCount(files, verbose_flag, METHOD="pnfs")
Definition: cache_state.py:21

Variable Documentation

cache_state.action

Definition at line 113 of file cache_state.py.

cache_state.announce

Definition at line 167 of file cache_state.py.

cache_state.args

Definition at line 116 of file cache_state.py.

cache_state.BULK_QUERY_SIZE

Definition at line 16 of file cache_state.py.

cache_state.cache_count

Definition at line 160 of file cache_state.py.

cache_state.cache_frac_str

Definition at line 227 of file cache_state.py.

cache_state.choices

Definition at line 114 of file cache_state.py.

cache_state.dataset_name
cache_state.default

Definition at line 101 of file cache_state.py.

cache_state.defname

Definition at line 149 of file cache_state.py.

cache_state.dest

Definition at line 107 of file cache_state.py.

cache_state.dimensions

Definition at line 141 of file cache_state.py.

cache_state.end

Definition at line 138 of file cache_state.py.

cache_state.f

Definition at line 189 of file cache_state.py.

cache_state.False

Definition at line 113 of file cache_state.py.

cache_state.file

Definition at line 189 of file cache_state.py.

cache_state.filelist

Definition at line 129 of file cache_state.py.

cache_state.files_to_check

Definition at line 173 of file cache_state.py.

cache_state.gp

Definition at line 98 of file cache_state.py.

cache_state.help

Definition at line 103 of file cache_state.py.

cache_state.loc

Definition at line 180 of file cache_state.py.

cache_state.metavar

Definition at line 102 of file cache_state.py.

cache_state.METHOD

Definition at line 127 of file cache_state.py.

cache_state.miss_count

Definition at line 161 of file cache_state.py.

cache_state.miss_frac_str

Definition at line 228 of file cache_state.py.

cache_state.n

Definition at line 174 of file cache_state.py.

cache_state.n_files

Definition at line 163 of file cache_state.py.

cache_state.nargs

Definition at line 100 of file cache_state.py.

cache_state.non_enstore

Definition at line 219 of file cache_state.py.

cache_state.PNFS_DIR_PATTERN

Definition at line 18 of file cache_state.py.

cache_state.progbar

Definition at line 171 of file cache_state.py.

cache_state.sam

Definition at line 131 of file cache_state.py.

cache_state.samlist

Definition at line 147 of file cache_state.py.

cache_state.stderr

Definition at line 122 of file cache_state.py.

cache_state.stdout

Definition at line 122 of file cache_state.py.

cache_state.thePath

Definition at line 213 of file cache_state.py.

cache_state.total

Definition at line 226 of file cache_state.py.

cache_state.WEBDAV_HOST

Definition at line 17 of file cache_state.py.