Functions | Variables
samweb_client.files Namespace Reference

Functions

def fileinfo (args)
 
def _make_file_info (lines)
 
def getAvailableDimensions (samweb)
 
def _callDimensions (samweb, url, dimensions, params=None, stream=False)
 
def listFiles (samweb, dimensions=None, defname=None, fileinfo=False, stream=False)
 
def listFilesSummary (samweb, dimensions=None, defname=None)
 
def parseDims (samweb, dimensions, mode=False)
 
def countFiles (samweb, dimensions=None, defname=None)
 
def _make_file_path (filenameorid)
 
def locateFile (samweb, filenameorid)
 
def locateFiles (samweb, filenameorids)
 
def addFileLocation (samweb, filenameorid, location)
 
def removeFileLocation (samweb, filenameorid, location)
 
def getFileAccessUrls (samweb, filenameorid, schema, locationfilter=None)
 
def getMetadata (samweb, filenameorid, locations=False)
 
def getMultipleMetadata (samweb, filenameorids, locations=False, asJSON=False)
 
def getMetadataText (samweb, filenameorid, format=None, locations=False)
 
def getFileLineage (samweb, lineagetype, filenameorid)
 
def validateFileMetadata (samweb, md=None, mdfile=None)
 
def declareFile (samweb, md=None, mdfile=None)
 
def modifyFileMetadata (samweb, filename, md=None, mdfile=None)
 
def retireFile (samweb, filename)
 

Variables

 fileinfo = namedtuple("fileinfo", ["file_name", "file_id", "file_size", "event_count"])
 

Function Documentation

def samweb_client.files._callDimensions (   samweb,
  url,
  dimensions,
  params = None,
  stream = False 
)
private
Call the requested method with a dimensions string, 
automatically switching from GET to POST as needed 

Definition at line 32 of file files.py.

32 def _callDimensions(samweb, url, dimensions, params=None,stream=False):
33  """ Call the requested method with a dimensions string,
34  automatically switching from GET to POST as needed """
35  if params is None: params = {}
36  else: params = params.copy()
37  kwargs = {'params':params, 'stream':stream}
38  if len(dimensions) > 1024:
39  kwargs['data'] = {'dims':dimensions}
40  method = samweb.postURL
41  else:
42  params['dims'] = dimensions
43  method = samweb.getURL
44  return method(url, **kwargs)
45 
46 @samweb_method
def _callDimensions(samweb, url, dimensions, params=None, stream=False)
Definition: files.py:32
def samweb_client.files._make_file_info (   lines)
private

Definition at line 16 of file files.py.

References samweb_client.files.fileinfo, and makeTrainCVSamples.int.

Referenced by samweb_client.files.listFiles().

16 def _make_file_info(lines):
17  for l in lines:
18  values = l.split()
19  if values:
20  try:
21  yield fileinfo( values[0], long(values[1]), long(values[2]), int(values[3]) )
22  except Exception:
23  raise Error("Error while decoding file list output from server")
24 
25 @samweb_method
def _make_file_info(lines)
Definition: files.py:16
def samweb_client.files._make_file_path (   filenameorid)
private

Definition at line 123 of file files.py.

References samweb_client.http_client.escape_url_component().

Referenced by samweb_client.files.addFileLocation(), samweb_client.files.getFileAccessUrls(), samweb_client.files.getFileLineage(), samweb_client.files.getMetadata(), samweb_client.files.getMetadataText(), samweb_client.files.locateFile(), samweb_client.files.modifyFileMetadata(), samweb_client.files.removeFileLocation(), and samweb_client.files.retireFile().

123 def _make_file_path(filenameorid):
124  try:
125  fileid = long(filenameorid)
126  path = '/files/id/%d' % fileid
127  except ValueError:
128  path = '/files/name/%s' % escape_url_component(filenameorid)
129  return path
130 
131 @samweb_method
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.addFileLocation (   samweb,
  filenameorid,
  location 
)
Add a location for a file
arguments:
    name or id of file
    location

Definition at line 162 of file files.py.

References samweb_client.files._make_file_path().

162 def addFileLocation(samweb, filenameorid, location):
163  """ Add a location for a file
164  arguments:
165  name or id of file
166  location
167  """
168  url = _make_file_path(filenameorid) + '/locations'
169  data = { "add" : location }
170  return samweb.postURL(url, data=data, secure=True, role='*')
171 
172 @samweb_method
def addFileLocation(samweb, filenameorid, location)
Definition: files.py:162
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.countFiles (   samweb,
  dimensions = None,
  defname = None 
)
return count of files matching either a dataset definition or a dimensions string
arguments:
  dimensions: string (default None)
  defname: string definition name (default None)

Definition at line 112 of file files.py.

References samweb_client.http_client.escape_url_component().

112 def countFiles(samweb, dimensions=None, defname=None):
113  """ return count of files matching either a dataset definition or a dimensions string
114  arguments:
115  dimensions: string (default None)
116  defname: string definition name (default None)"""
117  if defname is not None:
118  result = samweb.getURL('/definitions/name/%s/files/count' % escape_url_component(defname))
119  else:
120  result = samweb._callDimensions('/files/count', dimensions)
121  return long(result.text.strip())
122 
def countFiles(samweb, dimensions=None, defname=None)
Definition: files.py:112
def samweb_client.files.declareFile (   samweb,
  md = None,
  mdfile = None 
)
Declare a new file
arguments:
    md: dictionary containing metadata (default None)
    mdfile: file object containing metadata (must be in json format)

Definition at line 276 of file files.py.

276 def declareFile(samweb, md=None, mdfile=None):
277  """ Declare a new file
278  arguments:
279  md: dictionary containing metadata (default None)
280  mdfile: file object containing metadata (must be in json format)
281  """
282  if md:
283  data = json.dumps(md)
284  elif mdfile:
285  data = mdfile.read()
286  else:
287  raise ArgumentError('Must specify metadata dictionary or file object')
288  return samweb.postURL('/files', data=data, content_type='application/json', secure=True).text
289 
290 @samweb_method
def declareFile(samweb, md=None, mdfile=None)
Definition: files.py:276
def samweb_client.files.fileinfo (   args)

Definition at line 12 of file files.py.

References samweb_client.files.fileinfo.

12  def fileinfo(*args): return tuple(args)
13 else:
def samweb_client.files.getAvailableDimensions (   samweb)
List the available dimensions 

Definition at line 26 of file files.py.

References samweb_client.convert_from_unicode().

27  """ List the available dimensions """
28  result = samweb.getURL('/files/list/dimensions?format=json&descriptions=1')
29  return convert_from_unicode(result.json())
30 
31 @samweb_method
def getAvailableDimensions(samweb)
Definition: files.py:26
def convert_from_unicode(input)
Definition: __init__.py:14
def samweb_client.files.getFileAccessUrls (   samweb,
  filenameorid,
  schema,
  locationfilter = None 
)
return urls by which this file may be accessed
arguments:
    name or id of file
    schema
    locationfilter (default None)

Definition at line 184 of file files.py.

References samweb_client.files._make_file_path(), and samweb_client.convert_from_unicode().

184 def getFileAccessUrls(samweb, filenameorid, schema, locationfilter=None):
185  """ return urls by which this file may be accessed
186  arguments:
187  name or id of file
188  schema
189  locationfilter (default None)
190  """
191  params = { "schema": schema }
192  if locationfilter:
193  params["location"] = locationfilter
194  response = samweb.getURL(_make_file_path(filenameorid) + '/locations/url', params=params)
195  return convert_from_unicode(response.json())
196 
197 @samweb_method
def getFileAccessUrls(samweb, filenameorid, schema, locationfilter=None)
Definition: files.py:184
def convert_from_unicode(input)
Definition: __init__.py:14
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.getFileLineage (   samweb,
  lineagetype,
  filenameorid 
)
Return lineage information for a file
arguments:
    lineagetype (ie "parents", "children")
    name or id of file

Definition at line 251 of file files.py.

References samweb_client.files._make_file_path(), samweb_client.convert_from_unicode(), and samweb_client.http_client.escape_url_component().

251 def getFileLineage(samweb, lineagetype, filenameorid):
252  """ Return lineage information for a file
253  arguments:
254  lineagetype (ie "parents", "children")
255  name or id of file
256  """
257  result = samweb.getURL(_make_file_path(filenameorid) + '/lineage/' + escape_url_component(lineagetype))
258  return convert_from_unicode(result.json())
259 
260 @samweb_method
def getFileLineage(samweb, lineagetype, filenameorid)
Definition: files.py:251
def convert_from_unicode(input)
Definition: __init__.py:14
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.getMetadata (   samweb,
  filenameorid,
  locations = False 
)
Return metadata as a dictionary 
arguments:
    name or id of file
    locations: if True, also return file locations

Definition at line 198 of file files.py.

References samweb_client.files._make_file_path(), and samweb_client.convert_from_unicode().

198 def getMetadata(samweb, filenameorid, locations=False):
199  """ Return metadata as a dictionary
200  arguments:
201  name or id of file
202  locations: if True, also return file locations
203  """
204  params = {}
205  if locations: params['locations'] = True
206  response = samweb.getURL(_make_file_path(filenameorid) + '/metadata', params=params)
207  return convert_from_unicode(response.json())
208 
209 @samweb_method
def convert_from_unicode(input)
Definition: __init__.py:14
def _make_file_path(filenameorid)
Definition: files.py:123
def getMetadata(samweb, filenameorid, locations=False)
Definition: files.py:198
def samweb_client.files.getMetadataText (   samweb,
  filenameorid,
  format = None,
  locations = False 
)
Return metadata as a string
arguments:
    name or id of file

Definition at line 239 of file files.py.

References samweb_client.files._make_file_path().

239 def getMetadataText(samweb, filenameorid, format=None, locations=False):
240  """ Return metadata as a string
241  arguments:
242  name or id of file
243  """
244  if format is None: format='plain'
245  params = {'format':format}
246  if locations: params['locations'] = 1
247  result = samweb.getURL(_make_file_path(filenameorid) + '/metadata', params=params)
248  return result.text.rstrip()
249 
250 @samweb_method
def getMetadataText(samweb, filenameorid, format=None, locations=False)
Definition: files.py:239
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.getMultipleMetadata (   samweb,
  filenameorids,
  locations = False,
  asJSON = False 
)
Return a list of metadata dictionaries
(This method does not return an error if a
file does not exist; instead it returns no
result for that file)
arguments:
    list of file names or ids
    locations: if True include location information
    asJSON: return the undecoded JSON string instead of python objects

Definition at line 210 of file files.py.

References samweb_client.convert_from_unicode().

210 def getMultipleMetadata(samweb, filenameorids, locations=False, asJSON=False):
211  """ Return a list of metadata dictionaries
212  (This method does not return an error if a
213  file does not exist; instead it returns no
214  result for that file)
215  arguments:
216  list of file names or ids
217  locations: if True include location information
218  asJSON: return the undecoded JSON string instead of python objects
219  """
220  file_names = []
221  file_ids = []
222  for filenameorid in filenameorids:
223  try:
224  file_ids.append(long(filenameorid))
225  except ValueError:
226  file_names.append(filenameorid)
227 
228  params = {}
229  if file_names: params["file_name"] = file_names
230  if file_ids: params["file_id"] = file_ids
231  if locations: params["locations"] = 1
232  response = samweb.getURL("/files/metadata", params=params)
233  if asJSON:
234  return response.text.rstrip()
235  else:
236  return convert_from_unicode(response.json())
237 
238 @samweb_method
def getMultipleMetadata(samweb, filenameorids, locations=False, asJSON=False)
Definition: files.py:210
def convert_from_unicode(input)
Definition: __init__.py:14
def samweb_client.files.listFiles (   samweb,
  dimensions = None,
  defname = None,
  fileinfo = False,
  stream = False 
)
list files matching either a dataset definition or a dimensions string
arguments:
  dimensions: string (default None)
  defname: string definition name (default None)
  fileinfo: boolean; if True, return file_id, file_size, event_count 
  stream: boolean: if True the return value will be a generator and the results will
                   be progressively returned to the client. Note that this keeps the network connection open until
                   all the response has been read. (default False)

returns:
  a generator producing file names (note that the network connection may not be closed
    until you have read the entire list). If fileinfo is true, it will produce
    (file_name, file_id, file_size, event_count) tuples

Definition at line 47 of file files.py.

References samweb_client.files._make_file_info(), samweb_client.http_client.escape_url_component(), and parse_dependency_file_t.list.

Referenced by ProductionTestTools.getConfigFromDef(), and ProductionTestTools.getSamDefinitionFiles().

47 def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False):
48  """ list files matching either a dataset definition or a dimensions string
49  arguments:
50  dimensions: string (default None)
51  defname: string definition name (default None)
52  fileinfo: boolean; if True, return file_id, file_size, event_count
53  stream: boolean: if True the return value will be a generator and the results will
54  be progressively returned to the client. Note that this keeps the network connection open until
55  all the response has been read. (default False)
56 
57  returns:
58  a generator producing file names (note that the network connection may not be closed
59  until you have read the entire list). If fileinfo is true, it will produce
60  (file_name, file_id, file_size, event_count) tuples
61  """
62 
63  # This can return a potentially long list, so don't preload the result
64  # instead return a generator which reads it progressively
65  params = {'format':'plain'}
66  if fileinfo:
67  params['fileinfo'] = 1
68  if defname is not None:
69  result = samweb.getURL('/definitions/name/%s/files/list' % escape_url_component(defname), params=params,stream=True)
70  else:
71  result = samweb._callDimensions('/files/list', dimensions, params, stream=True)
72  if fileinfo:
73  output = _make_file_info(result.iter_lines())
74  else:
75  output = ifilter( None, (l.strip() for l in result.iter_lines()) )
76  if stream: return output
77  else: return list(output)
78 
79 @samweb_method
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False)
Definition: files.py:47
def _make_file_info(lines)
Definition: files.py:16
def samweb_client.files.listFilesSummary (   samweb,
  dimensions = None,
  defname = None 
)
return summary of files matching either a dataset definition or a dimensions string
arguments:
  dimensions: string (default None)
  defname: string definition name (default None)

Definition at line 80 of file files.py.

References samweb_client.convert_from_unicode(), and samweb_client.http_client.escape_url_component().

Referenced by FirstAnaPageMakers.makeDatasetPage(), and ProductionTestTools.samDefinitionSummary().

80 def listFilesSummary(samweb, dimensions=None, defname=None):
81  """ return summary of files matching either a dataset definition or a dimensions string
82  arguments:
83  dimensions: string (default None)
84  defname: string definition name (default None)"""
85  if defname is not None:
86  result = samweb.getURL('/definitions/name/%s/files/summary' % escape_url_component(defname))
87  else:
88  result = samweb._callDimensions('/files/summary', dimensions)
89  return convert_from_unicode(result.json())
90 
91 @samweb_method
def listFilesSummary(samweb, dimensions=None, defname=None)
Definition: files.py:80
def convert_from_unicode(input)
Definition: __init__.py:14
def samweb_client.files.locateFile (   samweb,
  filenameorid 
)
return locations for this file
arguments:
    name or id of file

Definition at line 132 of file files.py.

References samweb_client.files._make_file_path(), and samweb_client.convert_from_unicode().

132 def locateFile(samweb, filenameorid):
133  """ return locations for this file
134  arguments:
135  name or id of file
136  """
137  url = _make_file_path(filenameorid) + '/locations'
138  result = samweb.getURL(url)
139  return convert_from_unicode(result.json())
140 
141 @samweb_method
def convert_from_unicode(input)
Definition: __init__.py:14
def locateFile(samweb, filenameorid)
Definition: files.py:132
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.locateFiles (   samweb,
  filenameorids 
)
return the locations of multiple files
The return value is a dictionary of { file_name_or_id : location } pairs

Definition at line 142 of file files.py.

References samweb_client.convert_from_unicode().

142 def locateFiles(samweb, filenameorids):
143  """ return the locations of multiple files
144  The return value is a dictionary of { file_name_or_id : location } pairs
145  """
146 
147  file_names = []
148  file_ids = []
149  for filenameorid in filenameorids:
150  try:
151  file_ids.append(long(filenameorid))
152  except ValueError:
153  file_names.append(filenameorid)
154 
155  params = {}
156  if file_names: params["file_name"] = file_names
157  if file_ids: params["file_id"] = file_ids
158  response = samweb.getURL("/files/locations", params=params)
159  return convert_from_unicode(response.json())
160 
161 @samweb_method
def locateFiles(samweb, filenameorids)
Definition: files.py:142
def convert_from_unicode(input)
Definition: __init__.py:14
def samweb_client.files.modifyFileMetadata (   samweb,
  filename,
  md = None,
  mdfile = None 
)
Modify file metadata
arguments:
    filename
    md: dictionary containing metadata (default None)
    mdfile: file object containing metadata (must be in json format)

Definition at line 291 of file files.py.

References samweb_client.files._make_file_path().

291 def modifyFileMetadata(samweb, filename, md=None, mdfile=None):
292  """ Modify file metadata
293  arguments:
294  filename
295  md: dictionary containing metadata (default None)
296  mdfile: file object containing metadata (must be in json format)
297  """
298  if md:
299  data = json.dumps(md)
300  else:
301  data = mdfile.read()
302  url = _make_file_path(filename)
303  return samweb.putURL(url + "/metadata", data=data, content_type='application/json', secure=True, role='*').text
304 
305 @samweb_method
def modifyFileMetadata(samweb, filename, md=None, mdfile=None)
Definition: files.py:291
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.parseDims (   samweb,
  dimensions,
  mode = False 
)
For debugging only 

Definition at line 92 of file files.py.

92 def parseDims(samweb, dimensions, mode=False):
93  """ For debugging only """
94  if not mode:
95  params = { "parse_only": "1"}
96  result = samweb._callDimensions('/files/list', dimensions, params)
97  else:
98  params = { "diagnostics" : "1" }
99  if mode=='count':
100  path = '/files/count'
101  elif mode=='summary':
102  path = '/files/summary'
103  elif mode=='fileinfo':
104  params['fileinfo']="1"
105  path = '/files/list'
106  else:
107  path = '/files/list'
108  result = samweb._callDimensions(path, dimensions, params)
109  return result.text.rstrip()
110 
111 @samweb_method
def parseDims(samweb, dimensions, mode=False)
Definition: files.py:92
def samweb_client.files.removeFileLocation (   samweb,
  filenameorid,
  location 
)
Remove a location for a file
arguments:
    name or id of file
    location

Definition at line 173 of file files.py.

References samweb_client.files._make_file_path().

173 def removeFileLocation(samweb, filenameorid, location):
174  """ Remove a location for a file
175  arguments:
176  name or id of file
177  location
178  """
179  url = _make_file_path(filenameorid) + '/locations'
180  data = { "remove" : location }
181  return samweb.postURL(url, data=data, secure=True, role='*')
182 
183 @samweb_method
def removeFileLocation(samweb, filenameorid, location)
Definition: files.py:173
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.retireFile (   samweb,
  filename 
)
Retire a file:
arguments:
    filename

Definition at line 306 of file files.py.

References samweb_client.files._make_file_path().

306 def retireFile(samweb, filename):
307  """ Retire a file:
308  arguments:
309  filename
310  """
311  url = _make_file_path(filename) + '/retired_date'
312  return samweb.postURL(url, secure=True, role='*').text
313 
314 
def retireFile(samweb, filename)
Definition: files.py:306
def _make_file_path(filenameorid)
Definition: files.py:123
def samweb_client.files.validateFileMetadata (   samweb,
  md = None,
  mdfile = None 
)
Check the metadata for validity
arguments:
    md: dictionary containing metadata (default None)
    mdfile: file object containing metadata (must be in json format)

Definition at line 261 of file files.py.

261 def validateFileMetadata(samweb, md=None, mdfile=None):
262  """ Check the metadata for validity
263  arguments:
264  md: dictionary containing metadata (default None)
265  mdfile: file object containing metadata (must be in json format)
266  """
267  if md:
268  data = json.dumps(md)
269  elif mdfile:
270  data = mdfile.read()
271  else:
272  raise ArgumentError('Must specify metadata dictionary or file object')
273  return samweb.postURL('/files/validate_metadata', data=data, content_type='application/json').text
274 
275 @samweb_method
def validateFileMetadata(samweb, md=None, mdfile=None)
Definition: files.py:261

Variable Documentation

samweb_client.files.fileinfo = namedtuple("fileinfo", ["file_name", "file_id", "file_size", "event_count"])

Definition at line 14 of file files.py.

Referenced by samweb_client.files._make_file_info(), and samweb_client.files.fileinfo().