files.py
Go to the documentation of this file.
1 
2 from samweb_client import json, convert_from_unicode
3 from samweb_client.client import samweb_method
4 from samweb_client.http_client import escape_url_path, escape_url_component
5 from samweb_client.exceptions import *
6 
7 from itertools import ifilter
8 
9 try:
10  from collections import namedtuple
11 except ImportError:
12  def fileinfo(*args): return tuple(args)
13 else:
14  fileinfo = namedtuple("fileinfo", ["file_name", "file_id", "file_size", "event_count"])
15 
16 def _make_file_info(lines):
17  for l in lines:
18  values = l.split()
19  if values:
20  try:
21  yield fileinfo( values[0], long(values[1]), long(values[2]), int(values[3]) )
22  except Exception:
23  raise Error("Error while decoding file list output from server")
24 
25 @samweb_method
27  """ List the available dimensions """
28  result = samweb.getURL('/files/list/dimensions?format=json&descriptions=1')
29  return convert_from_unicode(result.json())
30 
31 @samweb_method
32 def _callDimensions(samweb, url, dimensions, params=None,stream=False):
33  """ Call the requested method with a dimensions string,
34  automatically switching from GET to POST as needed """
35  if params is None: params = {}
36  else: params = params.copy()
37  kwargs = {'params':params, 'stream':stream}
38  if len(dimensions) > 1024:
39  kwargs['data'] = {'dims':dimensions}
40  method = samweb.postURL
41  else:
42  params['dims'] = dimensions
43  method = samweb.getURL
44  return method(url, **kwargs)
45 
46 @samweb_method
47 def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False):
48  """ list files matching either a dataset definition or a dimensions string
49  arguments:
50  dimensions: string (default None)
51  defname: string definition name (default None)
52  fileinfo: boolean; if True, return file_id, file_size, event_count
53  stream: boolean: if True the return value will be a generator and the results will
54  be progressively returned to the client. Note that this keeps the network connection open until
55  all the response has been read. (default False)
56 
57  returns:
58  a generator producing file names (note that the network connection may not be closed
59  until you have read the entire list). If fileinfo is true, it will produce
60  (file_name, file_id, file_size, event_count) tuples
61  """
62 
63  # This can return a potentially long list, so don't preload the result
64  # instead return a generator which reads it progressively
65  params = {'format':'plain'}
66  if fileinfo:
67  params['fileinfo'] = 1
68  if defname is not None:
69  result = samweb.getURL('/definitions/name/%s/files/list' % escape_url_component(defname), params=params,stream=True)
70  else:
71  result = samweb._callDimensions('/files/list', dimensions, params, stream=True)
72  if fileinfo:
73  output = _make_file_info(result.iter_lines())
74  else:
75  output = ifilter( None, (l.strip() for l in result.iter_lines()) )
76  if stream: return output
77  else: return list(output)
78 
79 @samweb_method
80 def listFilesSummary(samweb, dimensions=None, defname=None):
81  """ return summary of files matching either a dataset definition or a dimensions string
82  arguments:
83  dimensions: string (default None)
84  defname: string definition name (default None)"""
85  if defname is not None:
86  result = samweb.getURL('/definitions/name/%s/files/summary' % escape_url_component(defname))
87  else:
88  result = samweb._callDimensions('/files/summary', dimensions)
89  return convert_from_unicode(result.json())
90 
91 @samweb_method
92 def parseDims(samweb, dimensions, mode=False):
93  """ For debugging only """
94  if not mode:
95  params = { "parse_only": "1"}
96  result = samweb._callDimensions('/files/list', dimensions, params)
97  else:
98  params = { "diagnostics" : "1" }
99  if mode=='count':
100  path = '/files/count'
101  elif mode=='summary':
102  path = '/files/summary'
103  elif mode=='fileinfo':
104  params['fileinfo']="1"
105  path = '/files/list'
106  else:
107  path = '/files/list'
108  result = samweb._callDimensions(path, dimensions, params)
109  return result.text.rstrip()
110 
111 @samweb_method
112 def countFiles(samweb, dimensions=None, defname=None):
113  """ return count of files matching either a dataset definition or a dimensions string
114  arguments:
115  dimensions: string (default None)
116  defname: string definition name (default None)"""
117  if defname is not None:
118  result = samweb.getURL('/definitions/name/%s/files/count' % escape_url_component(defname))
119  else:
120  result = samweb._callDimensions('/files/count', dimensions)
121  return long(result.text.strip())
122 
123 def _make_file_path(filenameorid):
124  try:
125  fileid = long(filenameorid)
126  path = '/files/id/%d' % fileid
127  except ValueError:
128  path = '/files/name/%s' % escape_url_component(filenameorid)
129  return path
130 
131 @samweb_method
132 def locateFile(samweb, filenameorid):
133  """ return locations for this file
134  arguments:
135  name or id of file
136  """
137  url = _make_file_path(filenameorid) + '/locations'
138  result = samweb.getURL(url)
139  return convert_from_unicode(result.json())
140 
141 @samweb_method
142 def locateFiles(samweb, filenameorids):
143  """ return the locations of multiple files
144  The return value is a dictionary of { file_name_or_id : location } pairs
145  """
146 
147  file_names = []
148  file_ids = []
149  for filenameorid in filenameorids:
150  try:
151  file_ids.append(long(filenameorid))
152  except ValueError:
153  file_names.append(filenameorid)
154 
155  params = {}
156  if file_names: params["file_name"] = file_names
157  if file_ids: params["file_id"] = file_ids
158  response = samweb.getURL("/files/locations", params=params)
159  return convert_from_unicode(response.json())
160 
161 @samweb_method
162 def addFileLocation(samweb, filenameorid, location):
163  """ Add a location for a file
164  arguments:
165  name or id of file
166  location
167  """
168  url = _make_file_path(filenameorid) + '/locations'
169  data = { "add" : location }
170  return samweb.postURL(url, data=data, secure=True, role='*')
171 
172 @samweb_method
173 def removeFileLocation(samweb, filenameorid, location):
174  """ Remove a location for a file
175  arguments:
176  name or id of file
177  location
178  """
179  url = _make_file_path(filenameorid) + '/locations'
180  data = { "remove" : location }
181  return samweb.postURL(url, data=data, secure=True, role='*')
182 
183 @samweb_method
184 def getFileAccessUrls(samweb, filenameorid, schema, locationfilter=None):
185  """ return urls by which this file may be accessed
186  arguments:
187  name or id of file
188  schema
189  locationfilter (default None)
190  """
191  params = { "schema": schema }
192  if locationfilter:
193  params["location"] = locationfilter
194  response = samweb.getURL(_make_file_path(filenameorid) + '/locations/url', params=params)
195  return convert_from_unicode(response.json())
196 
197 @samweb_method
198 def getMetadata(samweb, filenameorid, locations=False):
199  """ Return metadata as a dictionary
200  arguments:
201  name or id of file
202  locations: if True, also return file locations
203  """
204  params = {}
205  if locations: params['locations'] = True
206  response = samweb.getURL(_make_file_path(filenameorid) + '/metadata', params=params)
207  return convert_from_unicode(response.json())
208 
209 @samweb_method
210 def getMultipleMetadata(samweb, filenameorids, locations=False, asJSON=False):
211  """ Return a list of metadata dictionaries
212  (This method does not return an error if a
213  file does not exist; instead it returns no
214  result for that file)
215  arguments:
216  list of file names or ids
217  locations: if True include location information
218  asJSON: return the undecoded JSON string instead of python objects
219  """
220  file_names = []
221  file_ids = []
222  for filenameorid in filenameorids:
223  try:
224  file_ids.append(long(filenameorid))
225  except ValueError:
226  file_names.append(filenameorid)
227 
228  params = {}
229  if file_names: params["file_name"] = file_names
230  if file_ids: params["file_id"] = file_ids
231  if locations: params["locations"] = 1
232  response = samweb.getURL("/files/metadata", params=params)
233  if asJSON:
234  return response.text.rstrip()
235  else:
236  return convert_from_unicode(response.json())
237 
238 @samweb_method
239 def getMetadataText(samweb, filenameorid, format=None, locations=False):
240  """ Return metadata as a string
241  arguments:
242  name or id of file
243  """
244  if format is None: format='plain'
245  params = {'format':format}
246  if locations: params['locations'] = 1
247  result = samweb.getURL(_make_file_path(filenameorid) + '/metadata', params=params)
248  return result.text.rstrip()
249 
250 @samweb_method
251 def getFileLineage(samweb, lineagetype, filenameorid):
252  """ Return lineage information for a file
253  arguments:
254  lineagetype (ie "parents", "children")
255  name or id of file
256  """
257  result = samweb.getURL(_make_file_path(filenameorid) + '/lineage/' + escape_url_component(lineagetype))
258  return convert_from_unicode(result.json())
259 
260 @samweb_method
261 def validateFileMetadata(samweb, md=None, mdfile=None):
262  """ Check the metadata for validity
263  arguments:
264  md: dictionary containing metadata (default None)
265  mdfile: file object containing metadata (must be in json format)
266  """
267  if md:
268  data = json.dumps(md)
269  elif mdfile:
270  data = mdfile.read()
271  else:
272  raise ArgumentError('Must specify metadata dictionary or file object')
273  return samweb.postURL('/files/validate_metadata', data=data, content_type='application/json').text
274 
275 @samweb_method
276 def declareFile(samweb, md=None, mdfile=None):
277  """ Declare a new file
278  arguments:
279  md: dictionary containing metadata (default None)
280  mdfile: file object containing metadata (must be in json format)
281  """
282  if md:
283  data = json.dumps(md)
284  elif mdfile:
285  data = mdfile.read()
286  else:
287  raise ArgumentError('Must specify metadata dictionary or file object')
288  return samweb.postURL('/files', data=data, content_type='application/json', secure=True).text
289 
290 @samweb_method
291 def modifyFileMetadata(samweb, filename, md=None, mdfile=None):
292  """ Modify file metadata
293  arguments:
294  filename
295  md: dictionary containing metadata (default None)
296  mdfile: file object containing metadata (must be in json format)
297  """
298  if md:
299  data = json.dumps(md)
300  else:
301  data = mdfile.read()
302  url = _make_file_path(filename)
303  return samweb.putURL(url + "/metadata", data=data, content_type='application/json', secure=True, role='*').text
304 
305 @samweb_method
306 def retireFile(samweb, filename):
307  """ Retire a file:
308  arguments:
309  filename
310  """
311  url = _make_file_path(filename) + '/retired_date'
312  return samweb.postURL(url, secure=True, role='*').text
313 
def getMetadataText(samweb, filenameorid, format=None, locations=False)
Definition: files.py:239
def removeFileLocation(samweb, filenameorid, location)
Definition: files.py:173
def getFileAccessUrls(samweb, filenameorid, schema, locationfilter=None)
Definition: files.py:184
def listFilesSummary(samweb, dimensions=None, defname=None)
Definition: files.py:80
def declareFile(samweb, md=None, mdfile=None)
Definition: files.py:276
def getFileLineage(samweb, lineagetype, filenameorid)
Definition: files.py:251
def modifyFileMetadata(samweb, filename, md=None, mdfile=None)
Definition: files.py:291
def getMultipleMetadata(samweb, filenameorids, locations=False, asJSON=False)
Definition: files.py:210
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False)
Definition: files.py:47
def retireFile(samweb, filename)
Definition: files.py:306
def validateFileMetadata(samweb, md=None, mdfile=None)
Definition: files.py:261
def countFiles(samweb, dimensions=None, defname=None)
Definition: files.py:112
def locateFiles(samweb, filenameorids)
Definition: files.py:142
def getAvailableDimensions(samweb)
Definition: files.py:26
def convert_from_unicode(input)
Definition: __init__.py:14
def _make_file_info(lines)
Definition: files.py:16
def addFileLocation(samweb, filenameorid, location)
Definition: files.py:162
def _callDimensions(samweb, url, dimensions, params=None, stream=False)
Definition: files.py:32
def locateFile(samweb, filenameorid)
Definition: files.py:132
def _make_file_path(filenameorid)
Definition: files.py:123
def parseDims(samweb, dimensions, mode=False)
Definition: files.py:92
def getMetadata(samweb, filenameorid, locations=False)
Definition: files.py:198