2 from samweb_client
import json, convert_from_unicode
7 from itertools
import ifilter
10 from collections
import namedtuple
14 fileinfo = namedtuple(
"fileinfo", [
"file_name",
"file_id",
"file_size",
"event_count"])
21 yield fileinfo( values[0], long(values[1]), long(values[2]),
int(values[3]) )
23 raise Error(
"Error while decoding file list output from server")
27 """ List the available dimensions """ 28 result = samweb.getURL(
'/files/list/dimensions?format=json&descriptions=1')
33 """ Call the requested method with a dimensions string, 34 automatically switching from GET to POST as needed """ 35 if params
is None: params = {}
36 else: params = params.copy()
37 kwargs = {
'params':params,
'stream':stream}
38 if len(dimensions) > 1024:
39 kwargs[
'data'] = {
'dims':dimensions}
40 method = samweb.postURL
42 params[
'dims'] = dimensions
43 method = samweb.getURL
44 return method(url, **kwargs)
47 def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False):
48 """ list files matching either a dataset definition or a dimensions string 50 dimensions: string (default None) 51 defname: string definition name (default None) 52 fileinfo: boolean; if True, return file_id, file_size, event_count 53 stream: boolean: if True the return value will be a generator and the results will 54 be progressively returned to the client. Note that this keeps the network connection open until 55 all the response has been read. (default False) 58 a generator producing file names (note that the network connection may not be closed 59 until you have read the entire list). If fileinfo is true, it will produce 60 (file_name, file_id, file_size, event_count) tuples 65 params = {
'format':
'plain'}
67 params[
'fileinfo'] = 1
68 if defname
is not None:
69 result = samweb.getURL(
'/definitions/name/%s/files/list' %
escape_url_component(defname), params=params,stream=
True)
71 result = samweb._callDimensions(
'/files/list', dimensions, params, stream=
True)
75 output = ifilter(
None, (l.strip()
for l
in result.iter_lines()) )
76 if stream:
return output
77 else:
return list(output)
81 """ return summary of files matching either a dataset definition or a dimensions string 83 dimensions: string (default None) 84 defname: string definition name (default None)""" 85 if defname
is not None:
88 result = samweb._callDimensions(
'/files/summary', dimensions)
93 """ For debugging only """ 95 params = {
"parse_only":
"1"}
96 result = samweb._callDimensions(
'/files/list', dimensions, params)
98 params = {
"diagnostics" :
"1" }
100 path =
'/files/count' 101 elif mode==
'summary':
102 path =
'/files/summary' 103 elif mode==
'fileinfo':
104 params[
'fileinfo']=
"1" 108 result = samweb._callDimensions(path, dimensions, params)
109 return result.text.rstrip()
113 """ return count of files matching either a dataset definition or a dimensions string 115 dimensions: string (default None) 116 defname: string definition name (default None)""" 117 if defname
is not None:
120 result = samweb._callDimensions(
'/files/count', dimensions)
121 return long(result.text.strip())
125 fileid = long(filenameorid)
126 path =
'/files/id/%d' % fileid
133 """ return locations for this file 138 result = samweb.getURL(url)
143 """ return the locations of multiple files 144 The return value is a dictionary of { file_name_or_id : location } pairs 149 for filenameorid
in filenameorids:
151 file_ids.append(long(filenameorid))
153 file_names.append(filenameorid)
156 if file_names: params[
"file_name"] = file_names
157 if file_ids: params[
"file_id"] = file_ids
158 response = samweb.getURL(
"/files/locations", params=params)
163 """ Add a location for a file 169 data = {
"add" : location }
170 return samweb.postURL(url, data=data, secure=
True, role=
'*')
174 """ Remove a location for a file 180 data = {
"remove" : location }
181 return samweb.postURL(url, data=data, secure=
True, role=
'*')
185 """ return urls by which this file may be accessed 189 locationfilter (default None) 191 params = {
"schema": schema }
193 params[
"location"] = locationfilter
194 response = samweb.getURL(
_make_file_path(filenameorid) +
'/locations/url', params=params)
199 """ Return metadata as a dictionary 202 locations: if True, also return file locations 205 if locations: params[
'locations'] =
True 206 response = samweb.getURL(
_make_file_path(filenameorid) +
'/metadata', params=params)
211 """ Return a list of metadata dictionaries 212 (This method does not return an error if a 213 file does not exist; instead it returns no 214 result for that file) 216 list of file names or ids 217 locations: if True include location information 218 asJSON: return the undecoded JSON string instead of python objects 222 for filenameorid
in filenameorids:
224 file_ids.append(long(filenameorid))
226 file_names.append(filenameorid)
229 if file_names: params[
"file_name"] = file_names
230 if file_ids: params[
"file_id"] = file_ids
231 if locations: params[
"locations"] = 1
232 response = samweb.getURL(
"/files/metadata", params=params)
234 return response.text.rstrip()
240 """ Return metadata as a string 244 if format
is None: format=
'plain' 245 params = {
'format':format}
246 if locations: params[
'locations'] = 1
247 result = samweb.getURL(
_make_file_path(filenameorid) +
'/metadata', params=params)
248 return result.text.rstrip()
252 """ Return lineage information for a file 254 lineagetype (ie "parents", "children") 262 """ Check the metadata for validity 264 md: dictionary containing metadata (default None) 265 mdfile: file object containing metadata (must be in json format) 268 data = json.dumps(md)
272 raise ArgumentError(
'Must specify metadata dictionary or file object')
273 return samweb.postURL(
'/files/validate_metadata', data=data, content_type=
'application/json').text
277 """ Declare a new file 279 md: dictionary containing metadata (default None) 280 mdfile: file object containing metadata (must be in json format) 283 data = json.dumps(md)
287 raise ArgumentError(
'Must specify metadata dictionary or file object')
288 return samweb.postURL(
'/files', data=data, content_type=
'application/json', secure=
True).text
292 """ Modify file metadata 295 md: dictionary containing metadata (default None) 296 mdfile: file object containing metadata (must be in json format) 299 data = json.dumps(md)
303 return samweb.putURL(url +
"/metadata", data=data, content_type=
'application/json', secure=
True, role=
'*').text
312 return samweb.postURL(url, secure=
True, role=
'*').text
def getMetadataText(samweb, filenameorid, format=None, locations=False)
def removeFileLocation(samweb, filenameorid, location)
def getFileAccessUrls(samweb, filenameorid, schema, locationfilter=None)
def listFilesSummary(samweb, dimensions=None, defname=None)
def declareFile(samweb, md=None, mdfile=None)
def getFileLineage(samweb, lineagetype, filenameorid)
def modifyFileMetadata(samweb, filename, md=None, mdfile=None)
def getMultipleMetadata(samweb, filenameorids, locations=False, asJSON=False)
def listFiles(samweb, dimensions=None, defname=None, fileinfo=False, stream=False)
def retireFile(samweb, filename)
def validateFileMetadata(samweb, md=None, mdfile=None)
def countFiles(samweb, dimensions=None, defname=None)
def escape_url_component(s)
def locateFiles(samweb, filenameorids)
def getAvailableDimensions(samweb)
def convert_from_unicode(input)
def _make_file_info(lines)
def addFileLocation(samweb, filenameorid, location)
def _callDimensions(samweb, url, dimensions, params=None, stream=False)
def locateFile(samweb, filenameorid)
def _make_file_path(filenameorid)
def parseDims(samweb, dimensions, mode=False)
def getMetadata(samweb, filenameorid, locations=False)