2 from __future__
import print_function
3 from builtins
import str
4 from builtins
import range
5 from past.builtins
import basestring
6 from builtins
import object
7 import os, sys, stat, pwd
16 detectors[
"fd"] =
"fardet" 17 detectors[
"nd"] =
"neardet" 18 detectors[
"tb"] =
"testbeam" 19 detectors[
"ndos"] =
"ndos" 22 neutrinoGenerators=[
"genie",
"gibuu"]
23 neutrinoGenieTunes=[
"default",
"G1802a0211a",
"G1810b0211a",
"G1810i00000",
"G1801b0211a",
"G1802a00000",
"G1801a0211a",
"G1810a0211a",
"G0000b00000",
"G1802b0211a",
"G1801a00000",
"G1801b00000",
"G1810a00000",
"G1810j00000",
"G1802b00000",
"G1810b00000",
"N1810j0211a",
"NOVA1800a00000"]
24 cosmicGenerators=[
"cry"]
26 run_pattern = re.compile(
r"^.*?_r([0-9]+)_s([0-9]+).+")
27 sim_pattern = re.compile(
r"^.*_(20\d\d\d\d\d\d_\d\d\d\d\d\d)")
28 stream_pattern = re.compile(
r"^.*?_r[0-9]+_s[0-9]+_(.*?)[\.,_].*")
30 parent_pattern = re.compile(
r'(,\n.*)?(?P<parent_str>"parents":.*)', re.MULTILINE)
33 if value ==
None :
return True 34 if value ==
"none" :
return True 40 elif isNone( base):
return extension
41 elif isNone(extension):
return base
42 elif base == extension:
return base
44 return base +
"_" + extension
48 fclFile.write(
"\nphysics.analyzers.metadata.params.")
49 fclFile.write(parName)
51 fclFile.write(parValue)
55 """ Convert CamelCase to camel_case """ 58 return re.sub(
r'\B([A-Z])',
r'_\1', s).lower()
61 """ Numerous keys are stored in the files in ways that we need to 'translate' 62 so that they can be given to SAM. """ 65 for k, v
in list(in_md.items()):
69 if isinstance(v, basestring)
and len(v) == 0:
80 if k.lower() ==
"parents" and '","file_name":' in v:
81 v = v.replace(
'","file_name":',
'"},{"file_name":')
86 if isinstance(v, basestring)
and any(c
in v
for c
in "[]{}"):
95 v =
list(set([tuple(run)
for run
in v]))
99 if k
in (
'first_event',
'last_event'):
110 if k ==
'simulated.cycle':
114 if k
in (
'start_date',
'end_date'):
118 if k ==
'process_name' and md.get(
'application',[]).
get(
'name')
is None:
119 k =
'application_name' 120 elif k ==
'stream_name':
123 if k.startswith(
'application_'):
124 if 'application' not in md:
125 md[
'application'] = {}
126 md[
'application'][k[12:]] = v
127 elif k
in (
'file_format_era',
'file_format_version'):
135 filesize = os.path.getsize(inFile)
136 filename = os.path.basename(inFile)
137 if filename.endswith(
"caf.root"):
138 md = {
'file_name': filename,
'group':
'nova',
'file_size': filesize,
'file_format':
'unknown'}
141 data = subprocess.check_output([
"extractCAFMetadata",
str(inFile)])
142 except subprocess.CalledProcessError:
143 print(
"extractCAFMetadata failed!")
146 dumperDict = json.loads(data[data.find(b
"{"):])
152 elif filename.endswith(
".h5"):
153 md = {
'file_name': filename,
'group':
'nova',
'file_size': filesize,
'file_format':
'unknown'}
155 data = subprocess.check_output([
"extractHDF5Metadata",
str(inFile)])
156 except subprocess.CalledProcessError:
157 print(
"extractHDF5Metadata failed!")
160 dumperDict = json.loads(data[data.find(b
"{"):])
168 md = {
'file_name': filename,
'group':
'nova',
'file_size': filesize,
'file_format':
'artroot'}
170 data = subprocess.check_output([
"sam_metadata_dumper",
str(inFile)])
171 except subprocess.CalledProcessError:
172 print(
"sam_metadata_dumper failed!")
180 for matches
in parent_pattern.findall(data):
182 parent_dict = eval(
"{%s}" % match)
183 if isinstance(parent_dict[
"parents"][0], basestring):
187 data = data.replace(
"".join(matches),
"")
189 dumperDict = json.loads(data.replace(b
"\x1b[?1034h", b
""))
191 for dumperKey
in dumperDict:
192 if str(inFile)
in str(dumperKey):
193 tmpMD = dumperDict[dumperKey]
202 def __init__(self,inFileName, metadata, release, systematic, skim, cosmicsPolarity, subversion, specialName):
244 print(
"Error!! No detector name found!!")
247 if detID
in list(detectors.keys()):
250 print(
"No detector name found for detectorid:", detId)
259 print(
"No file_type set!")
262 elif self.
fileType ==
"importedDetector":
265 elif self.
fileType ==
"importedSimulated":
272 print(
"Unrecognized file_type:", file_type)
283 if "dataset.tag" in self.
metadata !=
"none":
286 TOTrunNum = subprocess.Popen( [
"samweb",
"count-files",
"dataset.tag " +
str(nameofThetag) ], shell=
False, stdout=subprocess.PIPE ).stdout.read()
288 for i
in range(0,Tot):
294 run_match = run_pattern.match(inFileBase)
295 if run_match !=
None:
300 print(
"No run number/subrun number found!")
308 tmpStream = self.
metadata[
"data_stream"]
309 elif "online.stream" in self.
metadata:
310 tmpStream = self.
metadata[
"online.stream"]
317 if tmpStream ==
'all':
318 stream_match = stream_pattern.match(inFileBase)
319 if stream_match !=
None:
320 tmpStream = stream_match.groups()[0]
321 if tmpStream[0] ==
't':
322 tmpStream = tmpStream[1:]
331 print(
"No data stream found!")
359 self.
fluxVersion=os.environ[
"NOVA_FLUX_VERSION"].replace(
"nova_",
"")
364 if self.
genietune not in neutrinoGenieTunes:
380 matchGroups=matches.groups()
381 if len(matchGroups)<1:
390 if (
not isNone(mdSystematic))
and (
not isNone(systematic)):
391 print(
"ERROR: you specified a systematic, but this is already a systematic file")
393 elif not isNone(mdSystematic):
395 elif not isNone(systematic):
398 if subversion !=
None:
415 outFileName = os.path.basename(self.inFileName.strip(tier+
'.root'))
416 outFileName +=
"_r%.8d_s%.2d" % (self.
runNum, self.
subNum)
424 print(
"SETTING FILE NAME DATA")
425 outFileName +=
"_r%.8d_s%.2d" % (self.
runNum, self.
subNum)
431 outFileName +=
"_%s" % self.
horn 435 outFileName +=
"_%s" % self.
release 437 outFileName +=
"_data" 447 outFileName +=
"_%s" % self.
genierw 451 outFileName +=
"_%s" % self.
horn 455 if self.
cycle ==
None:
456 outFileName +=
"_r%.8d_s%.2d" % (self.
runNum, self.
subNum)
460 outFileName +=
"_%s" % self.
release 463 outFileName +=
"_sim" 468 outFileName +=
"_%s" % self.
skim 470 outFileName +=
"_%s" % self.
special 472 if tier ==
"artdaq" :
474 outFileName +=
".daq.root" 476 outFileName +=
".%s.root" % tier
477 elif tier ==
"h5caf":
478 outFileName +=
".h5caf" 480 outFileName +=
".%s.root" % tier
def fileEnstoreChecksum(path)
std::string format(const int32_t &value, const int &ndigits=8)