Functions | Variables
runNovaSAM_cmake Namespace Reference

Functions

def unCamelCase (s)
 
def createMetadata (inFile)
 
def getOutDir (pathname, hashDirs=False)
 
def checkAndMoveFiles (inFile, declareFiles)
 
def declareFile (fileWPath)
 
def copyOutFiles (hashDirs=False)
 
def getOutputFileName (detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
def resolveFclPath (fcl)
 

Variables

 gErrorIgnoreLevel
 
 detectors = dict()
 
 _lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
 
 _skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
 
 parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')
 
 help
 
 type
 
 str
 
 action
 
 args = parser.parse_args()
 
 inFile = args.inFile
 
 samweb = samweb_client.SAMWebClient(experiment='nova')
 
 inFileBase = os.path.basename(inFile)
 
 metadata = samweb.getMetadata(inFileBase)
 
 runNum = int(metadata["runs"][0][0])
 
 subNum = int(metadata["runs"][0][1])
 
 run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")
 
 run_match = run_pattern.match(inFileBase)
 
 stream = int(metadata["data_stream"])
 
string streamEntry = 't{0:02d}'
 
 stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)
 
 stream_match = stream_pattern.match(inFileBase)
 
 detector = metadata["online.detector"].lower()
 
 detId = metadata["nova.detectorid"].lower()
 
 release = os.environ["NOVASOFT_VERSION"]
 
 file_type = metadata["file_type"]
 
string dataFlag = "data"
 
 subversion = os.environ["NPASS"]
 
 nevt = None
 
 fclFile = args.config
 
 fclPath = resolveFclPath(fclFile)
 
 tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")
 
 fclFileObj = open(tmpFclName, 'a')
 
list cmdList = ['nova']
 
 outTier
 
 cafTier
 
list outList = []
 
 output = outTier.split(":")[0]
 
 tier = outTier.split(":")[1]
 
 outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
 cafLabel = cafTier.split(":")[0]
 
 cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
 
string cmd = ' '
 
 retCode = subprocess.call(cmdList)
 
 dirList = os.listdir(".")
 
 skip_match = _skip_pattern.match(file)
 
 dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
 

Function Documentation

def runNovaSAM_cmake.checkAndMoveFiles (   inFile,
  declareFiles 
) 
Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease

Definition at line 106 of file runNovaSAM_cmake.py.

References declareFile().

106 def checkAndMoveFiles(inFile, declareFiles):
107  """Checks all root files in the current directories for zombie or recovered status. Bad files are deleted while good files are moved to the results subdirectory for copy out ease"""
108  inFileBase = os.path.basename(inFile)
109  baseDir = "."
110  for root, dirs, filenames in os.walk(baseDir):
111  if root == baseDir:
112  for file in filenames:
113  if file.endswith (".root") and file != inFileBase:
114  fileWPath = os.path.join(root, file)
115  rootFile = ROOT.TFile(fileWPath)
116  if rootFile.IsZombie() or rootFile.TestBit(ROOT.TFile.kRecovered):
117  #do not copy over, just delete
118  print "File", fileWPath, "is Zombie - remove it"
119  os.remove(fileWPath)
120  else:
121  if declareFiles == True:
122  declareFile(fileWPath)
123  newFilePath = os.path.join(root, "results", file)
124  os.renames(fileWPath, newFilePath)
125  rootFile.Close()
126  return
127 
runNovaSAM_cmake.declareFile
def declareFile(fileWPath)
Definition: runNovaSAM_cmake.py:128
runNovaSAM_cmake.checkAndMoveFiles
def checkAndMoveFiles(inFile, declareFiles)
Definition: runNovaSAM_cmake.py:106
def runNovaSAM_cmake.copyOutFiles (   hashDirs = False) 
Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output

Definition at line 150 of file runNovaSAM_cmake.py.

References getOutDir().

150 def copyOutFiles(hashDirs=False):
151  """Builtin facility to copy out art files (automatically excludes hist files). This adds in a subdirectories with a single hex digit each corresponding to the first three digits in the hash of the output file name. This splits up files into 4096 separate subdirectories, preventing overfull directories. Copy out does not happen if the file already exists in the output"""
152  print "Copying out files"
153  dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")
154  baseDir = "./results"
155  for root, dirs, filenames in os.walk(baseDir):
156  if root == baseDir:
157  for file in filenames:
158  if file.endswith (".root") and file != inFileBase:
159  fileWPath = os.path.join(root, file)
160  dest = getOutDir(file, hashDirs)
161  skip_match = _skip_pattern.match(file)
162  if skip_match == None:
163  try:
164  print "Checking if", dest + "/" + file, "exists"
165  dh.ls(dest + "/" + file, 1, "")
166  except RuntimeError:
167  print "It doesn't - copy", fileWPath, "to", dest
168  dh.cp(["-D", fileWPath, dest])
169  print "Removing", fileWPath
170  os.remove(fileWPath)
171  dh.cleanup()
172  return
173 
runNovaSAM_cmake.copyOutFiles
def copyOutFiles(hashDirs=False)
Definition: runNovaSAM_cmake.py:150
runNovaSAM_cmake.getOutDir
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM_cmake.py:93
def runNovaSAM_cmake.createMetadata (   inFile)

Definition at line 37 of file runNovaSAM_cmake.py.

References samweb_client.utility.fileEnstoreChecksum(), site_stats_from_log.get, str, and unCamelCase().

Referenced by declareFile().

37 def createMetadata(inFile):
38  filename = os.path.basename(inFile)
39  filesize = os.path.getsize(inFile)
40 
41  try:
42  data = subprocess.check_output(["sam_metadata_dumper", str(inFile)])
43  except subprocess.CalledProcessError:
44  print "sam_metadata_dumper failed!"
45  return None
46 
47  md = {'file_name': filename, 'group': 'nova', 'file_size': filesize, 'file_format': 'root'}
48 
49  # An ad-hoc parser for the output
50  mode = 'new'
51  for line in data.split('\n'):
52  m = _lineregex.match(line)
53  if mode == 'json':
54  if not m and not line.startswith('-'):
55  data += line
56  else:
57  md[key] = json.loads(data)
58  mode = 'new'
59 
60  if mode == 'new':
61  if m:
62  key = m.group(1)
63  value = m.group(2)
64 
65  if '.' not in key:
66  key = unCamelCase(key)
67 
68  # ART does not appear to set the application name
69  # but does provide a process name
70  if key == 'process_name' and md.get('application',[]).get('name') is None:
71  key = 'application_name'
72  elif key == 'stream_name':
73  key = 'data_stream'
74 
75  if key in ('start_date','end_date'):
76  value = long(value)
77 
78  if key == 'runs' or key == 'parents':
79  mode = 'json'
80  data = value
81  elif key.startswith('application_'):
82  # special handling for applications
83  if 'application' not in md:
84  md['application'] = {}
85  md['application'][key[12:]] = value
86  else:
87  md[key] = value
88 
89  md['crc'] = fileEnstoreChecksum(inFile)
90 
91  return md
92 
site_stats_from_log.get
get
Definition: site_stats_from_log.py:189
runNovaSAM_cmake.createMetadata
def createMetadata(inFile)
Definition: runNovaSAM_cmake.py:37
runNovaSAM_cmake.unCamelCase
def unCamelCase(s)
Definition: runNovaSAM_cmake.py:30
samweb_client.utility.fileEnstoreChecksum
def fileEnstoreChecksum(path)
Definition: utility.py:50
runNovaSAM_cmake.str
str
Definition: runNovaSAM_cmake.py:209
def runNovaSAM_cmake.declareFile (   fileWPath) 
Checks file for a TKey of RootFileDB.  If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM

Definition at line 128 of file runNovaSAM_cmake.py.

References createMetadata().

Referenced by checkAndMoveFiles().

128 def declareFile(fileWPath):
129  """Checks file for a TKey of RootFileDB. If it exists, run sam_metadata_dumper and construct appropriate metadata for the file. Use that metadata to declare the file to SAM"""
130  samweb = samweb_client.SAMWebClient(experiment='nova')
131  rootFile = ROOT.TFile(fileWPath)
132  if rootFile.FindKey("RootFileDB"):
133  print "Found key (RootFileDB): constructing metadata for", fileWPath
134  md = createMetadata(fileWPath)
135  if (md == None):
136  print "No metadata found!"
137  else:
138  pprint.pprint(md)
139  print ''
140  print "Declaring", fileWPath, "to SAM"
141  try:
142  samweb.declareFile(md)
143  except:
144  print fileWPath, "already exists in SAM"
145  else:
146  print fileWPath, "does not contain RootFileDB, do not try to declare"
147  rootFile.Close()
148  return
149 
runNovaSAM_cmake.createMetadata
def createMetadata(inFile)
Definition: runNovaSAM_cmake.py:37
runNovaSAM_cmake.declareFile
def declareFile(fileWPath)
Definition: runNovaSAM_cmake.py:128
def runNovaSAM_cmake.getOutDir (   pathname,
  hashDirs = False 
)

Definition at line 93 of file runNovaSAM_cmake.py.

References exit(), and parse_dependency_file_t.list.

Referenced by copyOutFiles().

93 def getOutDir(pathname, hashDirs=False):
94  if "DEST" in os.environ:
95  dest = os.environ["DEST"]
96  else:
97  print "No destination set!"
98  exit(1)
99  dirs = [dest]
100  if hashDirs:
101  head, tail = os.path.split(pathname)
102  hash = md5.new(tail)
103  dirs += list(hash.hexdigest()[:3])
104  return os.path.join(*dirs)
105 
parse_dependency_file_t.list
list
Definition: parse_dependency_file_t.py:9
exit
exit(0)
runNovaSAM_cmake.getOutDir
def getOutDir(pathname, hashDirs=False)
Definition: runNovaSAM_cmake.py:93
def runNovaSAM_cmake.getOutputFileName (   detector,
  runNum,
  subNum,
  streamEntry,
  release,
  subversion,
  dataFlag,
  tier,
  nevt 
)

Definition at line 174 of file runNovaSAM_cmake.py.

References exit(), and novadaq::HexUtils.format().

174 def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt):
175  if dataFlag == "sim":
176  pattern = re.compile(r"^.*?_(.*?)_\d+.*_v\d+_(.*?)(\.|\_)sim\..+")
177  match = pattern.match(inFileBase)
178  if match != None:
179  simLabel = match.groups()[0]
180  uniq = match.groups()[1]
181  outName = '{0}_{1}_{2}_r{3:08d}_s{4:02d}_{5}_v{6}_{7}_{8}.{9}.root'.format(detector, simLabel, nevt, runNum, subNum, release, subversion, uniq, dataFlag, tier)
182  else:
183  print "regex couldn't find the sim label and the uniqueifier"
184  exit(1)
185  else:
186  outName = '{0}_r{1:08d}_s{2:02d}_{3}_{4}_v{5}_{6}.{7}.root'.format(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier)
187  return outName
188 
novadaq::HexUtils::format
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
exit
exit(0)
runNovaSAM_cmake.getOutputFileName
def getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)
Definition: runNovaSAM_cmake.py:174
def runNovaSAM_cmake.resolveFclPath (   fcl)

Definition at line 189 of file runNovaSAM_cmake.py.

189 def resolveFclPath(fcl):
190  # Check if we have an absolute path name, return it if so
191  if fcl[0] == "/":
192  return fcl
193 
194  # SRT sets up FHICL_FILE_PATH with extra / at end of directories;
195  # cmake does not. Need to add + "/" + below to account for this.
196  fclPaths = os.environ["FHICL_FILE_PATH"].split(":")
197  for path in fclPaths:
198  if os.path.isfile(path + "/" + fcl):
199  return path + "/" + fcl
200 
201  # If we haven't found it, we have a problem.
202  raise IOError(sys.argv[0] + ": config file "+ fcl+" not found in FHICL_FILE_PATH: $FHICL_FILE_PATH")
203 
204 
split
void split(double tt, double *fr)
runNovaSAM_cmake.resolveFclPath
def resolveFclPath(fcl)
Definition: runNovaSAM_cmake.py:189
def runNovaSAM_cmake.unCamelCase (   s) 
Convert CamelCase to camel_case

Definition at line 30 of file runNovaSAM_cmake.py.

Referenced by createMetadata().

30 def unCamelCase(s):
31  """ Convert CamelCase to camel_case """
32  # \B matches an empty string which is NOT at the beginning of a word
33  # so requiring this means no _ will be inserted at the start
34  return re.sub( r'\B([A-Z])', r'_\1', s).lower()
35 
36 
runNovaSAM_cmake.unCamelCase
def unCamelCase(s)
Definition: runNovaSAM_cmake.py:30

Variable Documentation

runNovaSAM_cmake._lineregex = re.compile(r'\d+: ([^ \t]+)[ \t]+(.+)')
private

Definition at line 24 of file runNovaSAM_cmake.py.

runNovaSAM_cmake._skip_pattern = re.compile(r"^.*(hist|RootOutput).*\.root", re.IGNORECASE)
private

Definition at line 27 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.action

Definition at line 209 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.args = parser.parse_args()

Definition at line 217 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.cafLabel = cafTier.split(":")[0]

Definition at line 366 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.cafName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)

Definition at line 370 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.cafTier

Definition at line 346 of file runNovaSAM_cmake.py.

string runNovaSAM_cmake.cmd = ' '

Definition at line 387 of file runNovaSAM_cmake.py.

list runNovaSAM_cmake.cmdList = ['nova']

Definition at line 334 of file runNovaSAM_cmake.py.

string runNovaSAM_cmake.dataFlag = "data"

Definition at line 301 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.detector = metadata["online.detector"].lower()

Definition at line 274 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.detectors = dict()

Definition at line 18 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.detId = metadata["nova.detectorid"].lower()

Definition at line 276 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.dh = ifdh.ifdh("http://samweb.fnal.gov:8480/sam/nova/api")

Definition at line 419 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.dirList = os.listdir(".")

Definition at line 410 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.fclFile = args.config

Definition at line 319 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.fclFileObj = open(tmpFclName, 'a')

Definition at line 331 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.fclPath = resolveFclPath(fclFile)

Definition at line 323 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.file_type = metadata["file_type"]

Definition at line 295 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.gErrorIgnoreLevel

Definition at line 16 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.help

Definition at line 207 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.inFile = args.inFile

Definition at line 219 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.inFileBase = os.path.basename(inFile)

Definition at line 223 of file runNovaSAM_cmake.py.

dictionary runNovaSAM_cmake.metadata = samweb.getMetadata(inFileBase)

Definition at line 225 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.nevt = None

Definition at line 315 of file runNovaSAM_cmake.py.

list runNovaSAM_cmake.outList = []

Definition at line 348 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.outName = getOutputFileName(detector, runNum, subNum, streamEntry, release, subversion, dataFlag, tier, nevt)

Definition at line 358 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.output = outTier.split(":")[0]

Definition at line 352 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.outTier

Definition at line 344 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.parser = argparse.ArgumentParser(description='Run the nova command using SAM metadata')

Definition at line 206 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.release = os.environ["NOVASOFT_VERSION"]

Definition at line 289 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.retCode = subprocess.call(cmdList)

Definition at line 391 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.run_match = run_pattern.match(inFileBase)

Definition at line 240 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.run_pattern = re.compile(r"^.*?_r([0-9]+)_s([0-9]+).+")

Definition at line 239 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.runNum = int(metadata["runs"][0][0])

Definition at line 236 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.samweb = samweb_client.SAMWebClient(experiment='nova')

Definition at line 221 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.skip_match = _skip_pattern.match(file)

Definition at line 412 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.str

Definition at line 209 of file runNovaSAM_cmake.py.

Referenced by createMetadata().

runNovaSAM_cmake.stream = int(metadata["data_stream"])

Definition at line 253 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.stream_match = stream_pattern.match(inFileBase)

Definition at line 265 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.stream_pattern = re.compile(r"^.*?_(dd.*?)[_,.].+", re.IGNORECASE)

Definition at line 264 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.streamEntry = 't{0:02d}'

Definition at line 254 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.subNum = int(metadata["runs"][0][1])

Definition at line 237 of file runNovaSAM_cmake.py.

int runNovaSAM_cmake.subversion = os.environ["NPASS"]

Definition at line 309 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.tier = outTier.split(":")[1]

Definition at line 353 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.tmpFclName = os.path.basename(fclPath).replace(".fcl", "_" + os.path.splitext(inFileBase)[0] + ".fcl")

Definition at line 326 of file runNovaSAM_cmake.py.

runNovaSAM_cmake.type

Definition at line 207 of file runNovaSAM_cmake.py.

Generated on Tue Aug 11 2020 05:19:55 for NOvA by   doxygen 1.8.11