Classes | Functions | Variables
run_unfold_systs_multiprocess Namespace Reference

Classes

class  Consumer
 
class  Task
 

Functions

def make_cmd (nnodes, ranks_per_node, procid, nfiles_per_rank, systs_batch_idx, idx_first_file, totfiles)
 
def RunMultiProcess (nCMD, lCMD, num_consumers=0)
 

Variables

 parser
 
 default
 
 type
 
 help
 
 args = parser.parse_args()
 
 nranks_per_node = multiprocessing.cpu_count()
 
 jsub_id = os.environ['JOBSUBJOBID']
 
 jid = jsub_id.split('@')[0].split('.')[0]
 
 procid = jsub_id.split('@')[0].split('.')[1]
 
 ncmd
 
 cmd
 

Function Documentation

def run_unfold_systs_multiprocess.make_cmd (   nnodes,
  ranks_per_node,
  procid,
  nfiles_per_rank,
  systs_batch_idx,
  idx_first_file,
  totfiles 
)

Definition at line 7 of file run_unfold_systs_multiprocess.py.

References novadaq::HexUtils.format(), makeTrainCVSamples.int, and PandAna.Demos.demo1.range.

7 def make_cmd(nnodes, ranks_per_node, procid, nfiles_per_rank, systs_batch_idx, idx_first_file, totfiles):
8 
9  cmd = []
10  name_cmd = []
11  covered = False
12  nfiles_last_batch = totfiles%nfiles_per_rank
13 
14  # if int(procid) == ranks_per_node:
15  # ifilefirst = int(procid) * nfiles_per_rank * ranks_per_node + irank * nfiles_per_rank
16  # ifilelast = ifilefirst + nfiles_per_rank - 1
17  # else:
18  # ifilefirst = totfiles - nfiles_last_batch
19  # ifilelast = totfiles - 1
20 
21  for irank in range(ranks_per_node):
22  if nnodes > 1 and int(procid) == nnodes - 1 and irank == ranks_per_node - 1:
23  ifilefirst = totfiles - nfiles_last_batch + idx_first_file
24  ifilelast = ifilefirst + nfiles_last_batch -1
25  else:
26  ifilefirst = int(procid) * nfiles_per_rank * ranks_per_node + irank * nfiles_per_rank + idx_first_file
27  ifilelast = ifilefirst + nfiles_per_rank - 1
28 
29  icmd = "cafe -bq -nr --numuccinc /development/NDAna/numucc_inc/specprod_numuccinc.C systs -1 -1 -1 -1 1 {} {}".format(
30  ifilefirst, ifilelast)
31  iname_cmd = "ProcID: {} , Systs batch index: {}, cafe -bq -nr --numuccinc /development/NDAna/numucc_inc/specprod_numuccinc.C systs -1 -1 -1 -1 1 {} {}".format(
32  procid, systs_batch_idx, ifilefirst, ifilelast)
33  cmd.append(icmd)
34  name_cmd.append(iname_cmd)
35 
36  return (name_cmd, cmd)
37 
def make_cmd(nnodes, ranks_per_node, procid, nfiles_per_rank, systs_batch_idx, idx_first_file, totfiles)
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
def run_unfold_systs_multiprocess.RunMultiProcess (   nCMD,
  lCMD,
  num_consumers = 0 
)

Definition at line 72 of file run_unfold_systs_multiprocess.py.

References novadaq::HexUtils.format(), print, and PandAna.Demos.demo1.range.

72 def RunMultiProcess(nCMD, lCMD, num_consumers=0):
73  # Establish communication queues
74  tasks = multiprocessing.Queue()
75  results = multiprocessing.Queue()
76 
77  # Start consumers
78  if num_consumers == 0:
79  num_consumers = multiprocessing.cpu_count()
80  print('Creating {} consumers'.format(num_consumers))
81  consumers = [Consumer(tasks, results) for i in range(num_consumers)]
82  for w in consumers:
83  w.start()
84 
85  num_jobs = len(lCMD)
86  for i in range(num_jobs):
87  tasks.put(Task(nCMD[i], lCMD[i]))
88 
89  # Add a poison pill for each consumer
90  for i in range(num_consumers):
91  tasks.put(None)
92  # Start printing results
93  while num_jobs:
94  result = results.get()
95  print('Result: {}'.format(result))
96  num_jobs -= 1
97  return
98 
99 
def RunMultiProcess(nCMD, lCMD, num_consumers=0)
bool print
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14

Variable Documentation

run_unfold_systs_multiprocess.args = parser.parse_args()

Definition at line 120 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.cmd

Definition at line 135 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.default

Definition at line 107 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.help

Definition at line 108 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.jid = jsub_id.split('@')[0].split('.')[0]

Definition at line 129 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.jsub_id = os.environ['JOBSUBJOBID']

Definition at line 128 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.ncmd

Definition at line 135 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.nranks_per_node = multiprocessing.cpu_count()

Definition at line 123 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.parser
Initial value:
1 = argparse.ArgumentParser(
2  prog='make_cafe_cmd',
3  description=,
4  epilog="Questions and Comments to dingpf@fnal.gov")

Definition at line 101 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.procid = jsub_id.split('@')[0].split('.')[1]

Definition at line 130 of file run_unfold_systs_multiprocess.py.

run_unfold_systs_multiprocess.type

Definition at line 107 of file run_unfold_systs_multiprocess.py.