copy_new_pkl_files.py
Go to the documentation of this file.
1 import os
2 import glob
3 verbose = True
4 print "run : Copying new pkl files"
5 ########### inputs
6 in_root = "%s/production_testing/"%os.environ["NOVAPRODVALID_GRID"]
7 input_folders = glob.glob(in_root+"*")
8 if verbose:
9  print "run : Input folders"
10  for i in input_folders:
11  print "run : %s"%i
12 ########### outputs
13 out_root = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
14 output_folders = glob.glob(out_root+"*")
15 if verbose:
16  print "run : Output folders"
17  for o in output_folders:
18  print "run : %s"%o
19 o_names = []
20 for out_folder in output_folders:
21  o_names.append(out_folder.split("/")[-1])
22 if verbose:
23  print "run : Output base names:"
24  for o in o_names:
25  print "run : %s"%o
26 ########### run
27 for in_folder in input_folders:
28  f_name = in_folder.split("/")[-1]
29 
30  if f_name not in o_names:
31  if verbose: print "run : input folder %s not in outputs"%f_name
32  #c = "cp -r %s %s"%(in_folder,out_root)
33  c = "mkdir %s/%s"%(out_root,f_name)
34  print c
35  os.system(c)
36  c = "cp %s/*.pkl %s/%s/"%(in_folder, out_root,f_name)
37  print c
38  os.system(c)
39  else:
40  if verbose: print "run : input folder %s is in outputs, checking for new files"%f_name
41  out_pkl_files = glob.glob("%s%s/*.pkl"%(out_root,f_name))
42  o_pkl_names = []
43  for o in out_pkl_files: o_pkl_names.append(o.split("/")[-1])
44  if verbose:
45  print "run : output pkls:"
46  for p in o_pkl_names: print "run : - %s"%p
47 
48  in_pkl_files = glob.glob("%s/*.pkl"%(in_folder))
49  for i_pkl in in_pkl_files:
50  p_name = i_pkl.split("/")[-1]
51  if p_name not in o_pkl_names:
52  if verbose: print "run : - input pkl %s not in outputs"%p_name
53  c = "cp %s %s%s/"%(i_pkl,out_root,f_name)
54  #print c
55  os.system(c)
56  else:
57  if verbose: print "run : - input pkl %s is in outputs"%p_name
58