copy_new_pkl_files.py

Go to the documentation of this file.

import os
import glob
verbose = True
print "run  : Copying new pkl files"
########### inputs
in_root = "%s/production_testing/"%os.environ["NOVAPRODVALID_GRID"]
input_folders = glob.glob(in_root+"*")
if verbose:
    print "run  : Input folders"
    for i in input_folders:
        print "run  :     %s"%i
########### outputs
out_root = "%s/testing/"%os.environ["NOVAPRODVALID_WEB"]
output_folders = glob.glob(out_root+"*")
if verbose:
    print "run  : Output folders"
    for o in output_folders:
        print "run  :     %s"%o
o_names = []
for out_folder in output_folders:
    o_names.append(out_folder.split("/")[-1])
if verbose:
    print "run  : Output base names:"
    for o in o_names:
        print "run  :     %s"%o
########### run
for in_folder in input_folders:
    f_name = in_folder.split("/")[-1]
    
    if f_name not in o_names:
        if verbose: print "run  :     input folder %s not in outputs"%f_name
        #c = "cp -r %s %s"%(in_folder,out_root)
        c = "mkdir %s/%s"%(out_root,f_name)
        print c
        os.system(c)
        c = "cp %s/*.pkl %s/%s/"%(in_folder, out_root,f_name) 
        print c
        os.system(c)
    else:
        if verbose: print "run  :     input folder %s is in outputs, checking for new files"%f_name
        out_pkl_files = glob.glob("%s%s/*.pkl"%(out_root,f_name))
        o_pkl_names = []
        for o in out_pkl_files: o_pkl_names.append(o.split("/")[-1])
        if verbose:
            print "run  :     output pkls:"
            for p in o_pkl_names: print "run  :      - %s"%p
            
        in_pkl_files  = glob.glob("%s/*.pkl"%(in_folder))
        for i_pkl in in_pkl_files:
            p_name = i_pkl.split("/")[-1]
            if p_name not in o_pkl_names:
                if verbose: print "run  :      - input pkl %s not in outputs"%p_name
                c = "cp %s %s%s/"%(i_pkl,out_root,f_name)
                #print c
                os.system(c)
            else:
                if verbose: print "run  :      - input pkl %s is in outputs"%p_name