findDuplicateFiles.py

Go to the documentation of this file.

#!/bin/env python

import samweb_client
import itertools
import re
from operator import itemgetter, attrgetter

samweb = samweb_client.SAMWebClient(experiment='nova')

for stream in [0,2]:
    pattern = re.compile(r"^.*?_r(.*?)_s(.*?)_.*")
    query = "Nova.DetectorID = fd and data_tier = artdaq and data_stream = " + stream + " and file_type = importedDetector and run_number >= 11496 and DAQ2RawDigit.base_release = S14-01-20"
    listResult = samweb.listFiles(query)

    fileList = []

    for file in listResult:
        match = pattern.match(file)
        if match == None:
            print "None found"
        else:
            run = int(match.groups()[0])
            subrun = int(match.groups()[1])
            fileInfo = (run, subrun, file)
            fileList.append(fileInfo)
        
            fileList.sort(key=itemgetter(0,1))

        count = 0
        lastRun = 0
        lastSubrun = 0
        lastFile = ""
        for fileInfo in fileList:
            run = fileInfo[0]
            subrun = fileInfo[1]
            file = fileInfo[2]
    
        if count > 1 and run == lastRun and subrun == lastSubrun:
            print "Duplicate found!"
            print "File 1: ", file
            print "File 2: ", lastFile
            samweb.retireFile(lastFile)        
            print "-----------------------------"

        lastRun = run
        lastSubrun = subrun
        lastFile = file
        count = count + 1

        
print "Done!"