update_sam_good_runs_metadata.py

Go to the documentation of this file.

#!/usr/bin/env python

import os
import json
import samweb_client
import sys


#cuts = ' and (data_tier raw or data_tier artdaq) and (online.stream 0 or online.stream 2)'
cuts = ' and data_tier raw and online.stream 0'


def percentStr(x, y):
    return str(.1*int((1000*x+.5)/y))+'%'

def bulkMark(samweb, files, flag):
    assert flag == 'true' or flag == 'false'

    if len(files) == 0: return

    # Undocumented bulk metadata update. Private mail from Robert I

    mdlist = [ {'file_name': f, 'dq.isgoodrun': flag} for f in files ]

    samweb.putURL('/files',
                  params = {'continue_on_error': 1},
                  data = json.dumps(mdlist),
                  content_type = 'application/json',
                  secure = True,
                  role='*')


if len(sys.argv) != 4:
    print 'Usage: update_sam_good_runs_metadata.py [neardet|fardet] GRL.txt BRL.txt'
    sys.exit(1)

det = sys.argv[1]
assert det == 'neardet' or det == 'fardet'

goodname = sys.argv[2]
badname = sys.argv[3]

print 'Reading', goodname
fgood = open(goodname, 'r')

good = set()

for line in fgood:
    fields = line.split('\t') # ' '
    assert len(fields) == 2 #6

    run = int(fields[0])
    subrun = int(fields[1])
    assert subrun >= 0 and subrun <= 63

    good.update([(run, subrun)])

print len(good), 'good subruns in input file'


print 'Reading', badname
fbad = open(badname, 'r')

bad = set()

for line in fbad:
    fields = line.split('\t') # ' '
    assert len(fields) == 2 #6

    run = int(fields[0])
    subrun = int(fields[1])
    assert subrun >= 0 and subrun <= 63

    bad.update([(run, subrun)])


print len(bad), 'bad subruns in input file'

print

print 'Do you have all the necessary tickets? You\'ll be sad in 20 minutes or so if you don\'t.'

print

samweb = samweb_client.SAMWebClient(experiment='nova')

print 'Getting list of bad-runs files from SAM. This will take some time'
dbBad = samweb.listFiles('online.detector '+det+' and dq.isgoodrun false' + cuts)

print 'Found', len(dbBad), 'bad-runs files'

toMarkGood = []

for fname in dbBad:
    try:
        fields = fname.split('_')
        run = int(fields[1][1:])
        subrun = int(fields[2][1:])
    except:
        print "Couldn't parse filename", fname
        continue

    if (run, subrun) in good:
        toMarkGood += [fname]
    else:
        # Presumably bad, make sure
        if (run, subrun) not in bad:
            print 'Warning:', fname, 'not in either good or bad list'

print 'Need to flip goodruns field in', len(toMarkGood), 'of them ('+percentStr(len(toMarkGood), len(dbBad))+')'

print

print 'Getting list of good-runs files from SAM. This will take some time'
dbGood = samweb.listFiles('online.detector '+det+' and dq.isgoodrun true' + cuts)

print 'Found', len(dbGood), 'good-runs files'

toMarkBad = []

for fname in dbGood:
    try:
        fields = fname.split('_')
        run = int(fields[1][1:])
        subrun = int(fields[2][1:])
    except:
        print "Couldn't parse filename", fname
        continue

    if (run, subrun) in bad:
        toMarkBad += [fname]
    else:
        # Presumably good, make sure
        if (run, subrun) not in good:
            print 'Warning:', fname, 'not in good or bad list'

print 'Need to flip goodruns field in', len(toMarkBad), 'of them ('+percentStr(len(toMarkBad), len(dbGood))+')'

print

stride = 100

for i in range(0, len(toMarkGood), stride):
    print 'Marking good:', i, '/', len(toMarkGood)
    bulkMark(samweb, toMarkGood[i:i+stride], 'true')

for i in range(0, len(toMarkBad), stride):
    print 'Marking bad:', i, '/', len(toMarkBad)
    bulkMark(samweb, toMarkBad[i:i+stride], 'false')

#for fname in toMarkGood:
#    print 'Marking', fname, 'good'
#    samweb.modifyFileMetadata(fname, {"dq.isgoodrun": "true"})

#for fname in toMarkBad:
#    print 'Marking', fname, 'bad'
#    samweb.modifyFileMetadata(fname, {"dq.isgoodrun": "false"})