Functions
def	exit_handler ()

def	main ()

Variables
	pid = str(os.getpid())

string	pidfile = "/tmp/blessedplotsrunner.pid"

Function Documentation

def BlessedPlots.exit_handler ( )

Definition at line 34 of file BlessedPlots.py.

 def exit_handler():
     if os.path.isfile(pidfile):
         os.unlink(pidfile)
 

def BlessedPlots.main ( )

Definition at line 38 of file BlessedPlots.py.

References novadaq::HexUtils.format(), makeTrainCVSamples.int, open(), and print.

 def main():
     atexit.register(exit_handler)
     print("NOvA Blessed Plots Web Backend")
 
     # Parse command line arguments
     parser = argparse.ArgumentParser(description = 'NOvA Blessed Plots Backend')
 
     parser.add_argument('--docid',
                         dest    = 'docid',
                         action  = 'store',
                         help    = 'Process only a single docdb document (by id)')
     parser.add_argument('--metaonly',
                         dest    = 'metaonly',
                         default = False,
                         action  = 'store_true',
                         help    = 'Only update document meta information (no plot/thumbnail processing)')
     parser.add_argument('--force-regen',
                         dest    = 'regen',
                         default = False,
                         action  = 'store_true',
                         help    = 'Force reprocessing of documents')
     parser.add_argument('--full-lookback',
                         dest    = 'full_lookback',
                         default = False,
                         action  = 'store_true',
                         help    = 'Look for DocDB changes that occured anytime in the past.')
     args = parser.parse_args()
 
     # Set the umask to set permissions of any created files/directories
     os.umask(002)
 
     # Create our database interface object
     db = BlessedPlotsDB()
 
     # Create our DocDB interface and class object
     docdb = DocDB()
 
     # Get all doc ids for the "BlessedPlots" category (cat ID=10).
     query_category = 10
     print("Getting doc IDs for category {}".format(query_category))
 
     if (args.docid != None):
         doc_ids = [args.docid]
     else:
         doc_ids = docdb.GetDocIDsByCategory(query_category, args.full_lookback)
         if doc_ids != []:
                 time.sleep(5)
         else:
                 print('No new or revised documents found. All done!')
                 sys.exit()
 
 
 
     # Loop over doc ids and get the full document meta data
     docs    = []
     counter = 0
     print("Getting metadata for {} documents".format(len(doc_ids)))
 
     docsToNotTrack = []
     docsToProcess = []
     for doc_id in doc_ids:
         counter += 1
         utils.progressBar(counter, len(doc_ids), doc_id['id'])
 
         # Place a cursor, where we can execute queries
         c = db.conn.cursor()
         c.execute("SELECT revision,mod_time FROM bp_docs WHERE id={}".format(doc_id['id']))
         res = c.fetchall()
 
         if len(res) > 0:
             tracked = True
             changed = True if res[0][0] != doc_id['revision'] or res[0][1] != doc_id['modtime'] else False
         else:
             tracked = False
             changed = False
 
         if (changed or not tracked or args.regen):
             docToProcess = docdb.GetDocument(doc_id['id'])
             time.sleep(2)
             if config.DOCDB_USER not in docToProcess.modifygroup: continue
             if docToProcess.categories == None: continue
 
             # Check that this doc is still in the blessed category (work-around for DocDB bug)
             if docToProcess.categories == None:
                 catlist = []
             else:
                 catlist = [cat['id'] for cat in docToProcess.categories] + [cat['p_id'] for cat in docToProcess.categories]
             if query_category not in catlist:
                 docsToNotTrack.append(docToProcess.id)
                 continue
 
             for cat in docToProcess.categories:
                 # Fill bp_categories table
                 c.execute("INSERT OR IGNORE INTO bp_categories VALUES (?, ?, ?, ?)", (cat['id'], cat['p_id'], cat['name'], cat['description']))
 
                 # Fill bp_docs_categories
                 c.execute("INSERT OR REPLACE INTO bp_docs_categories VALUES (?, ?)", (docToProcess.id, cat['id']))
 
             db.conn.commit()
             docsToProcess.append(docToProcess)
 
     print('\n')
 
     # Check for documents that should no longer be tracked
     c.execute("SELECT id FROM bp_docs;")
     res = c.fetchall()
     trackedDocs = [doc[0] for doc in res]
     docsToTrack = []
     for doc in doc_ids:
         if doc['id'] in docsToNotTrack:
             print('\033[01;34mINFO:\033[00m Will not track docid {} -- it is not a blessed plot package.'.format(doc['id']))
         else:
             docsToTrack.append(int(doc['id']))
 
     print('')
 
     for doc in trackedDocs:
         # Do not remove things if we are only processing a single document since we will have no way to know what docs are no longer supposed to be tracked
         if args.docid == None and args.full_lookback and doc not in docsToTrack:
             print "\033[01;34mINFO:\033[00m Doc {} no longer blessed. Removing from database.".format(doc)
             c.execute("DELETE FROM bp_docs WHERE id={}".format(doc))
             c.execute("DELETE FROM bp_docs_preview WHERE doc_id={}".format(doc))
             c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc))
 
     # Loop over files in DocDB document if there are changes
     print "Found {} new or modified documents".format(len(docsToProcess))
     doc_counter = 0
     for doc in docsToProcess:
         doc_counter += 1
         print "\nProcessing DocDB #{} ({} of {})".format(doc.id, doc_counter, len(docsToProcess))
 
         if doc.is_dep:
             print "  \033[01;34mINFO:\033[00m This document is DEPRECATED. All contained plots are considered obsolete."
 
         if not args.metaonly:
 
             # Temp directory to hold files while we examine them
             tempDocDir = '/tmp/{}'.format(uuid.uuid4())
             os.mkdir(tempDocDir, 0775)
 
             # Download the doc tar file to temp directory and move there
             print "  Downloading & extracting files"
             docdbArchiveFile = 'archive_{}.tar.gz'.format(doc.id)
             docdb.DownloadArchive(doc.id, docdbArchiveFile, tempDocDir)
             os.chdir(tempDocDir)
 
             # Extract archive and then remove it
             tar = tarfile.open(docdbArchiveFile,'r')
             tar.extractall()
             tar.close()
             os.remove(docdbArchiveFile)
 
             # Scan dirs and files for other archives
             for root, dirs, files in os.walk('.'):
                 # Extract any found archives
                 for archive in files:
                     if os.path.splitext(archive)[1][1:] not in config.ZIP_EXTS: continue
 
 
                     if archive.find('.zip') > 0:
                         print "  Found embedded archive: {}. Extracting...".format(archive)
                         try:
                             zip = zipfile.ZipFile(archive, 'r')
                         except Exception as e:
                             print('  Error: could not extract archive')
                             continue
                         zip.extractall()
                         zip.close()
                     else:
                         print "  Found embedded archive: {}. Extracting...".format(archive)
                         try:
                             tar = tarfile.open(archive, 'r')
                         except Exception as e:
                             print('  Error: could not extract archive')
                             continue
                         tar.extractall()
                         tar.close()
 
             # Walk the directory structure
             extractedFiles = utils.ListBlessedFilesInDir('.')
 
             # Create subdirectories in web area, if needed
             if not os.path.isdir(config.WEB_PLOTS): os.mkdir(config.WEB_PLOTS, 0775)
             docPlotDir = config.WEB_PLOTS + '/{}'.format(doc.id)
             if not os.path.isdir(docPlotDir): os.mkdir(docPlotDir, 0775)
 
             # Remove any files that are currently in the plot dir from previous version
             for file in os.listdir(docPlotDir):
                 path = os.path.join(docPlotDir, file)
                 try:
                     if os.path.isfile(path):
                         os.unlink(path)
                 except Exception as e:
                     print(e)
 
             # Copy files from temp area to web area
             for file in extractedFiles:
                 os.chmod(file, 0644)
                 shutil.copy2(file, '{}/'.format(docPlotDir))
 
             os.chdir(docPlotDir)
             shutil.rmtree(tempDocDir)
 
             plots = []
             for capfile in utils.ListCaptionFilesInDir('.'):
                 plotname = os.path.splitext(os.path.basename(capfile))[0]
 
                 filelist = utils.ListPlotsInDirForCaption('.', capfile);
                 # Catch the case where there is a txt file, but no associated plot
                 if (len(filelist) == 0): continue
 
                 plot = Plot(plotname, capfile, filelist)
                 plots.append(plot)
 
             if len(plots) == 0:
                 print "  \033[01;33mWARNING:\033[00m This document contains NO valid plots. Please contact the author ({}).".format(doc.first_author)
             else:
                 print "  This document contains {} plots".format(len(plots))
 
                 # Create zip file for easy downloading
                 plotzip = zipfile.ZipFile('blessed_doc{}.zip'.format(doc.id), 'w')
 
             if doc.is_dep:
                 readmeFilename = 'DEPRECATED-README.txt'
                 readmeFile = open(readmeFilename, 'w')
                 readmeFile.write('THESE PLOTS HAVE BEEN BLESSED, BUT ARE CONSIDERED OBSOLETE.')
                 readmeFile.write('DO NOT USE THESE PLOTS IN TALKS.')
                 readmeFile.close()
                 plotzip.write(readmeFilename)
 
             # Clear out plot DB entries associated with this DocDB
             c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc.id))
             plot_counter = 0
             for plot in plots:
                 plot_counter += 1
                 src, ext = plot.PreferredExtension()
                 srcBase = os.path.splitext(os.path.basename(src))[0]
 
                 if ext == 'pdf':
                     print "  Converting [{}/{}] {}".format(plot_counter, len(plots), src)
                     os.system("convert -density 300 '{}' -trim +repage '{}.png'".format(src, srcBase))
                     plot.files.append('{}.png'.format(srcBase))
                 elif ext == 'png':
                     print "  No format conversion needed: {}".format(src)
                     pass
                 else:
                     print "  Converting [{}/{}] {}".format(plot_counter, len(plots), src)
                     os.system("convert '{}' -resize 1440> '{}.png'".format(src, srcBase))
                     plot.files.append('{}.png'.format(srcBase))
 
                 os.system("convert -flatten '{}.png' -resize '400x400^' -trim +repage '{}_thumb.png'".format(srcBase, srcBase))
 
                 # Fill bp_plots table
                 c.execute("INSERT OR REPLACE INTO bp_plots ('id', 'plot_name', 'docdb_id', 'caption') VALUES ("
                 "(SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}), "
                 "?, "
                 "?, "
                 "?)".format(plot.name, doc.id), (plot.name, doc.id, plot.caption))
                 db.conn.commit()
 
                 # Get this plot_id
                 c.execute("SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}".format(plot.name, doc.id))
                 res = c.fetchall()
                 plot_id = res[0][0]
 
                 # Clear out filename DB entries associated with this plot
                 c.execute("DELETE FROM bp_plots_filenames WHERE plot_id={}".format(plot_id))
                 # Fill bp_plots_filenames table and add to zip file
                 plotzip.write(plot.capfile)
                 for filename in plot.files:
                     plotzip.write(filename)
                     c.execute("INSERT OR REPLACE INTO bp_plots_filenames ('plot_id', 'filename') VALUES (?, ?)", (plot_id, filename))
                     db.conn.commit()
 
             if len(plots) > 0:
                 print "  Zipping up plots and captions"
                 plotzip.close()
 
         # Fill bp_doc table
         c.execute("INSERT OR REPLACE INTO bp_docs('id', 'name', 'revision', 'author_id', 'submitter_id', 'mod_time', 'abstract', 'blessed', 'deprecated') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (doc.id, doc.title, doc.rev, doc.first_author_id, doc.submitter_id, doc.modtime, doc.abstract, doc.is_blessed, doc.is_dep))
 
         for author in doc.authors:
             # Fill bp_authors table
             c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (int(author['@id']), author['fullname']))
             c.execute("INSERT OR IGNORE INTO bp_docs_authors VALUES (?, ?)", (doc.id, int(author['@id'])))
 
         c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (doc.submitter_id, doc.submitter))
 
         # Choose a plot from this doc to be the default thumbnail
         c.execute("INSERT OR REPLACE INTO bp_docs_preview ('doc_id', 'plot_id') VALUES (?, (SELECT id FROM bp_plots WHERE docdb_id={} LIMIT 1))".format(doc.id), (doc.id,))
         time.sleep(5)
 
     print "Finalizing database"
     db.conn.commit()
 
     print "\n\033[01;32mAll done.\033[00m Cheers!"
 

Variable Documentation

BlessedPlots.pid = str(os.getpid())

Definition at line 26 of file BlessedPlots.py.

string BlessedPlots.pidfile = "/tmp/blessedplotsrunner.pid"

Definition at line 27 of file BlessedPlots.py.

Functions

Variables

Function Documentation

Variable Documentation