BlessedPlots.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 
3 # BLESSED PLOTS WEB BACKEND
4 #
5 # This script maintains the database of blessed plots for presentation on the
6 # web.
7 #
8 
9 import os
10 import tarfile
11 import zipfile
12 import shutil
13 import uuid #/// Unique uid generator
14 import sys
15 import argparse
16 import atexit
17 import time
18 
19 import bp_config as config #/// User Configuration
20 import bp_utils as utils #/// Utility helper functions
21 from bp_database import * #/// SQLite3 database interface
22 from bp_docdb import * #/// DocDB interface and classes
23 from bp_plot import *
24 
25 # Check for lock file or open one if none exists
26 pid = str(os.getpid())
27 pidfile = "/tmp/blessedplotsrunner.pid"
28 
29 if os.path.isfile(pidfile):
30  print "Detected running instance of Blessed Plots. Exiting."
31  sys.exit()
32 file(pidfile, 'w').write(pid)
33 
35  if os.path.isfile(pidfile):
36  os.unlink(pidfile)
37 
38 def main():
39  atexit.register(exit_handler)
40  print("NOvA Blessed Plots Web Backend")
41 
42  # Parse command line arguments
43  parser = argparse.ArgumentParser(description = 'NOvA Blessed Plots Backend')
44 
45  parser.add_argument('--docid',
46  dest = 'docid',
47  action = 'store',
48  help = 'Process only a single docdb document (by id)')
49  parser.add_argument('--metaonly',
50  dest = 'metaonly',
51  default = False,
52  action = 'store_true',
53  help = 'Only update document meta information (no plot/thumbnail processing)')
54  parser.add_argument('--force-regen',
55  dest = 'regen',
56  default = False,
57  action = 'store_true',
58  help = 'Force reprocessing of documents')
59  parser.add_argument('--full-lookback',
60  dest = 'full_lookback',
61  default = False,
62  action = 'store_true',
63  help = 'Look for DocDB changes that occured anytime in the past.')
64  args = parser.parse_args()
65 
66  # Set the umask to set permissions of any created files/directories
67  os.umask(002)
68 
69  # Create our database interface object
70  db = BlessedPlotsDB()
71 
72  # Create our DocDB interface and class object
73  docdb = DocDB()
74 
75  # Get all doc ids for the "BlessedPlots" category (cat ID=10).
76  query_category = 10
77  print("Getting doc IDs for category {}".format(query_category))
78 
79  if (args.docid != None):
80  doc_ids = [args.docid]
81  else:
82  doc_ids = docdb.GetDocIDsByCategory(query_category, args.full_lookback)
83  if doc_ids != []:
84  time.sleep(5)
85  else:
86  print('No new or revised documents found. All done!')
87  sys.exit()
88 
89 
90 
91  # Loop over doc ids and get the full document meta data
92  docs = []
93  counter = 0
94  print("Getting metadata for {} documents".format(len(doc_ids)))
95 
96  docsToNotTrack = []
97  docsToProcess = []
98  for doc_id in doc_ids:
99  counter += 1
100  utils.progressBar(counter, len(doc_ids), doc_id['id'])
101 
102  # Place a cursor, where we can execute queries
103  c = db.conn.cursor()
104  c.execute("SELECT revision,mod_time FROM bp_docs WHERE id={}".format(doc_id['id']))
105  res = c.fetchall()
106 
107  if len(res) > 0:
108  tracked = True
109  changed = True if res[0][0] != doc_id['revision'] or res[0][1] != doc_id['modtime'] else False
110  else:
111  tracked = False
112  changed = False
113 
114  if (changed or not tracked or args.regen):
115  docToProcess = docdb.GetDocument(doc_id['id'])
116  time.sleep(2)
117  if config.DOCDB_USER not in docToProcess.modifygroup: continue
118  if docToProcess.categories == None: continue
119 
120  # Check that this doc is still in the blessed category (work-around for DocDB bug)
121  if docToProcess.categories == None:
122  catlist = []
123  else:
124  catlist = [cat['id'] for cat in docToProcess.categories] + [cat['p_id'] for cat in docToProcess.categories]
125  if query_category not in catlist:
126  docsToNotTrack.append(docToProcess.id)
127  continue
128 
129  for cat in docToProcess.categories:
130  # Fill bp_categories table
131  c.execute("INSERT OR IGNORE INTO bp_categories VALUES (?, ?, ?, ?)", (cat['id'], cat['p_id'], cat['name'], cat['description']))
132 
133  # Fill bp_docs_categories
134  c.execute("INSERT OR REPLACE INTO bp_docs_categories VALUES (?, ?)", (docToProcess.id, cat['id']))
135 
136  db.conn.commit()
137  docsToProcess.append(docToProcess)
138 
139  print('\n')
140 
141  # Check for documents that should no longer be tracked
142  c.execute("SELECT id FROM bp_docs;")
143  res = c.fetchall()
144  trackedDocs = [doc[0] for doc in res]
145  docsToTrack = []
146  for doc in doc_ids:
147  if doc['id'] in docsToNotTrack:
148  print('\033[01;34mINFO:\033[00m Will not track docid {} -- it is not a blessed plot package.'.format(doc['id']))
149  else:
150  docsToTrack.append(int(doc['id']))
151 
152  print('')
153 
154  for doc in trackedDocs:
155  # Do not remove things if we are only processing a single document since we will have no way to know what docs are no longer supposed to be tracked
156  if args.docid == None and args.full_lookback and doc not in docsToTrack:
157  print "\033[01;34mINFO:\033[00m Doc {} no longer blessed. Removing from database.".format(doc)
158  c.execute("DELETE FROM bp_docs WHERE id={}".format(doc))
159  c.execute("DELETE FROM bp_docs_preview WHERE doc_id={}".format(doc))
160  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc))
161 
162  # Loop over files in DocDB document if there are changes
163  print "Found {} new or modified documents".format(len(docsToProcess))
164  doc_counter = 0
165  for doc in docsToProcess:
166  doc_counter += 1
167  print "\nProcessing DocDB #{} ({} of {})".format(doc.id, doc_counter, len(docsToProcess))
168 
169  if doc.is_dep:
170  print " \033[01;34mINFO:\033[00m This document is DEPRECATED. All contained plots are considered obsolete."
171 
172  if not args.metaonly:
173 
174  # Temp directory to hold files while we examine them
175  tempDocDir = '/tmp/{}'.format(uuid.uuid4())
176  os.mkdir(tempDocDir, 0775)
177 
178  # Download the doc tar file to temp directory and move there
179  print " Downloading & extracting files"
180  docdbArchiveFile = 'archive_{}.tar.gz'.format(doc.id)
181  docdb.DownloadArchive(doc.id, docdbArchiveFile, tempDocDir)
182  os.chdir(tempDocDir)
183 
184  # Extract archive and then remove it
185  tar = tarfile.open(docdbArchiveFile,'r')
186  tar.extractall()
187  tar.close()
188  os.remove(docdbArchiveFile)
189 
190  # Scan dirs and files for other archives
191  for root, dirs, files in os.walk('.'):
192  # Extract any found archives
193  for archive in files:
194  if os.path.splitext(archive)[1][1:] not in config.ZIP_EXTS: continue
195 
196 
197  if archive.find('.zip') > 0:
198  print " Found embedded archive: {}. Extracting...".format(archive)
199  try:
200  zip = zipfile.ZipFile(archive, 'r')
201  except Exception as e:
202  print(' Error: could not extract archive')
203  continue
204  zip.extractall()
205  zip.close()
206  else:
207  print " Found embedded archive: {}. Extracting...".format(archive)
208  try:
209  tar = tarfile.open(archive, 'r')
210  except Exception as e:
211  print(' Error: could not extract archive')
212  continue
213  tar.extractall()
214  tar.close()
215 
216  # Walk the directory structure
217  extractedFiles = utils.ListBlessedFilesInDir('.')
218 
219  # Create subdirectories in web area, if needed
220  if not os.path.isdir(config.WEB_PLOTS): os.mkdir(config.WEB_PLOTS, 0775)
221  docPlotDir = config.WEB_PLOTS + '/{}'.format(doc.id)
222  if not os.path.isdir(docPlotDir): os.mkdir(docPlotDir, 0775)
223 
224  # Remove any files that are currently in the plot dir from previous version
225  for file in os.listdir(docPlotDir):
226  path = os.path.join(docPlotDir, file)
227  try:
228  if os.path.isfile(path):
229  os.unlink(path)
230  except Exception as e:
231  print(e)
232 
233  # Copy files from temp area to web area
234  for file in extractedFiles:
235  os.chmod(file, 0644)
236  shutil.copy2(file, '{}/'.format(docPlotDir))
237 
238  os.chdir(docPlotDir)
239  shutil.rmtree(tempDocDir)
240 
241  plots = []
242  for capfile in utils.ListCaptionFilesInDir('.'):
243  plotname = os.path.splitext(os.path.basename(capfile))[0]
244 
245  filelist = utils.ListPlotsInDirForCaption('.', capfile);
246  # Catch the case where there is a txt file, but no associated plot
247  if (len(filelist) == 0): continue
248 
249  plot = Plot(plotname, capfile, filelist)
250  plots.append(plot)
251 
252  if len(plots) == 0:
253  print " \033[01;33mWARNING:\033[00m This document contains NO valid plots. Please contact the author ({}).".format(doc.first_author)
254  else:
255  print " This document contains {} plots".format(len(plots))
256 
257  # Create zip file for easy downloading
258  plotzip = zipfile.ZipFile('blessed_doc{}.zip'.format(doc.id), 'w')
259 
260  if doc.is_dep:
261  readmeFilename = 'DEPRECATED-README.txt'
262  readmeFile = open(readmeFilename, 'w')
263  readmeFile.write('THESE PLOTS HAVE BEEN BLESSED, BUT ARE CONSIDERED OBSOLETE.')
264  readmeFile.write('DO NOT USE THESE PLOTS IN TALKS.')
265  readmeFile.close()
266  plotzip.write(readmeFilename)
267 
268  # Clear out plot DB entries associated with this DocDB
269  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc.id))
270  plot_counter = 0
271  for plot in plots:
272  plot_counter += 1
273  src, ext = plot.PreferredExtension()
274  srcBase = os.path.splitext(os.path.basename(src))[0]
275 
276  if ext == 'pdf':
277  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
278  os.system("convert -density 300 '{}' -trim +repage '{}.png'".format(src, srcBase))
279  plot.files.append('{}.png'.format(srcBase))
280  elif ext == 'png':
281  print " No format conversion needed: {}".format(src)
282  pass
283  else:
284  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
285  os.system("convert '{}' -resize 1440> '{}.png'".format(src, srcBase))
286  plot.files.append('{}.png'.format(srcBase))
287 
288  os.system("convert -flatten '{}.png' -resize '400x400^' -trim +repage '{}_thumb.png'".format(srcBase, srcBase))
289 
290  # Fill bp_plots table
291  c.execute("INSERT OR REPLACE INTO bp_plots ('id', 'plot_name', 'docdb_id', 'caption') VALUES ("
292  "(SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}), "
293  "?, "
294  "?, "
295  "?)".format(plot.name, doc.id), (plot.name, doc.id, plot.caption))
296  db.conn.commit()
297 
298  # Get this plot_id
299  c.execute("SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}".format(plot.name, doc.id))
300  res = c.fetchall()
301  plot_id = res[0][0]
302 
303  # Clear out filename DB entries associated with this plot
304  c.execute("DELETE FROM bp_plots_filenames WHERE plot_id={}".format(plot_id))
305  # Fill bp_plots_filenames table and add to zip file
306  plotzip.write(plot.capfile)
307  for filename in plot.files:
308  plotzip.write(filename)
309  c.execute("INSERT OR REPLACE INTO bp_plots_filenames ('plot_id', 'filename') VALUES (?, ?)", (plot_id, filename))
310  db.conn.commit()
311 
312  if len(plots) > 0:
313  print " Zipping up plots and captions"
314  plotzip.close()
315 
316  # Fill bp_doc table
317  c.execute("INSERT OR REPLACE INTO bp_docs('id', 'name', 'revision', 'author_id', 'submitter_id', 'mod_time', 'abstract', 'blessed', 'deprecated') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (doc.id, doc.title, doc.rev, doc.first_author_id, doc.submitter_id, doc.modtime, doc.abstract, doc.is_blessed, doc.is_dep))
318 
319  for author in doc.authors:
320  # Fill bp_authors table
321  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (int(author['@id']), author['fullname']))
322  c.execute("INSERT OR IGNORE INTO bp_docs_authors VALUES (?, ?)", (doc.id, int(author['@id'])))
323 
324  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (doc.submitter_id, doc.submitter))
325 
326  # Choose a plot from this doc to be the default thumbnail
327  c.execute("INSERT OR REPLACE INTO bp_docs_preview ('doc_id', 'plot_id') VALUES (?, (SELECT id FROM bp_plots WHERE docdb_id={} LIMIT 1))".format(doc.id), (doc.id,))
328  time.sleep(5)
329 
330  print "Finalizing database"
331  db.conn.commit()
332 
333  print "\n\033[01;32mAll done.\033[00m Cheers!"
334 
335 if __name__ == '__main__':
336  main()
write
Run ND cosmics.
====================================================================== ///
Definition: CutFlow_Data.C:28
bool print
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")
TFile * file
Definition: cellShifts.C:17
def exit_handler()
Definition: BlessedPlots.py:34