Functions | Variables
BlessedPlots Namespace Reference

Functions

def exit_handler ()
 
def main ()
 

Variables

 pid = str(os.getpid())
 
string pidfile = "/tmp/blessedplotsrunner.pid"
 

Function Documentation

def BlessedPlots.exit_handler ( )

Definition at line 34 of file BlessedPlots.py.

35  if os.path.isfile(pidfile):
36  os.unlink(pidfile)
37 
def exit_handler()
Definition: BlessedPlots.py:34
def BlessedPlots.main ( )

Definition at line 38 of file BlessedPlots.py.

Referenced by main(), build_template.rebuild(), build_test.rebuild(), and validation.recovalidate().

38 def main():
39  atexit.register(exit_handler)
40  print("NOvA Blessed Plots Web Backend")
41 
42  # Parse command line arguments
43  parser = argparse.ArgumentParser(description = 'NOvA Blessed Plots Backend')
44 
45  parser.add_argument('--docid',
46  dest = 'docid',
47  action = 'store',
48  help = 'Process only a single docdb document (by id)')
49  parser.add_argument('--metaonly',
50  dest = 'metaonly',
51  default = False,
52  action = 'store_true',
53  help = 'Only update document meta information (no plot/thumbnail processing)')
54  parser.add_argument('--force-regen',
55  dest = 'regen',
56  default = False,
57  action = 'store_true',
58  help = 'Force reprocessing of documents')
59  parser.add_argument('--full-lookback',
60  dest = 'full_lookback',
61  default = False,
62  action = 'store_true',
63  help = 'Look for DocDB changes that occured anytime in the past.')
64  args = parser.parse_args()
65 
66  # Set the umask to set permissions of any created files/directories
67  os.umask(002)
68 
69  # Create our database interface object
70  db = BlessedPlotsDB()
71 
72  # Create our DocDB interface and class object
73  docdb = DocDB()
74 
75  # Get all doc ids for the "BlessedPlots" category (cat ID=10).
76  query_category = 10
77  print("Getting doc IDs for category {}".format(query_category))
78 
79  if (args.docid != None):
80  doc_ids = [args.docid]
81  else:
82  doc_ids = docdb.GetDocIDsByCategory(query_category, args.full_lookback)
83  if doc_ids != []:
84  time.sleep(5)
85  else:
86  print('No new or revised documents found. All done!')
87  sys.exit()
88 
89 
90 
91  # Loop over doc ids and get the full document meta data
92  docs = []
93  counter = 0
94  print("Getting metadata for {} documents".format(len(doc_ids)))
95 
96  docsToNotTrack = []
97  docsToProcess = []
98  for doc_id in doc_ids:
99  counter += 1
100  utils.progressBar(counter, len(doc_ids), doc_id['id'])
101 
102  # Place a cursor, where we can execute queries
103  c = db.conn.cursor()
104  c.execute("SELECT revision,mod_time FROM bp_docs WHERE id={}".format(doc_id['id']))
105  res = c.fetchall()
106 
107  if len(res) > 0:
108  tracked = True
109  changed = True if res[0][0] != doc_id['revision'] or res[0][1] != doc_id['modtime'] else False
110  else:
111  tracked = False
112  changed = False
113 
114  if (changed or not tracked or args.regen):
115  docToProcess = docdb.GetDocument(doc_id['id'])
116  time.sleep(2)
117  if config.DOCDB_USER not in docToProcess.modifygroup: continue
118  if docToProcess.categories == None: continue
119 
120  # Check that this doc is still in the blessed category (work-around for DocDB bug)
121  if docToProcess.categories == None:
122  catlist = []
123  else:
124  catlist = [cat['id'] for cat in docToProcess.categories] + [cat['p_id'] for cat in docToProcess.categories]
125  if query_category not in catlist:
126  docsToNotTrack.append(docToProcess.id)
127  continue
128 
129  for cat in docToProcess.categories:
130  # Fill bp_categories table
131  c.execute("INSERT OR IGNORE INTO bp_categories VALUES (?, ?, ?, ?)", (cat['id'], cat['p_id'], cat['name'], cat['description']))
132 
133  # Fill bp_docs_categories
134  c.execute("INSERT OR REPLACE INTO bp_docs_categories VALUES (?, ?)", (docToProcess.id, cat['id']))
135 
136  db.conn.commit()
137  docsToProcess.append(docToProcess)
138 
139  print('\n')
140 
141  # Check for documents that should no longer be tracked
142  c.execute("SELECT id FROM bp_docs;")
143  res = c.fetchall()
144  trackedDocs = [doc[0] for doc in res]
145  docsToTrack = []
146  for doc in doc_ids:
147  if doc['id'] in docsToNotTrack:
148  print('\033[01;34mINFO:\033[00m Will not track docid {} -- it is not a blessed plot package.'.format(doc['id']))
149  else:
150  docsToTrack.append(int(doc['id']))
151 
152  print('')
153 
154  for doc in trackedDocs:
155  # Do not remove things if we are only processing a single document since we will have no way to know what docs are no longer supposed to be tracked
156  if args.docid == None and args.full_lookback and doc not in docsToTrack:
157  print "\033[01;34mINFO:\033[00m Doc {} no longer blessed. Removing from database.".format(doc)
158  c.execute("DELETE FROM bp_docs WHERE id={}".format(doc))
159  c.execute("DELETE FROM bp_docs_preview WHERE doc_id={}".format(doc))
160  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc))
161 
162  # Loop over files in DocDB document if there are changes
163  print "Found {} new or modified documents".format(len(docsToProcess))
164  doc_counter = 0
165  for doc in docsToProcess:
166  doc_counter += 1
167  print "\nProcessing DocDB #{} ({} of {})".format(doc.id, doc_counter, len(docsToProcess))
168 
169  if doc.is_dep:
170  print " \033[01;34mINFO:\033[00m This document is DEPRECATED. All contained plots are considered obsolete."
171 
172  if not args.metaonly:
173 
174  # Temp directory to hold files while we examine them
175  tempDocDir = '/tmp/{}'.format(uuid.uuid4())
176  os.mkdir(tempDocDir, 0775)
177 
178  # Download the doc tar file to temp directory and move there
179  print " Downloading & extracting files"
180  docdbArchiveFile = 'archive_{}.tar.gz'.format(doc.id)
181  docdb.DownloadArchive(doc.id, docdbArchiveFile, tempDocDir)
182  os.chdir(tempDocDir)
183 
184  # Extract archive and then remove it
185  tar = tarfile.open(docdbArchiveFile,'r')
186  tar.extractall()
187  tar.close()
188  os.remove(docdbArchiveFile)
189 
190  # Scan dirs and files for other archives
191  for root, dirs, files in os.walk('.'):
192  # Extract any found archives
193  for archive in files:
194  if os.path.splitext(archive)[1][1:] not in config.ZIP_EXTS: continue
195 
196 
197  if archive.find('.zip') > 0:
198  print " Found embedded archive: {}. Extracting...".format(archive)
199  zip = zipfile.ZipFile(archive, 'r')
200  zip.extractall()
201  zip.close()
202  else:
203  print " Found embedded archive: {}. Extracting...".format(archive)
204  tar = tarfile.open(archive, 'r')
205  tar.extractall()
206  tar.close()
207 
208  # Walk the directory structure
209  extractedFiles = utils.ListBlessedFilesInDir('.')
210 
211  # Create subdirectories in web area, if needed
212  if not os.path.isdir(config.WEB_PLOTS): os.mkdir(config.WEB_PLOTS, 0775)
213  docPlotDir = config.WEB_PLOTS + '/{}'.format(doc.id)
214  if not os.path.isdir(docPlotDir): os.mkdir(docPlotDir, 0775)
215 
216  # Remove any files that are currently in the plot dir from previous version
217  for file in os.listdir(docPlotDir):
218  path = os.path.join(docPlotDir, file)
219  try:
220  if os.path.isfile(path):
221  os.unlink(path)
222  except Exception as e:
223  print(e)
224 
225  # Copy files from temp area to web area
226  for file in extractedFiles:
227  os.chmod(file, 0644)
228  shutil.copy2(file, '{}/'.format(docPlotDir))
229 
230  os.chdir(docPlotDir)
231  shutil.rmtree(tempDocDir)
232 
233  plots = []
234  for capfile in utils.ListCaptionFilesInDir('.'):
235  plotname = os.path.splitext(os.path.basename(capfile))[0]
236 
237  filelist = utils.ListPlotsInDirForCaption('.', capfile);
238  # Catch the case where there is a txt file, but no associated plot
239  if (len(filelist) == 0): continue
240 
241  plot = Plot(plotname, capfile, filelist)
242  plots.append(plot)
243 
244  if len(plots) == 0:
245  print " \033[01;33mWARNING:\033[00m This document contains NO valid plots. Please contact the author ({}).".format(doc.first_author)
246  else:
247  print " This document contains {} plots".format(len(plots))
248 
249  # Create zip file for easy downloading
250  plotzip = zipfile.ZipFile('blessed_doc{}.zip'.format(doc.id), 'w')
251 
252  if doc.is_dep:
253  readmeFilename = 'DEPRECATED-README.txt'
254  readmeFile = open(readmeFilename, 'w')
255  readmeFile.write('THESE PLOTS HAVE BEEN BLESSED, BUT ARE CONSIDERED OBSOLETE.')
256  readmeFile.write('DO NOT USE THESE PLOTS IN TALKS.')
257  readmeFile.close()
258  plotzip.write(readmeFilename)
259 
260  # Clear out plot DB entries associated with this DocDB
261  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc.id))
262  plot_counter = 0
263  for plot in plots:
264  plot_counter += 1
265  src, ext = plot.PreferredExtension()
266  srcBase = os.path.splitext(os.path.basename(src))[0]
267 
268  if ext == 'pdf':
269  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
270  os.system("convert -density 300 '{}' -trim +repage '{}.png'".format(src, srcBase))
271  plot.files.append('{}.png'.format(srcBase))
272  elif ext == 'png':
273  print " No format conversion needed: {}".format(src)
274  pass
275  else:
276  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
277  os.system("convert '{}' -resize 1440> '{}.png'".format(src, srcBase))
278  plot.files.append('{}.png'.format(srcBase))
279 
280  os.system("convert -flatten '{}.png' -resize '400x400^' -trim +repage '{}_thumb.png'".format(srcBase, srcBase))
281 
282  # Fill bp_plots table
283  c.execute("INSERT OR REPLACE INTO bp_plots ('id', 'plot_name', 'docdb_id', 'caption') VALUES ("
284  "(SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}), "
285  "?, "
286  "?, "
287  "?)".format(plot.name, doc.id), (plot.name, doc.id, plot.caption))
288  db.conn.commit()
289 
290  # Get this plot_id
291  c.execute("SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}".format(plot.name, doc.id))
292  res = c.fetchall()
293  plot_id = res[0][0]
294 
295  # Clear out filename DB entries associated with this plot
296  c.execute("DELETE FROM bp_plots_filenames WHERE plot_id={}".format(plot_id))
297  # Fill bp_plots_filenames table and add to zip file
298  plotzip.write(plot.capfile)
299  for filename in plot.files:
300  plotzip.write(filename)
301  c.execute("INSERT OR REPLACE INTO bp_plots_filenames ('plot_id', 'filename') VALUES (?, ?)", (plot_id, filename))
302  db.conn.commit()
303 
304  if len(plots) > 0:
305  print " Zipping up plots and captions"
306  plotzip.close()
307 
308  # Fill bp_doc table
309  c.execute("INSERT OR REPLACE INTO bp_docs('id', 'name', 'revision', 'author_id', 'submitter_id', 'mod_time', 'abstract', 'blessed', 'deprecated') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (doc.id, doc.title, doc.rev, doc.first_author_id, doc.submitter_id, doc.modtime, doc.abstract, doc.is_blessed, doc.is_dep))
310 
311  for author in doc.authors:
312  # Fill bp_authors table
313  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (int(author['@id']), author['fullname']))
314  c.execute("INSERT OR IGNORE INTO bp_docs_authors VALUES (?, ?)", (doc.id, int(author['@id'])))
315 
316  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (doc.submitter_id, doc.submitter))
317 
318  # Choose a plot from this doc to be the default thumbnail
319  c.execute("INSERT OR REPLACE INTO bp_docs_preview ('doc_id', 'plot_id') VALUES (?, (SELECT id FROM bp_plots WHERE docdb_id={} LIMIT 1))".format(doc.id), (doc.id,))
320  time.sleep(5)
321 
322  print "Finalizing database"
323  db.conn.commit()
324 
325  print "\n\033[01;32mAll done.\033[00m Cheers!"
326 
====================================================================== ///
bool print
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
procfile open("FD_BRL_v0.txt")

Variable Documentation

BlessedPlots.pid = str(os.getpid())

Definition at line 26 of file BlessedPlots.py.

Referenced by remid::ReMIdTrain.analyze(), remid::ReMIdDedx.analyze(), remid::ReMIdDedxRock.analyze(), remid::ReMIdDedxStudies.analyze(), bpfit::BPFTmvaTrainer.analyze(), art::AssnsGroup.AssnsGroup(), fnex::ShifterAndWeighter.CalcNueSystWeights(), caf::Proxy< caf::SRBpfId >.CheckEquals(), caf::Proxy< caf::SRLem >.CheckEquals(), caf::Proxy< caf::SRRemid >.CheckEquals(), caf::Proxy< caf::SRRvp >.CheckEquals(), caf::Proxy< caf::SRBpfTrack >.CheckEquals(), skim::NueSkimmer.CopyProngCVN(), art::PtrMaker< T >.create(), jmshower::EID.EID(), art::Run.endTime(), art::SubRun.endTime(), art::FileCatalogMetadata.FileCatalogMetadata(), flat::FlatLem.Fill(), flat::FlatQepid.Fill(), flat::FlatRvp.Fill(), art::Principal.fillGroup(), flat::FlatLem.FlatLem(), flat::FlatQepid.FlatQepid(), flat::FlatRvp.FlatRvp(), novadaq::ProcessUtils.getChildProcessIds(), art::Group.Group(), om::HeaderBar.HandleTimer(), fnex::ShifterAndWeighter.HornCurrentWeight(), art::ProductMetaData.inputTag(), fnex::ShifterAndWeighter.LoadNueSystHists(), nerd::NERDProng.MakeCluster(), om::PlotOptions.MakeLabelText(), ana.MakeNueSystematicsFile(), art::SourceHelper.makePtr(), nue_background_decomposition(), nue_cafvsfnex_cosmics(), nue_signal_extrapolation(), caf::Proxy< caf::SRBpfId >.operator=(), caf::Proxy< caf::SRLem >.operator=(), caf::Proxy< caf::SRRemid >.operator=(), caf::Proxy< caf::SRRvp >.operator=(), caf::Proxy< caf::SRBpfTrack >.operator=(), ana::NuePlotStyle.PIDBinLabels(), PIDBinLabelsShortAxis(), g4n::ParticleListAction.PreTrackingAction(), lem::LEMAssociator.produce(), lem::MakePID.produce(), rvp::RecVarPID.produce(), xnue::XnuePID.produce(), cheat::MCCheater.produce(), bpfit::BPFEnergyEstimator.produce(), bpfit::BPFEnergyEstimatorOnly.produce(), remid::RecoMuon.produce(), bpfit::BPFPIdMaker.produce(), art::PtrMaker< T >.PtrMaker(), remakeresultsplots_nue(), murem::MuonRemove.RemoveByReMId(), cheat::SimTrack.SetPID(), jmshower::EID.SetSh1PID(), jmshower::EID.SetSh2PID(), systematics_table_from_pred_interp(), train_dectree_caf(), art::ProvenanceCheckerOutput.write(), write_product_node(), and slid::ShowerLID.~ShowerLID().

string BlessedPlots.pidfile = "/tmp/blessedplotsrunner.pid"

Definition at line 27 of file BlessedPlots.py.