39 atexit.register(exit_handler)
40 print(
"NOvA Blessed Plots Web Backend")
43 parser = argparse.ArgumentParser(description =
'NOvA Blessed Plots Backend')
45 parser.add_argument(
'--docid',
48 help =
'Process only a single docdb document (by id)')
49 parser.add_argument(
'--metaonly',
52 action =
'store_true',
53 help =
'Only update document meta information (no plot/thumbnail processing)')
54 parser.add_argument(
'--force-regen',
57 action =
'store_true',
58 help =
'Force reprocessing of documents')
59 parser.add_argument(
'--full-lookback',
60 dest =
'full_lookback',
62 action =
'store_true',
63 help =
'Look for DocDB changes that occured anytime in the past.')
64 args = parser.parse_args()
77 print(
"Getting doc IDs for category {}".
format(query_category))
79 if (args.docid !=
None):
80 doc_ids = [args.docid]
82 doc_ids = docdb.GetDocIDsByCategory(query_category, args.full_lookback)
86 print(
'No new or revised documents found. All done!')
94 print(
"Getting metadata for {} documents".
format(len(doc_ids)))
98 for doc_id
in doc_ids:
100 utils.progressBar(counter, len(doc_ids), doc_id[
'id'])
104 c.execute(
"SELECT revision,mod_time FROM bp_docs WHERE id={}".
format(doc_id[
'id']))
109 changed =
True if res[0][0] != doc_id[
'revision']
or res[0][1] != doc_id[
'modtime']
else False 114 if (changed
or not tracked
or args.regen):
115 docToProcess = docdb.GetDocument(doc_id[
'id'])
117 if config.DOCDB_USER
not in docToProcess.modifygroup:
continue 118 if docToProcess.categories ==
None:
continue 121 if docToProcess.categories ==
None:
124 catlist = [cat[
'id']
for cat
in docToProcess.categories] + [cat[
'p_id']
for cat
in docToProcess.categories]
125 if query_category
not in catlist:
126 docsToNotTrack.append(docToProcess.id)
129 for cat
in docToProcess.categories:
131 c.execute(
"INSERT OR IGNORE INTO bp_categories VALUES (?, ?, ?, ?)", (cat[
'id'], cat[
'p_id'], cat[
'name'], cat[
'description']))
134 c.execute(
"INSERT OR REPLACE INTO bp_docs_categories VALUES (?, ?)", (docToProcess.id, cat[
'id']))
137 docsToProcess.append(docToProcess)
142 c.execute(
"SELECT id FROM bp_docs;")
144 trackedDocs = [doc[0]
for doc
in res]
147 if doc[
'id']
in docsToNotTrack:
148 print(
'\033[01;34mINFO:\033[00m Will not track docid {} -- it is not a blessed plot package.'.
format(doc[
'id']))
150 docsToTrack.append(
int(doc[
'id']))
154 for doc
in trackedDocs:
156 if args.docid ==
None and args.full_lookback
and doc
not in docsToTrack:
157 print "\033[01;34mINFO:\033[00m Doc {} no longer blessed. Removing from database.".
format(doc)
158 c.execute(
"DELETE FROM bp_docs WHERE id={}".
format(doc))
159 c.execute(
"DELETE FROM bp_docs_preview WHERE doc_id={}".
format(doc))
160 c.execute(
"DELETE FROM bp_plots WHERE docdb_id={}".
format(doc))
163 print "Found {} new or modified documents".
format(len(docsToProcess))
165 for doc
in docsToProcess:
167 print "\nProcessing DocDB #{} ({} of {})".
format(doc.id, doc_counter, len(docsToProcess))
170 print " \033[01;34mINFO:\033[00m This document is DEPRECATED. All contained plots are considered obsolete." 172 if not args.metaonly:
175 tempDocDir =
'/tmp/{}'.
format(uuid.uuid4())
176 os.mkdir(tempDocDir, 0775)
179 print " Downloading & extracting files" 180 docdbArchiveFile =
'archive_{}.tar.gz'.
format(doc.id)
181 docdb.DownloadArchive(doc.id, docdbArchiveFile, tempDocDir)
185 tar = tarfile.open(docdbArchiveFile,
'r') 188 os.remove(docdbArchiveFile) 191 for root, dirs, files
in os.walk(
'.'):
193 for archive
in files:
194 if os.path.splitext(archive)[1][1:]
not in config.ZIP_EXTS:
continue 197 if archive.find(
'.zip') > 0:
198 print " Found embedded archive: {}. Extracting...".
format(archive)
200 zip = zipfile.ZipFile(archive,
'r') 201 except Exception
as e:
202 print(
' Error: could not extract archive')
207 print " Found embedded archive: {}. Extracting...".
format(archive)
209 tar = tarfile.open(archive,
'r') 210 except Exception
as e:
211 print(
' Error: could not extract archive')
217 extractedFiles = utils.ListBlessedFilesInDir(
'.')
220 if not os.path.isdir(config.WEB_PLOTS): os.mkdir(config.WEB_PLOTS, 0775)
221 docPlotDir = config.WEB_PLOTS +
'/{}'.
format(doc.id)
222 if not os.path.isdir(docPlotDir): os.mkdir(docPlotDir, 0775)
225 for file
in os.listdir(docPlotDir):
226 path = os.path.join(docPlotDir, file)
228 if os.path.isfile(path):
230 except Exception
as e:
234 for file
in extractedFiles:
236 shutil.copy2(file,
'{}/'.
format(docPlotDir))
239 shutil.rmtree(tempDocDir)
242 for capfile
in utils.ListCaptionFilesInDir(
'.'):
243 plotname = os.path.splitext(os.path.basename(capfile))[0]
245 filelist = utils.ListPlotsInDirForCaption(
'.', capfile);
247 if (len(filelist) == 0):
continue 249 plot =
Plot(plotname, capfile, filelist)
253 print " \033[01;33mWARNING:\033[00m This document contains NO valid plots. Please contact the author ({}).".
format(doc.first_author)
255 print " This document contains {} plots".
format(len(plots))
258 plotzip = zipfile.ZipFile(
'blessed_doc{}.zip'.
format(doc.id),
'w')
261 readmeFilename =
'DEPRECATED-README.txt' 262 readmeFile =
open(readmeFilename,
'w')
263 readmeFile.write(
'THESE PLOTS HAVE BEEN BLESSED, BUT ARE CONSIDERED OBSOLETE.')
264 readmeFile.write(
'DO NOT USE THESE PLOTS IN TALKS.')
266 plotzip.write(readmeFilename)
269 c.execute(
"DELETE FROM bp_plots WHERE docdb_id={}".
format(doc.id))
273 src, ext = plot.PreferredExtension()
274 srcBase = os.path.splitext(os.path.basename(src))[0]
277 print " Converting [{}/{}] {}".
format(plot_counter, len(plots), src)
278 os.system(
"convert -density 300 '{}' -trim +repage '{}.png'".
format(src, srcBase))
279 plot.files.append(
'{}.png'.
format(srcBase))
281 print " No format conversion needed: {}".
format(src)
284 print " Converting [{}/{}] {}".
format(plot_counter, len(plots), src)
285 os.system(
"convert '{}' -resize 1440> '{}.png'".
format(src, srcBase))
286 plot.files.append(
'{}.png'.
format(srcBase))
288 os.system(
"convert -flatten '{}.png' -resize '400x400^' -trim +repage '{}_thumb.png'".
format(srcBase, srcBase))
291 c.execute(
"INSERT OR REPLACE INTO bp_plots ('id', 'plot_name', 'docdb_id', 'caption') VALUES (" 292 "(SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}), " 295 "?)".
format(plot.name, doc.id), (plot.name, doc.id, plot.caption))
299 c.execute(
"SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}".
format(plot.name, doc.id))
304 c.execute(
"DELETE FROM bp_plots_filenames WHERE plot_id={}".
format(plot_id))
306 plotzip.write(plot.capfile)
307 for filename
in plot.files:
308 plotzip.write(filename)
309 c.execute(
"INSERT OR REPLACE INTO bp_plots_filenames ('plot_id', 'filename') VALUES (?, ?)", (plot_id, filename))
313 print " Zipping up plots and captions" 317 c.execute(
"INSERT OR REPLACE INTO bp_docs('id', 'name', 'revision', 'author_id', 'submitter_id', 'mod_time', 'abstract', 'blessed', 'deprecated') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (doc.id, doc.title, doc.rev, doc.first_author_id, doc.submitter_id, doc.modtime, doc.abstract, doc.is_blessed, doc.is_dep))
319 for author
in doc.authors:
321 c.execute(
"INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (
int(author[
'@id']), author[
'fullname']))
322 c.execute(
"INSERT OR IGNORE INTO bp_docs_authors VALUES (?, ?)", (doc.id,
int(author[
'@id'])))
324 c.execute(
"INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (doc.submitter_id, doc.submitter))
327 c.execute(
"INSERT OR REPLACE INTO bp_docs_preview ('doc_id', 'plot_id') VALUES (?, (SELECT id FROM bp_plots WHERE docdb_id={} LIMIT 1))".
format(doc.id), (doc.id,))
330 print "Finalizing database" 333 print "\n\033[01;32mAll done.\033[00m Cheers!"
====================================================================== ///
std::string format(const int32_t &value, const int &ndigits=8)
procfile open("FD_BRL_v0.txt")