Classes | Namespaces | Functions
ScheduleContext.h File Reference
#include "art/Framework/Services/Registry/ServiceMacros.h"
#include "art/Utilities/ScheduleID.h"
#include <atomic>

Go to the source code of this file.

Classes

class  art::ScheduleContext
 

Namespaces

 art
 Service to store calibration data products (CDP) in the SQLite3 metadatabase of a file.
 
 art::detail
 

Functions

int main ()
 MAIN FUNCTION. More...
 

Function Documentation

int main ( )

MAIN FUNCTION.

1D plots hits vs w //

Definition at line 38 of file BlessedPlots.py.

References genie::units::A, garching_flux_to_genie_root::Alpha(), POTGetData::append_json(), g4zmq::apply(), ana::assert(), GetFEBsFromWatchList::buildFEBList(), C, c1_x, c1_y, cd(), getBrightness::cell, CenterTitles(), om::cerr, plot_validation_datamc::Clone(), srt_ntbashutil::compress_path(), MCenterSpills::compute_pot(), nova_uptime::compute_pot(), nova_uptime::compute_uptime(), nova_uptime::connect_to_database(), GetFEBsFromWatchList::convertCsvToPythonObject(), novarwgt::ConvertGenieEvent(), novarwgt::ConvertNuToolsEvent(), novarwgt::ConvertSREvent(), pot-for-runs::count_subruns(), om::cout, DCS_db_parser::database_connect(), POTPopulateData::database_connect(), POTGetData::database_connect(), runNovaSAM::detector, Divide(), POTUptimeData::do_detector_mass(), uptime_graphic::do_fd_runs(), uptime_graphic::do_kiloton(), uptime_graphic::do_pot(), POTUptimeData::do_pot(), POTUptimeData::do_runs(), nova_uptime::downtime_report(), Draw(), E, allTimeWatchdog::endl, errAoverB(), evt, exit(), F, srt_path_util::file_type(), shutoffs::filename, Fill(), check_time_usage::float, novadaq::HexUtils::format(), makeBrightnessMap::fOut, util::frac(), emcalc_gui::g4_configure(), Gaus(), POTPopulateData::get_runs(), POTGetData::get_runs(), POTSpillRate::get_spills(), POTPopulateData::get_spills(), POTGetData::get_spills(), POTPopulateData::get_uptime(), POTGetData::get_uptime(), NNoiseSliceHitPerSRStudy::GetDataFromInputFile(), OnMonTriggerRateStudy::GetDataFromInputFile(), BeamWidthStudy::GetDataFromInputFile(), GetEntries(), cet::getenv(), GetHistMax(), NNoiseSliceHitPerSRStudy::GetInputFilePaths(), OnMonTriggerRateStudy::GetInputFilePaths(), BeamWidthStudy::GetInputFilePaths(), GetFEBsFromWatchList::getRecentFilesList(), GetXaxis(), python::gTerminate(), MECModelEnuComparisons::i, if(), singlekaon_xsec::init(), genie::RunOpt::Instance(), makeTrainCVSamples::int, srt_ntbashutil::is_absolute_path(), kBlue, kGreen, kOrange, genie::kScNull, genie::kScQuasiElastic, genie::kScResonant, Landau(), plotROC::legendX, plotROC::legendY, BlessedPlots::main(), MCenterSpills::mc7_spills_between(), nova_uptime::mc7_spills_between(), extractScale::mean, GetSpillData::merge_max_mappings(), POTSpillRate::num_spills_between(), one(), open(), MCenterSpills::options(), POTUptimeData::options(), POTPopulateData::options(), POTGetData::options(), nova_uptime::options(), confusionMatrixTree::out, path, DCS_db_parser::period, NDAPDHVSetting::plane, POTPopulateData::pot(), POTGetData::pot(), cet::pow(), Preliminary(), print, Print(), uptime_graphic::print_footer(), uptime_graphic::print_header(), uptime_graphic::print_table(), utilities::progressBar(), PandAna.Demos.demo1::range, pot-for-runs::read_file_list(), pot-for-runs::read_list(), GetFEBsFromWatchList::removeDuplicates(), pot-for-runs::run_minmax(), MCenterSpills::run_minmax(), nova_uptime::run_minmax(), nova_uptime::run_pot_summary(), POTPopulateData::run_startstop(), POTGetData::run_startstop(), MCenterSpills::runs_by_day(), nova_uptime::runs_by_day(), pot-for-runs::runs_db(), nova_uptime::runs_for_period(), SetBinContent(), SetLineColor(), SetMarkerColor(), SetTitle(), Simulation(), garching_flux_to_genie_root::SpectralFunction(), pot-for-runs::spills_between(), nova_uptime::spills_between(), sr, submit_syst::str, plotROC::title, novarwgt::test::ToNuSimData(), novarwgt::test::ToStandardRecord(), srt_ntf77::translate_command(), srt_ntbashutil::translate_path(), GetSpillData::unix_timestamp(), googlecode_upload::upload_find_auth(), checkCalibrationCSV::usage(), pot-for-runs::usage(), Validation(), train_latest_snapshot::values, POTSpillRate::view, w, POTUptimeData::write_json(), POTGetData::write_json(), GetFEBsFromWatchList::writeTextFile(), and submit_syst::x.

Referenced by absCal(), absCal_Diblocks(), COLZ(), srt_ntbashutil_t::do_me(), FD_LowLevelPlots(), main(), respCOLZ_WCell(), and respProfiles().

38 def main():
39  atexit.register(exit_handler)
40  print("NOvA Blessed Plots Web Backend")
41 
42  # Parse command line arguments
43  parser = argparse.ArgumentParser(description = 'NOvA Blessed Plots Backend')
44 
45  parser.add_argument('--docid',
46  dest = 'docid',
47  action = 'store',
48  help = 'Process only a single docdb document (by id)')
49  parser.add_argument('--metaonly',
50  dest = 'metaonly',
51  default = False,
52  action = 'store_true',
53  help = 'Only update document meta information (no plot/thumbnail processing)')
54  parser.add_argument('--force-regen',
55  dest = 'regen',
56  default = False,
57  action = 'store_true',
58  help = 'Force reprocessing of documents')
59  parser.add_argument('--full-lookback',
60  dest = 'full_lookback',
61  default = False,
62  action = 'store_true',
63  help = 'Look for DocDB changes that occured anytime in the past.')
65 
66  # Set the umask to set permissions of any created files/directories
67  os.umask(002)
68 
69  # Create our database interface object
70  db = BlessedPlotsDB()
71 
72  # Create our DocDB interface and class object
73  docdb = DocDB()
74 
75  # Get all doc ids for the "BlessedPlots" category (cat ID=10).
76  query_category = 10
77  print("Getting doc IDs for category {}".format(query_category))
78 
79  if (args.docid != None):
80  doc_ids = [args.docid]
81  else:
82  doc_ids = docdb.GetDocIDsByCategory(query_category, args.full_lookback)
83  if doc_ids != []:
84  time.sleep(5)
85  else:
86  print('No new or revised documents found. All done!')
87  sys.exit()
88 
89 
90 
91  # Loop over doc ids and get the full document meta data
92  docs = []
93  counter = 0
94  print("Getting metadata for {} documents".format(len(doc_ids)))
95 
96  docsToNotTrack = []
97  docsToProcess = []
98  for doc_id in doc_ids:
99  counter += 1
100  utils.progressBar(counter, len(doc_ids), doc_id['id'])
101 
102  # Place a cursor, where we can execute queries
103  c = db.conn.cursor()
104  c.execute("SELECT revision,mod_time FROM bp_docs WHERE id={}".format(doc_id['id']))
105  res = c.fetchall()
106 
107  if len(res) > 0:
108  tracked = True
109  changed = True if res[0][0] != doc_id['revision'] or res[0][1] != doc_id['modtime'] else False
110  else:
111  tracked = False
112  changed = False
113 
114  if (changed or not tracked or args.regen):
115  docToProcess = docdb.GetDocument(doc_id['id'])
116  time.sleep(2)
117  if config.DOCDB_USER not in docToProcess.modifygroup: continue
118  if docToProcess.categories == None: continue
119 
120  # Check that this doc is still in the blessed category (work-around for DocDB bug)
121  if docToProcess.categories == None:
122  catlist = []
123  else:
124  catlist = [cat['id'] for cat in docToProcess.categories] + [cat['p_id'] for cat in docToProcess.categories]
125  if query_category not in catlist:
126  docsToNotTrack.append(docToProcess.id)
127  continue
128 
129  for cat in docToProcess.categories:
130  # Fill bp_categories table
131  c.execute("INSERT OR IGNORE INTO bp_categories VALUES (?, ?, ?, ?)", (cat['id'], cat['p_id'], cat['name'], cat['description']))
132 
133  # Fill bp_docs_categories
134  c.execute("INSERT OR REPLACE INTO bp_docs_categories VALUES (?, ?)", (docToProcess.id, cat['id']))
135 
136  db.conn.commit()
137  docsToProcess.append(docToProcess)
138 
139  print('\n')
140 
141  # Check for documents that should no longer be tracked
142  c.execute("SELECT id FROM bp_docs;")
143  res = c.fetchall()
144  trackedDocs = [doc[0] for doc in res]
145  docsToTrack = []
146  for doc in doc_ids:
147  if doc['id'] in docsToNotTrack:
148  print('\033[01;34mINFO:\033[00m Will not track docid {} -- it is not a blessed plot package.'.format(doc['id']))
149  else:
150  docsToTrack.append(int(doc['id']))
151 
152  print('')
153 
154  for doc in trackedDocs:
155  # Do not remove things if we are only processing a single document since we will have no way to know what docs are no longer supposed to be tracked
156  if args.docid == None and args.full_lookback and doc not in docsToTrack:
157  print "\033[01;34mINFO:\033[00m Doc {} no longer blessed. Removing from database.".format(doc)
158  c.execute("DELETE FROM bp_docs WHERE id={}".format(doc))
159  c.execute("DELETE FROM bp_docs_preview WHERE doc_id={}".format(doc))
160  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc))
161 
162  # Loop over files in DocDB document if there are changes
163  print "Found {} new or modified documents".format(len(docsToProcess))
164  doc_counter = 0
165  for doc in docsToProcess:
166  doc_counter += 1
167  print "\nProcessing DocDB #{} ({} of {})".format(doc.id, doc_counter, len(docsToProcess))
168 
169  if doc.is_dep:
170  print " \033[01;34mINFO:\033[00m This document is DEPRECATED. All contained plots are considered obsolete."
171 
172  if not args.metaonly:
173 
174  # Temp directory to hold files while we examine them
175  tempDocDir = '/tmp/{}'.format(uuid.uuid4())
176  os.mkdir(tempDocDir, 0775)
177 
178  # Download the doc tar file to temp directory and move there
179  print " Downloading & extracting files"
180  docdbArchiveFile = 'archive_{}.tar.gz'.format(doc.id)
181  docdb.DownloadArchive(doc.id, docdbArchiveFile, tempDocDir)
182  os.chdir(tempDocDir)
183 
184  # Extract archive and then remove it
185  tar = tarfile.open(docdbArchiveFile,'r')
186  tar.extractall()
187  tar.close()
188  os.remove(docdbArchiveFile)
189 
190  # Scan dirs and files for other archives
191  for root, dirs, files in os.walk('.'):
192  # Extract any found archives
194  if os.path.splitext(archive)[1][1:] not in config.ZIP_EXTS: continue
195 
196 
197  if archive.find('.zip') > 0:
198  print " Found embedded archive: {}. Extracting...".format(archive)
199  try:
200  zip = zipfile.ZipFile(archive, 'r')
201  except Exception as e:
202  print(' Error: could not extract archive')
203  continue
204  zip.extractall()
205  zip.close()
206  else:
207  print " Found embedded archive: {}. Extracting...".format(archive)
208  try:
209  tar = tarfile.open(archive, 'r')
210  except Exception as e:
211  print(' Error: could not extract archive')
212  continue
213  tar.extractall()
214  tar.close()
215 
216  # Walk the directory structure
217  extractedFiles = utils.ListBlessedFilesInDir('.')
218 
219  # Create subdirectories in web area, if needed
220  if not os.path.isdir(config.WEB_PLOTS): os.mkdir(config.WEB_PLOTS, 0775)
221  docPlotDir = config.WEB_PLOTS + '/{}'.format(doc.id)
222  if not os.path.isdir(docPlotDir): os.mkdir(docPlotDir, 0775)
223 
224  # Remove any files that are currently in the plot dir from previous version
225  for file in os.listdir(docPlotDir):
226  path = os.path.join(docPlotDir, file)
227  try:
228  if os.path.isfile(path):
229  os.unlink(path)
230  except Exception as e:
231  print(e)
232 
233  # Copy files from temp area to web area
234  for file in extractedFiles:
235  os.chmod(file, 0644)
236  shutil.copy2(file, '{}/'.format(docPlotDir))
237 
238  os.chdir(docPlotDir)
239  shutil.rmtree(tempDocDir)
240 
241  plots = []
242  for capfile in utils.ListCaptionFilesInDir('.'):
243  plotname = os.path.splitext(os.path.basename(capfile))[0]
244 
245  filelist = utils.ListPlotsInDirForCaption('.', capfile);
246  # Catch the case where there is a txt file, but no associated plot
247  if (len(filelist) == 0): continue
248 
249  plot = Plot(plotname, capfile, filelist)
250  plots.append(plot)
251 
252  if len(plots) == 0:
253  print " \033[01;33mWARNING:\033[00m This document contains NO valid plots. Please contact the author ({}).".format(doc.first_author)
254  else:
255  print " This document contains {} plots".format(len(plots))
256 
257  # Create zip file for easy downloading
258  plotzip = zipfile.ZipFile('blessed_doc{}.zip'.format(doc.id), 'w')
259 
260  if doc.is_dep:
261  readmeFilename = 'DEPRECATED-README.txt'
262  readmeFile = open(readmeFilename, 'w')
263  readmeFile.write('THESE PLOTS HAVE BEEN BLESSED, BUT ARE CONSIDERED OBSOLETE.')
264  readmeFile.write('DO NOT USE THESE PLOTS IN TALKS.')
265  readmeFile.close()
266  plotzip.write(readmeFilename)
267 
268  # Clear out plot DB entries associated with this DocDB
269  c.execute("DELETE FROM bp_plots WHERE docdb_id={}".format(doc.id))
270  plot_counter = 0
271  for plot in plots:
272  plot_counter += 1
273  src, ext = plot.PreferredExtension()
274  srcBase = os.path.splitext(os.path.basename(src))[0]
275 
276  if ext == 'pdf':
277  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
278  os.system("convert -density 300 '{}' -trim +repage '{}.png'".format(src, srcBase))
279  plot.files.append('{}.png'.format(srcBase))
280  elif ext == 'png':
281  print " No format conversion needed: {}".format(src)
282  pass
283  else:
284  print " Converting [{}/{}] {}".format(plot_counter, len(plots), src)
285  os.system("convert '{}' -resize 1440\> '{}.png'".format(src, srcBase))
286  plot.files.append('{}.png'.format(srcBase))
287 
288  os.system("convert -flatten '{}.png' -resize '400x400^' -trim +repage '{}_thumb.png'".format(srcBase, srcBase))
289 
290  # Fill bp_plots table
291  c.execute("INSERT OR REPLACE INTO bp_plots ('id', 'plot_name', 'docdb_id', 'caption') VALUES ("
292  "(SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}), "
293  "?, "
294  "?, "
295  "?)".format(plot.name, doc.id), (plot.name, doc.id, plot.caption))
296  db.conn.commit()
297 
298  # Get this plot_id
299  c.execute("SELECT id FROM bp_plots WHERE plot_name='{}' AND docdb_id={}".format(plot.name, doc.id))
300  res = c.fetchall()
301  plot_id = res[0][0]
302 
303  # Clear out filename DB entries associated with this plot
304  c.execute("DELETE FROM bp_plots_filenames WHERE plot_id={}".format(plot_id))
305  # Fill bp_plots_filenames table and add to zip file
306  plotzip.write(plot.capfile)
307  for filename in plot.files:
308  plotzip.write(filename)
309  c.execute("INSERT OR REPLACE INTO bp_plots_filenames ('plot_id', 'filename') VALUES (?, ?)", (plot_id, filename))
310  db.conn.commit()
311 
312  if len(plots) > 0:
313  print " Zipping up plots and captions"
314  plotzip.close()
315 
316  # Fill bp_doc table
317  c.execute("INSERT OR REPLACE INTO bp_docs('id', 'name', 'revision', 'author_id', 'submitter_id', 'mod_time', 'abstract', 'blessed', 'deprecated') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (doc.id, doc.title, doc.rev, doc.first_author_id, doc.submitter_id, doc.modtime, doc.abstract, doc.is_blessed, doc.is_dep))
318 
319  for author in doc.authors:
320  # Fill bp_authors table
321  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (int(author['@id']), author['fullname']))
322  c.execute("INSERT OR IGNORE INTO bp_docs_authors VALUES (?, ?)", (doc.id, int(author['@id'])))
323 
324  c.execute("INSERT OR IGNORE INTO bp_authors VALUES (?, ?)", (doc.submitter_id, doc.submitter))
325 
326  # Choose a plot from this doc to be the default thumbnail
327  c.execute("INSERT OR REPLACE INTO bp_docs_preview ('doc_id', 'plot_id') VALUES (?, (SELECT id FROM bp_plots WHERE docdb_id={} LIMIT 1))".format(doc.id), (doc.id,))
328  time.sleep(5)
329 
330  print "Finalizing database"
331  db.conn.commit()
332 
333  print "\n\033[01;32mAll done.\033[00m Cheers!"
334 
const XML_Char int len
Definition: expat.h:262
::xsd::cxx::tree::id< char, ncname > id
Definition: Database.h:165
const XML_Char * name
Definition: expat.h:151
def progressBar(value, endvalue, docdb_id, bar_length=40)
Definition: bp_utils.py:6
struct Plot Plot
Definition: DrawUtils.h:15
set< int >::iterator it
string trim(string in)
Definition: rootgINukeVal.C:65
string directory
projection from multiple chains
cout<< "Opened file "<< fin<< " ixs= "<< ixs<< endl;if(ixs==0) hhh=(TH1F *) fff-> Get("h1")
Definition: AddMC.C:8
Definition: config.py:1
string filename
Definition: shutoffs.py:106
dictionary contains
bool Check(string tag, string name)
Definition: GenerateCovMx.C:75
f cosmic PID use this for the final state era of
Definition: Vars.h:69
void append()
Definition: append.C:24
void convert(std::string dir="cc_numu/C12")
Definition: convert.C:107
string WEB_PLOTS
Definition: bp_config.py:19
const XML_Char const XML_Char * data
Definition: expat.h:268
Definition: novas.h:112
ProcessPackage< L > Do
if(dump)
Definition: lz4.cxx:387
list ZIP_EXTS
Definition: bp_config.py:11
correl_yv Fill(-(dy[iP-1][iC-1]), hyv->GetBinContent(iP, iC))
root
Link up the nodes tree #####.
const double a
void resize(T &x, std::vector< int > dims)
Definition: resize.hpp:41
const std::vector< Plot > plots
bool print
def ListPlotsInDirForCaption(dir, cap)
Definition: bp_utils.py:57
Class for parsing *.fcl files for their metadata information.
def execute(command)
Definition: g4zmq.py:44
def ListCaptionFilesInDir(dir)
Definition: bp_utils.py:39
std::string format(const int32_t &value, const int &ndigits=8)
Definition: HexUtils.cpp:14
dictionary categories
Naming schema histograms should follow a logical naming convention: <category>-<observable>-<reconstr...
Definition: naming_schema.py:9
procfile open("FD_BRL_v0.txt")
const XML_Char * version
Definition: expat.h:187
const std::string path
Definition: plot_BEN.C:43
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:66
def mkdir(directory, verbose=False)
make a directory
ifstream in
Definition: comparison.C:7
def Download(url, destination)
Definition: helpers.py:65
TDirectory * dir
Definition: macro.C:5
exit(0)
while(!feof(fp))
TFile * file
Definition: cellShifts.C:17
string DOCDB_USER
Definition: bp_config.py:31
tuple density
Definition: barite.py:33
def parse_args()
Definition: regression.py:166
Float_t e
Definition: plot.C:35
#define for
Definition: msvc_pragmas.h:3
def ListBlessedFilesInDir(dir)
Definition: bp_utils.py:15
procfile close()
void plot(std::string label, std::map< std::string, std::map< std::string, Spectrum * >> &plots, bool log)
def exit_handler()
Definition: BlessedPlots.py:34