Ntuple.h
Go to the documentation of this file.
1 #ifndef cetlib_sqlite_Ntuple_h
2 #define cetlib_sqlite_Ntuple_h
3 
4 // =====================================================================
5 //
6 // Ntuple
7 //
8 // The Ntuple class template is an interface for inserting into an
9 // SQLite database in a type-safe and thread-safe manner. It is not
10 // intended for this facility to provide facilities for making SQLite
11 // database queries. For querying, consider using the
12 // 'cet::sqlite::select' utilities.
13 //
14 //
15 // Construction
16 // ------------
17 //
18 // The c'tor receives a reference to a cet::sqlite::Connection object,
19 // that must outlive the life of the Ntuple. It is the responsibility
20 // of the user to manage the Connection's lifetime. A Connection is
21 // created by a call to ConnectionFactory::make. Please see the
22 // documentation in cetlib/sqlite/Connection(Factory).h.
23 //
24 // In addition to the Connection object, the Ntuple c'tor receives an
25 // SQLite table name and a set of column names in the form of an
26 // std::array object.
27 //
28 // See the Ntuple definition below for the full c'tor signatures.
29 //
30 // N.B. Constructing an Ntuple in a multi-threaded context is NOT
31 // thread-safe if done in parallel with any operations being
32 // performed on the same database elsewhere.
33 //
34 // Template arguments
35 // ------------------
36 //
37 // The template arguments supplied for the Ntuple type indicate the
38 // type of the column. The following Ntuple specifications are
39 // identical in semantics:
40 //
41 // Ntuple<int, double, string> n1 {...};
42 // Ntuple<column<int>, column<double>, column<string>> n2 {...};
43 //
44 // both of which indicate three columns with SQLite types of INTEGER,
45 // NUMERIC, and TEXT, respectively. Specifying the 'column' template
46 // is necessary ONLY when a column constraint is desired (e.g.):
47 //
48 // Ntuple<column<int, primary_key>, double, string> n3 {...};
49 //
50 // The names of the columns are provided as a c'tor argument (see
51 // below).
52 //
53 // Intended use
54 // ------------
55 //
56 // There are two public modifying functions that can be called:
57 //
58 // Ntuple::insert(...)
59 // Ntuple::flush()
60 //
61 // Calling 'insert' will add items to the internal buffer until the
62 // maximum buffer size has been reached, at which point, the contents
63 // of the buffer will be automatically flushed to the SQLite database.
64 //
65 // Calling 'flush' is necessary only when no more items will be
66 // inserted into the Ntuple AND querying of the Ntuple is required
67 // immediately thereafter. The Ntuple d'tor automatically calls flush
68 // whenever the Ntuple object goes out of scope.
69 //
70 // Examples of use
71 // ---------------
72 //
73 // using namespace cet::sqlite;
74 //
75 // auto c = existing_connection_factory.make("languages.db");
76 // Ntuple<string, string> langs {c, "europe", {"Country","Language"}};
77 // langs.insert("Germany", "German");
78 // langs.insert("Switzerland", "German");
79 // langs.insert("Switzerland", "Italian");
80 // langs.insert("Switzerland", "French");
81 // langs.insert("Switzerland", "Romansh");
82 // langs.flush();
83 //
84 // query_result<string> ch;
85 // ch << select("Languange").from(c,
86 // "europe").where("Country='Switzerland'");
87 // // see cet::sqlite::select documentation regarding using query_result.
88 //
89 // -----------------------------------------------------------
90 //
91 // Technical notes:
92 //
93 // In principle, one could use a concurrent container to prevent
94 // having to lock whenever an insert is done. However, since a
95 // flush occurs whenever the buffer max is reached, the buffer must
96 // be protected from any modification until the flush is complete.
97 // A lock is therefore inevitable. We could probably optimize by
98 // using an atomic variable to protect against modification only
99 // when a flush is being done. This is a potential optimization to
100 // keep in mind.
101 // ===========================================================
102 
105 #include "cetlib/sqlite/column.h"
107 #include "cetlib/sqlite/helpers.h"
108 
109 #include "sqlite3.h"
110 
111 #include <iostream>
112 #include <memory>
113 #include <mutex>
114 #include <string>
115 #include <tuple>
116 #include <vector>
117 
118 namespace cet {
119  namespace sqlite {
120 
121  template <typename... Args>
122  class Ntuple {
123  public:
124  // Elements of row are unique_ptr's so that it is possible to bind
125  // to a null parameter.
126  template <typename T>
127  using element_t =
128  std::unique_ptr<typename sqlite::permissive_column<T>::element_type>;
129 
130  using row_t = std::tuple<element_t<Args>...>;
131  static constexpr auto nColumns = std::tuple_size<row_t>::value;
133 
134  Ntuple(Connection& connection,
135  std::string const& name,
136  name_array const& columns,
137  bool overwriteContents = false,
138  std::size_t bufsize = 1000ull);
139 
140  ~Ntuple() noexcept;
141 
142  std::string const&
143  name() const
144  {
145  return name_;
146  }
147 
148  void insert(Args const...);
149  void flush();
150 
151  // Disable copying
152  Ntuple(Ntuple const&) = delete;
153  Ntuple& operator=(Ntuple const&) = delete;
154 
155  private:
156  static constexpr auto iSequence = std::make_index_sequence<nColumns>();
157 
158  // This is the c'tor that does all of the work. It exists so that
159  // the Args... and column-names array can be expanded in parallel.
160  template <std::size_t... I>
162  std::string const& name,
163  name_array const& columns,
164  bool overwriteContents,
165  std::size_t bufsize,
166  std::index_sequence<I...>);
167 
168  int flush_no_throw();
169 
172  std::size_t max_;
173  std::vector<row_t> buffer_{};
174  sqlite3_stmt* insert_statement_{nullptr};
175  std::recursive_mutex mutex_{};
176  };
177 
178  } // sqlite
179 } // cet
180 
181 template <typename... Args>
182 template <std::size_t... I>
184  std::string const& name,
185  name_array const& cnames,
186  bool const overwriteContents,
187  std::size_t const bufsize,
188  std::index_sequence<I...>)
189  : connection_{connection}, name_{name}, max_{bufsize}
190 {
191  assert(connection);
192 
193  sqlite::createTableIfNeeded(connection,
194  overwriteContents,
195  name,
196  sqlite::permissive_column<Args>{cnames[I]}...);
197 
198  std::string sql{"INSERT INTO "};
199  sql += name;
200  sql += " VALUES (?";
201  for (std::size_t i = 1; i < nColumns; ++i) {
202  sql += ",?";
203  }
204  sql += ")";
205  int const rc{sqlite3_prepare_v2(
206  connection_, sql.c_str(), sql.size(), &insert_statement_, nullptr)};
207  if (rc != SQLITE_OK) {
208  auto const ec = sqlite3_step(insert_statement_);
210  << "Failed to prepare insertion statement.\n"
211  << "Return code: " << ec << '\n';
212  }
213 
214  buffer_.reserve(bufsize);
215 }
216 
217 template <typename... Args>
219  std::string const& name,
220  name_array const& cnames,
221  bool const overwriteContents,
222  std::size_t const bufsize)
223  : Ntuple{connection, name, cnames, overwriteContents, bufsize, iSequence}
224 {}
225 
226 template <typename... Args>
228 {
229  if (flush_no_throw() != SQLITE_OK) {
230  std::cerr << "SQLite step failure while flushing.\n";
231  }
232  sqlite3_finalize(insert_statement_);
233 }
234 
235 template <typename... Args>
236 void
238 {
239  std::lock_guard<decltype(mutex_)> lock{mutex_};
240  if (buffer_.size() == max_) {
241  flush();
242  }
243  buffer_.emplace_back(std::make_unique<Args>(args)...);
244 }
245 
246 template <typename... Args>
247 int
249 {
250  // Guard against any modifications to the buffer, which is about to
251  // be flushed to the database.
252  std::lock_guard<decltype(mutex_)> lock{mutex_};
253  int const rc{
255  if (rc != SQLITE_DONE) {
256  return rc;
257  }
258  buffer_.clear();
259  return SQLITE_OK;
260 }
261 
262 template <typename... Args>
263 void
265 {
266  // No lock here -- lock held by flush_no_throw;
267  if (flush_no_throw() != SQLITE_OK) {
269  << "SQLite step failure while flushing.";
270  }
271 }
272 
273 #endif /* cetlib_sqlite_Ntuple_h */
274 
275 // Local Variables:
276 // mode: c++
277 // End:
void insert(Args const ...)
Definition: Ntuple.h:237
const XML_Char * name
Definition: expat.h:151
int flush_no_throw()
Definition: Ntuple.h:248
void createTableIfNeeded(sqlite3 *db, bool const delete_contents, std::string const &tablename, permissive_column< Args > const &...cols)
Definition: helpers.h:49
std::recursive_mutex mutex_
Definition: Ntuple.h:175
std::string name_
Definition: Ntuple.h:171
std::size_t max_
Definition: Ntuple.h:172
cet::coded_exception< errors::ErrorCodes, ExceptionDetail::translate > Exception
Definition: Exception.h:33
OStream cerr
Definition: OStream.cxx:7
std::string const & name() const
Definition: Ntuple.h:143
static constexpr auto iSequence
Definition: Ntuple.h:156
sqlite::name_array< nColumns > name_array
Definition: Ntuple.h:132
Connection & connection_
Definition: Ntuple.h:170
std::vector< row_t > buffer_
Definition: Ntuple.h:173
const XML_Char int const XML_Char * value
Definition: expat.h:331
Ntuple(Connection &connection, std::string const &name, name_array const &columns, bool overwriteContents=false, std::size_t bufsize=1000ull)
Definition: Ntuple.h:218
static constexpr auto nColumns
Definition: Ntuple.h:131
std::unique_ptr< typename sqlite::permissive_column< T >::element_type > element_t
Definition: Ntuple.h:128
sqlite3_stmt * insert_statement_
Definition: Ntuple.h:174
std::tuple< element_t< Args >... > row_t
Definition: Ntuple.h:130
~Ntuple() noexcept
Definition: Ntuple.h:227
int flush_no_throw(std::vector< Row > const &buffer, sqlite3_stmt *&insertStmt)
Definition: Connection.h:86
std::array< std::string, N > name_array
Definition: column.h:42
assert(nhit_max >=nhit_nbins)
Ntuple & operator=(Ntuple const &)=delete
enum BeamMode string