logistic_test.cpp
Go to the documentation of this file.
1 /**
2  * Performance test: logistic.
3  *
4  * This test runs a simple logistic model 100 times with a fixed
5  * seed. The test records to a csv file information about the current
6  * build, whether the values match the expected known values, whether
7  * values are identical between runs, and the runtime in seconds for
8  * each of the 100 individual runs.
9  *
10  * The output file is a csv file located at
11  * test/performance/performance.csv. If the file is empty, a header
12  * line will be written. If the file is not empty and the header line
13  * of the file matches, a data row will be appended. If the file is
14  * not empty and the header line is different, a header row and a data
15  * row will be appended.
16  *
17  * Below are the test routines in this test.
18  * 1. run: Runs the model 100 times. This test should always
19  * succeed. Test failure indicates that the model could not be run.
20  * 2. values_from_tagged_version: Compares the first run's final
21  * iteration against known values. Test will succeed if the values
22  * match the known values; the test will fail otherwise.
23  * 3. values_same_run_to_run: Compares the first run's final iteration
24  * against all runs' final iterations. Test will succeed if all the
25  * values match; the test will fail otherwise.
26  * 4. write_results_to_disk: Writes the results to disk. Test will
27  * succeed if writing to an empty file or to one with a matching
28  * header; the test will fail if the header in the existing file
29  * does not match the current header.
30  *
31  * Note: the values_from_tagged_version will need to be updated
32  * if Stan's RNG changes, the seeding mechanism changes, or the
33  * algorithm changes. The other routines should be fine.
34  *
35  * Each run of this test produces a single, csv row with 106 columns
36  * in the output file:
37  * 1. current date. Formatted using std::ctime: "Www Mmm dd hh:mm:ss yyyy"
38  * 2. git hash. The current 40-character SHA-1 git hash or "NA".
39  * 3. git date. The date of the last commit or "NA".
40  * 4. model name. "logistic"
41  * 5. matches tagged version. Either "yes" or "no".
42  * 6. all values the same. Either "yes" or "no".
43  * 7--106. run time (in seconds).
44  */
45 
46 #include <gtest/gtest.h>
47 #include <test/test-models/performance/logistic.hpp>
48 #include <stan/version.hpp>
49 #include <ctime>
51 #include <boost/algorithm/string/trim.hpp>
52 
53 
54 class performance : public ::testing::Test {
55 public:
56  static void SetUpTestCase() {
57  N = 100;
58  seconds_per_run.resize(N);
59  last_draws_per_run.resize(N);
60  matches_tagged_version = false;
61  all_values_same = false;
62  }
63 
64  static int N;
65  static std::vector<double> seconds_per_run;
66  static std::vector<std::vector<double> > last_draws_per_run;
68  static bool all_values_same;
69 };
70 
71 int performance::N;
72 std::vector<double> performance::seconds_per_run;
73 std::vector<std::vector<double> > performance::last_draws_per_run;
76 
77 
85 
87  clock_t t;
88  for (int n = 0; n < N; ++n) {
89  std::cout << "iteration: " << n << " / " << N << std::endl;
90  t = clock(); // start timer
91  stan::test::performance::command<stan_model>(1000,
92  10000,
93  "src/test/test-models/performance/logistic.data.R",
94  "test/performance/logistic_output.csv",
95  0U);
96  t = clock() - t; // end timer
97  seconds_per_run[n] = static_cast<double>(t) / CLOCKS_PER_SEC;
99  = get_last_iteration_from_file("test/performance/logistic_output.csv");
100  }
101  SUCCEED();
102 }
103 
104 // evaluate
105 TEST_F(performance, values_from_tagged_version) {
106  int N_values = 9;
107  ASSERT_EQ(N_values, last_draws_per_run[0].size())
108  << "last tagged version, 2.17.0, had " << N_values << " elements";
109 
110  std::vector<double> first_run = last_draws_per_run[0];
111  EXPECT_FLOAT_EQ(-65.7658, first_run[0])
112  << "lp__: index 0";
113 
114  EXPECT_FLOAT_EQ(0.944063, first_run[1])
115  << "accept_stat__: index 1";
116 
117  EXPECT_FLOAT_EQ(0.76857001, first_run[2])
118  << "stepsize__: index 2";
119 
120  EXPECT_FLOAT_EQ(2, first_run[3])
121  << "treedepth__: index 3";
122 
123  EXPECT_FLOAT_EQ(3, first_run[4])
124  << "n_leapfrog__: index 4";
125 
126  EXPECT_FLOAT_EQ(0, first_run[5])
127  << "divergent__: index 5";
128 
129  EXPECT_FLOAT_EQ(65.779, first_run[6])
130  << "energy__: index 6";
131 
132  EXPECT_FLOAT_EQ(1.37067, first_run[7])
133  << "beta.1: index 7";
134 
135  EXPECT_FLOAT_EQ(-0.771606, first_run[8])
136  << "beta.2: index 8";
137 
138  matches_tagged_version = !HasNonfatalFailure();
139 }
140 
141 TEST_F(performance, values_same_run_to_run) {
142  int N_values = last_draws_per_run[0].size();
143 
144  for (int i = 0; i < N_values; i++) {
145  double expected_value = last_draws_per_run[0][i];
146  for (int n = 1; n < N; n++) {
147  EXPECT_FLOAT_EQ(expected_value, last_draws_per_run[n][i])
148  << "expecting run to run values to be the same. Found run "
149  << n << " to have different values than the 0th run for "
150  << "index: " << i;
151  }
152  }
153  all_values_same = !HasNonfatalFailure();
154 }
155 
156 TEST_F(performance, check_output_is_same) {
157  std::ifstream file_stream;
158  file_stream.open("test/performance/logistic_output.csv",
160  ASSERT_TRUE(file_stream.good());
161 
162  std::string line, expected;
163 
164  getline(file_stream, line);
165  ASSERT_EQ("lp__,accept_stat__,stepsize__,treedepth__,n_leapfrog__,divergent__,energy__,beta.1,beta.2", line);
166  ASSERT_TRUE(file_stream.good());
167 
168  getline(file_stream, line);
169  ASSERT_EQ("# Adaptation terminated", line);
170  ASSERT_TRUE(file_stream.good());
171 
172  file_stream.close();
173 }
174 
175 TEST_F(performance, write_results_to_disk) {
176  std::stringstream header;
177  std::stringstream line;
178 
179  // current date / time
180  header << quote("date");
181  line << quote(get_date());
182 
183  // git hash
184  header << "," << quote("git hash") << "," << quote("git date");
185  line << "," << quote(get_git_hash()) << "," << quote(get_git_date());
186 
187  // model name: "logistic"
188  header << "," << quote("model name");
189  line << "," << quote("logistic");
190 
191  // matches tagged values
192  header << "," << quote("matches tagged version");
193  line << "," << quote(matches_tagged_version ? "yes" : "no");
194 
195  // all values same
196  header << "," << quote("all values same");
197  line << "," << quote(all_values_same ? "yes" : "no");
198 
199  // N times
200  for (int n = 0; n < N; n++) {
201  std::stringstream ss;
202  ss << "run " << n+1;
203  header << "," << quote(ss.str());
204  line << "," << seconds_per_run[n];
205  }
206 
207 
208  // append output to: test/performance/performance.csv
209  bool write_header = false;
210  std::fstream file_stream;
211 
212  file_stream.open("test/performance/performance.csv",
214  if (file_stream.peek() == std::fstream::traits_type::eof()) {
215  write_header = true;
216  } else {
217  std::string file_header;
218  std::getline(file_stream, file_header);
219 
220  EXPECT_EQ(file_header, header.str())
221  << "header of file is different";
222  if (file_header != header.str())
223  write_header = true;
224  }
225  file_stream.close();
226 
227  file_stream.open("test/performance/performance.csv",
229  if (write_header)
230  file_stream << header.str() << std::endl;
231  file_stream << line.str() << std::endl;
232  file_stream.close();
233 }
static std::vector< double > seconds_per_run
std::string get_git_date()
Definition: utility.hpp:172
Float_t ss
Definition: plot.C:24
TEST_F(performance, run)
std::string get_git_hash()
Definition: utility.hpp:164
std::string quote(const T &val)
Definition: utility.hpp:156
static std::vector< std::vector< double > > last_draws_per_run
static int N
std::string get_date()
Definition: utility.hpp:192
Definition: run.py:1
OStream cout
Definition: OStream.cxx:6
static void SetUpTestCase()
ifstream in
Definition: comparison.C:7
app
Definition: demo.py:189
static bool matches_tagged_version
std::vector< double > get_last_iteration_from_file(const char *filename)
Definition: utility.hpp:128
static bool all_values_same
run_command_output run_command(std::string command)
Definition: utility.hpp:90
enum BeamMode string