/Users/deen/code/yugabyte-db/src/yb/util/test_util.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // |
18 | | // The following only applies to changes made to this file as part of YugaByte development. |
19 | | // |
20 | | // Portions Copyright (c) YugaByte, Inc. |
21 | | // |
22 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
23 | | // in compliance with the License. You may obtain a copy of the License at |
24 | | // |
25 | | // http://www.apache.org/licenses/LICENSE-2.0 |
26 | | // |
27 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
28 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
29 | | // or implied. See the License for the specific language governing permissions and limitations |
30 | | // under the License. |
31 | | // |
32 | | |
33 | | #include "yb/util/test_util.h" |
34 | | |
35 | | #include <glog/logging.h> |
36 | | #include <gtest/gtest-spi.h> |
37 | | |
38 | | #include "yb/gutil/casts.h" |
39 | | #include "yb/gutil/strings/strcat.h" |
40 | | #include "yb/gutil/strings/util.h" |
41 | | #include "yb/gutil/walltime.h" |
42 | | |
43 | | #include "yb/util/env.h" |
44 | | #include "yb/util/logging.h" |
45 | | #include "yb/util/path_util.h" |
46 | | #include "yb/util/spinlock_profiling.h" |
47 | | #include "yb/util/status_format.h" |
48 | | #include "yb/util/status_log.h" |
49 | | #include "yb/util/thread.h" |
50 | | #include "yb/util/debug/trace_event.h" |
51 | | |
52 | | DEFINE_string(test_leave_files, "on_failure", |
53 | | "Whether to leave test files around after the test run. " |
54 | | " Valid values are 'always', 'on_failure', or 'never'"); |
55 | | |
56 | | DEFINE_int32(test_random_seed, 0, "Random seed to use for randomized tests"); |
57 | | DECLARE_int64(memory_limit_hard_bytes); |
58 | | DECLARE_bool(enable_tracing); |
59 | | DECLARE_bool(TEST_running_test); |
60 | | DECLARE_bool(never_fsync); |
61 | | |
62 | | using std::string; |
63 | | using strings::Substitute; |
64 | | using gflags::FlagSaver; |
65 | | |
66 | | namespace yb { |
67 | | |
68 | | static const char* const kSlowTestsEnvVariable = "YB_ALLOW_SLOW_TESTS"; |
69 | | |
70 | | static const uint64 kTestBeganAtMicros = Env::Default()->NowMicros(); |
71 | | |
72 | | /////////////////////////////////////////////////// |
73 | | // YBTest |
74 | | /////////////////////////////////////////////////// |
75 | | |
76 | | YBTest::YBTest() |
77 | | : env_(new EnvWrapper(Env::Default())), |
78 | 2.29k | test_dir_(GetTestDataDirectory()) { |
79 | 2.29k | InitThreading(); |
80 | 2.29k | debug::EnableTraceEvents(); |
81 | 2.29k | } |
82 | | |
83 | | // env passed in from subclass, for tests that run in-memory |
84 | | YBTest::YBTest(Env *env) |
85 | | : env_(env), |
86 | 0 | test_dir_(GetTestDataDirectory()) { |
87 | 0 | } |
88 | | |
89 | 1.21k | YBTest::~YBTest() { |
90 | | // Clean up the test directory in the destructor instead of a TearDown |
91 | | // method. This is better because it ensures that the child-class |
92 | | // dtor runs first -- so, if the child class is using a minicluster, etc, |
93 | | // we will shut that down before we remove files underneath. |
94 | 1.21k | if (FLAGS_test_leave_files == "always") { |
95 | 8 | LOG(INFO) << "-----------------------------------------------"; |
96 | 8 | LOG(INFO) << "--test_leave_files specified, leaving files in " << test_dir_; |
97 | 1.21k | } else if (FLAGS_test_leave_files == "on_failure" && HasFatalFailure()) { |
98 | 86 | LOG(INFO) << "-----------------------------------------------"; |
99 | 86 | LOG(INFO) << "Had fatal failures, leaving test files at " << test_dir_; |
100 | 1.12k | } else { |
101 | 1.12k | VLOG(1) << "Cleaning up temporary test files..."3 ; |
102 | 1.12k | WARN_NOT_OK(env_->DeleteRecursively(test_dir_), |
103 | 1.12k | "Couldn't remove test files"); |
104 | 1.12k | } |
105 | 1.21k | } |
106 | | |
107 | 2.43k | void YBTest::SetUp() { |
108 | 2.43k | InitSpinLockContentionProfiling(); |
109 | 2.43k | InitGoogleLoggingSafeBasic("yb_test"); |
110 | 2.43k | FLAGS_enable_tracing = true; |
111 | 2.43k | FLAGS_memory_limit_hard_bytes = 8 * 1024 * 1024 * 1024L; |
112 | 2.43k | FLAGS_TEST_running_test = true; |
113 | 2.43k | FLAGS_never_fsync = true; |
114 | 2.43k | for (const char* env_var_name : { |
115 | 2.43k | "ASAN_OPTIONS", |
116 | 2.43k | "LSAN_OPTIONS", |
117 | 2.43k | "UBSAN_OPTIONS", |
118 | 2.43k | "TSAN_OPTIONS" |
119 | 9.74k | }) { |
120 | 9.74k | const char* value = getenv(env_var_name); |
121 | 9.74k | if (value && value[0]) { |
122 | 9.74k | LOG(INFO) << "Environment variable " << env_var_name << ": " << value; |
123 | 9.74k | } |
124 | 9.74k | } |
125 | 2.43k | } |
126 | | |
127 | 476 | string YBTest::GetTestPath(const string& relative_path) { |
128 | 476 | CHECK(!test_dir_.empty()) << "Call SetUp() first"0 ; |
129 | 476 | return JoinPathSegments(test_dir_, relative_path); |
130 | 476 | } |
131 | | |
132 | | /////////////////////////////////////////////////// |
133 | | // Test utility functions |
134 | | /////////////////////////////////////////////////// |
135 | | |
136 | 99 | bool AllowSlowTests() { |
137 | 99 | char *e = getenv(kSlowTestsEnvVariable); |
138 | 99 | if ((e == nullptr) || |
139 | 99 | (strlen(e) == 0)0 || |
140 | 99 | (strcasecmp(e, "false") == 0)0 || |
141 | 99 | (strcasecmp(e, "0") == 0)0 || |
142 | 99 | (strcasecmp(e, "no") == 0)0 ) { |
143 | 99 | return false; |
144 | 99 | } |
145 | 0 | if ((strcasecmp(e, "true") == 0) || |
146 | 0 | (strcasecmp(e, "1") == 0) || |
147 | 0 | (strcasecmp(e, "yes") == 0)) { |
148 | 0 | return true; |
149 | 0 | } |
150 | 0 | LOG(FATAL) << "Unrecognized value for " << kSlowTestsEnvVariable << ": " << e; |
151 | 0 | return false; |
152 | 0 | } |
153 | | |
154 | | void OverrideFlagForSlowTests(const std::string& flag_name, |
155 | 34 | const std::string& new_value) { |
156 | | // Ensure that the flag is valid. |
157 | 34 | google::GetCommandLineFlagInfoOrDie(flag_name.c_str()); |
158 | | |
159 | | // If we're not running slow tests, don't override it. |
160 | 34 | if (!AllowSlowTests()) { |
161 | 34 | return; |
162 | 34 | } |
163 | 0 | google::SetCommandLineOptionWithMode(flag_name.c_str(), new_value.c_str(), |
164 | 0 | google::SET_FLAG_IF_DEFAULT); |
165 | 0 | } |
166 | | |
167 | 310 | int SeedRandom() { |
168 | 310 | int seed; |
169 | | // Initialize random seed |
170 | 310 | if (FLAGS_test_random_seed == 0) { |
171 | | // Not specified by user |
172 | 310 | seed = static_cast<int>(GetCurrentTimeMicros()); |
173 | 310 | } else { |
174 | 0 | seed = FLAGS_test_random_seed; |
175 | 0 | } |
176 | 310 | LOG(INFO) << "Using random seed: " << seed; |
177 | 310 | srand(seed); |
178 | 310 | return seed; |
179 | 310 | } |
180 | | |
181 | 3.61k | string GetTestDataDirectory() { |
182 | 3.61k | const ::testing::TestInfo* const test_info = |
183 | 3.61k | ::testing::UnitTest::GetInstance()->current_test_info(); |
184 | 3.61k | CHECK(test_info) << "Must be running in a gtest unit test to call this function"0 ; |
185 | 3.61k | string dir; |
186 | 3.61k | CHECK_OK(Env::Default()->GetTestDirectory(&dir)); |
187 | | |
188 | | // The directory name includes some strings for specific reasons: |
189 | | // - program name: identifies the directory to the test invoker |
190 | | // - timestamp and pid: disambiguates with prior runs of the same test |
191 | | // |
192 | | // e.g. "env-test.TestEnv.TestReadFully.1409169025392361-23600" |
193 | 3.61k | dir += Substitute("/$0.$1.$2.$3-$4", |
194 | 3.61k | StringReplace(google::ProgramInvocationShortName(), "/", "_", true), |
195 | 3.61k | StringReplace(test_info->test_case_name(), "/", "_", true), |
196 | 3.61k | StringReplace(test_info->name(), "/", "_", true), |
197 | 3.61k | kTestBeganAtMicros, |
198 | 3.61k | getpid()); |
199 | 3.61k | Status s = Env::Default()->CreateDir(dir); |
200 | 3.61k | CHECK(s.IsAlreadyPresent() || s.ok()) |
201 | 0 | << "Could not create directory " << dir << ": " << s.ToString(); |
202 | 3.61k | if (s.ok()) { |
203 | 2.29k | string metadata; |
204 | | |
205 | 2.29k | StrAppend(&metadata, Substitute("PID=$0\n", getpid())); |
206 | | |
207 | 2.29k | StrAppend(&metadata, Substitute("PPID=$0\n", getppid())); |
208 | | |
209 | 2.29k | char* jenkins_build_id = getenv("BUILD_ID"); |
210 | 2.29k | if (jenkins_build_id) { |
211 | 0 | StrAppend(&metadata, Substitute("BUILD_ID=$0\n", jenkins_build_id)); |
212 | 0 | } |
213 | | |
214 | 2.29k | CHECK_OK(WriteStringToFile(Env::Default(), metadata, |
215 | 2.29k | Substitute("$0/test_metadata", dir))); |
216 | 2.29k | } |
217 | 3.61k | return dir; |
218 | 3.61k | } |
219 | | |
220 | | void AssertEventually(const std::function<void(void)>& f, |
221 | 14 | const MonoDelta& timeout) { |
222 | 14 | const MonoTime deadline = MonoTime::Now() + timeout; |
223 | 14 | { |
224 | 14 | FlagSaver flag_saver; |
225 | | // Disable --gtest_break_on_failure, or else the assertion failures |
226 | | // inside our attempts will cause the test to SEGV even though we |
227 | | // would like to retry. |
228 | 14 | testing::FLAGS_gtest_break_on_failure = false; |
229 | | |
230 | 112 | for (int attempts = 0; MonoTime::Now() < deadline; attempts++98 ) { |
231 | | // Capture any assertion failures within this scope (i.e. from their function) |
232 | | // into 'results' |
233 | 112 | testing::TestPartResultArray results; |
234 | 112 | testing::ScopedFakeTestPartResultReporter reporter( |
235 | 112 | testing::ScopedFakeTestPartResultReporter::INTERCEPT_ONLY_CURRENT_THREAD, |
236 | 112 | &results); |
237 | 112 | f(); |
238 | | |
239 | | // Determine whether their function produced any new test failure results. |
240 | 112 | bool has_failures = false; |
241 | 224 | for (int i = 0; i < results.size(); i++112 ) { |
242 | 112 | has_failures |= results.GetTestPartResult(i).failed(); |
243 | 112 | } |
244 | 112 | if (!has_failures) { |
245 | 14 | return; |
246 | 14 | } |
247 | | |
248 | | // If they had failures, sleep and try again. |
249 | 98 | int sleep_ms = (attempts < 10) ? (1 << attempts) : 10000 ; |
250 | 98 | SleepFor(MonoDelta::FromMilliseconds(sleep_ms)); |
251 | 98 | } |
252 | 14 | } |
253 | | |
254 | | // If we ran out of time looping, run their function one more time |
255 | | // without capturing its assertions. This way the assertions will |
256 | | // propagate back out to the normal test reporter. Of course it's |
257 | | // possible that it will pass on this last attempt, but that's OK |
258 | | // too, since we aren't trying to be that strict about the deadline. |
259 | 0 | f(); |
260 | 0 | if (testing::Test::HasFatalFailure()) { |
261 | 0 | ADD_FAILURE() << "Timed out waiting for assertion to pass."; |
262 | 0 | } |
263 | 0 | } |
264 | | |
265 | | Status Wait(const std::function<Result<bool>()>& condition, |
266 | | CoarseTimePoint deadline, |
267 | | const std::string& description, |
268 | | MonoDelta initial_delay, |
269 | | double delay_multiplier, |
270 | 2.51k | MonoDelta max_delay) { |
271 | 2.51k | auto start = CoarseMonoClock::Now(); |
272 | 2.51k | MonoDelta delay = initial_delay; |
273 | 9.39k | for (;;) { |
274 | 9.39k | const auto current = condition(); |
275 | 9.39k | if (!current.ok()) { |
276 | 5 | return current.status(); |
277 | 5 | } |
278 | 9.39k | if (current.get()) { |
279 | 2.50k | break; |
280 | 2.50k | } |
281 | 6.89k | const auto now = CoarseMonoClock::Now(); |
282 | 6.89k | const MonoDelta left(deadline - now); |
283 | 6.89k | if (left <= MonoDelta::kZero) { |
284 | 12 | return STATUS_FORMAT(TimedOut, |
285 | 12 | "Operation '$0' didn't complete within $1ms", |
286 | 12 | description, |
287 | 12 | MonoDelta(now - start).ToMilliseconds()); |
288 | 12 | } |
289 | 6.88k | delay = std::min(std::min(MonoDelta::FromSeconds(delay.ToSeconds() * delay_multiplier), left), |
290 | 6.88k | max_delay); |
291 | 6.88k | SleepFor(delay); |
292 | 6.88k | } |
293 | 2.50k | return Status::OK(); |
294 | 2.51k | } |
295 | | |
296 | | Status Wait(const std::function<Result<bool>()>& condition, |
297 | | MonoTime deadline, |
298 | | const std::string& description, |
299 | | MonoDelta initial_delay, |
300 | | double delay_multiplier, |
301 | 2.51k | MonoDelta max_delay) { |
302 | 2.51k | auto left = deadline - MonoTime::Now(); |
303 | 2.51k | return Wait(condition, CoarseMonoClock::Now() + left, description, initial_delay, |
304 | 2.51k | delay_multiplier, max_delay); |
305 | 2.51k | } |
306 | | |
307 | | Status LoggedWait( |
308 | | const std::function<Result<bool>()>& condition, |
309 | | CoarseTimePoint deadline, |
310 | | const string& description, |
311 | | MonoDelta initial_delay, |
312 | | double delay_multiplier, |
313 | 1 | MonoDelta max_delay) { |
314 | 1 | LOG(INFO) << description << " - started"; |
315 | 1 | auto status = |
316 | 1 | Wait(condition, deadline, description, initial_delay, delay_multiplier, max_delay); |
317 | 1 | LOG(INFO) << description << " - completed: " << status; |
318 | 1 | return status; |
319 | 1 | } |
320 | | |
321 | | // Waits for the given condition to be true or until the provided timeout has expired. |
322 | | Status WaitFor(const std::function<Result<bool>()>& condition, |
323 | | MonoDelta timeout, |
324 | | const string& description, |
325 | | MonoDelta initial_delay, |
326 | | double delay_multiplier, |
327 | 2.44k | MonoDelta max_delay) { |
328 | 2.44k | return Wait(condition, MonoTime::Now() + timeout, description, initial_delay, delay_multiplier, |
329 | 2.44k | max_delay); |
330 | 2.44k | } |
331 | | |
332 | | Status LoggedWaitFor( |
333 | | const std::function<Result<bool>()>& condition, |
334 | | MonoDelta timeout, |
335 | | const string& description, |
336 | | MonoDelta initial_delay, |
337 | | double delay_multiplier, |
338 | 171 | MonoDelta max_delay) { |
339 | 171 | LOG(INFO) << description << " - started"; |
340 | 171 | auto status = |
341 | 171 | WaitFor(condition, timeout, description, initial_delay, delay_multiplier, max_delay); |
342 | 171 | LOG(INFO) << description << " - completed: " << status; |
343 | 171 | return status; |
344 | 171 | } |
345 | | |
346 | 106 | string GetToolPath(const string& rel_path, const string& tool_name) { |
347 | 106 | string exe; |
348 | 106 | CHECK_OK(Env::Default()->GetExecutablePath(&exe)); |
349 | 106 | const string binroot = JoinPathSegments(DirName(exe), rel_path); |
350 | 106 | const string tool_path = JoinPathSegments(binroot, tool_name); |
351 | 106 | CHECK(Env::Default()->FileExists(tool_path)) << tool_name << " tool not found at " << tool_path0 ; |
352 | 106 | return tool_path; |
353 | 106 | } |
354 | | |
355 | 245 | int CalcNumTablets(size_t num_tablet_servers) { |
356 | | #ifdef NDEBUG |
357 | | return 0; // Will use the default. |
358 | | #elif defined(THREAD_SANITIZER) || defined(ADDRESS_SANITIZER) |
359 | | return narrow_cast<int>(num_tablet_servers); |
360 | | #else |
361 | 245 | return narrow_cast<int>(num_tablet_servers * 3); |
362 | 245 | #endif |
363 | 245 | } |
364 | | |
365 | | } // namespace yb |