/Users/deen/code/yugabyte-db/src/yb/util/stack_trace.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) YugaByte, Inc. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
4 | | // in compliance with the License. You may obtain a copy of the License at |
5 | | // |
6 | | // http://www.apache.org/licenses/LICENSE-2.0 |
7 | | // |
8 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
9 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
10 | | // or implied. See the License for the specific language governing permissions and limitations |
11 | | // under the License. |
12 | | // |
13 | | |
14 | | #include "yb/util/stack_trace.h" |
15 | | |
16 | | #include <signal.h> |
17 | | |
18 | | #ifdef __linux__ |
19 | | #include <linux/futex.h> |
20 | | #include <sys/syscall.h> |
21 | | #endif |
22 | | |
23 | | #include <mutex> |
24 | | |
25 | | #include "yb/gutil/casts.h" |
26 | | #include "yb/gutil/linux_syscall_support.h" |
27 | | |
28 | | #include "yb/util/lockfree.h" |
29 | | #include "yb/util/monotime.h" |
30 | | #include "yb/util/result.h" |
31 | | #include "yb/util/thread.h" |
32 | | |
33 | | using namespace std::literals; |
34 | | |
35 | | #if defined(__APPLE__) |
36 | | typedef sig_t sighandler_t; |
37 | | #endif |
38 | | |
39 | | namespace yb { |
40 | | |
41 | | namespace { |
42 | | |
43 | | YB_DEFINE_ENUM(ThreadStackState, (kNone)(kSendFailed)(kReady)); |
44 | | |
45 | | struct ThreadStackEntry : public MPSCQueueEntry<ThreadStackEntry> { |
46 | | ThreadIdForStack tid; |
47 | | StackTrace stack; |
48 | | }; |
49 | | |
50 | | class CompletionFlag { |
51 | | public: |
52 | 6.79k | void Signal() { |
53 | 6.79k | complete_.store(1, std::memory_order_release); |
54 | | #ifndef __APPLE__ |
55 | | sys_futex(reinterpret_cast<int32_t*>(&complete_), |
56 | | FUTEX_WAKE | FUTEX_PRIVATE_FLAG, |
57 | | INT_MAX, // wake all |
58 | | nullptr, nullptr, |
59 | | 0 /* ignored */); |
60 | | #endif |
61 | 6.79k | } |
62 | | |
63 | 6.69k | bool TimedWait(MonoDelta timeout) { |
64 | 6.69k | if (complete()) { |
65 | 42 | return true; |
66 | 42 | } |
67 | | |
68 | 6.65k | auto now = MonoTime::Now(); |
69 | 6.65k | auto deadline = now + timeout; |
70 | 2.48M | while (now < deadline) { |
71 | | #ifndef __APPLE__ |
72 | | MonoDelta rem = deadline - now; |
73 | | struct timespec ts; |
74 | | rem.ToTimeSpec(&ts); |
75 | | sys_futex(reinterpret_cast<int32_t*>(&complete_), |
76 | | FUTEX_WAIT | FUTEX_PRIVATE_FLAG, |
77 | | 0, // wait if value is still 0 |
78 | | reinterpret_cast<struct kernel_timespec *>(&ts), nullptr, 0); |
79 | | #else |
80 | 2.48M | sched_yield(); |
81 | 2.48M | #endif |
82 | 2.48M | if (complete()) { |
83 | 6.63k | return true; |
84 | 6.63k | } |
85 | 2.48M | now = MonoTime::Now(); |
86 | 2.48M | } |
87 | | |
88 | 22 | return complete(); |
89 | 6.65k | } |
90 | | |
91 | 6.79k | void Reset() { |
92 | 6.79k | complete_.store(0, std::memory_order_release); |
93 | 6.79k | } |
94 | | |
95 | 2.49M | bool complete() const { |
96 | 2.49M | return complete_.load(std::memory_order_acquire); |
97 | 2.49M | } |
98 | | private: |
99 | | std::atomic<int32_t> complete_ { 0 }; |
100 | | }; |
101 | | |
102 | | // Global structure used to communicate between the signal handler |
103 | | // and a dumping thread. |
104 | | struct ThreadStackHelper { |
105 | | std::mutex mutex; // Locked by ThreadStacks, so only one could be executed in parallel. |
106 | | |
107 | | LockFreeStack<ThreadStackEntry> collected; |
108 | | // Reuse previously allocated memory. We expect this size to be merely small, near 152 bytes |
109 | | // per application thread. |
110 | | LockFreeStack<ThreadStackEntry> allocated; |
111 | | CompletionFlag completion_flag; |
112 | | |
113 | | // Could be modified only by ThreadStacks. |
114 | | CoarseTimePoint deadline; |
115 | | size_t allocated_entries = 0; |
116 | | |
117 | | // Incremented by each signal handler. |
118 | | std::atomic<int64_t> left_to_collect{0}; |
119 | | |
120 | | std::vector<std::unique_ptr<ThreadStackEntry[]>> allocated_chunks; |
121 | | |
122 | 6.79k | void SetNumEntries(size_t len) { |
123 | 6.79k | len += 5; // We reserve several entries, because threads from previous request could still be |
124 | | // processing signal and write their results. |
125 | 6.79k | if (len <= allocated_entries) { |
126 | 6.23k | return; |
127 | 6.23k | } |
128 | | |
129 | 561 | size_t new_chunk_size = std::max<size_t>(len - allocated_entries, 0x10); |
130 | 561 | allocated_chunks.emplace_back(new ThreadStackEntry[new_chunk_size]); |
131 | 561 | allocated_entries += new_chunk_size; |
132 | | |
133 | 9.53k | for (auto entry = allocated_chunks.back().get(), end = entry + new_chunk_size; entry != end; |
134 | 8.97k | ++entry) { |
135 | 8.97k | allocated.Push(entry); |
136 | 8.97k | } |
137 | 561 | } |
138 | | |
139 | | void StoreResult( |
140 | 6.79k | const std::vector<ThreadIdForStack>& tids, std::vector<Result<StackTrace>>* out) { |
141 | | // We give the thread ~1s to respond. In testing, threads typically respond within |
142 | | // a few iterations of the loop, so this timeout is very conservative. |
143 | | // |
144 | | // The main reason that a thread would not respond is that it has blocked signals. For |
145 | | // example, glibc's timer_thread doesn't respond to our signal, so we always time out |
146 | | // on that one. |
147 | 6.79k | if (left_to_collect.load(std::memory_order_acquire) > 0) { |
148 | 6.69k | completion_flag.TimedWait(1s); |
149 | 6.69k | } |
150 | | |
151 | 13.5k | while (auto entry = collected.Pop()) { |
152 | 6.79k | auto it = std::lower_bound(tids.begin(), tids.end(), entry->tid); |
153 | 6.79k | if (it != tids.end() && *it == entry->tid6.78k ) { |
154 | 6.77k | (*out)[it - tids.begin()] = entry->stack; |
155 | 6.77k | } |
156 | 6.79k | allocated.Push(entry); |
157 | 6.79k | } |
158 | 6.79k | } |
159 | | |
160 | 6.79k | void RecordStackTrace(const StackTrace& stack_trace) { |
161 | 6.79k | auto* entry = allocated.Pop(); |
162 | 6.79k | if (entry) { |
163 | | // Not enough allocated entries, don't write log since we are in signal handler. |
164 | 6.79k | entry->tid = Thread::CurrentThreadIdForStack(); |
165 | 6.79k | entry->stack = stack_trace; |
166 | 6.79k | collected.Push(entry); |
167 | 6.79k | } |
168 | | |
169 | 6.79k | if (left_to_collect.fetch_sub(1, std::memory_order_acq_rel) - 1 <= 0) { |
170 | 6.79k | completion_flag.Signal(); |
171 | 6.79k | } |
172 | 6.79k | } |
173 | | }; |
174 | | |
175 | | ThreadStackHelper thread_stack_helper; |
176 | | |
177 | | // Signal handler for our stack trace signal. |
178 | | // We expect that the signal is only sent from DumpThreadStack() -- not by a user. |
179 | | |
180 | 6.79k | void HandleStackTraceSignal(int signum) { |
181 | 6.79k | int old_errno = errno; |
182 | 6.79k | StackTrace stack_trace; |
183 | 6.79k | stack_trace.Collect(2); |
184 | | |
185 | 6.79k | thread_stack_helper.RecordStackTrace(stack_trace); |
186 | 6.79k | errno = old_errno; |
187 | 6.79k | } |
188 | | |
189 | | // The signal that we'll use to communicate with our other threads. |
190 | | // This can't be in used by other libraries in the process. |
191 | | int g_stack_trace_signum = SIGUSR2; |
192 | | |
193 | 33.1k | bool InitSignalHandlerUnlocked(int signum) { |
194 | 33.1k | enum InitState { |
195 | 33.1k | UNINITIALIZED, |
196 | 33.1k | INIT_ERROR, |
197 | 33.1k | INITIALIZED |
198 | 33.1k | }; |
199 | 33.1k | static InitState state = UNINITIALIZED; |
200 | | |
201 | | // If we've already registered a handler, but we're being asked to |
202 | | // change our signal, unregister the old one. |
203 | 33.1k | if (signum != g_stack_trace_signum && state == INITIALIZED4 ) { |
204 | 3 | struct sigaction old_act; |
205 | 3 | PCHECK(sigaction(g_stack_trace_signum, nullptr, &old_act) == 0); |
206 | 3 | if (old_act.sa_handler == &HandleStackTraceSignal) { |
207 | 3 | signal(g_stack_trace_signum, SIG_DFL); |
208 | 3 | } |
209 | 3 | } |
210 | | |
211 | | // If we'd previously had an error, but the signal number |
212 | | // is changing, we should mark ourselves uninitialized. |
213 | 33.1k | if (signum != g_stack_trace_signum) { |
214 | 4 | g_stack_trace_signum = signum; |
215 | 4 | state = UNINITIALIZED; |
216 | 4 | } |
217 | | |
218 | 33.1k | if (state == UNINITIALIZED) { |
219 | 15.7k | struct sigaction old_act; |
220 | 15.7k | PCHECK(sigaction(g_stack_trace_signum, nullptr, &old_act) == 0); |
221 | 15.7k | if (old_act.sa_handler != SIG_DFL && |
222 | 15.7k | old_act.sa_handler != SIG_IGN1 ) { |
223 | 1 | state = INIT_ERROR; |
224 | 1 | LOG(WARNING) << "signal handler for stack trace signal " |
225 | 1 | << g_stack_trace_signum |
226 | 1 | << " is already in use: " |
227 | 1 | << "YB will not produce thread stack traces."; |
228 | 15.7k | } else { |
229 | | // No one appears to be using the signal. This is racy, but there is no |
230 | | // atomic swap capability. |
231 | 15.7k | sighandler_t old_handler = signal(g_stack_trace_signum, HandleStackTraceSignal); |
232 | 15.7k | if (old_handler != SIG_IGN && |
233 | 15.7k | old_handler != SIG_DFL) { |
234 | 0 | LOG(FATAL) << "raced against another thread installing a signal handler"; |
235 | 0 | } |
236 | 15.7k | state = INITIALIZED; |
237 | 15.7k | } |
238 | 15.7k | } |
239 | 33.1k | return state == INITIALIZED; |
240 | 33.1k | } |
241 | | |
242 | | } // namespace |
243 | | |
244 | 6.79k | Result<StackTrace> ThreadStack(ThreadIdForStack tid) { |
245 | 6.79k | return ThreadStacks({tid}).front(); |
246 | 6.79k | } |
247 | | |
248 | 6.79k | std::vector<Result<StackTrace>> ThreadStacks(const std::vector<ThreadIdForStack>& tids) { |
249 | 6.79k | static const Status status = STATUS( |
250 | 6.79k | RuntimeError, "Thread did not respond: maybe it is blocking signals"); |
251 | | |
252 | 6.79k | std::vector<Result<StackTrace>> result(tids.size(), status); |
253 | 6.79k | std::lock_guard<std::mutex> execution_lock(thread_stack_helper.mutex); |
254 | | |
255 | | // Ensure that our signal handler is installed. We don't need any fancy GoogleOnce here |
256 | | // because of the mutex above. |
257 | 6.79k | if (!InitSignalHandlerUnlocked(g_stack_trace_signum)) { |
258 | 1 | static const Status status = STATUS( |
259 | 1 | RuntimeError, "Unable to take thread stack: signal handler unavailable"); |
260 | 1 | std::fill_n(result.begin(), tids.size(), status); |
261 | 1 | return result; |
262 | 1 | } |
263 | | |
264 | 6.79k | thread_stack_helper.left_to_collect.store(tids.size(), std::memory_order_release); |
265 | 6.79k | thread_stack_helper.SetNumEntries(tids.size()); |
266 | 6.79k | thread_stack_helper.completion_flag.Reset(); |
267 | | |
268 | 13.5k | for (size_t i = 0; i != tids.size(); ++i6.79k ) { |
269 | | // We use the raw syscall here instead of kill() to ensure that we don't accidentally |
270 | | // send a signal to some other process in the case that the thread has exited and |
271 | | // the TID been recycled. |
272 | | #if defined(__linux__) |
273 | | int res = narrow_cast<int>(syscall(SYS_tgkill, getpid(), tids[i], g_stack_trace_signum)); |
274 | | #else |
275 | 6.79k | int res = pthread_kill(tids[i], g_stack_trace_signum); |
276 | 6.79k | #endif |
277 | 6.79k | if (res != 0) { |
278 | 1 | static const Status status = STATUS( |
279 | 1 | RuntimeError, "Unable to deliver signal: process may have exited"); |
280 | 1 | result[i] = status; |
281 | 1 | thread_stack_helper.left_to_collect.fetch_sub(1, std::memory_order_acq_rel); |
282 | 1 | } |
283 | 6.79k | } |
284 | | |
285 | 6.79k | thread_stack_helper.StoreResult(tids, &result); |
286 | | |
287 | 6.79k | return result; |
288 | 6.79k | } |
289 | | |
290 | 6.79k | std::string DumpThreadStack(ThreadIdForStack tid) { |
291 | 6.79k | auto stack_trace = ThreadStack(tid); |
292 | 6.79k | if (!stack_trace.ok()) { |
293 | 24 | return stack_trace.status().message().ToBuffer(); |
294 | 24 | } |
295 | 6.77k | return stack_trace->Symbolize(); |
296 | 6.79k | } |
297 | | |
298 | 26.4k | Status SetStackTraceSignal(int signum) { |
299 | 26.4k | std::lock_guard<decltype(thread_stack_helper.mutex)> lock(thread_stack_helper.mutex); |
300 | 26.4k | if (!InitSignalHandlerUnlocked(signum)) { |
301 | 1 | return STATUS(InvalidArgument, "Unable to install signal handler"); |
302 | 1 | } |
303 | 26.3k | return Status::OK(); |
304 | 26.4k | } |
305 | | |
306 | | } // namespace yb |