/Users/deen/code/yugabyte-db/src/yb/util/memory/memory.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright 2010 Google Inc. All Rights Reserved |
2 | | // |
3 | | // Licensed to the Apache Software Foundation (ASF) under one |
4 | | // or more contributor license agreements. See the NOTICE file |
5 | | // distributed with this work for additional information |
6 | | // regarding copyright ownership. The ASF licenses this file |
7 | | // to you under the Apache License, Version 2.0 (the |
8 | | // "License"); you may not use this file except in compliance |
9 | | // with the License. You may obtain a copy of the License at |
10 | | // |
11 | | // http://www.apache.org/licenses/LICENSE-2.0 |
12 | | // |
13 | | // Unless required by applicable law or agreed to in writing, |
14 | | // software distributed under the License is distributed on an |
15 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
16 | | // KIND, either express or implied. See the License for the |
17 | | // specific language governing permissions and limitations |
18 | | // under the License. |
19 | | // |
20 | | // The following only applies to changes made to this file as part of YugaByte development. |
21 | | // |
22 | | // Portions Copyright (c) YugaByte, Inc. |
23 | | // |
24 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
25 | | // in compliance with the License. You may obtain a copy of the License at |
26 | | // |
27 | | // http://www.apache.org/licenses/LICENSE-2.0 |
28 | | // |
29 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
30 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
31 | | // or implied. See the License for the specific language governing permissions and limitations |
32 | | // under the License. |
33 | | // |
34 | | |
35 | | #include "yb/util/memory/memory.h" |
36 | | |
37 | | #include <string.h> |
38 | | |
39 | | #include <algorithm> |
40 | | #include <cstdlib> |
41 | | |
42 | | #include "yb/util/alignment.h" |
43 | | #include "yb/util/flag_tags.h" |
44 | | #include "yb/util/mem_tracker.h" |
45 | | #include "yb/util/size_literals.h" |
46 | | |
47 | | using std::copy; |
48 | | using std::max; |
49 | | using std::min; |
50 | | using std::reverse; |
51 | | using std::sort; |
52 | | using std::swap; |
53 | | |
54 | | namespace yb { |
55 | | |
56 | | namespace { |
57 | | static char dummy_buffer[0] = {}; |
58 | | } |
59 | | |
60 | | // This function is micro-optimized a bit, since it helps debug |
61 | | // mode tests run much faster. |
62 | | #if defined(__GNUC__) && !defined(__clang__) |
63 | | #pragma GCC push_options |
64 | | #pragma GCC optimize("-O3") |
65 | | #endif |
66 | 353M | void OverwriteWithPattern(char* p, size_t len, GStringPiece pattern) { |
67 | 353M | size_t pat_len = pattern.size(); |
68 | 353M | CHECK_LT(0, pat_len); |
69 | 353M | size_t rem = len; |
70 | 353M | const char *pat_ptr = pattern.data(); |
71 | | |
72 | 27.5G | while (rem >= pat_len) { |
73 | 27.1G | memcpy(p, pat_ptr, pat_len); |
74 | 27.1G | p += pat_len; |
75 | 27.1G | rem -= pat_len; |
76 | 27.1G | } |
77 | | |
78 | 30.7G | while (rem-- > 0) { |
79 | 30.3G | *p++ = *pat_ptr++; |
80 | 30.3G | } |
81 | 353M | } |
82 | | #if defined(__GNUC__) && !defined(__clang__) |
83 | | #pragma GCC pop_options |
84 | | #endif |
85 | | |
86 | 236M | Buffer::~Buffer() { |
87 | 236M | #if !defined(NDEBUG) && !defined(ADDRESS_SANITIZER) |
88 | 236M | if (data_ != nullptr) { |
89 | | // "unrolling" the string "BAD" makes for a much more efficient |
90 | | // OverwriteWithPattern call in debug mode, so we can keep this |
91 | | // useful bit of code without tests going slower! |
92 | | // |
93 | | // In ASAN mode, we don't bother with this, because when we free the memory, ASAN will |
94 | | // prevent us from accessing it anyway. |
95 | 117M | OverwriteWithPattern(reinterpret_cast<char*>(data_), size_, |
96 | 117M | "BADBADBADBADBADBADBADBADBADBADBAD" |
97 | 117M | "BADBADBADBADBADBADBADBADBADBADBAD" |
98 | 117M | "BADBADBADBADBADBADBADBADBADBADBAD"); |
99 | 117M | } |
100 | 236M | #endif |
101 | 236M | if (allocator_ != nullptr) allocator_->FreeInternal(this)117M ; |
102 | 236M | } |
103 | | |
104 | | void BufferAllocator::LogAllocation(size_t requested, |
105 | | size_t minimal, |
106 | 118M | const Buffer& buffer) { |
107 | 118M | if (!buffer) { |
108 | 2 | LOG(WARNING) << "Memory allocation failed. " |
109 | 2 | << "Number of bytes requested: " << requested |
110 | 2 | << ", minimal: " << minimal; |
111 | 2 | return; |
112 | 2 | } |
113 | 118M | if (buffer.size() < requested) { |
114 | 0 | LOG(WARNING) << "Memory allocation was shorter than requested. " |
115 | 0 | << "Number of bytes requested to allocate: " << requested |
116 | 0 | << ", minimal: " << minimal |
117 | 0 | << ", and actually allocated: " << buffer.size(); |
118 | 0 | } |
119 | 118M | } |
120 | | |
121 | | // TODO(onufry) - test whether the code still tests OK if we set this to true, |
122 | | // or remove this code and add a test that Google allocator does not change it's |
123 | | // contract - 16-aligned in -c opt and %16 == 8 in debug. |
124 | | DEFINE_bool(allocator_aligned_mode, false, |
125 | | "Use 16-byte alignment instead of 8-byte, " |
126 | | "unless explicitly specified otherwise - to boost SIMD"); |
127 | | TAG_FLAG(allocator_aligned_mode, hidden); |
128 | | |
129 | | HeapBufferAllocator::HeapBufferAllocator() |
130 | 8.70k | : aligned_mode_(FLAGS_allocator_aligned_mode) { |
131 | 8.70k | } |
132 | | |
133 | | Buffer HeapBufferAllocator::AllocateInternal( |
134 | | const size_t requested, |
135 | | const size_t minimal, |
136 | 118M | BufferAllocator* const originator) { |
137 | 118M | DCHECK_LE(minimal, requested); |
138 | 118M | void* data; |
139 | 118M | size_t attempted = requested; |
140 | 118M | while (true118M ) { |
141 | 118M | data = (attempted == 0) ? &dummy_buffer[0]0 : Malloc(attempted); |
142 | 118M | if (data != nullptr118M ) { |
143 | 118M | return CreateBuffer(data, attempted, originator); |
144 | 118M | } |
145 | 18.4E | if (attempted == minimal) return Buffer()0 ; |
146 | 18.4E | attempted = (attempted + minimal) / 2; |
147 | 18.4E | } |
148 | 118M | } |
149 | | |
150 | | bool HeapBufferAllocator::ReallocateInternal( |
151 | | const size_t requested, |
152 | | const size_t minimal, |
153 | | Buffer* const buffer, |
154 | 0 | BufferAllocator* const originator) { |
155 | 0 | DCHECK_LE(minimal, requested); |
156 | 0 | void* data; |
157 | 0 | size_t attempted = requested; |
158 | 0 | while (true) { |
159 | 0 | if (attempted == 0) { |
160 | 0 | if (buffer->size() > 0) free(buffer->data()); |
161 | 0 | data = &dummy_buffer[0]; |
162 | 0 | } else { |
163 | 0 | if (buffer->size() > 0) { |
164 | 0 | data = Realloc(buffer->data(), buffer->size(), attempted); |
165 | 0 | } else { |
166 | 0 | data = Malloc(attempted); |
167 | 0 | } |
168 | 0 | } |
169 | 0 | if (data != nullptr) { |
170 | 0 | UpdateBuffer(data, attempted, buffer); |
171 | 0 | return true; |
172 | 0 | } |
173 | 0 | if (attempted == minimal) return false; |
174 | 0 | attempted = minimal + (attempted - minimal - 1) / 2; |
175 | 0 | } |
176 | 0 | } |
177 | | |
178 | 117M | void HeapBufferAllocator::FreeInternal(Buffer* buffer) { |
179 | 117M | if (buffer->size() > 0) free(buffer->data())117M ; |
180 | 117M | } |
181 | | |
182 | 118M | void* HeapBufferAllocator::Malloc(size_t size) { |
183 | 118M | if (aligned_mode_) { |
184 | 0 | void* data; |
185 | 0 | if (posix_memalign(&data, 16, align_up(size, 16))) { |
186 | 0 | return nullptr; |
187 | 0 | } |
188 | 0 | return data; |
189 | 118M | } else { |
190 | 118M | return malloc(size); |
191 | 118M | } |
192 | 118M | } |
193 | | |
194 | | void* HeapBufferAllocator::Realloc(void* previousData, size_t previousSize, |
195 | 0 | size_t newSize) { |
196 | 0 | if (aligned_mode_) { |
197 | 0 | void* data = Malloc(newSize); |
198 | 0 | if (data) { |
199 | | // NOTE(ptab): We should use realloc here to avoid memmory coping, |
200 | | // but it doesn't work on memory allocated by posix_memalign(...). |
201 | | // realloc reallocates the memory but doesn't preserve the content. |
202 | | // TODO(ptab): reiterate after some time to check if it is fixed (tcmalloc ?) |
203 | 0 | memcpy(data, previousData, min(previousSize, newSize)); |
204 | 0 | free(previousData); |
205 | 0 | return data; |
206 | 0 | } else { |
207 | 0 | return nullptr; |
208 | 0 | } |
209 | 0 | } else { |
210 | 0 | return realloc(previousData, newSize); |
211 | 0 | } |
212 | 0 | } |
213 | | |
214 | | Buffer ClearingBufferAllocator::AllocateInternal(size_t requested, |
215 | | size_t minimal, |
216 | 0 | BufferAllocator* originator) { |
217 | 0 | Buffer buffer = DelegateAllocate(delegate_, requested, minimal, |
218 | 0 | originator); |
219 | 0 | if (buffer) memset(buffer.data(), 0, buffer.size()); |
220 | 0 | return buffer; |
221 | 0 | } |
222 | | |
223 | | bool ClearingBufferAllocator::ReallocateInternal(size_t requested, |
224 | | size_t minimal, |
225 | | Buffer* buffer, |
226 | 0 | BufferAllocator* originator) { |
227 | 0 | size_t offset = (buffer != nullptr ? buffer->size() : 0); |
228 | 0 | bool success = DelegateReallocate(delegate_, requested, minimal, buffer, |
229 | 0 | originator); |
230 | | // We expect buffer to be non-null in case of success. |
231 | 0 | assert(!success || buffer != nullptr); |
232 | 0 | if (success && buffer->size() > offset) { |
233 | 0 | memset(static_cast<char*>(buffer->data()) + offset, 0, |
234 | 0 | buffer->size() - offset); |
235 | 0 | } |
236 | 0 | return success; |
237 | 0 | } |
238 | | |
239 | 0 | void ClearingBufferAllocator::FreeInternal(Buffer* buffer) { |
240 | 0 | DelegateFree(delegate_, buffer); |
241 | 0 | } |
242 | | |
243 | | Buffer MediatingBufferAllocator::AllocateInternal( |
244 | | const size_t requested, |
245 | | const size_t minimal, |
246 | 0 | BufferAllocator* const originator) { |
247 | | // Allow the mediator to trim the request. |
248 | 0 | size_t granted; |
249 | 0 | if (requested > 0) { |
250 | 0 | granted = mediator_->Allocate(requested, minimal); |
251 | 0 | if (granted < minimal) return Buffer(); |
252 | 0 | } else { |
253 | 0 | granted = 0; |
254 | 0 | } |
255 | 0 | Buffer buffer = DelegateAllocate(delegate_, granted, minimal, originator); |
256 | 0 | if (!buffer) { |
257 | 0 | mediator_->Free(granted); |
258 | 0 | } else if (buffer.size() < granted) { |
259 | 0 | mediator_->Free(granted - buffer.size()); |
260 | 0 | } |
261 | 0 | return buffer; |
262 | 0 | } |
263 | | |
264 | | bool MediatingBufferAllocator::ReallocateInternal( |
265 | | const size_t requested, |
266 | | const size_t minimal, |
267 | | Buffer* const buffer, |
268 | 0 | BufferAllocator* const originator) { |
269 | | // Allow the mediator to trim the request. Be conservative; assume that |
270 | | // realloc may degenerate to malloc-memcpy-free. |
271 | 0 | size_t granted; |
272 | 0 | if (requested > 0) { |
273 | 0 | granted = mediator_->Allocate(requested, minimal); |
274 | 0 | if (granted < minimal) return false; |
275 | 0 | } else { |
276 | 0 | granted = 0; |
277 | 0 | } |
278 | 0 | size_t old_size = buffer->size(); |
279 | 0 | if (DelegateReallocate(delegate_, granted, minimal, buffer, originator)) { |
280 | 0 | mediator_->Free(granted - buffer->size() + old_size); |
281 | 0 | return true; |
282 | 0 | } else { |
283 | 0 | mediator_->Free(granted); |
284 | 0 | return false; |
285 | 0 | } |
286 | 0 | } |
287 | | |
288 | 0 | void MediatingBufferAllocator::FreeInternal(Buffer* buffer) { |
289 | 0 | mediator_->Free(buffer->size()); |
290 | 0 | DelegateFree(delegate_, buffer); |
291 | 0 | } |
292 | | |
293 | | Buffer MemoryStatisticsCollectingBufferAllocator::AllocateInternal( |
294 | | const size_t requested, |
295 | | const size_t minimal, |
296 | 0 | BufferAllocator* const originator) { |
297 | 0 | Buffer buffer = DelegateAllocate(delegate_, requested, minimal, originator); |
298 | 0 | if (buffer) { |
299 | 0 | memory_stats_collector_->AllocatedMemoryBytes(buffer.size()); |
300 | 0 | } else { |
301 | 0 | memory_stats_collector_->RefusedMemoryBytes(minimal); |
302 | 0 | } |
303 | 0 | return buffer; |
304 | 0 | } |
305 | | |
306 | | bool MemoryStatisticsCollectingBufferAllocator::ReallocateInternal( |
307 | | const size_t requested, |
308 | | const size_t minimal, |
309 | | Buffer* const buffer, |
310 | 0 | BufferAllocator* const originator) { |
311 | 0 | const size_t old_size = buffer->size(); |
312 | 0 | bool outcome = DelegateReallocate(delegate_, requested, minimal, buffer, |
313 | 0 | originator); |
314 | 0 | if (buffer->size() > old_size) { |
315 | 0 | memory_stats_collector_->AllocatedMemoryBytes(buffer->size() - old_size); |
316 | 0 | } else if (buffer->size() < old_size) { |
317 | 0 | memory_stats_collector_->FreedMemoryBytes(old_size - buffer->size()); |
318 | 0 | } else if (!outcome && (minimal > buffer->size())) { |
319 | 0 | memory_stats_collector_->RefusedMemoryBytes(minimal - buffer->size()); |
320 | 0 | } |
321 | 0 | return outcome; |
322 | 0 | } |
323 | | |
324 | 0 | void MemoryStatisticsCollectingBufferAllocator::FreeInternal(Buffer* buffer) { |
325 | 0 | DelegateFree(delegate_, buffer); |
326 | 0 | memory_stats_collector_->FreedMemoryBytes(buffer->size()); |
327 | 0 | } |
328 | | |
329 | 0 | size_t MemoryTrackingBufferAllocator::Available() const { |
330 | 0 | return enforce_limit_ ? mem_tracker_->SpareCapacity() : std::numeric_limits<int64_t>::max(); |
331 | 0 | } |
332 | | |
333 | 7.08k | bool MemoryTrackingBufferAllocator::TryConsume(int64_t bytes) { |
334 | | // Calls TryConsume first, even if enforce_limit_ is false: this |
335 | | // will cause mem_tracker_ to try to free up more memory by GCing. |
336 | 7.08k | if (!mem_tracker_->TryConsume(bytes)) { |
337 | 5 | if (enforce_limit_) { |
338 | 4 | return false; |
339 | 4 | } else { |
340 | | // If enforce_limit_ is false, allocate memory anyway. |
341 | 1 | mem_tracker_->Consume(bytes); |
342 | 1 | } |
343 | 5 | } |
344 | 7.08k | return true; |
345 | 7.08k | } |
346 | | |
347 | | Buffer MemoryTrackingBufferAllocator::AllocateInternal(size_t requested, |
348 | | size_t minimal, |
349 | 7.08k | BufferAllocator* originator) { |
350 | 7.08k | if (TryConsume(requested)) { |
351 | 7.08k | Buffer buffer = DelegateAllocate(delegate_, requested, requested, originator); |
352 | 7.08k | if (!buffer) { |
353 | 0 | mem_tracker_->Release(requested); |
354 | 7.08k | } else { |
355 | 7.08k | return buffer; |
356 | 7.08k | } |
357 | 7.08k | } |
358 | | |
359 | 1 | if (TryConsume(minimal)) { |
360 | 0 | Buffer buffer = DelegateAllocate(delegate_, minimal, minimal, originator); |
361 | 0 | if (!buffer) { |
362 | 0 | mem_tracker_->Release(minimal); |
363 | 0 | } |
364 | 0 | return buffer; |
365 | 0 | } |
366 | | |
367 | 1 | return Buffer(); |
368 | 1 | } |
369 | | |
370 | | |
371 | | bool MemoryTrackingBufferAllocator::ReallocateInternal(size_t requested, |
372 | | size_t minimal, |
373 | | Buffer* buffer, |
374 | 0 | BufferAllocator* originator) { |
375 | 0 | LOG(FATAL) << "Not implemented"; |
376 | 0 | return false; |
377 | 0 | } |
378 | | |
379 | 1.39k | void MemoryTrackingBufferAllocator::FreeInternal(Buffer* buffer) { |
380 | 1.39k | DelegateFree(delegate_, buffer); |
381 | 1.39k | mem_tracker_->Release(buffer->size()); |
382 | 1.39k | } |
383 | | |
384 | 54 | std::string TcMallocStats() { |
385 | | #if defined(TCMALLOC_ENABLED) |
386 | | char buf[20_KB]; |
387 | | MallocExtension::instance()->GetStats(buf, sizeof(buf)); |
388 | | return buf; |
389 | | #else |
390 | 54 | return ""; |
391 | 54 | #endif |
392 | 54 | } |
393 | | |
394 | | |
395 | | } // namespace yb |