/Users/deen/code/yugabyte-db/src/yb/rocksdb/db/internal_stats.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | | // This source code is licensed under the BSD-style license found in the |
3 | | // LICENSE file in the root directory of this source tree. An additional grant |
4 | | // of patent rights can be found in the PATENTS file in the same directory. |
5 | | // |
6 | | // The following only applies to changes made to this file as part of YugaByte development. |
7 | | // |
8 | | // Portions Copyright (c) YugaByte, Inc. |
9 | | // |
10 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
11 | | // in compliance with the License. You may obtain a copy of the License at |
12 | | // |
13 | | // http://www.apache.org/licenses/LICENSE-2.0 |
14 | | // |
15 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
16 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
17 | | // or implied. See the License for the specific language governing permissions and limitations |
18 | | // under the License. |
19 | | // |
20 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
21 | | // Use of this source code is governed by a BSD-style license that can be |
22 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
23 | | // |
24 | | |
25 | | #ifndef YB_ROCKSDB_DB_INTERNAL_STATS_H |
26 | | #define YB_ROCKSDB_DB_INTERNAL_STATS_H |
27 | | |
28 | | #pragma once |
29 | | #include <vector> |
30 | | #include <string> |
31 | | |
32 | | #include "yb/rocksdb/db/version_set.h" |
33 | | |
34 | | class ColumnFamilyData; |
35 | | |
36 | | namespace rocksdb { |
37 | | |
38 | | class MemTableList; |
39 | | class DBImpl; |
40 | | |
41 | | // Config for retrieving a property's value. |
42 | | struct DBPropertyInfo { |
43 | | bool need_out_of_mutex; |
44 | | |
45 | | // gcc had an internal error for initializing union of pointer-to-member- |
46 | | // functions. Workaround is to populate exactly one of the following function |
47 | | // pointers with a non-nullptr value. |
48 | | |
49 | | // @param value Value-result argument for storing the property's string value |
50 | | // @param suffix Argument portion of the property. For example, suffix would |
51 | | // be "5" for the property "rocksdb.num-files-at-level5". So far, only |
52 | | // certain string properties take an argument. |
53 | | bool (InternalStats::*handle_string)(std::string* value, Slice suffix); |
54 | | |
55 | | // @param value Value-result argument for storing the property's uint64 value |
56 | | // @param db Many of the int properties rely on DBImpl methods. |
57 | | // @param version Version is needed in case the property is retrieved without |
58 | | // holding db mutex, which is only supported for int properties. |
59 | | bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db, |
60 | | Version* version); |
61 | | }; |
62 | | |
63 | | extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); |
64 | | |
65 | | #ifndef ROCKSDB_LITE |
66 | | enum class InternalDBStatsType { |
67 | | WAL_FILE_BYTES, |
68 | | WAL_FILE_SYNCED, |
69 | | BYTES_WRITTEN, |
70 | | NUMBER_KEYS_WRITTEN, |
71 | | WRITE_DONE_BY_OTHER, |
72 | | WRITE_DONE_BY_SELF, |
73 | | WRITE_WITH_WAL, |
74 | | WRITE_STALL_MICROS, |
75 | | INTERNAL_DB_STATS_ENUM_MAX, |
76 | | }; |
77 | | |
78 | | class InternalStats { |
79 | | public: |
80 | | enum InternalCFStatsType { |
81 | | LEVEL0_SLOWDOWN_TOTAL, |
82 | | LEVEL0_SLOWDOWN_WITH_COMPACTION, |
83 | | MEMTABLE_COMPACTION, |
84 | | MEMTABLE_SLOWDOWN, |
85 | | LEVEL0_NUM_FILES_TOTAL, |
86 | | LEVEL0_NUM_FILES_WITH_COMPACTION, |
87 | | SOFT_PENDING_COMPACTION_BYTES_LIMIT, |
88 | | HARD_PENDING_COMPACTION_BYTES_LIMIT, |
89 | | WRITE_STALLS_ENUM_MAX, |
90 | | BYTES_FLUSHED, |
91 | | INTERNAL_CF_STATS_ENUM_MAX, |
92 | | }; |
93 | | |
94 | | InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd); |
95 | | ~InternalStats(); |
96 | | |
97 | | // Per level compaction stats. comp_stats_[level] stores the stats for |
98 | | // compactions that produced data for the specified "level". |
99 | | struct CompactionStats { |
100 | | uint64_t micros; |
101 | | |
102 | | // The number of bytes read from all non-output levels |
103 | | uint64_t bytes_read_non_output_levels; |
104 | | |
105 | | // The number of bytes read from the compaction output level. |
106 | | uint64_t bytes_read_output_level; |
107 | | |
108 | | // Total number of bytes written during compaction |
109 | | uint64_t bytes_written; |
110 | | |
111 | | // Total number of bytes moved to the output level |
112 | | uint64_t bytes_moved; |
113 | | |
114 | | // The number of compaction input files in all non-output levels. |
115 | | int num_input_files_in_non_output_levels; |
116 | | |
117 | | // The number of compaction input files in the output level. |
118 | | int num_input_files_in_output_level; |
119 | | |
120 | | // The number of compaction output files. |
121 | | int num_output_files; |
122 | | |
123 | | // Total incoming entries during compaction between levels N and N+1 |
124 | | uint64_t num_input_records; |
125 | | |
126 | | // Accumulated diff number of entries |
127 | | // (num input entries - num output entires) for compaction levels N and N+1 |
128 | | uint64_t num_dropped_records; |
129 | | |
130 | | // Number of compactions done |
131 | | int count; |
132 | | |
133 | | explicit CompactionStats(int _count = 0) |
134 | | : micros(0), |
135 | | bytes_read_non_output_levels(0), |
136 | | bytes_read_output_level(0), |
137 | | bytes_written(0), |
138 | | bytes_moved(0), |
139 | | num_input_files_in_non_output_levels(0), |
140 | | num_input_files_in_output_level(0), |
141 | | num_output_files(0), |
142 | | num_input_records(0), |
143 | | num_dropped_records(0), |
144 | 834k | count(_count) {} |
145 | | |
146 | 39.4k | void Add(const CompactionStats& c) { |
147 | 39.4k | this->micros += c.micros; |
148 | 39.4k | this->bytes_read_non_output_levels += c.bytes_read_non_output_levels; |
149 | 39.4k | this->bytes_read_output_level += c.bytes_read_output_level; |
150 | 39.4k | this->bytes_written += c.bytes_written; |
151 | 39.4k | this->bytes_moved += c.bytes_moved; |
152 | 39.4k | this->num_input_files_in_non_output_levels += |
153 | 39.4k | c.num_input_files_in_non_output_levels; |
154 | 39.4k | this->num_input_files_in_output_level += |
155 | 39.4k | c.num_input_files_in_output_level; |
156 | 39.4k | this->num_output_files += c.num_output_files; |
157 | 39.4k | this->num_input_records += c.num_input_records; |
158 | 39.4k | this->num_dropped_records += c.num_dropped_records; |
159 | 39.4k | this->count += c.count; |
160 | 39.4k | } |
161 | | |
162 | 0 | void Subtract(const CompactionStats& c) { |
163 | 0 | this->micros -= c.micros; |
164 | 0 | this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels; |
165 | 0 | this->bytes_read_output_level -= c.bytes_read_output_level; |
166 | 0 | this->bytes_written -= c.bytes_written; |
167 | 0 | this->bytes_moved -= c.bytes_moved; |
168 | 0 | this->num_input_files_in_non_output_levels -= |
169 | 0 | c.num_input_files_in_non_output_levels; |
170 | 0 | this->num_input_files_in_output_level -= |
171 | 0 | c.num_input_files_in_output_level; |
172 | 0 | this->num_output_files -= c.num_output_files; |
173 | 0 | this->num_input_records -= c.num_input_records; |
174 | 0 | this->num_dropped_records -= c.num_dropped_records; |
175 | 0 | this->count -= c.count; |
176 | 0 | } |
177 | | }; |
178 | | |
179 | 39.4k | void AddCompactionStats(int level, const CompactionStats& stats) { |
180 | 39.4k | comp_stats_[level].Add(stats); |
181 | 39.4k | } |
182 | | |
183 | 12.2k | void IncBytesMoved(int level, uint64_t amount) { |
184 | 12.2k | comp_stats_[level].bytes_moved += amount; |
185 | 12.2k | } |
186 | | |
187 | 34.3k | void AddCFStats(InternalCFStatsType type, uint64_t value) { |
188 | 34.3k | cf_stats_value_[type] += value; |
189 | 34.3k | ++cf_stats_count_[type]; |
190 | 34.3k | } |
191 | | |
192 | 0 | uint64_t GetCFStats(InternalCFStatsType type) { |
193 | 0 | return cf_stats_value_[type]; |
194 | 0 | } |
195 | | |
196 | 98.7M | void AddDBStats(InternalDBStatsType type, uint64_t value) { |
197 | 98.7M | auto& v = db_stats_[static_cast<size_t>(type)]; |
198 | 98.7M | v.store(v.load(std::memory_order_relaxed) + value, |
199 | 98.7M | std::memory_order_relaxed); |
200 | 98.7M | } |
201 | | |
202 | 49 | uint64_t GetDBStats(InternalDBStatsType type) { |
203 | 49 | return db_stats_[static_cast<size_t>(type)].load(std::memory_order_relaxed); |
204 | 49 | } |
205 | | |
206 | | HistogramImpl* GetFileReadHist(int level); |
207 | | |
208 | 19 | uint64_t GetBackgroundErrorCount() const { return bg_error_count_; } |
209 | | |
210 | 74 | uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; } |
211 | | |
212 | | bool GetStringProperty(const DBPropertyInfo& property_info, |
213 | | const Slice& property, std::string* value); |
214 | | |
215 | | bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, |
216 | | DBImpl* db); |
217 | | |
218 | | bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info, |
219 | | Version* version, uint64_t* value); |
220 | | |
221 | | // Store a mapping from the user-facing DB::Properties string to our |
222 | | // DBPropertyInfo struct used internally for retrieving properties. |
223 | | static const std::unordered_map<std::string, DBPropertyInfo> ppt_name_to_info; |
224 | | |
225 | | private: |
226 | | void DumpDBStats(std::string* value); |
227 | | void DumpCFStats(std::string* value); |
228 | | |
229 | | // Per-DB stats |
230 | | std::atomic<uint64_t> db_stats_[ |
231 | | static_cast<size_t>(InternalDBStatsType::INTERNAL_DB_STATS_ENUM_MAX)]; |
232 | | // Per-ColumnFamily stats |
233 | | uint64_t cf_stats_value_[INTERNAL_CF_STATS_ENUM_MAX]; |
234 | | uint64_t cf_stats_count_[INTERNAL_CF_STATS_ENUM_MAX]; |
235 | | // Per-ColumnFamily/level compaction stats |
236 | | std::vector<CompactionStats> comp_stats_; |
237 | | std::vector<HistogramImpl> file_read_latency_; |
238 | | |
239 | | // Used to compute per-interval statistics |
240 | | struct CFStatsSnapshot { |
241 | | // ColumnFamily-level stats |
242 | | CompactionStats comp_stats; |
243 | | uint64_t ingest_bytes; // Bytes written to L0 |
244 | | uint64_t stall_count; // Stall count |
245 | | |
246 | | CFStatsSnapshot() |
247 | | : comp_stats(0), |
248 | | ingest_bytes(0), |
249 | 347k | stall_count(0) {} |
250 | | } cf_stats_snapshot_; |
251 | | |
252 | | struct DBStatsSnapshot { |
253 | | // DB-level stats |
254 | | uint64_t ingest_bytes; // Bytes written by user |
255 | | uint64_t wal_bytes; // Bytes written to WAL |
256 | | uint64_t wal_synced; // Number of times WAL is synced |
257 | | uint64_t write_with_wal; // Number of writes that request WAL |
258 | | // These count the number of writes processed by the calling thread or |
259 | | // another thread. |
260 | | uint64_t write_other; |
261 | | uint64_t write_self; |
262 | | // Stats from compaction jobs - bytes written, bytes read, duration. |
263 | | uint64_t compact_bytes_write; |
264 | | uint64_t compact_bytes_read; |
265 | | uint64_t compact_micros; |
266 | | // Total number of keys written. write_self and write_other measure number |
267 | | // of write requests written, Each of the write request can contain updates |
268 | | // to multiple keys. num_keys_written is total number of keys updated by all |
269 | | // those writes. |
270 | | uint64_t num_keys_written; |
271 | | // Total time writes delayed by stalls. |
272 | | uint64_t write_stall_micros; |
273 | | double seconds_up; |
274 | | |
275 | | DBStatsSnapshot() |
276 | | : ingest_bytes(0), |
277 | | wal_bytes(0), |
278 | | wal_synced(0), |
279 | | write_with_wal(0), |
280 | | write_other(0), |
281 | | write_self(0), |
282 | | compact_bytes_write(0), |
283 | | compact_bytes_read(0), |
284 | | compact_micros(0), |
285 | | num_keys_written(0), |
286 | | write_stall_micros(0), |
287 | 347k | seconds_up(0) {} |
288 | | } db_stats_snapshot_; |
289 | | |
290 | | // Handler functions for getting property values. They use "value" as a value- |
291 | | // result argument, and return true upon successfully setting "value". |
292 | | bool HandleNumFilesAtLevel(std::string* value, Slice suffix); |
293 | | bool HandleLevelStats(std::string* value, Slice suffix); |
294 | | bool HandleStats(std::string* value, Slice suffix); |
295 | | bool HandleCFStats(std::string* value, Slice suffix); |
296 | | bool HandleDBStats(std::string* value, Slice suffix); |
297 | | bool HandleSsTables(std::string* value, Slice suffix); |
298 | | bool HandleAggregatedTableProperties(std::string* value, Slice suffix); |
299 | | bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix); |
300 | | bool HandleNumImmutableMemTable(uint64_t* value, DBImpl* db, |
301 | | Version* version); |
302 | | bool HandleNumImmutableMemTableFlushed(uint64_t* value, DBImpl* db, |
303 | | Version* version); |
304 | | bool HandleMemTableFlushPending(uint64_t* value, DBImpl* db, |
305 | | Version* version); |
306 | | bool HandleNumRunningFlushes(uint64_t* value, DBImpl* db, Version* version); |
307 | | bool HandleCompactionPending(uint64_t* value, DBImpl* db, Version* version); |
308 | | bool HandleNumRunningCompactions(uint64_t* value, DBImpl* db, |
309 | | Version* version); |
310 | | bool HandleBackgroundErrors(uint64_t* value, DBImpl* db, Version* version); |
311 | | bool HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db, |
312 | | Version* version); |
313 | | bool HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version); |
314 | | bool HandleSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version); |
315 | | bool HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db, |
316 | | Version* version); |
317 | | bool HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db, |
318 | | Version* version); |
319 | | bool HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db, |
320 | | Version* version); |
321 | | bool HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db, |
322 | | Version* version); |
323 | | bool HandleEstimateNumKeys(uint64_t* value, DBImpl* db, Version* version); |
324 | | bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version); |
325 | | bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version); |
326 | | bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version); |
327 | | bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db, |
328 | | Version* version); |
329 | | bool HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db, |
330 | | Version* version); |
331 | | bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version); |
332 | | bool HandleTotalSstFilesSize(uint64_t* value, DBImpl* db, Version* version); |
333 | | bool HandleEstimatePendingCompactionBytes(uint64_t* value, DBImpl* db, |
334 | | Version* version); |
335 | | bool HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db, |
336 | | Version* version); |
337 | | bool HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db, |
338 | | Version* version); |
339 | | |
340 | | // Total number of background errors encountered. Every time a flush task |
341 | | // or compaction task fails, this counter is incremented. The failure can |
342 | | // be caused by any possible reason, including file system errors, out of |
343 | | // resources, or input file corruption. Failing when retrying the same flush |
344 | | // or compaction will cause the counter to increase too. |
345 | | uint64_t bg_error_count_; |
346 | | |
347 | | const int number_levels_; |
348 | | Env* env_; |
349 | | ColumnFamilyData* cfd_; |
350 | | const uint64_t started_at_; |
351 | | }; |
352 | | |
353 | | #else |
354 | | |
355 | | class InternalStats { |
356 | | public: |
357 | | enum InternalCFStatsType { |
358 | | LEVEL0_SLOWDOWN_TOTAL, |
359 | | LEVEL0_SLOWDOWN_WITH_COMPACTION, |
360 | | MEMTABLE_COMPACTION, |
361 | | MEMTABLE_SLOWDOWN, |
362 | | LEVEL0_NUM_FILES_TOTAL, |
363 | | LEVEL0_NUM_FILES_WITH_COMPACTION, |
364 | | SOFT_PENDING_COMPACTION_BYTES_LIMIT, |
365 | | HARD_PENDING_COMPACTION_BYTES_LIMIT, |
366 | | WRITE_STALLS_ENUM_MAX, |
367 | | BYTES_FLUSHED, |
368 | | INTERNAL_CF_STATS_ENUM_MAX, |
369 | | }; |
370 | | |
371 | | enum InternalDBStatsType { |
372 | | WAL_FILE_BYTES, |
373 | | WAL_FILE_SYNCED, |
374 | | BYTES_WRITTEN, |
375 | | NUMBER_KEYS_WRITTEN, |
376 | | WRITE_DONE_BY_OTHER, |
377 | | WRITE_DONE_BY_SELF, |
378 | | WRITE_WITH_WAL, |
379 | | WRITE_STALL_MICROS, |
380 | | INTERNAL_DB_STATS_ENUM_MAX, |
381 | | }; |
382 | | |
383 | | InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd) {} |
384 | | |
385 | | struct CompactionStats { |
386 | | uint64_t micros; |
387 | | uint64_t bytes_read_non_output_levels; |
388 | | uint64_t bytes_read_output_level; |
389 | | uint64_t bytes_written; |
390 | | uint64_t bytes_moved; |
391 | | int num_input_files_in_non_output_levels; |
392 | | int num_input_files_in_output_level; |
393 | | int num_output_files; |
394 | | uint64_t num_input_records; |
395 | | uint64_t num_dropped_records; |
396 | | int count; |
397 | | |
398 | | explicit CompactionStats(int _count = 0) {} |
399 | | |
400 | | explicit CompactionStats(const CompactionStats& c) {} |
401 | | |
402 | | void Add(const CompactionStats& c) {} |
403 | | |
404 | | void Subtract(const CompactionStats& c) {} |
405 | | }; |
406 | | |
407 | | void AddCompactionStats(int level, const CompactionStats& stats) {} |
408 | | |
409 | | void IncBytesMoved(int level, uint64_t amount) {} |
410 | | |
411 | | void AddCFStats(InternalCFStatsType type, uint64_t value) {} |
412 | | |
413 | | void AddDBStats(InternalDBStatsType type, uint64_t value) {} |
414 | | |
415 | | HistogramImpl* GetFileReadHist(int level) { return nullptr; } |
416 | | |
417 | | uint64_t GetBackgroundErrorCount() const { return 0; } |
418 | | |
419 | | uint64_t BumpAndGetBackgroundErrorCount() { return 0; } |
420 | | |
421 | | bool GetStringProperty(const DBPropertyInfo& property_info, |
422 | | const Slice& property, std::string* value) { |
423 | | return false; |
424 | | } |
425 | | |
426 | | bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value, |
427 | | DBImpl* db) const { |
428 | | return false; |
429 | | } |
430 | | |
431 | | bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info, |
432 | | Version* version, uint64_t* value) const { |
433 | | return false; |
434 | | } |
435 | | }; |
436 | | #endif // !ROCKSDB_LITE |
437 | | |
438 | | } // namespace rocksdb |
439 | | |
440 | | #endif // YB_ROCKSDB_DB_INTERNAL_STATS_H |