/Users/deen/code/yugabyte-db/src/yb/rocksdb/util/options.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | | // This source code is licensed under the BSD-style license found in the |
3 | | // LICENSE file in the root directory of this source tree. An additional grant |
4 | | // of patent rights can be found in the PATENTS file in the same directory. |
5 | | // |
6 | | // The following only applies to changes made to this file as part of YugaByte development. |
7 | | // |
8 | | // Portions Copyright (c) YugaByte, Inc. |
9 | | // |
10 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
11 | | // in compliance with the License. You may obtain a copy of the License at |
12 | | // |
13 | | // http://www.apache.org/licenses/LICENSE-2.0 |
14 | | // |
15 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
16 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
17 | | // or implied. See the License for the specific language governing permissions and limitations |
18 | | // under the License. |
19 | | // |
20 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
21 | | // Use of this source code is governed by a BSD-style license that can be |
22 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
23 | | |
24 | | #include "yb/rocksdb/options.h" |
25 | | #include "yb/rocksdb/immutable_options.h" |
26 | | |
27 | | #ifndef __STDC_FORMAT_MACROS |
28 | | #define __STDC_FORMAT_MACROS |
29 | | #endif |
30 | | |
31 | | #include <inttypes.h> |
32 | | #include <limits> |
33 | | |
34 | | #include "yb/rocksdb/cache.h" |
35 | | #include "yb/rocksdb/compaction_filter.h" |
36 | | #include "yb/rocksdb/comparator.h" |
37 | | #include "yb/rocksdb/env.h" |
38 | | #include "yb/rocksdb/filter_policy.h" |
39 | | #include "yb/rocksdb/sst_file_manager.h" |
40 | | #include "yb/rocksdb/memtablerep.h" |
41 | | #include "yb/rocksdb/merge_operator.h" |
42 | | #include "yb/util/slice.h" |
43 | | #include "yb/rocksdb/slice_transform.h" |
44 | | #include "yb/rocksdb/table.h" |
45 | | #include "yb/rocksdb/table_properties.h" |
46 | | #include "yb/rocksdb/wal_filter.h" |
47 | | #include "yb/rocksdb/table/block_based_table_factory.h" |
48 | | #include "yb/rocksdb/util/compression.h" |
49 | | #include "yb/rocksdb/util/statistics.h" |
50 | | #include "yb/rocksdb/util/xfunc.h" |
51 | | |
52 | | namespace rocksdb { |
53 | | |
54 | | ImmutableCFOptions::ImmutableCFOptions(const Options& options) |
55 | | : compaction_style(options.compaction_style), |
56 | | compaction_options_universal(options.compaction_options_universal), |
57 | | compaction_options_fifo(options.compaction_options_fifo), |
58 | | prefix_extractor(options.prefix_extractor.get()), |
59 | | comparator(options.comparator), |
60 | | merge_operator(options.merge_operator.get()), |
61 | | compaction_filter(options.compaction_filter), |
62 | | compaction_filter_factory(options.compaction_filter_factory.get()), |
63 | | inplace_update_support(options.inplace_update_support), |
64 | | inplace_callback(options.inplace_callback), |
65 | | info_log(options.info_log.get()), |
66 | | statistics(options.statistics.get()), |
67 | | env(options.env), |
68 | | delayed_write_rate(options.delayed_write_rate), |
69 | | allow_mmap_reads(options.allow_mmap_reads), |
70 | | allow_mmap_writes(options.allow_mmap_writes), |
71 | | db_paths(options.db_paths), |
72 | | memtable_factory(options.memtable_factory.get()), |
73 | | table_factory(options.table_factory.get()), |
74 | | table_properties_collector_factories( |
75 | | options.table_properties_collector_factories), |
76 | | advise_random_on_open(options.advise_random_on_open), |
77 | | bloom_locality(options.bloom_locality), |
78 | | purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush), |
79 | | min_partial_merge_operands(options.min_partial_merge_operands), |
80 | | disable_data_sync(options.disableDataSync), |
81 | | use_fsync(options.use_fsync), |
82 | | compression(options.compression), |
83 | | compression_per_level(options.compression_per_level), |
84 | | compression_opts(options.compression_opts), |
85 | | level_compaction_dynamic_level_bytes( |
86 | | options.level_compaction_dynamic_level_bytes), |
87 | | access_hint_on_compaction_start(options.access_hint_on_compaction_start), |
88 | | new_table_reader_for_compaction_inputs( |
89 | | options.new_table_reader_for_compaction_inputs), |
90 | | compaction_readahead_size(options.compaction_readahead_size), |
91 | | num_levels(options.num_levels), |
92 | | optimize_filters_for_hits(options.optimize_filters_for_hits), |
93 | | listeners(options.listeners), |
94 | | row_cache(options.row_cache), |
95 | | mem_tracker(options.mem_tracker), |
96 | | block_based_table_mem_tracker(options.block_based_table_mem_tracker), |
97 | | iterator_replacer(options.iterator_replacer), |
98 | 885k | compaction_file_filter_factory(options.compaction_file_filter_factory.get()) {} |
99 | | |
100 | | ColumnFamilyOptions::ColumnFamilyOptions() |
101 | | : comparator(BytewiseComparator()), |
102 | | merge_operator(nullptr), |
103 | | compaction_filter(nullptr), |
104 | | compaction_filter_factory(nullptr), |
105 | | write_buffer_size(4_MB), // Option expects bytes. |
106 | | max_write_buffer_number(2), |
107 | | min_write_buffer_number_to_merge(1), |
108 | | max_write_buffer_number_to_maintain(0), |
109 | | compression(Snappy_Supported() ? kSnappyCompression : kNoCompression), |
110 | | prefix_extractor(nullptr), |
111 | | num_levels(7), |
112 | | level0_file_num_compaction_trigger(4), |
113 | | level0_slowdown_writes_trigger(20), |
114 | | level0_stop_writes_trigger(24), |
115 | | target_file_size_base(2 * 1048576), |
116 | | target_file_size_multiplier(1), |
117 | | max_bytes_for_level_base(10 * 1048576), |
118 | | level_compaction_dynamic_level_bytes(false), |
119 | | max_bytes_for_level_multiplier(10), |
120 | | max_bytes_for_level_multiplier_additional(num_levels, 1), |
121 | | expanded_compaction_factor(25), |
122 | | source_compaction_factor(1), |
123 | | max_grandparent_overlap_factor(10), |
124 | | soft_rate_limit(0.0), |
125 | | hard_rate_limit(0.0), |
126 | | soft_pending_compaction_bytes_limit(0), |
127 | | hard_pending_compaction_bytes_limit(0), |
128 | | rate_limit_delay_max_milliseconds(1000), |
129 | | arena_block_size(0), |
130 | | disable_auto_compactions(false), |
131 | | purge_redundant_kvs_while_flush(true), |
132 | | compaction_style(kCompactionStyleLevel), |
133 | | compaction_pri(kByCompensatedSize), |
134 | | verify_checksums_in_compaction(true), |
135 | | filter_deletes(false), |
136 | | max_sequential_skip_in_iterations(8), |
137 | | memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)), |
138 | | table_factory( |
139 | | std::shared_ptr<TableFactory>(new BlockBasedTableFactory())), |
140 | | inplace_update_support(false), |
141 | | inplace_update_num_locks(10000), |
142 | | inplace_callback(nullptr), |
143 | | memtable_prefix_bloom_bits(0), |
144 | | memtable_prefix_bloom_probes(6), |
145 | | memtable_prefix_bloom_huge_page_tlb_size(0), |
146 | | bloom_locality(0), |
147 | | max_successive_merges(0), |
148 | | min_partial_merge_operands(2), |
149 | | optimize_filters_for_hits(false), |
150 | | paranoid_file_checks(false), |
151 | 3.62M | compaction_measure_io_stats(false) { |
152 | 3.62M | assert(memtable_factory.get() != nullptr); |
153 | 3.62M | } |
154 | | |
155 | | ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) |
156 | | : comparator(options.comparator), |
157 | | merge_operator(options.merge_operator), |
158 | | compaction_filter(options.compaction_filter), |
159 | | compaction_filter_factory(options.compaction_filter_factory), |
160 | | write_buffer_size(options.write_buffer_size), |
161 | | max_write_buffer_number(options.max_write_buffer_number), |
162 | | min_write_buffer_number_to_merge( |
163 | | options.min_write_buffer_number_to_merge), |
164 | | max_write_buffer_number_to_maintain( |
165 | | options.max_write_buffer_number_to_maintain), |
166 | | compression(options.compression), |
167 | | compression_per_level(options.compression_per_level), |
168 | | compression_opts(options.compression_opts), |
169 | | prefix_extractor(options.prefix_extractor), |
170 | | num_levels(options.num_levels), |
171 | | level0_file_num_compaction_trigger( |
172 | | options.level0_file_num_compaction_trigger), |
173 | | level0_slowdown_writes_trigger(options.level0_slowdown_writes_trigger), |
174 | | level0_stop_writes_trigger(options.level0_stop_writes_trigger), |
175 | | target_file_size_base(options.target_file_size_base), |
176 | | target_file_size_multiplier(options.target_file_size_multiplier), |
177 | | max_bytes_for_level_base(options.max_bytes_for_level_base), |
178 | | level_compaction_dynamic_level_bytes( |
179 | | options.level_compaction_dynamic_level_bytes), |
180 | | max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), |
181 | | max_bytes_for_level_multiplier_additional( |
182 | | options.max_bytes_for_level_multiplier_additional), |
183 | | expanded_compaction_factor(options.expanded_compaction_factor), |
184 | | source_compaction_factor(options.source_compaction_factor), |
185 | | max_grandparent_overlap_factor(options.max_grandparent_overlap_factor), |
186 | | soft_rate_limit(options.soft_rate_limit), |
187 | | soft_pending_compaction_bytes_limit( |
188 | | options.soft_pending_compaction_bytes_limit), |
189 | | hard_pending_compaction_bytes_limit( |
190 | | options.hard_pending_compaction_bytes_limit), |
191 | | rate_limit_delay_max_milliseconds( |
192 | | options.rate_limit_delay_max_milliseconds), |
193 | | arena_block_size(options.arena_block_size), |
194 | | disable_auto_compactions(options.disable_auto_compactions), |
195 | | purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush), |
196 | | compaction_style(options.compaction_style), |
197 | | compaction_pri(options.compaction_pri), |
198 | | verify_checksums_in_compaction(options.verify_checksums_in_compaction), |
199 | | compaction_options_universal(options.compaction_options_universal), |
200 | | compaction_options_fifo(options.compaction_options_fifo), |
201 | | filter_deletes(options.filter_deletes), |
202 | | max_sequential_skip_in_iterations( |
203 | | options.max_sequential_skip_in_iterations), |
204 | | memtable_factory(options.memtable_factory), |
205 | | table_factory(options.table_factory), |
206 | | table_properties_collector_factories( |
207 | | options.table_properties_collector_factories), |
208 | | inplace_update_support(options.inplace_update_support), |
209 | | inplace_update_num_locks(options.inplace_update_num_locks), |
210 | | inplace_callback(options.inplace_callback), |
211 | | memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), |
212 | | memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), |
213 | | memtable_prefix_bloom_huge_page_tlb_size( |
214 | | options.memtable_prefix_bloom_huge_page_tlb_size), |
215 | | bloom_locality(options.bloom_locality), |
216 | | max_successive_merges(options.max_successive_merges), |
217 | | min_partial_merge_operands(options.min_partial_merge_operands), |
218 | | optimize_filters_for_hits(options.optimize_filters_for_hits), |
219 | | paranoid_file_checks(options.paranoid_file_checks), |
220 | 2.14M | compaction_measure_io_stats(options.compaction_measure_io_stats) { |
221 | 2.14M | assert(memtable_factory.get() != nullptr); |
222 | 2.14M | if (max_bytes_for_level_multiplier_additional.size() < |
223 | 2.14M | static_cast<unsigned int>(num_levels)) { |
224 | 957 | max_bytes_for_level_multiplier_additional.resize(num_levels, 1); |
225 | 957 | } |
226 | 2.14M | } |
227 | | |
228 | | DBOptions::DBOptions() |
229 | | : create_if_missing(false), |
230 | | create_missing_column_families(false), |
231 | | error_if_exists(false), |
232 | | paranoid_checks(true), |
233 | | env(Env::Default()), |
234 | | checkpoint_env(nullptr), |
235 | | rate_limiter(nullptr), |
236 | | sst_file_manager(nullptr), |
237 | | info_log(nullptr), |
238 | | #ifdef NDEBUG |
239 | | info_log_level(INFO_LEVEL), |
240 | | #else |
241 | | info_log_level(DEBUG_LEVEL), |
242 | | #endif // NDEBUG |
243 | | max_open_files(5000), |
244 | | max_file_opening_threads(1), |
245 | | max_total_wal_size(0), |
246 | | statistics(nullptr), |
247 | | disableDataSync(false), |
248 | | use_fsync(false), |
249 | | db_log_dir(""), |
250 | | wal_dir(""), |
251 | | delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000), |
252 | | base_background_compactions(-1), |
253 | | max_background_compactions(1), |
254 | | num_reserved_small_compaction_threads(-1), |
255 | | compaction_size_threshold_bytes(std::numeric_limits<uint64_t>::max()), |
256 | | max_subcompactions(1), |
257 | | max_background_flushes(1), |
258 | | max_log_file_size(0), |
259 | | log_file_time_to_roll(0), |
260 | | keep_log_file_num(1000), |
261 | | recycle_log_file_num(0), |
262 | | max_manifest_file_size(std::numeric_limits<uint64_t>::max()), |
263 | | table_cache_numshardbits(4), |
264 | | WAL_ttl_seconds(0), |
265 | | WAL_size_limit_MB(0), |
266 | | manifest_preallocation_size(64 * 1024), |
267 | | allow_os_buffer(true), |
268 | | allow_mmap_reads(false), |
269 | | allow_mmap_writes(false), |
270 | | allow_fallocate(true), |
271 | | is_fd_close_on_exec(true), |
272 | | skip_log_error_on_recovery(false), |
273 | | stats_dump_period_sec(600), |
274 | | advise_random_on_open(true), |
275 | | db_write_buffer_size(0), |
276 | | access_hint_on_compaction_start(NORMAL), |
277 | | new_table_reader_for_compaction_inputs(false), |
278 | | compaction_readahead_size(0), |
279 | | random_access_max_buffer_size(1024 * 1024), |
280 | | writable_file_max_buffer_size(1024 * 1024), |
281 | | use_adaptive_mutex(false), |
282 | | bytes_per_sync(0), |
283 | | wal_bytes_per_sync(0), |
284 | | listeners(), |
285 | | enable_thread_tracking(false), |
286 | | delayed_write_rate(2 * 1024U * 1024U), |
287 | | allow_concurrent_memtable_write(false), |
288 | | enable_write_thread_adaptive_yield(false), |
289 | | write_thread_max_yield_usec(100), |
290 | | write_thread_slow_yield_usec(3), |
291 | | skip_stats_update_on_db_open(false), |
292 | | wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords), |
293 | | row_cache(nullptr), |
294 | | #ifndef ROCKSDB_LITE |
295 | | wal_filter(nullptr), |
296 | | #endif // ROCKSDB_LITE |
297 | 9.89M | fail_if_options_file_error(false) { |
298 | 9.89M | } |
299 | | |
300 | | static const char* const access_hints[] = { |
301 | | "NONE", "NORMAL", "SEQUENTIAL", "WILLNEED" |
302 | | }; |
303 | | |
304 | 0 | void DBOptions::Dump(Logger* log) const { |
305 | 0 | RHEADER(log, " Options.error_if_exists: %d", error_if_exists); |
306 | 0 | RHEADER(log, " Options.create_if_missing: %d", create_if_missing); |
307 | 0 | RHEADER(log, " Options.paranoid_checks: %d", paranoid_checks); |
308 | 0 | RHEADER(log, " Options.env: %p", env); |
309 | 0 | RHEADER(log, " Options.info_log: %p", info_log.get()); |
310 | 0 | RHEADER(log, " Options.max_open_files: %d", max_open_files); |
311 | 0 | RHEADER(log, |
312 | 0 | "Options.max_file_opening_threads: %d", max_file_opening_threads); |
313 | 0 | RHEADER(log, |
314 | 0 | " Options.max_total_wal_size: %" PRIu64, max_total_wal_size); |
315 | 0 | RHEADER(log, " Options.disableDataSync: %d", disableDataSync); |
316 | 0 | RHEADER(log, " Options.use_fsync: %d", use_fsync); |
317 | 0 | RHEADER(log, " Options.max_log_file_size: %" ROCKSDB_PRIszt, |
318 | 0 | max_log_file_size); |
319 | 0 | RHEADER(log, "Options.max_manifest_file_size: %" PRIu64, |
320 | 0 | max_manifest_file_size); |
321 | 0 | RHEADER(log, " Options.log_file_time_to_roll: %" ROCKSDB_PRIszt, |
322 | 0 | log_file_time_to_roll); |
323 | 0 | RHEADER(log, " Options.keep_log_file_num: %" ROCKSDB_PRIszt, |
324 | 0 | keep_log_file_num); |
325 | 0 | RHEADER(log, " Options.recycle_log_file_num: %" ROCKSDB_PRIszt, |
326 | 0 | recycle_log_file_num); |
327 | 0 | RHEADER(log, " Options.allow_os_buffer: %d", allow_os_buffer); |
328 | 0 | RHEADER(log, " Options.allow_mmap_reads: %d", allow_mmap_reads); |
329 | 0 | RHEADER(log, " Options.allow_fallocate: %d", allow_fallocate); |
330 | 0 | RHEADER(log, " Options.allow_mmap_writes: %d", allow_mmap_writes); |
331 | 0 | RHEADER(log, " Options.create_missing_column_families: %d", |
332 | 0 | create_missing_column_families); |
333 | 0 | RHEADER(log, " Options.db_log_dir: %s", |
334 | 0 | db_log_dir.c_str()); |
335 | 0 | RHEADER(log, " Options.wal_dir: %s", |
336 | 0 | wal_dir.c_str()); |
337 | 0 | RHEADER(log, " Options.table_cache_numshardbits: %d", |
338 | 0 | table_cache_numshardbits); |
339 | 0 | RHEADER(log, " Options.delete_obsolete_files_period_micros: %" PRIu64, |
340 | 0 | delete_obsolete_files_period_micros); |
341 | 0 | RHEADER(log, " Options.base_background_compactions: %d", |
342 | 0 | base_background_compactions); |
343 | 0 | RHEADER(log, " Options.max_background_compactions: %d", |
344 | 0 | max_background_compactions); |
345 | 0 | RHEADER(log, " Options.max_subcompactions: %" PRIu32, |
346 | 0 | max_subcompactions); |
347 | 0 | RHEADER(log, " Options.max_background_flushes: %d", |
348 | 0 | max_background_flushes); |
349 | 0 | RHEADER(log, " Options.WAL_ttl_seconds: %" PRIu64, |
350 | 0 | WAL_ttl_seconds); |
351 | 0 | RHEADER(log, " Options.WAL_size_limit_MB: %" PRIu64, |
352 | 0 | WAL_size_limit_MB); |
353 | 0 | RHEADER(log, |
354 | 0 | " Options.manifest_preallocation_size: %" ROCKSDB_PRIszt, |
355 | 0 | manifest_preallocation_size); |
356 | 0 | RHEADER(log, " Options.allow_os_buffer: %d", |
357 | 0 | allow_os_buffer); |
358 | 0 | RHEADER(log, " Options.allow_mmap_reads: %d", |
359 | 0 | allow_mmap_reads); |
360 | 0 | RHEADER(log, " Options.allow_mmap_writes: %d", |
361 | 0 | allow_mmap_writes); |
362 | 0 | RHEADER(log, " Options.is_fd_close_on_exec: %d", |
363 | 0 | is_fd_close_on_exec); |
364 | 0 | RHEADER(log, " Options.stats_dump_period_sec: %u", |
365 | 0 | stats_dump_period_sec); |
366 | 0 | RHEADER(log, " Options.advise_random_on_open: %d", |
367 | 0 | advise_random_on_open); |
368 | 0 | RHEADER(log, |
369 | 0 | " Options.db_write_buffer_size: %" ROCKSDB_PRIszt |
370 | 0 | "d", |
371 | 0 | db_write_buffer_size); |
372 | 0 | RHEADER(log, " Options.access_hint_on_compaction_start: %s", |
373 | 0 | access_hints[access_hint_on_compaction_start]); |
374 | 0 | RHEADER(log, " Options.new_table_reader_for_compaction_inputs: %d", |
375 | 0 | new_table_reader_for_compaction_inputs); |
376 | 0 | RHEADER(log, |
377 | 0 | " Options.compaction_readahead_size: %" ROCKSDB_PRIszt |
378 | 0 | "d", |
379 | 0 | compaction_readahead_size); |
380 | 0 | RHEADER( |
381 | 0 | log, |
382 | 0 | " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt |
383 | 0 | "d", |
384 | 0 | random_access_max_buffer_size); |
385 | 0 | RHEADER(log, |
386 | 0 | " Options.writable_file_max_buffer_size: %" ROCKSDB_PRIszt |
387 | 0 | "d", |
388 | 0 | writable_file_max_buffer_size); |
389 | 0 | RHEADER(log, " Options.use_adaptive_mutex: %d", |
390 | 0 | use_adaptive_mutex); |
391 | 0 | RHEADER(log, " Options.rate_limiter: %p", |
392 | 0 | rate_limiter.get()); |
393 | 0 | RHEADER( |
394 | 0 | log, " Options.sst_file_manager.rate_bytes_per_sec: %" PRIi64, |
395 | 0 | sst_file_manager ? sst_file_manager->GetDeleteRateBytesPerSecond() : 0); |
396 | 0 | RHEADER(log, " Options.bytes_per_sync: %" PRIu64, |
397 | 0 | bytes_per_sync); |
398 | 0 | RHEADER(log, " Options.wal_bytes_per_sync: %" PRIu64, |
399 | 0 | wal_bytes_per_sync); |
400 | 0 | RHEADER(log, " Options.wal_recovery_mode: %d", |
401 | 0 | wal_recovery_mode); |
402 | 0 | RHEADER(log, " Options.enable_thread_tracking: %d", |
403 | 0 | enable_thread_tracking); |
404 | 0 | RHEADER(log, " Options.allow_concurrent_memtable_write: %d", |
405 | 0 | allow_concurrent_memtable_write); |
406 | 0 | RHEADER(log, " Options.enable_write_thread_adaptive_yield: %d", |
407 | 0 | enable_write_thread_adaptive_yield); |
408 | 0 | RHEADER(log, " Options.write_thread_max_yield_usec: %" PRIu64, |
409 | 0 | write_thread_max_yield_usec); |
410 | 0 | RHEADER(log, " Options.write_thread_slow_yield_usec: %" PRIu64, |
411 | 0 | write_thread_slow_yield_usec); |
412 | 0 | if (row_cache) { |
413 | 0 | RHEADER(log, " Options.row_cache: %" PRIu64, |
414 | 0 | row_cache->GetCapacity()); |
415 | 0 | } else { |
416 | 0 | RHEADER(log, " Options.row_cache: None"); |
417 | 0 | } |
418 | 0 | RHEADER(log, " Options.initial_seqno: %" PRIu64, initial_seqno); |
419 | 0 | #ifndef ROCKSDB_LITE |
420 | 0 | RHEADER(log, " Options.wal_filter: %s", |
421 | 0 | wal_filter ? wal_filter->Name() : "None"); |
422 | 0 | #endif // ROCKDB_LITE |
423 | 0 | } // DBOptions::Dump |
424 | | |
425 | 17.7k | void ColumnFamilyOptions::Dump(Logger* log) const { |
426 | 17.7k | RHEADER(log, " Options.comparator: %s", comparator->Name()); |
427 | 17.7k | RHEADER(log, " Options.merge_operator: %s", |
428 | 17.7k | merge_operator ? merge_operator->Name() : "None"); |
429 | 17.7k | RHEADER(log, " Options.compaction_filter: %s", |
430 | 17.7k | compaction_filter ? compaction_filter->Name() : "None"); |
431 | 17.7k | RHEADER(log, " Options.compaction_filter_factory: %s", |
432 | 17.7k | compaction_filter_factory ? compaction_filter_factory->Name() : "None"); |
433 | 17.7k | RHEADER(log, " Options.memtable_factory: %s", memtable_factory->Name()); |
434 | 17.7k | RHEADER(log, " Options.table_factory: %s", table_factory->Name()); |
435 | 17.7k | RHEADER(log, " table_factory options: %s", |
436 | 17.7k | table_factory->GetPrintableTableOptions().c_str()); |
437 | 17.7k | RHEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt, |
438 | 17.7k | write_buffer_size); |
439 | 17.7k | RHEADER(log, " Options.max_write_buffer_number: %d", max_write_buffer_number); |
440 | 17.7k | if (!compression_per_level.empty()) { |
441 | 88 | for (unsigned int i = 0; i < compression_per_level.size(); i++66 ) { |
442 | 66 | RHEADER(log, " Options.compression[%d]: %s", i, |
443 | 66 | CompressionTypeToString(compression_per_level[i]).c_str()); |
444 | 66 | } |
445 | 17.6k | } else { |
446 | 17.6k | RHEADER(log, " Options.compression: %s", |
447 | 17.6k | CompressionTypeToString(compression).c_str()); |
448 | 17.6k | } |
449 | 17.7k | RHEADER(log, " Options.prefix_extractor: %s", |
450 | 17.7k | prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name()); |
451 | 17.7k | RHEADER(log, " Options.num_levels: %d", num_levels); |
452 | 17.7k | RHEADER(log, " Options.min_write_buffer_number_to_merge: %d", |
453 | 17.7k | min_write_buffer_number_to_merge); |
454 | 17.7k | RHEADER(log, " Options.max_write_buffer_number_to_maintain: %d", |
455 | 17.7k | max_write_buffer_number_to_maintain); |
456 | 17.7k | RHEADER(log, " Options.compression_opts.window_bits: %d", |
457 | 17.7k | compression_opts.window_bits); |
458 | 17.7k | RHEADER(log, " Options.compression_opts.level: %d", |
459 | 17.7k | compression_opts.level); |
460 | 17.7k | RHEADER(log, " Options.compression_opts.strategy: %d", |
461 | 17.7k | compression_opts.strategy); |
462 | 17.7k | RHEADER(log, " Options.level0_file_num_compaction_trigger: %d", |
463 | 17.7k | level0_file_num_compaction_trigger); |
464 | 17.7k | RHEADER(log, " Options.level0_slowdown_writes_trigger: %d", |
465 | 17.7k | level0_slowdown_writes_trigger); |
466 | 17.7k | RHEADER(log, " Options.level0_stop_writes_trigger: %d", |
467 | 17.7k | level0_stop_writes_trigger); |
468 | 17.7k | RHEADER(log, " Options.target_file_size_base: %" PRIu64, |
469 | 17.7k | target_file_size_base); |
470 | 17.7k | RHEADER(log, " Options.target_file_size_multiplier: %d", |
471 | 17.7k | target_file_size_multiplier); |
472 | 17.7k | RHEADER(log, " Options.max_bytes_for_level_base: %" PRIu64, |
473 | 17.7k | max_bytes_for_level_base); |
474 | 17.7k | RHEADER(log, "Options.level_compaction_dynamic_level_bytes: %d", |
475 | 17.7k | level_compaction_dynamic_level_bytes); |
476 | 17.7k | RHEADER(log, " Options.max_bytes_for_level_multiplier: %d", |
477 | 17.7k | max_bytes_for_level_multiplier); |
478 | 142k | for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size(); |
479 | 124k | i++) { |
480 | 124k | RHEADER(log, |
481 | 124k | "Options.max_bytes_for_level_multiplier_addtl[%" ROCKSDB_PRIszt |
482 | 124k | "]: %d", |
483 | 124k | i, max_bytes_for_level_multiplier_additional[i]); |
484 | 124k | } |
485 | 17.7k | RHEADER(log, " Options.max_sequential_skip_in_iterations: %" PRIu64, |
486 | 17.7k | max_sequential_skip_in_iterations); |
487 | 17.7k | RHEADER(log, " Options.expanded_compaction_factor: %d", |
488 | 17.7k | expanded_compaction_factor); |
489 | 17.7k | RHEADER(log, " Options.source_compaction_factor: %d", |
490 | 17.7k | source_compaction_factor); |
491 | 17.7k | RHEADER(log, " Options.max_grandparent_overlap_factor: %d", |
492 | 17.7k | max_grandparent_overlap_factor); |
493 | | |
494 | 17.7k | RHEADER(log, |
495 | 17.7k | " Options.arena_block_size: %" ROCKSDB_PRIszt, |
496 | 17.7k | arena_block_size); |
497 | 17.7k | RHEADER(log, " Options.soft_pending_compaction_bytes_limit: %" PRIu64, |
498 | 17.7k | soft_pending_compaction_bytes_limit); |
499 | 17.7k | RHEADER(log, " Options.hard_pending_compaction_bytes_limit: %" PRIu64, |
500 | 17.7k | hard_pending_compaction_bytes_limit); |
501 | 17.7k | RHEADER(log, " Options.rate_limit_delay_max_milliseconds: %u", |
502 | 17.7k | rate_limit_delay_max_milliseconds); |
503 | 17.7k | RHEADER(log, " Options.disable_auto_compactions: %d", |
504 | 17.7k | disable_auto_compactions); |
505 | 17.7k | RHEADER(log, " Options.filter_deletes: %d", |
506 | 17.7k | filter_deletes); |
507 | 17.7k | RHEADER(log, " Options.verify_checksums_in_compaction: %d", |
508 | 17.7k | verify_checksums_in_compaction); |
509 | 17.7k | RHEADER(log, " Options.compaction_style: %d", |
510 | 17.7k | compaction_style); |
511 | 17.7k | RHEADER(log, " Options.compaction_pri: %d", |
512 | 17.7k | compaction_pri); |
513 | 17.7k | RHEADER(log, " Options.compaction_options_universal.size_ratio: %u", |
514 | 17.7k | compaction_options_universal.size_ratio); |
515 | 17.7k | RHEADER(log, "Options.compaction_options_universal." |
516 | 17.7k | "always_include_size_threshold: %" ROCKSDB_PRIszt, |
517 | 17.7k | compaction_options_universal.always_include_size_threshold); |
518 | 17.7k | RHEADER(log, "Options.compaction_options_universal.min_merge_width: %u", |
519 | 17.7k | compaction_options_universal.min_merge_width); |
520 | 17.7k | RHEADER(log, "Options.compaction_options_universal.max_merge_width: %u", |
521 | 17.7k | compaction_options_universal.max_merge_width); |
522 | 17.7k | RHEADER(log, "Options.compaction_options_universal." |
523 | 17.7k | "max_size_amplification_percent: %u", |
524 | 17.7k | compaction_options_universal.max_size_amplification_percent); |
525 | 17.7k | RHEADER(log, |
526 | 17.7k | "Options.compaction_options_universal.compression_size_percent: %d", |
527 | 17.7k | compaction_options_universal.compression_size_percent); |
528 | 17.7k | RHEADER(log, |
529 | 17.7k | "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, |
530 | 17.7k | compaction_options_fifo.max_table_files_size); |
531 | 17.7k | std::string collector_names; |
532 | 17.7k | for (const auto& collector_factory : table_properties_collector_factories) { |
533 | 95 | collector_names.append(collector_factory->Name()); |
534 | 95 | collector_names.append("; "); |
535 | 95 | } |
536 | 17.7k | RHEADER(log, " Options.table_properties_collectors: %s", |
537 | 17.7k | collector_names.c_str()); |
538 | 17.7k | RHEADER(log, " Options.inplace_update_support: %d", |
539 | 17.7k | inplace_update_support); |
540 | 17.7k | RHEADER(log, |
541 | 17.7k | " Options.inplace_update_num_locks: %" ROCKSDB_PRIszt, |
542 | 17.7k | inplace_update_num_locks); |
543 | 17.7k | RHEADER(log, " Options.min_partial_merge_operands: %u", |
544 | 17.7k | min_partial_merge_operands); |
545 | | // TODO: easier config for bloom (maybe based on avg key/value size) |
546 | 17.7k | RHEADER(log, " Options.memtable_prefix_bloom_bits: %d", |
547 | 17.7k | memtable_prefix_bloom_bits); |
548 | 17.7k | RHEADER(log, " Options.memtable_prefix_bloom_probes: %d", |
549 | 17.7k | memtable_prefix_bloom_probes); |
550 | | |
551 | 17.7k | RHEADER(log, |
552 | 17.7k | " Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt, |
553 | 17.7k | memtable_prefix_bloom_huge_page_tlb_size); |
554 | 17.7k | RHEADER(log, " Options.bloom_locality: %d", |
555 | 17.7k | bloom_locality); |
556 | | |
557 | 17.7k | RHEADER(log, |
558 | 17.7k | " Options.max_successive_merges: %" ROCKSDB_PRIszt, |
559 | 17.7k | max_successive_merges); |
560 | 17.7k | RHEADER(log, " Options.optimize_filters_for_hits: %d", |
561 | 17.7k | optimize_filters_for_hits); |
562 | 17.7k | RHEADER(log, " Options.paranoid_file_checks: %d", |
563 | 17.7k | paranoid_file_checks); |
564 | 17.7k | RHEADER(log, " Options.compaction_measure_io_stats: %d", |
565 | 17.7k | compaction_measure_io_stats); |
566 | 17.7k | } // ColumnFamilyOptions::Dump |
567 | | |
568 | 0 | void Options::Dump(Logger* log) const { |
569 | 0 | DBOptions::Dump(log); |
570 | 0 | ColumnFamilyOptions::Dump(log); |
571 | 0 | } // Options::Dump |
572 | | |
573 | 17.7k | void Options::DumpCFOptions(Logger* log) const { |
574 | 17.7k | ColumnFamilyOptions::Dump(log); |
575 | 17.7k | } // Options::DumpCFOptions |
576 | | |
577 | | // |
578 | | // The goal of this method is to create a configuration that |
579 | | // allows an application to write all files into L0 and |
580 | | // then do a single compaction to output all files into L1. |
581 | | Options* |
582 | 0 | Options::PrepareForBulkLoad() { |
583 | | // never slowdown ingest. |
584 | 0 | level0_file_num_compaction_trigger = (1<<30); |
585 | 0 | level0_slowdown_writes_trigger = (1<<30); |
586 | 0 | level0_stop_writes_trigger = (1<<30); |
587 | | |
588 | | // no auto compactions please. The application should issue a |
589 | | // manual compaction after all data is loaded into L0. |
590 | 0 | disable_auto_compactions = true; |
591 | 0 | disableDataSync = true; |
592 | | |
593 | | // A manual compaction run should pick all files in L0 in |
594 | | // a single compaction run. |
595 | 0 | source_compaction_factor = (1<<30); |
596 | | |
597 | | // It is better to have only 2 levels, otherwise a manual |
598 | | // compaction would compact at every possible level, thereby |
599 | | // increasing the total time needed for compactions. |
600 | 0 | num_levels = 2; |
601 | | |
602 | | // Need to allow more write buffers to allow more parallism |
603 | | // of flushes. |
604 | 0 | max_write_buffer_number = 6; |
605 | 0 | min_write_buffer_number_to_merge = 1; |
606 | | |
607 | | // When compaction is disabled, more parallel flush threads can |
608 | | // help with write throughput. |
609 | 0 | max_background_flushes = 4; |
610 | | |
611 | | // Prevent a memtable flush to automatically promote files |
612 | | // to L1. This is helpful so that all files that are |
613 | | // input to the manual compaction are all at L0. |
614 | 0 | max_background_compactions = 2; |
615 | 0 | base_background_compactions = 2; |
616 | | |
617 | | // The compaction would create large files in L1. |
618 | 0 | target_file_size_base = 256 * 1024 * 1024; |
619 | 0 | return this; |
620 | 0 | } |
621 | | |
622 | | #ifndef ROCKSDB_LITE |
623 | | // Optimization functions |
624 | | ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup( |
625 | 0 | uint64_t block_cache_size_mb) { |
626 | 0 | prefix_extractor.reset(NewNoopTransform()); |
627 | 0 | BlockBasedTableOptions block_based_options; |
628 | 0 | block_based_options.index_type = IndexType::kHashSearch; |
629 | 0 | block_based_options.filter_policy.reset(NewBloomFilterPolicy(10)); |
630 | 0 | block_based_options.block_cache = |
631 | 0 | NewLRUCache(static_cast<size_t>(block_cache_size_mb * 1024 * 1024)); |
632 | 0 | table_factory.reset(new BlockBasedTableFactory(block_based_options)); |
633 | 0 | memtable_factory.reset(NewHashLinkListRepFactory()); |
634 | 0 | return this; |
635 | 0 | } |
636 | | |
637 | | ColumnFamilyOptions* ColumnFamilyOptions::OptimizeLevelStyleCompaction( |
638 | 1 | uint64_t memtable_memory_budget) { |
639 | 1 | write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4); |
640 | | // merge two memtables when flushing to L0 |
641 | 1 | min_write_buffer_number_to_merge = 2; |
642 | | // this means we'll use 50% extra memory in the worst case, but will reduce |
643 | | // write stalls. |
644 | 1 | max_write_buffer_number = 6; |
645 | | // start flushing L0->L1 as soon as possible. each file on level0 is |
646 | | // (memtable_memory_budget / 2). This will flush level 0 when it's bigger than |
647 | | // memtable_memory_budget. |
648 | 1 | level0_file_num_compaction_trigger = 2; |
649 | | // doesn't really matter much, but we don't want to create too many files |
650 | 1 | target_file_size_base = memtable_memory_budget / 8; |
651 | | // make Level1 size equal to Level0 size, so that L0->L1 compactions are fast |
652 | 1 | max_bytes_for_level_base = memtable_memory_budget; |
653 | | |
654 | | // level style compaction |
655 | 1 | compaction_style = kCompactionStyleLevel; |
656 | | |
657 | | // only compress levels >= 2 |
658 | 1 | compression_per_level.resize(num_levels); |
659 | 8 | for (int i = 0; i < num_levels; ++i7 ) { |
660 | 7 | if (i < 2) { |
661 | 2 | compression_per_level[i] = kNoCompression; |
662 | 5 | } else { |
663 | 5 | compression_per_level[i] = kSnappyCompression; |
664 | 5 | } |
665 | 7 | } |
666 | 1 | return this; |
667 | 1 | } |
668 | | |
669 | | ColumnFamilyOptions* ColumnFamilyOptions::OptimizeUniversalStyleCompaction( |
670 | 1 | uint64_t memtable_memory_budget) { |
671 | 1 | write_buffer_size = static_cast<size_t>(memtable_memory_budget / 4); |
672 | | // merge two memtables when flushing to L0 |
673 | 1 | min_write_buffer_number_to_merge = 2; |
674 | | // this means we'll use 50% extra memory in the worst case, but will reduce |
675 | | // write stalls. |
676 | 1 | max_write_buffer_number = 6; |
677 | | // universal style compaction |
678 | 1 | compaction_style = kCompactionStyleUniversal; |
679 | 1 | compaction_options_universal.compression_size_percent = 80; |
680 | 1 | return this; |
681 | 1 | } |
682 | | |
683 | 6 | DBOptions* DBOptions::IncreaseParallelism(int total_threads) { |
684 | 6 | max_background_compactions = total_threads - 1; |
685 | 6 | max_background_flushes = 1; |
686 | 6 | env->SetBackgroundThreads(total_threads, Env::LOW); |
687 | 6 | env->SetBackgroundThreads(1, Env::HIGH); |
688 | 6 | return this; |
689 | 6 | } |
690 | | |
691 | | #endif // !ROCKSDB_LITE |
692 | | |
693 | | const ReadOptions ReadOptions::kDefault; |
694 | | |
695 | | ReadOptions::ReadOptions() |
696 | | : verify_checksums(true), |
697 | | fill_cache(true), |
698 | | snapshot(nullptr), |
699 | | iterate_upper_bound(nullptr), |
700 | | read_tier(kReadAllTier), |
701 | | tailing(false), |
702 | | managed(false), |
703 | | total_order_seek(false), |
704 | | prefix_same_as_start(false), |
705 | | pin_data(false), |
706 | 46.5M | query_id(rocksdb::kDefaultQueryId) { |
707 | 46.5M | XFUNC_TEST("", "managed_options", managed_options, xf_manage_options, |
708 | 46.5M | reinterpret_cast<ReadOptions*>(this)); |
709 | 46.5M | } |
710 | | |
711 | | ReadOptions::ReadOptions(bool cksum, bool cache) |
712 | | : verify_checksums(cksum), |
713 | | fill_cache(cache), |
714 | | snapshot(nullptr), |
715 | | iterate_upper_bound(nullptr), |
716 | | read_tier(kReadAllTier), |
717 | | tailing(false), |
718 | | managed(false), |
719 | | total_order_seek(false), |
720 | | prefix_same_as_start(false), |
721 | | pin_data(false), |
722 | 9 | query_id(rocksdb::kDefaultQueryId) { |
723 | 9 | XFUNC_TEST("", "managed_options", managed_options, xf_manage_options, |
724 | 9 | reinterpret_cast<ReadOptions*>(this)); |
725 | 9 | } |
726 | | |
727 | | std::atomic<int64_t> flush_tick_(1); |
728 | | |
729 | 16.1k | int64_t FlushTick() { |
730 | 16.1k | return flush_tick_.fetch_add(1, std::memory_order_acq_rel); |
731 | 16.1k | } |
732 | | |
733 | | } // namespace rocksdb |