/Users/deen/code/yugabyte-db/src/yb/rocksdb/db/db_test_util.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | | // This source code is licensed under the BSD-style license found in the |
3 | | // LICENSE file in the root directory of this source tree. An additional grant |
4 | | // of patent rights can be found in the PATENTS file in the same directory. |
5 | | // |
6 | | // The following only applies to changes made to this file as part of YugaByte development. |
7 | | // |
8 | | // Portions Copyright (c) YugaByte, Inc. |
9 | | // |
10 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
11 | | // in compliance with the License. You may obtain a copy of the License at |
12 | | // |
13 | | // http://www.apache.org/licenses/LICENSE-2.0 |
14 | | // |
15 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
16 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
17 | | // or implied. See the License for the specific language governing permissions and limitations |
18 | | // under the License. |
19 | | // |
20 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
21 | | // Use of this source code is governed by a BSD-style license that can be |
22 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
23 | | |
24 | | #include "yb/rocksdb/db/db_test_util.h" |
25 | | |
26 | | #include "yb/encryption/header_manager_impl.h" |
27 | | #include "yb/encryption/universe_key_manager.h" |
28 | | |
29 | | #include "yb/rocksdb/util/logging.h" |
30 | | |
31 | | #include "yb/rocksutil/rocksdb_encrypted_file_factory.h" |
32 | | #include "yb/rocksutil/yb_rocksdb_logger.h" |
33 | | |
34 | | #include "yb/util/random_util.h" |
35 | | #include "yb/util/status_log.h" |
36 | | |
37 | | namespace rocksdb { |
38 | | |
39 | | // Special Env used to delay background operations |
40 | | |
41 | | SpecialEnv::SpecialEnv(Env* base) |
42 | | : EnvWrapper(base), |
43 | | rnd_(301), |
44 | | sleep_counter_(this), |
45 | | addon_time_(0), |
46 | | time_elapse_only_sleep_(false), |
47 | 552 | no_sleep_(false) { |
48 | 552 | delay_sstable_sync_.store(false, std::memory_order_release); |
49 | 552 | drop_writes_.store(false, std::memory_order_release); |
50 | 552 | no_space_.store(false, std::memory_order_release); |
51 | 552 | non_writable_.store(false, std::memory_order_release); |
52 | 552 | count_random_reads_ = false; |
53 | 552 | count_sequential_reads_ = false; |
54 | 552 | manifest_sync_error_.store(false, std::memory_order_release); |
55 | 552 | manifest_write_error_.store(false, std::memory_order_release); |
56 | 552 | log_write_error_.store(false, std::memory_order_release); |
57 | 552 | random_file_open_counter_.store(0, std::memory_order_relaxed); |
58 | 552 | log_write_slowdown_ = 0; |
59 | 552 | bytes_written_ = 0; |
60 | 552 | sync_counter_ = 0; |
61 | 552 | non_writeable_rate_ = 0; |
62 | 552 | new_writable_count_ = 0; |
63 | 552 | non_writable_count_ = 0; |
64 | 552 | table_write_callback_ = nullptr; |
65 | 552 | } |
66 | | |
67 | | const string DBHolder::kKeyId = "key_id"; |
68 | | const string DBHolder::kKeyFile = "universe_key_file"; |
69 | | |
70 | | DBHolder::DBHolder(const std::string path, bool encryption_enabled) |
71 | | : option_config_(kDefault), |
72 | | mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())), |
73 | 548 | env_(new SpecialEnv(mem_env_ ? mem_env_ : Env::Default())) { |
74 | 548 | if (encryption_enabled) { |
75 | 4 | CreateEncryptedEnv(); |
76 | 4 | } |
77 | 548 | env_->SetBackgroundThreads(1, Env::LOW); |
78 | 548 | env_->SetBackgroundThreads(1, Env::HIGH); |
79 | 548 | dbname_ = test::TmpDir(env_) + path; |
80 | 548 | alternative_wal_dir_ = dbname_ + "/wal"; |
81 | 548 | alternative_db_log_dir_ = dbname_ + "/db_log_dir"; |
82 | 548 | auto options = CurrentOptions(); |
83 | 548 | auto delete_options = options; |
84 | 548 | delete_options.wal_dir = alternative_wal_dir_; |
85 | 548 | WARN_NOT_OK(DestroyDB(dbname_, delete_options), "Cleanup failed " + dbname_); |
86 | | // Destroy it for not alternative WAL dir is used. |
87 | 548 | WARN_NOT_OK(DestroyDB(dbname_, options), "Cleanup failed " + dbname_); |
88 | 548 | db_ = nullptr; |
89 | 548 | Reopen(options); |
90 | 548 | Random::GetTLSInstance()->Reset(0xdeadbeef); |
91 | 548 | } |
92 | | |
93 | 542 | DBHolder::~DBHolder() { |
94 | 542 | Env::Default()->CleanupFile(kKeyFile); |
95 | 542 | rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
96 | 542 | rocksdb::SyncPoint::GetInstance()->LoadDependency({}); |
97 | 542 | rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); |
98 | 542 | Close(); |
99 | 542 | Options options; |
100 | 542 | options.db_paths.emplace_back(dbname_, 0); |
101 | 542 | options.db_paths.emplace_back(dbname_ + "_2", 0); |
102 | 542 | options.db_paths.emplace_back(dbname_ + "_3", 0); |
103 | 542 | options.db_paths.emplace_back(dbname_ + "_4", 0); |
104 | 542 | EXPECT_OK(DestroyDB(dbname_, options)); |
105 | 542 | delete env_; |
106 | 542 | } |
107 | | |
108 | 4 | void DBHolder::CreateEncryptedEnv() { |
109 | 4 | auto bytes = yb::RandomBytes(32); |
110 | 4 | yb::Slice key(bytes.data(), bytes.size()); |
111 | 4 | auto status = yb::WriteStringToFile(yb::Env::Default(), key, kKeyFile); |
112 | 4 | if (!status.ok()) { |
113 | 0 | LOG(FATAL) << "Could not write slice to file:" << status.ToString(); |
114 | 0 | } |
115 | | |
116 | 4 | auto res = yb::encryption::UniverseKeyManager::FromKey(kKeyId, key); |
117 | 4 | if (!res.ok()) { |
118 | 0 | LOG(FATAL) << "Could not get key from bytes:" << res.status().ToString(); |
119 | 0 | } |
120 | 4 | universe_key_manager_ = std::move(*res); |
121 | 4 | encrypted_env_ = yb::NewRocksDBEncryptedEnv( |
122 | 4 | yb::encryption::DefaultHeaderManager(universe_key_manager_.get())); |
123 | 4 | delete env_; |
124 | 4 | env_ = new rocksdb::SpecialEnv(encrypted_env_.get()); |
125 | 4 | } |
126 | | |
127 | 7.67k | bool DBHolder::ShouldSkipOptions(int option_config, int skip_mask) { |
128 | | #ifdef ROCKSDB_LITE |
129 | | // These options are not supported in ROCKSDB_LITE |
130 | | if (option_config == kHashSkipList || |
131 | | option_config == kPlainTableFirstBytePrefix || |
132 | | option_config == kPlainTableCappedPrefix || |
133 | | option_config == kPlainTableCappedPrefixNonMmap || |
134 | | option_config == kPlainTableAllBytesPrefix || |
135 | | option_config == kVectorRep || option_config == kHashLinkList || |
136 | | option_config == kUniversalCompaction || |
137 | | option_config == kUniversalCompactionMultiLevel || |
138 | | option_config == kUniversalSubcompactions || |
139 | | option_config == kFIFOCompaction || |
140 | | option_config == kConcurrentSkipList) { |
141 | | return true; |
142 | | } |
143 | | #endif |
144 | | |
145 | 7.67k | if ((skip_mask & kSkipDeletesFilterFirst) && |
146 | 7.67k | option_config == kDeletesFilterFirst6.60k ) { |
147 | 220 | return true; |
148 | 220 | } |
149 | 7.45k | if ((skip_mask & kSkipUniversalCompaction) && |
150 | 7.45k | (348 option_config == kUniversalCompaction348 || |
151 | 348 | option_config == kUniversalCompactionMultiLevel336 )) { |
152 | 24 | return true; |
153 | 24 | } |
154 | 7.42k | if ((skip_mask & kSkipMergePut) && option_config == kMergePut164 ) { |
155 | 6 | return true; |
156 | 6 | } |
157 | 7.42k | if ((skip_mask & kSkipNoSeekToLast) && |
158 | 7.42k | (6.49k option_config == kHashLinkList6.49k || option_config == kHashSkipList6.27k )) { |
159 | 448 | return true; |
160 | 448 | } |
161 | 6.97k | if ((skip_mask & kSkipPlainTable) && |
162 | 6.97k | (247 option_config == kPlainTableAllBytesPrefix247 || |
163 | 247 | option_config == kPlainTableFirstBytePrefix238 || |
164 | 247 | option_config == kPlainTableCappedPrefix229 || |
165 | 247 | option_config == kPlainTableCappedPrefixNonMmap220 )) { |
166 | 36 | return true; |
167 | 36 | } |
168 | 6.93k | if ((skip_mask & kSkipHashIndex) && |
169 | 6.93k | (48 option_config == kBlockBasedTableWithPrefixHashIndex48 || |
170 | 48 | option_config == kBlockBasedTableWithWholeKeyHashIndex46 )) { |
171 | 4 | return true; |
172 | 4 | } |
173 | 6.93k | if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction349 ) { |
174 | 14 | return true; |
175 | 14 | } |
176 | 6.92k | if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads46 ) { |
177 | 2 | return true; |
178 | 2 | } |
179 | 6.91k | return false; |
180 | 6.92k | } |
181 | | |
182 | | // Switch to a fresh database with the next option configuration to |
183 | | // test. Return false if there are no more configurations to test. |
184 | 1.01k | bool DBHolder::ChangeOptions(int skip_mask) { |
185 | 1.11k | for (option_config_++; option_config_ < kEnd; option_config_++94 ) { |
186 | 1.07k | if (ShouldSkipOptions(option_config_, skip_mask)) { |
187 | 94 | continue; |
188 | 94 | } |
189 | 979 | break; |
190 | 1.07k | } |
191 | | |
192 | 1.01k | if (option_config_ >= kEnd) { |
193 | 37 | Destroy(last_options_); |
194 | 37 | return false; |
195 | 979 | } else { |
196 | 979 | auto options = CurrentOptions(); |
197 | 979 | options.create_if_missing = true; |
198 | 979 | DestroyAndReopen(options); |
199 | 979 | return true; |
200 | 979 | } |
201 | 1.01k | } |
202 | | |
203 | | // Switch between different compaction styles. |
204 | 232 | bool DBHolder::ChangeCompactOptions() { |
205 | 232 | if (option_config_ == kDefault) { |
206 | 51 | option_config_ = kUniversalCompaction; |
207 | 51 | Destroy(last_options_); |
208 | 51 | auto options = CurrentOptions(); |
209 | 51 | options.create_if_missing = true; |
210 | 51 | CHECK_OK(TryReopen(options)); |
211 | 51 | return true; |
212 | 181 | } else if (option_config_ == kUniversalCompaction) { |
213 | 45 | option_config_ = kUniversalCompactionMultiLevel; |
214 | 45 | Destroy(last_options_); |
215 | 45 | auto options = CurrentOptions(); |
216 | 45 | options.create_if_missing = true; |
217 | 45 | CHECK_OK(TryReopen(options)); |
218 | 45 | return true; |
219 | 136 | } else if (option_config_ == kUniversalCompactionMultiLevel) { |
220 | 45 | option_config_ = kLevelSubcompactions; |
221 | 45 | Destroy(last_options_); |
222 | 45 | auto options = CurrentOptions(); |
223 | 45 | assert(options.max_subcompactions > 1); |
224 | 45 | CHECK_OK(TryReopen(options)); |
225 | 45 | return true; |
226 | 91 | } else if (option_config_ == kLevelSubcompactions) { |
227 | 45 | option_config_ = kUniversalSubcompactions; |
228 | 45 | Destroy(last_options_); |
229 | 45 | auto options = CurrentOptions(); |
230 | 45 | assert(options.max_subcompactions > 1); |
231 | 45 | CHECK_OK(TryReopen(options)); |
232 | 45 | return true; |
233 | 46 | } else { |
234 | 46 | return false; |
235 | 46 | } |
236 | 232 | } |
237 | | |
238 | | // Switch between different filter policy |
239 | | // Jump from kDefault to kFilter to kFullFilter |
240 | 6 | bool DBHolder::ChangeFilterOptions() { |
241 | 6 | if (option_config_ == kDefault) { |
242 | 2 | option_config_ = kFilter; |
243 | 4 | } else if (option_config_ == kFilter) { |
244 | 2 | option_config_ = kFullFilterWithNewTableReaderForCompactions; |
245 | 2 | } else { |
246 | 2 | return false; |
247 | 2 | } |
248 | 4 | Destroy(last_options_); |
249 | | |
250 | 4 | auto options = CurrentOptions(); |
251 | 4 | options.create_if_missing = true; |
252 | 4 | CHECK_OK(TryReopen(options)); |
253 | 4 | return true; |
254 | 6 | } |
255 | | |
256 | | // Return the current option configuration. |
257 | | Options DBHolder::CurrentOptions( |
258 | 5.05k | const anon::OptionsOverride& options_override) { |
259 | 5.05k | Options options; |
260 | 5.05k | options.write_buffer_size = 4090 * 4096; |
261 | 5.05k | return CurrentOptions(options, options_override); |
262 | 5.05k | } |
263 | | |
264 | | Options DBHolder::CurrentOptions( |
265 | | const Options& defaultOptions, |
266 | 5.50k | const anon::OptionsOverride& options_override) { |
267 | | // this redundant copy is to minimize code change w/o having lint error. |
268 | 5.50k | Options options = defaultOptions; |
269 | 5.50k | XFUNC_TEST("", "dbtest_options", inplace_options1, GetXFTestOptions, |
270 | 5.50k | reinterpret_cast<Options*>(&options), |
271 | 5.50k | options_override.skip_policy); |
272 | 5.50k | BlockBasedTableOptions table_options; |
273 | 5.50k | bool set_block_based_table_factory = true; |
274 | 5.50k | switch (option_config_) { |
275 | 0 | #ifndef ROCKSDB_LITE |
276 | 96 | case kHashSkipList: |
277 | 96 | options.prefix_extractor.reset(NewFixedPrefixTransform(1)); |
278 | 96 | options.memtable_factory.reset(NewHashSkipListRepFactory(16)); |
279 | 96 | break; |
280 | 87 | case kPlainTableFirstBytePrefix: |
281 | 87 | options.table_factory.reset(new PlainTableFactory()); |
282 | 87 | options.prefix_extractor.reset(NewFixedPrefixTransform(1)); |
283 | 87 | options.allow_mmap_reads = true; |
284 | 87 | options.max_sequential_skip_in_iterations = 999999; |
285 | 87 | set_block_based_table_factory = false; |
286 | 87 | break; |
287 | 87 | case kPlainTableCappedPrefix: |
288 | 87 | options.table_factory.reset(new PlainTableFactory()); |
289 | 87 | options.prefix_extractor.reset(NewCappedPrefixTransform(8)); |
290 | 87 | options.allow_mmap_reads = true; |
291 | 87 | options.max_sequential_skip_in_iterations = 999999; |
292 | 87 | set_block_based_table_factory = false; |
293 | 87 | break; |
294 | 87 | case kPlainTableCappedPrefixNonMmap: |
295 | 87 | options.table_factory.reset(new PlainTableFactory()); |
296 | 87 | options.prefix_extractor.reset(NewCappedPrefixTransform(8)); |
297 | 87 | options.allow_mmap_reads = false; |
298 | 87 | options.max_sequential_skip_in_iterations = 999999; |
299 | 87 | set_block_based_table_factory = false; |
300 | 87 | break; |
301 | 87 | case kPlainTableAllBytesPrefix: |
302 | 87 | options.table_factory.reset(new PlainTableFactory()); |
303 | 87 | options.prefix_extractor.reset(NewNoopTransform()); |
304 | 87 | options.allow_mmap_reads = true; |
305 | 87 | options.max_sequential_skip_in_iterations = 999999; |
306 | 87 | set_block_based_table_factory = false; |
307 | 87 | break; |
308 | 105 | case kVectorRep: |
309 | 105 | options.memtable_factory.reset(new VectorRepFactory(100)); |
310 | 105 | break; |
311 | 96 | case kHashLinkList: |
312 | 96 | options.prefix_extractor.reset(NewFixedPrefixTransform(1)); |
313 | 96 | options.memtable_factory.reset( |
314 | 96 | NewHashLinkListRepFactory(4, 0, 3, true, 4)); |
315 | 96 | break; |
316 | 0 | #endif // ROCKSDB_LITE |
317 | 93 | case kMergePut: |
318 | 93 | options.merge_operator = MergeOperators::CreatePutOperator(); |
319 | 93 | break; |
320 | 109 | case kFilter: |
321 | 109 | table_options.filter_policy.reset(NewBloomFilterPolicy(10, true)); |
322 | 109 | break; |
323 | 109 | case kFullFilterWithNewTableReaderForCompactions: |
324 | 109 | table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); |
325 | 109 | options.new_table_reader_for_compaction_inputs = true; |
326 | 109 | options.compaction_readahead_size = 10 * 1024 * 1024; |
327 | 109 | break; |
328 | 105 | case kUncompressed: |
329 | 105 | options.compression = kNoCompression; |
330 | 105 | break; |
331 | 105 | case kNumLevel_3: |
332 | 105 | options.num_levels = 3; |
333 | 105 | break; |
334 | 105 | case kDBLogDir: |
335 | 105 | options.db_log_dir = alternative_db_log_dir_; |
336 | 105 | break; |
337 | 101 | case kWalDirAndMmapReads: |
338 | 101 | options.wal_dir = alternative_wal_dir_; |
339 | | // mmap reads should be orthogonal to WalDir setting, so we piggyback to |
340 | | // this option config to test mmap reads as well |
341 | 101 | options.allow_mmap_reads = true; |
342 | 101 | break; |
343 | 105 | case kManifestFileSize: |
344 | 105 | options.max_manifest_file_size = 50; // 50 bytes |
345 | | // YugaByte change: the break statement was missing here in RocksDB before we made implicit |
346 | | // fallthrough an error. |
347 | 105 | break; |
348 | 105 | case kPerfOptions: |
349 | 105 | options.soft_rate_limit = 2.0; |
350 | 105 | options.delayed_write_rate = 8 * 1024 * 1024; |
351 | | // TODO(3.13) -- test more options |
352 | 105 | break; |
353 | 104 | case kDeletesFilterFirst: |
354 | 104 | options.filter_deletes = true; |
355 | 104 | break; |
356 | 333 | case kUniversalCompaction: |
357 | 333 | options.compaction_style = kCompactionStyleUniversal; |
358 | 333 | options.num_levels = 1; |
359 | 333 | break; |
360 | 356 | case kUniversalCompactionMultiLevel: |
361 | 356 | options.compaction_style = kCompactionStyleUniversal; |
362 | 356 | options.num_levels = 8; |
363 | 356 | break; |
364 | 105 | case kCompressedBlockCache: |
365 | 105 | options.allow_mmap_writes = true; |
366 | 105 | table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024); |
367 | 105 | break; |
368 | 105 | case kInfiniteMaxOpenFiles: |
369 | 105 | options.max_open_files = -1; |
370 | 105 | break; |
371 | 105 | case kxxHashChecksum: { |
372 | 105 | table_options.checksum = kxxHash; |
373 | 105 | break; |
374 | 0 | } |
375 | 77 | case kFIFOCompaction: { |
376 | 77 | options.compaction_style = kCompactionStyleFIFO; |
377 | 77 | break; |
378 | 0 | } |
379 | 101 | case kBlockBasedTableWithPrefixHashIndex: { |
380 | 101 | table_options.index_type = IndexType::kHashSearch; |
381 | 101 | options.prefix_extractor.reset(NewFixedPrefixTransform(1)); |
382 | 101 | break; |
383 | 0 | } |
384 | 101 | case kBlockBasedTableWithWholeKeyHashIndex: { |
385 | 101 | table_options.index_type = IndexType::kHashSearch; |
386 | 101 | options.prefix_extractor.reset(NewNoopTransform()); |
387 | 101 | break; |
388 | 0 | } |
389 | 1 | case kBlockBasedTableWithIndexRestartInterval: { |
390 | 1 | table_options.index_block_restart_interval = 8; |
391 | 1 | break; |
392 | 0 | } |
393 | 105 | case kOptimizeFiltersForHits: { |
394 | 105 | options.optimize_filters_for_hits = true; |
395 | 105 | set_block_based_table_factory = true; |
396 | 105 | break; |
397 | 0 | } |
398 | 105 | case kRowCache: { |
399 | 105 | options.row_cache = NewLRUCache(1024 * 1024); |
400 | 105 | break; |
401 | 0 | } |
402 | 105 | case kRecycleLogFiles: { |
403 | 105 | options.recycle_log_file_num = 2; |
404 | 105 | break; |
405 | 0 | } |
406 | 270 | case kLevelSubcompactions: { |
407 | 270 | options.max_subcompactions = 4; |
408 | 270 | break; |
409 | 0 | } |
410 | 276 | case kUniversalSubcompactions: { |
411 | 276 | options.compaction_style = kCompactionStyleUniversal; |
412 | 276 | options.num_levels = 8; |
413 | 276 | options.max_subcompactions = 4; |
414 | 276 | break; |
415 | 0 | } |
416 | 105 | case kConcurrentSkipList: { |
417 | 105 | options.allow_concurrent_memtable_write = true; |
418 | 105 | options.enable_write_thread_adaptive_yield = true; |
419 | 105 | break; |
420 | 0 | } |
421 | 0 | case kBlockBasedTableWithThreeSharedPartsKeyDeltaEncoding: { |
422 | 0 | table_options.use_delta_encoding = true; |
423 | 0 | table_options.data_block_key_value_encoding_format = |
424 | 0 | KeyValueEncodingFormat::kKeyDeltaEncodingThreeSharedParts; |
425 | 0 | break; |
426 | 0 | } |
427 | | |
428 | 1.57k | default: |
429 | 1.57k | break; |
430 | 5.50k | } |
431 | | |
432 | 5.50k | if (options_override.filter_policy) { |
433 | 28 | table_options.filter_policy = options_override.filter_policy; |
434 | 28 | } |
435 | 5.50k | if (set_block_based_table_factory) { |
436 | 5.15k | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); |
437 | 5.15k | } |
438 | 5.50k | options.env = env_; |
439 | 5.50k | options.checkpoint_env = env_->IsPlainText() ? env_5.50k : Env::Default()4 ; |
440 | 5.50k | options.create_if_missing = true; |
441 | 5.50k | options.fail_if_options_file_error = true; |
442 | 5.50k | return options; |
443 | 5.50k | } |
444 | | |
445 | | void DBHolder::CreateColumnFamilies(const std::vector<std::string>& cfs, |
446 | 1.28k | const Options& options) { |
447 | 1.28k | ColumnFamilyOptions cf_opts(options); |
448 | 1.28k | size_t cfi = handles_.size(); |
449 | 1.28k | handles_.resize(cfi + cfs.size()); |
450 | 1.56k | for (auto cf : cfs) { |
451 | 1.56k | ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++])); |
452 | 1.56k | } |
453 | 1.28k | } |
454 | | |
455 | | void DBHolder::CreateAndReopenWithCF(const std::vector<std::string>& cfs, |
456 | 1.28k | const Options& options) { |
457 | 1.28k | CreateColumnFamilies(cfs, options); |
458 | 1.28k | std::vector<std::string> cfs_plus_default = cfs; |
459 | 1.28k | cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); |
460 | 1.28k | ReopenWithColumnFamilies(cfs_plus_default, options); |
461 | 1.28k | } |
462 | | |
463 | | void DBHolder::ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
464 | 4 | const std::vector<Options>& options) { |
465 | 4 | ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
466 | 4 | } |
467 | | |
468 | | void DBHolder::ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
469 | 2.51k | const Options& options) { |
470 | 2.51k | ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
471 | 2.51k | } |
472 | | |
473 | | Status DBHolder::TryReopenWithColumnFamilies( |
474 | 2.53k | const std::vector<std::string>& cfs, const std::vector<Options>& options) { |
475 | 2.53k | Close(); |
476 | 2.53k | EXPECT_EQ(cfs.size(), options.size()); |
477 | 2.53k | std::vector<ColumnFamilyDescriptor> column_families; |
478 | 7.89k | for (size_t i = 0; i < cfs.size(); ++i5.35k ) { |
479 | 5.35k | column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i])); |
480 | 5.35k | } |
481 | 2.53k | DBOptions db_opts = DBOptions(options[0]); |
482 | 2.53k | return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); |
483 | 2.53k | } |
484 | | |
485 | | Status DBHolder::TryReopenWithColumnFamilies( |
486 | 2.52k | const std::vector<std::string>& cfs, const Options& options) { |
487 | 2.52k | Close(); |
488 | 2.52k | std::vector<Options> v_opts(cfs.size(), options); |
489 | 2.52k | return TryReopenWithColumnFamilies(cfs, v_opts); |
490 | 2.52k | } |
491 | | |
492 | 3.71k | void DBHolder::Reopen(const Options& options) { |
493 | 3.71k | ASSERT_OK(TryReopen(options)); |
494 | 3.71k | } |
495 | | |
496 | 14.4k | void DBHolder::Close() { |
497 | 14.4k | for (auto h : handles_) { |
498 | 6.89k | delete h; |
499 | 6.89k | } |
500 | 14.4k | handles_.clear(); |
501 | 14.4k | delete db_; |
502 | 14.4k | db_ = nullptr; |
503 | 14.4k | } |
504 | | |
505 | 1.86k | void DBHolder::DestroyAndReopen(const Options& options) { |
506 | | // Destroy using last options |
507 | 1.86k | Destroy(last_options_); |
508 | 1.86k | ASSERT_OK(TryReopen(options)); |
509 | 1.86k | } |
510 | | |
511 | 2.19k | void DBHolder::Destroy(const Options& options) { |
512 | 2.19k | Close(); |
513 | 2.19k | ASSERT_OK(DestroyDB(dbname_, options)); |
514 | 2.19k | } |
515 | | |
516 | 7 | Status DBHolder::ReadOnlyReopen(const Options& options) { |
517 | 7 | return DB::OpenForReadOnly(options, dbname_, &db_); |
518 | 7 | } |
519 | | |
520 | 6.12k | Status DBHolder::TryReopen(const Options& options) { |
521 | 6.12k | Close(); |
522 | 6.12k | last_options_ = options; |
523 | 6.12k | return DB::Open(options, dbname_, &db_); |
524 | 6.12k | } |
525 | | |
526 | 8.92k | Status DBHolder::Flush(int cf) { |
527 | 8.92k | if (cf == 0) { |
528 | 6.80k | return db_->Flush(FlushOptions()); |
529 | 6.80k | } else { |
530 | 2.12k | return db_->Flush(FlushOptions(), handles_[cf]); |
531 | 2.12k | } |
532 | 8.92k | } |
533 | | |
534 | 3.95M | Status DBHolder::Put(const Slice& k, const Slice& v, WriteOptions wo) { |
535 | 3.95M | if (kMergePut == option_config_) { |
536 | 3.11k | return db_->Merge(wo, k, v); |
537 | 3.95M | } else { |
538 | 3.95M | return db_->Put(wo, k, v); |
539 | 3.95M | } |
540 | 3.95M | } |
541 | | |
542 | | Status DBHolder::Put(int cf, const Slice& k, const Slice& v, |
543 | 9.34M | WriteOptions wo) { |
544 | 9.34M | if (kMergePut == option_config_) { |
545 | 227 | return db_->Merge(wo, handles_[cf], k, v); |
546 | 9.34M | } else { |
547 | 9.34M | return db_->Put(wo, handles_[cf], k, v); |
548 | 9.34M | } |
549 | 9.34M | } |
550 | | |
551 | 161k | Status DBHolder::Delete(const std::string& k) { |
552 | 161k | return db_->Delete(WriteOptions(), k); |
553 | 161k | } |
554 | | |
555 | 453 | Status DBHolder::Delete(int cf, const std::string& k) { |
556 | 453 | return db_->Delete(WriteOptions(), handles_[cf], k); |
557 | 453 | } |
558 | | |
559 | 0 | Status DBHolder::SingleDelete(const std::string& k) { |
560 | 0 | return db_->SingleDelete(WriteOptions(), k); |
561 | 0 | } |
562 | | |
563 | 182 | Status DBHolder::SingleDelete(int cf, const std::string& k) { |
564 | 182 | return db_->SingleDelete(WriteOptions(), handles_[cf], k); |
565 | 182 | } |
566 | | |
567 | 4.34M | std::string DBHolder::Get(const std::string& k, const Snapshot* snapshot) { |
568 | 4.34M | ReadOptions options; |
569 | 4.34M | options.verify_checksums = true; |
570 | 4.34M | options.snapshot = snapshot; |
571 | 4.34M | options.query_id = kInMultiTouchId; |
572 | 4.34M | std::string result; |
573 | 4.34M | Status s = db_->Get(options, k, &result); |
574 | 4.34M | if (s.IsNotFound()) { |
575 | 314k | result = "NOT_FOUND"; |
576 | 4.02M | } else if (!s.ok()) { |
577 | 520 | result = s.ToString(); |
578 | 520 | } |
579 | 4.34M | return result; |
580 | 4.34M | } |
581 | | |
582 | | std::string DBHolder::Get(int cf, const std::string& k, |
583 | 1.66M | const Snapshot* snapshot) { |
584 | 1.66M | ReadOptions options; |
585 | 1.66M | options.verify_checksums = true; |
586 | 1.66M | options.snapshot = snapshot; |
587 | 1.66M | std::string result; |
588 | 1.66M | Status s = db_->Get(options, handles_[cf], k, &result); |
589 | 1.66M | if (s.IsNotFound()) { |
590 | 207k | result = "NOT_FOUND"; |
591 | 1.45M | } else if (!s.ok()) { |
592 | 0 | result = s.ToString(); |
593 | 0 | } |
594 | 1.66M | return result; |
595 | 1.66M | } |
596 | | |
597 | 180 | uint64_t DBHolder::GetNumSnapshots() { |
598 | 180 | uint64_t int_num; |
599 | 180 | EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num)); |
600 | 180 | return int_num; |
601 | 180 | } |
602 | | |
603 | 150 | uint64_t DBHolder::GetTimeOldestSnapshots() { |
604 | 150 | uint64_t int_num; |
605 | 150 | EXPECT_TRUE( |
606 | 150 | dbfull()->GetIntProperty("rocksdb.oldest-snapshot-time", &int_num)); |
607 | 150 | return int_num; |
608 | 150 | } |
609 | | |
610 | | // Return a string that contains all key,value pairs in order, |
611 | | // formatted like "(k1->v1)(k2->v2)". |
612 | 20 | std::string DBHolder::Contents(int cf) { |
613 | 20 | std::vector<std::string> forward; |
614 | 20 | std::string result; |
615 | 20 | Iterator* iter = (cf == 0) ? db_->NewIterator(ReadOptions())0 |
616 | 20 | : db_->NewIterator(ReadOptions(), handles_[cf]); |
617 | 50 | for (iter->SeekToFirst(); iter->Valid(); iter->Next()30 ) { |
618 | 30 | std::string s = IterStatus(iter); |
619 | 30 | result.push_back('('); |
620 | 30 | result.append(s); |
621 | 30 | result.push_back(')'); |
622 | 30 | forward.push_back(s); |
623 | 30 | } |
624 | | |
625 | | // Check reverse iteration results are the reverse of forward results |
626 | 20 | unsigned int matched = 0; |
627 | 50 | for (iter->SeekToLast(); iter->Valid(); iter->Prev()30 ) { |
628 | 30 | EXPECT_LT(matched, forward.size()); |
629 | 30 | EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); |
630 | 30 | matched++; |
631 | 30 | } |
632 | 20 | EXPECT_EQ(matched, forward.size()); |
633 | | |
634 | 20 | delete iter; |
635 | 20 | return result; |
636 | 20 | } |
637 | | |
638 | 376 | std::string DBHolder::AllEntriesFor(const Slice& user_key, int cf) { |
639 | 376 | Arena arena; |
640 | 376 | ScopedArenaIterator iter; |
641 | 376 | if (cf == 0) { |
642 | 0 | iter.set(dbfull()->NewInternalIterator(&arena)); |
643 | 376 | } else { |
644 | 376 | iter.set(dbfull()->NewInternalIterator(&arena, handles_[cf])); |
645 | 376 | } |
646 | 376 | InternalKey target(user_key, kMaxSequenceNumber, kTypeValue); |
647 | 376 | iter->Seek(target.Encode()); |
648 | 376 | std::string result; |
649 | 376 | if (!iter->status().ok()) { |
650 | 0 | result = iter->status().ToString(); |
651 | 376 | } else { |
652 | 376 | result = "[ "; |
653 | 376 | bool first = true; |
654 | 1.04k | while (iter->Valid()) { |
655 | 952 | ParsedInternalKey ikey(Slice(), 0, kTypeValue); |
656 | 952 | if (!ParseInternalKey(iter->key(), &ikey)) { |
657 | 0 | result += "CORRUPTED"; |
658 | 952 | } else { |
659 | 952 | if (!last_options_.comparator->Equal(ikey.user_key, user_key)) { |
660 | 285 | break; |
661 | 285 | } |
662 | 667 | if (!first) { |
663 | 369 | result += ", "; |
664 | 369 | } |
665 | 667 | first = false; |
666 | 667 | switch (ikey.type) { |
667 | 583 | case kTypeValue: |
668 | 583 | result += iter->value().ToString(); |
669 | 583 | break; |
670 | 14 | case kTypeMerge: |
671 | | // keep it the same as kTypeValue for testing kMergePut |
672 | 14 | result += iter->value().ToString(); |
673 | 14 | break; |
674 | 44 | case kTypeDeletion: |
675 | 44 | result += "DEL"; |
676 | 44 | break; |
677 | 26 | case kTypeSingleDeletion: |
678 | 26 | result += "SDEL"; |
679 | 26 | break; |
680 | 0 | default: |
681 | 0 | assert(false); |
682 | 0 | break; |
683 | 667 | } |
684 | 667 | } |
685 | 667 | iter->Next(); |
686 | 667 | } |
687 | 376 | if (!first) { |
688 | 298 | result += " "; |
689 | 298 | } |
690 | 376 | result += "]"; |
691 | 376 | } |
692 | 376 | return result; |
693 | 376 | } |
694 | | |
695 | | #ifndef ROCKSDB_LITE |
696 | 286 | int DBHolder::NumSortedRuns(int cf) { |
697 | 286 | ColumnFamilyMetaData cf_meta; |
698 | 286 | if (cf == 0) { |
699 | 196 | db_->GetColumnFamilyMetaData(&cf_meta); |
700 | 196 | } else { |
701 | 90 | db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); |
702 | 90 | } |
703 | 286 | int num_sr = static_cast<int>(cf_meta.levels[0].files.size()); |
704 | 748 | for (size_t i = 1U; i < cf_meta.levels.size(); i++462 ) { |
705 | 462 | if (cf_meta.levels[i].files.size() > 0) { |
706 | 133 | num_sr++; |
707 | 133 | } |
708 | 462 | } |
709 | 286 | return num_sr; |
710 | 286 | } |
711 | | |
712 | 30 | uint64_t DBHolder::TotalSize(int cf) { |
713 | 30 | ColumnFamilyMetaData cf_meta; |
714 | 30 | if (cf == 0) { |
715 | 30 | db_->GetColumnFamilyMetaData(&cf_meta); |
716 | 30 | } else { |
717 | 0 | db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); |
718 | 0 | } |
719 | 30 | return cf_meta.size; |
720 | 30 | } |
721 | | |
722 | 14 | uint64_t DBHolder::SizeAtLevel(int level) { |
723 | 14 | std::vector<LiveFileMetaData> metadata; |
724 | 14 | db_->GetLiveFilesMetaData(&metadata); |
725 | 14 | uint64_t sum = 0; |
726 | 355 | for (const auto& m : metadata) { |
727 | 355 | if (m.level == level) { |
728 | 127 | sum += m.total_size; |
729 | 127 | } |
730 | 355 | } |
731 | 14 | return sum; |
732 | 14 | } |
733 | | |
734 | 24 | size_t DBHolder::TotalLiveFiles(int cf) { |
735 | 24 | ColumnFamilyMetaData cf_meta; |
736 | 24 | if (cf == 0) { |
737 | 0 | db_->GetColumnFamilyMetaData(&cf_meta); |
738 | 24 | } else { |
739 | 24 | db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta); |
740 | 24 | } |
741 | 24 | size_t num_files = 0; |
742 | 108 | for (auto& level : cf_meta.levels) { |
743 | 108 | num_files += level.files.size(); |
744 | 108 | } |
745 | 24 | return num_files; |
746 | 24 | } |
747 | | |
748 | 16 | size_t DBHolder::CountLiveFiles() { |
749 | 16 | std::vector<LiveFileMetaData> metadata; |
750 | 16 | db_->GetLiveFilesMetaData(&metadata); |
751 | 16 | return metadata.size(); |
752 | 16 | } |
753 | | #endif // ROCKSDB_LITE |
754 | | |
755 | 8.76k | int DBHolder::NumTableFilesAtLevel(int level, int cf) { |
756 | 8.76k | std::string property; |
757 | 8.76k | if (cf == 0) { |
758 | | // default cfd |
759 | 1.66k | EXPECT_TRUE(db_->GetProperty( |
760 | 1.66k | "rocksdb.num-files-at-level" + NumberToString(level), &property)); |
761 | 7.10k | } else { |
762 | 7.10k | EXPECT_TRUE(db_->GetProperty( |
763 | 7.10k | handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level), |
764 | 7.10k | &property)); |
765 | 7.10k | } |
766 | 8.76k | return atoi(property.c_str()); |
767 | 8.76k | } |
768 | | |
769 | 769 | int DBHolder::TotalTableFiles(int cf, int levels) { |
770 | 769 | if (levels == -1) { |
771 | 765 | levels = CurrentOptions().num_levels; |
772 | 765 | } |
773 | 769 | int result = 0; |
774 | 5.70k | for (int level = 0; level < levels; level++4.93k ) { |
775 | 4.93k | result += NumTableFilesAtLevel(level, cf); |
776 | 4.93k | } |
777 | 769 | return result; |
778 | 769 | } |
779 | | |
780 | | // Return spread of files per level |
781 | 510 | std::string DBHolder::FilesPerLevel(int cf) { |
782 | 510 | int num_levels = |
783 | 510 | (cf == 0) ? db_->NumberLevels()270 : db_->NumberLevels(handles_[1])240 ; |
784 | 510 | std::string result; |
785 | 510 | size_t last_non_zero_offset = 0; |
786 | 3.43k | for (int level = 0; level < num_levels; level++2.92k ) { |
787 | 2.92k | int f = NumTableFilesAtLevel(level, cf); |
788 | 2.92k | char buf[100]; |
789 | 2.92k | snprintf(buf, sizeof(buf), "%s%d", (level ? ","2.41k : ""510 ), f); |
790 | 2.92k | result += buf; |
791 | 2.92k | if (f > 0) { |
792 | 883 | last_non_zero_offset = result.size(); |
793 | 883 | } |
794 | 2.92k | } |
795 | 510 | result.resize(last_non_zero_offset); |
796 | 510 | return result; |
797 | 510 | } |
798 | | |
799 | 12 | size_t DBHolder::CountFiles() { |
800 | 12 | std::vector<std::string> files; |
801 | 12 | env_->GetChildrenWarnNotOk(dbname_, &files); |
802 | | |
803 | 12 | std::vector<std::string> logfiles; |
804 | 12 | if (dbname_ != last_options_.wal_dir) { |
805 | 12 | env_->GetChildrenWarnNotOk(last_options_.wal_dir, &logfiles); |
806 | 12 | } |
807 | | |
808 | 12 | return files.size() + logfiles.size(); |
809 | 12 | } |
810 | | |
811 | 14.0k | uint64_t DBHolder::Size(const Slice& start, const Slice& limit, int cf) { |
812 | 14.0k | Range r(start, limit); |
813 | 14.0k | uint64_t size; |
814 | 14.0k | if (cf == 0) { |
815 | 54 | db_->GetApproximateSizes(&r, 1, &size); |
816 | 13.9k | } else { |
817 | 13.9k | db_->GetApproximateSizes(handles_[1], &r, 1, &size); |
818 | 13.9k | } |
819 | 14.0k | return size; |
820 | 14.0k | } |
821 | | |
822 | | void DBHolder::Compact(int cf, const Slice& start, const Slice& limit, |
823 | 16 | uint32_t target_path_id) { |
824 | 16 | CompactRangeOptions compact_options; |
825 | 16 | compact_options.target_path_id = target_path_id; |
826 | 16 | ASSERT_OK(db_->CompactRange(compact_options, handles_[cf], &start, &limit)); |
827 | 16 | } |
828 | | |
829 | 248 | void DBHolder::Compact(int cf, const Slice& start, const Slice& limit) { |
830 | 248 | ASSERT_OK( |
831 | 248 | db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit)); |
832 | 248 | } |
833 | | |
834 | 311 | void DBHolder::Compact(const Slice& start, const Slice& limit) { |
835 | 311 | ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &limit)); |
836 | 311 | } |
837 | | |
838 | | // Do n memtable compactions, each of which produces an sstable |
839 | | // covering the range [small,large]. |
840 | | void DBHolder::MakeTables(int n, const std::string& small, |
841 | 149 | const std::string& large, int cf) { |
842 | 1.01k | for (int i = 0; i < n; i++862 ) { |
843 | 862 | ASSERT_OK(Put(cf, small, "begin")); |
844 | 862 | ASSERT_OK(Put(cf, large, "end")); |
845 | 862 | ASSERT_OK(Flush(cf)); |
846 | 862 | MoveFilesToLevel(n - i - 1, cf); |
847 | 862 | } |
848 | 149 | } |
849 | | |
850 | | // Prevent pushing of new sstables into deeper levels by adding |
851 | | // tables that cover a specified range to all levels. |
852 | | void DBHolder::FillLevels(const std::string& smallest, |
853 | 115 | const std::string& largest, int cf) { |
854 | 115 | MakeTables(db_->NumberLevels(handles_[cf]), smallest, largest, cf); |
855 | 115 | } |
856 | | |
857 | 950 | void DBHolder::MoveFilesToLevel(int level, int cf) { |
858 | 3.54k | for (int l = 0; l < level; ++l2.59k ) { |
859 | 2.59k | if (cf > 0) { |
860 | 2.50k | CHECK_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr, handles_[cf])); |
861 | 2.50k | } else { |
862 | 94 | CHECK_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr)); |
863 | 94 | } |
864 | 2.59k | } |
865 | 950 | } |
866 | | |
867 | 0 | void DBHolder::DumpFileCounts(const char* label) { |
868 | 0 | fprintf(stderr, "---\n%s:\n", label); |
869 | 0 | fprintf(stderr, "maxoverlap: %" PRIu64 "\n", |
870 | 0 | dbfull()->TEST_MaxNextLevelOverlappingBytes()); |
871 | 0 | for (int level = 0; level < db_->NumberLevels(); level++) { |
872 | 0 | int num = NumTableFilesAtLevel(level); |
873 | 0 | if (num > 0) { |
874 | 0 | fprintf(stderr, " level %3d : %d files\n", level, num); |
875 | 0 | } |
876 | 0 | } |
877 | 0 | } |
878 | | |
879 | 0 | std::string DBHolder::DumpSSTableList() { |
880 | 0 | std::string property; |
881 | 0 | db_->GetProperty("rocksdb.sstables", &property); |
882 | 0 | return property; |
883 | 0 | } |
884 | | |
885 | | void DBHolder::GetSstFiles(std::string path, |
886 | 623 | std::vector<std::string>* files) { |
887 | 623 | env_->GetChildrenWarnNotOk(path, files); |
888 | | |
889 | 623 | files->erase( |
890 | 4.96k | std::remove_if(files->begin(), files->end(), [](std::string name) { |
891 | 4.96k | uint64_t number; |
892 | 4.96k | FileType type; |
893 | 4.96k | return !(ParseFileName(name, &number, &type) && type == kTableFile3.24k ); |
894 | 4.96k | }), files->end()); |
895 | 623 | } |
896 | | |
897 | 622 | int DBHolder::GetSstFileCount(std::string path) { |
898 | 622 | std::vector<std::string> files; |
899 | 622 | GetSstFiles(path, &files); |
900 | 622 | return static_cast<int>(files.size()); |
901 | 622 | } |
902 | | |
903 | | // this will generate non-overlapping files since it keeps increasing key_idx |
904 | | void DBHolder::GenerateNewFile(int cf, Random* rnd, int* key_idx, |
905 | 60 | bool nowait) { |
906 | 6.06k | for (int i = 0; i < KNumKeysByGenerateNewFile; i++6.00k ) { |
907 | 6.00k | ASSERT_OK(Put(cf, Key(*key_idx), RandomString(rnd, (i == 99) ? 1 : 990))); |
908 | 6.00k | (*key_idx)++; |
909 | 6.00k | } |
910 | 60 | if (!nowait) { |
911 | 60 | CHECK_OK(dbfull()->TEST_WaitForFlushMemTable()); |
912 | 60 | CHECK_OK(dbfull()->TEST_WaitForCompact()); |
913 | 60 | } |
914 | 60 | } |
915 | | |
916 | | // this will generate non-overlapping files since it keeps increasing key_idx |
917 | 541 | void DBHolder::GenerateNewFile(Random* rnd, int* key_idx, bool nowait) { |
918 | 54.6k | for (int i = 0; i < KNumKeysByGenerateNewFile; i++54.1k ) { |
919 | 54.1k | ASSERT_OK(Put(Key(*key_idx), RandomString(rnd, (i == 99) ? 1 : 990))); |
920 | 54.1k | (*key_idx)++; |
921 | 54.1k | } |
922 | 541 | if (!nowait) { |
923 | 526 | CHECK_OK(dbfull()->TEST_WaitForFlushMemTable()); |
924 | 526 | CHECK_OK(dbfull()->TEST_WaitForCompact()); |
925 | 526 | } |
926 | 541 | } |
927 | | |
928 | | const int DBHolder::kNumKeysByGenerateNewRandomFile = 51; |
929 | | |
930 | 135 | void DBHolder::GenerateNewRandomFile(Random* rnd, bool nowait) { |
931 | 7.02k | for (int i = 0; i < kNumKeysByGenerateNewRandomFile; i++6.88k ) { |
932 | 6.88k | ASSERT_OK(Put("key" + RandomString(rnd, 7), RandomString(rnd, 2000))); |
933 | 6.88k | } |
934 | 135 | ASSERT_OK(Put("key" + RandomString(rnd, 7), RandomString(rnd, 200))); |
935 | 135 | if (!nowait) { |
936 | 115 | CHECK_OK(dbfull()->TEST_WaitForFlushMemTable()); |
937 | 115 | CHECK_OK(dbfull()->TEST_WaitForCompact()); |
938 | 115 | } |
939 | 135 | } |
940 | | |
941 | 712 | std::string DBHolder::IterStatus(Iterator* iter) { |
942 | 712 | std::string result; |
943 | 712 | if (iter->Valid()) { |
944 | 590 | result = iter->key().ToString() + "->" + iter->value().ToString(); |
945 | 590 | } else { |
946 | 122 | result = "(invalid)"; |
947 | 122 | } |
948 | 712 | return result; |
949 | 712 | } |
950 | | |
951 | 50 | Options DBHolder::OptionsForLogIterTest() { |
952 | 50 | Options options = CurrentOptions(); |
953 | 50 | options.create_if_missing = true; |
954 | 50 | options.WAL_ttl_seconds = 1000; |
955 | 50 | return options; |
956 | 50 | } |
957 | | |
958 | 3.19M | std::string DBHolder::DummyString(size_t len, char c) { |
959 | 3.19M | return std::string(len, c); |
960 | 3.19M | } |
961 | | |
962 | 162 | void DBHolder::VerifyIterLast(std::string expected_key, int cf) { |
963 | 162 | Iterator* iter; |
964 | 162 | ReadOptions ro; |
965 | 162 | if (cf == 0) { |
966 | 0 | iter = db_->NewIterator(ro); |
967 | 162 | } else { |
968 | 162 | iter = db_->NewIterator(ro, handles_[cf]); |
969 | 162 | } |
970 | 162 | iter->SeekToLast(); |
971 | 162 | ASSERT_EQ(IterStatus(iter), expected_key); |
972 | 162 | delete iter; |
973 | 162 | } |
974 | | |
975 | | // Used to test InplaceUpdate |
976 | | |
977 | | // If previous value is nullptr or delta is > than previous value, |
978 | | // sets newValue with delta |
979 | | // If previous value is not empty, |
980 | | // updates previous value with 'b' string of previous value size - 1. |
981 | | UpdateStatus DBHolder::updateInPlaceSmallerSize(char* prevValue, |
982 | | uint32_t* prevSize, |
983 | | Slice delta, |
984 | 55 | std::string* newValue) { |
985 | 55 | if (prevValue == nullptr) { |
986 | 5 | *newValue = std::string(delta.size(), 'c'); |
987 | 5 | return UpdateStatus::UPDATED; |
988 | 50 | } else { |
989 | 50 | *prevSize = *prevSize - 1; |
990 | 50 | std::string str_b = std::string(*prevSize, 'b'); |
991 | 50 | memcpy(prevValue, str_b.c_str(), str_b.size()); |
992 | 50 | return UpdateStatus::UPDATED_INPLACE; |
993 | 50 | } |
994 | 55 | } |
995 | | |
996 | | UpdateStatus DBHolder::updateInPlaceSmallerVarintSize(char* prevValue, |
997 | | uint32_t* prevSize, |
998 | | Slice delta, |
999 | 1.33k | std::string* newValue) { |
1000 | 1.33k | if (prevValue == nullptr) { |
1001 | 5 | *newValue = std::string(delta.size(), 'c'); |
1002 | 5 | return UpdateStatus::UPDATED; |
1003 | 1.32k | } else { |
1004 | 1.32k | *prevSize = 1; |
1005 | 1.32k | std::string str_b = std::string(*prevSize, 'b'); |
1006 | 1.32k | memcpy(prevValue, str_b.c_str(), str_b.size()); |
1007 | 1.32k | return UpdateStatus::UPDATED_INPLACE; |
1008 | 1.32k | } |
1009 | 1.33k | } |
1010 | | |
1011 | | UpdateStatus DBHolder::updateInPlaceLargerSize(char* prevValue, |
1012 | | uint32_t* prevSize, |
1013 | | Slice delta, |
1014 | 50 | std::string* newValue) { |
1015 | 50 | *newValue = std::string(delta.size(), 'c'); |
1016 | 50 | return UpdateStatus::UPDATED; |
1017 | 50 | } |
1018 | | |
1019 | | UpdateStatus DBHolder::updateInPlaceNoAction(char* prevValue, |
1020 | | uint32_t* prevSize, Slice delta, |
1021 | 5 | std::string* newValue) { |
1022 | 5 | return UpdateStatus::UPDATE_FAILED; |
1023 | 5 | } |
1024 | | |
1025 | | // Utility method to test InplaceUpdate |
1026 | 25 | void DBHolder::validateNumberOfEntries(int numValues, int cf) { |
1027 | 25 | Arena arena; |
1028 | 25 | ScopedArenaIterator iter; |
1029 | 25 | if (cf != 0) { |
1030 | 25 | iter.set(dbfull()->NewInternalIterator(&arena, handles_[cf])); |
1031 | 25 | } else { |
1032 | 0 | iter.set(dbfull()->NewInternalIterator(&arena)); |
1033 | 0 | } |
1034 | 25 | iter->SeekToFirst(); |
1035 | 25 | ASSERT_EQ(iter->status().ok(), true); |
1036 | 25 | int seq = numValues; |
1037 | 140 | while (iter->Valid()) { |
1038 | 115 | ParsedInternalKey ikey; |
1039 | 115 | ikey.sequence = -1; |
1040 | 115 | ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true); |
1041 | | |
1042 | | // checks sequence number for updates |
1043 | 115 | ASSERT_EQ(ikey.sequence, (unsigned)seq--); |
1044 | 115 | iter->Next(); |
1045 | 115 | } |
1046 | 25 | ASSERT_EQ(0, seq); |
1047 | 25 | } |
1048 | | |
1049 | | void DBHolder::CopyFile(const std::string& source, |
1050 | 150 | const std::string& destination, uint64_t size) { |
1051 | 150 | const EnvOptions soptions; |
1052 | 150 | unique_ptr<SequentialFile> srcfile; |
1053 | 150 | ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions)); |
1054 | 150 | unique_ptr<WritableFile> destfile; |
1055 | 150 | ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions)); |
1056 | | |
1057 | 150 | if (size == 0) { |
1058 | | // default argument means copy everything |
1059 | 120 | ASSERT_OK(env_->GetFileSize(source, &size)); |
1060 | 120 | } |
1061 | | |
1062 | 150 | uint8_t buffer[4096]; |
1063 | 150 | Slice slice; |
1064 | 10.0k | while (size > 0) { |
1065 | 9.91k | uint64_t one = std::min(uint64_t(sizeof(buffer)), size); |
1066 | 9.91k | ASSERT_OK(srcfile->Read(one, &slice, buffer)); |
1067 | 9.91k | ASSERT_OK(destfile->Append(slice)); |
1068 | 9.91k | size -= slice.size(); |
1069 | 9.91k | } |
1070 | 150 | ASSERT_OK(destfile->Close()); |
1071 | 150 | } |
1072 | | |
1073 | | std::unordered_map<std::string, uint64_t> DBHolder::GetAllSSTFiles( |
1074 | 32 | uint64_t* total_size) { |
1075 | 32 | std::unordered_map<std::string, uint64_t> res; |
1076 | | |
1077 | 32 | if (total_size) { |
1078 | 6 | *total_size = 0; |
1079 | 6 | } |
1080 | 32 | std::vector<std::string> files; |
1081 | 32 | env_->GetChildrenWarnNotOk(dbname_, &files); |
1082 | 486 | for (auto& file_name : files) { |
1083 | 486 | uint64_t number; |
1084 | 486 | FileType type; |
1085 | 486 | std::string file_path = dbname_ + "/" + file_name; |
1086 | 486 | if (ParseFileName(file_name, &number, &type) && |
1087 | 486 | (390 type == kTableFile390 || type == kTableSBlockFile292 )) { |
1088 | 196 | uint64_t file_size = 0; |
1089 | 196 | CHECK_OK(env_->GetFileSize(file_path, &file_size)); |
1090 | 196 | res[file_path] = file_size; |
1091 | 196 | if (total_size) { |
1092 | 70 | *total_size += file_size; |
1093 | 70 | } |
1094 | 196 | } |
1095 | 486 | } |
1096 | 32 | return res; |
1097 | 32 | } |
1098 | | |
1099 | 1 | void ConfigureLoggingToGlog(Options* options, const std::string& log_prefix) { |
1100 | 1 | options->log_prefix = log_prefix; |
1101 | 1 | options->info_log_level = InfoLogLevel::INFO_LEVEL; |
1102 | 1 | options->info_log = std::make_shared<yb::YBRocksDBLogger>(options->log_prefix); |
1103 | 1 | } |
1104 | | |
1105 | | } // namespace rocksdb |