/Users/deen/code/yugabyte-db/src/yb/rocksdb/utilities/ttl/ttl_test.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. |
2 | | // Use of this source code is governed by a BSD-style license that can be |
3 | | // found in the LICENSE file. See the AUTHORS file for names of contributors. |
4 | | // |
5 | | // The following only applies to changes made to this file as part of YugaByte development. |
6 | | // |
7 | | // Portions Copyright (c) YugaByte, Inc. |
8 | | // |
9 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
10 | | // in compliance with the License. You may obtain a copy of the License at |
11 | | // |
12 | | // http://www.apache.org/licenses/LICENSE-2.0 |
13 | | // |
14 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
15 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
16 | | // or implied. See the License for the specific language governing permissions and limitations |
17 | | // under the License. |
18 | | // |
19 | | |
20 | | #ifndef ROCKSDB_LITE |
21 | | |
22 | | #include <map> |
23 | | #include <memory> |
24 | | |
25 | | #ifndef OS_WIN |
26 | | #include <unistd.h> |
27 | | #endif |
28 | | |
29 | | #include "yb/rocksdb/compaction_filter.h" |
30 | | #include "yb/rocksdb/utilities/db_ttl.h" |
31 | | #include "yb/rocksdb/util/testharness.h" |
32 | | #include "yb/rocksdb/util/testutil.h" |
33 | | #include "yb/rocksdb/util/logging.h" |
34 | | |
35 | | #include "yb/util/test_util.h" |
36 | | |
37 | | namespace rocksdb { |
38 | | |
39 | | namespace { |
40 | | |
41 | | typedef std::map<std::string, std::string> KVMap; |
42 | | |
43 | | enum BatchOperation { OP_PUT = 0, OP_DELETE = 1 }; |
44 | | } |
45 | | |
46 | | class SpecialTimeEnv : public EnvWrapper { |
47 | | public: |
48 | 15 | explicit SpecialTimeEnv(Env* base) : EnvWrapper(base) { |
49 | 15 | CHECK_OK(base->GetCurrentTime(¤t_time_)); |
50 | 15 | } |
51 | | |
52 | 28 | void Sleep(int64_t sleep_time) { current_time_ += sleep_time; } |
53 | 3.87k | Status GetCurrentTime(int64_t* current_time) override { |
54 | 3.87k | *current_time = current_time_; |
55 | 3.87k | return Status::OK(); |
56 | 3.87k | } |
57 | | |
58 | | private: |
59 | | int64_t current_time_; |
60 | | }; |
61 | | |
62 | | class TtlTest : public RocksDBTest { |
63 | | public: |
64 | 15 | TtlTest() { |
65 | 15 | env_.reset(new SpecialTimeEnv(Env::Default())); |
66 | 15 | dbname_ = test::TmpDir() + "/db_ttl"; |
67 | 15 | options_.create_if_missing = true; |
68 | 15 | options_.env = env_.get(); |
69 | | // ensure that compaction is kicked in to always strip timestamp from kvs |
70 | 15 | options_.max_grandparent_overlap_factor = 0; |
71 | | // compaction should take place always from level0 for determinism |
72 | 15 | db_ttl_ = nullptr; |
73 | 15 | CHECK_OK(DestroyDB(dbname_, Options())); |
74 | 15 | } |
75 | | |
76 | 15 | ~TtlTest() { |
77 | 15 | CloseTtl(); |
78 | 15 | CHECK_OK(DestroyDB(dbname_, Options())); |
79 | 15 | } |
80 | | |
81 | | // Open database with TTL support when TTL not provided with db_ttl_ pointer |
82 | 3 | void OpenTtl() { |
83 | 3 | ASSERT_TRUE(db_ttl_ == |
84 | 3 | nullptr); // db should be closed before opening again |
85 | 3 | ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_)); |
86 | 3 | read_only_ = false; |
87 | 3 | } |
88 | | |
89 | | // Open database with TTL support when TTL provided with db_ttl_ pointer |
90 | 17 | void OpenTtl(int32_t ttl) { |
91 | 17 | ASSERT_TRUE(db_ttl_ == nullptr); |
92 | 17 | ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl)); |
93 | 17 | read_only_ = false; |
94 | 17 | } |
95 | | |
96 | | // Open with TestFilter compaction filter |
97 | 2 | void OpenTtlWithTestCompaction(int32_t ttl) { |
98 | 2 | options_.compaction_filter_factory = |
99 | 2 | std::shared_ptr<CompactionFilterFactory>( |
100 | 2 | new TestFilterFactory(kSampleSize_, kNewValue_)); |
101 | 2 | OpenTtl(ttl); |
102 | 2 | } |
103 | | |
104 | | // Open database with TTL support in read_only mode |
105 | 1 | void OpenReadOnlyTtl(int32_t ttl) { |
106 | 1 | ASSERT_TRUE(db_ttl_ == nullptr); |
107 | 1 | ASSERT_OK(DBWithTTL::Open(options_, dbname_, &db_ttl_, ttl, true)); |
108 | 1 | read_only_ = true; |
109 | 1 | } |
110 | | |
111 | 36 | void CloseTtl() { |
112 | 36 | delete db_ttl_; |
113 | 36 | db_ttl_ = nullptr; |
114 | 36 | } |
115 | | |
116 | | // Populates and returns a kv-map |
117 | 15 | void MakeKVMap(int64_t num_entries) { |
118 | 15 | kvmap_.clear(); |
119 | 15 | int digits = 1; |
120 | 45 | for (int64_t dummy = num_entries; dummy /= 10; ++digits) { |
121 | 30 | } |
122 | 15 | int digits_in_i = 1; |
123 | 1.51k | for (int64_t i = 0; i < num_entries; i++) { |
124 | 1.50k | std::string key = "key"; |
125 | 1.50k | std::string value = "value"; |
126 | 1.50k | if (i % 10 == 0) { |
127 | 150 | digits_in_i++; |
128 | 150 | } |
129 | 1.65k | for(int j = digits_in_i; j < digits; j++) { |
130 | 150 | key.append("0"); |
131 | 150 | value.append("0"); |
132 | 150 | } |
133 | 1.50k | AppendNumberTo(&key, i); |
134 | 1.50k | AppendNumberTo(&value, i); |
135 | 1.50k | kvmap_[key] = value; |
136 | 1.50k | } |
137 | 15 | ASSERT_EQ(static_cast<int64_t>(kvmap_.size()), |
138 | 15 | num_entries); // check all insertions done |
139 | 15 | } |
140 | | |
141 | | // Makes a write-batch with key-vals from kvmap_ and 'Write''s it |
142 | 2 | void MakePutWriteBatch(const BatchOperation* batch_ops, int64_t num_ops) { |
143 | 2 | ASSERT_LE(num_ops, static_cast<int64_t>(kvmap_.size())); |
144 | 2 | static WriteOptions wopts; |
145 | 2 | static FlushOptions flush_opts; |
146 | 2 | WriteBatch batch; |
147 | 2 | kv_it_ = kvmap_.begin(); |
148 | 152 | for (int64_t i = 0; i < num_ops && kv_it_ != kvmap_.end(); i++, ++kv_it_) { |
149 | 150 | switch (batch_ops[i]) { |
150 | 100 | case OP_PUT: |
151 | 100 | batch.Put(kv_it_->first, kv_it_->second); |
152 | 100 | break; |
153 | 50 | case OP_DELETE: |
154 | 50 | batch.Delete(kv_it_->first); |
155 | 50 | break; |
156 | 0 | default: |
157 | 0 | ASSERT_TRUE(false); |
158 | 150 | } |
159 | 150 | } |
160 | 2 | ASSERT_OK(db_ttl_->Write(wopts, &batch)); |
161 | 2 | ASSERT_OK(db_ttl_->Flush(flush_opts)); |
162 | 2 | } |
163 | | |
164 | | // Puts num_entries starting from start_pos_map from kvmap_ into the database |
165 | | void PutValues(int64_t start_pos_map, int64_t num_entries, bool flush = true, |
166 | 20 | ColumnFamilyHandle* cf = nullptr) { |
167 | 20 | ASSERT_TRUE(db_ttl_); |
168 | 20 | ASSERT_LE(start_pos_map + num_entries, static_cast<int64_t>(kvmap_.size())); |
169 | 20 | static WriteOptions wopts; |
170 | 20 | static FlushOptions flush_opts; |
171 | 20 | kv_it_ = kvmap_.begin(); |
172 | 20 | advance(kv_it_, start_pos_map); |
173 | 1.82k | for (int64_t i = 0; kv_it_ != kvmap_.end() && i < num_entries; |
174 | 1.80k | i++, ++kv_it_) { |
175 | 1.80k | ASSERT_OK(cf == nullptr |
176 | 1.80k | ? db_ttl_->Put(wopts, kv_it_->first, kv_it_->second) |
177 | 1.80k | : db_ttl_->Put(wopts, cf, kv_it_->first, kv_it_->second)); |
178 | 1.80k | } |
179 | | // Put a mock kv at the end because CompactionFilter doesn't delete last key |
180 | 20 | ASSERT_OK(cf == nullptr ? db_ttl_->Put(wopts, "keymock", "valuemock") |
181 | 20 | : db_ttl_->Put(wopts, cf, "keymock", "valuemock")); |
182 | 20 | if (flush) { |
183 | 15 | if (cf == nullptr) { |
184 | 15 | ASSERT_OK(db_ttl_->Flush(flush_opts)); |
185 | 0 | } else { |
186 | 0 | ASSERT_OK(db_ttl_->Flush(flush_opts, cf)); |
187 | 0 | } |
188 | 15 | } |
189 | 20 | } |
190 | | |
191 | | // Runs a manual compaction |
192 | 27 | void ManualCompact(ColumnFamilyHandle* cf = nullptr) { |
193 | 27 | Status status; |
194 | 27 | if (cf == nullptr) { |
195 | 18 | status = db_ttl_->CompactRange(CompactRangeOptions(), nullptr, nullptr); |
196 | 9 | } else { |
197 | 9 | status = db_ttl_->CompactRange(CompactRangeOptions(), cf, nullptr, nullptr); |
198 | 9 | } |
199 | 27 | if (read_only_) { |
200 | 1 | ASSERT_TRUE(status.IsNotSupported()); |
201 | 26 | } else { |
202 | 26 | ASSERT_OK(status); |
203 | 26 | } |
204 | 27 | } |
205 | | |
206 | | // checks the whole kvmap_ to return correct values using KeyMayExist |
207 | 1 | void SimpleKeyMayExistCheck() { |
208 | 1 | static ReadOptions ropts; |
209 | 1 | bool value_found; |
210 | 1 | std::string val; |
211 | 100 | for(auto &kv : kvmap_) { |
212 | 100 | bool ret = db_ttl_->KeyMayExist(ropts, kv.first, &val, &value_found); |
213 | 100 | if (ret == false || value_found == false) { |
214 | 0 | fprintf(stderr, "KeyMayExist could not find key=%s in the database but" |
215 | 0 | " should have\n", kv.first.c_str()); |
216 | 0 | ASSERT_TRUE(false); |
217 | 100 | } else if (val.compare(kv.second) != 0) { |
218 | 0 | fprintf(stderr, " value for key=%s present in database is %s but" |
219 | 0 | " should be %s\n", kv.first.c_str(), val.c_str(), |
220 | 0 | kv.second.c_str()); |
221 | 0 | ASSERT_TRUE(false); |
222 | 0 | } |
223 | 100 | } |
224 | 1 | } |
225 | | |
226 | | // checks the whole kvmap_ to return correct values using MultiGet |
227 | 1 | void SimpleMultiGetTest() { |
228 | 1 | static ReadOptions ropts; |
229 | 1 | std::vector<Slice> keys; |
230 | 1 | std::vector<std::string> values; |
231 | | |
232 | 100 | for (auto& kv : kvmap_) { |
233 | 100 | keys.emplace_back(kv.first); |
234 | 100 | } |
235 | | |
236 | 1 | auto statuses = db_ttl_->MultiGet(ropts, keys, &values); |
237 | 1 | size_t i = 0; |
238 | 100 | for (auto& kv : kvmap_) { |
239 | 100 | ASSERT_OK(statuses[i]); |
240 | 100 | ASSERT_EQ(values[i], kv.second); |
241 | 100 | ++i; |
242 | 100 | } |
243 | 1 | } |
244 | | |
245 | | // Sleeps for slp_tim then runs a manual compaction |
246 | | // Checks span starting from st_pos from kvmap_ in the db and |
247 | | // Gets should return true if check is true and false otherwise |
248 | | // Also checks that value that we got is the same as inserted; and =kNewValue |
249 | | // if test_compaction_change is true |
250 | | void SleepCompactCheck(int slp_tim, int64_t st_pos, int64_t span, |
251 | | bool check = true, bool test_compaction_change = false, |
252 | 25 | ColumnFamilyHandle* cf = nullptr) { |
253 | 25 | ASSERT_TRUE(db_ttl_); |
254 | | |
255 | 25 | env_->Sleep(slp_tim); |
256 | 25 | ManualCompact(cf); |
257 | 25 | static ReadOptions ropts; |
258 | 25 | kv_it_ = kvmap_.begin(); |
259 | 25 | advance(kv_it_, st_pos); |
260 | 25 | std::string v; |
261 | 2.12k | for (int64_t i = 0; kv_it_ != kvmap_.end() && i < span; i++, ++kv_it_) { |
262 | 1.19k | Status s = (cf == nullptr) ? db_ttl_->Get(ropts, kv_it_->first, &v) |
263 | 900 | : db_ttl_->Get(ropts, cf, kv_it_->first, &v); |
264 | 2.09k | if (s.ok() != check) { |
265 | 0 | fprintf(stderr, "key=%s ", kv_it_->first.c_str()); |
266 | 0 | if (!s.ok()) { |
267 | 0 | fprintf(stderr, "is absent from db but was expected to be present\n"); |
268 | 0 | } else { |
269 | 0 | fprintf(stderr, "is present in db but was expected to be absent\n"); |
270 | 0 | } |
271 | 0 | ASSERT_TRUE(false); |
272 | 2.09k | } else if (s.ok()) { |
273 | 1.21k | if (test_compaction_change && v.compare(kNewValue_) != 0) { |
274 | 0 | fprintf(stderr, " value for key=%s present in database is %s but " |
275 | 0 | " should be %s\n", kv_it_->first.c_str(), v.c_str(), |
276 | 0 | kNewValue_.c_str()); |
277 | 0 | ASSERT_TRUE(false); |
278 | 1.21k | } else if (!test_compaction_change && v.compare(kv_it_->second) !=0) { |
279 | 0 | fprintf(stderr, " value for key=%s present in database is %s but " |
280 | 0 | " should be %s\n", kv_it_->first.c_str(), v.c_str(), |
281 | 0 | kv_it_->second.c_str()); |
282 | 0 | ASSERT_TRUE(false); |
283 | 0 | } |
284 | 1.21k | } |
285 | 2.09k | } |
286 | 25 | } |
287 | | |
288 | | // Similar as SleepCompactCheck but uses TtlIterator to read from db |
289 | | void SleepCompactCheckIter(int slp, int st_pos, int64_t span, |
290 | 2 | bool check = true) { |
291 | 2 | ASSERT_TRUE(db_ttl_); |
292 | 2 | env_->Sleep(slp); |
293 | 2 | ManualCompact(); |
294 | 2 | static ReadOptions ropts; |
295 | 2 | Iterator *dbiter = db_ttl_->NewIterator(ropts); |
296 | 2 | kv_it_ = kvmap_.begin(); |
297 | 2 | advance(kv_it_, st_pos); |
298 | | |
299 | 2 | dbiter->Seek(kv_it_->first); |
300 | 2 | if (!check) { |
301 | 1 | if (dbiter->Valid()) { |
302 | 0 | ASSERT_NE(dbiter->value().compare(kv_it_->second), 0); |
303 | 0 | } |
304 | 1 | } else { // dbiter should have found out kvmap_[st_pos] |
305 | 101 | for (int64_t i = st_pos; kv_it_ != kvmap_.end() && i < st_pos + span; |
306 | 100 | i++, ++kv_it_) { |
307 | 100 | ASSERT_TRUE(dbiter->Valid()); |
308 | 100 | ASSERT_EQ(dbiter->value().compare(kv_it_->second), 0); |
309 | 100 | dbiter->Next(); |
310 | 100 | } |
311 | 1 | } |
312 | 2 | delete dbiter; |
313 | 2 | } |
314 | | |
315 | | class TestFilter : public CompactionFilter { |
316 | | public: |
317 | | TestFilter(const int64_t kSampleSize, const std::string& kNewValue) |
318 | | : kSampleSize_(kSampleSize), |
319 | 4 | kNewValue_(kNewValue) { |
320 | 4 | } |
321 | | |
322 | | // Works on keys of the form "key<number>" |
323 | | // Drops key if number at the end of key is in [0, kSampleSize_/3), |
324 | | // Keeps key if it is in [kSampleSize_/3, 2*kSampleSize_/3), |
325 | | // Change value if it is in [2*kSampleSize_/3, kSampleSize_) |
326 | | // Eg. kSampleSize_=6. Drop:key0-1...Keep:key2-3...Change:key4-5... |
327 | | FilterDecision Filter(int level, const Slice& key, |
328 | | const Slice& value, std::string* new_value, |
329 | 237 | bool* value_changed) override { |
330 | 237 | assert(new_value != nullptr); |
331 | | |
332 | 237 | std::string search_str = "0123456789"; |
333 | 237 | std::string key_string = key.ToString(); |
334 | 237 | size_t pos = key_string.find_first_of(search_str); |
335 | 237 | int num_key_end; |
336 | 237 | if (pos != std::string::npos) { |
337 | 234 | auto key_substr = key_string.substr(pos, key.size() - pos); |
338 | 234 | #ifndef CYGWIN |
339 | 234 | num_key_end = std::stoi(key_substr); |
340 | | #else |
341 | | num_key_end = std::strtol(key_substr.c_str(), 0, 10); |
342 | | #endif |
343 | | |
344 | 3 | } else { |
345 | 3 | return FilterDecision::kKeep; // Keep keys not matching the format "key<NUMBER>" |
346 | 3 | } |
347 | | |
348 | 234 | int64_t partition = kSampleSize_ / 3; |
349 | 234 | if (num_key_end < partition) { |
350 | 33 | return FilterDecision::kDiscard; |
351 | 201 | } else if (num_key_end < partition * 2) { |
352 | 99 | return FilterDecision::kKeep; |
353 | 102 | } else { |
354 | 102 | *new_value = kNewValue_; |
355 | 102 | *value_changed = true; |
356 | 102 | return FilterDecision::kKeep; |
357 | 102 | } |
358 | 234 | } |
359 | | |
360 | 0 | const char* Name() const override { |
361 | 0 | return "TestFilter"; |
362 | 0 | } |
363 | | |
364 | | private: |
365 | | const int64_t kSampleSize_; |
366 | | const std::string kNewValue_; |
367 | | }; |
368 | | |
369 | | class TestFilterFactory : public CompactionFilterFactory { |
370 | | public: |
371 | | TestFilterFactory(const int64_t kSampleSize, const std::string& kNewValue) |
372 | | : kSampleSize_(kSampleSize), |
373 | 2 | kNewValue_(kNewValue) { |
374 | 2 | } |
375 | | |
376 | | std::unique_ptr<CompactionFilter> CreateCompactionFilter( |
377 | 4 | const CompactionFilter::Context& context) override { |
378 | 4 | return std::unique_ptr<CompactionFilter>(new TestFilter(kSampleSize_, kNewValue_)); |
379 | 4 | } |
380 | | |
381 | 0 | const char* Name() const override { |
382 | 0 | return "TestFilterFactory"; |
383 | 0 | } |
384 | | |
385 | | private: |
386 | | const int64_t kSampleSize_; |
387 | | const std::string kNewValue_; |
388 | | }; |
389 | | |
390 | | // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer |
391 | | static const int64_t kSampleSize_ = 100; |
392 | | std::string dbname_; |
393 | | DBWithTTL* db_ttl_; |
394 | | unique_ptr<SpecialTimeEnv> env_; |
395 | | |
396 | | private: |
397 | | Options options_; |
398 | | KVMap kvmap_; |
399 | | KVMap::iterator kv_it_; |
400 | | const std::string kNewValue_ = "new_value"; |
401 | | unique_ptr<CompactionFilter> test_comp_filter_; |
402 | | bool read_only_ = false; |
403 | | }; // class TtlTest |
404 | | |
405 | | // If TTL is non positive or not provided, the behaviour is TTL = infinity |
406 | | // This test opens the db 3 times with such default behavior and inserts a |
407 | | // bunch of kvs each time. All kvs should accumulate in the db till the end |
408 | | // Partitions the sample-size provided into 3 sets over boundary1 and boundary2 |
409 | 1 | TEST_F(TtlTest, NoEffect) { |
410 | 1 | MakeKVMap(kSampleSize_); |
411 | 1 | int64_t boundary1 = kSampleSize_ / 3; |
412 | 1 | int64_t boundary2 = 2 * boundary1; |
413 | | |
414 | 1 | OpenTtl(); |
415 | 1 | PutValues(0, boundary1); // T=0: Set1 never deleted |
416 | 1 | SleepCompactCheck(1, 0, boundary1); // T=1: Set1 still there |
417 | 1 | CloseTtl(); |
418 | | |
419 | 1 | OpenTtl(0); |
420 | 1 | PutValues(boundary1, boundary2 - boundary1); // T=1: Set2 never deleted |
421 | 1 | SleepCompactCheck(1, 0, boundary2); // T=2: Sets1 & 2 still there |
422 | 1 | CloseTtl(); |
423 | | |
424 | 1 | OpenTtl(-1); |
425 | 1 | PutValues(boundary2, kSampleSize_ - boundary2); // T=3: Set3 never deleted |
426 | 1 | SleepCompactCheck(1, 0, kSampleSize_, true); // T=4: Sets 1,2,3 still there |
427 | 1 | CloseTtl(); |
428 | 1 | } |
429 | | |
430 | | // Puts a set of values and checks its presence using Get during ttl |
431 | 1 | TEST_F(TtlTest, PresentDuringTTL) { |
432 | 1 | MakeKVMap(kSampleSize_); |
433 | | |
434 | 1 | OpenTtl(2); // T=0:Open the db with ttl = 2 |
435 | 1 | PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=2 |
436 | 1 | SleepCompactCheck(1, 0, kSampleSize_, true); // T=1:Set1 should still be there |
437 | 1 | CloseTtl(); |
438 | 1 | } |
439 | | |
440 | | // Puts a set of values and checks its absence using Get after ttl |
441 | 1 | TEST_F(TtlTest, AbsentAfterTTL) { |
442 | 1 | MakeKVMap(kSampleSize_); |
443 | | |
444 | 1 | OpenTtl(1); // T=0:Open the db with ttl = 2 |
445 | 1 | PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=2 |
446 | 1 | SleepCompactCheck(2, 0, kSampleSize_, false); // T=2:Set1 should not be there |
447 | 1 | CloseTtl(); |
448 | 1 | } |
449 | | |
450 | | // Resets the timestamp of a set of kvs by updating them and checks that they |
451 | | // are not deleted according to the old timestamp |
452 | 1 | TEST_F(TtlTest, ResetTimestamp) { |
453 | 1 | MakeKVMap(kSampleSize_); |
454 | | |
455 | 1 | OpenTtl(3); |
456 | 1 | PutValues(0, kSampleSize_); // T=0: Insert Set1. Delete at t=3 |
457 | 1 | env_->Sleep(2); // T=2 |
458 | 1 | PutValues(0, kSampleSize_); // T=2: Insert Set1. Delete at t=5 |
459 | 1 | SleepCompactCheck(2, 0, kSampleSize_); // T=4: Set1 should still be there |
460 | 1 | CloseTtl(); |
461 | 1 | } |
462 | | |
463 | | // Similar to PresentDuringTTL but uses Iterator |
464 | 1 | TEST_F(TtlTest, IterPresentDuringTTL) { |
465 | 1 | MakeKVMap(kSampleSize_); |
466 | | |
467 | 1 | OpenTtl(2); |
468 | 1 | PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=2 |
469 | 1 | SleepCompactCheckIter(1, 0, kSampleSize_); // T=1: Set should be there |
470 | 1 | CloseTtl(); |
471 | 1 | } |
472 | | |
473 | | // Similar to AbsentAfterTTL but uses Iterator |
474 | 1 | TEST_F(TtlTest, IterAbsentAfterTTL) { |
475 | 1 | MakeKVMap(kSampleSize_); |
476 | | |
477 | 1 | OpenTtl(1); |
478 | 1 | PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1 |
479 | 1 | SleepCompactCheckIter(2, 0, kSampleSize_, false); // T=2: Should not be there |
480 | 1 | CloseTtl(); |
481 | 1 | } |
482 | | |
483 | | // Checks presence while opening the same db more than once with the same ttl |
484 | | // Note: The second open will open the same db |
485 | 1 | TEST_F(TtlTest, MultiOpenSamePresent) { |
486 | 1 | MakeKVMap(kSampleSize_); |
487 | | |
488 | 1 | OpenTtl(2); |
489 | 1 | PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=2 |
490 | 1 | CloseTtl(); |
491 | | |
492 | 1 | OpenTtl(2); // T=0. Delete at t=2 |
493 | 1 | SleepCompactCheck(1, 0, kSampleSize_); // T=1: Set should be there |
494 | 1 | CloseTtl(); |
495 | 1 | } |
496 | | |
497 | | // Checks absence while opening the same db more than once with the same ttl |
498 | | // Note: The second open will open the same db |
499 | 1 | TEST_F(TtlTest, MultiOpenSameAbsent) { |
500 | 1 | MakeKVMap(kSampleSize_); |
501 | | |
502 | 1 | OpenTtl(1); |
503 | 1 | PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1 |
504 | 1 | CloseTtl(); |
505 | | |
506 | 1 | OpenTtl(1); // T=0.Delete at t=1 |
507 | 1 | SleepCompactCheck(2, 0, kSampleSize_, false); // T=2: Set should not be there |
508 | 1 | CloseTtl(); |
509 | 1 | } |
510 | | |
511 | | // Checks presence while opening the same db more than once with bigger ttl |
512 | 1 | TEST_F(TtlTest, MultiOpenDifferent) { |
513 | 1 | MakeKVMap(kSampleSize_); |
514 | | |
515 | 1 | OpenTtl(1); |
516 | 1 | PutValues(0, kSampleSize_); // T=0: Insert. Delete at t=1 |
517 | 1 | CloseTtl(); |
518 | | |
519 | 1 | OpenTtl(3); // T=0: Set deleted at t=3 |
520 | 1 | SleepCompactCheck(2, 0, kSampleSize_); // T=2: Set should be there |
521 | 1 | CloseTtl(); |
522 | 1 | } |
523 | | |
524 | | // Checks presence during ttl in read_only mode |
525 | 1 | TEST_F(TtlTest, ReadOnlyPresentForever) { |
526 | 1 | MakeKVMap(kSampleSize_); |
527 | | |
528 | 1 | OpenTtl(1); // T=0:Open the db normally |
529 | 1 | PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=1 |
530 | 1 | CloseTtl(); |
531 | | |
532 | 1 | OpenReadOnlyTtl(1); |
533 | 1 | SleepCompactCheck(2, 0, kSampleSize_); // T=2:Set1 should still be there |
534 | 1 | CloseTtl(); |
535 | 1 | } |
536 | | |
537 | | // Checks whether WriteBatch works well with TTL |
538 | | // Puts all kvs in kvmap_ in a batch and writes first, then deletes first half |
539 | 1 | TEST_F(TtlTest, WriteBatchTest) { |
540 | 1 | MakeKVMap(kSampleSize_); |
541 | 1 | BatchOperation batch_ops[kSampleSize_]; |
542 | 101 | for (int i = 0; i < kSampleSize_; i++) { |
543 | 100 | batch_ops[i] = OP_PUT; |
544 | 100 | } |
545 | | |
546 | 1 | OpenTtl(2); |
547 | 1 | MakePutWriteBatch(batch_ops, kSampleSize_); |
548 | 51 | for (int i = 0; i < kSampleSize_ / 2; i++) { |
549 | 50 | batch_ops[i] = OP_DELETE; |
550 | 50 | } |
551 | 1 | MakePutWriteBatch(batch_ops, kSampleSize_ / 2); |
552 | 1 | SleepCompactCheck(0, 0, kSampleSize_ / 2, false); |
553 | 1 | SleepCompactCheck(0, kSampleSize_ / 2, kSampleSize_ - kSampleSize_ / 2); |
554 | 1 | CloseTtl(); |
555 | 1 | } |
556 | | |
557 | | // Checks user's compaction filter for correctness with TTL logic |
558 | 1 | TEST_F(TtlTest, CompactionFilter) { |
559 | 1 | MakeKVMap(kSampleSize_); |
560 | | |
561 | 1 | OpenTtlWithTestCompaction(1); |
562 | 1 | PutValues(0, kSampleSize_); // T=0:Insert Set1. Delete at t=1 |
563 | | // T=2: TTL logic takes precedence over TestFilter:-Set1 should not be there |
564 | 1 | SleepCompactCheck(2, 0, kSampleSize_, false); |
565 | 1 | CloseTtl(); |
566 | | |
567 | 1 | OpenTtlWithTestCompaction(3); |
568 | 1 | PutValues(0, kSampleSize_); // T=0:Insert Set1. |
569 | 1 | int64_t partition = kSampleSize_ / 3; |
570 | 1 | SleepCompactCheck(1, 0, partition, false); // Part dropped |
571 | 1 | SleepCompactCheck(0, partition, partition); // Part kept |
572 | 1 | SleepCompactCheck(0, 2 * partition, partition, true, true); // Part changed |
573 | 1 | CloseTtl(); |
574 | 1 | } |
575 | | |
576 | | // Insert some key-values which KeyMayExist should be able to get and check that |
577 | | // values returned are fine |
578 | 1 | TEST_F(TtlTest, KeyMayExist) { |
579 | 1 | MakeKVMap(kSampleSize_); |
580 | | |
581 | 1 | OpenTtl(); |
582 | 1 | PutValues(0, kSampleSize_, false); |
583 | | |
584 | 1 | SimpleKeyMayExistCheck(); |
585 | | |
586 | 1 | CloseTtl(); |
587 | 1 | } |
588 | | |
589 | 1 | TEST_F(TtlTest, MultiGetTest) { |
590 | 1 | MakeKVMap(kSampleSize_); |
591 | | |
592 | 1 | OpenTtl(); |
593 | 1 | PutValues(0, kSampleSize_, false); |
594 | | |
595 | 1 | SimpleMultiGetTest(); |
596 | | |
597 | 1 | CloseTtl(); |
598 | 1 | } |
599 | | |
600 | 1 | TEST_F(TtlTest, ColumnFamiliesTest) { |
601 | 1 | DB* db; |
602 | 1 | Options options; |
603 | 1 | options.create_if_missing = true; |
604 | 1 | options.env = env_.get(); |
605 | | |
606 | 1 | ASSERT_OK(DB::Open(options, dbname_, &db)); |
607 | 1 | ColumnFamilyHandle* handle; |
608 | 1 | ASSERT_OK(db->CreateColumnFamily(ColumnFamilyOptions(options), |
609 | 1 | "ttl_column_family", &handle)); |
610 | | |
611 | 1 | delete handle; |
612 | 1 | delete db; |
613 | | |
614 | 1 | std::vector<ColumnFamilyDescriptor> column_families; |
615 | 1 | column_families.push_back(ColumnFamilyDescriptor( |
616 | 1 | kDefaultColumnFamilyName, ColumnFamilyOptions(options))); |
617 | 1 | column_families.push_back(ColumnFamilyDescriptor( |
618 | 1 | "ttl_column_family", ColumnFamilyOptions(options))); |
619 | | |
620 | 1 | std::vector<ColumnFamilyHandle*> handles; |
621 | | |
622 | 1 | ASSERT_OK(DBWithTTL::Open(DBOptions(options), dbname_, column_families, |
623 | 1 | &handles, &db_ttl_, {3, 5}, false)); |
624 | 1 | ASSERT_EQ(handles.size(), 2U); |
625 | 1 | ColumnFamilyHandle* new_handle; |
626 | 1 | ASSERT_OK(db_ttl_->CreateColumnFamilyWithTtl(options, "ttl_column_family_2", |
627 | 1 | &new_handle, 2)); |
628 | 1 | handles.push_back(new_handle); |
629 | | |
630 | 1 | MakeKVMap(kSampleSize_); |
631 | 1 | PutValues(0, kSampleSize_, false, handles[0]); |
632 | 1 | PutValues(0, kSampleSize_, false, handles[1]); |
633 | 1 | PutValues(0, kSampleSize_, false, handles[2]); |
634 | | |
635 | | // everything should be there after 1 second |
636 | 1 | SleepCompactCheck(1, 0, kSampleSize_, true, false, handles[0]); |
637 | 1 | SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]); |
638 | 1 | SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[2]); |
639 | | |
640 | | // only column family 1 should be alive after 4 seconds |
641 | 1 | SleepCompactCheck(3, 0, kSampleSize_, false, false, handles[0]); |
642 | 1 | SleepCompactCheck(0, 0, kSampleSize_, true, false, handles[1]); |
643 | 1 | SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]); |
644 | | |
645 | | // nothing should be there after 6 seconds |
646 | 1 | SleepCompactCheck(2, 0, kSampleSize_, false, false, handles[0]); |
647 | 1 | SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[1]); |
648 | 1 | SleepCompactCheck(0, 0, kSampleSize_, false, false, handles[2]); |
649 | | |
650 | 3 | for (auto h : handles) { |
651 | 3 | delete h; |
652 | 3 | } |
653 | 1 | delete db_ttl_; |
654 | 1 | db_ttl_ = nullptr; |
655 | 1 | } |
656 | | |
657 | | } // namespace rocksdb |
658 | | |
659 | | // A black-box test for the ttl wrapper around rocksdb |
660 | 13.2k | int main(int argc, char** argv) { |
661 | 13.2k | ::testing::InitGoogleTest(&argc, argv); |
662 | 13.2k | return RUN_ALL_TESTS(); |
663 | 13.2k | } |
664 | | |
665 | | #else |
666 | | #include <stdio.h> |
667 | | |
668 | | int main(int argc, char** argv) { |
669 | | fprintf(stderr, "SKIPPED as DBWithTTL is not supported in ROCKSDB_LITE\n"); |
670 | | return 0; |
671 | | } |
672 | | |
673 | | #endif // !ROCKSDB_LITE |