/Users/deen/code/yugabyte-db/src/yb/docdb/in_mem_docdb.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) YugaByte, Inc. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
4 | | // in compliance with the License. You may obtain a copy of the License at |
5 | | // |
6 | | // http://www.apache.org/licenses/LICENSE-2.0 |
7 | | // |
8 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
9 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
10 | | // or implied. See the License for the specific language governing permissions and limitations |
11 | | // under the License. |
12 | | // |
13 | | |
14 | | #include "yb/docdb/in_mem_docdb.h" |
15 | | |
16 | | #include <sstream> |
17 | | |
18 | | #include "yb/common/hybrid_time.h" |
19 | | |
20 | | #include "yb/docdb/doc_key.h" |
21 | | #include "yb/docdb/doc_reader.h" |
22 | | #include "yb/docdb/docdb.h" |
23 | | #include "yb/docdb/docdb_rocksdb_util.h" |
24 | | #include "yb/docdb/docdb_test_util.h" |
25 | | |
26 | | #include "yb/gutil/strings/substitute.h" |
27 | | |
28 | | #include "yb/rocksdb/db.h" |
29 | | |
30 | | #include "yb/util/status_format.h" |
31 | | #include "yb/util/status_log.h" |
32 | | #include "yb/util/test_macros.h" |
33 | | |
34 | | using std::endl; |
35 | | using std::string; |
36 | | using std::stringstream; |
37 | | using strings::Substitute; |
38 | | |
39 | | namespace yb { |
40 | | namespace docdb { |
41 | | |
42 | 228k | Status InMemDocDbState::SetPrimitive(const DocPath& doc_path, const PrimitiveValue& value) { |
43 | 228k | VLOG(2) << __func__ << ": doc_path=" << doc_path.ToString() << ", value=" << value.ToString()0 ; |
44 | 228k | const PrimitiveValue encoded_doc_key_as_primitive(doc_path.encoded_doc_key().AsSlice()); |
45 | 228k | const bool is_deletion = value.value_type() == ValueType::kTombstone; |
46 | 228k | if (doc_path.num_subkeys() == 0) { |
47 | 41.6k | if (is_deletion) { |
48 | 20.8k | root_.DeleteChild(encoded_doc_key_as_primitive); |
49 | 20.8k | } else { |
50 | 20.7k | root_.SetChildPrimitive(encoded_doc_key_as_primitive, value); |
51 | 20.7k | } |
52 | | |
53 | 41.6k | return Status::OK(); |
54 | 41.6k | } |
55 | 186k | SubDocument* current_subdoc = nullptr; |
56 | | |
57 | 186k | if (is_deletion) { |
58 | 1.81k | current_subdoc = root_.GetChild(encoded_doc_key_as_primitive); |
59 | 1.81k | if (current_subdoc == nullptr) { |
60 | | // The subdocument we're trying to delete does not exist, nothing to do. |
61 | 219 | return Status::OK(); |
62 | 219 | } |
63 | 185k | } else { |
64 | 185k | current_subdoc = root_.GetOrAddChild(encoded_doc_key_as_primitive).first; |
65 | 185k | } |
66 | | |
67 | 186k | const auto num_subkeys = doc_path.num_subkeys(); |
68 | 485k | for (size_t subkey_index = 0; subkey_index < num_subkeys - 1; ++subkey_index299k ) { |
69 | 299k | const PrimitiveValue& subkey = doc_path.subkey(subkey_index); |
70 | 299k | if (subkey.value_type() == ValueType::kArrayIndex) { |
71 | 0 | return STATUS(NotSupported, "Setting values at a given array index is not supported yet."); |
72 | 0 | } |
73 | | |
74 | 299k | if (current_subdoc->value_type() != ValueType::kObject) { |
75 | 0 | return STATUS_FORMAT(IllegalState, |
76 | 0 | "Cannot set or delete values inside a subdocument of type $0", |
77 | 0 | current_subdoc->value_type()); |
78 | 0 | } |
79 | | |
80 | 299k | if (is_deletion) { |
81 | 1.35k | current_subdoc = current_subdoc->GetChild(subkey); |
82 | 1.35k | if (current_subdoc == nullptr) { |
83 | | // Document does not exist, nothing to do. |
84 | 882 | return Status::OK(); |
85 | 882 | } |
86 | 298k | } else { |
87 | 298k | current_subdoc = current_subdoc->GetOrAddChild(subkey).first; |
88 | 298k | } |
89 | 299k | } |
90 | | |
91 | 185k | if (is_deletion) { |
92 | 717 | current_subdoc->DeleteChild(doc_path.last_subkey()); |
93 | 185k | } else { |
94 | 185k | current_subdoc->SetChildPrimitive(doc_path.last_subkey(), value); |
95 | 185k | } |
96 | | |
97 | 185k | return Status::OK(); |
98 | 186k | } |
99 | | |
100 | 22.7k | Status InMemDocDbState::DeleteSubDoc(const DocPath &doc_path) { |
101 | 22.7k | return SetPrimitive(doc_path, PrimitiveValue::kTombstone); |
102 | 22.7k | } |
103 | | |
104 | 244k | void InMemDocDbState::SetDocument(const KeyBytes& encoded_doc_key, SubDocument&& doc) { |
105 | 244k | root_.SetChild(PrimitiveValue(encoded_doc_key.AsSlice()), std::move(doc)); |
106 | 244k | } |
107 | | |
108 | 1.07M | const SubDocument* InMemDocDbState::GetSubDocument(const SubDocKey& subdoc_key) const { |
109 | 1.07M | const SubDocument* current = |
110 | 1.07M | root_.GetChild(PrimitiveValue(subdoc_key.doc_key().Encode().AsSlice())); |
111 | 1.07M | for (const auto& subkey : subdoc_key.subkeys()) { |
112 | 0 | if (current == nullptr) { |
113 | 0 | return nullptr; |
114 | 0 | } |
115 | 0 | current = current->GetChild(subkey); |
116 | 0 | } |
117 | 1.07M | return current; |
118 | 1.07M | } |
119 | | |
120 | | void InMemDocDbState::CaptureAt(const DocDB& doc_db, HybridTime hybrid_time, |
121 | 6.49k | rocksdb::QueryId query_id) { |
122 | | // Clear the internal state. |
123 | 6.49k | root_ = SubDocument(); |
124 | | |
125 | 6.49k | auto rocksdb_iter = CreateRocksDBIterator( |
126 | 6.49k | doc_db.regular, doc_db.key_bounds, BloomFilterMode::DONT_USE_BLOOM_FILTER, |
127 | 6.49k | boost::none /* user_key_for_filter */, query_id); |
128 | 6.49k | rocksdb_iter.SeekToFirst(); |
129 | 6.49k | KeyBytes prev_key; |
130 | 294k | while (rocksdb_iter.Valid()) { |
131 | 287k | const auto key = rocksdb_iter.key(); |
132 | 287k | CHECK_NE(0, prev_key.CompareTo(key)) << "Infinite loop detected on key " << prev_key.ToString()0 ; |
133 | 287k | prev_key = KeyBytes(key); |
134 | | |
135 | 287k | SubDocKey subdoc_key; |
136 | 287k | CHECK_OK(subdoc_key.FullyDecodeFrom(key)); |
137 | 287k | CHECK_EQ(0, subdoc_key.num_subkeys()) |
138 | 0 | << "Expected to be positioned at the first key of a new document with no subkeys, " |
139 | 0 | << "but found " << subdoc_key.num_subkeys() << " subkeys: " << subdoc_key.ToString(); |
140 | 287k | subdoc_key.remove_hybrid_time(); |
141 | | |
142 | | // TODO: It would be good to be able to refer to a slice of the original key whenever we need |
143 | | // to extract document key out of a subdocument key. |
144 | 287k | auto encoded_doc_key = subdoc_key.doc_key().Encode(); |
145 | | // TODO(dtxn) Pass real TransactionOperationContext when we need to support cross-shard |
146 | | // transactions write intents resolution during DocDbState capturing. |
147 | | // For now passing kNonTransactionalOperationContext in order to fail if there are any intents, |
148 | | // since this is not supported. |
149 | 287k | auto encoded_subdoc_key = subdoc_key.EncodeWithoutHt(); |
150 | 287k | auto doc_from_rocksdb_opt = ASSERT_RESULT(yb::docdb::TEST_GetSubDocument( |
151 | 287k | encoded_subdoc_key, doc_db, query_id, kNonTransactionalOperationContext, |
152 | 287k | CoarseTimePoint::max() /* deadline */, ReadHybridTime::SingleTime(hybrid_time))); |
153 | | // doc_found can be false for deleted documents, and that is perfectly valid. |
154 | 287k | if (doc_from_rocksdb_opt) { |
155 | 244k | SetDocument(encoded_doc_key, std::move(*doc_from_rocksdb_opt)); |
156 | 244k | } |
157 | | // Go to the next top-level document key. |
158 | 287k | ROCKSDB_SEEK(&rocksdb_iter, subdoc_key.AdvanceOutOfSubDoc().AsSlice()); |
159 | | |
160 | 287k | VLOG(4) << "After performing a seek: IsValid=" << rocksdb_iter.Valid()0 ; |
161 | 287k | if (VLOG_IS_ON(4) && rocksdb_iter.Valid()0 ) { |
162 | 0 | VLOG(4) << "Next key: " << FormatSliceAsStr(rocksdb_iter.key()); |
163 | 0 | SubDocKey tmp_subdoc_key; |
164 | 0 | CHECK_OK(tmp_subdoc_key.FullyDecodeFrom(rocksdb_iter.key())); |
165 | 0 | VLOG(4) << "Parsed as SubDocKey: " << tmp_subdoc_key.ToString(); |
166 | 0 | } |
167 | 287k | } |
168 | | |
169 | | // Initialize the "captured at" hybrid_time now, even though we expect it to be overwritten in |
170 | | // many cases. One common usage pattern is that this will be called with HybridTime::kMax, but |
171 | | // we'll later call SetCaptureHybridTime and set the hybrid_time to the last known hybrid_time of |
172 | | // an operation performed on DocDB. |
173 | 6.49k | captured_at_ = hybrid_time; |
174 | | |
175 | | // Ensure we don't get any funny value types in the root node (had a test failure like this). |
176 | 6.49k | CHECK_EQ(root_.value_type(), ValueType::kObject); |
177 | 6.49k | } |
178 | | |
179 | 1.29k | void InMemDocDbState::SetCaptureHybridTime(HybridTime hybrid_time) { |
180 | 1.29k | CHECK(hybrid_time.is_valid()); |
181 | 1.29k | captured_at_ = hybrid_time; |
182 | 1.29k | } |
183 | | |
184 | 5.15k | bool InMemDocDbState::EqualsAndLogDiff(const InMemDocDbState &expected, bool log_diff) { |
185 | 5.15k | bool matches = true; |
186 | 5.15k | if (num_docs() != expected.num_docs()) { |
187 | 26 | if (log_diff) { |
188 | 0 | LOG(WARNING) << "Found " << num_docs() << " documents but expected to find " |
189 | 0 | << expected.num_docs(); |
190 | 0 | } |
191 | 26 | matches = false; |
192 | 26 | } |
193 | | |
194 | | // As an optimization, a SubDocument won't even maintain a map if it is an empty object that no |
195 | | // operations have been performed on. As we are using a SubDocument to represent the top-level |
196 | | // mapping of encoded document keys to SubDocuments here, we need to check for that situation. |
197 | 5.15k | if (expected.root_.has_valid_object_container()) { |
198 | 193k | for (const auto& expected_kv : expected.root_.object_container()) { |
199 | 193k | const KeyBytes encoded_doc_key(expected_kv.first.GetString()); |
200 | 193k | const SubDocument& expected_doc = expected_kv.second; |
201 | 193k | DocKey doc_key; |
202 | 193k | CHECK_OK(doc_key.FullyDecodeFrom(encoded_doc_key.AsSlice())); |
203 | 193k | const SubDocument* child_from_this = GetDocument(doc_key); |
204 | 193k | if (child_from_this == nullptr) { |
205 | 577 | if (log_diff) { |
206 | 0 | LOG(WARNING) << "Document with key " << doc_key.ToString() << " is missing but is " |
207 | 0 | << "expected to be " << expected_doc.ToString(); |
208 | 0 | } |
209 | 577 | matches = false; |
210 | 577 | continue; |
211 | 577 | } |
212 | 192k | if (*child_from_this != expected_kv.second) { |
213 | 248 | if (log_diff) { |
214 | 0 | LOG(WARNING) << "Expected document with key " << doc_key.ToString() << " to be " |
215 | 0 | << expected_doc.ToString() << " but found " << *child_from_this; |
216 | 0 | } |
217 | 248 | matches = false; |
218 | 248 | } |
219 | 192k | } |
220 | 5.15k | } |
221 | | |
222 | | // Also report all document keys that are present in this ("actual") database but are absent from |
223 | | // the other ("expected") database. |
224 | 5.15k | if (root_.has_valid_object_container()) { |
225 | 192k | for (const auto& actual_kv : root_.object_container()) { |
226 | 192k | const KeyBytes encoded_doc_key(actual_kv.first.GetString()); |
227 | 192k | DocKey doc_key; |
228 | 192k | CHECK_OK(doc_key.FullyDecodeFrom(encoded_doc_key.AsSlice())); |
229 | 192k | const SubDocument* child_from_expected = GetDocument(doc_key); |
230 | 192k | if (child_from_expected == nullptr) { |
231 | 0 | DocKey doc_key; |
232 | 0 | CHECK_OK(doc_key.FullyDecodeFrom(encoded_doc_key.AsSlice())); |
233 | 0 | if (log_diff) { |
234 | 0 | LOG(WARNING) << "Unexpected document found with key " << doc_key.ToString() << ":" |
235 | 0 | << actual_kv.second.ToString(); |
236 | 0 | } |
237 | 0 | matches = false; |
238 | 0 | } |
239 | 192k | } |
240 | 5.15k | } |
241 | | |
242 | | // A brute-force way to check that the comparison logic above is correct. |
243 | | // TODO: disable this if it makes tests much slower. |
244 | 5.15k | CHECK_EQ(matches, ToDebugString() == expected.ToDebugString()); |
245 | 5.15k | return matches; |
246 | 5.15k | } |
247 | | |
248 | 10.3k | string InMemDocDbState::ToDebugString() const { |
249 | 10.3k | stringstream ss; |
250 | 10.3k | if (root_.has_valid_object_container()) { |
251 | 10.3k | int i = 1; |
252 | 386k | for (const auto& kv : root_.object_container()) { |
253 | 386k | DocKey doc_key; |
254 | 386k | CHECK_OK(doc_key.FullyDecodeFrom(rocksdb::Slice(kv.first.GetString()))); |
255 | 386k | ss << i << ". " << doc_key.ToString() << " => " << kv.second.ToString() << endl; |
256 | 386k | ++i; |
257 | 386k | } |
258 | 10.3k | } |
259 | 10.3k | string dump_str = ss.str(); |
260 | 10.3k | return dump_str.empty() ? "<Empty>"0 : dump_str; |
261 | 10.3k | } |
262 | | |
263 | 9.27k | HybridTime InMemDocDbState::captured_at() const { |
264 | 9.27k | CHECK(captured_at_.is_valid()); |
265 | 9.27k | return captured_at_; |
266 | 9.27k | } |
267 | | |
268 | 588 | void InMemDocDbState::SanityCheck() const { |
269 | 588 | CHECK_EQ(root_.value_type(), ValueType::kObject); |
270 | 588 | } |
271 | | |
272 | 1.07M | const SubDocument* InMemDocDbState::GetDocument(const DocKey& doc_key) const { |
273 | 1.07M | return GetSubDocument(SubDocKey(doc_key)); |
274 | 1.07M | } |
275 | | |
276 | | } // namespace docdb |
277 | | } // namespace yb |