/Users/deen/code/yugabyte-db/src/yb/docdb/docdb_util.h
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) YugaByte, Inc. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
4 | | // in compliance with the License. You may obtain a copy of the License at |
5 | | // |
6 | | // http://www.apache.org/licenses/LICENSE-2.0 |
7 | | // |
8 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
9 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
10 | | // or implied. See the License for the specific language governing permissions and limitations |
11 | | // under the License. |
12 | | // |
13 | | |
14 | | // Utilities for docdb operations. |
15 | | |
16 | | #ifndef YB_DOCDB_DOCDB_UTIL_H |
17 | | #define YB_DOCDB_DOCDB_UTIL_H |
18 | | |
19 | | #include "yb/common/schema.h" |
20 | | |
21 | | #include "yb/docdb/docdb_fwd.h" |
22 | | #include "yb/docdb/doc_path.h" |
23 | | #include "yb/docdb/shared_lock_manager_fwd.h" |
24 | | #include "yb/docdb/doc_write_batch.h" |
25 | | #include "yb/docdb/docdb_compaction_filter.h" |
26 | | #include "yb/rocksdb/compaction_filter.h" |
27 | | |
28 | | namespace yb { |
29 | | namespace docdb { |
30 | | |
31 | | void SetValueFromQLBinaryWrapper( |
32 | | QLValuePB ql_value, |
33 | | const int pg_data_type, |
34 | | DatumMessagePB* cdc_datum_message = NULL); |
35 | | |
36 | | struct ExternalIntent { |
37 | | DocPath doc_path; |
38 | | Value value; |
39 | | }; |
40 | | |
41 | | // A wrapper around a RocksDB instance and provides utility functions on top of it, such as |
42 | | // compacting the history until a certain point. This is used in the bulk load tool. This is also |
43 | | // convenient base class for GTest test classes, because it exposes member functions such as |
44 | | // rocksdb() and write_options(). |
45 | | class DocDBRocksDBUtil { |
46 | | |
47 | | public: |
48 | 0 | DocDBRocksDBUtil() {} |
49 | | explicit DocDBRocksDBUtil(InitMarkerBehavior init_marker_behavior) |
50 | 0 | : init_marker_behavior_(init_marker_behavior) { |
51 | 0 | } |
52 | | |
53 | 0 | virtual ~DocDBRocksDBUtil() {} |
54 | | virtual CHECKED_STATUS InitRocksDBDir() = 0; |
55 | | |
56 | | // Initializes RocksDB options, should be called after constructor, because it uses virtual |
57 | | // function BlockCacheSize. |
58 | | virtual CHECKED_STATUS InitRocksDBOptions() = 0; |
59 | | virtual std::string tablet_id() = 0; |
60 | | |
61 | | // Size of block cache for RocksDB, 0 means don't use block cache. |
62 | 0 | virtual size_t block_cache_size() const { return 16 * 1024 * 1024; } |
63 | | |
64 | | rocksdb::DB* rocksdb(); |
65 | | rocksdb::DB* intents_db(); |
66 | 0 | DocDB doc_db() { return { rocksdb(), intents_db(), &KeyBounds::kNoBounds }; } |
67 | | |
68 | | CHECKED_STATUS InitCommonRocksDBOptionsForTests(); |
69 | | CHECKED_STATUS InitCommonRocksDBOptionsForBulkLoad(); |
70 | | |
71 | 0 | const rocksdb::WriteOptions& write_options() const { return write_options_; } |
72 | | |
73 | 0 | const rocksdb::Options& regular_db_options() const { return regular_db_options_; } |
74 | | |
75 | | CHECKED_STATUS OpenRocksDB(); |
76 | | |
77 | | void CloseRocksDB(); |
78 | | CHECKED_STATUS ReopenRocksDB(); |
79 | | CHECKED_STATUS DestroyRocksDB(); |
80 | | void ResetMonotonicCounter(); |
81 | | |
82 | | // Populates the given RocksDB write batch from the given DocWriteBatch. If a valid hybrid_time is |
83 | | // specified, it is appended to every key. |
84 | | CHECKED_STATUS PopulateRocksDBWriteBatch( |
85 | | const DocWriteBatch& dwb, |
86 | | rocksdb::WriteBatch *rocksdb_write_batch, |
87 | | HybridTime hybrid_time = HybridTime::kInvalid, |
88 | | bool decode_dockey = true, |
89 | | bool increment_write_id = true, |
90 | | PartialRangeKeyIntents partial_range_key_intents = PartialRangeKeyIntents::kTrue) const; |
91 | | |
92 | | // Writes the given DocWriteBatch to RocksDB. We substitue the hybrid time, if provided. |
93 | | CHECKED_STATUS WriteToRocksDB( |
94 | | const DocWriteBatch& write_batch, |
95 | | const HybridTime& hybrid_time, |
96 | | bool decode_dockey = true, |
97 | | bool increment_write_id = true, |
98 | | PartialRangeKeyIntents partial_range_key_intents = PartialRangeKeyIntents::kTrue); |
99 | | |
100 | | // The same as WriteToRocksDB but also clears the write batch afterwards. |
101 | | CHECKED_STATUS WriteToRocksDBAndClear(DocWriteBatch* dwb, const HybridTime& hybrid_time, |
102 | | bool decode_dockey = true, bool increment_write_id = true); |
103 | | |
104 | | // Writes value fully determined by its index using DefaultWriteBatch. |
105 | | CHECKED_STATUS WriteSimple(int index); |
106 | | |
107 | | void SetHistoryCutoffHybridTime(HybridTime history_cutoff); |
108 | | |
109 | | // Produces a string listing the contents of the entire RocksDB database, with every key and value |
110 | | // decoded as a DocDB key/value and converted to a human-readable string representation. |
111 | | std::string DocDBDebugDumpToStr(); |
112 | | |
113 | | // ---------------------------------------------------------------------------------------------- |
114 | | // SetPrimitive taking a Value |
115 | | |
116 | | CHECKED_STATUS SetPrimitive( |
117 | | const DocPath& doc_path, |
118 | | const Value& value, |
119 | | HybridTime hybrid_time, |
120 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
121 | | |
122 | | CHECKED_STATUS SetPrimitive( |
123 | | const DocPath& doc_path, |
124 | | const PrimitiveValue& value, |
125 | | HybridTime hybrid_time, |
126 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
127 | | |
128 | | CHECKED_STATUS AddExternalIntents( |
129 | | const TransactionId& txn_id, |
130 | | const std::vector<ExternalIntent>& intents, |
131 | | const Uuid& involved_tablet, |
132 | | HybridTime hybrid_time); |
133 | | |
134 | | CHECKED_STATUS InsertSubDocument( |
135 | | const DocPath& doc_path, |
136 | | const SubDocument& value, |
137 | | HybridTime hybrid_time, |
138 | | MonoDelta ttl = Value::kMaxTtl, |
139 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
140 | | |
141 | | CHECKED_STATUS ExtendSubDocument( |
142 | | const DocPath& doc_path, |
143 | | const SubDocument& value, |
144 | | HybridTime hybrid_time, |
145 | | MonoDelta ttl = Value::kMaxTtl, |
146 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
147 | | |
148 | | CHECKED_STATUS ExtendList( |
149 | | const DocPath& doc_path, |
150 | | const SubDocument& value, |
151 | | HybridTime hybrid_time, |
152 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
153 | | |
154 | | CHECKED_STATUS ReplaceInList( |
155 | | const DocPath &doc_path, |
156 | | const int target_cql_index, |
157 | | const SubDocument& value, |
158 | | const ReadHybridTime& read_ht, |
159 | | const HybridTime& hybrid_time, |
160 | | const rocksdb::QueryId query_id, |
161 | | MonoDelta default_ttl = Value::kMaxTtl, |
162 | | MonoDelta ttl = Value::kMaxTtl, |
163 | | UserTimeMicros user_timestamp = Value::kInvalidUserTimestamp); |
164 | | |
165 | | CHECKED_STATUS DeleteSubDoc( |
166 | | const DocPath& doc_path, |
167 | | HybridTime hybrid_time, |
168 | | const ReadHybridTime& read_ht = ReadHybridTime::Max()); |
169 | | |
170 | | void DocDBDebugDumpToConsole(); |
171 | | |
172 | | CHECKED_STATUS FlushRocksDbAndWait(); |
173 | | |
174 | | void SetTableTTL(uint64_t ttl_msec); |
175 | | |
176 | 0 | void SetCurrentTransactionId(const TransactionId& txn_id) { |
177 | 0 | current_txn_id_ = txn_id; |
178 | 0 | } |
179 | | |
180 | 0 | void SetTransactionIsolationLevel(IsolationLevel isolation_level) { |
181 | 0 | txn_isolation_level_ = isolation_level; |
182 | 0 | } |
183 | | |
184 | 0 | void ResetCurrentTransactionId() { |
185 | 0 | current_txn_id_.reset(); |
186 | 0 | } |
187 | | |
188 | 0 | CHECKED_STATUS DisableCompactions() { |
189 | 0 | regular_db_options_.compaction_style = rocksdb::kCompactionStyleNone; |
190 | 0 | intents_db_options_.compaction_style = rocksdb::kCompactionStyleNone; |
191 | 0 | return ReopenRocksDB(); |
192 | 0 | } |
193 | | |
194 | | CHECKED_STATUS ReinitDBOptions(); |
195 | | |
196 | 0 | std::atomic<int64_t>& monotonic_counter() { |
197 | 0 | return monotonic_counter_; |
198 | 0 | } |
199 | | |
200 | | DocWriteBatch MakeDocWriteBatch(); |
201 | | DocWriteBatch MakeDocWriteBatch(InitMarkerBehavior init_marker_behavior); |
202 | | |
203 | | void SetInitMarkerBehavior(InitMarkerBehavior init_marker_behavior); |
204 | | |
205 | | // Returns DocWriteBatch created with MakeDocWriteBatch. |
206 | | // Could be used when single DocWriteBatch is enough. |
207 | | DocWriteBatch& DefaultDocWriteBatch(); |
208 | | |
209 | | protected: |
210 | | std::string IntentsDBDir(); |
211 | | |
212 | | std::unique_ptr<rocksdb::DB> regular_db_; |
213 | | std::unique_ptr<rocksdb::DB> intents_db_; |
214 | | rocksdb::Options regular_db_options_; |
215 | | rocksdb::Options intents_db_options_; |
216 | | std::string rocksdb_dir_; |
217 | | |
218 | | // This is used for auto-assigning op ids to RocksDB write batches to emulate what a tablet would |
219 | | // do in production. |
220 | | rocksdb::OpId op_id_; |
221 | | |
222 | | std::shared_ptr<rocksdb::Cache> block_cache_; |
223 | | std::shared_ptr<ManualHistoryRetentionPolicy> retention_policy_ { |
224 | | std::make_shared<ManualHistoryRetentionPolicy>() }; |
225 | | std::shared_ptr<rocksdb::CompactionFileFilterFactory> compaction_file_filter_factory_; |
226 | | std::shared_ptr<std::function<uint64_t()>> max_file_size_for_compaction_; |
227 | | |
228 | | rocksdb::WriteOptions write_options_; |
229 | | Schema schema_; |
230 | | boost::optional<TransactionId> current_txn_id_; |
231 | | mutable IntraTxnWriteId intra_txn_write_id_ = 0; |
232 | | IsolationLevel txn_isolation_level_ = IsolationLevel::NON_TRANSACTIONAL; |
233 | | InitMarkerBehavior init_marker_behavior_ = InitMarkerBehavior::kOptional; |
234 | | |
235 | | private: |
236 | | std::atomic<int64_t> monotonic_counter_{0}; |
237 | | boost::optional<DocWriteBatch> doc_write_batch_; |
238 | | }; |
239 | | |
240 | | } // namespace docdb |
241 | | } // namespace yb |
242 | | |
243 | | #endif // YB_DOCDB_DOCDB_UTIL_H |