/Users/deen/code/yugabyte-db/src/yb/integration-tests/tablet-split-itest-base.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) YugaByte, Inc. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
4 | | // in compliance with the License. You may obtain a copy of the License at |
5 | | // |
6 | | // http://www.apache.org/licenses/LICENSE-2.0 |
7 | | // |
8 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
9 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
10 | | // or implied. See the License for the specific language governing permissions and limitations |
11 | | // under the License. |
12 | | // |
13 | | |
14 | | #include "yb/integration-tests/tablet-split-itest-base.h" |
15 | | |
16 | | #include <signal.h> |
17 | | |
18 | | #include <boost/range/adaptor/transformed.hpp> |
19 | | |
20 | | #include "yb/client/client-test-util.h" |
21 | | #include "yb/client/session.h" |
22 | | #include "yb/client/snapshot_test_util.h" |
23 | | #include "yb/client/table_info.h" |
24 | | #include "yb/client/transaction.h" |
25 | | #include "yb/client/yb_op.h" |
26 | | |
27 | | #include "yb/common/schema.h" |
28 | | #include "yb/common/ql_expr.h" |
29 | | #include "yb/common/ql_value.h" |
30 | | #include "yb/common/wire_protocol.h" |
31 | | |
32 | | #include "yb/consensus/consensus.h" |
33 | | #include "yb/consensus/consensus_util.h" |
34 | | |
35 | | #include "yb/docdb/doc_key.h" |
36 | | #include "yb/docdb/ql_rowwise_iterator_interface.h" |
37 | | |
38 | | #include "yb/integration-tests/mini_cluster.h" |
39 | | #include "yb/integration-tests/test_workload.h" |
40 | | |
41 | | #include "yb/master/catalog_entity_info.h" |
42 | | #include "yb/master/master_admin.proxy.h" |
43 | | #include "yb/master/master_client.pb.h" |
44 | | |
45 | | #include "yb/rocksdb/db.h" |
46 | | |
47 | | #include "yb/rpc/messenger.h" |
48 | | |
49 | | #include "yb/tablet/tablet.h" |
50 | | #include "yb/tablet/tablet_metadata.h" |
51 | | #include "yb/tablet/tablet_peer.h" |
52 | | |
53 | | #include "yb/tserver/mini_tablet_server.h" |
54 | | #include "yb/tserver/tserver_service.pb.h" |
55 | | #include "yb/tserver/tserver_service.proxy.h" |
56 | | |
57 | | #include "yb/yql/cql/ql/util/statement_result.h" |
58 | | |
59 | | DECLARE_int32(cleanup_split_tablets_interval_sec); |
60 | | DECLARE_int64(db_block_size_bytes); |
61 | | DECLARE_int64(db_filter_block_size_bytes); |
62 | | DECLARE_int64(db_index_block_size_bytes); |
63 | | DECLARE_int64(db_write_buffer_size); |
64 | | DECLARE_bool(enable_automatic_tablet_splitting); |
65 | | DECLARE_int32(raft_heartbeat_interval_ms); |
66 | | DECLARE_int32(replication_factor); |
67 | | DECLARE_int32(tserver_heartbeat_metrics_interval_ms); |
68 | | DECLARE_bool(TEST_do_not_start_election_test_only); |
69 | | DECLARE_bool(TEST_skip_deleting_split_tablets); |
70 | | DECLARE_bool(TEST_validate_all_tablet_candidates); |
71 | | |
72 | | namespace yb { |
73 | | |
74 | | Result<size_t> SelectRowsCount( |
75 | 0 | const client::YBSessionPtr& session, const client::TableHandle& table) { |
76 | 0 | LOG(INFO) << "Running full scan on test table..."; |
77 | 0 | session->SetTimeout(5s * kTimeMultiplier); |
78 | 0 | QLPagingStatePB paging_state; |
79 | 0 | size_t row_count = 0; |
80 | 0 | for (;;) { |
81 | 0 | const auto op = table.NewReadOp(); |
82 | 0 | auto* const req = op->mutable_request(); |
83 | 0 | req->set_return_paging_state(true); |
84 | 0 | if (paging_state.has_table_id()) { |
85 | 0 | if (paging_state.has_read_time()) { |
86 | 0 | ReadHybridTime read_time = ReadHybridTime::FromPB(paging_state.read_time()); |
87 | 0 | if (read_time) { |
88 | 0 | session->SetReadPoint(read_time); |
89 | 0 | } |
90 | 0 | } |
91 | 0 | session->SetForceConsistentRead(client::ForceConsistentRead::kTrue); |
92 | 0 | *req->mutable_paging_state() = std::move(paging_state); |
93 | 0 | } |
94 | 0 | RETURN_NOT_OK(session->ApplyAndFlush(op)); |
95 | 0 | auto rowblock = ql::RowsResult(op.get()).GetRowBlock(); |
96 | 0 | row_count += rowblock->row_count(); |
97 | 0 | if (!op->response().has_paging_state()) { |
98 | 0 | break; |
99 | 0 | } |
100 | 0 | paging_state = op->response().paging_state(); |
101 | 0 | } |
102 | 0 | return row_count; |
103 | 0 | } |
104 | | |
105 | | void DumpTableLocations( |
106 | 0 | master::CatalogManagerIf* catalog_mgr, const client::YBTableName& table_name) { |
107 | 0 | master::GetTableLocationsResponsePB resp; |
108 | 0 | master::GetTableLocationsRequestPB req; |
109 | 0 | table_name.SetIntoTableIdentifierPB(req.mutable_table()); |
110 | 0 | req.set_max_returned_locations(std::numeric_limits<int32_t>::max()); |
111 | 0 | ASSERT_OK(catalog_mgr->GetTableLocations(&req, &resp)); |
112 | 0 | LOG(INFO) << "Table locations:"; |
113 | 0 | for (auto& tablet : resp.tablet_locations()) { |
114 | 0 | LOG(INFO) << "Tablet: " << tablet.tablet_id() |
115 | 0 | << " partition: " << tablet.partition().ShortDebugString(); |
116 | 0 | } |
117 | 0 | } |
118 | | |
119 | 0 | void DumpWorkloadStats(const TestWorkload& workload) { |
120 | 0 | LOG(INFO) << "Rows inserted: " << workload.rows_inserted(); |
121 | 0 | LOG(INFO) << "Rows insert failed: " << workload.rows_insert_failed(); |
122 | 0 | LOG(INFO) << "Rows read ok: " << workload.rows_read_ok(); |
123 | 0 | LOG(INFO) << "Rows read empty: " << workload.rows_read_empty(); |
124 | 0 | LOG(INFO) << "Rows read error: " << workload.rows_read_error(); |
125 | 0 | LOG(INFO) << "Rows read try again: " << workload.rows_read_try_again(); |
126 | 0 | } |
127 | | |
128 | 0 | Status SplitTablet(master::CatalogManagerIf* catalog_mgr, const tablet::Tablet& tablet) { |
129 | 0 | const auto& tablet_id = tablet.tablet_id(); |
130 | 0 | LOG(INFO) << "Tablet: " << tablet_id; |
131 | 0 | LOG(INFO) << "Number of SST files: " << tablet.TEST_db()->GetCurrentVersionNumSSTFiles(); |
132 | 0 | std::string properties; |
133 | 0 | tablet.TEST_db()->GetProperty(rocksdb::DB::Properties::kAggregatedTableProperties, &properties); |
134 | 0 | LOG(INFO) << "DB properties: " << properties; |
135 | |
|
136 | 0 | return catalog_mgr->SplitTablet(tablet_id, true /* select_all_tablets_for_split */); |
137 | 0 | } |
138 | | |
139 | 0 | Status DoSplitTablet(master::CatalogManagerIf* catalog_mgr, const tablet::Tablet& tablet) { |
140 | 0 | const auto& tablet_id = tablet.tablet_id(); |
141 | 0 | LOG(INFO) << "Tablet: " << tablet_id; |
142 | 0 | LOG(INFO) << "Number of SST files: " << tablet.TEST_db()->GetCurrentVersionNumSSTFiles(); |
143 | 0 | std::string properties; |
144 | 0 | tablet.TEST_db()->GetProperty(rocksdb::DB::Properties::kAggregatedTableProperties, &properties); |
145 | 0 | LOG(INFO) << "DB properties: " << properties; |
146 | |
|
147 | 0 | const auto encoded_split_key = VERIFY_RESULT(tablet.GetEncodedMiddleSplitKey()); |
148 | 0 | std::string partition_split_key = encoded_split_key; |
149 | 0 | if (tablet.metadata()->partition_schema()->IsHashPartitioning()) { |
150 | 0 | const auto doc_key_hash = VERIFY_RESULT(docdb::DecodeDocKeyHash(encoded_split_key)).value(); |
151 | 0 | LOG(INFO) << "Middle hash key: " << doc_key_hash; |
152 | 0 | partition_split_key = PartitionSchema::EncodeMultiColumnHashValue(doc_key_hash); |
153 | 0 | } |
154 | 0 | LOG(INFO) << "Partition split key: " << Slice(partition_split_key).ToDebugHexString(); |
155 | |
|
156 | 0 | return catalog_mgr->TEST_SplitTablet(tablet_id, encoded_split_key, partition_split_key); |
157 | 0 | } |
158 | | |
159 | | // |
160 | | // TabletSplitITestBase |
161 | | // |
162 | | |
163 | | // Need to define the static constexpr members as well. |
164 | | template<class MiniClusterType> |
165 | | constexpr std::chrono::duration<int64> TabletSplitITestBase<MiniClusterType>::kRpcTimeout; |
166 | | |
167 | | template<class MiniClusterType> |
168 | | constexpr int TabletSplitITestBase<MiniClusterType>::kDefaultNumRows; |
169 | | |
170 | | template<class MiniClusterType> |
171 | | constexpr size_t TabletSplitITestBase<MiniClusterType>::kDbBlockSizeBytes; |
172 | | |
173 | | template <class MiniClusterType> |
174 | 51 | void TabletSplitITestBase<MiniClusterType>::SetUp() { |
175 | 51 | this->SetNumTablets(3); |
176 | 51 | this->create_table_ = false; |
177 | 51 | this->mini_cluster_opt_.num_tablet_servers = GetRF(); |
178 | 51 | client::TransactionTestBase<MiniClusterType>::SetUp(); |
179 | 51 | proxy_cache_ = std::make_unique<rpc::ProxyCache>(this->client_->messenger()); |
180 | 51 | } _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE5SetUpEv Line | Count | Source | 174 | 43 | void TabletSplitITestBase<MiniClusterType>::SetUp() { | 175 | 43 | this->SetNumTablets(3); | 176 | 43 | this->create_table_ = false; | 177 | 43 | this->mini_cluster_opt_.num_tablet_servers = GetRF(); | 178 | 43 | client::TransactionTestBase<MiniClusterType>::SetUp(); | 179 | 43 | proxy_cache_ = std::make_unique<rpc::ProxyCache>(this->client_->messenger()); | 180 | 43 | } |
_ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE5SetUpEv Line | Count | Source | 174 | 8 | void TabletSplitITestBase<MiniClusterType>::SetUp() { | 175 | 8 | this->SetNumTablets(3); | 176 | 8 | this->create_table_ = false; | 177 | 8 | this->mini_cluster_opt_.num_tablet_servers = GetRF(); | 178 | 8 | client::TransactionTestBase<MiniClusterType>::SetUp(); | 179 | 8 | proxy_cache_ = std::make_unique<rpc::ProxyCache>(this->client_->messenger()); | 180 | 8 | } |
|
181 | | |
182 | | template <class MiniClusterType> |
183 | | Result<tserver::ReadRequestPB> TabletSplitITestBase<MiniClusterType>::CreateReadRequest( |
184 | 0 | const TabletId& tablet_id, int32_t key) { |
185 | 0 | tserver::ReadRequestPB req; |
186 | 0 | auto op = client::CreateReadOp(key, this->table_, this->kValueColumn); |
187 | 0 | auto* ql_batch = req.add_ql_batch(); |
188 | 0 | *ql_batch = op->request(); |
189 | |
|
190 | 0 | std::string partition_key; |
191 | 0 | RETURN_NOT_OK(op->GetPartitionKey(&partition_key)); |
192 | 0 | const auto& hash_code = PartitionSchema::DecodeMultiColumnHashValue(partition_key); |
193 | 0 | ql_batch->set_hash_code(hash_code); |
194 | 0 | ql_batch->set_max_hash_code(hash_code); |
195 | 0 | req.set_tablet_id(tablet_id); |
196 | 0 | req.set_consistency_level(YBConsistencyLevel::CONSISTENT_PREFIX); |
197 | 0 | return req; |
198 | 0 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE17CreateReadRequestERKNSt3__112basic_stringIcNS3_11char_traitsIcEENS3_9allocatorIcEEEEi Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE17CreateReadRequestERKNSt3__112basic_stringIcNS3_11char_traitsIcEENS3_9allocatorIcEEEEi |
199 | | |
200 | | template <class MiniClusterType> |
201 | | tserver::WriteRequestPB TabletSplitITestBase<MiniClusterType>::CreateInsertRequest( |
202 | 0 | const TabletId& tablet_id, int32_t key, int32_t value) { |
203 | 0 | tserver::WriteRequestPB req; |
204 | 0 | auto op = this->table_.NewWriteOp(QLWriteRequestPB::QL_STMT_INSERT); |
205 | |
|
206 | 0 | { |
207 | 0 | auto op_req = op->mutable_request(); |
208 | 0 | QLAddInt32HashValue(op_req, key); |
209 | 0 | this->table_.AddInt32ColumnValue(op_req, this->kValueColumn, value); |
210 | 0 | } |
211 | |
|
212 | 0 | auto* ql_batch = req.add_ql_write_batch(); |
213 | 0 | *ql_batch = op->request(); |
214 | |
|
215 | 0 | std::string partition_key; |
216 | 0 | EXPECT_OK(op->GetPartitionKey(&partition_key)); |
217 | 0 | const auto& hash_code = PartitionSchema::DecodeMultiColumnHashValue(partition_key); |
218 | 0 | ql_batch->set_hash_code(hash_code); |
219 | 0 | req.set_tablet_id(tablet_id); |
220 | 0 | return req; |
221 | 0 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE19CreateInsertRequestERKNSt3__112basic_stringIcNS3_11char_traitsIcEENS3_9allocatorIcEEEEii Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE19CreateInsertRequestERKNSt3__112basic_stringIcNS3_11char_traitsIcEENS3_9allocatorIcEEEEii |
222 | | |
223 | | template <class MiniClusterType> |
224 | | Result<std::pair<docdb::DocKeyHash, docdb::DocKeyHash>> |
225 | | TabletSplitITestBase<MiniClusterType>::WriteRows( |
226 | | client::TableHandle* table, const uint32_t num_rows, |
227 | 15 | const int32_t start_key, const int32_t start_value) { |
228 | 15 | auto min_hash_code = std::numeric_limits<docdb::DocKeyHash>::max(); |
229 | 15 | auto max_hash_code = std::numeric_limits<docdb::DocKeyHash>::min(); |
230 | | |
231 | 15 | LOG(INFO) << "Writing " << num_rows << " rows..."; |
232 | | |
233 | 15 | auto txn = this->CreateTransaction(); |
234 | 15 | auto session = this->CreateSession(txn); |
235 | 15 | for (int32_t i = start_key, v = start_value; |
236 | 22.5k | i < start_key + static_cast<int32_t>(num_rows); |
237 | 22.5k | ++i, ++v) { |
238 | 22.5k | client::YBqlWriteOpPtr op = VERIFY_RESULT( |
239 | 22.5k | client::kv_table_test::WriteRow(table, |
240 | 22.5k | session, |
241 | 22.5k | i /* key */, |
242 | 22.5k | v /* value */, |
243 | 22.5k | client::WriteOpType::INSERT)); |
244 | 22.5k | const auto hash_code = op->GetHashCode(); |
245 | 22.5k | min_hash_code = std::min(min_hash_code, hash_code); |
246 | 22.5k | max_hash_code = std::max(max_hash_code, hash_code); |
247 | 22.5k | YB_LOG_EVERY_N_SECS(INFO, 10) << "Rows written: " << start_key << "..." << i; |
248 | 22.5k | } |
249 | 15 | if (txn) { |
250 | 15 | RETURN_NOT_OK(txn->CommitFuture().get()); |
251 | 15 | LOG(INFO) << "Committed: " << txn->id(); |
252 | 15 | } |
253 | | |
254 | 15 | LOG(INFO) << num_rows << " rows has been written"; |
255 | 15 | LOG(INFO) << "min_hash_code = " << min_hash_code; |
256 | 15 | LOG(INFO) << "max_hash_code = " << max_hash_code; |
257 | 15 | return std::make_pair(min_hash_code, max_hash_code); |
258 | 15 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE9WriteRowsEPNS_6client11TableHandleEjii _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE9WriteRowsEPNS_6client11TableHandleEjii Line | Count | Source | 227 | 15 | const int32_t start_key, const int32_t start_value) { | 228 | 15 | auto min_hash_code = std::numeric_limits<docdb::DocKeyHash>::max(); | 229 | 15 | auto max_hash_code = std::numeric_limits<docdb::DocKeyHash>::min(); | 230 | | | 231 | 15 | LOG(INFO) << "Writing " << num_rows << " rows..."; | 232 | | | 233 | 15 | auto txn = this->CreateTransaction(); | 234 | 15 | auto session = this->CreateSession(txn); | 235 | 15 | for (int32_t i = start_key, v = start_value; | 236 | 22.5k | i < start_key + static_cast<int32_t>(num_rows); | 237 | 22.5k | ++i, ++v) { | 238 | 22.5k | client::YBqlWriteOpPtr op = VERIFY_RESULT( | 239 | 22.5k | client::kv_table_test::WriteRow(table, | 240 | 22.5k | session, | 241 | 22.5k | i /* key */, | 242 | 22.5k | v /* value */, | 243 | 22.5k | client::WriteOpType::INSERT)); | 244 | 22.5k | const auto hash_code = op->GetHashCode(); | 245 | 22.5k | min_hash_code = std::min(min_hash_code, hash_code); | 246 | 22.5k | max_hash_code = std::max(max_hash_code, hash_code); | 247 | 22.5k | YB_LOG_EVERY_N_SECS(INFO, 10) << "Rows written: " << start_key << "..." << i; | 248 | 22.5k | } | 249 | 15 | if (txn) { | 250 | 15 | RETURN_NOT_OK(txn->CommitFuture().get()); | 251 | 15 | LOG(INFO) << "Committed: " << txn->id(); | 252 | 15 | } | 253 | | | 254 | 15 | LOG(INFO) << num_rows << " rows has been written"; | 255 | 15 | LOG(INFO) << "min_hash_code = " << min_hash_code; | 256 | 15 | LOG(INFO) << "max_hash_code = " << max_hash_code; | 257 | 15 | return std::make_pair(min_hash_code, max_hash_code); | 258 | 15 | } |
|
259 | | |
260 | | template <class MiniClusterType> |
261 | 7 | Status TabletSplitITestBase<MiniClusterType>::FlushTestTable() { |
262 | 7 | return this->client_->FlushTables( |
263 | 7 | {this->table_->id()}, /* add_indexes = */ false, /* timeout_secs = */ 30, |
264 | 7 | /* is_compaction = */ false); |
265 | 7 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE14FlushTestTableEv _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE14FlushTestTableEv Line | Count | Source | 261 | 7 | Status TabletSplitITestBase<MiniClusterType>::FlushTestTable() { | 262 | 7 | return this->client_->FlushTables( | 263 | 7 | {this->table_->id()}, /* add_indexes = */ false, /* timeout_secs = */ 30, | 264 | 7 | /* is_compaction = */ false); | 265 | 7 | } |
|
266 | | |
267 | | template <class MiniClusterType> |
268 | | Result<std::pair<docdb::DocKeyHash, docdb::DocKeyHash>> |
269 | | TabletSplitITestBase<MiniClusterType>::WriteRowsAndFlush( |
270 | 7 | const uint32_t num_rows, const int32_t start_key) { |
271 | 7 | auto result = VERIFY_RESULT(WriteRows(num_rows, start_key)); |
272 | 7 | RETURN_NOT_OK(FlushTestTable()); |
273 | 7 | return result; |
274 | 7 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE17WriteRowsAndFlushEji _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE17WriteRowsAndFlushEji Line | Count | Source | 270 | 7 | const uint32_t num_rows, const int32_t start_key) { | 271 | 7 | auto result = VERIFY_RESULT(WriteRows(num_rows, start_key)); | 272 | 7 | RETURN_NOT_OK(FlushTestTable()); | 273 | 7 | return result; | 274 | 7 | } |
|
275 | | |
276 | | template <class MiniClusterType> |
277 | | Result<docdb::DocKeyHash> TabletSplitITestBase<MiniClusterType>::WriteRowsAndGetMiddleHashCode( |
278 | 0 | uint32_t num_rows) { |
279 | 0 | auto min_max_hash_code = VERIFY_RESULT(WriteRowsAndFlush(num_rows, 1)); |
280 | 0 | const auto split_hash_code = (min_max_hash_code.first + min_max_hash_code.second) / 2; |
281 | 0 | LOG(INFO) << "Split hash code: " << split_hash_code; |
282 | |
|
283 | 0 | RETURN_NOT_OK(CheckRowsCount(num_rows)); |
284 | |
|
285 | 0 | return split_hash_code; |
286 | 0 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE29WriteRowsAndGetMiddleHashCodeEj Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE29WriteRowsAndGetMiddleHashCodeEj |
287 | | |
288 | | template <class MiniClusterType> |
289 | | Result<scoped_refptr<master::TabletInfo>> |
290 | | TabletSplitITestBase<MiniClusterType>::GetSingleTestTabletInfo( |
291 | 0 | master::CatalogManagerIf* catalog_mgr) { |
292 | 0 | auto tablet_infos = catalog_mgr->GetTableInfo(this->table_->id())->GetTablets(); |
293 | |
|
294 | 0 | SCHECK_EQ(tablet_infos.size(), 1U, IllegalState, "Expect test table to have only 1 tablet"); |
295 | 0 | return tablet_infos.front(); |
296 | 0 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE23GetSingleTestTabletInfoEPNS_6master16CatalogManagerIfE Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE23GetSingleTestTabletInfoEPNS_6master16CatalogManagerIfE |
297 | | |
298 | | template <class MiniClusterType> |
299 | 0 | void TabletSplitITestBase<MiniClusterType>::CheckTableKeysInRange(const size_t num_keys) { |
300 | 0 | client::TableHandle table; |
301 | 0 | ASSERT_OK(table.Open(client::kTableName, this->client_.get())); |
302 | |
|
303 | 0 | std::vector<int32> keys; |
304 | 0 | for (const auto& row : client::TableRange(table)) { |
305 | 0 | keys.push_back(row.column(0).int32_value()); |
306 | 0 | } |
307 | |
|
308 | 0 | LOG(INFO) << "Total rows read: " << keys.size(); |
309 | |
|
310 | 0 | std::sort(keys.begin(), keys.end()); |
311 | 0 | int32 prev_key = 0; |
312 | 0 | for (const auto& key : keys) { |
313 | 0 | if (key != prev_key + 1) { |
314 | 0 | LOG(ERROR) << "Keys missed: " << prev_key + 1 << "..." << key - 1; |
315 | 0 | } |
316 | 0 | prev_key = key; |
317 | 0 | } |
318 | 0 | LOG(INFO) << "Last key: " << prev_key; |
319 | |
|
320 | 0 | ASSERT_EQ(prev_key, num_keys); |
321 | 0 | ASSERT_EQ(keys.size(), num_keys); |
322 | 0 | } Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_11MiniClusterEE21CheckTableKeysInRangeEm Unexecuted instantiation: _ZN2yb20TabletSplitITestBaseINS_19ExternalMiniClusterEE21CheckTableKeysInRangeEm |
323 | | |
324 | | template class TabletSplitITestBase<MiniCluster>; |
325 | | template class TabletSplitITestBase<ExternalMiniCluster>; |
326 | | |
327 | | // |
328 | | // TabletSplitITest |
329 | | // |
330 | | |
331 | 43 | TabletSplitITest::TabletSplitITest() = default; |
332 | 2 | TabletSplitITest::~TabletSplitITest() = default; |
333 | | |
334 | 43 | void TabletSplitITest::SetUp() { |
335 | 43 | FLAGS_cleanup_split_tablets_interval_sec = 1; |
336 | 43 | FLAGS_enable_automatic_tablet_splitting = false; |
337 | 43 | FLAGS_TEST_validate_all_tablet_candidates = true; |
338 | 43 | FLAGS_db_block_size_bytes = kDbBlockSizeBytes; |
339 | | // We set other block sizes to be small for following test reasons: |
340 | | // 1) To have more granular change of SST file size depending on number of rows written. |
341 | | // This helps to do splits earlier and have faster tests. |
342 | | // 2) To don't have long flushes when simulating slow compaction/flush. This way we can |
343 | | // test compaction abort faster. |
344 | 43 | FLAGS_db_filter_block_size_bytes = 2_KB; |
345 | 43 | FLAGS_db_index_block_size_bytes = 2_KB; |
346 | | // Split size threshold less than memstore size is not effective, because splits are triggered |
347 | | // based on flushed SST files size. |
348 | 43 | FLAGS_db_write_buffer_size = 100_KB; |
349 | 43 | TabletSplitITestBase<MiniCluster>::SetUp(); |
350 | 43 | snapshot_util_ = std::make_unique<client::SnapshotTestUtil>(); |
351 | 43 | snapshot_util_->SetProxy(&client_->proxy_cache()); |
352 | 43 | snapshot_util_->SetCluster(cluster_.get()); |
353 | 43 | } |
354 | | |
355 | 0 | Result<master::TabletInfos> TabletSplitITest::GetTabletInfosForTable(const TableId& table_id) { |
356 | 0 | return VERIFY_RESULT(catalog_manager())->GetTableInfo(table_id)->GetTablets(); |
357 | 0 | } |
358 | | |
359 | 0 | Result<TabletId> TabletSplitITest::CreateSingleTabletAndSplit(uint32_t num_rows) { |
360 | 0 | CreateSingleTablet(); |
361 | 0 | const auto split_hash_code = VERIFY_RESULT(WriteRowsAndGetMiddleHashCode(num_rows)); |
362 | 0 | return SplitTabletAndValidate(split_hash_code, num_rows); |
363 | 0 | } |
364 | | |
365 | 0 | Result<tserver::GetSplitKeyResponsePB> TabletSplitITest::GetSplitKey(const std::string& tablet_id) { |
366 | 0 | auto tserver = cluster_->mini_tablet_server(0); |
367 | 0 | auto ts_service_proxy = std::make_unique<tserver::TabletServerServiceProxy>( |
368 | 0 | proxy_cache_.get(), HostPort::FromBoundEndpoint(tserver->bound_rpc_addr())); |
369 | 0 | tserver::GetSplitKeyRequestPB req; |
370 | 0 | req.set_tablet_id(tablet_id); |
371 | 0 | rpc::RpcController controller; |
372 | 0 | controller.set_timeout(kRpcTimeout); |
373 | 0 | tserver::GetSplitKeyResponsePB resp; |
374 | 0 | RETURN_NOT_OK(ts_service_proxy->GetSplitKey(req, &resp, &controller)); |
375 | 0 | return resp; |
376 | 0 | } |
377 | | |
378 | | Status TabletSplitITest::WaitForTabletSplitCompletion( |
379 | | const size_t expected_non_split_tablets, |
380 | | const size_t expected_split_tablets, |
381 | | size_t num_replicas_online, |
382 | | const client::YBTableName& table, |
383 | 0 | bool core_dump_on_failure) { |
384 | 0 | if (num_replicas_online == 0) { |
385 | 0 | num_replicas_online = FLAGS_replication_factor; |
386 | 0 | } |
387 | |
|
388 | 0 | LOG(INFO) << "Waiting for tablet split to be completed... "; |
389 | 0 | LOG(INFO) << "expected_non_split_tablets: " << expected_non_split_tablets; |
390 | 0 | LOG(INFO) << "expected_split_tablets: " << expected_split_tablets; |
391 | |
|
392 | 0 | const auto expected_total_tablets = expected_non_split_tablets + expected_split_tablets; |
393 | 0 | LOG(INFO) << "expected_total_tablets: " << expected_total_tablets; |
394 | |
|
395 | 0 | std::vector<tablet::TabletPeerPtr> peers; |
396 | 0 | auto s = WaitFor([&] { |
397 | 0 | peers = ListTabletPeers(cluster_.get(), ListPeersFilter::kAll); |
398 | 0 | size_t num_peers_running = 0; |
399 | 0 | size_t num_peers_split = 0; |
400 | 0 | size_t num_peers_leader_ready = 0; |
401 | 0 | for (const auto& peer : peers) { |
402 | 0 | const auto tablet = peer->shared_tablet(); |
403 | 0 | const auto consensus = peer->shared_consensus(); |
404 | 0 | if (!tablet || !consensus) { |
405 | 0 | break; |
406 | 0 | } |
407 | 0 | if (tablet->metadata()->table_name() != table.table_name() || |
408 | 0 | tablet->table_type() == TRANSACTION_STATUS_TABLE_TYPE) { |
409 | 0 | continue; |
410 | 0 | } |
411 | 0 | const auto raft_group_state = peer->state(); |
412 | 0 | const auto tablet_data_state = tablet->metadata()->tablet_data_state(); |
413 | 0 | const auto leader_status = consensus->GetLeaderStatus(/* allow_stale =*/true); |
414 | 0 | if (raft_group_state == tablet::RaftGroupStatePB::RUNNING) { |
415 | 0 | ++num_peers_running; |
416 | 0 | } else { |
417 | 0 | return false; |
418 | 0 | } |
419 | 0 | num_peers_leader_ready += leader_status == consensus::LeaderStatus::LEADER_AND_READY; |
420 | 0 | num_peers_split += |
421 | 0 | tablet_data_state == tablet::TabletDataState::TABLET_DATA_SPLIT_COMPLETED; |
422 | 0 | } |
423 | 0 | VLOG(1) << "num_peers_running: " << num_peers_running; |
424 | 0 | VLOG(1) << "num_peers_split: " << num_peers_split; |
425 | 0 | VLOG(1) << "num_peers_leader_ready: " << num_peers_leader_ready; |
426 | |
|
427 | 0 | return num_peers_running == num_replicas_online * expected_total_tablets && |
428 | 0 | num_peers_split == num_replicas_online * expected_split_tablets && |
429 | 0 | num_peers_leader_ready == expected_total_tablets; |
430 | 0 | }, split_completion_timeout_, "Wait for tablet split to be completed"); |
431 | 0 | if (!s.ok()) { |
432 | 0 | for (const auto& peer : peers) { |
433 | 0 | const auto tablet = peer->shared_tablet(); |
434 | 0 | const auto consensus = peer->shared_consensus(); |
435 | 0 | if (!tablet || !consensus) { |
436 | 0 | LOG(INFO) << consensus::MakeTabletLogPrefix(peer->tablet_id(), peer->permanent_uuid()) |
437 | 0 | << "no tablet"; |
438 | 0 | continue; |
439 | 0 | } |
440 | 0 | if (tablet->table_type() == TRANSACTION_STATUS_TABLE_TYPE) { |
441 | 0 | continue; |
442 | 0 | } |
443 | 0 | LOG(INFO) << consensus::MakeTabletLogPrefix(peer->tablet_id(), peer->permanent_uuid()) |
444 | 0 | << "raft_group_state: " << AsString(peer->state()) |
445 | 0 | << " tablet_data_state: " |
446 | 0 | << TabletDataState_Name(tablet->metadata()->tablet_data_state()) |
447 | 0 | << " leader status: " |
448 | 0 | << AsString(consensus->GetLeaderStatus(/* allow_stale =*/true)); |
449 | 0 | } |
450 | 0 | if (core_dump_on_failure) { |
451 | 0 | LOG(INFO) << "Tablet splitting did not complete. Crashing test with core dump. " |
452 | 0 | << "Received error: " << s.ToString(); |
453 | 0 | raise(SIGSEGV); |
454 | 0 | } else { |
455 | 0 | LOG(INFO) << "Tablet splitting did not complete. Received error: " << s.ToString(); |
456 | 0 | return s; |
457 | 0 | } |
458 | 0 | } |
459 | 0 | LOG(INFO) << "Waiting for tablet split to be completed - DONE"; |
460 | |
|
461 | 0 | DumpTableLocations(VERIFY_RESULT(catalog_manager()), table); |
462 | 0 | return Status::OK(); |
463 | 0 | } |
464 | | |
465 | 0 | Result<TabletId> TabletSplitITest::SplitSingleTablet(docdb::DocKeyHash split_hash_code) { |
466 | 0 | auto* catalog_mgr = VERIFY_RESULT(catalog_manager()); |
467 | |
|
468 | 0 | auto source_tablet_info = VERIFY_RESULT(GetSingleTestTabletInfo(catalog_mgr)); |
469 | 0 | const auto source_tablet_id = source_tablet_info->id(); |
470 | |
|
471 | 0 | RETURN_NOT_OK(catalog_mgr->TEST_SplitTablet(source_tablet_info, split_hash_code)); |
472 | 0 | return source_tablet_id; |
473 | 0 | } |
474 | | |
475 | | Result<TabletId> TabletSplitITest::SplitTabletAndValidate( |
476 | | docdb::DocKeyHash split_hash_code, |
477 | | size_t num_rows, |
478 | 0 | bool parent_tablet_protected_from_deletion) { |
479 | 0 | auto source_tablet_id = VERIFY_RESULT(SplitSingleTablet(split_hash_code)); |
480 | | |
481 | | // If the parent tablet will not be deleted, then we will expect another tablet at the end. |
482 | 0 | const auto expected_split_tablets = |
483 | 0 | (FLAGS_TEST_skip_deleting_split_tablets || parent_tablet_protected_from_deletion) ? 1 : 0; |
484 | |
|
485 | 0 | RETURN_NOT_OK( |
486 | 0 | WaitForTabletSplitCompletion(/* expected_non_split_tablets =*/2, expected_split_tablets)); |
487 | |
|
488 | 0 | RETURN_NOT_OK(CheckPostSplitTabletReplicasData(num_rows)); |
489 | |
|
490 | 0 | if (expected_split_tablets > 0) { |
491 | 0 | RETURN_NOT_OK(CheckSourceTabletAfterSplit(source_tablet_id)); |
492 | 0 | } |
493 | |
|
494 | 0 | return source_tablet_id; |
495 | 0 | } |
496 | | |
497 | 0 | Status TabletSplitITest::CheckSourceTabletAfterSplit(const TabletId& source_tablet_id) { |
498 | 0 | LOG(INFO) << "Checking source tablet behavior after split..."; |
499 | 0 | google::FlagSaver saver; |
500 | 0 | ANNOTATE_UNPROTECTED_WRITE(FLAGS_TEST_do_not_start_election_test_only) = true; |
501 | |
|
502 | 0 | size_t tablet_split_insert_error_count = 0; |
503 | 0 | size_t not_the_leader_insert_error_count = 0; |
504 | 0 | size_t ts_online_count = 0; |
505 | 0 | for (auto mini_ts : this->cluster_->mini_tablet_servers()) { |
506 | 0 | if (!mini_ts->is_started()) { |
507 | 0 | continue; |
508 | 0 | } |
509 | 0 | ++ts_online_count; |
510 | 0 | auto ts_service_proxy = std::make_unique<tserver::TabletServerServiceProxy>( |
511 | 0 | proxy_cache_.get(), HostPort::FromBoundEndpoint(mini_ts->bound_rpc_addr())); |
512 | |
|
513 | 0 | { |
514 | 0 | tserver::ReadRequestPB req = VERIFY_RESULT(CreateReadRequest(source_tablet_id, 1 /* key */)); |
515 | |
|
516 | 0 | rpc::RpcController controller; |
517 | 0 | controller.set_timeout(kRpcTimeout); |
518 | 0 | tserver::ReadResponsePB resp; |
519 | 0 | RETURN_NOT_OK(ts_service_proxy->Read(req, &resp, &controller)); |
520 | |
|
521 | 0 | SCHECK(resp.has_error(), InternalError, "Expected error on read from split tablet"); |
522 | 0 | SCHECK_EQ( |
523 | 0 | resp.error().code(), |
524 | 0 | tserver::TabletServerErrorPB::TABLET_SPLIT, |
525 | 0 | InternalError, |
526 | 0 | "Expected error on read from split tablet to be " |
527 | 0 | "tserver::TabletServerErrorPB::TABLET_SPLIT"); |
528 | 0 | } |
529 | |
|
530 | 0 | { |
531 | 0 | tserver::WriteRequestPB req = |
532 | 0 | CreateInsertRequest(source_tablet_id, 0 /* key */, 0 /* value */); |
533 | |
|
534 | 0 | rpc::RpcController controller; |
535 | 0 | controller.set_timeout(kRpcTimeout); |
536 | 0 | tserver::WriteResponsePB resp; |
537 | 0 | RETURN_NOT_OK(ts_service_proxy->Write(req, &resp, &controller)); |
538 | |
|
539 | 0 | SCHECK(resp.has_error(), InternalError, "Expected error on write to split tablet"); |
540 | 0 | LOG(INFO) << "Error: " << AsString(resp.error()); |
541 | 0 | switch (resp.error().code()) { |
542 | 0 | case tserver::TabletServerErrorPB::TABLET_SPLIT: |
543 | 0 | SCHECK_EQ( |
544 | 0 | resp.error().status().code(), |
545 | 0 | AppStatusPB::ILLEGAL_STATE, |
546 | 0 | InternalError, |
547 | 0 | "tserver::TabletServerErrorPB::TABLET_SPLIT error should have " |
548 | 0 | "AppStatusPB::ILLEGAL_STATE on write to split tablet"); |
549 | 0 | tablet_split_insert_error_count++; |
550 | 0 | break; |
551 | 0 | case tserver::TabletServerErrorPB::NOT_THE_LEADER: |
552 | 0 | not_the_leader_insert_error_count++; |
553 | 0 | break; |
554 | 0 | case tserver::TabletServerErrorPB::TABLET_NOT_FOUND: |
555 | | // In the case that the source tablet was just hidden instead of deleted. |
556 | 0 | tablet_split_insert_error_count++; |
557 | 0 | break; |
558 | 0 | default: |
559 | 0 | return STATUS_FORMAT(InternalError, "Unexpected error: $0", resp.error()); |
560 | 0 | } |
561 | 0 | } |
562 | 0 | } |
563 | 0 | SCHECK_EQ( |
564 | 0 | tablet_split_insert_error_count, 1U, InternalError, |
565 | 0 | "Leader should return \"try again\" error on insert."); |
566 | 0 | SCHECK_EQ( |
567 | 0 | not_the_leader_insert_error_count, ts_online_count - 1, InternalError, |
568 | 0 | "Followers should return \"not the leader\" error."); |
569 | 0 | return Status::OK(); |
570 | 0 | } |
571 | | |
572 | 0 | Result<std::vector<tablet::TabletPeerPtr>> TabletSplitITest::ListSplitCompleteTabletPeers() { |
573 | 0 | return ListTableInactiveSplitTabletPeers(this->cluster_.get(), VERIFY_RESULT(GetTestTableId())); |
574 | 0 | } |
575 | | |
576 | 0 | Result<std::vector<tablet::TabletPeerPtr>> TabletSplitITest::ListPostSplitChildrenTabletPeers() { |
577 | 0 | return ListTableActiveTabletPeers(this->cluster_.get(), VERIFY_RESULT(GetTestTableId())); |
578 | 0 | } |
579 | | |
580 | 0 | Status TabletSplitITest::WaitForTestTablePostSplitTabletsFullyCompacted(MonoDelta timeout) { |
581 | 0 | auto peer_to_str = [](const tablet::TabletPeerPtr& peer) { |
582 | 0 | return peer->LogPrefix() + |
583 | 0 | (peer->tablet_metadata()->has_been_fully_compacted() ? "Compacted" : "NotCompacted"); |
584 | 0 | }; |
585 | 0 | std::vector<std::string> not_compacted_peers; |
586 | 0 | auto s = LoggedWaitFor( |
587 | 0 | [this, ¬_compacted_peers, &peer_to_str]() -> Result<bool> { |
588 | 0 | auto peers = ListPostSplitChildrenTabletPeers(); |
589 | 0 | if (!peers.ok()) { |
590 | 0 | return false; |
591 | 0 | } |
592 | 0 | LOG(INFO) << "Verifying post-split tablet peers:\n" |
593 | 0 | << JoinStrings(*peers | boost::adaptors::transformed(peer_to_str), "\n"); |
594 | 0 | not_compacted_peers.clear(); |
595 | 0 | for (auto peer : *peers) { |
596 | 0 | if (!peer->tablet_metadata()->has_been_fully_compacted()) { |
597 | 0 | not_compacted_peers.push_back(peer_to_str(peer)); |
598 | 0 | } |
599 | 0 | } |
600 | 0 | return not_compacted_peers.empty(); |
601 | 0 | }, |
602 | 0 | timeout, "Wait for post tablet split compaction to be completed"); |
603 | 0 | if (!s.ok()) { |
604 | 0 | LOG(ERROR) << "Following post-split tablet peers have not been fully compacted:\n" |
605 | 0 | << JoinStrings(not_compacted_peers, "\n"); |
606 | 0 | } |
607 | 0 | return s; |
608 | 0 | } |
609 | | |
610 | 0 | Result<int> TabletSplitITest::NumPostSplitTabletPeersFullyCompacted() { |
611 | 0 | int count = 0; |
612 | 0 | for (auto peer : VERIFY_RESULT(ListPostSplitChildrenTabletPeers())) { |
613 | 0 | const auto* tablet = peer->tablet(); |
614 | 0 | if (tablet->metadata()->has_been_fully_compacted()) { |
615 | 0 | ++count; |
616 | 0 | } |
617 | 0 | } |
618 | 0 | return count; |
619 | 0 | } |
620 | | |
621 | 0 | Result<uint64_t> TabletSplitITest::GetMinSstFileSizeAmongAllReplicas(const std::string& tablet_id) { |
622 | 0 | const auto test_table_id = VERIFY_RESULT(GetTestTableId()); |
623 | 0 | auto peers = ListTabletPeers(this->cluster_.get(), [&tablet_id](auto peer) { |
624 | 0 | return peer->tablet_id() == tablet_id; |
625 | 0 | }); |
626 | 0 | if (peers.size() == 0) { |
627 | 0 | return STATUS(IllegalState, "Table has no active peer tablets"); |
628 | 0 | } |
629 | 0 | uint64_t min_file_size = std::numeric_limits<uint64_t>::max(); |
630 | 0 | for (const auto& peer : peers) { |
631 | 0 | min_file_size = std::min( |
632 | 0 | min_file_size, |
633 | 0 | peer->shared_tablet()->GetCurrentVersionSstFilesSize()); |
634 | 0 | } |
635 | 0 | return min_file_size; |
636 | 0 | } |
637 | | |
638 | | Status TabletSplitITest::CheckPostSplitTabletReplicasData( |
639 | 0 | size_t num_rows, size_t num_replicas_online, size_t num_active_tablets) { |
640 | 0 | LOG(INFO) << "Checking post-split tablet replicas data..."; |
641 | |
|
642 | 0 | if (num_replicas_online == 0) { |
643 | 0 | num_replicas_online = FLAGS_replication_factor; |
644 | 0 | } |
645 | |
|
646 | 0 | const auto test_table_id = VERIFY_RESULT(GetTestTableId()); |
647 | 0 | auto active_leader_peers = VERIFY_RESULT(WaitForTableActiveTabletLeadersPeers( |
648 | 0 | this->cluster_.get(), test_table_id, num_active_tablets)); |
649 | |
|
650 | 0 | std::unordered_map<TabletId, OpId> last_on_leader; |
651 | 0 | for (auto peer : active_leader_peers) { |
652 | 0 | last_on_leader[peer->tablet_id()] = peer->shared_consensus()->GetLastReceivedOpId(); |
653 | 0 | } |
654 | |
|
655 | 0 | const auto active_peers = ListTableActiveTabletPeers(this->cluster_.get(), test_table_id); |
656 | |
|
657 | 0 | std::vector<size_t> keys(num_rows, num_replicas_online); |
658 | 0 | std::unordered_map<size_t, std::vector<std::string>> key_replicas; |
659 | 0 | const auto key_column_id = this->table_.ColumnId(this->kKeyColumn); |
660 | 0 | const auto value_column_id = this->table_.ColumnId(this->kValueColumn); |
661 | 0 | for (auto peer : active_peers) { |
662 | 0 | RETURN_NOT_OK(LoggedWaitFor( |
663 | 0 | [&] { |
664 | 0 | return peer->shared_consensus()->GetLastAppliedOpId() >= |
665 | 0 | last_on_leader[peer->tablet_id()]; |
666 | 0 | }, |
667 | 0 | 15s * kTimeMultiplier, |
668 | 0 | Format( |
669 | 0 | "Waiting for tablet replica $0 to apply all ops from leader ...", peer->LogPrefix()))); |
670 | 0 | LOG(INFO) << "Last applied op id for " << peer->LogPrefix() << ": " |
671 | 0 | << AsString(peer->shared_consensus()->GetLastAppliedOpId()); |
672 | |
|
673 | 0 | const auto shared_tablet = peer->shared_tablet(); |
674 | 0 | const SchemaPtr schema = shared_tablet->metadata()->schema(); |
675 | 0 | auto client_schema = schema->CopyWithoutColumnIds(); |
676 | 0 | auto iter = VERIFY_RESULT(shared_tablet->NewRowIterator(client_schema)); |
677 | 0 | QLTableRow row; |
678 | 0 | std::unordered_set<size_t> tablet_keys; |
679 | 0 | while (VERIFY_RESULT(iter->HasNext())) { |
680 | 0 | RETURN_NOT_OK(iter->NextRow(&row)); |
681 | 0 | auto key_opt = row.GetValue(key_column_id); |
682 | 0 | SCHECK(key_opt.is_initialized(), InternalError, "Key is not initialized"); |
683 | 0 | SCHECK_EQ(key_opt, row.GetValue(value_column_id), InternalError, "Wrong value for key"); |
684 | 0 | auto key = key_opt->int32_value(); |
685 | 0 | SCHECK( |
686 | 0 | tablet_keys.insert(key).second, |
687 | 0 | InternalError, |
688 | 0 | Format("Duplicate key $0 in tablet $1", key, shared_tablet->tablet_id())); |
689 | 0 | SCHECK_GT( |
690 | 0 | keys[key - 1]--, |
691 | 0 | 0U, |
692 | 0 | InternalError, |
693 | 0 | Format("Extra key $0 in tablet $1", key, shared_tablet->tablet_id())); |
694 | 0 | key_replicas[key - 1].push_back(peer->LogPrefix()); |
695 | 0 | } |
696 | 0 | } |
697 | 0 | for (size_t key = 1; key <= num_rows; ++key) { |
698 | 0 | const auto key_missing_in_replicas = keys[key - 1]; |
699 | 0 | if (key_missing_in_replicas > 0) { |
700 | 0 | LOG(INFO) << Format("Key $0 replicas: $1", key, key_replicas[key - 1]); |
701 | 0 | return STATUS_FORMAT( |
702 | 0 | InternalError, "Missing key: $0 in $1 replicas", key, key_missing_in_replicas); |
703 | 0 | } |
704 | 0 | } |
705 | 0 | return Status::OK(); |
706 | 0 | } |
707 | | |
708 | | // |
709 | | // TabletSplitExternalMiniClusterITest |
710 | | // |
711 | | |
712 | 8 | void TabletSplitExternalMiniClusterITest::SetFlags() { |
713 | 8 | TabletSplitITestBase<ExternalMiniCluster>::SetFlags(); |
714 | 8 | for (const auto& master_flag : { |
715 | 8 | "--enable_automatic_tablet_splitting=false", |
716 | 8 | "--tablet_split_low_phase_shard_count_per_node=-1", |
717 | 8 | "--tablet_split_high_phase_shard_count_per_node=-1", |
718 | 8 | "--tablet_split_low_phase_size_threshold_bytes=-1", |
719 | 8 | "--tablet_split_high_phase_size_threshold_bytes=-1", |
720 | 8 | "--tablet_force_split_threshold_bytes=-1", |
721 | 48 | }) { |
722 | 48 | mini_cluster_opt_.extra_master_flags.push_back(master_flag); |
723 | 48 | } |
724 | | |
725 | 8 | for (const auto& tserver_flag : std::initializer_list<std::string>{ |
726 | 8 | Format("--db_block_size_bytes=$0", kDbBlockSizeBytes), |
727 | 8 | "--cleanup_split_tablets_interval_sec=1", |
728 | 8 | "--tserver_heartbeat_metrics_interval_ms=100", |
729 | 24 | }) { |
730 | 24 | mini_cluster_opt_.extra_tserver_flags.push_back(tserver_flag); |
731 | 24 | } |
732 | 8 | } |
733 | | |
734 | 69 | Status TabletSplitExternalMiniClusterITest::SplitTablet(const std::string& tablet_id) { |
735 | 69 | master::SplitTabletRequestPB req; |
736 | 69 | req.set_tablet_id(tablet_id); |
737 | 69 | master::SplitTabletResponsePB resp; |
738 | 69 | rpc::RpcController rpc; |
739 | 69 | rpc.set_timeout(30s * kTimeMultiplier); |
740 | | |
741 | 69 | RETURN_NOT_OK(cluster_->GetMasterProxy<master::MasterAdminProxy>().SplitTablet(req, &resp, &rpc)); |
742 | 69 | if (resp.has_error()) { |
743 | 59 | RETURN_NOT_OK(StatusFromPB(resp.error().status())); |
744 | 59 | } |
745 | 10 | return Status::OK(); |
746 | 69 | } |
747 | | |
748 | | Status TabletSplitExternalMiniClusterITest::FlushTabletsOnSingleTServer( |
749 | 5 | size_t tserver_idx, const std::vector<yb::TabletId> tablet_ids, bool is_compaction) { |
750 | 5 | auto tserver = cluster_->tablet_server(tserver_idx); |
751 | 5 | RETURN_NOT_OK(cluster_->FlushTabletsOnSingleTServer(tserver, tablet_ids, is_compaction)); |
752 | 5 | return Status::OK(); |
753 | 5 | } |
754 | | |
755 | | Result<std::set<TabletId>> TabletSplitExternalMiniClusterITest::GetTestTableTabletIds( |
756 | 848 | size_t tserver_idx) { |
757 | 848 | std::set<TabletId> tablet_ids; |
758 | 848 | auto res = VERIFY_RESULT(cluster_->GetTablets(cluster_->tablet_server(tserver_idx))); |
759 | 3.44k | for (const auto& tablet : res) { |
760 | 3.44k | if (tablet.table_name() == table_->name().table_name()) { |
761 | 1.06k | tablet_ids.insert(tablet.tablet_id()); |
762 | 1.06k | } |
763 | 3.44k | } |
764 | 848 | return tablet_ids; |
765 | 848 | } |
766 | | |
767 | 262 | Result<std::set<TabletId>> TabletSplitExternalMiniClusterITest::GetTestTableTabletIds() { |
768 | 262 | std::set<TabletId> tablet_ids; |
769 | 1.04k | for (size_t i = 0; i < cluster_->num_tablet_servers(); ++i) { |
770 | 786 | if (cluster_->tablet_server(i)->IsShutdown()) { |
771 | 4 | continue; |
772 | 4 | } |
773 | 782 | auto res = VERIFY_RESULT(GetTestTableTabletIds(i)); |
774 | 1.03k | for (const auto& id : res) { |
775 | 1.03k | tablet_ids.insert(id); |
776 | 1.03k | } |
777 | 782 | } |
778 | 262 | return tablet_ids; |
779 | 262 | } |
780 | | |
781 | | Result<vector<tserver::ListTabletsResponsePB_StatusAndSchemaPB>> |
782 | 3 | TabletSplitExternalMiniClusterITest::ListTablets(size_t tserver_idx) { |
783 | 3 | vector<tserver::ListTabletsResponsePB_StatusAndSchemaPB> tablets; |
784 | 3 | std::set<TabletId> tablet_ids; |
785 | 3 | auto res = VERIFY_RESULT(cluster_->ListTablets(cluster_->tablet_server(tserver_idx))); |
786 | 15 | for (const auto& tablet : res.status_and_schema()) { |
787 | 15 | auto tablet_id = tablet.tablet_status().tablet_id(); |
788 | 15 | if (tablet.tablet_status().table_name() == table_->name().table_name() && |
789 | 6 | tablet_ids.find(tablet_id) == tablet_ids.end()) { |
790 | 6 | tablets.push_back(tablet); |
791 | 6 | tablet_ids.insert(tablet_id); |
792 | 6 | } |
793 | 15 | } |
794 | 3 | return tablets; |
795 | 3 | } |
796 | | |
797 | | Result<vector<tserver::ListTabletsResponsePB_StatusAndSchemaPB>> |
798 | 1 | TabletSplitExternalMiniClusterITest::ListTablets() { |
799 | 1 | vector<tserver::ListTabletsResponsePB_StatusAndSchemaPB> tablets; |
800 | 1 | std::set<TabletId> tablet_ids; |
801 | 4 | for (size_t i = 0; i < cluster_->num_tablet_servers(); ++i) { |
802 | 3 | auto res = VERIFY_RESULT(ListTablets(i)); |
803 | 6 | for (const auto& tablet : res) { |
804 | 6 | auto tablet_id = tablet.tablet_status().tablet_id(); |
805 | 6 | if (tablet_ids.find(tablet_id) == tablet_ids.end()) { |
806 | 2 | tablets.push_back(tablet); |
807 | 2 | tablet_ids.insert(tablet_id); |
808 | 2 | } |
809 | 6 | } |
810 | 3 | } |
811 | 1 | return tablets; |
812 | 1 | } |
813 | | |
814 | | Status TabletSplitExternalMiniClusterITest::WaitForTabletsExcept( |
815 | 8 | size_t num_tablets, size_t tserver_idx, const TabletId& exclude_tablet) { |
816 | 8 | std::set<TabletId> tablets; |
817 | 8 | auto status = WaitFor( |
818 | 65 | [&]() -> Result<bool> { |
819 | 65 | tablets = VERIFY_RESULT(GetTestTableTabletIds(tserver_idx)); |
820 | 65 | size_t count = 0; |
821 | 24 | for (auto& tablet_id : tablets) { |
822 | 24 | if (tablet_id != exclude_tablet) { |
823 | 21 | count++; |
824 | 21 | } |
825 | 24 | } |
826 | 65 | return count == num_tablets; |
827 | 65 | }, |
828 | 8 | 20s * kTimeMultiplier, |
829 | 8 | Format( |
830 | 8 | "Waiting for tablet count: $0 at tserver: $1", |
831 | 8 | num_tablets, |
832 | 8 | cluster_->tablet_server(tserver_idx)->uuid())); |
833 | 8 | if (!status.ok()) { |
834 | 0 | status = status.CloneAndAppend(Format("Got tablets: $0", tablets)); |
835 | 0 | } |
836 | 8 | return status; |
837 | 8 | } |
838 | | |
839 | 5 | Status TabletSplitExternalMiniClusterITest::WaitForTablets(size_t num_tablets, size_t tserver_idx) { |
840 | 5 | return WaitForTabletsExcept(num_tablets, tserver_idx, ""); |
841 | 5 | } |
842 | | |
843 | 10 | Status TabletSplitExternalMiniClusterITest::WaitForTablets(size_t num_tablets) { |
844 | 10 | std::set<TabletId> tablets; |
845 | 255 | auto status = WaitFor([&]() -> Result<bool> { |
846 | 255 | tablets = VERIFY_RESULT(GetTestTableTabletIds()); |
847 | 255 | return tablets.size() == num_tablets; |
848 | 255 | }, 20s * kTimeMultiplier, Format("Waiting for tablet count: $0", num_tablets)); |
849 | 10 | if (!status.ok()) { |
850 | 2 | status = status.CloneAndAppend(Format("Got tablets: $0", tablets)); |
851 | 2 | } |
852 | 10 | return status; |
853 | 10 | } |
854 | | |
855 | 1 | Result<TabletId> TabletSplitExternalMiniClusterITest::GetOnlyTestTabletId(size_t tserver_idx) { |
856 | 1 | auto tablet_ids = VERIFY_RESULT(GetTestTableTabletIds(tserver_idx)); |
857 | 1 | if (tablet_ids.size() != 1) { |
858 | 0 | return STATUS(InternalError, "Expected one tablet"); |
859 | 0 | } |
860 | 1 | return *tablet_ids.begin(); |
861 | 1 | } |
862 | | |
863 | 7 | Result<TabletId> TabletSplitExternalMiniClusterITest::GetOnlyTestTabletId() { |
864 | 7 | auto tablet_ids = VERIFY_RESULT(GetTestTableTabletIds()); |
865 | 7 | if (tablet_ids.size() != 1) { |
866 | 0 | return STATUS(InternalError, Format("Expected one tablet, got $0", tablet_ids.size())); |
867 | 0 | } |
868 | 7 | return *tablet_ids.begin(); |
869 | 7 | } |
870 | | |
871 | | Status TabletSplitExternalMiniClusterITest::SplitTabletCrashMaster( |
872 | 2 | bool change_split_boundary, string* split_partition_key) { |
873 | 2 | CreateSingleTablet(); |
874 | 2 | int key = 1, num_rows = 2000; |
875 | 2 | RETURN_NOT_OK(WriteRowsAndFlush(num_rows, key)); |
876 | 2 | key += num_rows; |
877 | 2 | auto tablet_id = CHECK_RESULT(GetOnlyTestTabletId()); |
878 | | |
879 | 2 | RETURN_NOT_OK(cluster_->SetFlagOnMasters("TEST_crash_after_creating_single_split_tablet", "1.0")); |
880 | | // Split tablet should crash before creating either tablet |
881 | 2 | if (split_partition_key) { |
882 | 1 | auto res = VERIFY_RESULT(cluster_->GetSplitKey(tablet_id)); |
883 | 1 | *split_partition_key = res.split_partition_key(); |
884 | 1 | } |
885 | 2 | RETURN_NOT_OK(SplitTablet(tablet_id)); |
886 | 2 | auto status = WaitForTablets(3); |
887 | 2 | if (status.ok()) { |
888 | 0 | return STATUS(IllegalState, "Tablet should not have split"); |
889 | 0 | } |
890 | | |
891 | 2 | RETURN_NOT_OK(RestartAllMasters(cluster_.get())); |
892 | 2 | RETURN_NOT_OK(cluster_->SetFlagOnMasters("TEST_crash_after_creating_single_split_tablet", "0.0")); |
893 | | |
894 | 2 | if (change_split_boundary) { |
895 | 1 | RETURN_NOT_OK(WriteRows(num_rows * 2, key)); |
896 | 4 | for (size_t i = 0; i < cluster_->num_tablet_servers(); i++) { |
897 | 3 | RETURN_NOT_OK(FlushTabletsOnSingleTServer(i, {tablet_id}, false)); |
898 | 3 | } |
899 | 1 | } |
900 | | |
901 | | // Wait for tablet split to complete |
902 | 2 | auto raft_heartbeat_roundtrip_time = FLAGS_raft_heartbeat_interval_ms * 2ms; |
903 | 2 | RETURN_NOT_OK(LoggedWaitFor( |
904 | 2 | [this, tablet_id]() -> Result<bool> { |
905 | 2 | auto status = SplitTablet(tablet_id); |
906 | 2 | if (!status.ok()) { |
907 | 2 | return false; |
908 | 2 | } |
909 | 2 | return WaitForTablets(3).ok(); |
910 | 2 | }, |
911 | 2 | 5 * raft_heartbeat_roundtrip_time * kTimeMultiplier |
912 | 2 | + 2ms * FLAGS_tserver_heartbeat_metrics_interval_ms, |
913 | 2 | Format("Wait for tablet to be split: $0", tablet_id))); |
914 | | |
915 | | // Wait for parent tablet clean up |
916 | 2 | std::this_thread::sleep_for(5 * raft_heartbeat_roundtrip_time * kTimeMultiplier); |
917 | 2 | RETURN_NOT_OK(WaitForTablets(2)); |
918 | | |
919 | 2 | return Status::OK(); |
920 | 2 | } |
921 | | |
922 | | } // namespace yb |