/Users/deen/code/yugabyte-db/src/yb/master/sys_catalog.h
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // |
18 | | // The following only applies to changes made to this file as part of YugaByte development. |
19 | | // |
20 | | // Portions Copyright (c) YugaByte, Inc. |
21 | | // |
22 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
23 | | // in compliance with the License. You may obtain a copy of the License at |
24 | | // |
25 | | // http://www.apache.org/licenses/LICENSE-2.0 |
26 | | // |
27 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
28 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
29 | | // or implied. See the License for the specific language governing permissions and limitations |
30 | | // under the License. |
31 | | // |
32 | | #ifndef YB_MASTER_SYS_CATALOG_H_ |
33 | | #define YB_MASTER_SYS_CATALOG_H_ |
34 | | |
35 | | #include <string> |
36 | | #include <vector> |
37 | | #include <unordered_map> |
38 | | |
39 | | #include "yb/common/ql_protocol.pb.h" |
40 | | |
41 | | #include "yb/consensus/consensus_fwd.h" |
42 | | #include "yb/consensus/metadata.pb.h" |
43 | | |
44 | | #include "yb/gutil/callback.h" |
45 | | |
46 | | #include "yb/master/master_fwd.h" |
47 | | #include "yb/master/sys_catalog_constants.h" |
48 | | |
49 | | #include "yb/tablet/snapshot_coordinator.h" |
50 | | |
51 | | #include "yb/tserver/tablet_memory_manager.h" |
52 | | |
53 | | #include "yb/util/mem_tracker.h" |
54 | | #include "yb/util/metrics_fwd.h" |
55 | | #include "yb/util/pb_util.h" |
56 | | #include "yb/util/status_fwd.h" |
57 | | |
58 | | namespace yb { |
59 | | |
60 | | class Schema; |
61 | | class FsManager; |
62 | | class ThreadPool; |
63 | | |
64 | | namespace tserver { |
65 | | class WriteRequestPB; |
66 | | class WriteResponsePB; |
67 | | } |
68 | | |
69 | | namespace master { |
70 | | class Master; |
71 | | class MasterOptions; |
72 | | |
73 | | // Forward declaration from internal header file. |
74 | | class VisitorBase; |
75 | | class SysCatalogWriter; |
76 | | |
77 | | // SysCatalogTable is a YB table that keeps track of table and |
78 | | // tablet metadata. |
79 | | // - SysCatalogTable has only one tablet. |
80 | | // - SysCatalogTable is managed by the master and not exposed to the user |
81 | | // as a "normal table", instead we have Master APIs to query the table. |
82 | | class SysCatalogTable { |
83 | | public: |
84 | | typedef Callback<Status()> ElectedLeaderCallback; |
85 | | |
86 | | // 'leader_cb_' is invoked whenever this node is elected as a leader |
87 | | // of the consensus configuration for this tablet, including for local standalone |
88 | | // master consensus configurations. It used to initialize leader state, submit any |
89 | | // leader-specific tasks and so forth. |
90 | | // |
91 | | /// NOTE: Since 'leader_cb_' is invoked synchronously and can block |
92 | | // the consensus configuration's progress, any long running tasks (e.g., scanning |
93 | | // tablets) should be performed asynchronously (by, e.g., submitting |
94 | | // them to a to a separate threadpool). |
95 | | SysCatalogTable(Master* master, MetricRegistry* metrics, ElectedLeaderCallback leader_cb); |
96 | | |
97 | | ~SysCatalogTable(); |
98 | | |
99 | | // Allow for orderly shutdown of tablet peer, etc. |
100 | | void StartShutdown(); |
101 | | void CompleteShutdown(); |
102 | | |
103 | | // Load the Metadata from disk, and initialize the TabletPeer for the sys-table |
104 | | CHECKED_STATUS Load(FsManager *fs_manager); |
105 | | |
106 | | // Create the new Metadata and initialize the TabletPeer for the sys-table. |
107 | | CHECKED_STATUS CreateNew(FsManager *fs_manager); |
108 | | |
109 | | // ================================================================== |
110 | | // Templated CRUD methods for items in sys.catalog. |
111 | | // ================================================================== |
112 | | template <class... Items> |
113 | | CHECKED_STATUS Upsert(int64_t leader_term, Items&&... items); |
114 | | |
115 | | template <class... Items> |
116 | | CHECKED_STATUS Delete(int64_t leader_term, Items&&... items); |
117 | | |
118 | | template <class... Items> |
119 | | CHECKED_STATUS Mutate( |
120 | | QLWriteRequestPB::QLStmtType op_type, int64_t leader_term, Items&&... items); |
121 | | |
122 | | // ================================================================== |
123 | | // Static schema related methods. |
124 | | // ================================================================== |
125 | | static std::string schema_column_type(); |
126 | | static std::string schema_column_id(); |
127 | | static std::string schema_column_metadata(); |
128 | | |
129 | 5.35k | ThreadPool* raft_pool() const { return raft_pool_.get(); } |
130 | 5.35k | ThreadPool* tablet_prepare_pool() const { return tablet_prepare_pool_.get(); } |
131 | 5.35k | ThreadPool* append_pool() const { return append_pool_.get(); } |
132 | | |
133 | 17.3M | std::shared_ptr<tablet::TabletPeer> tablet_peer() const { |
134 | 17.3M | return std::atomic_load(&tablet_peer_); |
135 | 17.3M | } |
136 | | |
137 | | // Create a new tablet peer with information from the metadata |
138 | | void SetupTabletPeer(const scoped_refptr<tablet::RaftGroupMetadata>& metadata); |
139 | | |
140 | | // Update the in-memory master addresses. Report missing uuid's in the |
141 | | // config when check_missing_uuids is set to true. |
142 | | CHECKED_STATUS ConvertConfigToMasterAddresses( |
143 | | const yb::consensus::RaftConfigPB& config, |
144 | | bool check_missing_uuids = false); |
145 | | |
146 | | // Create consensus metadata object and flush it to disk. |
147 | | CHECKED_STATUS CreateAndFlushConsensusMeta( |
148 | | FsManager* fs_manager, |
149 | | const yb::consensus::RaftConfigPB& config, |
150 | | int64_t current_term); |
151 | | |
152 | | CHECKED_STATUS Visit(VisitorBase* visitor); |
153 | | |
154 | | // Read the ysql catalog version info from the pg_yb_catalog_version catalog table. |
155 | | CHECKED_STATUS ReadYsqlCatalogVersion(TableId ysql_catalog_table_id, |
156 | | uint64_t* catalog_version, |
157 | | uint64_t* last_breaking_version); |
158 | | |
159 | | // Read the pg_class catalog table. There is a separate pg_class table in each |
160 | | // YSQL database, read the information in the pg_class table for the database |
161 | | // 'database_oid' and load this information into 'table_to_tablespace_map'. |
162 | | // 'is_colocated' indicates whether this database is colocated or not. |
163 | | CHECKED_STATUS ReadPgClassInfo(const uint32_t database_oid, |
164 | | const bool is_colocated, |
165 | | TableToTablespaceIdMap* table_to_tablespace_map); |
166 | | |
167 | | CHECKED_STATUS ReadTablespaceInfoFromPgYbTablegroup( |
168 | | const uint32_t database_oid, |
169 | | TableToTablespaceIdMap *table_tablespace_map); |
170 | | |
171 | | // Read relnamespace OID from the pg_class catalog table. |
172 | | Result<uint32_t> ReadPgClassRelnamespace(const uint32_t database_oid, |
173 | | const uint32_t table_oid); |
174 | | |
175 | | // Read nspname string from the pg_namespace catalog table. |
176 | | Result<std::string> ReadPgNamespaceNspname(const uint32_t database_oid, |
177 | | const uint32_t relnamespace_oid); |
178 | | |
179 | | // Read the pg_tablespace catalog table and return a map with all the tablespaces and their |
180 | | // respective placement information. |
181 | | Result<std::shared_ptr<TablespaceIdToReplicationInfoMap>> ReadPgTablespaceInfo(); |
182 | | |
183 | | // Copy the content of co-located tables in sys catalog as a batch. |
184 | | CHECKED_STATUS CopyPgsqlTables(const std::vector<TableId>& source_table_ids, |
185 | | const std::vector<TableId>& target_table_ids, |
186 | | int64_t leader_term); |
187 | | |
188 | | // Drop YSQL table by removing the table metadata in sys-catalog. |
189 | | CHECKED_STATUS DeleteYsqlSystemTable(const string& table_id); |
190 | | |
191 | | const Schema& schema(); |
192 | | |
193 | 5.35k | const scoped_refptr<MetricEntity>& GetMetricEntity() const { return metric_entity_; } |
194 | | |
195 | | CHECKED_STATUS FetchDdlLog(google::protobuf::RepeatedPtrField<DdlLogEntryPB>* entries); |
196 | | |
197 | | private: |
198 | | friend class CatalogManager; |
199 | | friend class enterprise::CatalogManager; |
200 | | |
201 | | inline std::unique_ptr<SysCatalogWriter> NewWriter(int64_t leader_term); |
202 | | |
203 | 49.4k | const char *table_name() const { return kSysCatalogTableName; } |
204 | | |
205 | | // Return the schema of the table. |
206 | | // NOTE: This is the "server-side" schema, so it must have the column IDs. |
207 | | Schema BuildTableSchema(); |
208 | | |
209 | | // Returns 'Status::OK()' if the WriteTranasction completed |
210 | | CHECKED_STATUS SyncWrite(SysCatalogWriter* writer); |
211 | | |
212 | | void SysCatalogStateChanged(const std::string& tablet_id, |
213 | | std::shared_ptr<consensus::StateChangeContext> context); |
214 | | |
215 | | CHECKED_STATUS SetupTablet(const scoped_refptr<tablet::RaftGroupMetadata>& metadata); |
216 | | |
217 | | CHECKED_STATUS OpenTablet(const scoped_refptr<tablet::RaftGroupMetadata>& metadata); |
218 | | |
219 | | // Use the master options to generate a new consensus configuration. |
220 | | // In addition, resolve all UUIDs of this consensus configuration. |
221 | | CHECKED_STATUS SetupConfig(const MasterOptions& options, |
222 | | consensus::RaftConfigPB* committed_config); |
223 | | |
224 | | std::string tablet_id() const; |
225 | | |
226 | | // Conventional "T xxx P xxxx..." prefix for logging. |
227 | | std::string LogPrefix() const; |
228 | | |
229 | | // Waits for the tablet to reach 'RUNNING' state. |
230 | | // |
231 | | // Contrary to tablet servers, in master we actually wait for the master tablet |
232 | | // to become online synchronously, this allows us to fail fast if something fails |
233 | | // and shouldn't induce the all-workers-blocked-waiting-for-tablets problem |
234 | | // that we've seen in tablet servers since the master only has to boot a few |
235 | | // tablets. |
236 | | CHECKED_STATUS WaitUntilRunning(); |
237 | | |
238 | | // Shutdown the tablet peer and apply pool which are not needed in shell mode for this master. |
239 | | CHECKED_STATUS GoIntoShellMode(); |
240 | | |
241 | | // Initializes the RaftPeerPB for the local peer. |
242 | | // Crashes due to an invariant check if the rpc server is not running. |
243 | | void InitLocalRaftPeerPB(); |
244 | | |
245 | | // Table schema, with IDs, used for the YQL write path. |
246 | | std::unique_ptr<Schema> schema_; |
247 | | |
248 | | MetricRegistry* metric_registry_; |
249 | | |
250 | | scoped_refptr<MetricEntity> metric_entity_; |
251 | | |
252 | | std::unique_ptr<ThreadPool> inform_removed_master_pool_; |
253 | | |
254 | | // Thread pool for Raft-related operations |
255 | | std::unique_ptr<ThreadPool> raft_pool_; |
256 | | |
257 | | // Thread pool for preparing transactions, shared between all tablets. |
258 | | std::unique_ptr<ThreadPool> tablet_prepare_pool_; |
259 | | |
260 | | // Thread pool for appender tasks |
261 | | std::unique_ptr<ThreadPool> append_pool_; |
262 | | |
263 | | std::unique_ptr<ThreadPool> allocation_pool_; |
264 | | |
265 | | std::shared_ptr<tablet::TabletPeer> tablet_peer_; |
266 | | |
267 | | Master* master_; |
268 | | |
269 | | ElectedLeaderCallback leader_cb_; |
270 | | |
271 | | consensus::RaftPeerPB local_peer_pb_; |
272 | | |
273 | | scoped_refptr<Histogram> setup_config_dns_histogram_; |
274 | | |
275 | | scoped_refptr<Counter> peer_write_count; |
276 | | |
277 | | std::unordered_map<std::string, scoped_refptr<AtomicGauge<uint64>>> visitor_duration_metrics_; |
278 | | |
279 | | std::shared_ptr<tserver::TabletMemoryManager> mem_manager_; |
280 | | |
281 | | std::unique_ptr<consensus::MultiRaftManager> multi_raft_manager_; |
282 | | |
283 | | DISALLOW_COPY_AND_ASSIGN(SysCatalogTable); |
284 | | }; |
285 | | |
286 | | } // namespace master |
287 | | } // namespace yb |
288 | | |
289 | | #include "yb/master/sys_catalog-internal.h" |
290 | | |
291 | | #endif // YB_MASTER_SYS_CATALOG_H_ |