/Users/deen/code/yugabyte-db/src/yb/master/master-test_base.cc

Source (jump to first uncovered line)
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
// The following only applies to changes made to this file as part of YugaByte development.
//
// Portions Copyright (c) YugaByte, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
// in compliance with the License.  You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied.  See the License for the specific language governing permissions and limitations
// under the License.
//

#include "yb/master/master-test_base.h"

#include <memory>

#include <gtest/gtest.h>

#include "yb/common/wire_protocol.h"

#include "yb/gutil/strings/substitute.h"

#include "yb/master/master-test-util.h"
#include "yb/master/master.h"
#include "yb/master/master_client.proxy.h"
#include "yb/master/master_cluster.proxy.h"
#include "yb/master/master_ddl.proxy.h"
#include "yb/master/master_heartbeat.proxy.h"
#include "yb/master/master_replication.proxy.h"
#include "yb/master/mini_master.h"
#include "yb/master/ts_descriptor.h"

#include "yb/rpc/messenger.h"
#include "yb/rpc/proxy.h"

#include "yb/util/status.h"
#include "yb/util/status_format.h"
#include "yb/util/status_log.h"
#include "yb/util/test_util.h"

DECLARE_bool(catalog_manager_check_ts_count_for_create_table);
DECLARE_bool(TEST_disable_cdc_state_insert_on_setup);

namespace yb {
namespace master {

MasterTestBase::MasterTestBase() = default;
MasterTestBase::~MasterTestBase() = default;

void MasterTestBase::SetUp() {
  YBTest::SetUp();

  // Set an RPC timeout for the controllers.
  controller_ = make_shared<RpcController>();
  controller_->set_timeout(MonoDelta::FromSeconds(10));

  // In this test, we create tables to test catalog manager behavior,
  // but we have no tablet servers. Typically this would be disallowed.
  FLAGS_catalog_manager_check_ts_count_for_create_table = false;
  // Since this is a master-only test, don't do any operations on cdc state for xCluster tests.
  FLAGS_TEST_disable_cdc_state_insert_on_setup = true;

  // Start master with the create flag on.
  mini_master_.reset(new MiniMaster(Env::Default(), GetTestPath("Master"),
                                    AllocateFreePort(), AllocateFreePort(), 0));
  ASSERT_OK(mini_master_->Start());
  ASSERT_OK(mini_master_->master()->WaitUntilCatalogManagerIsLeaderAndReadyForTests());

  // Create a client proxy to it.
  client_messenger_ = ASSERT_RESULT(MessengerBuilder("Client").Build());
  rpc::ProxyCache proxy_cache(client_messenger_.get());
  proxy_client_ = std::make_unique<MasterClientProxy>(
      &proxy_cache, mini_master_->bound_rpc_addr());
  proxy_cluster_ = std::make_unique<MasterClusterProxy>(
      &proxy_cache, mini_master_->bound_rpc_addr());
  proxy_ddl_ = std::make_unique<MasterDdlProxy>(
      &proxy_cache, mini_master_->bound_rpc_addr());
  proxy_heartbeat_ = std::make_unique<MasterHeartbeatProxy>(
      &proxy_cache, mini_master_->bound_rpc_addr());
  proxy_replication_ = std::make_unique<MasterReplicationProxy>(
      &proxy_cache, mini_master_->bound_rpc_addr());

  // Create the default test namespace.
  CreateNamespaceResponsePB resp;
  ASSERT_OK(CreateNamespace(default_namespace_name, &resp));
  default_namespace_id = resp.id();
}

void MasterTestBase::TearDown() {
  if (client_messenger_) {
    client_messenger_->Shutdown();
  }
  mini_master_->Shutdown();
  YBTest::TearDown();
}

Status MasterTestBase::CreateTable(const NamespaceName& namespace_name,
                                   const TableName& table_name,
                                   const Schema& schema,
                                   TableId* table_id /* = nullptr */) {
  CreateTableRequestPB req;
  return DoCreateTable(namespace_name, table_name, schema, &req, table_id);
}

Status MasterTestBase::CreatePgsqlTable(const NamespaceId& namespace_id,
                                        const TableName& table_name,
                                        const Schema& schema) {
  CreateTableRequestPB req, *request;
  request = &req;
  CreateTableResponsePB resp;

  request->set_table_type(TableType::PGSQL_TABLE_TYPE);
  request->set_name(table_name);
  SchemaToPB(schema, request->mutable_schema());

  if (!namespace_id.empty()) {
    request->mutable_namespace_()->set_id(namespace_id);
  }
  request->mutable_partition_schema()->set_hash_schema(PartitionSchemaPB::PGSQL_HASH_SCHEMA);
  request->mutable_schema()->mutable_table_properties()->set_num_tablets(8);

  // Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
  // though, as that helps with readability and standardization.
  RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::CreateTablegroupTable(const NamespaceId& namespace_id,
                                             const TableName& table_name,
                                             const TablegroupId& tablegroup_id,
                                             const Schema& schema,
                                             TableId* table_id /* = nullptr */) {
  CreateTableRequestPB req, *request;
  request = &req;
  CreateTableResponsePB resp;

  request->set_table_type(TableType::PGSQL_TABLE_TYPE);
  request->set_name(table_name);
  request->set_colocated(true);
  request->set_tablegroup_id(tablegroup_id);
  SchemaToPB(schema, request->mutable_schema());

  if (!namespace_id.empty()) {
    request->mutable_namespace_()->set_id(namespace_id);
  }

  // Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
  // though, as that helps with readability and standardization.
  RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
  if (table_id) {
    *table_id = resp.table_id();
  }
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::DoCreateTable(const NamespaceName& namespace_name,
                                     const TableName& table_name,
                                     const Schema& schema,
                                     CreateTableRequestPB* request,
                                     TableId* table_id /* = nullptr */) {
  CreateTableResponsePB resp;

  request->set_name(table_name);
  SchemaToPB(schema, request->mutable_schema());

  if (!namespace_name.empty()) {
    request->mutable_namespace_()->set_name(namespace_name);
  }
  request->mutable_partition_schema()->set_hash_schema(PartitionSchemaPB::MULTI_COLUMN_HASH_SCHEMA);
  request->mutable_schema()->mutable_table_properties()->set_num_tablets(8);

  // Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
  // though, as that helps with readability and standardization.
  RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
  if (table_id) {
    *table_id = resp.table_id();
  }
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }

  return Status::OK();
}

void MasterTestBase::DoListTables(const ListTablesRequestPB& req, ListTablesResponsePB* resp) {
  ASSERT_OK(proxy_ddl_->ListTables(req, resp, ResetAndGetController()));
  SCOPED_TRACE(resp->DebugString());
  ASSERT_FALSE(resp->has_error());
}

void MasterTestBase::DoListAllTables(ListTablesResponsePB* resp,
                                     const NamespaceName& namespace_name /*= ""*/) {
  ListTablesRequestPB req;

  if (!namespace_name.empty()) {
    req.mutable_namespace_()->set_name(namespace_name);
  }

  DoListTables(req, resp);
}

Status MasterTestBase::DeleteTableById(const TableId& table_id) {
  DeleteTableRequestPB req;
  DeleteTableResponsePB resp;
  req.mutable_table()->set_table_id(table_id);
  RETURN_NOT_OK(proxy_ddl_->DeleteTable(req, &resp, ResetAndGetController()));
  SCOPED_TRACE(resp.DebugString());
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::DeleteTable(const NamespaceName& namespace_name,
                                   const TableName& table_name,
                                   TableId* table_id /* = nullptr */) {
  SCHECK(!namespace_name.empty(), InvalidArgument, "Namespace name was empty");
  DeleteTableRequestPB req;
  DeleteTableResponsePB resp;
  req.mutable_table()->set_table_name(table_name);

  req.mutable_table()->mutable_namespace_()->set_name(namespace_name);

  RETURN_NOT_OK(proxy_ddl_->DeleteTable(req, &resp, ResetAndGetController()));
  SCOPED_TRACE(resp.DebugString());
  if (table_id) {
    *table_id = resp.table_id();
  }

  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::CreateTablegroup(const TablegroupId& tablegroup_id,
                                        const NamespaceId& namespace_id,
                                        const NamespaceName& namespace_name,
                                        const TablespaceId& tablespace_id) {
  CreateTablegroupRequestPB req, *request;
  request = &req;
  CreateTablegroupResponsePB resp;

  request->set_id(tablegroup_id);
  request->set_namespace_id(namespace_id);
  request->set_namespace_name(namespace_name);
  request->set_tablespace_id(tablespace_id);

  // Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
  // though, as that helps with readability and standardization.
  RETURN_NOT_OK(proxy_ddl_->CreateTablegroup(*request, &resp, ResetAndGetController()));
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::DeleteTablegroup(const TablegroupId& tablegroup_id,
                                        const NamespaceId& namespace_id) {
  DeleteTablegroupRequestPB req;
  DeleteTablegroupResponsePB resp;
  req.set_id(tablegroup_id);
  req.set_namespace_id(namespace_id);

  RETURN_NOT_OK(proxy_ddl_->DeleteTablegroup(req, &resp, ResetAndGetController()));
  if (resp.has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp.error().status()));
  }
  return Status::OK();
}

void MasterTestBase::DoListTablegroups(const ListTablegroupsRequestPB& req,
                                       ListTablegroupsResponsePB* resp) {
  ASSERT_OK(proxy_ddl_->ListTablegroups(req, resp, ResetAndGetController()));
  SCOPED_TRACE(resp->DebugString());
  ASSERT_FALSE(resp->has_error());
}

void MasterTestBase::DoListAllNamespaces(ListNamespacesResponsePB* resp) {
  DoListAllNamespaces(boost::none, resp);
}

void MasterTestBase::DoListAllNamespaces(const boost::optional<YQLDatabase>& database_type,
                                         ListNamespacesResponsePB* resp) {
  ListNamespacesRequestPB req;
  if (database_type) {
    req.set_database_type(*database_type);
  }

  ASSERT_OK(proxy_ddl_->ListNamespaces(req, resp, ResetAndGetController()));
  SCOPED_TRACE(resp->DebugString());
  ASSERT_FALSE(resp->has_error());
}

Status MasterTestBase::CreateNamespace(const NamespaceName& ns_name,
                                       CreateNamespaceResponsePB* resp) {
  return CreateNamespace(ns_name, boost::none, resp);
}

Status MasterTestBase::CreateNamespace(const NamespaceName& ns_name,
                                       const boost::optional<YQLDatabase>& database_type,
                                       CreateNamespaceResponsePB* resp) {
  RETURN_NOT_OK(CreateNamespaceAsync(ns_name, database_type, resp));
  return CreateNamespaceWait(resp->id(), database_type);
}

Status MasterTestBase::CreateNamespaceAsync(const NamespaceName& ns_name,
                                            const boost::optional<YQLDatabase>& database_type,
                                            CreateNamespaceResponsePB* resp) {
  CreateNamespaceRequestPB req;
  req.set_name(ns_name);
  if (database_type) {
    req.set_database_type(*database_type);
  }

  RETURN_NOT_OK(proxy_ddl_->CreateNamespace(req, resp, ResetAndGetController()));
  if (resp->has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp->error().status()));
  }
  return Status::OK();
}

Status MasterTestBase::CreateNamespaceWait(const NamespaceId& ns_id,
                                           const boost::optional<YQLDatabase>& database_type) {
  Status status = Status::OK();

  IsCreateNamespaceDoneRequestPB is_req;
  is_req.mutable_namespace_()->set_id(ns_id);
  if (database_type) {
    is_req.mutable_namespace_()->set_database_type(*database_type);
  }

  return LoggedWaitFor([&]() -> Result<bool> {
    IsCreateNamespaceDoneResponsePB is_resp;
    status = proxy_ddl_->IsCreateNamespaceDone(is_req, &is_resp, ResetAndGetController());
    WARN_NOT_OK(status, "IsCreateNamespaceDone returned unexpected error");
    if (!status.ok()) {
      return status;
    }
    if (is_resp.has_done() && is_resp.done()) {
      if (is_resp.has_error()) {
        return StatusFromPB(is_resp.error().status());
      }
      return true;
    }
    return false;
  }, MonoDelta::FromSeconds(60), "Wait for create namespace to finish async setup tasks.");
}

Status MasterTestBase::AlterNamespace(const NamespaceName& ns_name,
                                      const NamespaceId& ns_id,
                                      const boost::optional<YQLDatabase>& database_type,
                                      const std::string& new_name,
                                      AlterNamespaceResponsePB* resp) {
  AlterNamespaceRequestPB req;
  req.mutable_namespace_()->set_id(ns_id);
  req.mutable_namespace_()->set_name(ns_name);
  if (database_type) {
    req.mutable_namespace_()->set_database_type(*database_type);
  }
  req.set_new_name(new_name);

  RETURN_NOT_OK(proxy_ddl_->AlterNamespace(req, resp, ResetAndGetController()));
  if (resp->has_error()) {
    RETURN_NOT_OK(StatusFromPB(resp->error().status()));
  }
  return Status::OK();
}

// PGSQL Namespaces are deleted asynchronously since they may delete a large number of tables.
// CQL Namespaces don't need to call this function and return success if present.
Status MasterTestBase::DeleteNamespaceWait(IsDeleteNamespaceDoneRequestPB const& del_req) {
  return LoggedWaitFor([&]() -> Result<bool> {
    IsDeleteNamespaceDoneResponsePB del_resp;
    auto status = proxy_ddl_->IsDeleteNamespaceDone(del_req, &del_resp, ResetAndGetController());
    if (!status.ok()) {
      WARN_NOT_OK(status, "IsDeleteNamespaceDone returned unexpected error");
      return status;
    }
    if (del_resp.has_done() && del_resp.done()) {
      if (del_resp.has_error()) {
        return StatusFromPB(del_resp.error().status());
      }
      return true;
    }
    return false;
  }, MonoDelta::FromSeconds(10), "Wait for delete namespace to finish async cleanup tasks.");
}

Status MasterTestBase::DeleteTableSync(const NamespaceName& ns_name, const TableName& table_name,
                                       TableId* table_id) {
  RETURN_NOT_OK(DeleteTable(ns_name, table_name, table_id));

  IsDeleteTableDoneRequestPB done_req;
  done_req.set_table_id(*table_id);
  IsDeleteTableDoneResponsePB done_resp;
  bool delete_done = false;

  for (int num_retries = 0; num_retries < 30; ++num_retries0) {
    RETURN_NOT_OK(proxy_ddl_->IsDeleteTableDone(done_req, &done_resp, ResetAndGetController()));
    if (!done_resp.has_done()) {
      return STATUS_FORMAT(
          IllegalState, "Expected IsDeleteTableDone response to set value for done ($0.$1)",
          ns_name, table_name);
    }
    if (done_resp.done()) {
      LOG(INFO) << "Done on retry " << num_retries;
      delete_done = true;
      break;
    }

    SleepFor(MonoDelta::FromMilliseconds(10 * num_retries)); // sleep a bit more with each attempt.
  }

  if (!delete_done) {
    return STATUS_FORMAT(IllegalState, "Delete Table did not complete ($0.$1)",
                         ns_name, table_name);
  }
  return Status::OK();
}

void MasterTestBase::CheckNamespaces(
    const std::set<std::tuple<NamespaceName, NamespaceId>>& namespace_info,
    const ListNamespacesResponsePB& namespaces) {
  for (int i = 0; i < namespaces.namespaces_size(); i++135) {
    auto search_key = std::make_tuple(namespaces.namespaces(i).name(),
                                      namespaces.namespaces(i).id());
    ASSERT_TRUE(namespace_info.find(search_key) != namespace_info.end())
                  << strings::Substitute("Couldn't find namespace $0", namespaces.namespaces(i)
                      .name());
  }

  ASSERT_EQ(namespaces.namespaces_size(), namespace_info.size());
}

bool MasterTestBase::FindNamespace(
    const std::tuple<NamespaceName, NamespaceId>& namespace_info,
    const ListNamespacesResponsePB& namespaces) {
  for (int i = 0; i < namespaces.namespaces_size(); i++16) {
    auto cur_ns = std::make_tuple(namespaces.namespaces(i).name(),
                                  namespaces.namespaces(i).id());
    if (cur_ns == namespace_info) {
      return true; // found!
    }
  }
  return false; // namespace not found.
}

void MasterTestBase::CheckTables(
    const std::set<std::tuple<TableName, NamespaceName, NamespaceId, bool>>& table_info,
    const ListTablesResponsePB& tables) {
  for (int i = 0; i < tables.tables_size(); i++412) {
    auto search_key = std::make_tuple(tables.tables(i).name(),
                                      tables.tables(i).namespace_().name(),
                                      tables.tables(i).namespace_().id(),
                                      tables.tables(i).relation_type());
    ASSERT_TRUE(table_info.find(search_key) != table_info.end())
        << strings::Substitute("Couldn't find table $0.$1",
            tables.tables(i).namespace_().name(), tables.tables(i).name());
  }

  ASSERT_EQ(tables.tables_size(), table_info.size());
}

void MasterTestBase::UpdateMasterClusterConfig(SysClusterConfigEntryPB* cluster_config) {
  ChangeMasterClusterConfigRequestPB change_req;
  change_req.mutable_cluster_config()->CopyFrom(*cluster_config);
  ChangeMasterClusterConfigResponsePB change_resp;
  rpc::ProxyCache proxy_cache(client_messenger_.get());
  master::MasterClusterProxy proxy(&proxy_cache, mini_master_->bound_rpc_addr());
  ASSERT_OK(proxy.ChangeMasterClusterConfig(change_req, &change_resp, ResetAndGetController()));
  // Bump version number by 1, so we do not have to re-query.
  cluster_config->set_version(cluster_config->version() + 1);
  LOG(INFO) << "Update cluster config to: " << cluster_config->ShortDebugString();
}

} // namespace master
} // namespace yb

YugabyteDB (2.13.1.0-b60, 21121d69985fbf76aa6958d8f04a9bfa936293b5)

Coverage Report

Created: 2022-03-22 16:43

Line	Count	Source (jump to first uncovered line)
1		// Licensed to the Apache Software Foundation (ASF) under one
2		// or more contributor license agreements. See the NOTICE file
3		// distributed with this work for additional information
4		// regarding copyright ownership. The ASF licenses this file
5		// to you under the Apache License, Version 2.0 (the
6		// "License"); you may not use this file except in compliance
7		// with the License. You may obtain a copy of the License at
8		//
9		// http://www.apache.org/licenses/LICENSE-2.0
10		//
11		// Unless required by applicable law or agreed to in writing,
12		// software distributed under the License is distributed on an
13		// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14		// KIND, either express or implied. See the License for the
15		// specific language governing permissions and limitations
16		// under the License.
17		//
18		// The following only applies to changes made to this file as part of YugaByte development.
19		//
20		// Portions Copyright (c) YugaByte, Inc.
21		//
22		// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
23		// in compliance with the License. You may obtain a copy of the License at
24		//
25		// http://www.apache.org/licenses/LICENSE-2.0
26		//
27		// Unless required by applicable law or agreed to in writing, software distributed under the License
28		// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
29		// or implied. See the License for the specific language governing permissions and limitations
30		// under the License.
31		//
32
33		#include "yb/master/master-test_base.h"
34
35		#include <memory>
36
37		#include <gtest/gtest.h>
38
39		#include "yb/common/wire_protocol.h"
40
41		#include "yb/gutil/strings/substitute.h"
42
43		#include "yb/master/master-test-util.h"
44		#include "yb/master/master.h"
45		#include "yb/master/master_client.proxy.h"
46		#include "yb/master/master_cluster.proxy.h"
47		#include "yb/master/master_ddl.proxy.h"
48		#include "yb/master/master_heartbeat.proxy.h"
49		#include "yb/master/master_replication.proxy.h"
50		#include "yb/master/mini_master.h"
51		#include "yb/master/ts_descriptor.h"
52
53		#include "yb/rpc/messenger.h"
54		#include "yb/rpc/proxy.h"
55
56		#include "yb/util/status.h"
57		#include "yb/util/status_format.h"
58		#include "yb/util/status_log.h"
59		#include "yb/util/test_util.h"
60
61		DECLARE_bool(catalog_manager_check_ts_count_for_create_table);
62		DECLARE_bool(TEST_disable_cdc_state_insert_on_setup);
63
64		namespace yb {
65		namespace master {
66
67	35	MasterTestBase::MasterTestBase() = default;
68	33	MasterTestBase::~MasterTestBase() = default;
69
70	35	void MasterTestBase::SetUp() {
71	35	YBTest::SetUp();
72
73		// Set an RPC timeout for the controllers.
74	35	controller_ = make_shared<RpcController>();
75	35	controller_->set_timeout(MonoDelta::FromSeconds(10));
76
77		// In this test, we create tables to test catalog manager behavior,
78		// but we have no tablet servers. Typically this would be disallowed.
79	35	FLAGS_catalog_manager_check_ts_count_for_create_table = false;
80		// Since this is a master-only test, don't do any operations on cdc state for xCluster tests.
81	35	FLAGS_TEST_disable_cdc_state_insert_on_setup = true;
82
83		// Start master with the create flag on.
84	35	mini_master_.reset(new MiniMaster(Env::Default(), GetTestPath("Master"),
85	35	AllocateFreePort(), AllocateFreePort(), 0));
86	35	ASSERT_OK(mini_master_->Start());
87	35	ASSERT_OK(mini_master_->master()->WaitUntilCatalogManagerIsLeaderAndReadyForTests());
88
89		// Create a client proxy to it.
90	35	client_messenger_ = ASSERT_RESULT(MessengerBuilder("Client").Build());
91	0	rpc::ProxyCache proxy_cache(client_messenger_.get());
92	35	proxy_client_ = std::make_unique<MasterClientProxy>(
93	35	&proxy_cache, mini_master_->bound_rpc_addr());
94	35	proxy_cluster_ = std::make_unique<MasterClusterProxy>(
95	35	&proxy_cache, mini_master_->bound_rpc_addr());
96	35	proxy_ddl_ = std::make_unique<MasterDdlProxy>(
97	35	&proxy_cache, mini_master_->bound_rpc_addr());
98	35	proxy_heartbeat_ = std::make_unique<MasterHeartbeatProxy>(
99	35	&proxy_cache, mini_master_->bound_rpc_addr());
100	35	proxy_replication_ = std::make_unique<MasterReplicationProxy>(
101	35	&proxy_cache, mini_master_->bound_rpc_addr());
102
103		// Create the default test namespace.
104	35	CreateNamespaceResponsePB resp;
105	35	ASSERT_OK(CreateNamespace(default_namespace_name, &resp));
106	35	default_namespace_id = resp.id();
107	35	}
108
109	35	void MasterTestBase::TearDown() {
110	35	if (client_messenger_) {
111	35	client_messenger_->Shutdown();
112	35	}
113	35	mini_master_->Shutdown();
114	35	YBTest::TearDown();
115	35	}
116
117		Status MasterTestBase::CreateTable(const NamespaceName& namespace_name,
118		const TableName& table_name,
119		const Schema& schema,
120	18	TableId* table_id /* = nullptr */) {
121	18	CreateTableRequestPB req;
122	18	return DoCreateTable(namespace_name, table_name, schema, &req, table_id);
123	18	}
124
125		Status MasterTestBase::CreatePgsqlTable(const NamespaceId& namespace_id,
126		const TableName& table_name,
127	4	const Schema& schema) {
128	4	CreateTableRequestPB req, *request;
129	4	request = &req;
130	4	CreateTableResponsePB resp;
131
132	4	request->set_table_type(TableType::PGSQL_TABLE_TYPE);
133	4	request->set_name(table_name);
134	4	SchemaToPB(schema, request->mutable_schema());
135
136	4	if (!namespace_id.empty()) {
137	4	request->mutable_namespace_()->set_id(namespace_id);
138	4	}
139	4	request->mutable_partition_schema()->set_hash_schema(PartitionSchemaPB::PGSQL_HASH_SCHEMA);
140	4	request->mutable_schema()->mutable_table_properties()->set_num_tablets(8);
141
142		// Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
143		// though, as that helps with readability and standardization.
144	4	RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
145	4	if (resp.has_error()) {
146	1	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
147	1	}
148	3	return Status::OK();
149	4	}
150
151		Status MasterTestBase::CreateTablegroupTable(const NamespaceId& namespace_id,
152		const TableName& table_name,
153		const TablegroupId& tablegroup_id,
154		const Schema& schema,
155	1	TableId* table_id /* = nullptr */) {
156	1	CreateTableRequestPB req, *request;
157	1	request = &req;
158	1	CreateTableResponsePB resp;
159
160	1	request->set_table_type(TableType::PGSQL_TABLE_TYPE);
161	1	request->set_name(table_name);
162	1	request->set_colocated(true);
163	1	request->set_tablegroup_id(tablegroup_id);
164	1	SchemaToPB(schema, request->mutable_schema());
165
166	1	if (!namespace_id.empty()) {
167	1	request->mutable_namespace_()->set_id(namespace_id);
168	1	}
169
170		// Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
171		// though, as that helps with readability and standardization.
172	1	RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
173	1	if (table_id) {
174	1	*table_id = resp.table_id();
175	1	}
176	1	if (resp.has_error()) {
177	0	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
178	0	}
179	1	return Status::OK();
180	1	}
181
182		Status MasterTestBase::DoCreateTable(const NamespaceName& namespace_name,
183		const TableName& table_name,
184		const Schema& schema,
185		CreateTableRequestPB* request,
186	21	TableId* table_id /* = nullptr */) {
187	21	CreateTableResponsePB resp;
188
189	21	request->set_name(table_name);
190	21	SchemaToPB(schema, request->mutable_schema());
191
192	21	if (!namespace_name.empty()) {
193	21	request->mutable_namespace_()->set_name(namespace_name);
194	21	}
195	21	request->mutable_partition_schema()->set_hash_schema(PartitionSchemaPB::MULTI_COLUMN_HASH_SCHEMA);
196	21	request->mutable_schema()->mutable_table_properties()->set_num_tablets(8);
197
198		// Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
199		// though, as that helps with readability and standardization.
200	21	RETURN_NOT_OK(proxy_ddl_->CreateTable(*request, &resp, ResetAndGetController()));
201	21	if (table_id) {
202	4	*table_id = resp.table_id();
203	4	}
204	21	if (resp.has_error()) {
205	4	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
206	4	}
207
208	17	return Status::OK();
209	21	}
210
211	35	void MasterTestBase::DoListTables(const ListTablesRequestPB& req, ListTablesResponsePB* resp) {
212	35	ASSERT_OK(proxy_ddl_->ListTables(req, resp, ResetAndGetController()));
213	35	SCOPED_TRACE(resp->DebugString());
214	35	ASSERT_FALSE(resp->has_error());
215	35	}
216
217		void MasterTestBase::DoListAllTables(ListTablesResponsePB* resp,
218	25	const NamespaceName& namespace_name /= ""/) {
219	25	ListTablesRequestPB req;
220
221	25	if (!namespace_name.empty()) {
222	2	req.mutable_namespace_()->set_name(namespace_name);
223	2	}
224
225	25	DoListTables(req, resp);
226	25	}
227
228	1	Status MasterTestBase::DeleteTableById(const TableId& table_id) {
229	1	DeleteTableRequestPB req;
230	1	DeleteTableResponsePB resp;
231	1	req.mutable_table()->set_table_id(table_id);
232	1	RETURN_NOT_OK(proxy_ddl_->DeleteTable(req, &resp, ResetAndGetController()));
233	1	SCOPED_TRACE(resp.DebugString());
234	1	if (resp.has_error()) {
235	0	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
236	0	}
237	1	return Status::OK();
238	1	}
239
240		Status MasterTestBase::DeleteTable(const NamespaceName& namespace_name,
241		const TableName& table_name,
242	10	TableId* table_id /* = nullptr */) {
243	10	SCHECK(!namespace_name.empty(), InvalidArgument, "Namespace name was empty");
244	10	DeleteTableRequestPB req;
245	10	DeleteTableResponsePB resp;
246	10	req.mutable_table()->set_table_name(table_name);
247
248	10	req.mutable_table()->mutable_namespace_()->set_name(namespace_name);
249
250	10	RETURN_NOT_OK(proxy_ddl_->DeleteTable(req, &resp, ResetAndGetController()));
251	10	SCOPED_TRACE(resp.DebugString());
252	10	if (table_id) {
253	3	*table_id = resp.table_id();
254	3	}
255
256	10	if (resp.has_error()) {
257	1	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
258	1	}
259	9	return Status::OK();
260	10	}
261
262		Status MasterTestBase::CreateTablegroup(const TablegroupId& tablegroup_id,
263		const NamespaceId& namespace_id,
264		const NamespaceName& namespace_name,
265	1	const TablespaceId& tablespace_id) {
266	1	CreateTablegroupRequestPB req, *request;
267	1	request = &req;
268	1	CreateTablegroupResponsePB resp;
269
270	1	request->set_id(tablegroup_id);
271	1	request->set_namespace_id(namespace_id);
272	1	request->set_namespace_name(namespace_name);
273	1	request->set_tablespace_id(tablespace_id);
274
275		// Dereferencing as the RPCs require const ref for request. Keeping request param as pointer
276		// though, as that helps with readability and standardization.
277	1	RETURN_NOT_OK(proxy_ddl_->CreateTablegroup(*request, &resp, ResetAndGetController()));
278	1	if (resp.has_error()) {
279	0	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
280	0	}
281	1	return Status::OK();
282	1	}
283
284		Status MasterTestBase::DeleteTablegroup(const TablegroupId& tablegroup_id,
285	1	const NamespaceId& namespace_id) {
286	1	DeleteTablegroupRequestPB req;
287	1	DeleteTablegroupResponsePB resp;
288	1	req.set_id(tablegroup_id);
289	1	req.set_namespace_id(namespace_id);
290
291	1	RETURN_NOT_OK(proxy_ddl_->DeleteTablegroup(req, &resp, ResetAndGetController()));
292	1	if (resp.has_error()) {
293	0	RETURN_NOT_OK(StatusFromPB(resp.error().status()));
294	0	}
295	1	return Status::OK();
296	1	}
297
298		void MasterTestBase::DoListTablegroups(const ListTablegroupsRequestPB& req,
299	2	ListTablegroupsResponsePB* resp) {
300	2	ASSERT_OK(proxy_ddl_->ListTablegroups(req, resp, ResetAndGetController()));
301	2	SCOPED_TRACE(resp->DebugString());
302	2	ASSERT_FALSE(resp->has_error());
303	2	}
304
305	31	void MasterTestBase::DoListAllNamespaces(ListNamespacesResponsePB* resp) {
306	31	DoListAllNamespaces(boost::none, resp);
307	31	}
308
309		void MasterTestBase::DoListAllNamespaces(const boost::optional<YQLDatabase>& database_type,
310	36	ListNamespacesResponsePB* resp) {
311	36	ListNamespacesRequestPB req;
312	36	if (database_type) {
313	5	req.set_database_type(*database_type);
314	5	}
315
316	36	ASSERT_OK(proxy_ddl_->ListNamespaces(req, resp, ResetAndGetController()));
317	36	SCOPED_TRACE(resp->DebugString());
318	36	ASSERT_FALSE(resp->has_error());
319	36	}
320
321		Status MasterTestBase::CreateNamespace(const NamespaceName& ns_name,
322	43	CreateNamespaceResponsePB* resp) {
323	43	return CreateNamespace(ns_name, boost::none, resp);
324	43	}
325
326		Status MasterTestBase::CreateNamespace(const NamespaceName& ns_name,
327		const boost::optional<YQLDatabase>& database_type,
328	64	CreateNamespaceResponsePB* resp) {
329	64	RETURN_NOT_OK(CreateNamespaceAsync(ns_name, database_type, resp));
330	52	return CreateNamespaceWait(resp->id(), database_type);
331	64	}
332
333		Status MasterTestBase::CreateNamespaceAsync(const NamespaceName& ns_name,
334		const boost::optional<YQLDatabase>& database_type,
335	77	CreateNamespaceResponsePB* resp) {
336	77	CreateNamespaceRequestPB req;
337	77	req.set_name(ns_name);
338	77	if (database_type) {
339	34	req.set_database_type(*database_type);
340	34	}
341
342	77	RETURN_NOT_OK(proxy_ddl_->CreateNamespace(req, resp, ResetAndGetController()));
343	77	if (resp->has_error()) {
344	12	RETURN_NOT_OK(StatusFromPB(resp->error().status()));
345	12	}
346	65	return Status::OK();
347	77	}
348
349		Status MasterTestBase::CreateNamespaceWait(const NamespaceId& ns_id,
350	54	const boost::optional<YQLDatabase>& database_type) {
351	54	Status status = Status::OK();
352
353	54	IsCreateNamespaceDoneRequestPB is_req;
354	54	is_req.mutable_namespace_()->set_id(ns_id);
355	54	if (database_type) {
356	13	is_req.mutable_namespace_()->set_database_type(*database_type);
357	13	}
358
359	99	return LoggedWaitFor([&]() -> Result<bool> {
360	99	IsCreateNamespaceDoneResponsePB is_resp;
361	99	status = proxy_ddl_->IsCreateNamespaceDone(is_req, &is_resp, ResetAndGetController());
362	99	WARN_NOT_OK(status, "IsCreateNamespaceDone returned unexpected error");
363	99	if (!status.ok()) {
364	0	return status;
365	0	}
366	99	if (is_resp.has_done() && is_resp.done()) {
367	54	if (is_resp.has_error()) {
368	0	return StatusFromPB(is_resp.error().status());
369	0	}
370	54	return true;
371	54	}
372	45	return false;
373	99	}, MonoDelta::FromSeconds(60), "Wait for create namespace to finish async setup tasks.");
374	54	}
375
376		Status MasterTestBase::AlterNamespace(const NamespaceName& ns_name,
377		const NamespaceId& ns_id,
378		const boost::optional<YQLDatabase>& database_type,
379		const std::string& new_name,
380	4	AlterNamespaceResponsePB* resp) {
381	4	AlterNamespaceRequestPB req;
382	4	req.mutable_namespace_()->set_id(ns_id);
383	4	req.mutable_namespace_()->set_name(ns_name);
384	4	if (database_type) {
385	2	req.mutable_namespace_()->set_database_type(*database_type);
386	2	}
387	4	req.set_new_name(new_name);
388
389	4	RETURN_NOT_OK(proxy_ddl_->AlterNamespace(req, resp, ResetAndGetController()));
390	4	if (resp->has_error()) {
391	1	RETURN_NOT_OK(StatusFromPB(resp->error().status()));
392	1	}
393	3	return Status::OK();
394	4	}
395
396		// PGSQL Namespaces are deleted asynchronously since they may delete a large number of tables.
397		// CQL Namespaces don't need to call this function and return success if present.
398	24	Status MasterTestBase::DeleteNamespaceWait(IsDeleteNamespaceDoneRequestPB const& del_req) {
399	62	return LoggedWaitFor([&]() -> Result<bool> {
400	62	IsDeleteNamespaceDoneResponsePB del_resp;
401	62	auto status = proxy_ddl_->IsDeleteNamespaceDone(del_req, &del_resp, ResetAndGetController());
402	62	if (!status.ok()) {
403	0	WARN_NOT_OK(status, "IsDeleteNamespaceDone returned unexpected error");
404	0	return status;
405	0	}
406	62	if (del_resp.has_done() && del_resp.done()) {
407	24	if (del_resp.has_error()) {
408	5	return StatusFromPB(del_resp.error().status());
409	5	}
410	19	return true;
411	24	}
412	38	return false;
413	62	}, MonoDelta::FromSeconds(10), "Wait for delete namespace to finish async cleanup tasks.");
414	24	}
415
416		Status MasterTestBase::DeleteTableSync(const NamespaceName& ns_name, const TableName& table_name,
417	2	TableId* table_id) {
418	2	RETURN_NOT_OK(DeleteTable(ns_name, table_name, table_id));
419
420	1	IsDeleteTableDoneRequestPB done_req;
421	1	done_req.set_table_id(*table_id);
422	1	IsDeleteTableDoneResponsePB done_resp;
423	1	bool delete_done = false;
424
425	1	for (int num_retries = 0; num_retries < 30; ++num_retries0 ) {
426	1	RETURN_NOT_OK(proxy_ddl_->IsDeleteTableDone(done_req, &done_resp, ResetAndGetController()));
427	1	if (!done_resp.has_done()) {
428	0	return STATUS_FORMAT(
429	0	IllegalState, "Expected IsDeleteTableDone response to set value for done ($0.$1)",
430	0	ns_name, table_name);
431	0	}
432	1	if (done_resp.done()) {
433	1	LOG(INFO) << "Done on retry " << num_retries;
434	1	delete_done = true;
435	1	break;
436	1	}
437
438	0	SleepFor(MonoDelta::FromMilliseconds(10 * num_retries)); // sleep a bit more with each attempt.
439	0	}
440
441	1	if (!delete_done) {
442	0	return STATUS_FORMAT(IllegalState, "Delete Table did not complete ($0.$1)",
443	0	ns_name, table_name);
444	0	}
445	1	return Status::OK();
446	1	}
447
448		void MasterTestBase::CheckNamespaces(
449		const std::set<std::tuple<NamespaceName, NamespaceId>>& namespace_info,
450	32	const ListNamespacesResponsePB& namespaces) {
451	167	for (int i = 0; i < namespaces.namespaces_size(); i++135 ) {
452	135	auto search_key = std::make_tuple(namespaces.namespaces(i).name(),
453	135	namespaces.namespaces(i).id());
454	270	ASSERT_TRUE(namespace_info.find(search_key) != namespace_info.end())
455	270	<< strings::Substitute("Couldn't find namespace $0", namespaces.namespaces(i)
456	270	.name());
457	135	}
458
459	32	ASSERT_EQ(namespaces.namespaces_size(), namespace_info.size());
460	32	}
461
462		bool MasterTestBase::FindNamespace(
463		const std::tuple<NamespaceName, NamespaceId>& namespace_info,
464	4	const ListNamespacesResponsePB& namespaces) {
465	20	for (int i = 0; i < namespaces.namespaces_size(); i++16 ) {
466	16	auto cur_ns = std::make_tuple(namespaces.namespaces(i).name(),
467	16	namespaces.namespaces(i).id());
468	16	if (cur_ns == namespace_info) {
469	0	return true; // found!
470	0	}
471	16	}
472	4	return false; // namespace not found.
473	4	}
474
475		void MasterTestBase::CheckTables(
476		const std::set<std::tuple<TableName, NamespaceName, NamespaceId, bool>>& table_info,
477	25	const ListTablesResponsePB& tables) {
478	437	for (int i = 0; i < tables.tables_size(); i++412 ) {
479	412	auto search_key = std::make_tuple(tables.tables(i).name(),
480	412	tables.tables(i).namespace_().name(),
481	412	tables.tables(i).namespace_().id(),
482	412	tables.tables(i).relation_type());
483	824	ASSERT_TRUE(table_info.find(search_key) != table_info.end())
484	824	<< strings::Substitute("Couldn't find table $0.$1",
485	824	tables.tables(i).namespace_().name(), tables.tables(i).name());
486	412	}
487
488	25	ASSERT_EQ(tables.tables_size(), table_info.size());
489	25	}
490
491	3	void MasterTestBase::UpdateMasterClusterConfig(SysClusterConfigEntryPB* cluster_config) {
492	3	ChangeMasterClusterConfigRequestPB change_req;
493	3	change_req.mutable_cluster_config()->CopyFrom(*cluster_config);
494	3	ChangeMasterClusterConfigResponsePB change_resp;
495	3	rpc::ProxyCache proxy_cache(client_messenger_.get());
496	3	master::MasterClusterProxy proxy(&proxy_cache, mini_master_->bound_rpc_addr());
497	3	ASSERT_OK(proxy.ChangeMasterClusterConfig(change_req, &change_resp, ResetAndGetController()));
498		// Bump version number by 1, so we do not have to re-query.
499	3	cluster_config->set_version(cluster_config->version() + 1);
500	3	LOG(INFO) << "Update cluster config to: " << cluster_config->ShortDebugString();
501	3	}
502
503		} // namespace master
504		} // namespace yb