YugabyteDB (2.13.0.0-b42, bfc6a6643e7399ac8a0e81d06a3ee6d6571b33ab)

// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//

//   http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.

//

// The following only applies to changes made to this file as part of YugaByte development.

//

// Portions Copyright (c) YugaByte, Inc.

//

// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except

// in compliance with the License.  You may obtain a copy of the License at

//

// http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software distributed under the License

// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express

// or implied.  See the License for the specific language governing permissions and limitations

// under the License.

//

#include "yb/master/master-path-handlers.h"

#include <algorithm>

#include <functional>

#include <iomanip>

#include <map>

#include <sstream>

#include <unordered_set>

#include "yb/common/hybrid_time.h"

#include "yb/common/partition.h"

#include "yb/common/schema.h"

#include "yb/common/transaction.h"

#include "yb/common/wire_protocol.h"

#include "yb/gutil/map-util.h"

#include "yb/gutil/stringprintf.h"

#include "yb/gutil/strings/join.h"

#include "yb/gutil/strings/numbers.h"

#include "yb/gutil/strings/substitute.h"

#include "yb/master/master_fwd.h"

#include "yb/master/catalog_entity_info.h"

#include "yb/master/catalog_manager_if.h"

#include "yb/master/master.h"

#include "yb/master/master_cluster.pb.h"

#include "yb/master/master_util.h"

#include "yb/master/scoped_leader_shared_lock.h"

#include "yb/master/sys_catalog.h"

#include "yb/master/ts_descriptor.h"

#include "yb/master/ts_manager.h"

#include "yb/server/webserver.h"

#include "yb/server/webui_util.h"

#include "yb/util/curl_util.h"

#include "yb/util/jsonwriter.h"

#include "yb/util/status_log.h"

#include "yb/util/string_case.h"

#include "yb/util/timestamp.h"

#include "yb/util/url-coding.h"

#include "yb/util/version_info.h"

#include "yb/util/version_info.pb.h"

DEFINE_int32(

    hide_dead_node_threshold_mins, 60 * 24,

    "After this many minutes of no heartbeat from a node, hide it from the UI "

    "(we presume it has been removed from the cluster). If -1, this flag is ignored and node is "

    "never hidden from the UI");

DECLARE_int32(ysql_tablespace_info_refresh_secs);

namespace yb {

namespace {

static constexpr const char* kDBTypeNameUnknown = "unknown";

static constexpr const char* kDBTypeNameCql = "ycql";

static constexpr const char* kDBTypeNamePgsql = "ysql";

static constexpr const char* kDBTypeNameRedis = "yedis";

const int64_t kCurlTimeoutSec = 180;

const char* DatabaseTypeName(YQLDatabase db) {

  switch (db) {

    case YQL_DATABASE_UNKNOWN: break;

    case YQL_DATABASE_CQL: return kDBTypeNameCql;

    case YQL_DATABASE_PGSQL: return kDBTypeNamePgsql;

    case YQL_DATABASE_REDIS: return kDBTypeNameRedis;

  }

  CHECK(false) << "Unexpected db type " << db;

  return kDBTypeNameUnknown;

}

YQLDatabase DatabaseTypeByName(const string& db_type_name) {

  static const std::array<pair<const char*, YQLDatabase>, 3> db_types{

      make_pair(kDBTypeNameCql, YQLDatabase::YQL_DATABASE_CQL),

      make_pair(kDBTypeNamePgsql, YQLDatabase::YQL_DATABASE_PGSQL),

      make_pair(kDBTypeNameRedis, YQLDatabase::YQL_DATABASE_REDIS)};

  for (const auto& db : db_types) {

    if (db_type_name == db.first) {

      return db.second;

    }

  }

  return YQLDatabase::YQL_DATABASE_UNKNOWN;

}

} // namespace

using consensus::RaftPeerPB;

using std::vector;

using std::map;

using std::string;

using std::stringstream;

using std::unique_ptr;

using strings::Substitute;

using server::MonitoredTask;

using namespace std::placeholders;

namespace master {

MasterPathHandlers::~MasterPathHandlers() {

}

void MasterPathHandlers::TabletCounts::operator+=(const TabletCounts& other) {

  user_tablet_leaders += other.user_tablet_leaders;

  user_tablet_followers += other.user_tablet_followers;

  system_tablet_leaders += other.system_tablet_leaders;

  system_tablet_followers += other.system_tablet_followers;

}

MasterPathHandlers::ZoneTabletCounts::ZoneTabletCounts(

  const TabletCounts& tablet_counts,

  uint32_t active_tablets_count) : tablet_counts(tablet_counts),

                                   active_tablets_count(active_tablets_count) {

}

void MasterPathHandlers::ZoneTabletCounts::operator+=(const ZoneTabletCounts& other) {

  tablet_counts += other.tablet_counts;

  node_count += other.node_count;

  active_tablets_count += other.active_tablets_count;

}

// Retrieve the specified URL response from the leader master

void MasterPathHandlers::RedirectToLeader(const Webserver::WebRequest& req,

                                          Webserver::WebResponse* resp) {

  std::stringstream *output = &resp->output;

  vector<ServerEntryPB> masters;

  Status s = master_->ListMasters(&masters);

  if (!s.ok()) {

    s = s.CloneAndPrepend("Unable to list masters during web request handling");

    LOG(WARNING) << s.ToString();

    *output << "<h2>" << s.ToString() << "</h2>\n";

    return;

  }

  string redirect;

  for (const ServerEntryPB& master : masters) {

    if (master.has_error()) {

      continue;

    }

    if (master.role() == PeerRole::LEADER) {

      // URI already starts with a /, so none is needed between $1 and $2.

      if (master.registration().http_addresses().size() > 0) {

        redirect = Substitute(

            "http://$0$1$2",

            HostPortPBToString(master.registration().http_addresses(0)),

            req.redirect_uri,

            req.query_string.empty() ? "?raw" : "?" + req.query_string + "&raw");

      }

      break;

    }

  }

  if (redirect.empty()) {

    string error = "Unable to locate leader master to redirect this request: " + redirect;

    LOG(WARNING) << error;

    *output << error << "<br>";

    return;

  }

  EasyCurl curl;

  faststring buf;

  s = curl.FetchURL(redirect, &buf, kCurlTimeoutSec);

  if (!s.ok()) {

    LOG(WARNING) << "Error retrieving leader master URL: " << redirect

                 << ", error :" << s.ToString();

    *output << "Error retrieving leader master URL: <a href=\"" << redirect

            << "\">" + redirect + "</a><br> Error: " << s.ToString() << ".<br>";

    return;

  }

  *output << buf.ToString();

}

void MasterPathHandlers::CallIfLeaderOrPrintRedirect(

    const Webserver::WebRequest& req, Webserver::WebResponse* resp,

    const Webserver::PathHandlerCallback& callback) {

  string redirect;

  // Lock the CatalogManager in a self-contained block, to prevent double-locking on callbacks.

  {

    SCOPED_LEADER_SHARED_LOCK(l, master_->catalog_manager_impl());

    // If we are not the master leader, redirect the URL.

    if (!l.first_failed_status().ok()) {

      RedirectToLeader(req, resp);

      return;

    }

    // Handle the request as a leader master.

    callback(req, resp);

    return;

  }

}

inline void MasterPathHandlers::TServerTable(std::stringstream* output,

                                             TServersViewType viewType) {

  *output << "<table class='table table-striped'>\n";

  *output << "    <tr>\n"

          << "      <th>Server</th>\n"

          << "      <th>Time since </br>heartbeat</th>\n"

          << "      <th>Status & Uptime</th>\n";

  if (viewType == TServersViewType::kTServersClocksView) {

    *output << "      <th>Physical Time (UTC)</th>\n"

            << "      <th>Hybrid Time (UTC)</th>\n"

            << "      <th>Heartbeat RTT</th>\n";

  } else {

    DCHECK_EQ(viewType, TServersViewType::kTServersDefaultView);

    *output << "      <th>User Tablet-Peers / Leaders</th>\n"

            << "      <th>RAM Used</th>\n"

            << "      <th>Num SST Files</th>\n"

            << "      <th>Total SST Files Size</th>\n"

            << "      <th>Uncompressed SST </br>Files Size</th>\n"

            << "      <th>Read ops/sec</th>\n"

            << "      <th>Write ops/sec</th>\n";

  }

  *output << "      <th>Cloud</th>\n"

          << "      <th>Region</th>\n"

          << "      <th>Zone</th>\n";

  if (viewType == TServersViewType::kTServersDefaultView) {

    *output << "      <th>System Tablet-Peers / Leaders</th>\n"

            << "      <th>Active Tablet-Peers</th>\n";

  }

  *output << "    </tr>\n";

}

namespace {

constexpr int kHoursPerDay = 24;

constexpr int kSecondsPerMinute = 60;

constexpr int kMinutesPerHour = 60;

constexpr int kSecondsPerHour = kSecondsPerMinute * kMinutesPerHour;

constexpr int kMinutesPerDay = kMinutesPerHour * kHoursPerDay;

constexpr int kSecondsPerDay = kSecondsPerHour * kHoursPerDay;

string UptimeString(uint64_t seconds) {

  auto days = seconds / kSecondsPerDay;

  auto hours = (seconds / kSecondsPerHour) - (days * kHoursPerDay);

  auto mins = (seconds / kSecondsPerMinute) - (days * kMinutesPerDay) - (hours * kMinutesPerHour);

  std::ostringstream uptime_string_stream;

  uptime_string_stream << " ";

  if (days > 0) {

    uptime_string_stream << days << "days, ";

  }

  uptime_string_stream << hours << ":" << std::setw(2) << std::setfill('0') << mins <<

      ":" << std::setw(2) << std::setfill('0') << (seconds % 60);

  return uptime_string_stream.str();

}

bool ShouldHideTserverNodeFromDisplay(

    const TSDescriptor* ts, int hide_dead_node_threshold_mins) {

  return hide_dead_node_threshold_mins > 0

      && !ts->IsLive()

      && ts->TimeSinceHeartbeat().ToMinutes() > hide_dead_node_threshold_mins;

}

int GetTserverCountForDisplay(const TSManager* ts_manager) {

  int count = 0;

  for (const auto& tserver : ts_manager->GetAllDescriptors()) {

    if (!ShouldHideTserverNodeFromDisplay(tserver.get(), FLAGS_hide_dead_node_threshold_mins)) {

      count++;

    }

  }

  return count;

}

} // anonymous namespace

string MasterPathHandlers::GetHttpHostPortFromServerRegistration(

    const ServerRegistrationPB& reg) const {

  if (reg.http_addresses().size() > 0) {

    return HostPortPBToString(reg.http_addresses(0));

  }

  return "";

}

namespace {

bool TabletServerComparator(

    const std::shared_ptr<TSDescriptor>& a, const std::shared_ptr<TSDescriptor>& b) {

  auto a_cloud_info = a->GetRegistration().common().cloud_info();

  auto b_cloud_info = b->GetRegistration().common().cloud_info();

  if (a_cloud_info.placement_cloud() == b_cloud_info.placement_cloud()) {

    if (a_cloud_info.placement_region() == b_cloud_info.placement_region()) {

      if (a_cloud_info.placement_zone() == b_cloud_info.placement_zone()) {

        return a->permanent_uuid() < b->permanent_uuid();

      }

      return a_cloud_info.placement_zone() < b_cloud_info.placement_zone();

    }

    return a_cloud_info.placement_region() < b_cloud_info.placement_region();

  }

  return a_cloud_info.placement_cloud() < b_cloud_info.placement_cloud();

}

} // anonymous namespace

void MasterPathHandlers::TServerDisplay(const std::string& current_uuid,

                                        std::vector<std::shared_ptr<TSDescriptor>>* descs,

                                        TabletCountMap* tablet_map,

                                        std::stringstream* output,

                                        const int hide_dead_node_threshold_mins,

                                        TServersViewType viewType) {

  // Copy vector to avoid changes to the reference descs passed

  std::vector<std::shared_ptr<TSDescriptor>> local_descs(*descs);

  // Comparator orders by cloud, region, zone and uuid fields.

  std::sort(local_descs.begin(), local_descs.end(), &TabletServerComparator);

  for (auto desc : local_descs) {

    if (desc->placement_uuid() == current_uuid) {

      if (ShouldHideTserverNodeFromDisplay(desc.get(), hide_dead_node_threshold_mins)) {

        continue;

      }

      const string time_since_hb = StringPrintf("%.1fs", desc->TimeSinceHeartbeat().ToSeconds());

      TSRegistrationPB reg = desc->GetRegistration();

      string host_port = GetHttpHostPortFromServerRegistration(reg.common());

      *output << "  <tr>\n";

      *output << "  <td>" << RegistrationToHtml(reg.common(), host_port) << "</br>";

      *output << "  " << desc->permanent_uuid() << "</td>";

      *output << "<td>" << time_since_hb << "</td>";

      if (desc->IsLive()) {

        *output << "    <td style=\"color:Green\">" << kTserverAlive << ":" <<

                UptimeString(desc->uptime_seconds()) << "</td>";

      } else {

        *output << "    <td style=\"color:Red\">" << kTserverDead << "</td>";

      }

      auto tserver = tablet_map->find(desc->permanent_uuid());

      bool no_tablets = tserver == tablet_map->end();

      if (viewType == TServersViewType::kTServersClocksView) {

        // Render physical time.

        const Timestamp p_ts(desc->physical_time());

        *output << "    <td>" << p_ts.ToHumanReadableTime() << "</td>";

        // Render the physical and logical components of the hybrid time.

        const HybridTime ht = desc->hybrid_time();

        const Timestamp h_ts(ht.GetPhysicalValueMicros());

        *output << "    <td>" << h_ts.ToHumanReadableTime();

        if (ht.GetLogicalValue()) {

          *output << " / Logical: " << ht.GetLogicalValue();

        }

        *output << "</td>";

        // Render the roundtrip time of previous heartbeat.

        double rtt_ms = desc->heartbeat_rtt().ToMicroseconds()/1000.0;

        *output << "    <td>" <<  StringPrintf("%.2fms", rtt_ms) << "</td>";

      } else {

        DCHECK_EQ(viewType, TServersViewType::kTServersDefaultView);

        *output << "    <td>" << (no_tablets ? 0

                : tserver->second.user_tablet_leaders + tserver->second.user_tablet_followers)

                << " / " << (no_tablets ? 0 : tserver->second.user_tablet_leaders) << "</td>";

        *output << "    <td>" << HumanizeBytes(desc->total_memory_usage()) << "</td>";

        *output << "    <td>" << desc->num_sst_files() << "</td>";

        *output << "    <td>" << HumanizeBytes(desc->total_sst_file_size()) << "</td>";

        *output << "    <td>" << HumanizeBytes(desc->uncompressed_sst_file_size()) << "</td>";

        *output << "    <td>" << desc->read_ops_per_sec() << "</td>";

        *output << "    <td>" << desc->write_ops_per_sec() << "</td>";

      }

      *output << "    <td>" << reg.common().cloud_info().placement_cloud() << "</td>";

      *output << "    <td>" << reg.common().cloud_info().placement_region() << "</td>";

      *output << "    <td>" << reg.common().cloud_info().placement_zone() << "</td>";

      if (viewType == TServersViewType::kTServersDefaultView) {

        *output << "    <td>" << (no_tablets ? 0

                : tserver->second.system_tablet_leaders + tserver->second.system_tablet_followers)

                << " / " << (no_tablets ? 0 : tserver->second.system_tablet_leaders) << "</td>";

        *output << "    <td>" << (no_tablets ? 0 : desc->num_live_replicas()) << "</td>";

      }

      *output << "  </tr>\n";

    }

  }

  *output << "</table>\n";

}

void MasterPathHandlers::DisplayTabletZonesTable(

  const ZoneTabletCounts::CloudTree& cloud_tree,

  std::stringstream* output

) {

  *output << "<h3>Tablet-Peers by Availability Zone</h3>\n"

          << "<table class='table table-striped'>\n"

          << "  <tr>\n"

          << "    <th>Cloud</th>\n"

          << "    <th>Region</th>\n"

          << "    <th>Zone</th>\n"

          << "    <th>Total Nodes</th>\n"

          << "    <th>User Tablet-Peers / Leaders</th>\n"

          << "    <th>System Tablet-Peers / Leaders</th>\n"

          << "    <th>Active Tablet-Peers</th>\n"

          << "  </tr>\n";

  for (const auto& cloud_iter : cloud_tree) {

    const auto& region_tree = cloud_iter.second;

    bool needs_new_row = false;

    int total_size_rows = 0;

    for (const auto& region_iter : region_tree) {

      total_size_rows += region_iter.second.size();

    }

    *output << "<tr>\n"

            << "  <td rowspan=\"" << total_size_rows <<"\">" << cloud_iter.first << "</td>\n";

    for (const auto& region_iter : region_tree) {

      const auto& zone_tree = region_iter.second;

      if (needs_new_row) {

        *output << "<tr>\n";

        needs_new_row = false;

      }

      *output << "  <td rowspan=\"" << zone_tree.size() <<"\">" << region_iter.first

              << "</td>\n";

      for (const auto& zone_iter : zone_tree) {

        const auto& counts = zone_iter.second;

        if (needs_new_row) {

          *output << "<tr>\n";

        }

        *output << "  <td>" << zone_iter.first << "</td>\n";

        uint32_t user_leaders = counts.tablet_counts.user_tablet_leaders;

        uint32_t user_total = user_leaders + counts.tablet_counts.user_tablet_followers;

        uint32_t system_leaders = counts.tablet_counts.system_tablet_leaders;

        uint32_t system_total = system_leaders + counts.tablet_counts.system_tablet_followers;

        *output << "  <td>" << counts.node_count << "</td>\n"

                << "  <td>" << user_total << " / " << user_leaders << "</td>\n"

                << "  <td>" << system_total << " / " << system_leaders << "</td>\n"

                << "  <td>" << counts.active_tablets_count << "</td>\n"

                << "</tr>\n";

        needs_new_row = true;

      }

    }

  }

  *output << "</table>\n";

}

MasterPathHandlers::ZoneTabletCounts::CloudTree MasterPathHandlers::CalculateTabletCountsTree(

  const std::vector<std::shared_ptr<TSDescriptor>>& descriptors,

  const TabletCountMap& tablet_count_map

) {

  ZoneTabletCounts::CloudTree cloud_tree;

  for (const auto& descriptor : descriptors) {

    CloudInfoPB cloud_info = descriptor->GetRegistration().common().cloud_info();

    std::string cloud = cloud_info.placement_cloud();

    std::string region = cloud_info.placement_region();

    std::string zone = cloud_info.placement_zone();

    auto tablet_count_search = tablet_count_map.find(descriptor->permanent_uuid());

    ZoneTabletCounts counts = tablet_count_search == tablet_count_map.end()

        ? ZoneTabletCounts()

        : ZoneTabletCounts(tablet_count_search->second, descriptor->num_live_replicas());

    auto cloud_iter = cloud_tree.find(cloud);

    if (cloud_iter == cloud_tree.end()) {

      ZoneTabletCounts::RegionTree region_tree;

      ZoneTabletCounts::ZoneTree zone_tree;

      zone_tree.emplace(zone, std::move(counts));

      region_tree.emplace(region, std::move(zone_tree));

      cloud_tree.emplace(cloud, std::move(region_tree));

    } else {

      ZoneTabletCounts::RegionTree& region_tree = cloud_iter->second;

      auto region_iter = region_tree.find(region);

      if (region_iter == region_tree.end()) {

        ZoneTabletCounts::ZoneTree zone_tree;

        zone_tree.emplace(zone, std::move(counts));

        region_tree.emplace(region, std::move(zone_tree));

      } else {

        ZoneTabletCounts::ZoneTree& zone_tree = region_iter->second;

        auto zone_iter = zone_tree.find(zone);

        if (zone_iter == zone_tree.end()) {

          zone_tree.emplace(zone, std::move(counts));

        } else {

          zone_iter->second += counts;

        }

      }

    }

  }

  return cloud_tree;

}

void MasterPathHandlers::HandleTabletServers(const Webserver::WebRequest& req,

                                             Webserver::WebResponse* resp,

                                             TServersViewType viewType) {

  std::stringstream *output = &resp->output;

  master_->catalog_manager()->AssertLeaderLockAcquiredForReading();

  auto threshold_arg = req.parsed_args.find("live_threshold_mins");

  int hide_dead_node_threshold_override = FLAGS_hide_dead_node_threshold_mins;

  if (threshold_arg != req.parsed_args.end()) {

    hide_dead_node_threshold_override = atoi(threshold_arg->second.c_str());

  }

  SysClusterConfigEntryPB config;

  Status s = master_->catalog_manager()->GetClusterConfig(&config);

  if (!s.ok()) {

    *output << "<div class=\"alert alert-warning\">" << s.ToString() << "</div>";

    return;

  }

  auto live_id = config.replication_info().live_replicas().placement_uuid();

  vector<std::shared_ptr<TSDescriptor> > descs;

  const auto& ts_manager = master_->ts_manager();

  ts_manager->GetAllDescriptors(&descs);

  // Get user and system tablet leader and follower counts for each TabletServer

  TabletCountMap tablet_map;

  CalculateTabletMap(&tablet_map);

  std::unordered_set<string> read_replica_uuids;

  for (auto desc : descs) {

    if (!read_replica_uuids.count(desc->placement_uuid()) && desc->placement_uuid() != live_id) {

      read_replica_uuids.insert(desc->placement_uuid());

    }

  }

  *output << std::setprecision(output_precision_);

  *output << "<h2>Tablet Servers</h2>\n";

  if (!live_id.empty()) {

    *output << "<h3 style=\"color:" << kYBDarkBlue << "\">Primary Cluster UUID: "

            << live_id << "</h3>\n";

  }

  TServerTable(output, viewType);

  TServerDisplay(live_id, &descs, &tablet_map, output, hide_dead_node_threshold_override,

                 viewType);

  for (const auto& read_replica_uuid : read_replica_uuids) {

    *output << "<h3 style=\"color:" << kYBDarkBlue << "\">Read Replica UUID: "

            << (read_replica_uuid.empty() ? kNoPlacementUUID : read_replica_uuid) << "</h3>\n";

    TServerTable(output, viewType);

    TServerDisplay(read_replica_uuid, &descs, &tablet_map, output,

                   hide_dead_node_threshold_override, viewType);

  }

  ZoneTabletCounts::CloudTree counts_tree = CalculateTabletCountsTree(descs, tablet_map);

  DisplayTabletZonesTable(counts_tree, output);

}

void MasterPathHandlers::HandleGetTserverStatus(const Webserver::WebRequest& req,

                                             Webserver::WebResponse* resp) {

  std::stringstream *output = &resp->output;

  master_->catalog_manager()->AssertLeaderLockAcquiredForReading();

  JsonWriter jw(output, JsonWriter::COMPACT);

  SysClusterConfigEntryPB config;

  Status s = master_->catalog_manager()->GetClusterConfig(&config);

  if (!s.ok()) {

    jw.StartObject();

    jw.String("error");

    jw.String(s.ToString());

    return;

  }

  vector<std::shared_ptr<TSDescriptor> > descs;

  const auto& ts_manager = master_->ts_manager();

  ts_manager->GetAllDescriptors(&descs);

  // Get user and system tablet leader and follower counts for each TabletServer.

  TabletCountMap tablet_map;

  CalculateTabletMap(&tablet_map);

  std::unordered_set<string> cluster_uuids;

  auto primary_uuid = config.replication_info().live_replicas().placement_uuid();

  cluster_uuids.insert(primary_uuid);

  for (auto desc : descs) {

    cluster_uuids.insert(desc->placement_uuid());

  }

  jw.StartObject();

  for (const auto& cur_uuid : cluster_uuids) {

    jw.String(cur_uuid);

    jw.StartObject();

    for (auto desc : descs) {

      if (desc->placement_uuid() == cur_uuid) {

        TSRegistrationPB reg = desc->GetRegistration();

        string host_port = GetHttpHostPortFromServerRegistration(reg.common());

        jw.String(host_port);

        jw.StartObject();

        // Some stats may be repeated as strings due to backwards compatability.

        jw.String("time_since_hb");

        jw.String(StringPrintf("%.1fs", desc->TimeSinceHeartbeat().ToSeconds()));

        jw.String("time_since_hb_sec");

        jw.Double(desc->TimeSinceHeartbeat().ToSeconds());

        if (desc->IsLive()) {

          jw.String("status");

          jw.String(kTserverAlive);

          jw.String("uptime_seconds");

          jw.Uint64(desc->uptime_seconds());

        } else {

          jw.String("status");

          jw.String(kTserverDead);

          jw.String("uptime_seconds");

          jw.Uint(0);

        }

        jw.String("ram_used");

        jw.String(HumanizeBytes(desc->total_memory_usage()));

        jw.String("ram_used_bytes");

        jw.Uint64(desc->total_memory_usage());

        jw.String("num_sst_files");

        jw.Uint64(desc->num_sst_files());

        jw.String("total_sst_file_size");

        jw.String(HumanizeBytes(desc->total_sst_file_size()));

        jw.String("total_sst_file_size_bytes");

        jw.Uint64(desc->total_sst_file_size());

        jw.String("uncompressed_sst_file_size");

        jw.String(HumanizeBytes(desc->uncompressed_sst_file_size()));

        jw.String("uncompressed_sst_file_size_bytes");

        jw.Uint64(desc->uncompressed_sst_file_size());

        jw.String("path_metrics");

        jw.StartArray();

        for(const auto& path_metric : desc->path_metrics()) {

          jw.StartObject();

          jw.String("path");

          jw.String(path_metric.first);

          jw.String("space_used");

          jw.Uint64(path_metric.second.used_space);

          jw.String("total_space_size");

          jw.Uint64(path_metric.second.total_space);

          jw.EndObject();

        }

        jw.EndArray();

        jw.String("read_ops_per_sec");

        jw.Double(desc->read_ops_per_sec());

        jw.String("write_ops_per_sec");

        jw.Double(desc->write_ops_per_sec());

        auto tserver = tablet_map.find(desc->permanent_uuid());

        uint user_tablets_total = 0;

        uint user_tablets_leaders = 0;

        uint system_tablets_total = 0;

        uint system_tablets_leaders = 0;

        int active_tablets = 0;

        if (!(tserver == tablet_map.end())) {

          user_tablets_total = tserver->second.user_tablet_leaders +

            tserver->second.user_tablet_followers;

          user_tablets_leaders = tserver->second.user_tablet_leaders;

          system_tablets_total = tserver->second.system_tablet_leaders +

            tserver->second.system_tablet_followers;

          system_tablets_leaders = tserver->second.system_tablet_leaders;

          active_tablets = desc->num_live_replicas();

        }

        jw.String("user_tablets_total");

        jw.Uint(user_tablets_total);

        jw.String("user_tablets_leaders");

        jw.Uint(user_tablets_leaders);

        jw.String("system_tablets_total");

        jw.Uint(system_tablets_total);

        jw.String("system_tablets_leaders");

        jw.Uint(system_tablets_leaders);

        jw.String("active_tablets");

        jw.Int(active_tablets);

        CloudInfoPB cloud_info = reg.common().cloud_info();

        jw.String("cloud");

        jw.String(cloud_info.placement_cloud());

        jw.String("region");

        jw.String(cloud_info.placement_region());

        jw.String("zone");

        jw.String(cloud_info.placement_zone());

        jw.EndObject();

      }

    }

    jw.EndObject();

  }

  jw.EndObject();

}

void MasterPathHandlers::HandleHealthCheck(

    const Webserver::WebRequest& req, Webserver::WebResponse* resp) {

  // TODO: Lock not needed since other APIs handle it.  Refactor other functions accordingly

  std::stringstream *output = &resp->output;

  JsonWriter jw(output, JsonWriter::COMPACT);

  SysClusterConfigEntryPB config;

  Status s = master_->catalog_manager()->GetClusterConfig(&config);

  if (!s.ok()) {

    jw.StartObject();

    jw.String("error");

    jw.String(s.ToString());

    return;

  }

  auto replication_factor = master_->catalog_manager()->GetReplicationFactor();

  if (!replication_factor.ok()) {

    jw.StartObject();

    jw.String("error");

    jw.String(replication_factor.status().ToString());

    return;

  }

  vector<std::shared_ptr<TSDescriptor> > descs;

  const auto* ts_manager = master_->ts_manager();

  ts_manager->GetAllDescriptors(&descs);

  const auto& live_placement_uuid = config.replication_info().live_replicas().placement_uuid();

  // Ignore read replica health for V1.

  vector<std::shared_ptr<TSDescriptor> > dead_nodes;

  uint64_t most_recent_uptime = std::numeric_limits<uint64_t>::max();

  jw.StartObject();

  {

    // Iterate TabletServers, looking for health anomalies.

    for (const auto & desc : descs) {

      if (desc->placement_uuid() == live_placement_uuid) {

        if (!desc->IsLive()) {

          // 1. Are any of the TS marked dead in the master?

          dead_nodes.push_back(desc);

        } else {

          // 2. Have any of the servers restarted lately?

          most_recent_uptime = min(most_recent_uptime, desc->uptime_seconds());

        }

      }

    }

    jw.String("dead_nodes");

    jw.StartArray();

    for (auto const & ts_desc : dead_nodes) {

      jw.String(ts_desc->permanent_uuid());

    }

    jw.EndArray();

    jw.String("most_recent_uptime");

    jw.Uint64(most_recent_uptime);

    auto time_arg = req.parsed_args.find("tserver_death_interval_msecs");

    int64 death_interval_msecs = 0;

    if (time_arg != req.parsed_args.end()) {

      death_interval_msecs = atoi(time_arg->second.c_str());

    }

    // Get all the tablets and add the tablet id for each tablet that has

    // replication locations lesser than 'replication_factor'.

    jw.String("under_replicated_tablets");

    jw.StartArray();

    auto tables = master_->catalog_manager()->GetTables(GetTablesMode::kRunning);

    for (const auto& table : tables) {

      // Ignore tables that are neither user tables nor user indexes.

      // However there are a bunch of system tables that still need to be investigated:

      // 1. Redis system table.

      // 2. Transaction status table.

      // 3. Metrics table.

      if (!master_->catalog_manager()->IsUserTable(*table) &&

          table->GetTableType() != REDIS_TABLE_TYPE &&

          table->GetTableType() != TRANSACTION_STATUS_TABLE_TYPE &&

          !(table->namespace_id() == kSystemNamespaceId &&

            table->name() == kMetricsSnapshotsTableName)) {

        continue;

      }

      TabletInfos tablets = table->GetTablets();

      for (const auto& tablet : tablets) {

        auto replication_locations = tablet->GetReplicaLocations();

        if (replication_locations->size() < *replication_factor) {

          // These tablets don't have the required replication locations needed.

          jw.String(tablet->tablet_id());

          continue;

        }

        // Check if we have tablets that have replicas on the dead node.

        if (dead_nodes.size() == 0) {

          continue;

        }

        size_t recent_replica_count = 0;

        for (const auto& iter : *replication_locations) {

          if (std::find_if(dead_nodes.begin(),

                           dead_nodes.end(),

                           [iter, death_interval_msecs] (const auto& ts) {

                               return (ts->permanent_uuid() == iter.first &&

                                       ts->TimeSinceHeartbeat().ToMilliseconds() >

                                           death_interval_msecs); }) ==

                  dead_nodes.end()) {

            ++recent_replica_count;

          }

        }

        if (recent_replica_count < *replication_factor) {

          jw.String(tablet->tablet_id());

        }

      }

    }

    jw.EndArray();

    // TODO: Add these health checks in a subsequent diff

    //

    // 4. is the load balancer busy moving tablets/leaders around

    /* Use: CHECKED_STATUS IsLoadBalancerIdle(const IsLoadBalancerIdleRequestPB* req,

                                              IsLoadBalancerIdleResponsePB* resp);

     */

    // 5. do any of the TS have tablets they were not able to start up

  }

  jw.EndObject();

}

string MasterPathHandlers::GetParentTableOid(scoped_refptr<TableInfo> parent_table) {

  TableId t_id = parent_table->id();;

  if (parent_table->IsColocatedParentTable()) {

    // No YSQL parent id for colocated database parent table

    return "";

  }

  const auto parent_result = GetPgsqlTablegroupOidByTableId(t_id);

  RETURN_NOT_OK_RET(ResultToStatus(parent_result), "");

  return std::to_string(*parent_result);

}

void MasterPathHandlers::HandleCatalogManager(const Webserver::WebRequest& req,

                                              Webserver::WebResponse* resp,

                                              bool only_user_tables) {

  std::stringstream *output = &resp->output;

  master_->catalog_manager()->AssertLeaderLockAcquiredForReading();

  auto tables = master_->catalog_manager()->GetTables(GetTablesMode::kAll);

  bool has_tablegroups = master_->catalog_manager()->HasTablegroups();

  typedef map<string, string> StringMap;

  // The first stores user tables, the second index tables, and the third system tables.

  std::unique_ptr<StringMap> ordered_tables[kNumTypes];

  for (int i = 0; i < kNumTypes; ++i) {

    ordered_tables[i] = std::make_unique<StringMap>();

  }

  for (const auto& table : tables) {

    auto l = table->LockForRead();

    if (!l->is_running()) {

      continue;

    }

    string keyspace = master_->catalog_manager()->GetNamespaceName(table->namespace_id());

    bool is_platform = keyspace.compare(kSystemPlatformNamespace) == 0;

    TableType table_cat;

    // Determine the table category. YugaWare tables should be displayed as system tables.

    if (is_platform) {

      table_cat = kSystemTable;

    } else if (master_->catalog_manager()->IsUserIndex(*table)) {

      table_cat = kUserIndex;

    } else if (master_->catalog_manager()->IsUserTable(*table)) {

      table_cat = kUserTable;

    } else if (table->IsTablegroupParentTable() ||

               table->IsColocatedParentTable()) {

      table_cat = kColocatedParentTable;

    } else {

      table_cat = kSystemTable;

    }

    // Skip non-user tables if we should.

    if (only_user_tables && (table_cat != kUserIndex && table_cat != kUserTable)) {

      continue;

    }

    string table_uuid = table->id();

    string state = SysTablesEntryPB_State_Name(l->pb.state());

    Capitalize(&state);

    string ysql_table_oid;

    string ysql_parent_oid;

    string display_info = Substitute(

                          "<tr>" \

                          "<td>$0</td>",

                          EscapeForHtmlToString(keyspace));

    if (table->GetTableType() == PGSQL_TABLE_TYPE &&

        !table->IsColocatedParentTable() &&

        !table->IsTablegroupParentTable()) {

      const auto result = GetPgsqlTableOid(table_uuid);

      if (result.ok()) {

        ysql_table_oid = std::to_string(*result);

      } else {

        LOG(ERROR) << "Failed to get OID of '" << table_uuid << "' ysql table";

      }

      display_info += Substitute(

                      "<td><a href=\"/table?id=$3\">$0</a></td>" \

                      "<td>$1</td>" \

                      "<td>$2</td>" \

                      "<td>$3</td>" \

                      "<td>$4</td>",

                      EscapeForHtmlToString(l->name()),

                      state,

                      EscapeForHtmlToString(l->pb.state_msg()),

                      EscapeForHtmlToString(table_uuid),

                      ysql_table_oid);

      if (has_tablegroups) {

        if (table->IsColocatedUserTable()) {

          const auto parent_table = table->GetColocatedTablet()->table();

          ysql_parent_oid = GetParentTableOid(parent_table);

          display_info += Substitute("<td>$0</td>", ysql_parent_oid);

        } else {

          display_info += Substitute("<td></td>");

        }

      }

    } else if (table->IsTablegroupParentTable() ||

               table->IsColocatedParentTable()) {

      // Colocated parent table.

      ysql_table_oid = GetParentTableOid(table);

      // Insert a newline in id and name to wrap long tablegroup text.

      std::string parent_name = l->name();

      display_info += Substitute(

                      "<td><a href=\"/table?id=$0\">$1</a></td>" \