YugabyteDB (2.13.1.0-b60, 21121d69985fbf76aa6958d8f04a9bfa936293b5)

//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.

//  This source code is licensed under the BSD-style license found in the

//  LICENSE file in the root directory of this source tree. An additional grant

//  of patent rights can be found in the PATENTS file in the same directory.

//

// The following only applies to changes made to this file as part of YugaByte development.

//

// Portions Copyright (c) YugaByte, Inc.

//

// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except

// in compliance with the License.  You may obtain a copy of the License at

//

// http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software distributed under the License

// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express

// or implied.  See the License for the specific language governing permissions and limitations

// under the License.

//

#include <cmath>

#include "yb/rocksdb/options.h"

namespace rocksdb {

namespace {

// For now, always use 1-0 as level bytes multiplier.

const int kBytesForLevelMultiplier = 10;

const size_t kBytesForOneMb = 1024 * 1024;

// Pick compaction style

CompactionStyle PickCompactionStyle(size_t write_buffer_size,

                                    int read_amp_threshold,

                                    int write_amp_threshold,

                                    uint64_t target_db_size) {

#ifndef ROCKSDB_LITE

  // Estimate read amplification and write amplification of two compaction

  // styles. If there is hard limit to force a choice, make the choice.

  // Otherwise, calculate a score based on threshold and expected value of

  // two styles, weighing reads 4X important than writes.

  int expected_levels = static_cast<int>(ceil(

      std::log(target_db_size / write_buffer_size) / std::log(kBytesForLevelMultiplier)));

  int expected_max_files_universal =

      static_cast<int>(ceil(log2(target_db_size / write_buffer_size)));

  const int kEstimatedLevel0FilesInLevelStyle = 2;

  // Estimate write amplification:

  // (1) 1 for every L0 file

  // (2) 2 for L1

  // (3) kBytesForLevelMultiplier for the last level. It's really hard to

  //     predict.

  // (3) kBytesForLevelMultiplier for other levels.

  int expected_write_amp_level = kEstimatedLevel0FilesInLevelStyle + 2

      + (expected_levels - 2) * kBytesForLevelMultiplier

      + kBytesForLevelMultiplier;

  int expected_read_amp_level =

      kEstimatedLevel0FilesInLevelStyle + expected_levels;

  int max_read_amp_uni = expected_max_files_universal;

  if (read_amp_threshold <= max_read_amp_uni) {

    return kCompactionStyleLevel;

  } else if (write_amp_threshold <= expected_write_amp_level) {

    return kCompactionStyleUniversal;

  }

  const double kReadWriteWeight = 4;

  double level_ratio =

      static_cast<double>(read_amp_threshold) / expected_read_amp_level *

          kReadWriteWeight +

      static_cast<double>(write_amp_threshold) / expected_write_amp_level;

  int expected_write_amp_uni = expected_max_files_universal / 2 + 2;

  int expected_read_amp_uni = expected_max_files_universal / 2 + 1;

  double uni_ratio =

      static_cast<double>(read_amp_threshold) / expected_read_amp_uni *

          kReadWriteWeight +

      static_cast<double>(write_amp_threshold) / expected_write_amp_uni;

  if (level_ratio > uni_ratio) {

    return kCompactionStyleLevel;

  } else {

    return kCompactionStyleUniversal;

  }

#else

  return kCompactionStyleLevel;

#endif  // !ROCKSDB_LITE

}

// Pick mem table size

void PickWriteBufferSize(size_t total_write_buffer_limit, Options* options) {

  const size_t kMaxWriteBufferSize = 128 * kBytesForOneMb;

  const size_t kMinWriteBufferSize = 4 * kBytesForOneMb;

  // Try to pick up a buffer size between 4MB and 128MB.

  // And try to pick 4 as the total number of write buffers.

  size_t write_buffer_size = total_write_buffer_limit / 4;

  if (write_buffer_size > kMaxWriteBufferSize) {

    write_buffer_size = kMaxWriteBufferSize;

  } else if (write_buffer_size < kMinWriteBufferSize) {

    write_buffer_size = std::min(static_cast<size_t>(kMinWriteBufferSize),

                                 total_write_buffer_limit / 2);

  }

  // Truncate to multiple of 1MB.

  if (write_buffer_size % kBytesForOneMb != 0) {

    write_buffer_size =

        (write_buffer_size / kBytesForOneMb + 1) * kBytesForOneMb;

  }

  options->write_buffer_size = write_buffer_size;

  options->max_write_buffer_number =

      static_cast<int>(total_write_buffer_limit / write_buffer_size);

  options->min_write_buffer_number_to_merge = 1;

}

#ifndef ROCKSDB_LITE

void OptimizeForUniversal(Options* options) {

  options->level0_file_num_compaction_trigger = 2;

  options->level0_slowdown_writes_trigger = 30;

  options->level0_stop_writes_trigger = 40;

  options->max_open_files = -1;

}

#endif

// Optimize parameters for level-based compaction

void OptimizeForLevel(int read_amplification_threshold,

                      int write_amplification_threshold,

                      uint64_t target_db_size, Options* options) {

  int expected_levels_one_level0_file =

      static_cast<int>(ceil(std::log(target_db_size / options->write_buffer_size) /

                            std::log(kBytesForLevelMultiplier)));

  int level0_stop_writes_trigger =

      read_amplification_threshold - expected_levels_one_level0_file;

  const size_t kInitialLevel0TotalSize = 128 * kBytesForOneMb;

  const int kMaxFileNumCompactionTrigger = 4;

  const int kMinLevel0StopTrigger = 3;

  int file_num_buffer = static_cast<int>(

      kInitialLevel0TotalSize / options->write_buffer_size + 1);

  if (level0_stop_writes_trigger > file_num_buffer) {

    // Have sufficient room for multiple level 0 files

    // Try enlarge the buffer up to 1GB

    // Try to enlarge the buffer up to 1GB, if still have sufficient headroom.

    file_num_buffer *=

        1 << std::max(0, std::min(3, level0_stop_writes_trigger -

                                       file_num_buffer - 2));

    options->level0_stop_writes_trigger = level0_stop_writes_trigger;

    options->level0_slowdown_writes_trigger = level0_stop_writes_trigger - 2;

    options->level0_file_num_compaction_trigger =

        std::min(kMaxFileNumCompactionTrigger, file_num_buffer / 2);

  } else {

    options->level0_stop_writes_trigger =

        std::max(kMinLevel0StopTrigger, file_num_buffer);

    options->level0_slowdown_writes_trigger =

        options->level0_stop_writes_trigger - 1;

    options->level0_file_num_compaction_trigger = 1;

  }

  // This doesn't consider compaction and overheads of mem tables. But usually

  // it is in the same order of magnitude.

  size_t expected_level0_compaction_size =

      options->level0_file_num_compaction_trigger * options->write_buffer_size;

  // Enlarge level1 target file size if level0 compaction size is larger.

  uint64_t max_bytes_for_level_base = 10 * kBytesForOneMb;

  if (expected_level0_compaction_size > max_bytes_for_level_base) {

    max_bytes_for_level_base = expected_level0_compaction_size;

  }

  options->max_bytes_for_level_base = max_bytes_for_level_base;

  // Now always set level multiplier to be 10

  options->max_bytes_for_level_multiplier = kBytesForLevelMultiplier;

  const uint64_t kMinFileSize = 2 * kBytesForOneMb;

  // Allow at least 3-way parallelism for compaction between level 1 and 2.

  uint64_t max_file_size = max_bytes_for_level_base / 3;

  if (max_file_size < kMinFileSize) {

    options->target_file_size_base = kMinFileSize;

  } else {

    if (max_file_size % kBytesForOneMb != 0) {

      max_file_size = (max_file_size / kBytesForOneMb + 1) * kBytesForOneMb;

    }

    options->target_file_size_base = max_file_size;

  }

  // TODO: consider to tune num_levels too.

}

}  // namespace

Options GetOptions(size_t total_write_buffer_limit,

                   int read_amplification_threshold,

                   int write_amplification_threshold, uint64_t target_db_size) {

  Options options;

  PickWriteBufferSize(total_write_buffer_limit, &options);

  size_t write_buffer_size = options.write_buffer_size;

  options.compaction_style =

      PickCompactionStyle(write_buffer_size, read_amplification_threshold,

                          write_amplification_threshold, target_db_size);

#ifndef ROCKSDB_LITE

  if (options.compaction_style == kCompactionStyleUniversal) {

    OptimizeForUniversal(&options);

  } else {

#else

  {

#endif  // !ROCKSDB_LITE

    OptimizeForLevel(read_amplification_threshold,

                     write_amplification_threshold, target_db_size, &options);

  }

  return options;

}

}  // namespace rocksdb