YugabyteDB (2.13.0.0-b42, bfc6a6643e7399ac8a0e81d06a3ee6d6571b33ab)

Coverage Report

Created: 2022-03-09 17:30

/Users/deen/code/yugabyte-db/src/yb/fs/fs_manager.cc
Line
Count
Source (jump to first uncovered line)
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
//
18
// The following only applies to changes made to this file as part of YugaByte development.
19
//
20
// Portions Copyright (c) YugaByte, Inc.
21
//
22
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
23
// in compliance with the License.  You may obtain a copy of the License at
24
//
25
// http://www.apache.org/licenses/LICENSE-2.0
26
//
27
// Unless required by applicable law or agreed to in writing, software distributed under the License
28
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
29
// or implied.  See the License for the specific language governing permissions and limitations
30
// under the License.
31
//
32
33
#include "yb/fs/fs_manager.h"
34
35
#include <map>
36
#include <set>
37
#include <unordered_set>
38
39
#include <boost/algorithm/string/predicate.hpp>
40
#include <boost/preprocessor/cat.hpp>
41
#include <glog/logging.h>
42
#include <glog/stl_logging.h>
43
#include <google/protobuf/message.h>
44
45
#include "yb/fs/fs.pb.h"
46
47
#include "yb/gutil/map-util.h"
48
#include "yb/gutil/strings/join.h"
49
#include "yb/gutil/strings/numbers.h"
50
#include "yb/gutil/strings/split.h"
51
#include "yb/gutil/strings/strip.h"
52
#include "yb/gutil/strings/util.h"
53
#include "yb/gutil/walltime.h"
54
55
#include "yb/util/env_util.h"
56
#include "yb/util/flag_tags.h"
57
#include "yb/util/format.h"
58
#include "yb/util/metric_entity.h"
59
#include "yb/util/net/net_util.h"
60
#include "yb/util/oid_generator.h"
61
#include "yb/util/path_util.h"
62
#include "yb/util/pb_util.h"
63
#include "yb/util/result.h"
64
65
DEFINE_bool(enable_data_block_fsync, true,
66
            "Whether to enable fsync() of data blocks, metadata, and their parent directories. "
67
            "Disabling this flag may cause data loss in the event of a system crash.");
68
TAG_FLAG(enable_data_block_fsync, unsafe);
69
70
DECLARE_string(fs_data_dirs);
71
72
DEFINE_string(fs_wal_dirs, "",
73
              "Comma-separated list of directories for write-ahead logs. This is an optional "
74
                  "argument. If this is not specified, fs_data_dirs is used for write-ahead logs "
75
                  "also and that's a reasonable default for most use cases.");
76
TAG_FLAG(fs_wal_dirs, stable);
77
78
DEFINE_string(instance_uuid_override, "",
79
              "When creating local instance metadata (for master or tserver) in an empty data "
80
              "directory, use this UUID instead of randomly-generated one. Can be used to replace "
81
              "a node that had its disk wiped in some scenarios.");
82
83
DEFINE_test_flag(bool, simulate_fs_create_failure, false,
84
                 "Simulate failure during initial creation of fs during the first time "
85
                 "process creation.");
86
87
using google::protobuf::Message;
88
using yb::env_util::ScopedFileDeleter;
89
using std::map;
90
using std::unordered_set;
91
using strings::Substitute;
92
93
namespace yb {
94
// ==========================================================================
95
//  FS Paths
96
// ==========================================================================
97
const char *FsManager::kWalDirName = "wals";
98
const char *FsManager::kWalFileNamePrefix = "wal";
99
const char *FsManager::kWalsRecoveryDirSuffix = ".recovery";
100
const char *FsManager::kRocksDBDirName = "rocksdb";
101
const char *FsManager::kDataDirName = "data";
102
103
namespace {
104
105
static const char kRaftGroupMetadataDirName[] = "tablet-meta";
106
static const char kInstanceMetadataFileName[] = "instance";
107
static const char kFsLockFileName[] = "fs-lock";
108
static const char kConsensusMetadataDirName[] = "consensus-meta";
109
static const char kLogsDirName[] = "logs";
110
static const char kTmpInfix[] = ".tmp";
111
112
114k
std::string DataDir(const std::string& root, const std::string& server_type) {
113
114k
  return JoinPathSegments(GetServerTypeDataPath(root, server_type), FsManager::kDataDirName);
114
114k
}
115
116
102k
std::string WalDir(const std::string& root, const std::string& server_type) {
117
102k
  return JoinPathSegments(GetServerTypeDataPath(root, server_type), FsManager::kWalDirName);
118
102k
}
119
120
} // namespace
121
122
FsManagerOpts::FsManagerOpts()
123
36.7k
    : read_only(false) {
124
36.7k
  if (FLAGS_fs_wal_dirs.empty() && !FLAGS_fs_data_dirs.empty()) {
125
    // It is sufficient if user sets the data dirs. By default we use the same
126
    // directories for WALs as well.
127
5.73k
    FLAGS_fs_wal_dirs = FLAGS_fs_data_dirs;
128
5.73k
  }
129
36.7k
  wal_paths = strings::Split(FLAGS_fs_wal_dirs, ",", strings::SkipEmpty());
130
36.7k
  data_paths = strings::Split(FLAGS_fs_data_dirs, ",", strings::SkipEmpty());
131
36.7k
}
132
133
33.1k
FsManagerOpts::~FsManagerOpts() = default;
134
46.7k
FsManagerOpts::FsManagerOpts(const FsManagerOpts&) = default;
135
3
FsManagerOpts& FsManagerOpts::operator=(const FsManagerOpts&) = default;
136
137
FsManager::FsManager(Env* env, const string& root_path, const std::string& server_type)
138
    : env_(DCHECK_NOTNULL(env)),
139
      read_only_(false),
140
      wal_fs_roots_({ root_path }),
141
      data_fs_roots_({ root_path }),
142
      server_type_(server_type),
143
      metric_entity_(nullptr),
144
155
      initted_(false) {
145
155
}
146
147
FsManager::FsManager(Env* env,
148
                     const FsManagerOpts& opts)
149
    : env_(DCHECK_NOTNULL(env)),
150
      read_only_(opts.read_only),
151
      wal_fs_roots_(opts.wal_paths),
152
      data_fs_roots_(opts.data_paths),
153
      server_type_(opts.server_type),
154
      metric_entity_(opts.metric_entity),
155
      parent_mem_tracker_(opts.parent_mem_tracker),
156
17.5k
      initted_(false) {
157
17.5k
}
158
159
407
FsManager::~FsManager() {
160
407
}
161
162
41.2k
Status FsManager::Init() {
163
41.2k
  if (initted_) {
164
23.5k
    return Status::OK();
165
23.5k
  }
166
167
  // The wal root must be set.
168
17.7k
  if (data_fs_roots_.empty()) {
169
2
    return STATUS(IOError, "List of data directories (fs_data_dirs) not provided");
170
2
  }
171
172
  // Deduplicate all of the roots.
173
17.7k
  set<string> all_roots;
174
17.8k
  for (const string& wal_fs_root : wal_fs_roots_) {
175
17.8k
    all_roots.insert(wal_fs_root);
176
17.8k
  }
177
17.8k
  for (const string& data_fs_root : data_fs_roots_) {
178
17.8k
    all_roots.insert(data_fs_root);
179
17.8k
  }
180
181
  // Build a map of original root --> canonicalized root, sanitizing each
182
  // root a bit as we go.
183
17.7k
  typedef map<string, string> RootMap;
184
17.7k
  RootMap canonicalized_roots;
185
17.8k
  for (const string& root : all_roots) {
186
17.8k
    if (root.empty()) {
187
1
      return STATUS(IOError, "Empty string provided for filesystem root");
188
1
    }
189
17.8k
    if (root[0] != '/') {
190
1
      return STATUS(IOError,
191
1
          Substitute("Relative path $0 provided for filesystem root", root));
192
1
    }
193
17.8k
    {
194
17.8k
      string root_copy = root;
195
17.8k
      StripWhiteSpace(&root_copy);
196
17.8k
      if (root != root_copy) {
197
1
        return STATUS(IOError,
198
1
                  Substitute("Filesystem root $0 contains illegal whitespace", root));
199
1
      }
200
17.8k
    }
201
202
    // Strip the basename when canonicalizing, as it may not exist. The
203
    // dirname, however, must exist.
204
17.8k
    string canonicalized;
205
17.8k
    Status s = env_->Canonicalize(DirName(root), &canonicalized);
206
17.8k
    if (!s.ok()) {
207
0
      return STATUS(
208
0
          InvalidArgument, strings::Substitute(
209
0
          "Cannot create directory for YB data, please check the --fs_data_dirs parameter "
210
0
          "(Passed: $0). Path does not exist: $1\nDetails: $2",
211
0
          FLAGS_fs_data_dirs, root, s.ToString()));
212
0
    }
213
17.8k
    canonicalized = JoinPathSegments(canonicalized, BaseName(root));
214
17.8k
    InsertOrDie(&canonicalized_roots, root, canonicalized);
215
17.8k
  }
216
217
  // All done, use the map to set the canonicalized state.
218
17.8k
  for (const auto& wal_fs_root : wal_fs_roots_) {
219
17.8k
    canonicalized_wal_fs_roots_.insert(FindOrDie(canonicalized_roots, wal_fs_root));
220
17.8k
  }
221
17.7k
  if (!data_fs_roots_.empty()) {
222
17.7k
    canonicalized_metadata_fs_root_ = FindOrDie(canonicalized_roots, data_fs_roots_[0]);
223
17.8k
    for (const string& data_fs_root : data_fs_roots_) {
224
17.8k
      canonicalized_data_fs_roots_.insert(FindOrDie(canonicalized_roots, data_fs_root));
225
17.8k
    }
226
0
  } else {
227
0
    LOG(FATAL) << "Data directories (fs_data_dirs) must be specified";
228
0
  }
229
230
17.8k
  for (const RootMap::value_type& e : canonicalized_roots) {
231
17.8k
    canonicalized_all_fs_roots_.insert(e.second);
232
17.8k
  }
233
234
0
  if (VLOG_IS_ON(1)) {
235
0
    VLOG(1) << "WAL roots: " << canonicalized_wal_fs_roots_;
236
0
    VLOG(1) << "Metadata root: " << canonicalized_metadata_fs_root_;
237
0
    VLOG(1) << "Data roots: " << canonicalized_data_fs_roots_;
238
0
    VLOG(1) << "All roots: " << canonicalized_all_fs_roots_;
239
0
  }
240
241
17.7k
  initted_ = true;
242
17.7k
  return Status::OK();
243
17.7k
}
244
245
29.1k
Status FsManager::Open() {
246
29.1k
  RETURN_NOT_OK(Init());
247
248
29.1k
  if (HasAnyLockFiles()) {
249
0
    return STATUS(Corruption, "Lock file is present, filesystem may be in inconsistent state");
250
0
  }
251
252
29.1k
  bool create_roots = false;
253
29.4k
  for (const string& root : canonicalized_all_fs_roots_) {
254
29.4k
    auto pb = std::make_unique<InstanceMetadataPB>();
255
29.4k
    auto read_result = pb_util::ReadPBContainerFromPath(env_, GetInstanceMetadataPath(root),
256
29.4k
                                                        pb.get());
257
29.4k
    if (read_result.IsNotFound()) {
258
11.5k
      create_roots = true;
259
11.5k
      continue;
260
11.5k
    }
261
17.8k
    RETURN_NOT_OK(read_result);
262
17.8k
    if (!metadata_) {
263
17.6k
      metadata_.reset(pb.release());
264
177
    } else if (pb->uuid() != metadata_->uuid()) {
265
0
      return STATUS(Corruption, Substitute(
266
0
          "Mismatched UUIDs across filesystem roots: $0 vs. $1",
267
0
          metadata_->uuid(), pb->uuid()));
268
0
    }
269
17.8k
  }
270
29.1k
  if (!metadata_) {
271
11.4k
    return STATUS(NotFound, "Metadata wasn't found");
272
11.4k
  }
273
17.6k
  if (create_roots) {
274
1
    RETURN_NOT_OK(CreateFileSystemRoots(canonicalized_all_fs_roots_,
275
1
                                        GetAncillaryDirs(/* add_metadata_dirs = */ false),
276
1
                                        *metadata_.get()));
277
1
  }
278
279
17.6k
  LOG(INFO) << "Opened local filesystem: " << JoinStrings(canonicalized_all_fs_roots_, ",")
280
17.6k
            << std::endl << metadata_->DebugString();
281
17.6k
  return Status::OK();
282
17.6k
}
283
284
40.6k
bool FsManager::HasAnyLockFiles() {
285
40.9k
  for (const string& root : canonicalized_all_fs_roots_) {
286
40.9k
    if (Exists(GetFsLockFilePath(root))) {
287
0
      LOG(INFO) << "Found lock file in dir " << root;
288
0
      return true;
289
0
    }
290
40.9k
  }
291
292
40.6k
  return false;
293
40.6k
}
294
295
12.4k
Status FsManager::DeleteLockFiles() {
296
12.4k
  CHECK(!read_only_);
297
12.4k
  vector<string> removal_list;
298
12.5k
  for (const string& root : canonicalized_all_fs_roots_) {
299
12.5k
    std::string lock_file_path = GetFsLockFilePath(root);
300
12.5k
    if (Exists(lock_file_path)) {
301
0
      removal_list.push_back(lock_file_path);
302
0
    }
303
12.5k
  }
304
305
0
  for (const string& target : removal_list) {
306
0
    RETURN_NOT_OK_PREPEND(env_->DeleteFile(target), "Lock file delete failed");
307
0
  }
308
309
12.4k
  return Status::OK();
310
12.4k
}
311
312
428
Status FsManager::DeleteFileSystemLayout(ShouldDeleteLogs also_delete_logs) {
313
428
  CHECK(!read_only_);
314
428
  set<string> removal_set;
315
428
  if (also_delete_logs) {
316
401
    removal_set = canonicalized_all_fs_roots_;
317
27
  } else {
318
27
    auto removal_list = GetWalRootDirs();
319
27
    removal_list.push_back(GetRaftGroupMetadataDir());
320
27
    removal_list.push_back(GetConsensusMetadataDir());
321
27
    for (const string& root : canonicalized_all_fs_roots_) {
322
27
      removal_list.push_back(GetInstanceMetadataPath(root));
323
27
    }
324
27
    auto data_dirs = GetDataRootDirs();
325
27
    removal_list.insert(removal_list.begin(), data_dirs.begin(), data_dirs.end());
326
27
    removal_set.insert(removal_list.begin(), removal_list.end());
327
27
  }
328
329
535
  for (const string& target : removal_set) {
330
535
    bool is_dir = false;
331
535
    Status s = env_->IsDirectory(target, &is_dir);
332
535
    if (!s.ok()) {
333
0
      LOG(WARNING) << "Error: " << s.ToString() << " when checking if " << target
334
0
                   << " is a directory.";
335
0
      continue;
336
0
    }
337
535
    if (is_dir) {
338
508
      RETURN_NOT_OK(env_->DeleteRecursively(target));
339
27
    } else {
340
27
      RETURN_NOT_OK(env_->DeleteFile(target));
341
27
    }
342
535
  }
343
344
428
  RETURN_NOT_OK(DeleteLockFiles());
345
346
428
  return Status::OK();
347
428
}
348
349
12.0k
Status FsManager::CreateInitialFileSystemLayout(bool delete_fs_if_lock_found) {
350
12.0k
  CHECK(!read_only_);
351
352
12.0k
  RETURN_NOT_OK(Init());
353
354
12.0k
  bool fs_cleaned = false;
355
356
  // If lock file is present, delete existing filesystem layout before continuing.
357
12.0k
  if (delete_fs_if_lock_found && HasAnyLockFiles()) {
358
0
    RETURN_NOT_OK(DeleteFileSystemLayout());
359
0
    fs_cleaned = true;
360
0
  }
361
362
  // It's OK if a root already exists as long as there's nothing in it.
363
12.1k
  for (const string& root : canonicalized_all_fs_roots_) {
364
12.1k
    if (!env_->FileExists(GetServerTypeDataPath(root, server_type_))) {
365
      // We'll create the directory below.
366
11.1k
      continue;
367
11.1k
    }
368
993
    bool is_empty;
369
993
    RETURN_NOT_OK_PREPEND(IsDirectoryEmpty(GetServerTypeDataPath(root, server_type_), &is_empty),
370
993
                          "Unable to check if FSManager root is empty");
371
993
    if (!is_empty) {
372
1
      return STATUS(AlreadyPresent, "FSManager root is not empty", root);
373
1
    }
374
993
  }
375
376
  // All roots are either empty or non-existent. Create missing roots and all
377
  // subdirectories.
378
  //
379
380
12.0k
  InstanceMetadataPB metadata;
381
12.0k
  CreateInstanceMetadata(&metadata);
382
12.0k
  RETURN_NOT_OK(CreateFileSystemRoots(canonicalized_all_fs_roots_,
383
12.0k
                                      GetAncillaryDirs(/* add_metadata_dirs = */ true),
384
12.0k
                                      metadata,
385
12.0k
                                      /* create_lock = */ fs_cleaned));
386
387
12.0k
  if (FLAGS_TEST_simulate_fs_create_failure) {
388
1
    return STATUS(IOError, "Simulated fs creation error");
389
1
  }
390
12.0k
  RETURN_NOT_OK(DeleteLockFiles());
391
12.0k
  return Status::OK();
392
12.0k
}
393
394
Status FsManager::CreateFileSystemRoots(const std::set<std::string>& roots,
395
                                        const std::set<std::string>& ancillary_dirs,
396
                                        const InstanceMetadataPB& metadata,
397
12.0k
                                        bool create_lock) {
398
  // In the event of failure, delete everything we created.
399
12.0k
  std::deque<ScopedFileDeleter> delete_on_failure;
400
12.0k
  unordered_set<string> to_sync;
401
402
  // All roots are either empty or non-existent. Create missing roots and all
403
  // subdirectories.
404
12.1k
  for (const auto& root : roots) {
405
12.1k
    bool created;
406
12.1k
    std::string out_dir;
407
12.1k
    RETURN_NOT_OK(SetupRootDir(env_, root, server_type_, &out_dir, &created));
408
12.1k
    if (created) {
409
11.1k
      delete_on_failure.emplace_front(env_, out_dir);
410
11.1k
      to_sync.insert(DirName(out_dir));
411
11.1k
    }
412
413
12.1k
    const string lock_file_path = GetFsLockFilePath(root);
414
12.1k
    if (create_lock && !Exists(lock_file_path)) {
415
0
      std::unique_ptr<WritableFile> file;
416
0
      RETURN_NOT_OK_PREPEND(env_->NewWritableFile(lock_file_path, &file),
417
0
                            "Unable to create lock file.");
418
      // Do not delete lock file on error. It is used to detect failed initial create.
419
0
    }
420
12.1k
    const string instance_metadata_path = GetInstanceMetadataPath(root);
421
12.1k
    if (env_->FileExists(instance_metadata_path)) {
422
2
      continue;
423
2
    }
424
12.1k
    RETURN_NOT_OK_PREPEND(WriteInstanceMetadata(metadata, instance_metadata_path),
425
12.1k
                            "Unable to write instance metadata");
426
12.1k
    delete_on_failure.emplace_front(env_, instance_metadata_path);
427
12.1k
  }
428
429
60.4k
  for (const auto& dir : ancillary_dirs) {
430
60.4k
    bool created;
431
60.4k
    RETURN_NOT_OK_PREPEND(CreateDirIfMissing(dir, &created),
432
60.4k
                          Substitute("Unable to create directory $0", dir));
433
60.4k
    if (created) {
434
60.4k
      delete_on_failure.emplace_front(env_, dir);
435
60.4k
      to_sync.insert(DirName(dir));
436
60.4k
    }
437
60.4k
  }
438
439
  // Ensure newly created directories are synchronized to disk.
440
12.0k
  if (FLAGS_enable_data_block_fsync) {
441
34.2k
    for (const string& dir : to_sync) {
442
34.2k
      RETURN_NOT_OK_PREPEND(env_->SyncDir(dir),
443
34.2k
                            Substitute("Unable to synchronize directory $0", dir));
444
34.2k
    }
445
11.6k
  }
446
447
  // Success: don't delete any files.
448
83.7k
  for (auto& deleter : delete_on_failure) {
449
83.7k
    deleter.Cancel();
450
83.7k
  }
451
452
12.0k
  return Status::OK();
453
12.0k
}
454
455
12.0k
std::set<std::string> FsManager::GetAncillaryDirs(bool add_metadata_dirs) const {
456
12.0k
  std::set<std::string> ancillary_dirs;
457
12.0k
  if (add_metadata_dirs) {
458
12.0k
    ancillary_dirs.emplace(GetRaftGroupMetadataDir());
459
12.0k
    ancillary_dirs.emplace(GetConsensusMetadataDir());
460
12.0k
  }
461
12.1k
  for (const auto& wal_fs_root : canonicalized_wal_fs_roots_) {
462
12.1k
    ancillary_dirs.emplace(WalDir(wal_fs_root, server_type_));
463
12.1k
  }
464
12.1k
  for (const string& data_fs_root : canonicalized_data_fs_roots_) {
465
12.1k
    const string data_dir = DataDir(data_fs_root, server_type_);
466
12.1k
    ancillary_dirs.emplace(data_dir);
467
12.1k
    ancillary_dirs.emplace(JoinPathSegments(data_dir, kRocksDBDirName));
468
12.1k
  }
469
12.0k
  return ancillary_dirs;
470
12.0k
}
471
472
12.0k
void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) {
473
12.0k
  if (!FLAGS_instance_uuid_override.empty()) {
474
0
    metadata->set_uuid(FLAGS_instance_uuid_override);
475
12.0k
  } else {
476
12.0k
    metadata->set_uuid(GenerateObjectId());
477
12.0k
  }
478
479
12.0k
  string time_str;
480
12.0k
  StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false);
481
12.0k
  string hostname;
482
12.0k
  if (!GetHostname(&hostname).ok()) {
483
0
    hostname = "<unknown host>";
484
0
  }
485
12.0k
  metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname));
486
12.0k
}
487
488
Status FsManager::WriteInstanceMetadata(const InstanceMetadataPB& metadata,
489
12.1k
                                        const string& path) {
490
  // The instance metadata is written effectively once per TS, so the
491
  // durability cost is negligible.
492
12.1k
  RETURN_NOT_OK(pb_util::WritePBContainerToPath(env_, path,
493
12.1k
                                                metadata,
494
12.1k
                                                pb_util::NO_OVERWRITE,
495
12.1k
                                                pb_util::SYNC));
496
12.1k
  LOG(INFO) << "Generated new instance metadata in path " << path << ":\n"
497
12.1k
            << metadata.DebugString();
498
12.1k
  return Status::OK();
499
12.1k
}
500
501
993
Status FsManager::IsDirectoryEmpty(const string& path, bool* is_empty) {
502
993
  vector<string> children;
503
993
  RETURN_NOT_OK(env_->GetChildren(path, &children));
504
2.97k
  for (const string& child : children) {
505
    // Excluding logs directory from the list of things to check for.
506
2.97k
    if (child == "." || child == ".." || child == kLogsDirName) {
507
2.97k
      continue;
508
1
    } else {
509
1
      LOG(INFO) << "Found data " << child;
510
1
      *is_empty = false;
511
1
      return Status::OK();
512
1
    }
513
2.97k
  }
514
992
  *is_empty = true;
515
992
  return Status::OK();
516
993
}
517
518
1.25M
Status FsManager::CreateDirIfMissing(const string& path, bool* created) {
519
1.25M
  return env_util::CreateDirIfMissing(env_, path, created);
520
1.25M
}
521
522
884k
Status FsManager::CreateDirIfMissingAndSync(const std::string& path, bool* created) {
523
884k
  RETURN_NOT_OK_PREPEND(CreateDirIfMissing(path, created),
524
884k
                        Substitute("Failed to create directory $0", path));
525
884k
  RETURN_NOT_OK_PREPEND(env_->SyncDir(DirName(path)),
526
884k
                        Substitute("Failed to sync root directory $0", DirName(path)));
527
884k
  return Status::OK();
528
884k
}
529
530
147M
const string& FsManager::uuid() const {
531
147M
  return CHECK_NOTNULL(metadata_.get())->uuid();
532
147M
}
533
534
37.7k
set<string> FsManager::GetFsRootDirs() const {
535
37.7k
  return canonicalized_all_fs_roots_;
536
37.7k
}
537
538
101k
vector<string> FsManager::GetDataRootDirs() const {
539
  // Add the data subdirectory to each data root.
540
101k
  vector<string> data_paths;
541
102k
  for (const string& data_fs_root : canonicalized_data_fs_roots_) {
542
102k
    data_paths.push_back(DataDir(data_fs_root, server_type_));
543
102k
  }
544
101k
  return data_paths;
545
101k
}
546
547
89.9k
vector<string> FsManager::GetWalRootDirs() const {
548
89.9k
  DCHECK(initted_);
549
89.9k
  vector<string> wal_dirs;
550
90.5k
  for (const auto& canonicalized_wal_fs_root : canonicalized_wal_fs_roots_) {
551
90.5k
    wal_dirs.push_back(WalDir(canonicalized_wal_fs_root, server_type_));
552
90.5k
  }
553
89.9k
  return wal_dirs;
554
89.9k
}
555
556
952k
std::string FsManager::GetRaftGroupMetadataDir(const std::string& data_dir) {
557
952k
  return JoinPathSegments(data_dir, kRaftGroupMetadataDirName);
558
952k
}
559
560
951k
string FsManager::GetRaftGroupMetadataDir() const {
561
951k
  DCHECK(initted_);
562
951k
  return GetRaftGroupMetadataDir(
563
951k
      GetServerTypeDataPath(canonicalized_metadata_fs_root_, server_type_));
564
951k
}
565
566
916k
string FsManager::GetRaftGroupMetadataPath(const string& tablet_id) const {
567
916k
  return JoinPathSegments(GetRaftGroupMetadataDir(), tablet_id);
568
916k
}
569
570
namespace {
571
// Return true if 'fname' is a valid tablet ID.
572
11.8k
bool IsValidTabletId(const std::string& fname) {
573
11.8k
  if (fname.find(kTmpInfix) != string::npos) {
574
2
    LOG(WARNING) << "Ignoring tmp file in tablet metadata dir: " << fname;
575
2
    return false;
576
2
  }
577
578
11.8k
  if (HasPrefixString(fname, ".")) {
579
    // Hidden file or ./..
580
0
    VLOG(1) << "Ignoring hidden file in tablet metadata dir: " << fname;
581
11.6k
    return false;
582
11.6k
  }
583
584
229
  return true;
585
229
}
586
} // anonymous namespace
587
588
5.81k
Status FsManager::ListTabletIds(vector<string>* tablet_ids) {
589
5.81k
  string dir = GetRaftGroupMetadataDir();
590
5.81k
  vector<string> children;
591
5.81k
  RETURN_NOT_OK_PREPEND(ListDir(dir, &children),
592
5.81k
                        Substitute("Couldn't list tablets in metadata directory $0", dir));
593
594
5.81k
  vector<string> tablets;
595
11.8k
  for (const string& child : children) {
596
11.8k
    if (!IsValidTabletId(child)) {
597
11.6k
      continue;
598
11.6k
    }
599
229
    tablet_ids->push_back(child);
600
229
  }
601
5.81k
  return Status::OK();
602
5.81k
}
603
604
41.5k
std::string FsManager::GetInstanceMetadataPath(const string& root) const {
605
41.5k
  return JoinPathSegments(GetServerTypeDataPath(root, server_type_), kInstanceMetadataFileName);
606
41.5k
}
607
608
65.6k
std::string FsManager::GetFsLockFilePath(const string& root) const {
609
65.6k
  return JoinPathSegments(GetServerTypeDataPath(root, server_type_), kFsLockFileName);
610
65.6k
}
611
612
1.31M
std::string FsManager::GetConsensusMetadataDir() const {
613
1.31M
  DCHECK(initted_);
614
1.31M
  return GetConsensusMetadataDir(
615
1.31M
      GetServerTypeDataPath(canonicalized_metadata_fs_root_, server_type_));
616
1.31M
}
617
618
1.31M
std::string FsManager::GetConsensusMetadataDir(const std::string& data_dir) {
619
1.31M
  return JoinPathSegments(data_dir, kConsensusMetadataDirName);
620
1.31M
}
621
622
std::string FsManager::GetFirstTabletWalDirOrDie(const std::string& table_id,
623
107
                                                 const std::string& tablet_id) const {
624
107
  auto wal_root_dirs = GetWalRootDirs();
625
0
  CHECK(!wal_root_dirs.empty()) << "No WAL directories specified";
626
107
  auto table_wal_dir = JoinPathSegments(wal_root_dirs[0], Substitute("table-$0", table_id));
627
107
  return JoinPathSegments(table_wal_dir, Substitute("tablet-$0", tablet_id));
628
107
}
629
630
90.9k
std::string FsManager::GetTabletWalRecoveryDir(const string& tablet_wal_path) {
631
90.9k
  return tablet_wal_path + kWalsRecoveryDirSuffix;
632
90.9k
}
633
634
namespace {
635
636
const auto kWalFileNameFullPrefix = std::string(FsManager::kWalFileNamePrefix) + "-";
637
638
} // namespace
639
640
99.9k
std::string FsManager::GetWalSegmentFileName(uint64_t sequence_number) {
641
99.9k
  return Format("$0$1", kWalFileNameFullPrefix, StringPrintf("%09" PRIu64, sequence_number));
642
99.9k
}
643
644
830
bool FsManager::IsWalSegmentFileName(const std::string& file_name) {
645
830
  return boost::starts_with(file_name, kWalFileNameFullPrefix);
646
830
}
647
648
// ==========================================================================
649
//  Dump/Debug utils
650
// ==========================================================================
651
652
0
void FsManager::DumpFileSystemTree(ostream& out) {
653
0
  DCHECK(initted_);
654
655
0
  for (const string& root : canonicalized_all_fs_roots_) {
656
0
    out << "File-System Root: " << root << std::endl;
657
658
0
    std::vector<string> objects;
659
0
    Status s = env_->GetChildren(root, &objects);
660
0
    if (!s.ok()) {
661
0
      LOG(ERROR) << "Unable to list the fs-tree: " << s.ToString();
662
0
      return;
663
0
    }
664
665
0
    DumpFileSystemTree(out, "|-", root, objects);
666
0
  }
667
0
}
668
669
void FsManager::DumpFileSystemTree(ostream& out, const string& prefix,
670
0
                                   const string& path, const vector<string>& objects) {
671
0
  for (const string& name : objects) {
672
0
    if (name == "." || name == "..") continue;
673
674
0
    std::vector<string> sub_objects;
675
0
    string sub_path = JoinPathSegments(path, name);
676
0
    Status s = env_->GetChildren(sub_path, &sub_objects);
677
0
    if (s.ok()) {
678
0
      out << prefix << name << "/" << std::endl;
679
0
      DumpFileSystemTree(out, prefix + "---", sub_path, sub_objects);
680
0
    } else {
681
0
      out << prefix << name << std::endl;
682
0
    }
683
0
  }
684
0
}
685
686
1.31k
Result<std::vector<std::string>> FsManager::ListDir(const std::string& path) const {
687
1.31k
  std::vector<std::string> result;
688
1.31k
  RETURN_NOT_OK(env_->GetChildren(path, ExcludeDots::kTrue, &result));
689
1.31k
  return result;
690
1.31k
}
691
692
5.81k
Status FsManager::ListDir(const std::string& path, std::vector<std::string> *objects) const {
693
5.81k
  return env_->GetChildren(path, objects);
694
5.81k
}
695
696
} // namespace yb