/Users/deen/code/yugabyte-db/src/yb/util/rolling_log.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | // |
18 | | // The following only applies to changes made to this file as part of YugaByte development. |
19 | | // |
20 | | // Portions Copyright (c) YugaByte, Inc. |
21 | | // |
22 | | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except |
23 | | // in compliance with the License. You may obtain a copy of the License at |
24 | | // |
25 | | // http://www.apache.org/licenses/LICENSE-2.0 |
26 | | // |
27 | | // Unless required by applicable law or agreed to in writing, software distributed under the License |
28 | | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
29 | | // or implied. See the License for the specific language governing permissions and limitations |
30 | | // under the License. |
31 | | // |
32 | | #include "yb/util/rolling_log.h" |
33 | | |
34 | | #include <zlib.h> |
35 | | |
36 | | #include <iomanip> |
37 | | #include <string> |
38 | | |
39 | | #include "yb/gutil/casts.h" |
40 | | #include "yb/gutil/strings/numbers.h" |
41 | | #include "yb/gutil/walltime.h" |
42 | | #include "yb/util/env.h" |
43 | | #include "yb/util/net/net_util.h" |
44 | | #include "yb/util/path_util.h" |
45 | | #include "yb/util/result.h" |
46 | | #include "yb/util/status_log.h" |
47 | | #include "yb/util/user.h" |
48 | | |
49 | | using std::ostringstream; |
50 | | using std::setw; |
51 | | using std::string; |
52 | | using strings::Substitute; |
53 | | |
54 | | static const int kDefaultSizeLimitBytes = 64 * 1024 * 1024; // 64MB |
55 | | |
56 | | namespace yb { |
57 | | |
58 | | RollingLog::RollingLog(Env* env, string log_dir, string log_name) |
59 | | : env_(env), |
60 | | log_dir_(std::move(log_dir)), |
61 | | log_name_(std::move(log_name)), |
62 | | size_limit_bytes_(kDefaultSizeLimitBytes), |
63 | 2 | compress_after_close_(true) {} |
64 | | |
65 | 2 | RollingLog::~RollingLog() { |
66 | 2 | WARN_NOT_OK(Close(), "Unable to close RollingLog"); |
67 | 2 | } |
68 | | |
69 | 1 | void RollingLog::SetSizeLimitBytes(size_t size) { |
70 | 1 | CHECK_GT(size, 0); |
71 | 1 | size_limit_bytes_ = size; |
72 | 1 | } |
73 | | |
74 | 1 | void RollingLog::SetCompressionEnabled(bool compress) { |
75 | 1 | compress_after_close_ = compress; |
76 | 1 | } |
77 | | |
78 | 4 | string RollingLog::GetLogFileName(int sequence) const { |
79 | 4 | ostringstream str; |
80 | | |
81 | | // 1. Program name. |
82 | 4 | str << google::ProgramInvocationShortName(); |
83 | | |
84 | | // 2. Host name. |
85 | 4 | string hostname; |
86 | 4 | Status s = GetHostname(&hostname); |
87 | 4 | if (!s.ok()) { |
88 | 0 | hostname = "unknown_host"; |
89 | 0 | } |
90 | 4 | str << "." << hostname; |
91 | | |
92 | | // 3. User name. |
93 | 4 | auto user_name = GetLoggedInUser(); |
94 | 4 | str << "." << (user_name.ok() ? *user_name : "unknown_user"); |
95 | | |
96 | | // 4. Log name. |
97 | 4 | str << "." << log_name_; |
98 | | |
99 | | // 5. Timestamp. |
100 | | // Implementation cribbed from glog/logging.cc |
101 | 4 | time_t time = static_cast<time_t>(WallTime_Now()); |
102 | 4 | struct ::tm tm_time; |
103 | 4 | localtime_r(&time, &tm_time); |
104 | | |
105 | 4 | str << "."; |
106 | 4 | str.fill('0'); |
107 | 4 | str << 1900+tm_time.tm_year |
108 | 4 | << setw(2) << 1+tm_time.tm_mon |
109 | 4 | << setw(2) << tm_time.tm_mday |
110 | 4 | << '-' |
111 | 4 | << setw(2) << tm_time.tm_hour |
112 | 4 | << setw(2) << tm_time.tm_min |
113 | 4 | << setw(2) << tm_time.tm_sec; |
114 | 4 | str.clear(); // resets formatting flags |
115 | | |
116 | | // 6. Sequence number. |
117 | 4 | str << "." << sequence; |
118 | | |
119 | | // 7. Pid. |
120 | 4 | str << "." << getpid(); |
121 | | |
122 | 4 | return str.str(); |
123 | 4 | } |
124 | | |
125 | 3 | Status RollingLog::Open() { |
126 | 3 | CHECK(!file_); |
127 | | |
128 | 4 | for (int sequence = 0; ; sequence++) { |
129 | | |
130 | 4 | string path = JoinPathSegments(log_dir_, |
131 | 4 | GetLogFileName(sequence)); |
132 | | |
133 | 4 | WritableFileOptions opts; |
134 | | // Logs aren't worth the performance cost of durability. |
135 | 4 | opts.sync_on_close = false; |
136 | 4 | opts.mode = Env::CREATE_NON_EXISTING; |
137 | | |
138 | 4 | Status s = env_->NewWritableFile(opts, path, &file_); |
139 | 4 | if (s.IsAlreadyPresent()) { |
140 | | // We already rolled once at this same timestamp. |
141 | | // Try again with a new sequence number. |
142 | 1 | continue; |
143 | 1 | } |
144 | 3 | RETURN_NOT_OK(s); |
145 | | |
146 | 0 | VLOG(1) << "Rolled " << log_name_ << " log to new file: " << path; |
147 | 3 | break; |
148 | 3 | } |
149 | 3 | return Status::OK(); |
150 | 3 | } |
151 | | |
152 | 4 | Status RollingLog::Close() { |
153 | 4 | if (!file_) { |
154 | 1 | return Status::OK(); |
155 | 1 | } |
156 | 3 | string path = file_->filename(); |
157 | 3 | RETURN_NOT_OK_PREPEND(file_->Close(), |
158 | 3 | Substitute("Unable to close $0", path)); |
159 | 3 | file_.reset(); |
160 | 3 | if (compress_after_close_) { |
161 | 1 | WARN_NOT_OK(CompressFile(path), "Unable to compress old log file"); |
162 | 1 | } |
163 | 3 | return Status::OK(); |
164 | 3 | } |
165 | | |
166 | 1.01k | Status RollingLog::Append(GStringPiece s) { |
167 | 1.01k | if (!file_) { |
168 | 1 | RETURN_NOT_OK_PREPEND(Open(), "Unable to open log"); |
169 | 1 | } |
170 | | |
171 | 1.01k | if (file_->Size() + s.size() > size_limit_bytes_) { |
172 | 1 | RETURN_NOT_OK_PREPEND(Close(), "Unable to roll log"); |
173 | 1 | RETURN_NOT_OK_PREPEND(Open(), "Unable to roll log"); |
174 | 1 | } |
175 | 1.01k | RETURN_NOT_OK(file_->Append(s)); |
176 | 1.01k | return Status::OK(); |
177 | 1.01k | } |
178 | | |
179 | | namespace { |
180 | | |
181 | 1 | Status GzClose(gzFile f) { |
182 | 1 | int err = gzclose(f); |
183 | 1 | switch (err) { |
184 | 1 | case Z_OK: |
185 | 1 | return Status::OK(); |
186 | 0 | case Z_STREAM_ERROR: |
187 | 0 | return STATUS(InvalidArgument, "Stream not valid"); |
188 | 0 | case Z_ERRNO: |
189 | 0 | return STATUS(IOError, "IO Error closing stream"); |
190 | 0 | case Z_MEM_ERROR: |
191 | 0 | return STATUS(RuntimeError, "Out of memory"); |
192 | 0 | case Z_BUF_ERROR: |
193 | 0 | return STATUS(IOError, "read ended in the middle of a stream"); |
194 | 0 | default: |
195 | 0 | return STATUS(IOError, "Unknown zlib error", SimpleItoa(err)); |
196 | 1 | } |
197 | 1 | } |
198 | | |
199 | | class ScopedGzipCloser { |
200 | | public: |
201 | | explicit ScopedGzipCloser(gzFile f) |
202 | 1 | : file_(f) { |
203 | 1 | } |
204 | | |
205 | 1 | ~ScopedGzipCloser() { |
206 | 1 | if (file_) { |
207 | 0 | WARN_NOT_OK(GzClose(file_), "Unable to close gzip stream"); |
208 | 0 | } |
209 | 1 | } |
210 | | |
211 | 1 | void Cancel() { |
212 | 1 | file_ = nullptr; |
213 | 1 | } |
214 | | |
215 | | private: |
216 | | gzFile file_; |
217 | | }; |
218 | | } // anonymous namespace |
219 | | |
220 | | // We implement CompressFile() manually using zlib APIs rather than forking |
221 | | // out to '/bin/gzip' since fork() can be expensive on processes that use a large |
222 | | // amount of memory. During the time of the fork, other threads could end up |
223 | | // blocked. Implementing it using the zlib stream APIs isn't too much code |
224 | | // and is less likely to be problematic. |
225 | 1 | Status RollingLog::CompressFile(const std::string& path) const { |
226 | 1 | std::unique_ptr<SequentialFile> in_file; |
227 | 1 | RETURN_NOT_OK_PREPEND(env_->NewSequentialFile(path, &in_file), |
228 | 1 | "Unable to open input file to compress"); |
229 | | |
230 | 1 | string gz_path = path + ".gz"; |
231 | 1 | gzFile gzf = gzopen(gz_path.c_str(), "w"); |
232 | 1 | if (!gzf) { |
233 | 0 | return STATUS(IOError, "Unable to open gzip stream"); |
234 | 0 | } |
235 | | |
236 | 1 | ScopedGzipCloser closer(gzf); |
237 | | |
238 | | // Loop reading data from the input file and writing to the gzip stream. |
239 | 1 | uint8_t buf[32 * 1024]; |
240 | 2 | while (true) { |
241 | 2 | Slice result; |
242 | 2 | RETURN_NOT_OK_PREPEND(in_file->Read(arraysize(buf), &result, buf), |
243 | 2 | "Unable to read from gzip input"); |
244 | 2 | if (result.size() == 0) { |
245 | 1 | break; |
246 | 1 | } |
247 | 1 | int n = gzwrite(gzf, result.data(), narrow_cast<unsigned>(result.size())); |
248 | 1 | if (n == 0) { |
249 | 0 | int errnum; |
250 | 0 | return STATUS(IOError, "Unable to write to gzip output", |
251 | 0 | gzerror(gzf, &errnum)); |
252 | 0 | } |
253 | 1 | } |
254 | 1 | closer.Cancel(); |
255 | 1 | RETURN_NOT_OK_PREPEND(GzClose(gzf), |
256 | 1 | "Unable to close gzip output"); |
257 | | |
258 | 1 | WARN_NOT_OK(env_->DeleteFile(path), |
259 | 1 | "Unable to delete gzip input file after compression"); |
260 | 1 | return Status::OK(); |
261 | 1 | } |
262 | | |
263 | | } // namespace yb |