001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.server.namenode;
020
021 import java.io.File;
022 import java.io.FileInputStream;
023 import java.io.IOException;
024 import java.io.BufferedInputStream;
025 import java.io.EOFException;
026 import java.io.DataInputStream;
027 import org.apache.commons.logging.Log;
028 import org.apache.commons.logging.LogFactory;
029 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
030 import org.apache.hadoop.hdfs.server.common.Storage;
031 import org.apache.hadoop.io.IOUtils;
032 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
033
034 import com.google.common.annotations.VisibleForTesting;
035 import com.google.common.base.Preconditions;
036 import com.google.common.base.Throwables;
037
038 /**
039 * An implementation of the abstract class {@link EditLogInputStream}, which
040 * reads edits from a local file.
041 */
042 public class EditLogFileInputStream extends EditLogInputStream {
043 private final File file;
044 private final long firstTxId;
045 private final long lastTxId;
046 private final boolean isInProgress;
047 static private enum State {
048 UNINIT,
049 OPEN,
050 CLOSED
051 }
052 private State state = State.UNINIT;
053 private FileInputStream fStream = null;
054 private int logVersion = 0;
055 private FSEditLogOp.Reader reader = null;
056 private FSEditLogLoader.PositionTrackingInputStream tracker = null;
057 private DataInputStream dataIn = null;
058 static final Log LOG = LogFactory.getLog(EditLogInputStream.class);
059
060 /**
061 * Open an EditLogInputStream for the given file.
062 * The file is pretransactional, so has no txids
063 * @param name filename to open
064 * @throws LogHeaderCorruptException if the header is either missing or
065 * appears to be corrupt/truncated
066 * @throws IOException if an actual IO error occurs while reading the
067 * header
068 */
069 EditLogFileInputStream(File name)
070 throws LogHeaderCorruptException, IOException {
071 this(name, HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID, false);
072 }
073
074 /**
075 * Open an EditLogInputStream for the given file.
076 * @param name filename to open
077 * @param firstTxId first transaction found in file
078 * @param lastTxId last transaction id found in file
079 * @throws LogHeaderCorruptException if the header is either missing or
080 * appears to be corrupt/truncated
081 * @throws IOException if an actual IO error occurs while reading the
082 * header
083 */
084 public EditLogFileInputStream(File name, long firstTxId, long lastTxId,
085 boolean isInProgress) {
086 this.file = name;
087 this.firstTxId = firstTxId;
088 this.lastTxId = lastTxId;
089 this.isInProgress = isInProgress;
090 }
091
092 private void init() throws LogHeaderCorruptException, IOException {
093 Preconditions.checkState(state == State.UNINIT);
094 BufferedInputStream bin = null;
095 try {
096 fStream = new FileInputStream(file);
097 bin = new BufferedInputStream(fStream);
098 tracker = new FSEditLogLoader.PositionTrackingInputStream(bin);
099 dataIn = new DataInputStream(tracker);
100 try {
101 logVersion = readLogVersion(dataIn);
102 } catch (EOFException eofe) {
103 throw new LogHeaderCorruptException("No header found in log");
104 }
105 reader = new FSEditLogOp.Reader(dataIn, tracker, logVersion);
106 state = State.OPEN;
107 } finally {
108 if (reader == null) {
109 IOUtils.cleanup(LOG, dataIn, tracker, bin, fStream);
110 state = State.CLOSED;
111 }
112 }
113 }
114
115 @Override
116 public long getFirstTxId() {
117 return firstTxId;
118 }
119
120 @Override
121 public long getLastTxId() {
122 return lastTxId;
123 }
124
125 @Override
126 public String getName() {
127 return file.getPath();
128 }
129
130 private FSEditLogOp nextOpImpl(boolean skipBrokenEdits) throws IOException {
131 FSEditLogOp op = null;
132 switch (state) {
133 case UNINIT:
134 try {
135 init();
136 } catch (Throwable e) {
137 LOG.error("caught exception initializing " + this, e);
138 if (skipBrokenEdits) {
139 return null;
140 }
141 Throwables.propagateIfPossible(e, IOException.class);
142 }
143 Preconditions.checkState(state != State.UNINIT);
144 return nextOpImpl(skipBrokenEdits);
145 case OPEN:
146 op = reader.readOp(skipBrokenEdits);
147 if ((op != null) && (op.hasTransactionId())) {
148 long txId = op.getTransactionId();
149 if ((txId >= lastTxId) &&
150 (lastTxId != HdfsConstants.INVALID_TXID)) {
151 //
152 // Sometimes, the NameNode crashes while it's writing to the
153 // edit log. In that case, you can end up with an unfinalized edit log
154 // which has some garbage at the end.
155 // JournalManager#recoverUnfinalizedSegments will finalize these
156 // unfinished edit logs, giving them a defined final transaction
157 // ID. Then they will be renamed, so that any subsequent
158 // readers will have this information.
159 //
160 // Since there may be garbage at the end of these "cleaned up"
161 // logs, we want to be sure to skip it here if we've read everything
162 // we were supposed to read out of the stream.
163 // So we force an EOF on all subsequent reads.
164 //
165 long skipAmt = file.length() - tracker.getPos();
166 if (skipAmt > 0) {
167 if (LOG.isDebugEnabled()) {
168 LOG.debug("skipping " + skipAmt + " bytes at the end " +
169 "of edit log '" + getName() + "': reached txid " + txId +
170 " out of " + lastTxId);
171 }
172 tracker.clearLimit();
173 IOUtils.skipFully(tracker, skipAmt);
174 }
175 }
176 }
177 break;
178 case CLOSED:
179 break; // return null
180 }
181 return op;
182 }
183
184 @Override
185 protected FSEditLogOp nextOp() throws IOException {
186 return nextOpImpl(false);
187 }
188
189 @Override
190 protected FSEditLogOp nextValidOp() {
191 try {
192 return nextOpImpl(true);
193 } catch (Throwable e) {
194 LOG.error("nextValidOp: got exception while reading " + this, e);
195 return null;
196 }
197 }
198
199 @Override
200 public int getVersion() throws IOException {
201 if (state == State.UNINIT) {
202 init();
203 }
204 return logVersion;
205 }
206
207 @Override
208 public long getPosition() {
209 if (state == State.OPEN) {
210 return tracker.getPos();
211 } else {
212 return 0;
213 }
214 }
215
216 @Override
217 public void close() throws IOException {
218 if (state == State.OPEN) {
219 dataIn.close();
220 }
221 state = State.CLOSED;
222 }
223
224 @Override
225 public long length() throws IOException {
226 // file size + size of both buffers
227 return file.length();
228 }
229
230 @Override
231 public boolean isInProgress() {
232 return isInProgress;
233 }
234
235 @Override
236 public String toString() {
237 return getName();
238 }
239
240 static FSEditLogLoader.EditLogValidation validateEditLog(File file) throws IOException {
241 EditLogFileInputStream in;
242 try {
243 in = new EditLogFileInputStream(file);
244 in.getVersion(); // causes us to read the header
245 } catch (LogHeaderCorruptException e) {
246 // If the header is malformed or the wrong value, this indicates a corruption
247 LOG.warn("Log file " + file + " has no valid header", e);
248 return new FSEditLogLoader.EditLogValidation(0,
249 HdfsConstants.INVALID_TXID, true);
250 }
251
252 try {
253 return FSEditLogLoader.validateEditLog(in);
254 } finally {
255 IOUtils.closeStream(in);
256 }
257 }
258
259 /**
260 * Read the header of fsedit log
261 * @param in fsedit stream
262 * @return the edit log version number
263 * @throws IOException if error occurs
264 */
265 @VisibleForTesting
266 static int readLogVersion(DataInputStream in)
267 throws IOException, LogHeaderCorruptException {
268 int logVersion;
269 try {
270 logVersion = in.readInt();
271 } catch (EOFException eofe) {
272 throw new LogHeaderCorruptException(
273 "Reached EOF when reading log header");
274 }
275 if (logVersion < HdfsConstants.LAYOUT_VERSION || // future version
276 logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION) { // unsupported
277 throw new LogHeaderCorruptException(
278 "Unexpected version of the file system log file: "
279 + logVersion + ". Current version = "
280 + HdfsConstants.LAYOUT_VERSION + ".");
281 }
282 return logVersion;
283 }
284
285 /**
286 * Exception indicating that the header of an edits log file is
287 * corrupted. This can be because the header is not present,
288 * or because the header data is invalid (eg claims to be
289 * over a newer version than the running NameNode)
290 */
291 static class LogHeaderCorruptException extends IOException {
292 private static final long serialVersionUID = 1L;
293
294 private LogHeaderCorruptException(String msg) {
295 super(msg);
296 }
297 }
298 }