001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs.server.datanode;
019
020
021 import java.io.Closeable;
022 import java.io.FilterInputStream;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.OutputStream;
026
027 import org.apache.hadoop.classification.InterfaceAudience;
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.hadoop.hdfs.protocol.Block;
030 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
031 import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
032 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
033 import org.apache.hadoop.hdfs.server.datanode.metrics.FSDatasetMBean;
034 import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
035 import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
036 import org.apache.hadoop.io.IOUtils;
037 import org.apache.hadoop.util.DataChecksum;
038 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
039
040 /**
041 * This is an interface for the underlying storage that stores blocks for
042 * a data node.
043 * Examples are the FSDataset (which stores blocks on dirs) and
044 * SimulatedFSDataset (which simulates data).
045 *
046 */
047 @InterfaceAudience.Private
048 public interface FSDatasetInterface extends FSDatasetMBean {
049
050
051 /**
052 * Returns the length of the metadata file of the specified block
053 * @param b - the block for which the metadata length is desired
054 * @return the length of the metadata file for the specified block.
055 * @throws IOException
056 */
057 public long getMetaDataLength(ExtendedBlock b) throws IOException;
058
059 /**
060 * This class provides the input stream and length of the metadata
061 * of a block
062 *
063 */
064 static class MetaDataInputStream extends FilterInputStream {
065 MetaDataInputStream(InputStream stream, long len) {
066 super(stream);
067 length = len;
068 }
069 private long length;
070
071 public long getLength() {
072 return length;
073 }
074 }
075
076 /**
077 * Returns metaData of block b as an input stream (and its length)
078 * @param b - the block
079 * @return the metadata input stream;
080 * @throws IOException
081 */
082 public MetaDataInputStream getMetaDataInputStream(ExtendedBlock b)
083 throws IOException;
084
085 /**
086 * Does the meta file exist for this block?
087 * @param b - the block
088 * @return true of the metafile for specified block exits
089 * @throws IOException
090 */
091 public boolean metaFileExists(ExtendedBlock b) throws IOException;
092
093
094 /**
095 * Returns the specified block's on-disk length (excluding metadata)
096 * @param b
097 * @return the specified block's on-disk length (excluding metadta)
098 * @throws IOException
099 */
100 public long getLength(ExtendedBlock b) throws IOException;
101
102 /**
103 * Get reference to the replica meta info in the replicasMap.
104 * To be called from methods that are synchronized on {@link FSDataset}
105 * @param blockId
106 * @return replica from the replicas map
107 */
108 @Deprecated
109 public Replica getReplica(String bpid, long blockId);
110
111 /**
112 * @return replica meta information
113 */
114 public String getReplicaString(String bpid, long blockId);
115
116 /**
117 * @return the generation stamp stored with the block.
118 */
119 public Block getStoredBlock(String bpid, long blkid)
120 throws IOException;
121
122 /**
123 * Returns an input stream to read the contents of the specified block
124 * @param b
125 * @return an input stream to read the contents of the specified block
126 * @throws IOException
127 */
128 public InputStream getBlockInputStream(ExtendedBlock b) throws IOException;
129
130 /**
131 * Returns an input stream at specified offset of the specified block
132 * @param b
133 * @param seekOffset
134 * @return an input stream to read the contents of the specified block,
135 * starting at the offset
136 * @throws IOException
137 */
138 public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset)
139 throws IOException;
140
141 /**
142 * Returns an input stream at specified offset of the specified block
143 * The block is still in the tmp directory and is not finalized
144 * @param b
145 * @param blkoff
146 * @param ckoff
147 * @return an input stream to read the contents of the specified block,
148 * starting at the offset
149 * @throws IOException
150 */
151 public BlockInputStreams getTmpInputStreams(ExtendedBlock b, long blkoff,
152 long ckoff) throws IOException;
153
154 /**
155 *
156 * This class contains the output streams for the data and checksum
157 * of a block
158 *
159 */
160 static class BlockWriteStreams {
161 OutputStream dataOut;
162 OutputStream checksumOut;
163 DataChecksum checksum;
164
165 BlockWriteStreams(OutputStream dOut, OutputStream cOut,
166 DataChecksum checksum) {
167 dataOut = dOut;
168 checksumOut = cOut;
169 this.checksum = checksum;
170 }
171
172 void close() throws IOException {
173 IOUtils.closeStream(dataOut);
174 IOUtils.closeStream(checksumOut);
175 }
176
177 DataChecksum getChecksum() {
178 return checksum;
179 }
180 }
181
182 /**
183 * This class contains the input streams for the data and checksum
184 * of a block
185 */
186 static class BlockInputStreams implements Closeable {
187 final InputStream dataIn;
188 final InputStream checksumIn;
189
190 BlockInputStreams(InputStream dataIn, InputStream checksumIn) {
191 this.dataIn = dataIn;
192 this.checksumIn = checksumIn;
193 }
194
195 /** {@inheritDoc} */
196 public void close() {
197 IOUtils.closeStream(dataIn);
198 IOUtils.closeStream(checksumIn);
199 }
200 }
201
202 /**
203 * Creates a temporary replica and returns the meta information of the replica
204 *
205 * @param b block
206 * @return the meta info of the replica which is being written to
207 * @throws IOException if an error occurs
208 */
209 public ReplicaInPipelineInterface createTemporary(ExtendedBlock b)
210 throws IOException;
211
212 /**
213 * Creates a RBW replica and returns the meta info of the replica
214 *
215 * @param b block
216 * @return the meta info of the replica which is being written to
217 * @throws IOException if an error occurs
218 */
219 public ReplicaInPipelineInterface createRbw(ExtendedBlock b) throws IOException;
220
221 /**
222 * Recovers a RBW replica and returns the meta info of the replica
223 *
224 * @param b block
225 * @param newGS the new generation stamp for the replica
226 * @param minBytesRcvd the minimum number of bytes that the replica could have
227 * @param maxBytesRcvd the maximum number of bytes that the replica could have
228 * @return the meta info of the replica which is being written to
229 * @throws IOException if an error occurs
230 */
231 public ReplicaInPipelineInterface recoverRbw(ExtendedBlock b,
232 long newGS, long minBytesRcvd, long maxBytesRcvd)
233 throws IOException;
234
235 /**
236 * Covert a temporary replica to a RBW.
237 * @param temporary the temporary replica being converted
238 * @return the result RBW
239 */
240 public ReplicaInPipelineInterface convertTemporaryToRbw(
241 ExtendedBlock temporary) throws IOException;
242
243 /**
244 * Append to a finalized replica and returns the meta info of the replica
245 *
246 * @param b block
247 * @param newGS the new generation stamp for the replica
248 * @param expectedBlockLen the number of bytes the replica is expected to have
249 * @return the meata info of the replica which is being written to
250 * @throws IOException
251 */
252 public ReplicaInPipelineInterface append(ExtendedBlock b,
253 long newGS, long expectedBlockLen) throws IOException;
254
255 /**
256 * Recover a failed append to a finalized replica
257 * and returns the meta info of the replica
258 *
259 * @param b block
260 * @param newGS the new generation stamp for the replica
261 * @param expectedBlockLen the number of bytes the replica is expected to have
262 * @return the meta info of the replica which is being written to
263 * @throws IOException
264 */
265 public ReplicaInPipelineInterface recoverAppend(ExtendedBlock b,
266 long newGS, long expectedBlockLen) throws IOException;
267
268 /**
269 * Recover a failed pipeline close
270 * It bumps the replica's generation stamp and finalize it if RBW replica
271 *
272 * @param b block
273 * @param newGS the new generation stamp for the replica
274 * @param expectedBlockLen the number of bytes the replica is expected to have
275 * @throws IOException
276 */
277 public void recoverClose(ExtendedBlock b,
278 long newGS, long expectedBlockLen) throws IOException;
279
280 /**
281 * Finalizes the block previously opened for writing using writeToBlock.
282 * The block size is what is in the parameter b and it must match the amount
283 * of data written
284 * @param b
285 * @throws IOException
286 */
287 public void finalizeBlock(ExtendedBlock b) throws IOException;
288
289 /**
290 * Unfinalizes the block previously opened for writing using writeToBlock.
291 * The temporary file associated with this block is deleted.
292 * @param b
293 * @throws IOException
294 */
295 public void unfinalizeBlock(ExtendedBlock b) throws IOException;
296
297 /**
298 * Returns the block report - the full list of blocks stored under a
299 * block pool
300 * @param bpid Block Pool Id
301 * @return - the block report - the full list of blocks stored
302 */
303 public BlockListAsLongs getBlockReport(String bpid);
304
305 /** Does the dataset contain the block? */
306 public boolean contains(ExtendedBlock block);
307
308 /**
309 * Is the block valid?
310 * @param b
311 * @return - true if the specified block is valid
312 */
313 public boolean isValidBlock(ExtendedBlock b);
314
315 /**
316 * Is the block a valid RBW?
317 * @param b
318 * @return - true if the specified block is a valid RBW
319 */
320 public boolean isValidRbw(ExtendedBlock b);
321
322 /**
323 * Invalidates the specified blocks
324 * @param bpid Block pool Id
325 * @param invalidBlks - the blocks to be invalidated
326 * @throws IOException
327 */
328 public void invalidate(String bpid, Block invalidBlks[]) throws IOException;
329
330 /**
331 * Check if all the data directories are healthy
332 * @throws DiskErrorException
333 */
334 public void checkDataDir() throws DiskErrorException;
335
336 /**
337 * Stringifies the name of the storage
338 */
339 public String toString();
340
341 /**
342 * Shutdown the FSDataset
343 */
344 public void shutdown();
345
346 /**
347 * Sets the file pointer of the checksum stream so that the last checksum
348 * will be overwritten
349 * @param b block
350 * @param stream The stream for the data file and checksum file
351 * @param checksumSize number of bytes each checksum has
352 * @throws IOException
353 */
354 public void adjustCrcChannelPosition(ExtendedBlock b, BlockWriteStreams stream,
355 int checksumSize) throws IOException;
356
357 /**
358 * Checks how many valid storage volumes there are in the DataNode.
359 * @return true if more than the minimum number of valid volumes are left
360 * in the FSDataSet.
361 */
362 public boolean hasEnoughResource();
363
364 /**
365 * Get visible length of the specified replica.
366 */
367 long getReplicaVisibleLength(final ExtendedBlock block) throws IOException;
368
369 /**
370 * Initialize a replica recovery.
371 * @return actual state of the replica on this data-node or
372 * null if data-node does not have the replica.
373 */
374 public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
375 throws IOException;
376
377 /**
378 * Update replica's generation stamp and length and finalize it.
379 */
380 public ReplicaInfo updateReplicaUnderRecovery(
381 ExtendedBlock oldBlock,
382 long recoveryId,
383 long newLength) throws IOException;
384 /**
385 * add new block pool ID
386 * @param bpid Block pool Id
387 * @param conf Configuration
388 */
389 public void addBlockPool(String bpid, Configuration conf) throws IOException;
390
391 /**
392 * Shutdown and remove the block pool from underlying storage.
393 * @param bpid Block pool Id to be removed
394 */
395 public void shutdownBlockPool(String bpid) ;
396
397 /**
398 * Deletes the block pool directories. If force is false, directories are
399 * deleted only if no block files exist for the block pool. If force
400 * is true entire directory for the blockpool is deleted along with its
401 * contents.
402 * @param bpid BlockPool Id to be deleted.
403 * @param force If force is false, directories are deleted only if no
404 * block files exist for the block pool, otherwise entire
405 * directory for the blockpool is deleted along with its contents.
406 * @throws IOException
407 */
408 public void deleteBlockPool(String bpid, boolean force) throws IOException;
409
410 /**
411 * Get {@link BlockLocalPathInfo} for the given block.
412 **/
413 public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b) throws IOException;
414 }