001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs;
019
020 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT;
027 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY;
028 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT;
029 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_KEY;
030 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
031 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
032 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
033 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
034 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
035 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
036 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT;
037 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY;
038 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY;
039 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE;
040 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT;
041 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY;
042 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
043 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER;
044 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT;
045 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
046 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
047 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
048 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
049 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
050 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
051 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
052
053 import java.io.BufferedOutputStream;
054 import java.io.DataInputStream;
055 import java.io.DataOutputStream;
056 import java.io.FileNotFoundException;
057 import java.io.IOException;
058 import java.io.InputStream;
059 import java.io.OutputStream;
060 import java.net.InetAddress;
061 import java.net.InetSocketAddress;
062 import java.net.NetworkInterface;
063 import java.net.Socket;
064 import java.net.SocketException;
065 import java.net.SocketAddress;
066 import java.net.URI;
067 import java.net.UnknownHostException;
068 import java.util.ArrayList;
069 import java.util.Collections;
070 import java.util.EnumSet;
071 import java.util.HashMap;
072 import java.util.LinkedHashMap;
073 import java.util.List;
074 import java.util.Map;
075 import java.util.Random;
076
077 import javax.net.SocketFactory;
078
079 import org.apache.commons.logging.Log;
080 import org.apache.commons.logging.LogFactory;
081 import org.apache.hadoop.classification.InterfaceAudience;
082 import org.apache.hadoop.conf.Configuration;
083 import org.apache.hadoop.fs.BlockLocation;
084 import org.apache.hadoop.fs.BlockStorageLocation;
085 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
086 import org.apache.hadoop.fs.ContentSummary;
087 import org.apache.hadoop.fs.CreateFlag;
088 import org.apache.hadoop.fs.FileAlreadyExistsException;
089 import org.apache.hadoop.fs.FileSystem;
090 import org.apache.hadoop.fs.FsServerDefaults;
091 import org.apache.hadoop.fs.FsStatus;
092 import org.apache.hadoop.fs.HdfsBlockLocation;
093 import org.apache.hadoop.fs.InvalidPathException;
094 import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
095 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
096 import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
097 import org.apache.hadoop.fs.Options;
098 import org.apache.hadoop.fs.Options.ChecksumOpt;
099 import org.apache.hadoop.fs.ParentNotDirectoryException;
100 import org.apache.hadoop.fs.Path;
101 import org.apache.hadoop.fs.UnresolvedLinkException;
102 import org.apache.hadoop.fs.VolumeId;
103 import org.apache.hadoop.fs.permission.FsPermission;
104 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
105 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
106 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
107 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
108 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
109 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
110 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
111 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
112 import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
113 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
114 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
115 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
116 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
117 import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil;
118 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
119 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
120 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
121 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
122 import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor;
123 import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
124 import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
125 import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
126 import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
127 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
128 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto;
129 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
130 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
131 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
132 import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
133 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
134 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
135 import org.apache.hadoop.hdfs.server.namenode.NameNode;
136 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
137 import org.apache.hadoop.io.DataOutputBuffer;
138 import org.apache.hadoop.io.EnumSetWritable;
139 import org.apache.hadoop.io.IOUtils;
140 import org.apache.hadoop.io.MD5Hash;
141 import org.apache.hadoop.io.Text;
142 import org.apache.hadoop.ipc.Client;
143 import org.apache.hadoop.ipc.RPC;
144 import org.apache.hadoop.ipc.RemoteException;
145 import org.apache.hadoop.net.DNS;
146 import org.apache.hadoop.net.NetUtils;
147 import org.apache.hadoop.security.AccessControlException;
148 import org.apache.hadoop.security.UserGroupInformation;
149 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
150 import org.apache.hadoop.security.token.Token;
151 import org.apache.hadoop.security.token.TokenRenewer;
152 import org.apache.hadoop.util.DataChecksum;
153 import org.apache.hadoop.util.Progressable;
154 import org.apache.hadoop.util.Time;
155
156 import com.google.common.annotations.VisibleForTesting;
157 import com.google.common.base.Joiner;
158 import com.google.common.base.Preconditions;
159 import com.google.common.net.InetAddresses;
160
161 /********************************************************
162 * DFSClient can connect to a Hadoop Filesystem and
163 * perform basic file tasks. It uses the ClientProtocol
164 * to communicate with a NameNode daemon, and connects
165 * directly to DataNodes to read/write block data.
166 *
167 * Hadoop DFS users should obtain an instance of
168 * DistributedFileSystem, which uses DFSClient to handle
169 * filesystem tasks.
170 *
171 ********************************************************/
172 @InterfaceAudience.Private
173 public class DFSClient implements java.io.Closeable {
174 public static final Log LOG = LogFactory.getLog(DFSClient.class);
175 public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; // 1 hour
176 static final int TCP_WINDOW_SIZE = 128 * 1024; // 128 KB
177 final ClientProtocol namenode;
178 /* The service used for delegation tokens */
179 private Text dtService;
180
181 final UserGroupInformation ugi;
182 volatile boolean clientRunning = true;
183 volatile long lastLeaseRenewal;
184 private volatile FsServerDefaults serverDefaults;
185 private volatile long serverDefaultsLastUpdate;
186 final String clientName;
187 Configuration conf;
188 SocketFactory socketFactory;
189 final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
190 final FileSystem.Statistics stats;
191 final int hdfsTimeout; // timeout value for a DFS operation.
192 private final String authority;
193 final SocketCache socketCache;
194 final Conf dfsClientConf;
195 private Random r = new Random();
196 private SocketAddress[] localInterfaceAddrs;
197 private DataEncryptionKey encryptionKey;
198
199 /**
200 * DFSClient configuration
201 */
202 static class Conf {
203 final int maxFailoverAttempts;
204 final int failoverSleepBaseMillis;
205 final int failoverSleepMaxMillis;
206 final int maxBlockAcquireFailures;
207 final int confTime;
208 final int ioBufferSize;
209 final ChecksumOpt defaultChecksumOpt;
210 final int writePacketSize;
211 final int socketTimeout;
212 final int socketCacheCapacity;
213 /** Wait time window (in msec) if BlockMissingException is caught */
214 final int timeWindow;
215 final int nCachedConnRetry;
216 final int nBlockWriteRetry;
217 final int nBlockWriteLocateFollowingRetry;
218 final long defaultBlockSize;
219 final long prefetchSize;
220 final short defaultReplication;
221 final String taskId;
222 final FsPermission uMask;
223 final boolean useLegacyBlockReader;
224 final boolean connectToDnViaHostname;
225 final boolean getHdfsBlocksMetadataEnabled;
226 final int getFileBlockStorageLocationsNumThreads;
227 final int getFileBlockStorageLocationsTimeout;
228
229 Conf(Configuration conf) {
230 maxFailoverAttempts = conf.getInt(
231 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
232 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
233 failoverSleepBaseMillis = conf.getInt(
234 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
235 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
236 failoverSleepMaxMillis = conf.getInt(
237 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
238 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
239
240 maxBlockAcquireFailures = conf.getInt(
241 DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY,
242 DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT);
243 confTime = conf.getInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
244 HdfsServerConstants.WRITE_TIMEOUT);
245 ioBufferSize = conf.getInt(
246 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
247 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
248 defaultChecksumOpt = getChecksumOptFromConf(conf);
249 socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
250 HdfsServerConstants.READ_TIMEOUT);
251 /** dfs.write.packet.size is an internal config variable */
252 writePacketSize = conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY,
253 DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT);
254 defaultBlockSize = conf.getLongBytes(DFS_BLOCK_SIZE_KEY,
255 DFS_BLOCK_SIZE_DEFAULT);
256 defaultReplication = (short) conf.getInt(
257 DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT);
258 taskId = conf.get("mapreduce.task.attempt.id", "NONMAPREDUCE");
259 socketCacheCapacity = conf.getInt(DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY,
260 DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT);
261 prefetchSize = conf.getLong(DFS_CLIENT_READ_PREFETCH_SIZE_KEY,
262 10 * defaultBlockSize);
263 timeWindow = conf
264 .getInt(DFS_CLIENT_RETRY_WINDOW_BASE, 3000);
265 nCachedConnRetry = conf.getInt(DFS_CLIENT_CACHED_CONN_RETRY_KEY,
266 DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT);
267 nBlockWriteRetry = conf.getInt(DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY,
268 DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT);
269 nBlockWriteLocateFollowingRetry = conf
270 .getInt(DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
271 DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT);
272 uMask = FsPermission.getUMask(conf);
273 useLegacyBlockReader = conf.getBoolean(
274 DFS_CLIENT_USE_LEGACY_BLOCKREADER,
275 DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT);
276 connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
277 DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
278 getHdfsBlocksMetadataEnabled = conf.getBoolean(
279 DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
280 DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
281 getFileBlockStorageLocationsNumThreads = conf.getInt(
282 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS,
283 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT);
284 getFileBlockStorageLocationsTimeout = conf.getInt(
285 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT,
286 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT);
287 }
288
289 private DataChecksum.Type getChecksumType(Configuration conf) {
290 final String checksum = conf.get(
291 DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY,
292 DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
293 try {
294 return DataChecksum.Type.valueOf(checksum);
295 } catch(IllegalArgumentException iae) {
296 LOG.warn("Bad checksum type: " + checksum + ". Using default "
297 + DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
298 return DataChecksum.Type.valueOf(
299 DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
300 }
301 }
302
303 // Construct a checksum option from conf
304 private ChecksumOpt getChecksumOptFromConf(Configuration conf) {
305 DataChecksum.Type type = getChecksumType(conf);
306 int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY,
307 DFS_BYTES_PER_CHECKSUM_DEFAULT);
308 return new ChecksumOpt(type, bytesPerChecksum);
309 }
310
311 // create a DataChecksum with the default option.
312 private DataChecksum createChecksum() throws IOException {
313 return createChecksum(null);
314 }
315
316 private DataChecksum createChecksum(ChecksumOpt userOpt)
317 throws IOException {
318 // Fill in any missing field with the default.
319 ChecksumOpt myOpt = ChecksumOpt.processChecksumOpt(
320 defaultChecksumOpt, userOpt);
321 DataChecksum dataChecksum = DataChecksum.newDataChecksum(
322 myOpt.getChecksumType(),
323 myOpt.getBytesPerChecksum());
324 if (dataChecksum == null) {
325 throw new IOException("Invalid checksum type specified: "
326 + myOpt.getChecksumType().name());
327 }
328 return dataChecksum;
329 }
330 }
331
332 Conf getConf() {
333 return dfsClientConf;
334 }
335
336 /**
337 * A map from file names to {@link DFSOutputStream} objects
338 * that are currently being written by this client.
339 * Note that a file can only be written by a single client.
340 */
341 private final Map<String, DFSOutputStream> filesBeingWritten
342 = new HashMap<String, DFSOutputStream>();
343
344 private boolean shortCircuitLocalReads;
345
346 /**
347 * Same as this(NameNode.getAddress(conf), conf);
348 * @see #DFSClient(InetSocketAddress, Configuration)
349 * @deprecated Deprecated at 0.21
350 */
351 @Deprecated
352 public DFSClient(Configuration conf) throws IOException {
353 this(NameNode.getAddress(conf), conf);
354 }
355
356 public DFSClient(InetSocketAddress address, Configuration conf) throws IOException {
357 this(NameNode.getUri(address), conf);
358 }
359
360 /**
361 * Same as this(nameNodeUri, conf, null);
362 * @see #DFSClient(InetSocketAddress, Configuration, org.apache.hadoop.fs.FileSystem.Statistics)
363 */
364 public DFSClient(URI nameNodeUri, Configuration conf
365 ) throws IOException {
366 this(nameNodeUri, conf, null);
367 }
368
369 /**
370 * Same as this(nameNodeUri, null, conf, stats);
371 * @see #DFSClient(InetSocketAddress, ClientProtocol, Configuration, org.apache.hadoop.fs.FileSystem.Statistics)
372 */
373 public DFSClient(URI nameNodeUri, Configuration conf,
374 FileSystem.Statistics stats)
375 throws IOException {
376 this(nameNodeUri, null, conf, stats);
377 }
378
379 /**
380 * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode.
381 * Exactly one of nameNodeUri or rpcNamenode must be null.
382 */
383 DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
384 Configuration conf, FileSystem.Statistics stats)
385 throws IOException {
386 // Copy only the required DFSClient configuration
387 this.dfsClientConf = new Conf(conf);
388 this.conf = conf;
389 this.stats = stats;
390 this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
391 this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
392
393 // The hdfsTimeout is currently the same as the ipc timeout
394 this.hdfsTimeout = Client.getTimeout(conf);
395 this.ugi = UserGroupInformation.getCurrentUser();
396
397 this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
398 this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
399 DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
400
401 if (rpcNamenode != null) {
402 // This case is used for testing.
403 Preconditions.checkArgument(nameNodeUri == null);
404 this.namenode = rpcNamenode;
405 dtService = null;
406 } else {
407 Preconditions.checkArgument(nameNodeUri != null,
408 "null URI");
409 NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo =
410 NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class);
411 this.dtService = proxyInfo.getDelegationTokenService();
412 this.namenode = proxyInfo.getProxy();
413 }
414
415 // read directly from the block file if configured.
416 this.shortCircuitLocalReads = conf.getBoolean(
417 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
418 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT);
419 if (LOG.isDebugEnabled()) {
420 LOG.debug("Short circuit read is " + shortCircuitLocalReads);
421 }
422 String localInterfaces[] =
423 conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES);
424 localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces);
425 if (LOG.isDebugEnabled() && 0 != localInterfaces.length) {
426 LOG.debug("Using local interfaces [" +
427 Joiner.on(',').join(localInterfaces)+ "] with addresses [" +
428 Joiner.on(',').join(localInterfaceAddrs) + "]");
429 }
430
431 this.socketCache = new SocketCache(dfsClientConf.socketCacheCapacity);
432 }
433
434 /**
435 * Return the socket addresses to use with each configured
436 * local interface. Local interfaces may be specified by IP
437 * address, IP address range using CIDR notation, interface
438 * name (e.g. eth0) or sub-interface name (e.g. eth0:0).
439 * The socket addresses consist of the IPs for the interfaces
440 * and the ephemeral port (port 0). If an IP, IP range, or
441 * interface name matches an interface with sub-interfaces
442 * only the IP of the interface is used. Sub-interfaces can
443 * be used by specifying them explicitly (by IP or name).
444 *
445 * @return SocketAddresses for the configured local interfaces,
446 * or an empty array if none are configured
447 * @throws UnknownHostException if a given interface name is invalid
448 */
449 private static SocketAddress[] getLocalInterfaceAddrs(
450 String interfaceNames[]) throws UnknownHostException {
451 List<SocketAddress> localAddrs = new ArrayList<SocketAddress>();
452 for (String interfaceName : interfaceNames) {
453 if (InetAddresses.isInetAddress(interfaceName)) {
454 localAddrs.add(new InetSocketAddress(interfaceName, 0));
455 } else if (NetUtils.isValidSubnet(interfaceName)) {
456 for (InetAddress addr : NetUtils.getIPs(interfaceName, false)) {
457 localAddrs.add(new InetSocketAddress(addr, 0));
458 }
459 } else {
460 for (String ip : DNS.getIPs(interfaceName, false)) {
461 localAddrs.add(new InetSocketAddress(ip, 0));
462 }
463 }
464 }
465 return localAddrs.toArray(new SocketAddress[localAddrs.size()]);
466 }
467
468 /**
469 * Select one of the configured local interfaces at random. We use a random
470 * interface because other policies like round-robin are less effective
471 * given that we cache connections to datanodes.
472 *
473 * @return one of the local interface addresses at random, or null if no
474 * local interfaces are configured
475 */
476 SocketAddress getRandomLocalInterfaceAddr() {
477 if (localInterfaceAddrs.length == 0) {
478 return null;
479 }
480 final int idx = r.nextInt(localInterfaceAddrs.length);
481 final SocketAddress addr = localInterfaceAddrs[idx];
482 if (LOG.isDebugEnabled()) {
483 LOG.debug("Using local interface " + addr);
484 }
485 return addr;
486 }
487
488 /**
489 * Return the number of times the client should go back to the namenode
490 * to retrieve block locations when reading.
491 */
492 int getMaxBlockAcquireFailures() {
493 return dfsClientConf.maxBlockAcquireFailures;
494 }
495
496 /**
497 * Return the timeout that clients should use when writing to datanodes.
498 * @param numNodes the number of nodes in the pipeline.
499 */
500 int getDatanodeWriteTimeout(int numNodes) {
501 return (dfsClientConf.confTime > 0) ?
502 (dfsClientConf.confTime + HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * numNodes) : 0;
503 }
504
505 int getDatanodeReadTimeout(int numNodes) {
506 return dfsClientConf.socketTimeout > 0 ?
507 (HdfsServerConstants.READ_TIMEOUT_EXTENSION * numNodes +
508 dfsClientConf.socketTimeout) : 0;
509 }
510
511 int getHdfsTimeout() {
512 return hdfsTimeout;
513 }
514
515 String getClientName() {
516 return clientName;
517 }
518
519 /**
520 * @return whether the client should use hostnames instead of IPs
521 * when connecting to DataNodes
522 */
523 boolean connectToDnViaHostname() {
524 return dfsClientConf.connectToDnViaHostname;
525 }
526
527 void checkOpen() throws IOException {
528 if (!clientRunning) {
529 IOException result = new IOException("Filesystem closed");
530 throw result;
531 }
532 }
533
534 /** Return the lease renewer instance. The renewer thread won't start
535 * until the first output stream is created. The same instance will
536 * be returned until all output streams are closed.
537 */
538 public LeaseRenewer getLeaseRenewer() throws IOException {
539 return LeaseRenewer.getInstance(authority, ugi, this);
540 }
541
542 /** Get a lease and start automatic renewal */
543 private void beginFileLease(final String src, final DFSOutputStream out)
544 throws IOException {
545 getLeaseRenewer().put(src, out, this);
546 }
547
548 /** Stop renewal of lease for the file. */
549 void endFileLease(final String src) throws IOException {
550 getLeaseRenewer().closeFile(src, this);
551 }
552
553
554 /** Put a file. Only called from LeaseRenewer, where proper locking is
555 * enforced to consistently update its local dfsclients array and
556 * client's filesBeingWritten map.
557 */
558 void putFileBeingWritten(final String src, final DFSOutputStream out) {
559 synchronized(filesBeingWritten) {
560 filesBeingWritten.put(src, out);
561 // update the last lease renewal time only when there was no
562 // writes. once there is one write stream open, the lease renewer
563 // thread keeps it updated well with in anyone's expiration time.
564 if (lastLeaseRenewal == 0) {
565 updateLastLeaseRenewal();
566 }
567 }
568 }
569
570 /** Remove a file. Only called from LeaseRenewer. */
571 void removeFileBeingWritten(final String src) {
572 synchronized(filesBeingWritten) {
573 filesBeingWritten.remove(src);
574 if (filesBeingWritten.isEmpty()) {
575 lastLeaseRenewal = 0;
576 }
577 }
578 }
579
580 /** Is file-being-written map empty? */
581 boolean isFilesBeingWrittenEmpty() {
582 synchronized(filesBeingWritten) {
583 return filesBeingWritten.isEmpty();
584 }
585 }
586
587 /** @return true if the client is running */
588 boolean isClientRunning() {
589 return clientRunning;
590 }
591
592 long getLastLeaseRenewal() {
593 return lastLeaseRenewal;
594 }
595
596 void updateLastLeaseRenewal() {
597 synchronized(filesBeingWritten) {
598 if (filesBeingWritten.isEmpty()) {
599 return;
600 }
601 lastLeaseRenewal = Time.now();
602 }
603 }
604
605 /**
606 * Renew leases.
607 * @return true if lease was renewed. May return false if this
608 * client has been closed or has no files open.
609 **/
610 boolean renewLease() throws IOException {
611 if (clientRunning && !isFilesBeingWrittenEmpty()) {
612 try {
613 namenode.renewLease(clientName);
614 updateLastLeaseRenewal();
615 return true;
616 } catch (IOException e) {
617 // Abort if the lease has already expired.
618 final long elapsed = Time.now() - getLastLeaseRenewal();
619 if (elapsed > HdfsConstants.LEASE_SOFTLIMIT_PERIOD) {
620 LOG.warn("Failed to renew lease for " + clientName + " for "
621 + (elapsed/1000) + " seconds (>= soft-limit ="
622 + (HdfsConstants.LEASE_SOFTLIMIT_PERIOD/1000) + " seconds.) "
623 + "Closing all files being written ...", e);
624 closeAllFilesBeingWritten(true);
625 } else {
626 // Let the lease renewer handle it and retry.
627 throw e;
628 }
629 }
630 }
631 return false;
632 }
633
634 /**
635 * Close connections the Namenode.
636 */
637 void closeConnectionToNamenode() {
638 RPC.stopProxy(namenode);
639 }
640
641 /** Abort and release resources held. Ignore all errors. */
642 void abort() {
643 clientRunning = false;
644 closeAllFilesBeingWritten(true);
645 socketCache.clear();
646
647 try {
648 // remove reference to this client and stop the renewer,
649 // if there is no more clients under the renewer.
650 getLeaseRenewer().closeClient(this);
651 } catch (IOException ioe) {
652 LOG.info("Exception occurred while aborting the client. " + ioe);
653 }
654 closeConnectionToNamenode();
655 }
656
657 /** Close/abort all files being written. */
658 private void closeAllFilesBeingWritten(final boolean abort) {
659 for(;;) {
660 final String src;
661 final DFSOutputStream out;
662 synchronized(filesBeingWritten) {
663 if (filesBeingWritten.isEmpty()) {
664 return;
665 }
666 src = filesBeingWritten.keySet().iterator().next();
667 out = filesBeingWritten.remove(src);
668 }
669 if (out != null) {
670 try {
671 if (abort) {
672 out.abort();
673 } else {
674 out.close();
675 }
676 } catch(IOException ie) {
677 LOG.error("Failed to " + (abort? "abort": "close") + " file " + src,
678 ie);
679 }
680 }
681 }
682 }
683
684 /**
685 * Close the file system, abandoning all of the leases and files being
686 * created and close connections to the namenode.
687 */
688 @Override
689 public synchronized void close() throws IOException {
690 if(clientRunning) {
691 closeAllFilesBeingWritten(false);
692 socketCache.clear();
693 clientRunning = false;
694 getLeaseRenewer().closeClient(this);
695 // close connections to the namenode
696 closeConnectionToNamenode();
697 }
698 }
699
700 /**
701 * Get the default block size for this cluster
702 * @return the default block size in bytes
703 */
704 public long getDefaultBlockSize() {
705 return dfsClientConf.defaultBlockSize;
706 }
707
708 /**
709 * @see ClientProtocol#getPreferredBlockSize(String)
710 */
711 public long getBlockSize(String f) throws IOException {
712 try {
713 return namenode.getPreferredBlockSize(f);
714 } catch (IOException ie) {
715 LOG.warn("Problem getting block size", ie);
716 throw ie;
717 }
718 }
719
720 /**
721 * Get server default values for a number of configuration params.
722 * @see ClientProtocol#getServerDefaults()
723 */
724 public FsServerDefaults getServerDefaults() throws IOException {
725 long now = Time.now();
726 if (now - serverDefaultsLastUpdate > SERVER_DEFAULTS_VALIDITY_PERIOD) {
727 serverDefaults = namenode.getServerDefaults();
728 serverDefaultsLastUpdate = now;
729 }
730 return serverDefaults;
731 }
732
733 /**
734 * Get a canonical token service name for this client's tokens. Null should
735 * be returned if the client is not using tokens.
736 * @return the token service for the client
737 */
738 @InterfaceAudience.LimitedPrivate( { "HDFS" })
739 public String getCanonicalServiceName() {
740 return (dtService != null) ? dtService.toString() : null;
741 }
742
743 /**
744 * @see ClientProtocol#getDelegationToken(Text)
745 */
746 public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
747 throws IOException {
748 assert dtService != null;
749 Token<DelegationTokenIdentifier> token =
750 namenode.getDelegationToken(renewer);
751 token.setService(this.dtService);
752
753 LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
754 return token;
755 }
756
757 /**
758 * Renew a delegation token
759 * @param token the token to renew
760 * @return the new expiration time
761 * @throws InvalidToken
762 * @throws IOException
763 * @deprecated Use Token.renew instead.
764 */
765 public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
766 throws InvalidToken, IOException {
767 LOG.info("Renewing " + DelegationTokenIdentifier.stringifyToken(token));
768 try {
769 return token.renew(conf);
770 } catch (InterruptedException ie) {
771 throw new RuntimeException("caught interrupted", ie);
772 } catch (RemoteException re) {
773 throw re.unwrapRemoteException(InvalidToken.class,
774 AccessControlException.class);
775 }
776 }
777
778 /**
779 * Get {@link BlockReader} for short circuited local reads.
780 */
781 static BlockReader getLocalBlockReader(Configuration conf,
782 String src, ExtendedBlock blk, Token<BlockTokenIdentifier> accessToken,
783 DatanodeInfo chosenNode, int socketTimeout, long offsetIntoBlock,
784 boolean connectToDnViaHostname) throws InvalidToken, IOException {
785 try {
786 return BlockReaderLocal.newBlockReader(conf, src, blk, accessToken,
787 chosenNode, socketTimeout, offsetIntoBlock, blk.getNumBytes()
788 - offsetIntoBlock, connectToDnViaHostname);
789 } catch (RemoteException re) {
790 throw re.unwrapRemoteException(InvalidToken.class,
791 AccessControlException.class);
792 }
793 }
794
795 private static Map<String, Boolean> localAddrMap = Collections
796 .synchronizedMap(new HashMap<String, Boolean>());
797
798 private static boolean isLocalAddress(InetSocketAddress targetAddr) {
799 InetAddress addr = targetAddr.getAddress();
800 Boolean cached = localAddrMap.get(addr.getHostAddress());
801 if (cached != null) {
802 if (LOG.isTraceEnabled()) {
803 LOG.trace("Address " + targetAddr +
804 (cached ? " is local" : " is not local"));
805 }
806 return cached;
807 }
808
809 // Check if the address is any local or loop back
810 boolean local = addr.isAnyLocalAddress() || addr.isLoopbackAddress();
811
812 // Check if the address is defined on any interface
813 if (!local) {
814 try {
815 local = NetworkInterface.getByInetAddress(addr) != null;
816 } catch (SocketException e) {
817 local = false;
818 }
819 }
820 if (LOG.isTraceEnabled()) {
821 LOG.trace("Address " + targetAddr +
822 (local ? " is local" : " is not local"));
823 }
824 localAddrMap.put(addr.getHostAddress(), local);
825 return local;
826 }
827
828 /**
829 * Should the block access token be refetched on an exception
830 *
831 * @param ex Exception received
832 * @param targetAddr Target datanode address from where exception was received
833 * @return true if block access token has expired or invalid and it should be
834 * refetched
835 */
836 private static boolean tokenRefetchNeeded(IOException ex,
837 InetSocketAddress targetAddr) {
838 /*
839 * Get a new access token and retry. Retry is needed in 2 cases. 1) When
840 * both NN and DN re-started while DFSClient holding a cached access token.
841 * 2) In the case that NN fails to update its access key at pre-set interval
842 * (by a wide margin) and subsequently restarts. In this case, DN
843 * re-registers itself with NN and receives a new access key, but DN will
844 * delete the old access key from its memory since it's considered expired
845 * based on the estimated expiration date.
846 */
847 if (ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) {
848 LOG.info("Access token was invalid when connecting to " + targetAddr
849 + " : " + ex);
850 return true;
851 }
852 return false;
853 }
854
855 /**
856 * Cancel a delegation token
857 * @param token the token to cancel
858 * @throws InvalidToken
859 * @throws IOException
860 * @deprecated Use Token.cancel instead.
861 */
862 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
863 throws InvalidToken, IOException {
864 LOG.info("Cancelling " + DelegationTokenIdentifier.stringifyToken(token));
865 try {
866 token.cancel(conf);
867 } catch (InterruptedException ie) {
868 throw new RuntimeException("caught interrupted", ie);
869 } catch (RemoteException re) {
870 throw re.unwrapRemoteException(InvalidToken.class,
871 AccessControlException.class);
872 }
873 }
874
875 @InterfaceAudience.Private
876 public static class Renewer extends TokenRenewer {
877
878 static {
879 //Ensure that HDFS Configuration files are loaded before trying to use
880 // the renewer.
881 HdfsConfiguration.init();
882 }
883
884 @Override
885 public boolean handleKind(Text kind) {
886 return DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind);
887 }
888
889 @SuppressWarnings("unchecked")
890 @Override
891 public long renew(Token<?> token, Configuration conf) throws IOException {
892 Token<DelegationTokenIdentifier> delToken =
893 (Token<DelegationTokenIdentifier>) token;
894 ClientProtocol nn = getNNProxy(delToken, conf);
895 try {
896 return nn.renewDelegationToken(delToken);
897 } catch (RemoteException re) {
898 throw re.unwrapRemoteException(InvalidToken.class,
899 AccessControlException.class);
900 }
901 }
902
903 @SuppressWarnings("unchecked")
904 @Override
905 public void cancel(Token<?> token, Configuration conf) throws IOException {
906 Token<DelegationTokenIdentifier> delToken =
907 (Token<DelegationTokenIdentifier>) token;
908 LOG.info("Cancelling " +
909 DelegationTokenIdentifier.stringifyToken(delToken));
910 ClientProtocol nn = getNNProxy(delToken, conf);
911 try {
912 nn.cancelDelegationToken(delToken);
913 } catch (RemoteException re) {
914 throw re.unwrapRemoteException(InvalidToken.class,
915 AccessControlException.class);
916 }
917 }
918
919 private static ClientProtocol getNNProxy(
920 Token<DelegationTokenIdentifier> token, Configuration conf)
921 throws IOException {
922 URI uri = HAUtil.getServiceUriFromToken(token);
923 if (HAUtil.isTokenForLogicalUri(token) &&
924 !HAUtil.isLogicalUri(conf, uri)) {
925 // If the token is for a logical nameservice, but the configuration
926 // we have disagrees about that, we can't actually renew it.
927 // This can be the case in MR, for example, if the RM doesn't
928 // have all of the HA clusters configured in its configuration.
929 throw new IOException("Unable to map logical nameservice URI '" +
930 uri + "' to a NameNode. Local configuration does not have " +
931 "a failover proxy provider configured.");
932 }
933
934 NameNodeProxies.ProxyAndInfo<ClientProtocol> info =
935 NameNodeProxies.createProxy(conf, uri, ClientProtocol.class);
936 assert info.getDelegationTokenService().equals(token.getService()) :
937 "Returned service '" + info.getDelegationTokenService().toString() +
938 "' doesn't match expected service '" +
939 token.getService().toString() + "'";
940
941 return info.getProxy();
942 }
943
944 @Override
945 public boolean isManaged(Token<?> token) throws IOException {
946 return true;
947 }
948
949 }
950
951 /**
952 * Report corrupt blocks that were discovered by the client.
953 * @see ClientProtocol#reportBadBlocks(LocatedBlock[])
954 */
955 public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
956 namenode.reportBadBlocks(blocks);
957 }
958
959 public short getDefaultReplication() {
960 return dfsClientConf.defaultReplication;
961 }
962
963 /*
964 * This is just a wrapper around callGetBlockLocations, but non-static so that
965 * we can stub it out for tests.
966 */
967 @VisibleForTesting
968 public LocatedBlocks getLocatedBlocks(String src, long start, long length)
969 throws IOException {
970 return callGetBlockLocations(namenode, src, start, length);
971 }
972
973 /**
974 * @see ClientProtocol#getBlockLocations(String, long, long)
975 */
976 static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
977 String src, long start, long length)
978 throws IOException {
979 try {
980 return namenode.getBlockLocations(src, start, length);
981 } catch(RemoteException re) {
982 throw re.unwrapRemoteException(AccessControlException.class,
983 FileNotFoundException.class,
984 UnresolvedPathException.class);
985 }
986 }
987
988 /**
989 * Recover a file's lease
990 * @param src a file's path
991 * @return true if the file is already closed
992 * @throws IOException
993 */
994 boolean recoverLease(String src) throws IOException {
995 checkOpen();
996
997 try {
998 return namenode.recoverLease(src, clientName);
999 } catch (RemoteException re) {
1000 throw re.unwrapRemoteException(FileNotFoundException.class,
1001 AccessControlException.class);
1002 }
1003 }
1004
1005 /**
1006 * Get block location info about file
1007 *
1008 * getBlockLocations() returns a list of hostnames that store
1009 * data for a specific file region. It returns a set of hostnames
1010 * for every block within the indicated region.
1011 *
1012 * This function is very useful when writing code that considers
1013 * data-placement when performing operations. For example, the
1014 * MapReduce system tries to schedule tasks on the same machines
1015 * as the data-block the task processes.
1016 */
1017 public BlockLocation[] getBlockLocations(String src, long start,
1018 long length) throws IOException, UnresolvedLinkException {
1019 LocatedBlocks blocks = getLocatedBlocks(src, start, length);
1020 BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks);
1021 HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
1022 for (int i = 0; i < locations.length; i++) {
1023 hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
1024 }
1025 return hdfsLocations;
1026 }
1027
1028 /**
1029 * Get block location information about a list of {@link HdfsBlockLocation}.
1030 * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to
1031 * get {@link BlockStorageLocation}s for blocks returned by
1032 * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)}
1033 * .
1034 *
1035 * This is done by making a round of RPCs to the associated datanodes, asking
1036 * the volume of each block replica. The returned array of
1037 * {@link BlockStorageLocation} expose this information as a
1038 * {@link VolumeId}.
1039 *
1040 * @param blockLocations
1041 * target blocks on which to query volume location information
1042 * @return volumeBlockLocations original block array augmented with additional
1043 * volume location information for each replica.
1044 */
1045 public BlockStorageLocation[] getBlockStorageLocations(
1046 List<BlockLocation> blockLocations) throws IOException,
1047 UnsupportedOperationException, InvalidBlockTokenException {
1048 if (!getConf().getHdfsBlocksMetadataEnabled) {
1049 throw new UnsupportedOperationException("Datanode-side support for " +
1050 "getVolumeBlockLocations() must also be enabled in the client " +
1051 "configuration.");
1052 }
1053 // Downcast blockLocations and fetch out required LocatedBlock(s)
1054 List<LocatedBlock> blocks = new ArrayList<LocatedBlock>();
1055 for (BlockLocation loc : blockLocations) {
1056 if (!(loc instanceof HdfsBlockLocation)) {
1057 throw new ClassCastException("DFSClient#getVolumeBlockLocations " +
1058 "expected to be passed HdfsBlockLocations");
1059 }
1060 HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc;
1061 blocks.add(hdfsLoc.getLocatedBlock());
1062 }
1063
1064 // Re-group the LocatedBlocks to be grouped by datanodes, with the values
1065 // a list of the LocatedBlocks on the datanode.
1066 Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks =
1067 new LinkedHashMap<DatanodeInfo, List<LocatedBlock>>();
1068 for (LocatedBlock b : blocks) {
1069 for (DatanodeInfo info : b.getLocations()) {
1070 if (!datanodeBlocks.containsKey(info)) {
1071 datanodeBlocks.put(info, new ArrayList<LocatedBlock>());
1072 }
1073 List<LocatedBlock> l = datanodeBlocks.get(info);
1074 l.add(b);
1075 }
1076 }
1077
1078 // Make RPCs to the datanodes to get volume locations for its replicas
1079 List<HdfsBlocksMetadata> metadatas = BlockStorageLocationUtil
1080 .queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
1081 getConf().getFileBlockStorageLocationsNumThreads,
1082 getConf().getFileBlockStorageLocationsTimeout,
1083 getConf().connectToDnViaHostname);
1084
1085 // Regroup the returned VolumeId metadata to again be grouped by
1086 // LocatedBlock rather than by datanode
1087 Map<LocatedBlock, List<VolumeId>> blockVolumeIds = BlockStorageLocationUtil
1088 .associateVolumeIdsWithBlocks(blocks, datanodeBlocks, metadatas);
1089
1090 // Combine original BlockLocations with new VolumeId information
1091 BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtil
1092 .convertToVolumeBlockLocations(blocks, blockVolumeIds);
1093
1094 return volumeBlockLocations;
1095 }
1096
1097 public DFSInputStream open(String src)
1098 throws IOException, UnresolvedLinkException {
1099 return open(src, dfsClientConf.ioBufferSize, true, null);
1100 }
1101
1102 /**
1103 * Create an input stream that obtains a nodelist from the
1104 * namenode, and then reads from all the right places. Creates
1105 * inner subclass of InputStream that does the right out-of-band
1106 * work.
1107 * @deprecated Use {@link #open(String, int, boolean)} instead.
1108 */
1109 @Deprecated
1110 public DFSInputStream open(String src, int buffersize, boolean verifyChecksum,
1111 FileSystem.Statistics stats)
1112 throws IOException, UnresolvedLinkException {
1113 return open(src, buffersize, verifyChecksum);
1114 }
1115
1116
1117 /**
1118 * Create an input stream that obtains a nodelist from the
1119 * namenode, and then reads from all the right places. Creates
1120 * inner subclass of InputStream that does the right out-of-band
1121 * work.
1122 */
1123 public DFSInputStream open(String src, int buffersize, boolean verifyChecksum)
1124 throws IOException, UnresolvedLinkException {
1125 checkOpen();
1126 // Get block info from namenode
1127 return new DFSInputStream(this, src, buffersize, verifyChecksum);
1128 }
1129
1130 /**
1131 * Get the namenode associated with this DFSClient object
1132 * @return the namenode associated with this DFSClient object
1133 */
1134 public ClientProtocol getNamenode() {
1135 return namenode;
1136 }
1137
1138 /**
1139 * Call {@link #create(String, boolean, short, long, Progressable)} with
1140 * default <code>replication</code> and <code>blockSize<code> and null <code>
1141 * progress</code>.
1142 */
1143 public OutputStream create(String src, boolean overwrite)
1144 throws IOException {
1145 return create(src, overwrite, dfsClientConf.defaultReplication,
1146 dfsClientConf.defaultBlockSize, null);
1147 }
1148
1149 /**
1150 * Call {@link #create(String, boolean, short, long, Progressable)} with
1151 * default <code>replication</code> and <code>blockSize<code>.
1152 */
1153 public OutputStream create(String src,
1154 boolean overwrite,
1155 Progressable progress) throws IOException {
1156 return create(src, overwrite, dfsClientConf.defaultReplication,
1157 dfsClientConf.defaultBlockSize, progress);
1158 }
1159
1160 /**
1161 * Call {@link #create(String, boolean, short, long, Progressable)} with
1162 * null <code>progress</code>.
1163 */
1164 public OutputStream create(String src,
1165 boolean overwrite,
1166 short replication,
1167 long blockSize) throws IOException {
1168 return create(src, overwrite, replication, blockSize, null);
1169 }
1170
1171 /**
1172 * Call {@link #create(String, boolean, short, long, Progressable, int)}
1173 * with default bufferSize.
1174 */
1175 public OutputStream create(String src, boolean overwrite, short replication,
1176 long blockSize, Progressable progress) throws IOException {
1177 return create(src, overwrite, replication, blockSize, progress,
1178 dfsClientConf.ioBufferSize);
1179 }
1180
1181 /**
1182 * Call {@link #create(String, FsPermission, EnumSet, short, long,
1183 * Progressable, int)} with default <code>permission</code>
1184 * {@link FsPermission#getDefault()}.
1185 *
1186 * @param src File name
1187 * @param overwrite overwrite an existing file if true
1188 * @param replication replication factor for the file
1189 * @param blockSize maximum block size
1190 * @param progress interface for reporting client progress
1191 * @param buffersize underlying buffersize
1192 *
1193 * @return output stream
1194 */
1195 public OutputStream create(String src,
1196 boolean overwrite,
1197 short replication,
1198 long blockSize,
1199 Progressable progress,
1200 int buffersize)
1201 throws IOException {
1202 return create(src, FsPermission.getDefault(),
1203 overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1204 : EnumSet.of(CreateFlag.CREATE), replication, blockSize, progress,
1205 buffersize, null);
1206 }
1207
1208 /**
1209 * Call {@link #create(String, FsPermission, EnumSet, boolean, short,
1210 * long, Progressable, int, ChecksumOpt)} with <code>createParent</code>
1211 * set to true.
1212 */
1213 public DFSOutputStream create(String src,
1214 FsPermission permission,
1215 EnumSet<CreateFlag> flag,
1216 short replication,
1217 long blockSize,
1218 Progressable progress,
1219 int buffersize,
1220 ChecksumOpt checksumOpt)
1221 throws IOException {
1222 return create(src, permission, flag, true,
1223 replication, blockSize, progress, buffersize, checksumOpt);
1224 }
1225
1226 /**
1227 * Create a new dfs file with the specified block replication
1228 * with write-progress reporting and return an output stream for writing
1229 * into the file.
1230 *
1231 * @param src File name
1232 * @param permission The permission of the directory being created.
1233 * If null, use default permission {@link FsPermission#getDefault()}
1234 * @param flag indicates create a new file or create/overwrite an
1235 * existing file or append to an existing file
1236 * @param createParent create missing parent directory if true
1237 * @param replication block replication
1238 * @param blockSize maximum block size
1239 * @param progress interface for reporting client progress
1240 * @param buffersize underlying buffer size
1241 * @param checksumOpt checksum options
1242 *
1243 * @return output stream
1244 *
1245 * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
1246 * boolean, short, long) for detailed description of exceptions thrown
1247 */
1248 public DFSOutputStream create(String src,
1249 FsPermission permission,
1250 EnumSet<CreateFlag> flag,
1251 boolean createParent,
1252 short replication,
1253 long blockSize,
1254 Progressable progress,
1255 int buffersize,
1256 ChecksumOpt checksumOpt) throws IOException {
1257 checkOpen();
1258 if (permission == null) {
1259 permission = FsPermission.getDefault();
1260 }
1261 FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
1262 if(LOG.isDebugEnabled()) {
1263 LOG.debug(src + ": masked=" + masked);
1264 }
1265 final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
1266 src, masked, flag, createParent, replication, blockSize, progress,
1267 buffersize, dfsClientConf.createChecksum(checksumOpt));
1268 beginFileLease(src, result);
1269 return result;
1270 }
1271
1272 /**
1273 * Append to an existing file if {@link CreateFlag#APPEND} is present
1274 */
1275 private DFSOutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
1276 int buffersize, Progressable progress) throws IOException {
1277 if (flag.contains(CreateFlag.APPEND)) {
1278 HdfsFileStatus stat = getFileInfo(src);
1279 if (stat == null) { // No file to append to
1280 // New file needs to be created if create option is present
1281 if (!flag.contains(CreateFlag.CREATE)) {
1282 throw new FileNotFoundException("failed to append to non-existent file "
1283 + src + " on client " + clientName);
1284 }
1285 return null;
1286 }
1287 return callAppend(stat, src, buffersize, progress);
1288 }
1289 return null;
1290 }
1291
1292 /**
1293 * Same as {{@link #create(String, FsPermission, EnumSet, short, long,
1294 * Progressable, int)} except that the permission
1295 * is absolute (ie has already been masked with umask.
1296 */
1297 public DFSOutputStream primitiveCreate(String src,
1298 FsPermission absPermission,
1299 EnumSet<CreateFlag> flag,
1300 boolean createParent,
1301 short replication,
1302 long blockSize,
1303 Progressable progress,
1304 int buffersize,
1305 ChecksumOpt checksumOpt)
1306 throws IOException, UnresolvedLinkException {
1307 checkOpen();
1308 CreateFlag.validate(flag);
1309 DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
1310 if (result == null) {
1311 DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
1312 result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
1313 flag, createParent, replication, blockSize, progress, buffersize,
1314 checksum);
1315 }
1316 beginFileLease(src, result);
1317 return result;
1318 }
1319
1320 /**
1321 * Creates a symbolic link.
1322 *
1323 * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean)
1324 */
1325 public void createSymlink(String target, String link, boolean createParent)
1326 throws IOException {
1327 try {
1328 FsPermission dirPerm =
1329 FsPermission.getDefault().applyUMask(dfsClientConf.uMask);
1330 namenode.createSymlink(target, link, dirPerm, createParent);
1331 } catch (RemoteException re) {
1332 throw re.unwrapRemoteException(AccessControlException.class,
1333 FileAlreadyExistsException.class,
1334 FileNotFoundException.class,
1335 ParentNotDirectoryException.class,
1336 NSQuotaExceededException.class,
1337 DSQuotaExceededException.class,
1338 UnresolvedPathException.class);
1339 }
1340 }
1341
1342 /**
1343 * Resolve the *first* symlink, if any, in the path.
1344 *
1345 * @see ClientProtocol#getLinkTarget(String)
1346 */
1347 public String getLinkTarget(String path) throws IOException {
1348 checkOpen();
1349 try {
1350 return namenode.getLinkTarget(path);
1351 } catch (RemoteException re) {
1352 throw re.unwrapRemoteException(AccessControlException.class,
1353 FileNotFoundException.class);
1354 }
1355 }
1356
1357 /** Method to get stream returned by append call */
1358 private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
1359 int buffersize, Progressable progress) throws IOException {
1360 LocatedBlock lastBlock = null;
1361 try {
1362 lastBlock = namenode.append(src, clientName);
1363 } catch(RemoteException re) {
1364 throw re.unwrapRemoteException(AccessControlException.class,
1365 FileNotFoundException.class,
1366 SafeModeException.class,
1367 DSQuotaExceededException.class,
1368 UnsupportedOperationException.class,
1369 UnresolvedPathException.class);
1370 }
1371 return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
1372 lastBlock, stat, dfsClientConf.createChecksum());
1373 }
1374
1375 /**
1376 * Append to an existing HDFS file.
1377 *
1378 * @param src file name
1379 * @param buffersize buffer size
1380 * @param progress for reporting write-progress; null is acceptable.
1381 * @param statistics file system statistics; null is acceptable.
1382 * @return an output stream for writing into the file
1383 *
1384 * @see ClientProtocol#append(String, String)
1385 */
1386 public HdfsDataOutputStream append(final String src, final int buffersize,
1387 final Progressable progress, final FileSystem.Statistics statistics
1388 ) throws IOException {
1389 final DFSOutputStream out = append(src, buffersize, progress);
1390 return new HdfsDataOutputStream(out, statistics, out.getInitialLen());
1391 }
1392
1393 private DFSOutputStream append(String src, int buffersize, Progressable progress)
1394 throws IOException {
1395 checkOpen();
1396 HdfsFileStatus stat = getFileInfo(src);
1397 if (stat == null) { // No file found
1398 throw new FileNotFoundException("failed to append to non-existent file "
1399 + src + " on client " + clientName);
1400 }
1401 final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
1402 beginFileLease(src, result);
1403 return result;
1404 }
1405
1406 /**
1407 * Set replication for an existing file.
1408 * @param src file name
1409 * @param replication
1410 *
1411 * @see ClientProtocol#setReplication(String, short)
1412 */
1413 public boolean setReplication(String src, short replication)
1414 throws IOException {
1415 try {
1416 return namenode.setReplication(src, replication);
1417 } catch(RemoteException re) {
1418 throw re.unwrapRemoteException(AccessControlException.class,
1419 FileNotFoundException.class,
1420 SafeModeException.class,
1421 DSQuotaExceededException.class,
1422 UnresolvedPathException.class);
1423 }
1424 }
1425
1426 /**
1427 * Rename file or directory.
1428 * @see ClientProtocol#rename(String, String)
1429 * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
1430 */
1431 @Deprecated
1432 public boolean rename(String src, String dst) throws IOException {
1433 checkOpen();
1434 try {
1435 return namenode.rename(src, dst);
1436 } catch(RemoteException re) {
1437 throw re.unwrapRemoteException(AccessControlException.class,
1438 NSQuotaExceededException.class,
1439 DSQuotaExceededException.class,
1440 UnresolvedPathException.class);
1441 }
1442 }
1443
1444 /**
1445 * Move blocks from src to trg and delete src
1446 * See {@link ClientProtocol#concat(String, String [])}.
1447 */
1448 public void concat(String trg, String [] srcs) throws IOException {
1449 checkOpen();
1450 try {
1451 namenode.concat(trg, srcs);
1452 } catch(RemoteException re) {
1453 throw re.unwrapRemoteException(AccessControlException.class,
1454 UnresolvedPathException.class);
1455 }
1456 }
1457 /**
1458 * Rename file or directory.
1459 * @see ClientProtocol#rename2(String, String, Options.Rename...)
1460 */
1461 public void rename(String src, String dst, Options.Rename... options)
1462 throws IOException {
1463 checkOpen();
1464 try {
1465 namenode.rename2(src, dst, options);
1466 } catch(RemoteException re) {
1467 throw re.unwrapRemoteException(AccessControlException.class,
1468 DSQuotaExceededException.class,
1469 FileAlreadyExistsException.class,
1470 FileNotFoundException.class,
1471 ParentNotDirectoryException.class,
1472 SafeModeException.class,
1473 NSQuotaExceededException.class,
1474 UnresolvedPathException.class);
1475 }
1476 }
1477 /**
1478 * Delete file or directory.
1479 * See {@link ClientProtocol#delete(String)}.
1480 */
1481 @Deprecated
1482 public boolean delete(String src) throws IOException {
1483 checkOpen();
1484 return namenode.delete(src, true);
1485 }
1486
1487 /**
1488 * delete file or directory.
1489 * delete contents of the directory if non empty and recursive
1490 * set to true
1491 *
1492 * @see ClientProtocol#delete(String, boolean)
1493 */
1494 public boolean delete(String src, boolean recursive) throws IOException {
1495 checkOpen();
1496 try {
1497 return namenode.delete(src, recursive);
1498 } catch(RemoteException re) {
1499 throw re.unwrapRemoteException(AccessControlException.class,
1500 FileNotFoundException.class,
1501 SafeModeException.class,
1502 UnresolvedPathException.class);
1503 }
1504 }
1505
1506 /** Implemented using getFileInfo(src)
1507 */
1508 public boolean exists(String src) throws IOException {
1509 checkOpen();
1510 return getFileInfo(src) != null;
1511 }
1512
1513 /**
1514 * Get a partial listing of the indicated directory
1515 * No block locations need to be fetched
1516 */
1517 public DirectoryListing listPaths(String src, byte[] startAfter)
1518 throws IOException {
1519 return listPaths(src, startAfter, false);
1520 }
1521
1522 /**
1523 * Get a partial listing of the indicated directory
1524 *
1525 * Recommend to use HdfsFileStatus.EMPTY_NAME as startAfter
1526 * if the application wants to fetch a listing starting from
1527 * the first entry in the directory
1528 *
1529 * @see ClientProtocol#getListing(String, byte[], boolean)
1530 */
1531 public DirectoryListing listPaths(String src, byte[] startAfter,
1532 boolean needLocation)
1533 throws IOException {
1534 checkOpen();
1535 try {
1536 return namenode.getListing(src, startAfter, needLocation);
1537 } catch(RemoteException re) {
1538 throw re.unwrapRemoteException(AccessControlException.class,
1539 FileNotFoundException.class,
1540 UnresolvedPathException.class);
1541 }
1542 }
1543
1544 /**
1545 * Get the file info for a specific file or directory.
1546 * @param src The string representation of the path to the file
1547 * @return object containing information regarding the file
1548 * or null if file not found
1549 *
1550 * @see ClientProtocol#getFileInfo(String) for description of exceptions
1551 */
1552 public HdfsFileStatus getFileInfo(String src) throws IOException {
1553 checkOpen();
1554 try {
1555 return namenode.getFileInfo(src);
1556 } catch(RemoteException re) {
1557 throw re.unwrapRemoteException(AccessControlException.class,
1558 FileNotFoundException.class,
1559 UnresolvedPathException.class);
1560 }
1561 }
1562
1563 /**
1564 * Get the file info for a specific file or directory. If src
1565 * refers to a symlink then the FileStatus of the link is returned.
1566 * @param src path to a file or directory.
1567 *
1568 * For description of exceptions thrown
1569 * @see ClientProtocol#getFileLinkInfo(String)
1570 */
1571 public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
1572 checkOpen();
1573 try {
1574 return namenode.getFileLinkInfo(src);
1575 } catch(RemoteException re) {
1576 throw re.unwrapRemoteException(AccessControlException.class,
1577 UnresolvedPathException.class);
1578 }
1579 }
1580
1581 /**
1582 * Get the checksum of a file.
1583 * @param src The file path
1584 * @return The checksum
1585 * @see DistributedFileSystem#getFileChecksum(Path)
1586 */
1587 public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
1588 checkOpen();
1589 return getFileChecksum(src, namenode, socketFactory,
1590 dfsClientConf.socketTimeout, getDataEncryptionKey(),
1591 dfsClientConf.connectToDnViaHostname);
1592 }
1593
1594 @InterfaceAudience.Private
1595 public void clearDataEncryptionKey() {
1596 LOG.debug("Clearing encryption key");
1597 synchronized (this) {
1598 encryptionKey = null;
1599 }
1600 }
1601
1602 /**
1603 * @return true if data sent between this client and DNs should be encrypted,
1604 * false otherwise.
1605 * @throws IOException in the event of error communicating with the NN
1606 */
1607 boolean shouldEncryptData() throws IOException {
1608 FsServerDefaults d = getServerDefaults();
1609 return d == null ? false : d.getEncryptDataTransfer();
1610 }
1611
1612 @InterfaceAudience.Private
1613 public DataEncryptionKey getDataEncryptionKey()
1614 throws IOException {
1615 if (shouldEncryptData()) {
1616 synchronized (this) {
1617 if (encryptionKey == null ||
1618 (encryptionKey != null &&
1619 encryptionKey.expiryDate < Time.now())) {
1620 LOG.debug("Getting new encryption token from NN");
1621 encryptionKey = namenode.getDataEncryptionKey();
1622 }
1623 return encryptionKey;
1624 }
1625 } else {
1626 return null;
1627 }
1628 }
1629
1630 /**
1631 * Get the checksum of a file.
1632 * @param src The file path
1633 * @return The checksum
1634 */
1635 public static MD5MD5CRC32FileChecksum getFileChecksum(String src,
1636 ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
1637 DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
1638 throws IOException {
1639 //get all block locations
1640 LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1641 if (null == blockLocations) {
1642 throw new FileNotFoundException("File does not exist: " + src);
1643 }
1644 List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
1645 final DataOutputBuffer md5out = new DataOutputBuffer();
1646 int bytesPerCRC = -1;
1647 DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
1648 long crcPerBlock = 0;
1649 boolean refetchBlocks = false;
1650 int lastRetriedIndex = -1;
1651
1652 //get block checksum for each block
1653 for(int i = 0; i < locatedblocks.size(); i++) {
1654 if (refetchBlocks) { // refetch to get fresh tokens
1655 blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1656 if (null == blockLocations) {
1657 throw new FileNotFoundException("File does not exist: " + src);
1658 }
1659 locatedblocks = blockLocations.getLocatedBlocks();
1660 refetchBlocks = false;
1661 }
1662 LocatedBlock lb = locatedblocks.get(i);
1663 final ExtendedBlock block = lb.getBlock();
1664 final DatanodeInfo[] datanodes = lb.getLocations();
1665
1666 //try each datanode location of the block
1667 final int timeout = 3000 * datanodes.length + socketTimeout;
1668 boolean done = false;
1669 for(int j = 0; !done && j < datanodes.length; j++) {
1670 Socket sock = null;
1671 DataOutputStream out = null;
1672 DataInputStream in = null;
1673
1674 try {
1675 //connect to a datanode
1676 sock = socketFactory.createSocket();
1677 String dnAddr = datanodes[j].getXferAddr(connectToDnViaHostname);
1678 if (LOG.isDebugEnabled()) {
1679 LOG.debug("Connecting to datanode " + dnAddr);
1680 }
1681 NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
1682 sock.setSoTimeout(timeout);
1683
1684 OutputStream unbufOut = NetUtils.getOutputStream(sock);
1685 InputStream unbufIn = NetUtils.getInputStream(sock);
1686 if (encryptionKey != null) {
1687 IOStreamPair encryptedStreams =
1688 DataTransferEncryptor.getEncryptedStreams(
1689 unbufOut, unbufIn, encryptionKey);
1690 unbufOut = encryptedStreams.out;
1691 unbufIn = encryptedStreams.in;
1692 }
1693 out = new DataOutputStream(new BufferedOutputStream(unbufOut,
1694 HdfsConstants.SMALL_BUFFER_SIZE));
1695 in = new DataInputStream(unbufIn);
1696
1697 if (LOG.isDebugEnabled()) {
1698 LOG.debug("write to " + datanodes[j] + ": "
1699 + Op.BLOCK_CHECKSUM + ", block=" + block);
1700 }
1701 // get block MD5
1702 new Sender(out).blockChecksum(block, lb.getBlockToken());
1703
1704 final BlockOpResponseProto reply =
1705 BlockOpResponseProto.parseFrom(HdfsProtoUtil.vintPrefixed(in));
1706
1707 if (reply.getStatus() != Status.SUCCESS) {
1708 if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN
1709 && i > lastRetriedIndex) {
1710 if (LOG.isDebugEnabled()) {
1711 LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
1712 + "for file " + src + " for block " + block
1713 + " from datanode " + datanodes[j]
1714 + ". Will retry the block once.");
1715 }
1716 lastRetriedIndex = i;
1717 done = true; // actually it's not done; but we'll retry
1718 i--; // repeat at i-th block
1719 refetchBlocks = true;
1720 break;
1721 } else {
1722 throw new IOException("Bad response " + reply + " for block "
1723 + block + " from datanode " + datanodes[j]);
1724 }
1725 }
1726
1727 OpBlockChecksumResponseProto checksumData =
1728 reply.getChecksumResponse();
1729
1730 //read byte-per-checksum
1731 final int bpc = checksumData.getBytesPerCrc();
1732 if (i == 0) { //first block
1733 bytesPerCRC = bpc;
1734 }
1735 else if (bpc != bytesPerCRC) {
1736 throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
1737 + " but bytesPerCRC=" + bytesPerCRC);
1738 }
1739
1740 //read crc-per-block
1741 final long cpb = checksumData.getCrcPerBlock();
1742 if (locatedblocks.size() > 1 && i == 0) {
1743 crcPerBlock = cpb;
1744 }
1745
1746 //read md5
1747 final MD5Hash md5 = new MD5Hash(
1748 checksumData.getMd5().toByteArray());
1749 md5.write(md5out);
1750
1751 // read crc-type
1752 final DataChecksum.Type ct = HdfsProtoUtil.
1753 fromProto(checksumData.getCrcType());
1754 if (i == 0) { // first block
1755 crcType = ct;
1756 } else if (crcType != DataChecksum.Type.MIXED
1757 && crcType != ct) {
1758 // if crc types are mixed in a file
1759 crcType = DataChecksum.Type.MIXED;
1760 }
1761
1762 done = true;
1763
1764 if (LOG.isDebugEnabled()) {
1765 if (i == 0) {
1766 LOG.debug("set bytesPerCRC=" + bytesPerCRC
1767 + ", crcPerBlock=" + crcPerBlock);
1768 }
1769 LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
1770 }
1771 } catch (IOException ie) {
1772 LOG.warn("src=" + src + ", datanodes["+j+"]=" + datanodes[j], ie);
1773 } finally {
1774 IOUtils.closeStream(in);
1775 IOUtils.closeStream(out);
1776 IOUtils.closeSocket(sock);
1777 }
1778 }
1779
1780 if (!done) {
1781 throw new IOException("Fail to get block MD5 for " + block);
1782 }
1783 }
1784
1785 //compute file MD5
1786 final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
1787 switch (crcType) {
1788 case CRC32:
1789 return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
1790 crcPerBlock, fileMD5);
1791 case CRC32C:
1792 return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC,
1793 crcPerBlock, fileMD5);
1794 default:
1795 // we should never get here since the validity was checked
1796 // when getCrcType() was called above.
1797 return null;
1798 }
1799 }
1800
1801 /**
1802 * Set permissions to a file or directory.
1803 * @param src path name.
1804 * @param permission
1805 *
1806 * @see ClientProtocol#setPermission(String, FsPermission)
1807 */
1808 public void setPermission(String src, FsPermission permission)
1809 throws IOException {
1810 checkOpen();
1811 try {
1812 namenode.setPermission(src, permission);
1813 } catch(RemoteException re) {
1814 throw re.unwrapRemoteException(AccessControlException.class,
1815 FileNotFoundException.class,
1816 SafeModeException.class,
1817 UnresolvedPathException.class);
1818 }
1819 }
1820
1821 /**
1822 * Set file or directory owner.
1823 * @param src path name.
1824 * @param username user id.
1825 * @param groupname user group.
1826 *
1827 * @see ClientProtocol#setOwner(String, String, String)
1828 */
1829 public void setOwner(String src, String username, String groupname)
1830 throws IOException {
1831 checkOpen();
1832 try {
1833 namenode.setOwner(src, username, groupname);
1834 } catch(RemoteException re) {
1835 throw re.unwrapRemoteException(AccessControlException.class,
1836 FileNotFoundException.class,
1837 SafeModeException.class,
1838 UnresolvedPathException.class);
1839 }
1840 }
1841
1842 /**
1843 * @see ClientProtocol#getStats()
1844 */
1845 public FsStatus getDiskStatus() throws IOException {
1846 long rawNums[] = namenode.getStats();
1847 return new FsStatus(rawNums[0], rawNums[1], rawNums[2]);
1848 }
1849
1850 /**
1851 * Returns count of blocks with no good replicas left. Normally should be
1852 * zero.
1853 * @throws IOException
1854 */
1855 public long getMissingBlocksCount() throws IOException {
1856 return namenode.getStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
1857 }
1858
1859 /**
1860 * Returns count of blocks with one of more replica missing.
1861 * @throws IOException
1862 */
1863 public long getUnderReplicatedBlocksCount() throws IOException {
1864 return namenode.getStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
1865 }
1866
1867 /**
1868 * Returns count of blocks with at least one replica marked corrupt.
1869 * @throws IOException
1870 */
1871 public long getCorruptBlocksCount() throws IOException {
1872 return namenode.getStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
1873 }
1874
1875 /**
1876 * @return a list in which each entry describes a corrupt file/block
1877 * @throws IOException
1878 */
1879 public CorruptFileBlocks listCorruptFileBlocks(String path,
1880 String cookie)
1881 throws IOException {
1882 return namenode.listCorruptFileBlocks(path, cookie);
1883 }
1884
1885 public DatanodeInfo[] datanodeReport(DatanodeReportType type)
1886 throws IOException {
1887 return namenode.getDatanodeReport(type);
1888 }
1889
1890 /**
1891 * Enter, leave or get safe mode.
1892 *
1893 * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction)
1894 */
1895 public boolean setSafeMode(SafeModeAction action) throws IOException {
1896 return namenode.setSafeMode(action);
1897 }
1898
1899 /**
1900 * Save namespace image.
1901 *
1902 * @see ClientProtocol#saveNamespace()
1903 */
1904 void saveNamespace() throws AccessControlException, IOException {
1905 try {
1906 namenode.saveNamespace();
1907 } catch(RemoteException re) {
1908 throw re.unwrapRemoteException(AccessControlException.class);
1909 }
1910 }
1911
1912 /**
1913 * Rolls the edit log on the active NameNode.
1914 * @return the txid of the new log segment
1915 *
1916 * @see ClientProtocol#rollEdits()
1917 */
1918 long rollEdits() throws AccessControlException, IOException {
1919 try {
1920 return namenode.rollEdits();
1921 } catch(RemoteException re) {
1922 throw re.unwrapRemoteException(AccessControlException.class);
1923 }
1924 }
1925
1926 /**
1927 * enable/disable restore failed storage.
1928 *
1929 * @see ClientProtocol#restoreFailedStorage(String arg)
1930 */
1931 boolean restoreFailedStorage(String arg)
1932 throws AccessControlException, IOException{
1933 return namenode.restoreFailedStorage(arg);
1934 }
1935
1936 /**
1937 * Refresh the hosts and exclude files. (Rereads them.)
1938 * See {@link ClientProtocol#refreshNodes()}
1939 * for more details.
1940 *
1941 * @see ClientProtocol#refreshNodes()
1942 */
1943 public void refreshNodes() throws IOException {
1944 namenode.refreshNodes();
1945 }
1946
1947 /**
1948 * Dumps DFS data structures into specified file.
1949 *
1950 * @see ClientProtocol#metaSave(String)
1951 */
1952 public void metaSave(String pathname) throws IOException {
1953 namenode.metaSave(pathname);
1954 }
1955
1956 /**
1957 * Requests the namenode to tell all datanodes to use a new, non-persistent
1958 * bandwidth value for dfs.balance.bandwidthPerSec.
1959 * See {@link ClientProtocol#setBalancerBandwidth(long)}
1960 * for more details.
1961 *
1962 * @see ClientProtocol#setBalancerBandwidth(long)
1963 */
1964 public void setBalancerBandwidth(long bandwidth) throws IOException {
1965 namenode.setBalancerBandwidth(bandwidth);
1966 }
1967
1968 /**
1969 * @see ClientProtocol#finalizeUpgrade()
1970 */
1971 public void finalizeUpgrade() throws IOException {
1972 namenode.finalizeUpgrade();
1973 }
1974
1975 /**
1976 */
1977 @Deprecated
1978 public boolean mkdirs(String src) throws IOException {
1979 return mkdirs(src, null, true);
1980 }
1981
1982 /**
1983 * Create a directory (or hierarchy of directories) with the given
1984 * name and permission.
1985 *
1986 * @param src The path of the directory being created
1987 * @param permission The permission of the directory being created.
1988 * If permission == null, use {@link FsPermission#getDefault()}.
1989 * @param createParent create missing parent directory if true
1990 *
1991 * @return True if the operation success.
1992 *
1993 * @see ClientProtocol#mkdirs(String, FsPermission, boolean)
1994 */
1995 public boolean mkdirs(String src, FsPermission permission,
1996 boolean createParent) throws IOException {
1997 if (permission == null) {
1998 permission = FsPermission.getDefault();
1999 }
2000 FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
2001 return primitiveMkdir(src, masked, createParent);
2002 }
2003
2004 /**
2005 * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2006 * that the permissions has already been masked against umask.
2007 */
2008 public boolean primitiveMkdir(String src, FsPermission absPermission)
2009 throws IOException {
2010 return primitiveMkdir(src, absPermission, true);
2011 }
2012
2013 /**
2014 * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2015 * that the permissions has already been masked against umask.
2016 */
2017 public boolean primitiveMkdir(String src, FsPermission absPermission,
2018 boolean createParent)
2019 throws IOException {
2020 checkOpen();
2021 if (absPermission == null) {
2022 absPermission =
2023 FsPermission.getDefault().applyUMask(dfsClientConf.uMask);
2024 }
2025
2026 if(LOG.isDebugEnabled()) {
2027 LOG.debug(src + ": masked=" + absPermission);
2028 }
2029 try {
2030 return namenode.mkdirs(src, absPermission, createParent);
2031 } catch(RemoteException re) {
2032 throw re.unwrapRemoteException(AccessControlException.class,
2033 InvalidPathException.class,
2034 FileAlreadyExistsException.class,
2035 FileNotFoundException.class,
2036 ParentNotDirectoryException.class,
2037 SafeModeException.class,
2038 NSQuotaExceededException.class,
2039 DSQuotaExceededException.class,
2040 UnresolvedPathException.class);
2041 }
2042 }
2043
2044 /**
2045 * Get {@link ContentSummary} rooted at the specified directory.
2046 * @param path The string representation of the path
2047 *
2048 * @see ClientProtocol#getContentSummary(String)
2049 */
2050 ContentSummary getContentSummary(String src) throws IOException {
2051 try {
2052 return namenode.getContentSummary(src);
2053 } catch(RemoteException re) {
2054 throw re.unwrapRemoteException(AccessControlException.class,
2055 FileNotFoundException.class,
2056 UnresolvedPathException.class);
2057 }
2058 }
2059
2060 /**
2061 * Sets or resets quotas for a directory.
2062 * @see ClientProtocol#setQuota(String, long, long)
2063 */
2064 void setQuota(String src, long namespaceQuota, long diskspaceQuota)
2065 throws IOException {
2066 // sanity check
2067 if ((namespaceQuota <= 0 && namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2068 namespaceQuota != HdfsConstants.QUOTA_RESET) ||
2069 (diskspaceQuota <= 0 && diskspaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2070 diskspaceQuota != HdfsConstants.QUOTA_RESET)) {
2071 throw new IllegalArgumentException("Invalid values for quota : " +
2072 namespaceQuota + " and " +
2073 diskspaceQuota);
2074
2075 }
2076 try {
2077 namenode.setQuota(src, namespaceQuota, diskspaceQuota);
2078 } catch(RemoteException re) {
2079 throw re.unwrapRemoteException(AccessControlException.class,
2080 FileNotFoundException.class,
2081 NSQuotaExceededException.class,
2082 DSQuotaExceededException.class,
2083 UnresolvedPathException.class);
2084 }
2085 }
2086
2087 /**
2088 * set the modification and access time of a file
2089 *
2090 * @see ClientProtocol#setTimes(String, long, long)
2091 */
2092 public void setTimes(String src, long mtime, long atime) throws IOException {
2093 checkOpen();
2094 try {
2095 namenode.setTimes(src, mtime, atime);
2096 } catch(RemoteException re) {
2097 throw re.unwrapRemoteException(AccessControlException.class,
2098 FileNotFoundException.class,
2099 UnresolvedPathException.class);
2100 }
2101 }
2102
2103 /**
2104 * @deprecated use {@link HdfsDataInputStream} instead.
2105 */
2106 @Deprecated
2107 public static class DFSDataInputStream extends HdfsDataInputStream {
2108
2109 public DFSDataInputStream(DFSInputStream in) throws IOException {
2110 super(in);
2111 }
2112 }
2113
2114 boolean shouldTryShortCircuitRead(InetSocketAddress targetAddr) {
2115 return shortCircuitLocalReads && isLocalAddress(targetAddr);
2116 }
2117
2118 void reportChecksumFailure(String file, ExtendedBlock blk, DatanodeInfo dn) {
2119 DatanodeInfo [] dnArr = { dn };
2120 LocatedBlock [] lblocks = { new LocatedBlock(blk, dnArr) };
2121 reportChecksumFailure(file, lblocks);
2122 }
2123
2124 // just reports checksum failure and ignores any exception during the report.
2125 void reportChecksumFailure(String file, LocatedBlock lblocks[]) {
2126 try {
2127 reportBadBlocks(lblocks);
2128 } catch (IOException ie) {
2129 LOG.info("Found corruption while reading " + file
2130 + ". Error repairing corrupt blocks. Bad blocks remain.", ie);
2131 }
2132 }
2133
2134 @Override
2135 public String toString() {
2136 return getClass().getSimpleName() + "[clientName=" + clientName
2137 + ", ugi=" + ugi + "]";
2138 }
2139
2140 void disableShortCircuit() {
2141 shortCircuitLocalReads = false;
2142 }
2143 }