001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs;
019
020 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT;
027 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY;
028 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT;
029 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_KEY;
030 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS;
031 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES;
032 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD;
033 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
034 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
035 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
036 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
037 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
038 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
039 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT;
040 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY;
041 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT;
042 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY;
043 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY;
044 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE;
045 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT;
046 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY;
047 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT;
048 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
049 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
050 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
051 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
052 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL;
053 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT;
054 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
055 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
056 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
057 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
058 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
059
060 import java.io.BufferedOutputStream;
061 import java.io.DataInputStream;
062 import java.io.DataOutputStream;
063 import java.io.FileNotFoundException;
064 import java.io.IOException;
065 import java.io.InputStream;
066 import java.io.OutputStream;
067 import java.net.InetAddress;
068 import java.net.InetSocketAddress;
069 import java.net.NetworkInterface;
070 import java.net.Socket;
071 import java.net.SocketException;
072 import java.net.SocketAddress;
073 import java.net.URI;
074 import java.net.UnknownHostException;
075 import java.util.ArrayList;
076 import java.util.Collections;
077 import java.util.EnumSet;
078 import java.util.HashMap;
079 import java.util.LinkedHashMap;
080 import java.util.List;
081 import java.util.Map;
082 import java.util.Random;
083
084 import javax.net.SocketFactory;
085
086 import org.apache.commons.logging.Log;
087 import org.apache.commons.logging.LogFactory;
088 import org.apache.hadoop.classification.InterfaceAudience;
089 import org.apache.hadoop.conf.Configuration;
090 import org.apache.hadoop.fs.BlockLocation;
091 import org.apache.hadoop.fs.BlockStorageLocation;
092 import org.apache.hadoop.fs.CacheFlag;
093 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
094 import org.apache.hadoop.fs.ContentSummary;
095 import org.apache.hadoop.fs.CreateFlag;
096 import org.apache.hadoop.fs.FileAlreadyExistsException;
097 import org.apache.hadoop.fs.FileSystem;
098 import org.apache.hadoop.fs.FsServerDefaults;
099 import org.apache.hadoop.fs.FsStatus;
100 import org.apache.hadoop.fs.HdfsBlockLocation;
101 import org.apache.hadoop.fs.InvalidPathException;
102 import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
103 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
104 import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
105 import org.apache.hadoop.fs.Options;
106 import org.apache.hadoop.fs.RemoteIterator;
107 import org.apache.hadoop.fs.Options.ChecksumOpt;
108 import org.apache.hadoop.fs.ParentNotDirectoryException;
109 import org.apache.hadoop.fs.Path;
110 import org.apache.hadoop.fs.UnresolvedLinkException;
111 import org.apache.hadoop.fs.VolumeId;
112 import org.apache.hadoop.fs.permission.FsPermission;
113 import org.apache.hadoop.hdfs.client.ClientMmapManager;
114 import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
115 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
116 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
117 import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator;
118 import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
119 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
120 import org.apache.hadoop.hdfs.protocol.CachePoolIterator;
121 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
122 import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
123 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
124 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
125 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
126 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
127 import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
128 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
129 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
130 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
131 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
132 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
133 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
134 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
135 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
136 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
137 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
138 import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
139 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
140 import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor;
141 import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
142 import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
143 import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
144 import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
145 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
146 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto;
147 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
148 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
149 import org.apache.hadoop.hdfs.protocolPB.PBHelper;
150 import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
151 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
152 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
153 import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
154 import org.apache.hadoop.hdfs.server.namenode.NameNode;
155 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
156 import org.apache.hadoop.io.DataOutputBuffer;
157 import org.apache.hadoop.io.EnumSetWritable;
158 import org.apache.hadoop.io.IOUtils;
159 import org.apache.hadoop.io.MD5Hash;
160 import org.apache.hadoop.io.Text;
161 import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
162 import org.apache.hadoop.ipc.Client;
163 import org.apache.hadoop.ipc.RPC;
164 import org.apache.hadoop.ipc.RemoteException;
165 import org.apache.hadoop.net.DNS;
166 import org.apache.hadoop.net.NetUtils;
167 import org.apache.hadoop.security.AccessControlException;
168 import org.apache.hadoop.security.UserGroupInformation;
169 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
170 import org.apache.hadoop.security.token.Token;
171 import org.apache.hadoop.security.token.TokenRenewer;
172 import org.apache.hadoop.util.DataChecksum;
173 import org.apache.hadoop.util.DataChecksum.Type;
174 import org.apache.hadoop.util.Progressable;
175 import org.apache.hadoop.util.Time;
176
177 import com.google.common.annotations.VisibleForTesting;
178 import com.google.common.base.Joiner;
179 import com.google.common.base.Preconditions;
180 import com.google.common.net.InetAddresses;
181
182 /********************************************************
183 * DFSClient can connect to a Hadoop Filesystem and
184 * perform basic file tasks. It uses the ClientProtocol
185 * to communicate with a NameNode daemon, and connects
186 * directly to DataNodes to read/write block data.
187 *
188 * Hadoop DFS users should obtain an instance of
189 * DistributedFileSystem, which uses DFSClient to handle
190 * filesystem tasks.
191 *
192 ********************************************************/
193 @InterfaceAudience.Private
194 public class DFSClient implements java.io.Closeable {
195 public static final Log LOG = LogFactory.getLog(DFSClient.class);
196 public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; // 1 hour
197 static final int TCP_WINDOW_SIZE = 128 * 1024; // 128 KB
198
199 private final Configuration conf;
200 private final Conf dfsClientConf;
201 final ClientProtocol namenode;
202 /* The service used for delegation tokens */
203 private Text dtService;
204
205 final UserGroupInformation ugi;
206 volatile boolean clientRunning = true;
207 volatile long lastLeaseRenewal;
208 private volatile FsServerDefaults serverDefaults;
209 private volatile long serverDefaultsLastUpdate;
210 final String clientName;
211 SocketFactory socketFactory;
212 final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
213 final FileSystem.Statistics stats;
214 private final String authority;
215 final PeerCache peerCache;
216 private Random r = new Random();
217 private SocketAddress[] localInterfaceAddrs;
218 private DataEncryptionKey encryptionKey;
219 private boolean shouldUseLegacyBlockReaderLocal;
220 private final CachingStrategy defaultReadCachingStrategy;
221 private final CachingStrategy defaultWriteCachingStrategy;
222 private ClientMmapManager mmapManager;
223
224 private static final ClientMmapManagerFactory MMAP_MANAGER_FACTORY =
225 new ClientMmapManagerFactory();
226
227 private static final class ClientMmapManagerFactory {
228 private ClientMmapManager mmapManager = null;
229 /**
230 * Tracks the number of users of mmapManager.
231 */
232 private int refcnt = 0;
233
234 synchronized ClientMmapManager get(Configuration conf) {
235 if (refcnt++ == 0) {
236 mmapManager = ClientMmapManager.fromConf(conf);
237 } else {
238 String mismatches = mmapManager.verifyConfigurationMatches(conf);
239 if (!mismatches.isEmpty()) {
240 LOG.warn("The ClientMmapManager settings you specified " +
241 "have been ignored because another thread created the " +
242 "ClientMmapManager first. " + mismatches);
243 }
244 }
245 return mmapManager;
246 }
247
248 synchronized void unref(ClientMmapManager mmapManager) {
249 if (this.mmapManager != mmapManager) {
250 throw new IllegalArgumentException();
251 }
252 if (--refcnt == 0) {
253 IOUtils.cleanup(LOG, mmapManager);
254 mmapManager = null;
255 }
256 }
257 }
258
259 /**
260 * DFSClient configuration
261 */
262 public static class Conf {
263 final int hdfsTimeout; // timeout value for a DFS operation.
264 final int maxFailoverAttempts;
265 final int maxRetryAttempts;
266 final int failoverSleepBaseMillis;
267 final int failoverSleepMaxMillis;
268 final int maxBlockAcquireFailures;
269 final int confTime;
270 final int ioBufferSize;
271 final ChecksumOpt defaultChecksumOpt;
272 final int writePacketSize;
273 final int socketTimeout;
274 final int socketCacheCapacity;
275 final long socketCacheExpiry;
276 final long excludedNodesCacheExpiry;
277 /** Wait time window (in msec) if BlockMissingException is caught */
278 final int timeWindow;
279 final int nCachedConnRetry;
280 final int nBlockWriteRetry;
281 final int nBlockWriteLocateFollowingRetry;
282 final long defaultBlockSize;
283 final long prefetchSize;
284 final short defaultReplication;
285 final String taskId;
286 final FsPermission uMask;
287 final boolean connectToDnViaHostname;
288 final boolean getHdfsBlocksMetadataEnabled;
289 final int getFileBlockStorageLocationsNumThreads;
290 final int getFileBlockStorageLocationsTimeout;
291 final int retryTimesForGetLastBlockLength;
292 final int retryIntervalForGetLastBlockLength;
293
294 final boolean useLegacyBlockReader;
295 final boolean useLegacyBlockReaderLocal;
296 final String domainSocketPath;
297 final boolean skipShortCircuitChecksums;
298 final int shortCircuitBufferSize;
299 final boolean shortCircuitLocalReads;
300 final boolean domainSocketDataTraffic;
301 final int shortCircuitStreamsCacheSize;
302 final long shortCircuitStreamsCacheExpiryMs;
303
304 public Conf(Configuration conf) {
305 // The hdfsTimeout is currently the same as the ipc timeout
306 hdfsTimeout = Client.getTimeout(conf);
307
308 maxFailoverAttempts = conf.getInt(
309 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
310 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
311 maxRetryAttempts = conf.getInt(
312 DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
313 DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
314 failoverSleepBaseMillis = conf.getInt(
315 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
316 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
317 failoverSleepMaxMillis = conf.getInt(
318 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
319 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
320
321 maxBlockAcquireFailures = conf.getInt(
322 DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY,
323 DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT);
324 confTime = conf.getInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
325 HdfsServerConstants.WRITE_TIMEOUT);
326 ioBufferSize = conf.getInt(
327 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
328 CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
329 defaultChecksumOpt = getChecksumOptFromConf(conf);
330 socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
331 HdfsServerConstants.READ_TIMEOUT);
332 /** dfs.write.packet.size is an internal config variable */
333 writePacketSize = conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY,
334 DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT);
335 defaultBlockSize = conf.getLongBytes(DFS_BLOCK_SIZE_KEY,
336 DFS_BLOCK_SIZE_DEFAULT);
337 defaultReplication = (short) conf.getInt(
338 DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT);
339 taskId = conf.get("mapreduce.task.attempt.id", "NONMAPREDUCE");
340 socketCacheCapacity = conf.getInt(DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY,
341 DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT);
342 socketCacheExpiry = conf.getLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY,
343 DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT);
344 excludedNodesCacheExpiry = conf.getLong(
345 DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL,
346 DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT);
347 prefetchSize = conf.getLong(DFS_CLIENT_READ_PREFETCH_SIZE_KEY,
348 10 * defaultBlockSize);
349 timeWindow = conf.getInt(DFS_CLIENT_RETRY_WINDOW_BASE, 3000);
350 nCachedConnRetry = conf.getInt(DFS_CLIENT_CACHED_CONN_RETRY_KEY,
351 DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT);
352 nBlockWriteRetry = conf.getInt(DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY,
353 DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT);
354 nBlockWriteLocateFollowingRetry = conf.getInt(
355 DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
356 DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT);
357 uMask = FsPermission.getUMask(conf);
358 connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
359 DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
360 getHdfsBlocksMetadataEnabled = conf.getBoolean(
361 DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
362 DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
363 getFileBlockStorageLocationsNumThreads = conf.getInt(
364 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS,
365 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT);
366 getFileBlockStorageLocationsTimeout = conf.getInt(
367 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT,
368 DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT);
369 retryTimesForGetLastBlockLength = conf.getInt(
370 DFSConfigKeys.DFS_CLIENT_RETRY_TIMES_GET_LAST_BLOCK_LENGTH,
371 DFSConfigKeys.DFS_CLIENT_RETRY_TIMES_GET_LAST_BLOCK_LENGTH_DEFAULT);
372 retryIntervalForGetLastBlockLength = conf.getInt(
373 DFSConfigKeys.DFS_CLIENT_RETRY_INTERVAL_GET_LAST_BLOCK_LENGTH,
374 DFSConfigKeys.DFS_CLIENT_RETRY_INTERVAL_GET_LAST_BLOCK_LENGTH_DEFAULT);
375
376 useLegacyBlockReader = conf.getBoolean(
377 DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER,
378 DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT);
379 useLegacyBlockReaderLocal = conf.getBoolean(
380 DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
381 DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT);
382 shortCircuitLocalReads = conf.getBoolean(
383 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
384 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT);
385 domainSocketDataTraffic = conf.getBoolean(
386 DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC,
387 DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT);
388 domainSocketPath = conf.getTrimmed(
389 DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
390 DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
391
392 if (BlockReaderLocal.LOG.isDebugEnabled()) {
393 BlockReaderLocal.LOG.debug(
394 DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL
395 + " = " + useLegacyBlockReaderLocal);
396 BlockReaderLocal.LOG.debug(
397 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY
398 + " = " + shortCircuitLocalReads);
399 BlockReaderLocal.LOG.debug(
400 DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC
401 + " = " + domainSocketDataTraffic);
402 BlockReaderLocal.LOG.debug(
403 DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY
404 + " = " + domainSocketPath);
405 }
406
407 skipShortCircuitChecksums = conf.getBoolean(
408 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY,
409 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_DEFAULT);
410 shortCircuitBufferSize = conf.getInt(
411 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY,
412 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_DEFAULT);
413 shortCircuitStreamsCacheSize = conf.getInt(
414 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_KEY,
415 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_DEFAULT);
416 shortCircuitStreamsCacheExpiryMs = conf.getLong(
417 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,
418 DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_DEFAULT);
419 }
420
421 private DataChecksum.Type getChecksumType(Configuration conf) {
422 final String checksum = conf.get(
423 DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY,
424 DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
425 try {
426 return DataChecksum.Type.valueOf(checksum);
427 } catch(IllegalArgumentException iae) {
428 LOG.warn("Bad checksum type: " + checksum + ". Using default "
429 + DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
430 return DataChecksum.Type.valueOf(
431 DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
432 }
433 }
434
435 // Construct a checksum option from conf
436 private ChecksumOpt getChecksumOptFromConf(Configuration conf) {
437 DataChecksum.Type type = getChecksumType(conf);
438 int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY,
439 DFS_BYTES_PER_CHECKSUM_DEFAULT);
440 return new ChecksumOpt(type, bytesPerChecksum);
441 }
442
443 // create a DataChecksum with the default option.
444 private DataChecksum createChecksum() throws IOException {
445 return createChecksum(null);
446 }
447
448 private DataChecksum createChecksum(ChecksumOpt userOpt)
449 throws IOException {
450 // Fill in any missing field with the default.
451 ChecksumOpt myOpt = ChecksumOpt.processChecksumOpt(
452 defaultChecksumOpt, userOpt);
453 DataChecksum dataChecksum = DataChecksum.newDataChecksum(
454 myOpt.getChecksumType(),
455 myOpt.getBytesPerChecksum());
456 if (dataChecksum == null) {
457 throw new IOException("Invalid checksum type specified: "
458 + myOpt.getChecksumType().name());
459 }
460 return dataChecksum;
461 }
462 }
463
464 public Conf getConf() {
465 return dfsClientConf;
466 }
467
468 Configuration getConfiguration() {
469 return conf;
470 }
471
472 /**
473 * A map from file names to {@link DFSOutputStream} objects
474 * that are currently being written by this client.
475 * Note that a file can only be written by a single client.
476 */
477 private final Map<String, DFSOutputStream> filesBeingWritten
478 = new HashMap<String, DFSOutputStream>();
479
480 private final DomainSocketFactory domainSocketFactory;
481
482 /**
483 * Same as this(NameNode.getAddress(conf), conf);
484 * @see #DFSClient(InetSocketAddress, Configuration)
485 * @deprecated Deprecated at 0.21
486 */
487 @Deprecated
488 public DFSClient(Configuration conf) throws IOException {
489 this(NameNode.getAddress(conf), conf);
490 }
491
492 public DFSClient(InetSocketAddress address, Configuration conf) throws IOException {
493 this(NameNode.getUri(address), conf);
494 }
495
496 /**
497 * Same as this(nameNodeUri, conf, null);
498 * @see #DFSClient(URI, Configuration, FileSystem.Statistics)
499 */
500 public DFSClient(URI nameNodeUri, Configuration conf
501 ) throws IOException {
502 this(nameNodeUri, conf, null);
503 }
504
505 /**
506 * Same as this(nameNodeUri, null, conf, stats);
507 * @see #DFSClient(URI, ClientProtocol, Configuration, FileSystem.Statistics)
508 */
509 public DFSClient(URI nameNodeUri, Configuration conf,
510 FileSystem.Statistics stats)
511 throws IOException {
512 this(nameNodeUri, null, conf, stats);
513 }
514
515 /**
516 * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode.
517 * If HA is enabled and a positive value is set for
518 * {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the
519 * configuration, the DFSClient will use {@link LossyRetryInvocationHandler}
520 * as its RetryInvocationHandler. Otherwise one of nameNodeUri or rpcNamenode
521 * must be null.
522 */
523 @VisibleForTesting
524 public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
525 Configuration conf, FileSystem.Statistics stats)
526 throws IOException {
527 // Copy only the required DFSClient configuration
528 this.dfsClientConf = new Conf(conf);
529 this.shouldUseLegacyBlockReaderLocal =
530 this.dfsClientConf.useLegacyBlockReaderLocal;
531 if (this.dfsClientConf.useLegacyBlockReaderLocal) {
532 LOG.debug("Using legacy short-circuit local reads.");
533 }
534 this.conf = conf;
535 this.stats = stats;
536 this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
537 this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
538
539 this.ugi = UserGroupInformation.getCurrentUser();
540
541 this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
542 this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
543 DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
544
545 int numResponseToDrop = conf.getInt(
546 DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
547 DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
548 NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null;
549 if (numResponseToDrop > 0) {
550 // This case is used for testing.
551 LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY
552 + " is set to " + numResponseToDrop
553 + ", this hacked client will proactively drop responses");
554 proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf,
555 nameNodeUri, ClientProtocol.class, numResponseToDrop);
556 }
557
558 if (proxyInfo != null) {
559 this.dtService = proxyInfo.getDelegationTokenService();
560 this.namenode = proxyInfo.getProxy();
561 } else if (rpcNamenode != null) {
562 // This case is used for testing.
563 Preconditions.checkArgument(nameNodeUri == null);
564 this.namenode = rpcNamenode;
565 dtService = null;
566 } else {
567 Preconditions.checkArgument(nameNodeUri != null,
568 "null URI");
569 proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
570 ClientProtocol.class);
571 this.dtService = proxyInfo.getDelegationTokenService();
572 this.namenode = proxyInfo.getProxy();
573 }
574
575 // read directly from the block file if configured.
576 this.domainSocketFactory = new DomainSocketFactory(dfsClientConf);
577
578 String localInterfaces[] =
579 conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES);
580 localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces);
581 if (LOG.isDebugEnabled() && 0 != localInterfaces.length) {
582 LOG.debug("Using local interfaces [" +
583 Joiner.on(',').join(localInterfaces)+ "] with addresses [" +
584 Joiner.on(',').join(localInterfaceAddrs) + "]");
585 }
586
587 this.peerCache = PeerCache.getInstance(dfsClientConf.socketCacheCapacity, dfsClientConf.socketCacheExpiry);
588 Boolean readDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ?
589 null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false);
590 Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ?
591 null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0);
592 Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ?
593 null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false);
594 this.defaultReadCachingStrategy =
595 new CachingStrategy(readDropBehind, readahead);
596 this.defaultWriteCachingStrategy =
597 new CachingStrategy(writeDropBehind, readahead);
598 this.mmapManager = MMAP_MANAGER_FACTORY.get(conf);
599 }
600
601 /**
602 * Return the socket addresses to use with each configured
603 * local interface. Local interfaces may be specified by IP
604 * address, IP address range using CIDR notation, interface
605 * name (e.g. eth0) or sub-interface name (e.g. eth0:0).
606 * The socket addresses consist of the IPs for the interfaces
607 * and the ephemeral port (port 0). If an IP, IP range, or
608 * interface name matches an interface with sub-interfaces
609 * only the IP of the interface is used. Sub-interfaces can
610 * be used by specifying them explicitly (by IP or name).
611 *
612 * @return SocketAddresses for the configured local interfaces,
613 * or an empty array if none are configured
614 * @throws UnknownHostException if a given interface name is invalid
615 */
616 private static SocketAddress[] getLocalInterfaceAddrs(
617 String interfaceNames[]) throws UnknownHostException {
618 List<SocketAddress> localAddrs = new ArrayList<SocketAddress>();
619 for (String interfaceName : interfaceNames) {
620 if (InetAddresses.isInetAddress(interfaceName)) {
621 localAddrs.add(new InetSocketAddress(interfaceName, 0));
622 } else if (NetUtils.isValidSubnet(interfaceName)) {
623 for (InetAddress addr : NetUtils.getIPs(interfaceName, false)) {
624 localAddrs.add(new InetSocketAddress(addr, 0));
625 }
626 } else {
627 for (String ip : DNS.getIPs(interfaceName, false)) {
628 localAddrs.add(new InetSocketAddress(ip, 0));
629 }
630 }
631 }
632 return localAddrs.toArray(new SocketAddress[localAddrs.size()]);
633 }
634
635 /**
636 * Select one of the configured local interfaces at random. We use a random
637 * interface because other policies like round-robin are less effective
638 * given that we cache connections to datanodes.
639 *
640 * @return one of the local interface addresses at random, or null if no
641 * local interfaces are configured
642 */
643 SocketAddress getRandomLocalInterfaceAddr() {
644 if (localInterfaceAddrs.length == 0) {
645 return null;
646 }
647 final int idx = r.nextInt(localInterfaceAddrs.length);
648 final SocketAddress addr = localInterfaceAddrs[idx];
649 if (LOG.isDebugEnabled()) {
650 LOG.debug("Using local interface " + addr);
651 }
652 return addr;
653 }
654
655 /**
656 * Return the number of times the client should go back to the namenode
657 * to retrieve block locations when reading.
658 */
659 int getMaxBlockAcquireFailures() {
660 return dfsClientConf.maxBlockAcquireFailures;
661 }
662
663 /**
664 * Return the timeout that clients should use when writing to datanodes.
665 * @param numNodes the number of nodes in the pipeline.
666 */
667 int getDatanodeWriteTimeout(int numNodes) {
668 return (dfsClientConf.confTime > 0) ?
669 (dfsClientConf.confTime + HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * numNodes) : 0;
670 }
671
672 int getDatanodeReadTimeout(int numNodes) {
673 return dfsClientConf.socketTimeout > 0 ?
674 (HdfsServerConstants.READ_TIMEOUT_EXTENSION * numNodes +
675 dfsClientConf.socketTimeout) : 0;
676 }
677
678 int getHdfsTimeout() {
679 return dfsClientConf.hdfsTimeout;
680 }
681
682 @VisibleForTesting
683 public String getClientName() {
684 return clientName;
685 }
686
687 void checkOpen() throws IOException {
688 if (!clientRunning) {
689 IOException result = new IOException("Filesystem closed");
690 throw result;
691 }
692 }
693
694 /** Return the lease renewer instance. The renewer thread won't start
695 * until the first output stream is created. The same instance will
696 * be returned until all output streams are closed.
697 */
698 public LeaseRenewer getLeaseRenewer() throws IOException {
699 return LeaseRenewer.getInstance(authority, ugi, this);
700 }
701
702 /** Get a lease and start automatic renewal */
703 private void beginFileLease(final String src, final DFSOutputStream out)
704 throws IOException {
705 getLeaseRenewer().put(src, out, this);
706 }
707
708 /** Stop renewal of lease for the file. */
709 void endFileLease(final String src) throws IOException {
710 getLeaseRenewer().closeFile(src, this);
711 }
712
713
714 /** Put a file. Only called from LeaseRenewer, where proper locking is
715 * enforced to consistently update its local dfsclients array and
716 * client's filesBeingWritten map.
717 */
718 void putFileBeingWritten(final String src, final DFSOutputStream out) {
719 synchronized(filesBeingWritten) {
720 filesBeingWritten.put(src, out);
721 // update the last lease renewal time only when there was no
722 // writes. once there is one write stream open, the lease renewer
723 // thread keeps it updated well with in anyone's expiration time.
724 if (lastLeaseRenewal == 0) {
725 updateLastLeaseRenewal();
726 }
727 }
728 }
729
730 /** Remove a file. Only called from LeaseRenewer. */
731 void removeFileBeingWritten(final String src) {
732 synchronized(filesBeingWritten) {
733 filesBeingWritten.remove(src);
734 if (filesBeingWritten.isEmpty()) {
735 lastLeaseRenewal = 0;
736 }
737 }
738 }
739
740 /** Is file-being-written map empty? */
741 boolean isFilesBeingWrittenEmpty() {
742 synchronized(filesBeingWritten) {
743 return filesBeingWritten.isEmpty();
744 }
745 }
746
747 /** @return true if the client is running */
748 boolean isClientRunning() {
749 return clientRunning;
750 }
751
752 long getLastLeaseRenewal() {
753 return lastLeaseRenewal;
754 }
755
756 void updateLastLeaseRenewal() {
757 synchronized(filesBeingWritten) {
758 if (filesBeingWritten.isEmpty()) {
759 return;
760 }
761 lastLeaseRenewal = Time.now();
762 }
763 }
764
765 /**
766 * Renew leases.
767 * @return true if lease was renewed. May return false if this
768 * client has been closed or has no files open.
769 **/
770 boolean renewLease() throws IOException {
771 if (clientRunning && !isFilesBeingWrittenEmpty()) {
772 try {
773 namenode.renewLease(clientName);
774 updateLastLeaseRenewal();
775 return true;
776 } catch (IOException e) {
777 // Abort if the lease has already expired.
778 final long elapsed = Time.now() - getLastLeaseRenewal();
779 if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) {
780 LOG.warn("Failed to renew lease for " + clientName + " for "
781 + (elapsed/1000) + " seconds (>= hard-limit ="
782 + (HdfsConstants.LEASE_HARDLIMIT_PERIOD/1000) + " seconds.) "
783 + "Closing all files being written ...", e);
784 closeAllFilesBeingWritten(true);
785 } else {
786 // Let the lease renewer handle it and retry.
787 throw e;
788 }
789 }
790 }
791 return false;
792 }
793
794 /**
795 * Close connections the Namenode.
796 */
797 void closeConnectionToNamenode() {
798 RPC.stopProxy(namenode);
799 }
800
801 /** Abort and release resources held. Ignore all errors. */
802 void abort() {
803 if (mmapManager != null) {
804 MMAP_MANAGER_FACTORY.unref(mmapManager);
805 mmapManager = null;
806 }
807 clientRunning = false;
808 closeAllFilesBeingWritten(true);
809 try {
810 // remove reference to this client and stop the renewer,
811 // if there is no more clients under the renewer.
812 getLeaseRenewer().closeClient(this);
813 } catch (IOException ioe) {
814 LOG.info("Exception occurred while aborting the client " + ioe);
815 }
816 closeConnectionToNamenode();
817 }
818
819 /** Close/abort all files being written. */
820 private void closeAllFilesBeingWritten(final boolean abort) {
821 for(;;) {
822 final String src;
823 final DFSOutputStream out;
824 synchronized(filesBeingWritten) {
825 if (filesBeingWritten.isEmpty()) {
826 return;
827 }
828 src = filesBeingWritten.keySet().iterator().next();
829 out = filesBeingWritten.remove(src);
830 }
831 if (out != null) {
832 try {
833 if (abort) {
834 out.abort();
835 } else {
836 out.close();
837 }
838 } catch(IOException ie) {
839 LOG.error("Failed to " + (abort? "abort": "close") + " file " + src,
840 ie);
841 }
842 }
843 }
844 }
845
846 /**
847 * Close the file system, abandoning all of the leases and files being
848 * created and close connections to the namenode.
849 */
850 @Override
851 public synchronized void close() throws IOException {
852 if (mmapManager != null) {
853 MMAP_MANAGER_FACTORY.unref(mmapManager);
854 mmapManager = null;
855 }
856 if(clientRunning) {
857 closeAllFilesBeingWritten(false);
858 clientRunning = false;
859 getLeaseRenewer().closeClient(this);
860 // close connections to the namenode
861 closeConnectionToNamenode();
862 }
863 }
864
865 /**
866 * Get the default block size for this cluster
867 * @return the default block size in bytes
868 */
869 public long getDefaultBlockSize() {
870 return dfsClientConf.defaultBlockSize;
871 }
872
873 /**
874 * @see ClientProtocol#getPreferredBlockSize(String)
875 */
876 public long getBlockSize(String f) throws IOException {
877 try {
878 return namenode.getPreferredBlockSize(f);
879 } catch (IOException ie) {
880 LOG.warn("Problem getting block size", ie);
881 throw ie;
882 }
883 }
884
885 /**
886 * Get server default values for a number of configuration params.
887 * @see ClientProtocol#getServerDefaults()
888 */
889 public FsServerDefaults getServerDefaults() throws IOException {
890 long now = Time.now();
891 if (now - serverDefaultsLastUpdate > SERVER_DEFAULTS_VALIDITY_PERIOD) {
892 serverDefaults = namenode.getServerDefaults();
893 serverDefaultsLastUpdate = now;
894 }
895 return serverDefaults;
896 }
897
898 /**
899 * Get a canonical token service name for this client's tokens. Null should
900 * be returned if the client is not using tokens.
901 * @return the token service for the client
902 */
903 @InterfaceAudience.LimitedPrivate( { "HDFS" })
904 public String getCanonicalServiceName() {
905 return (dtService != null) ? dtService.toString() : null;
906 }
907
908 /**
909 * @see ClientProtocol#getDelegationToken(Text)
910 */
911 public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
912 throws IOException {
913 assert dtService != null;
914 Token<DelegationTokenIdentifier> token =
915 namenode.getDelegationToken(renewer);
916
917 if (token != null) {
918 token.setService(this.dtService);
919 LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
920 } else {
921 LOG.info("Cannot get delegation token from " + renewer);
922 }
923 return token;
924
925 }
926
927 /**
928 * Renew a delegation token
929 * @param token the token to renew
930 * @return the new expiration time
931 * @throws InvalidToken
932 * @throws IOException
933 * @deprecated Use Token.renew instead.
934 */
935 @Deprecated
936 public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
937 throws InvalidToken, IOException {
938 LOG.info("Renewing " + DelegationTokenIdentifier.stringifyToken(token));
939 try {
940 return token.renew(conf);
941 } catch (InterruptedException ie) {
942 throw new RuntimeException("caught interrupted", ie);
943 } catch (RemoteException re) {
944 throw re.unwrapRemoteException(InvalidToken.class,
945 AccessControlException.class);
946 }
947 }
948
949 private static Map<String, Boolean> localAddrMap = Collections
950 .synchronizedMap(new HashMap<String, Boolean>());
951
952 static boolean isLocalAddress(InetSocketAddress targetAddr) {
953 InetAddress addr = targetAddr.getAddress();
954 Boolean cached = localAddrMap.get(addr.getHostAddress());
955 if (cached != null) {
956 if (LOG.isTraceEnabled()) {
957 LOG.trace("Address " + targetAddr +
958 (cached ? " is local" : " is not local"));
959 }
960 return cached;
961 }
962
963 boolean local = NetUtils.isLocalAddress(addr);
964
965 if (LOG.isTraceEnabled()) {
966 LOG.trace("Address " + targetAddr +
967 (local ? " is local" : " is not local"));
968 }
969 localAddrMap.put(addr.getHostAddress(), local);
970 return local;
971 }
972
973 /**
974 * Should the block access token be refetched on an exception
975 *
976 * @param ex Exception received
977 * @param targetAddr Target datanode address from where exception was received
978 * @return true if block access token has expired or invalid and it should be
979 * refetched
980 */
981 private static boolean tokenRefetchNeeded(IOException ex,
982 InetSocketAddress targetAddr) {
983 /*
984 * Get a new access token and retry. Retry is needed in 2 cases. 1) When
985 * both NN and DN re-started while DFSClient holding a cached access token.
986 * 2) In the case that NN fails to update its access key at pre-set interval
987 * (by a wide margin) and subsequently restarts. In this case, DN
988 * re-registers itself with NN and receives a new access key, but DN will
989 * delete the old access key from its memory since it's considered expired
990 * based on the estimated expiration date.
991 */
992 if (ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) {
993 LOG.info("Access token was invalid when connecting to " + targetAddr
994 + " : " + ex);
995 return true;
996 }
997 return false;
998 }
999
1000 /**
1001 * Cancel a delegation token
1002 * @param token the token to cancel
1003 * @throws InvalidToken
1004 * @throws IOException
1005 * @deprecated Use Token.cancel instead.
1006 */
1007 @Deprecated
1008 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
1009 throws InvalidToken, IOException {
1010 LOG.info("Cancelling " + DelegationTokenIdentifier.stringifyToken(token));
1011 try {
1012 token.cancel(conf);
1013 } catch (InterruptedException ie) {
1014 throw new RuntimeException("caught interrupted", ie);
1015 } catch (RemoteException re) {
1016 throw re.unwrapRemoteException(InvalidToken.class,
1017 AccessControlException.class);
1018 }
1019 }
1020
1021 @InterfaceAudience.Private
1022 public static class Renewer extends TokenRenewer {
1023
1024 static {
1025 //Ensure that HDFS Configuration files are loaded before trying to use
1026 // the renewer.
1027 HdfsConfiguration.init();
1028 }
1029
1030 @Override
1031 public boolean handleKind(Text kind) {
1032 return DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind);
1033 }
1034
1035 @SuppressWarnings("unchecked")
1036 @Override
1037 public long renew(Token<?> token, Configuration conf) throws IOException {
1038 Token<DelegationTokenIdentifier> delToken =
1039 (Token<DelegationTokenIdentifier>) token;
1040 ClientProtocol nn = getNNProxy(delToken, conf);
1041 try {
1042 return nn.renewDelegationToken(delToken);
1043 } catch (RemoteException re) {
1044 throw re.unwrapRemoteException(InvalidToken.class,
1045 AccessControlException.class);
1046 }
1047 }
1048
1049 @SuppressWarnings("unchecked")
1050 @Override
1051 public void cancel(Token<?> token, Configuration conf) throws IOException {
1052 Token<DelegationTokenIdentifier> delToken =
1053 (Token<DelegationTokenIdentifier>) token;
1054 LOG.info("Cancelling " +
1055 DelegationTokenIdentifier.stringifyToken(delToken));
1056 ClientProtocol nn = getNNProxy(delToken, conf);
1057 try {
1058 nn.cancelDelegationToken(delToken);
1059 } catch (RemoteException re) {
1060 throw re.unwrapRemoteException(InvalidToken.class,
1061 AccessControlException.class);
1062 }
1063 }
1064
1065 private static ClientProtocol getNNProxy(
1066 Token<DelegationTokenIdentifier> token, Configuration conf)
1067 throws IOException {
1068 URI uri = HAUtil.getServiceUriFromToken(token);
1069 if (HAUtil.isTokenForLogicalUri(token) &&
1070 !HAUtil.isLogicalUri(conf, uri)) {
1071 // If the token is for a logical nameservice, but the configuration
1072 // we have disagrees about that, we can't actually renew it.
1073 // This can be the case in MR, for example, if the RM doesn't
1074 // have all of the HA clusters configured in its configuration.
1075 throw new IOException("Unable to map logical nameservice URI '" +
1076 uri + "' to a NameNode. Local configuration does not have " +
1077 "a failover proxy provider configured.");
1078 }
1079
1080 NameNodeProxies.ProxyAndInfo<ClientProtocol> info =
1081 NameNodeProxies.createProxy(conf, uri, ClientProtocol.class);
1082 assert info.getDelegationTokenService().equals(token.getService()) :
1083 "Returned service '" + info.getDelegationTokenService().toString() +
1084 "' doesn't match expected service '" +
1085 token.getService().toString() + "'";
1086
1087 return info.getProxy();
1088 }
1089
1090 @Override
1091 public boolean isManaged(Token<?> token) throws IOException {
1092 return true;
1093 }
1094
1095 }
1096
1097 /**
1098 * Report corrupt blocks that were discovered by the client.
1099 * @see ClientProtocol#reportBadBlocks(LocatedBlock[])
1100 */
1101 public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
1102 namenode.reportBadBlocks(blocks);
1103 }
1104
1105 public short getDefaultReplication() {
1106 return dfsClientConf.defaultReplication;
1107 }
1108
1109 public LocatedBlocks getLocatedBlocks(String src, long start)
1110 throws IOException {
1111 return getLocatedBlocks(src, start, dfsClientConf.prefetchSize);
1112 }
1113
1114 /*
1115 * This is just a wrapper around callGetBlockLocations, but non-static so that
1116 * we can stub it out for tests.
1117 */
1118 @VisibleForTesting
1119 public LocatedBlocks getLocatedBlocks(String src, long start, long length)
1120 throws IOException {
1121 return callGetBlockLocations(namenode, src, start, length);
1122 }
1123
1124 /**
1125 * @see ClientProtocol#getBlockLocations(String, long, long)
1126 */
1127 static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
1128 String src, long start, long length)
1129 throws IOException {
1130 try {
1131 return namenode.getBlockLocations(src, start, length);
1132 } catch(RemoteException re) {
1133 throw re.unwrapRemoteException(AccessControlException.class,
1134 FileNotFoundException.class,
1135 UnresolvedPathException.class);
1136 }
1137 }
1138
1139 /**
1140 * Recover a file's lease
1141 * @param src a file's path
1142 * @return true if the file is already closed
1143 * @throws IOException
1144 */
1145 boolean recoverLease(String src) throws IOException {
1146 checkOpen();
1147
1148 try {
1149 return namenode.recoverLease(src, clientName);
1150 } catch (RemoteException re) {
1151 throw re.unwrapRemoteException(FileNotFoundException.class,
1152 AccessControlException.class,
1153 UnresolvedPathException.class);
1154 }
1155 }
1156
1157 /**
1158 * Get block location info about file
1159 *
1160 * getBlockLocations() returns a list of hostnames that store
1161 * data for a specific file region. It returns a set of hostnames
1162 * for every block within the indicated region.
1163 *
1164 * This function is very useful when writing code that considers
1165 * data-placement when performing operations. For example, the
1166 * MapReduce system tries to schedule tasks on the same machines
1167 * as the data-block the task processes.
1168 */
1169 public BlockLocation[] getBlockLocations(String src, long start,
1170 long length) throws IOException, UnresolvedLinkException {
1171 LocatedBlocks blocks = getLocatedBlocks(src, start, length);
1172 BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks);
1173 HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
1174 for (int i = 0; i < locations.length; i++) {
1175 hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
1176 }
1177 return hdfsLocations;
1178 }
1179
1180 /**
1181 * Get block location information about a list of {@link HdfsBlockLocation}.
1182 * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to
1183 * get {@link BlockStorageLocation}s for blocks returned by
1184 * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)}
1185 * .
1186 *
1187 * This is done by making a round of RPCs to the associated datanodes, asking
1188 * the volume of each block replica. The returned array of
1189 * {@link BlockStorageLocation} expose this information as a
1190 * {@link VolumeId}.
1191 *
1192 * @param blockLocations
1193 * target blocks on which to query volume location information
1194 * @return volumeBlockLocations original block array augmented with additional
1195 * volume location information for each replica.
1196 */
1197 public BlockStorageLocation[] getBlockStorageLocations(
1198 List<BlockLocation> blockLocations) throws IOException,
1199 UnsupportedOperationException, InvalidBlockTokenException {
1200 if (!getConf().getHdfsBlocksMetadataEnabled) {
1201 throw new UnsupportedOperationException("Datanode-side support for " +
1202 "getVolumeBlockLocations() must also be enabled in the client " +
1203 "configuration.");
1204 }
1205 // Downcast blockLocations and fetch out required LocatedBlock(s)
1206 List<LocatedBlock> blocks = new ArrayList<LocatedBlock>();
1207 for (BlockLocation loc : blockLocations) {
1208 if (!(loc instanceof HdfsBlockLocation)) {
1209 throw new ClassCastException("DFSClient#getVolumeBlockLocations " +
1210 "expected to be passed HdfsBlockLocations");
1211 }
1212 HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc;
1213 blocks.add(hdfsLoc.getLocatedBlock());
1214 }
1215
1216 // Re-group the LocatedBlocks to be grouped by datanodes, with the values
1217 // a list of the LocatedBlocks on the datanode.
1218 Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks =
1219 new LinkedHashMap<DatanodeInfo, List<LocatedBlock>>();
1220 for (LocatedBlock b : blocks) {
1221 for (DatanodeInfo info : b.getLocations()) {
1222 if (!datanodeBlocks.containsKey(info)) {
1223 datanodeBlocks.put(info, new ArrayList<LocatedBlock>());
1224 }
1225 List<LocatedBlock> l = datanodeBlocks.get(info);
1226 l.add(b);
1227 }
1228 }
1229
1230 // Make RPCs to the datanodes to get volume locations for its replicas
1231 List<HdfsBlocksMetadata> metadatas = BlockStorageLocationUtil
1232 .queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
1233 getConf().getFileBlockStorageLocationsNumThreads,
1234 getConf().getFileBlockStorageLocationsTimeout,
1235 getConf().connectToDnViaHostname);
1236
1237 // Regroup the returned VolumeId metadata to again be grouped by
1238 // LocatedBlock rather than by datanode
1239 Map<LocatedBlock, List<VolumeId>> blockVolumeIds = BlockStorageLocationUtil
1240 .associateVolumeIdsWithBlocks(blocks, datanodeBlocks, metadatas);
1241
1242 // Combine original BlockLocations with new VolumeId information
1243 BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtil
1244 .convertToVolumeBlockLocations(blocks, blockVolumeIds);
1245
1246 return volumeBlockLocations;
1247 }
1248
1249 public DFSInputStream open(String src)
1250 throws IOException, UnresolvedLinkException {
1251 return open(src, dfsClientConf.ioBufferSize, true, null);
1252 }
1253
1254 /**
1255 * Create an input stream that obtains a nodelist from the
1256 * namenode, and then reads from all the right places. Creates
1257 * inner subclass of InputStream that does the right out-of-band
1258 * work.
1259 * @deprecated Use {@link #open(String, int, boolean)} instead.
1260 */
1261 @Deprecated
1262 public DFSInputStream open(String src, int buffersize, boolean verifyChecksum,
1263 FileSystem.Statistics stats)
1264 throws IOException, UnresolvedLinkException {
1265 return open(src, buffersize, verifyChecksum);
1266 }
1267
1268
1269 /**
1270 * Create an input stream that obtains a nodelist from the
1271 * namenode, and then reads from all the right places. Creates
1272 * inner subclass of InputStream that does the right out-of-band
1273 * work.
1274 */
1275 public DFSInputStream open(String src, int buffersize, boolean verifyChecksum)
1276 throws IOException, UnresolvedLinkException {
1277 checkOpen();
1278 // Get block info from namenode
1279 return new DFSInputStream(this, src, buffersize, verifyChecksum);
1280 }
1281
1282 /**
1283 * Get the namenode associated with this DFSClient object
1284 * @return the namenode associated with this DFSClient object
1285 */
1286 public ClientProtocol getNamenode() {
1287 return namenode;
1288 }
1289
1290 /**
1291 * Call {@link #create(String, boolean, short, long, Progressable)} with
1292 * default <code>replication</code> and <code>blockSize<code> and null <code>
1293 * progress</code>.
1294 */
1295 public OutputStream create(String src, boolean overwrite)
1296 throws IOException {
1297 return create(src, overwrite, dfsClientConf.defaultReplication,
1298 dfsClientConf.defaultBlockSize, null);
1299 }
1300
1301 /**
1302 * Call {@link #create(String, boolean, short, long, Progressable)} with
1303 * default <code>replication</code> and <code>blockSize<code>.
1304 */
1305 public OutputStream create(String src,
1306 boolean overwrite,
1307 Progressable progress) throws IOException {
1308 return create(src, overwrite, dfsClientConf.defaultReplication,
1309 dfsClientConf.defaultBlockSize, progress);
1310 }
1311
1312 /**
1313 * Call {@link #create(String, boolean, short, long, Progressable)} with
1314 * null <code>progress</code>.
1315 */
1316 public OutputStream create(String src,
1317 boolean overwrite,
1318 short replication,
1319 long blockSize) throws IOException {
1320 return create(src, overwrite, replication, blockSize, null);
1321 }
1322
1323 /**
1324 * Call {@link #create(String, boolean, short, long, Progressable, int)}
1325 * with default bufferSize.
1326 */
1327 public OutputStream create(String src, boolean overwrite, short replication,
1328 long blockSize, Progressable progress) throws IOException {
1329 return create(src, overwrite, replication, blockSize, progress,
1330 dfsClientConf.ioBufferSize);
1331 }
1332
1333 /**
1334 * Call {@link #create(String, FsPermission, EnumSet, short, long,
1335 * Progressable, int, ChecksumOpt)} with default <code>permission</code>
1336 * {@link FsPermission#getFileDefault()}.
1337 *
1338 * @param src File name
1339 * @param overwrite overwrite an existing file if true
1340 * @param replication replication factor for the file
1341 * @param blockSize maximum block size
1342 * @param progress interface for reporting client progress
1343 * @param buffersize underlying buffersize
1344 *
1345 * @return output stream
1346 */
1347 public OutputStream create(String src,
1348 boolean overwrite,
1349 short replication,
1350 long blockSize,
1351 Progressable progress,
1352 int buffersize)
1353 throws IOException {
1354 return create(src, FsPermission.getFileDefault(),
1355 overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1356 : EnumSet.of(CreateFlag.CREATE), replication, blockSize, progress,
1357 buffersize, null);
1358 }
1359
1360 /**
1361 * Call {@link #create(String, FsPermission, EnumSet, boolean, short,
1362 * long, Progressable, int, ChecksumOpt)} with <code>createParent</code>
1363 * set to true.
1364 */
1365 public DFSOutputStream create(String src,
1366 FsPermission permission,
1367 EnumSet<CreateFlag> flag,
1368 short replication,
1369 long blockSize,
1370 Progressable progress,
1371 int buffersize,
1372 ChecksumOpt checksumOpt)
1373 throws IOException {
1374 return create(src, permission, flag, true,
1375 replication, blockSize, progress, buffersize, checksumOpt, null);
1376 }
1377
1378 /**
1379 * Create a new dfs file with the specified block replication
1380 * with write-progress reporting and return an output stream for writing
1381 * into the file.
1382 *
1383 * @param src File name
1384 * @param permission The permission of the directory being created.
1385 * If null, use default permission {@link FsPermission#getFileDefault()}
1386 * @param flag indicates create a new file or create/overwrite an
1387 * existing file or append to an existing file
1388 * @param createParent create missing parent directory if true
1389 * @param replication block replication
1390 * @param blockSize maximum block size
1391 * @param progress interface for reporting client progress
1392 * @param buffersize underlying buffer size
1393 * @param checksumOpt checksum options
1394 *
1395 * @return output stream
1396 *
1397 * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
1398 * boolean, short, long) for detailed description of exceptions thrown
1399 */
1400 public DFSOutputStream create(String src,
1401 FsPermission permission,
1402 EnumSet<CreateFlag> flag,
1403 boolean createParent,
1404 short replication,
1405 long blockSize,
1406 Progressable progress,
1407 int buffersize,
1408 ChecksumOpt checksumOpt) throws IOException {
1409 return create(src, permission, flag, createParent, replication, blockSize,
1410 progress, buffersize, checksumOpt, null);
1411 }
1412
1413 /**
1414 * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long,
1415 * Progressable, int, ChecksumOpt)} with the addition of favoredNodes that is
1416 * a hint to where the namenode should place the file blocks.
1417 * The favored nodes hint is not persisted in HDFS. Hence it may be honored
1418 * at the creation time only. HDFS could move the blocks during balancing or
1419 * replication, to move the blocks from favored nodes. A value of null means
1420 * no favored nodes for this create
1421 */
1422 public DFSOutputStream create(String src,
1423 FsPermission permission,
1424 EnumSet<CreateFlag> flag,
1425 boolean createParent,
1426 short replication,
1427 long blockSize,
1428 Progressable progress,
1429 int buffersize,
1430 ChecksumOpt checksumOpt,
1431 InetSocketAddress[] favoredNodes) throws IOException {
1432 checkOpen();
1433 if (permission == null) {
1434 permission = FsPermission.getFileDefault();
1435 }
1436 FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
1437 if(LOG.isDebugEnabled()) {
1438 LOG.debug(src + ": masked=" + masked);
1439 }
1440 String[] favoredNodeStrs = null;
1441 if (favoredNodes != null) {
1442 favoredNodeStrs = new String[favoredNodes.length];
1443 for (int i = 0; i < favoredNodes.length; i++) {
1444 favoredNodeStrs[i] =
1445 favoredNodes[i].getHostName() + ":"
1446 + favoredNodes[i].getPort();
1447 }
1448 }
1449 final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
1450 src, masked, flag, createParent, replication, blockSize, progress,
1451 buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs);
1452 beginFileLease(src, result);
1453 return result;
1454 }
1455
1456 /**
1457 * Append to an existing file if {@link CreateFlag#APPEND} is present
1458 */
1459 private DFSOutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
1460 int buffersize, Progressable progress) throws IOException {
1461 if (flag.contains(CreateFlag.APPEND)) {
1462 HdfsFileStatus stat = getFileInfo(src);
1463 if (stat == null) { // No file to append to
1464 // New file needs to be created if create option is present
1465 if (!flag.contains(CreateFlag.CREATE)) {
1466 throw new FileNotFoundException("failed to append to non-existent file "
1467 + src + " on client " + clientName);
1468 }
1469 return null;
1470 }
1471 return callAppend(stat, src, buffersize, progress);
1472 }
1473 return null;
1474 }
1475
1476 /**
1477 * Same as {{@link #create(String, FsPermission, EnumSet, short, long,
1478 * Progressable, int, ChecksumOpt)} except that the permission
1479 * is absolute (ie has already been masked with umask.
1480 */
1481 public DFSOutputStream primitiveCreate(String src,
1482 FsPermission absPermission,
1483 EnumSet<CreateFlag> flag,
1484 boolean createParent,
1485 short replication,
1486 long blockSize,
1487 Progressable progress,
1488 int buffersize,
1489 ChecksumOpt checksumOpt)
1490 throws IOException, UnresolvedLinkException {
1491 checkOpen();
1492 CreateFlag.validate(flag);
1493 DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
1494 if (result == null) {
1495 DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
1496 result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
1497 flag, createParent, replication, blockSize, progress, buffersize,
1498 checksum);
1499 }
1500 beginFileLease(src, result);
1501 return result;
1502 }
1503
1504 /**
1505 * Creates a symbolic link.
1506 *
1507 * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean)
1508 */
1509 public void createSymlink(String target, String link, boolean createParent)
1510 throws IOException {
1511 try {
1512 FsPermission dirPerm =
1513 FsPermission.getDefault().applyUMask(dfsClientConf.uMask);
1514 namenode.createSymlink(target, link, dirPerm, createParent);
1515 } catch (RemoteException re) {
1516 throw re.unwrapRemoteException(AccessControlException.class,
1517 FileAlreadyExistsException.class,
1518 FileNotFoundException.class,
1519 ParentNotDirectoryException.class,
1520 NSQuotaExceededException.class,
1521 DSQuotaExceededException.class,
1522 UnresolvedPathException.class,
1523 SnapshotAccessControlException.class);
1524 }
1525 }
1526
1527 /**
1528 * Resolve the *first* symlink, if any, in the path.
1529 *
1530 * @see ClientProtocol#getLinkTarget(String)
1531 */
1532 public String getLinkTarget(String path) throws IOException {
1533 checkOpen();
1534 try {
1535 return namenode.getLinkTarget(path);
1536 } catch (RemoteException re) {
1537 throw re.unwrapRemoteException(AccessControlException.class,
1538 FileNotFoundException.class);
1539 }
1540 }
1541
1542 /** Method to get stream returned by append call */
1543 private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
1544 int buffersize, Progressable progress) throws IOException {
1545 LocatedBlock lastBlock = null;
1546 try {
1547 lastBlock = namenode.append(src, clientName);
1548 } catch(RemoteException re) {
1549 throw re.unwrapRemoteException(AccessControlException.class,
1550 FileNotFoundException.class,
1551 SafeModeException.class,
1552 DSQuotaExceededException.class,
1553 UnsupportedOperationException.class,
1554 UnresolvedPathException.class,
1555 SnapshotAccessControlException.class);
1556 }
1557 return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
1558 lastBlock, stat, dfsClientConf.createChecksum());
1559 }
1560
1561 /**
1562 * Append to an existing HDFS file.
1563 *
1564 * @param src file name
1565 * @param buffersize buffer size
1566 * @param progress for reporting write-progress; null is acceptable.
1567 * @param statistics file system statistics; null is acceptable.
1568 * @return an output stream for writing into the file
1569 *
1570 * @see ClientProtocol#append(String, String)
1571 */
1572 public HdfsDataOutputStream append(final String src, final int buffersize,
1573 final Progressable progress, final FileSystem.Statistics statistics
1574 ) throws IOException {
1575 final DFSOutputStream out = append(src, buffersize, progress);
1576 return new HdfsDataOutputStream(out, statistics, out.getInitialLen());
1577 }
1578
1579 private DFSOutputStream append(String src, int buffersize, Progressable progress)
1580 throws IOException {
1581 checkOpen();
1582 HdfsFileStatus stat = getFileInfo(src);
1583 if (stat == null) { // No file found
1584 throw new FileNotFoundException("failed to append to non-existent file "
1585 + src + " on client " + clientName);
1586 }
1587 final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
1588 beginFileLease(src, result);
1589 return result;
1590 }
1591
1592 /**
1593 * Set replication for an existing file.
1594 * @param src file name
1595 * @param replication
1596 *
1597 * @see ClientProtocol#setReplication(String, short)
1598 */
1599 public boolean setReplication(String src, short replication)
1600 throws IOException {
1601 try {
1602 return namenode.setReplication(src, replication);
1603 } catch(RemoteException re) {
1604 throw re.unwrapRemoteException(AccessControlException.class,
1605 FileNotFoundException.class,
1606 SafeModeException.class,
1607 DSQuotaExceededException.class,
1608 UnresolvedPathException.class,
1609 SnapshotAccessControlException.class);
1610 }
1611 }
1612
1613 /**
1614 * Rename file or directory.
1615 * @see ClientProtocol#rename(String, String)
1616 * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
1617 */
1618 @Deprecated
1619 public boolean rename(String src, String dst) throws IOException {
1620 checkOpen();
1621 try {
1622 return namenode.rename(src, dst);
1623 } catch(RemoteException re) {
1624 throw re.unwrapRemoteException(AccessControlException.class,
1625 NSQuotaExceededException.class,
1626 DSQuotaExceededException.class,
1627 UnresolvedPathException.class,
1628 SnapshotAccessControlException.class);
1629 }
1630 }
1631
1632 /**
1633 * Move blocks from src to trg and delete src
1634 * See {@link ClientProtocol#concat(String, String [])}.
1635 */
1636 public void concat(String trg, String [] srcs) throws IOException {
1637 checkOpen();
1638 try {
1639 namenode.concat(trg, srcs);
1640 } catch(RemoteException re) {
1641 throw re.unwrapRemoteException(AccessControlException.class,
1642 UnresolvedPathException.class,
1643 SnapshotAccessControlException.class);
1644 }
1645 }
1646 /**
1647 * Rename file or directory.
1648 * @see ClientProtocol#rename2(String, String, Options.Rename...)
1649 */
1650 public void rename(String src, String dst, Options.Rename... options)
1651 throws IOException {
1652 checkOpen();
1653 try {
1654 namenode.rename2(src, dst, options);
1655 } catch(RemoteException re) {
1656 throw re.unwrapRemoteException(AccessControlException.class,
1657 DSQuotaExceededException.class,
1658 FileAlreadyExistsException.class,
1659 FileNotFoundException.class,
1660 ParentNotDirectoryException.class,
1661 SafeModeException.class,
1662 NSQuotaExceededException.class,
1663 UnresolvedPathException.class,
1664 SnapshotAccessControlException.class);
1665 }
1666 }
1667 /**
1668 * Delete file or directory.
1669 * See {@link ClientProtocol#delete(String, boolean)}.
1670 */
1671 @Deprecated
1672 public boolean delete(String src) throws IOException {
1673 checkOpen();
1674 return namenode.delete(src, true);
1675 }
1676
1677 /**
1678 * delete file or directory.
1679 * delete contents of the directory if non empty and recursive
1680 * set to true
1681 *
1682 * @see ClientProtocol#delete(String, boolean)
1683 */
1684 public boolean delete(String src, boolean recursive) throws IOException {
1685 checkOpen();
1686 try {
1687 return namenode.delete(src, recursive);
1688 } catch(RemoteException re) {
1689 throw re.unwrapRemoteException(AccessControlException.class,
1690 FileNotFoundException.class,
1691 SafeModeException.class,
1692 UnresolvedPathException.class,
1693 SnapshotAccessControlException.class);
1694 }
1695 }
1696
1697 /** Implemented using getFileInfo(src)
1698 */
1699 public boolean exists(String src) throws IOException {
1700 checkOpen();
1701 return getFileInfo(src) != null;
1702 }
1703
1704 /**
1705 * Get a partial listing of the indicated directory
1706 * No block locations need to be fetched
1707 */
1708 public DirectoryListing listPaths(String src, byte[] startAfter)
1709 throws IOException {
1710 return listPaths(src, startAfter, false);
1711 }
1712
1713 /**
1714 * Get a partial listing of the indicated directory
1715 *
1716 * Recommend to use HdfsFileStatus.EMPTY_NAME as startAfter
1717 * if the application wants to fetch a listing starting from
1718 * the first entry in the directory
1719 *
1720 * @see ClientProtocol#getListing(String, byte[], boolean)
1721 */
1722 public DirectoryListing listPaths(String src, byte[] startAfter,
1723 boolean needLocation)
1724 throws IOException {
1725 checkOpen();
1726 try {
1727 return namenode.getListing(src, startAfter, needLocation);
1728 } catch(RemoteException re) {
1729 throw re.unwrapRemoteException(AccessControlException.class,
1730 FileNotFoundException.class,
1731 UnresolvedPathException.class);
1732 }
1733 }
1734
1735 /**
1736 * Get the file info for a specific file or directory.
1737 * @param src The string representation of the path to the file
1738 * @return object containing information regarding the file
1739 * or null if file not found
1740 *
1741 * @see ClientProtocol#getFileInfo(String) for description of exceptions
1742 */
1743 public HdfsFileStatus getFileInfo(String src) throws IOException {
1744 checkOpen();
1745 try {
1746 return namenode.getFileInfo(src);
1747 } catch(RemoteException re) {
1748 throw re.unwrapRemoteException(AccessControlException.class,
1749 FileNotFoundException.class,
1750 UnresolvedPathException.class);
1751 }
1752 }
1753
1754 /**
1755 * Close status of a file
1756 * @return true if file is already closed
1757 */
1758 public boolean isFileClosed(String src) throws IOException{
1759 checkOpen();
1760 try {
1761 return namenode.isFileClosed(src);
1762 } catch(RemoteException re) {
1763 throw re.unwrapRemoteException(AccessControlException.class,
1764 FileNotFoundException.class,
1765 UnresolvedPathException.class);
1766 }
1767 }
1768
1769 /**
1770 * Get the file info for a specific file or directory. If src
1771 * refers to a symlink then the FileStatus of the link is returned.
1772 * @param src path to a file or directory.
1773 *
1774 * For description of exceptions thrown
1775 * @see ClientProtocol#getFileLinkInfo(String)
1776 */
1777 public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
1778 checkOpen();
1779 try {
1780 return namenode.getFileLinkInfo(src);
1781 } catch(RemoteException re) {
1782 throw re.unwrapRemoteException(AccessControlException.class,
1783 UnresolvedPathException.class);
1784 }
1785 }
1786
1787 /**
1788 * Get the checksum of a file.
1789 * @param src The file path
1790 * @return The checksum
1791 * @see DistributedFileSystem#getFileChecksum(Path)
1792 */
1793 public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
1794 checkOpen();
1795 return getFileChecksum(src, clientName, namenode, socketFactory,
1796 dfsClientConf.socketTimeout, getDataEncryptionKey(),
1797 dfsClientConf.connectToDnViaHostname);
1798 }
1799
1800 @InterfaceAudience.Private
1801 public void clearDataEncryptionKey() {
1802 LOG.debug("Clearing encryption key");
1803 synchronized (this) {
1804 encryptionKey = null;
1805 }
1806 }
1807
1808 /**
1809 * @return true if data sent between this client and DNs should be encrypted,
1810 * false otherwise.
1811 * @throws IOException in the event of error communicating with the NN
1812 */
1813 boolean shouldEncryptData() throws IOException {
1814 FsServerDefaults d = getServerDefaults();
1815 return d == null ? false : d.getEncryptDataTransfer();
1816 }
1817
1818 @InterfaceAudience.Private
1819 public DataEncryptionKey getDataEncryptionKey()
1820 throws IOException {
1821 if (shouldEncryptData()) {
1822 synchronized (this) {
1823 if (encryptionKey == null ||
1824 encryptionKey.expiryDate < Time.now()) {
1825 LOG.debug("Getting new encryption token from NN");
1826 encryptionKey = namenode.getDataEncryptionKey();
1827 }
1828 return encryptionKey;
1829 }
1830 } else {
1831 return null;
1832 }
1833 }
1834
1835 /**
1836 * Get the checksum of a file.
1837 * @param src The file path
1838 * @param clientName the name of the client requesting the checksum.
1839 * @param namenode the RPC proxy for the namenode
1840 * @param socketFactory to create sockets to connect to DNs
1841 * @param socketTimeout timeout to use when connecting and waiting for a response
1842 * @param encryptionKey the key needed to communicate with DNs in this cluster
1843 * @param connectToDnViaHostname whether the client should use hostnames instead of IPs
1844 * @return The checksum
1845 */
1846 private static MD5MD5CRC32FileChecksum getFileChecksum(String src,
1847 String clientName,
1848 ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
1849 DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
1850 throws IOException {
1851 //get all block locations
1852 LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1853 if (null == blockLocations) {
1854 throw new FileNotFoundException("File does not exist: " + src);
1855 }
1856 List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
1857 final DataOutputBuffer md5out = new DataOutputBuffer();
1858 int bytesPerCRC = -1;
1859 DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
1860 long crcPerBlock = 0;
1861 boolean refetchBlocks = false;
1862 int lastRetriedIndex = -1;
1863
1864 //get block checksum for each block
1865 for(int i = 0; i < locatedblocks.size(); i++) {
1866 if (refetchBlocks) { // refetch to get fresh tokens
1867 blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1868 if (null == blockLocations) {
1869 throw new FileNotFoundException("File does not exist: " + src);
1870 }
1871 locatedblocks = blockLocations.getLocatedBlocks();
1872 refetchBlocks = false;
1873 }
1874 LocatedBlock lb = locatedblocks.get(i);
1875 final ExtendedBlock block = lb.getBlock();
1876 final DatanodeInfo[] datanodes = lb.getLocations();
1877
1878 //try each datanode location of the block
1879 final int timeout = 3000 * datanodes.length + socketTimeout;
1880 boolean done = false;
1881 for(int j = 0; !done && j < datanodes.length; j++) {
1882 DataOutputStream out = null;
1883 DataInputStream in = null;
1884
1885 try {
1886 //connect to a datanode
1887 IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
1888 encryptionKey, datanodes[j], timeout);
1889 out = new DataOutputStream(new BufferedOutputStream(pair.out,
1890 HdfsConstants.SMALL_BUFFER_SIZE));
1891 in = new DataInputStream(pair.in);
1892
1893 if (LOG.isDebugEnabled()) {
1894 LOG.debug("write to " + datanodes[j] + ": "
1895 + Op.BLOCK_CHECKSUM + ", block=" + block);
1896 }
1897 // get block MD5
1898 new Sender(out).blockChecksum(block, lb.getBlockToken());
1899
1900 final BlockOpResponseProto reply =
1901 BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
1902
1903 if (reply.getStatus() != Status.SUCCESS) {
1904 if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
1905 throw new InvalidBlockTokenException();
1906 } else {
1907 throw new IOException("Bad response " + reply + " for block "
1908 + block + " from datanode " + datanodes[j]);
1909 }
1910 }
1911
1912 OpBlockChecksumResponseProto checksumData =
1913 reply.getChecksumResponse();
1914
1915 //read byte-per-checksum
1916 final int bpc = checksumData.getBytesPerCrc();
1917 if (i == 0) { //first block
1918 bytesPerCRC = bpc;
1919 }
1920 else if (bpc != bytesPerCRC) {
1921 throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
1922 + " but bytesPerCRC=" + bytesPerCRC);
1923 }
1924
1925 //read crc-per-block
1926 final long cpb = checksumData.getCrcPerBlock();
1927 if (locatedblocks.size() > 1 && i == 0) {
1928 crcPerBlock = cpb;
1929 }
1930
1931 //read md5
1932 final MD5Hash md5 = new MD5Hash(
1933 checksumData.getMd5().toByteArray());
1934 md5.write(md5out);
1935
1936 // read crc-type
1937 final DataChecksum.Type ct;
1938 if (checksumData.hasCrcType()) {
1939 ct = PBHelper.convert(checksumData
1940 .getCrcType());
1941 } else {
1942 LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
1943 "inferring checksum by reading first byte");
1944 ct = inferChecksumTypeByReading(
1945 clientName, socketFactory, socketTimeout, lb, datanodes[j],
1946 encryptionKey, connectToDnViaHostname);
1947 }
1948
1949 if (i == 0) { // first block
1950 crcType = ct;
1951 } else if (crcType != DataChecksum.Type.MIXED
1952 && crcType != ct) {
1953 // if crc types are mixed in a file
1954 crcType = DataChecksum.Type.MIXED;
1955 }
1956
1957 done = true;
1958
1959 if (LOG.isDebugEnabled()) {
1960 if (i == 0) {
1961 LOG.debug("set bytesPerCRC=" + bytesPerCRC
1962 + ", crcPerBlock=" + crcPerBlock);
1963 }
1964 LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
1965 }
1966 } catch (InvalidBlockTokenException ibte) {
1967 if (i > lastRetriedIndex) {
1968 if (LOG.isDebugEnabled()) {
1969 LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
1970 + "for file " + src + " for block " + block
1971 + " from datanode " + datanodes[j]
1972 + ". Will retry the block once.");
1973 }
1974 lastRetriedIndex = i;
1975 done = true; // actually it's not done; but we'll retry
1976 i--; // repeat at i-th block
1977 refetchBlocks = true;
1978 break;
1979 }
1980 } catch (IOException ie) {
1981 LOG.warn("src=" + src + ", datanodes["+j+"]=" + datanodes[j], ie);
1982 } finally {
1983 IOUtils.closeStream(in);
1984 IOUtils.closeStream(out);
1985 }
1986 }
1987
1988 if (!done) {
1989 throw new IOException("Fail to get block MD5 for " + block);
1990 }
1991 }
1992
1993 //compute file MD5
1994 final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData());
1995 switch (crcType) {
1996 case CRC32:
1997 return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
1998 crcPerBlock, fileMD5);
1999 case CRC32C:
2000 return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC,
2001 crcPerBlock, fileMD5);
2002 default:
2003 // If there is no block allocated for the file,
2004 // return one with the magic entry that matches what previous
2005 // hdfs versions return.
2006 if (locatedblocks.size() == 0) {
2007 return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
2008 }
2009
2010 // we should never get here since the validity was checked
2011 // when getCrcType() was called above.
2012 return null;
2013 }
2014 }
2015
2016 /**
2017 * Connect to the given datanode's datantrasfer port, and return
2018 * the resulting IOStreamPair. This includes encryption wrapping, etc.
2019 */
2020 private static IOStreamPair connectToDN(
2021 SocketFactory socketFactory, boolean connectToDnViaHostname,
2022 DataEncryptionKey encryptionKey, DatanodeInfo dn, int timeout)
2023 throws IOException
2024 {
2025 boolean success = false;
2026 Socket sock = null;
2027 try {
2028 sock = socketFactory.createSocket();
2029 String dnAddr = dn.getXferAddr(connectToDnViaHostname);
2030 if (LOG.isDebugEnabled()) {
2031 LOG.debug("Connecting to datanode " + dnAddr);
2032 }
2033 NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
2034 sock.setSoTimeout(timeout);
2035
2036 OutputStream unbufOut = NetUtils.getOutputStream(sock);
2037 InputStream unbufIn = NetUtils.getInputStream(sock);
2038 IOStreamPair ret;
2039 if (encryptionKey != null) {
2040 ret = DataTransferEncryptor.getEncryptedStreams(
2041 unbufOut, unbufIn, encryptionKey);
2042 } else {
2043 ret = new IOStreamPair(unbufIn, unbufOut);
2044 }
2045 success = true;
2046 return ret;
2047 } finally {
2048 if (!success) {
2049 IOUtils.closeSocket(sock);
2050 }
2051 }
2052 }
2053
2054 /**
2055 * Infer the checksum type for a replica by sending an OP_READ_BLOCK
2056 * for the first byte of that replica. This is used for compatibility
2057 * with older HDFS versions which did not include the checksum type in
2058 * OpBlockChecksumResponseProto.
2059 *
2060 * @param in input stream from datanode
2061 * @param out output stream to datanode
2062 * @param lb the located block
2063 * @param clientName the name of the DFSClient requesting the checksum
2064 * @param dn the connected datanode
2065 * @return the inferred checksum type
2066 * @throws IOException if an error occurs
2067 */
2068 private static Type inferChecksumTypeByReading(
2069 String clientName, SocketFactory socketFactory, int socketTimeout,
2070 LocatedBlock lb, DatanodeInfo dn,
2071 DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
2072 throws IOException {
2073 IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
2074 encryptionKey, dn, socketTimeout);
2075
2076 try {
2077 DataOutputStream out = new DataOutputStream(new BufferedOutputStream(pair.out,
2078 HdfsConstants.SMALL_BUFFER_SIZE));
2079 DataInputStream in = new DataInputStream(pair.in);
2080
2081 new Sender(out).readBlock(lb.getBlock(), lb.getBlockToken(), clientName,
2082 0, 1, true, CachingStrategy.newDefaultStrategy());
2083 final BlockOpResponseProto reply =
2084 BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
2085
2086 if (reply.getStatus() != Status.SUCCESS) {
2087 if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
2088 throw new InvalidBlockTokenException();
2089 } else {
2090 throw new IOException("Bad response " + reply + " trying to read "
2091 + lb.getBlock() + " from datanode " + dn);
2092 }
2093 }
2094
2095 return PBHelper.convert(reply.getReadOpChecksumInfo().getChecksum().getType());
2096 } finally {
2097 IOUtils.cleanup(null, pair.in, pair.out);
2098 }
2099 }
2100
2101 /**
2102 * Set permissions to a file or directory.
2103 * @param src path name.
2104 * @param permission
2105 *
2106 * @see ClientProtocol#setPermission(String, FsPermission)
2107 */
2108 public void setPermission(String src, FsPermission permission)
2109 throws IOException {
2110 checkOpen();
2111 try {
2112 namenode.setPermission(src, permission);
2113 } catch(RemoteException re) {
2114 throw re.unwrapRemoteException(AccessControlException.class,
2115 FileNotFoundException.class,
2116 SafeModeException.class,
2117 UnresolvedPathException.class,
2118 SnapshotAccessControlException.class);
2119 }
2120 }
2121
2122 /**
2123 * Set file or directory owner.
2124 * @param src path name.
2125 * @param username user id.
2126 * @param groupname user group.
2127 *
2128 * @see ClientProtocol#setOwner(String, String, String)
2129 */
2130 public void setOwner(String src, String username, String groupname)
2131 throws IOException {
2132 checkOpen();
2133 try {
2134 namenode.setOwner(src, username, groupname);
2135 } catch(RemoteException re) {
2136 throw re.unwrapRemoteException(AccessControlException.class,
2137 FileNotFoundException.class,
2138 SafeModeException.class,
2139 UnresolvedPathException.class,
2140 SnapshotAccessControlException.class);
2141 }
2142 }
2143
2144 /**
2145 * @see ClientProtocol#getStats()
2146 */
2147 public FsStatus getDiskStatus() throws IOException {
2148 long rawNums[] = namenode.getStats();
2149 return new FsStatus(rawNums[0], rawNums[1], rawNums[2]);
2150 }
2151
2152 /**
2153 * Returns count of blocks with no good replicas left. Normally should be
2154 * zero.
2155 * @throws IOException
2156 */
2157 public long getMissingBlocksCount() throws IOException {
2158 return namenode.getStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
2159 }
2160
2161 /**
2162 * Returns count of blocks with one of more replica missing.
2163 * @throws IOException
2164 */
2165 public long getUnderReplicatedBlocksCount() throws IOException {
2166 return namenode.getStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
2167 }
2168
2169 /**
2170 * Returns count of blocks with at least one replica marked corrupt.
2171 * @throws IOException
2172 */
2173 public long getCorruptBlocksCount() throws IOException {
2174 return namenode.getStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
2175 }
2176
2177 /**
2178 * @return a list in which each entry describes a corrupt file/block
2179 * @throws IOException
2180 */
2181 public CorruptFileBlocks listCorruptFileBlocks(String path,
2182 String cookie)
2183 throws IOException {
2184 return namenode.listCorruptFileBlocks(path, cookie);
2185 }
2186
2187 public DatanodeInfo[] datanodeReport(DatanodeReportType type)
2188 throws IOException {
2189 return namenode.getDatanodeReport(type);
2190 }
2191
2192 /**
2193 * Enter, leave or get safe mode.
2194 *
2195 * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction,boolean)
2196 */
2197 public boolean setSafeMode(SafeModeAction action) throws IOException {
2198 return setSafeMode(action, false);
2199 }
2200
2201 /**
2202 * Enter, leave or get safe mode.
2203 *
2204 * @param action
2205 * One of SafeModeAction.GET, SafeModeAction.ENTER and
2206 * SafeModeActiob.LEAVE
2207 * @param isChecked
2208 * If true, then check only active namenode's safemode status, else
2209 * check first namenode's status.
2210 * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction, boolean)
2211 */
2212 public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException{
2213 return namenode.setSafeMode(action, isChecked);
2214 }
2215
2216 /**
2217 * Create one snapshot.
2218 *
2219 * @param snapshotRoot The directory where the snapshot is to be taken
2220 * @param snapshotName Name of the snapshot
2221 * @return the snapshot path.
2222 * @see ClientProtocol#createSnapshot(String, String)
2223 */
2224 public String createSnapshot(String snapshotRoot, String snapshotName)
2225 throws IOException {
2226 checkOpen();
2227 try {
2228 return namenode.createSnapshot(snapshotRoot, snapshotName);
2229 } catch(RemoteException re) {
2230 throw re.unwrapRemoteException();
2231 }
2232 }
2233
2234 /**
2235 * Delete a snapshot of a snapshottable directory.
2236 *
2237 * @param snapshotRoot The snapshottable directory that the
2238 * to-be-deleted snapshot belongs to
2239 * @param snapshotName The name of the to-be-deleted snapshot
2240 * @throws IOException
2241 * @see ClientProtocol#deleteSnapshot(String, String)
2242 */
2243 public void deleteSnapshot(String snapshotRoot, String snapshotName)
2244 throws IOException {
2245 try {
2246 namenode.deleteSnapshot(snapshotRoot, snapshotName);
2247 } catch(RemoteException re) {
2248 throw re.unwrapRemoteException();
2249 }
2250 }
2251
2252 /**
2253 * Rename a snapshot.
2254 * @param snapshotDir The directory path where the snapshot was taken
2255 * @param snapshotOldName Old name of the snapshot
2256 * @param snapshotNewName New name of the snapshot
2257 * @throws IOException
2258 * @see ClientProtocol#renameSnapshot(String, String, String)
2259 */
2260 public void renameSnapshot(String snapshotDir, String snapshotOldName,
2261 String snapshotNewName) throws IOException {
2262 checkOpen();
2263 try {
2264 namenode.renameSnapshot(snapshotDir, snapshotOldName, snapshotNewName);
2265 } catch(RemoteException re) {
2266 throw re.unwrapRemoteException();
2267 }
2268 }
2269
2270 /**
2271 * Get all the current snapshottable directories.
2272 * @return All the current snapshottable directories
2273 * @throws IOException
2274 * @see ClientProtocol#getSnapshottableDirListing()
2275 */
2276 public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
2277 throws IOException {
2278 checkOpen();
2279 try {
2280 return namenode.getSnapshottableDirListing();
2281 } catch(RemoteException re) {
2282 throw re.unwrapRemoteException();
2283 }
2284 }
2285
2286 /**
2287 * Allow snapshot on a directory.
2288 *
2289 * @see ClientProtocol#allowSnapshot(String snapshotRoot)
2290 */
2291 public void allowSnapshot(String snapshotRoot) throws IOException {
2292 checkOpen();
2293 try {
2294 namenode.allowSnapshot(snapshotRoot);
2295 } catch (RemoteException re) {
2296 throw re.unwrapRemoteException();
2297 }
2298 }
2299
2300 /**
2301 * Disallow snapshot on a directory.
2302 *
2303 * @see ClientProtocol#disallowSnapshot(String snapshotRoot)
2304 */
2305 public void disallowSnapshot(String snapshotRoot) throws IOException {
2306 checkOpen();
2307 try {
2308 namenode.disallowSnapshot(snapshotRoot);
2309 } catch (RemoteException re) {
2310 throw re.unwrapRemoteException();
2311 }
2312 }
2313
2314 /**
2315 * Get the difference between two snapshots, or between a snapshot and the
2316 * current tree of a directory.
2317 * @see ClientProtocol#getSnapshotDiffReport(String, String, String)
2318 */
2319 public SnapshotDiffReport getSnapshotDiffReport(String snapshotDir,
2320 String fromSnapshot, String toSnapshot) throws IOException {
2321 checkOpen();
2322 try {
2323 return namenode.getSnapshotDiffReport(snapshotDir,
2324 fromSnapshot, toSnapshot);
2325 } catch(RemoteException re) {
2326 throw re.unwrapRemoteException();
2327 }
2328 }
2329
2330 public long addCacheDirective(
2331 CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
2332 checkOpen();
2333 try {
2334 return namenode.addCacheDirective(info, flags);
2335 } catch (RemoteException re) {
2336 throw re.unwrapRemoteException();
2337 }
2338 }
2339
2340 public void modifyCacheDirective(
2341 CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
2342 checkOpen();
2343 try {
2344 namenode.modifyCacheDirective(info, flags);
2345 } catch (RemoteException re) {
2346 throw re.unwrapRemoteException();
2347 }
2348 }
2349
2350 public void removeCacheDirective(long id)
2351 throws IOException {
2352 checkOpen();
2353 try {
2354 namenode.removeCacheDirective(id);
2355 } catch (RemoteException re) {
2356 throw re.unwrapRemoteException();
2357 }
2358 }
2359
2360 public RemoteIterator<CacheDirectiveEntry> listCacheDirectives(
2361 CacheDirectiveInfo filter) throws IOException {
2362 return new CacheDirectiveIterator(namenode, filter);
2363 }
2364
2365 public void addCachePool(CachePoolInfo info) throws IOException {
2366 checkOpen();
2367 try {
2368 namenode.addCachePool(info);
2369 } catch (RemoteException re) {
2370 throw re.unwrapRemoteException();
2371 }
2372 }
2373
2374 public void modifyCachePool(CachePoolInfo info) throws IOException {
2375 checkOpen();
2376 try {
2377 namenode.modifyCachePool(info);
2378 } catch (RemoteException re) {
2379 throw re.unwrapRemoteException();
2380 }
2381 }
2382
2383 public void removeCachePool(String poolName) throws IOException {
2384 checkOpen();
2385 try {
2386 namenode.removeCachePool(poolName);
2387 } catch (RemoteException re) {
2388 throw re.unwrapRemoteException();
2389 }
2390 }
2391
2392 public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
2393 return new CachePoolIterator(namenode);
2394 }
2395
2396 /**
2397 * Save namespace image.
2398 *
2399 * @see ClientProtocol#saveNamespace()
2400 */
2401 void saveNamespace() throws AccessControlException, IOException {
2402 try {
2403 namenode.saveNamespace();
2404 } catch(RemoteException re) {
2405 throw re.unwrapRemoteException(AccessControlException.class);
2406 }
2407 }
2408
2409 /**
2410 * Rolls the edit log on the active NameNode.
2411 * @return the txid of the new log segment
2412 *
2413 * @see ClientProtocol#rollEdits()
2414 */
2415 long rollEdits() throws AccessControlException, IOException {
2416 try {
2417 return namenode.rollEdits();
2418 } catch(RemoteException re) {
2419 throw re.unwrapRemoteException(AccessControlException.class);
2420 }
2421 }
2422
2423 @VisibleForTesting
2424 ExtendedBlock getPreviousBlock(String file) {
2425 return filesBeingWritten.get(file).getBlock();
2426 }
2427
2428 /**
2429 * enable/disable restore failed storage.
2430 *
2431 * @see ClientProtocol#restoreFailedStorage(String arg)
2432 */
2433 boolean restoreFailedStorage(String arg)
2434 throws AccessControlException, IOException{
2435 return namenode.restoreFailedStorage(arg);
2436 }
2437
2438 /**
2439 * Refresh the hosts and exclude files. (Rereads them.)
2440 * See {@link ClientProtocol#refreshNodes()}
2441 * for more details.
2442 *
2443 * @see ClientProtocol#refreshNodes()
2444 */
2445 public void refreshNodes() throws IOException {
2446 namenode.refreshNodes();
2447 }
2448
2449 /**
2450 * Dumps DFS data structures into specified file.
2451 *
2452 * @see ClientProtocol#metaSave(String)
2453 */
2454 public void metaSave(String pathname) throws IOException {
2455 namenode.metaSave(pathname);
2456 }
2457
2458 /**
2459 * Requests the namenode to tell all datanodes to use a new, non-persistent
2460 * bandwidth value for dfs.balance.bandwidthPerSec.
2461 * See {@link ClientProtocol#setBalancerBandwidth(long)}
2462 * for more details.
2463 *
2464 * @see ClientProtocol#setBalancerBandwidth(long)
2465 */
2466 public void setBalancerBandwidth(long bandwidth) throws IOException {
2467 namenode.setBalancerBandwidth(bandwidth);
2468 }
2469
2470 /**
2471 * @see ClientProtocol#finalizeUpgrade()
2472 */
2473 public void finalizeUpgrade() throws IOException {
2474 namenode.finalizeUpgrade();
2475 }
2476
2477 /**
2478 */
2479 @Deprecated
2480 public boolean mkdirs(String src) throws IOException {
2481 return mkdirs(src, null, true);
2482 }
2483
2484 /**
2485 * Create a directory (or hierarchy of directories) with the given
2486 * name and permission.
2487 *
2488 * @param src The path of the directory being created
2489 * @param permission The permission of the directory being created.
2490 * If permission == null, use {@link FsPermission#getDefault()}.
2491 * @param createParent create missing parent directory if true
2492 *
2493 * @return True if the operation success.
2494 *
2495 * @see ClientProtocol#mkdirs(String, FsPermission, boolean)
2496 */
2497 public boolean mkdirs(String src, FsPermission permission,
2498 boolean createParent) throws IOException {
2499 if (permission == null) {
2500 permission = FsPermission.getDefault();
2501 }
2502 FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
2503 return primitiveMkdir(src, masked, createParent);
2504 }
2505
2506 /**
2507 * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2508 * that the permissions has already been masked against umask.
2509 */
2510 public boolean primitiveMkdir(String src, FsPermission absPermission)
2511 throws IOException {
2512 return primitiveMkdir(src, absPermission, true);
2513 }
2514
2515 /**
2516 * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2517 * that the permissions has already been masked against umask.
2518 */
2519 public boolean primitiveMkdir(String src, FsPermission absPermission,
2520 boolean createParent)
2521 throws IOException {
2522 checkOpen();
2523 if (absPermission == null) {
2524 absPermission =
2525 FsPermission.getDefault().applyUMask(dfsClientConf.uMask);
2526 }
2527
2528 if(LOG.isDebugEnabled()) {
2529 LOG.debug(src + ": masked=" + absPermission);
2530 }
2531 try {
2532 return namenode.mkdirs(src, absPermission, createParent);
2533 } catch(RemoteException re) {
2534 throw re.unwrapRemoteException(AccessControlException.class,
2535 InvalidPathException.class,
2536 FileAlreadyExistsException.class,
2537 FileNotFoundException.class,
2538 ParentNotDirectoryException.class,
2539 SafeModeException.class,
2540 NSQuotaExceededException.class,
2541 DSQuotaExceededException.class,
2542 UnresolvedPathException.class,
2543 SnapshotAccessControlException.class);
2544 }
2545 }
2546
2547 /**
2548 * Get {@link ContentSummary} rooted at the specified directory.
2549 * @param path The string representation of the path
2550 *
2551 * @see ClientProtocol#getContentSummary(String)
2552 */
2553 ContentSummary getContentSummary(String src) throws IOException {
2554 try {
2555 return namenode.getContentSummary(src);
2556 } catch(RemoteException re) {
2557 throw re.unwrapRemoteException(AccessControlException.class,
2558 FileNotFoundException.class,
2559 UnresolvedPathException.class);
2560 }
2561 }
2562
2563 /**
2564 * Sets or resets quotas for a directory.
2565 * @see ClientProtocol#setQuota(String, long, long)
2566 */
2567 void setQuota(String src, long namespaceQuota, long diskspaceQuota)
2568 throws IOException {
2569 // sanity check
2570 if ((namespaceQuota <= 0 && namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2571 namespaceQuota != HdfsConstants.QUOTA_RESET) ||
2572 (diskspaceQuota <= 0 && diskspaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2573 diskspaceQuota != HdfsConstants.QUOTA_RESET)) {
2574 throw new IllegalArgumentException("Invalid values for quota : " +
2575 namespaceQuota + " and " +
2576 diskspaceQuota);
2577
2578 }
2579 try {
2580 namenode.setQuota(src, namespaceQuota, diskspaceQuota);
2581 } catch(RemoteException re) {
2582 throw re.unwrapRemoteException(AccessControlException.class,
2583 FileNotFoundException.class,
2584 NSQuotaExceededException.class,
2585 DSQuotaExceededException.class,
2586 UnresolvedPathException.class,
2587 SnapshotAccessControlException.class);
2588 }
2589 }
2590
2591 /**
2592 * set the modification and access time of a file
2593 *
2594 * @see ClientProtocol#setTimes(String, long, long)
2595 */
2596 public void setTimes(String src, long mtime, long atime) throws IOException {
2597 checkOpen();
2598 try {
2599 namenode.setTimes(src, mtime, atime);
2600 } catch(RemoteException re) {
2601 throw re.unwrapRemoteException(AccessControlException.class,
2602 FileNotFoundException.class,
2603 UnresolvedPathException.class,
2604 SnapshotAccessControlException.class);
2605 }
2606 }
2607
2608 /**
2609 * @deprecated use {@link HdfsDataInputStream} instead.
2610 */
2611 @Deprecated
2612 public static class DFSDataInputStream extends HdfsDataInputStream {
2613
2614 public DFSDataInputStream(DFSInputStream in) throws IOException {
2615 super(in);
2616 }
2617 }
2618
2619 void reportChecksumFailure(String file, ExtendedBlock blk, DatanodeInfo dn) {
2620 DatanodeInfo [] dnArr = { dn };
2621 LocatedBlock [] lblocks = { new LocatedBlock(blk, dnArr) };
2622 reportChecksumFailure(file, lblocks);
2623 }
2624
2625 // just reports checksum failure and ignores any exception during the report.
2626 void reportChecksumFailure(String file, LocatedBlock lblocks[]) {
2627 try {
2628 reportBadBlocks(lblocks);
2629 } catch (IOException ie) {
2630 LOG.info("Found corruption while reading " + file
2631 + ". Error repairing corrupt blocks. Bad blocks remain.", ie);
2632 }
2633 }
2634
2635 @Override
2636 public String toString() {
2637 return getClass().getSimpleName() + "[clientName=" + clientName
2638 + ", ugi=" + ugi + "]";
2639 }
2640
2641 public DomainSocketFactory getDomainSocketFactory() {
2642 return domainSocketFactory;
2643 }
2644
2645 public void disableLegacyBlockReaderLocal() {
2646 shouldUseLegacyBlockReaderLocal = false;
2647 }
2648
2649 public boolean useLegacyBlockReaderLocal() {
2650 return shouldUseLegacyBlockReaderLocal;
2651 }
2652
2653 public CachingStrategy getDefaultReadCachingStrategy() {
2654 return defaultReadCachingStrategy;
2655 }
2656
2657 public CachingStrategy getDefaultWriteCachingStrategy() {
2658 return defaultWriteCachingStrategy;
2659 }
2660
2661 @VisibleForTesting
2662 public ClientMmapManager getMmapManager() {
2663 return mmapManager;
2664 }
2665 }