001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs;
020
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
027 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
028 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
029 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
030 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
031 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
032 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
033
034 import java.io.IOException;
035 import java.io.PrintStream;
036 import java.io.UnsupportedEncodingException;
037 import java.net.InetSocketAddress;
038 import java.net.URI;
039 import java.net.URISyntaxException;
040 import java.security.SecureRandom;
041 import java.util.Collection;
042 import java.util.Collections;
043 import java.util.Comparator;
044 import java.util.HashMap;
045 import java.util.HashSet;
046 import java.util.List;
047 import java.util.Map;
048 import java.util.Random;
049 import java.util.Set;
050 import java.util.concurrent.TimeUnit;
051
052 import javax.net.SocketFactory;
053
054 import org.apache.commons.cli.CommandLine;
055 import org.apache.commons.cli.CommandLineParser;
056 import org.apache.commons.cli.Option;
057 import org.apache.commons.cli.Options;
058 import org.apache.commons.cli.ParseException;
059 import org.apache.commons.cli.PosixParser;
060 import org.apache.commons.logging.Log;
061 import org.apache.commons.logging.LogFactory;
062 import org.apache.hadoop.HadoopIllegalArgumentException;
063 import org.apache.hadoop.classification.InterfaceAudience;
064 import org.apache.hadoop.conf.Configuration;
065 import org.apache.hadoop.fs.BlockLocation;
066 import org.apache.hadoop.fs.FileSystem;
067 import org.apache.hadoop.fs.Path;
068 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
069 import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
070 import org.apache.hadoop.hdfs.protocol.DatanodeID;
071 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
072 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
073 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
074 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
075 import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
076 import org.apache.hadoop.hdfs.server.namenode.NameNode;
077 import org.apache.hadoop.io.retry.RetryPolicies;
078 import org.apache.hadoop.io.retry.RetryPolicy;
079 import org.apache.hadoop.io.retry.RetryProxy;
080 import org.apache.hadoop.ipc.ProtobufRpcEngine;
081 import org.apache.hadoop.ipc.RPC;
082 import org.apache.hadoop.net.NetUtils;
083 import org.apache.hadoop.net.NodeBase;
084 import org.apache.hadoop.security.SecurityUtil;
085 import org.apache.hadoop.security.UserGroupInformation;
086 import org.apache.hadoop.util.StringUtils;
087 import org.apache.hadoop.util.ToolRunner;
088
089 import com.google.common.base.Charsets;
090 import com.google.common.base.Joiner;
091 import com.google.common.base.Preconditions;
092 import com.google.common.collect.Lists;
093 import com.google.common.collect.Maps;
094 import com.google.common.primitives.SignedBytes;
095 import com.google.protobuf.BlockingService;
096
097 @InterfaceAudience.Private
098 public class DFSUtil {
099 public static final Log LOG = LogFactory.getLog(DFSUtil.class.getName());
100
101 public static final byte[] EMPTY_BYTES = {};
102
103 /** Compare two byte arrays by lexicographical order. */
104 public static int compareBytes(byte[] left, byte[] right) {
105 if (left == null) {
106 left = EMPTY_BYTES;
107 }
108 if (right == null) {
109 right = EMPTY_BYTES;
110 }
111 return SignedBytes.lexicographicalComparator().compare(left, right);
112 }
113
114 private DFSUtil() { /* Hidden constructor */ }
115 private static final ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
116 @Override
117 protected Random initialValue() {
118 return new Random();
119 }
120 };
121
122 private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() {
123 @Override
124 protected SecureRandom initialValue() {
125 return new SecureRandom();
126 }
127 };
128
129 /** @return a pseudo random number generator. */
130 public static Random getRandom() {
131 return RANDOM.get();
132 }
133
134 /** @return a pseudo secure random number generator. */
135 public static SecureRandom getSecureRandom() {
136 return SECURE_RANDOM.get();
137 }
138
139 /**
140 * Compartor for sorting DataNodeInfo[] based on decommissioned states.
141 * Decommissioned nodes are moved to the end of the array on sorting with
142 * this compartor.
143 */
144 public static final Comparator<DatanodeInfo> DECOM_COMPARATOR =
145 new Comparator<DatanodeInfo>() {
146 @Override
147 public int compare(DatanodeInfo a, DatanodeInfo b) {
148 return a.isDecommissioned() == b.isDecommissioned() ? 0 :
149 a.isDecommissioned() ? 1 : -1;
150 }
151 };
152
153
154 /**
155 * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states.
156 * Decommissioned/stale nodes are moved to the end of the array on sorting
157 * with this comparator.
158 */
159 @InterfaceAudience.Private
160 public static class DecomStaleComparator implements Comparator<DatanodeInfo> {
161 private long staleInterval;
162
163 /**
164 * Constructor of DecomStaleComparator
165 *
166 * @param interval
167 * The time interval for marking datanodes as stale is passed from
168 * outside, since the interval may be changed dynamically
169 */
170 public DecomStaleComparator(long interval) {
171 this.staleInterval = interval;
172 }
173
174 @Override
175 public int compare(DatanodeInfo a, DatanodeInfo b) {
176 // Decommissioned nodes will still be moved to the end of the list
177 if (a.isDecommissioned()) {
178 return b.isDecommissioned() ? 0 : 1;
179 } else if (b.isDecommissioned()) {
180 return -1;
181 }
182 // Stale nodes will be moved behind the normal nodes
183 boolean aStale = a.isStale(staleInterval);
184 boolean bStale = b.isStale(staleInterval);
185 return aStale == bStale ? 0 : (aStale ? 1 : -1);
186 }
187 }
188
189 /**
190 * Address matcher for matching an address to local address
191 */
192 static final AddressMatcher LOCAL_ADDRESS_MATCHER = new AddressMatcher() {
193 @Override
194 public boolean match(InetSocketAddress s) {
195 return NetUtils.isLocalAddress(s.getAddress());
196 };
197 };
198
199 /**
200 * Whether the pathname is valid. Currently prohibits relative paths,
201 * names which contain a ":" or "//", or other non-canonical paths.
202 */
203 public static boolean isValidName(String src) {
204 // Path must be absolute.
205 if (!src.startsWith(Path.SEPARATOR)) {
206 return false;
207 }
208
209 // Check for ".." "." ":" "/"
210 String[] components = StringUtils.split(src, '/');
211 for (int i = 0; i < components.length; i++) {
212 String element = components[i];
213 if (element.equals("..") ||
214 element.equals(".") ||
215 (element.indexOf(":") >= 0) ||
216 (element.indexOf("/") >= 0)) {
217 return false;
218 }
219
220 // The string may start or end with a /, but not have
221 // "//" in the middle.
222 if (element.isEmpty() && i != components.length - 1 &&
223 i != 0) {
224 return false;
225 }
226 }
227 return true;
228 }
229
230 /**
231 * Converts a byte array to a string using UTF8 encoding.
232 */
233 public static String bytes2String(byte[] bytes) {
234 return bytes2String(bytes, 0, bytes.length);
235 }
236
237 /**
238 * Decode a specific range of bytes of the given byte array to a string
239 * using UTF8.
240 *
241 * @param bytes The bytes to be decoded into characters
242 * @param offset The index of the first byte to decode
243 * @param length The number of bytes to decode
244 * @return The decoded string
245 */
246 public static String bytes2String(byte[] bytes, int offset, int length) {
247 try {
248 return new String(bytes, offset, length, "UTF8");
249 } catch(UnsupportedEncodingException e) {
250 assert false : "UTF8 encoding is not supported ";
251 }
252 return null;
253 }
254
255 /**
256 * Converts a string to a byte array using UTF8 encoding.
257 */
258 public static byte[] string2Bytes(String str) {
259 return str.getBytes(Charsets.UTF_8);
260 }
261
262 /**
263 * Given a list of path components returns a path as a UTF8 String
264 */
265 public static String byteArray2PathString(byte[][] pathComponents) {
266 if (pathComponents.length == 0) {
267 return "";
268 } else if (pathComponents.length == 1
269 && (pathComponents[0] == null || pathComponents[0].length == 0)) {
270 return Path.SEPARATOR;
271 }
272 StringBuilder result = new StringBuilder();
273 for (int i = 0; i < pathComponents.length; i++) {
274 result.append(new String(pathComponents[i], Charsets.UTF_8));
275 if (i < pathComponents.length - 1) {
276 result.append(Path.SEPARATOR_CHAR);
277 }
278 }
279 return result.toString();
280 }
281
282 /**
283 * Given a list of path components returns a byte array
284 */
285 public static byte[] byteArray2bytes(byte[][] pathComponents) {
286 if (pathComponents.length == 0) {
287 return EMPTY_BYTES;
288 } else if (pathComponents.length == 1
289 && (pathComponents[0] == null || pathComponents[0].length == 0)) {
290 return new byte[]{(byte) Path.SEPARATOR_CHAR};
291 }
292 int length = 0;
293 for (int i = 0; i < pathComponents.length; i++) {
294 length += pathComponents[i].length;
295 if (i < pathComponents.length - 1) {
296 length++; // for SEPARATOR
297 }
298 }
299 byte[] path = new byte[length];
300 int index = 0;
301 for (int i = 0; i < pathComponents.length; i++) {
302 System.arraycopy(pathComponents[i], 0, path, index,
303 pathComponents[i].length);
304 index += pathComponents[i].length;
305 if (i < pathComponents.length - 1) {
306 path[index] = (byte) Path.SEPARATOR_CHAR;
307 index++;
308 }
309 }
310 return path;
311 }
312
313 /** Convert an object representing a path to a string. */
314 public static String path2String(final Object path) {
315 return path == null? null
316 : path instanceof String? (String)path
317 : path instanceof byte[][]? byteArray2PathString((byte[][])path)
318 : path.toString();
319 }
320
321 /**
322 * Splits the array of bytes into array of arrays of bytes
323 * on byte separator
324 * @param bytes the array of bytes to split
325 * @param separator the delimiting byte
326 */
327 public static byte[][] bytes2byteArray(byte[] bytes, byte separator) {
328 return bytes2byteArray(bytes, bytes.length, separator);
329 }
330
331 /**
332 * Splits first len bytes in bytes to array of arrays of bytes
333 * on byte separator
334 * @param bytes the byte array to split
335 * @param len the number of bytes to split
336 * @param separator the delimiting byte
337 */
338 public static byte[][] bytes2byteArray(byte[] bytes,
339 int len,
340 byte separator) {
341 assert len <= bytes.length;
342 int splits = 0;
343 if (len == 0) {
344 return new byte[][]{null};
345 }
346 // Count the splits. Omit multiple separators and the last one
347 for (int i = 0; i < len; i++) {
348 if (bytes[i] == separator) {
349 splits++;
350 }
351 }
352 int last = len - 1;
353 while (last > -1 && bytes[last--] == separator) {
354 splits--;
355 }
356 if (splits == 0 && bytes[0] == separator) {
357 return new byte[][]{null};
358 }
359 splits++;
360 byte[][] result = new byte[splits][];
361 int startIndex = 0;
362 int nextIndex = 0;
363 int index = 0;
364 // Build the splits
365 while (index < splits) {
366 while (nextIndex < len && bytes[nextIndex] != separator) {
367 nextIndex++;
368 }
369 result[index] = new byte[nextIndex - startIndex];
370 System.arraycopy(bytes, startIndex, result[index], 0, nextIndex
371 - startIndex);
372 index++;
373 startIndex = nextIndex + 1;
374 nextIndex = startIndex;
375 }
376 return result;
377 }
378
379 /**
380 * Convert a LocatedBlocks to BlockLocations[]
381 * @param blocks a LocatedBlocks
382 * @return an array of BlockLocations
383 */
384 public static BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks) {
385 if (blocks == null) {
386 return new BlockLocation[0];
387 }
388 return locatedBlocks2Locations(blocks.getLocatedBlocks());
389 }
390
391 /**
392 * Convert a List<LocatedBlock> to BlockLocation[]
393 * @param blocks A List<LocatedBlock> to be converted
394 * @return converted array of BlockLocation
395 */
396 public static BlockLocation[] locatedBlocks2Locations(List<LocatedBlock> blocks) {
397 if (blocks == null) {
398 return new BlockLocation[0];
399 }
400 int nrBlocks = blocks.size();
401 BlockLocation[] blkLocations = new BlockLocation[nrBlocks];
402 if (nrBlocks == 0) {
403 return blkLocations;
404 }
405 int idx = 0;
406 for (LocatedBlock blk : blocks) {
407 assert idx < nrBlocks : "Incorrect index";
408 DatanodeInfo[] locations = blk.getLocations();
409 String[] hosts = new String[locations.length];
410 String[] xferAddrs = new String[locations.length];
411 String[] racks = new String[locations.length];
412 for (int hCnt = 0; hCnt < locations.length; hCnt++) {
413 hosts[hCnt] = locations[hCnt].getHostName();
414 xferAddrs[hCnt] = locations[hCnt].getXferAddr();
415 NodeBase node = new NodeBase(xferAddrs[hCnt],
416 locations[hCnt].getNetworkLocation());
417 racks[hCnt] = node.toString();
418 }
419 blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks,
420 blk.getStartOffset(),
421 blk.getBlockSize(),
422 blk.isCorrupt());
423 idx++;
424 }
425 return blkLocations;
426 }
427
428 /**
429 * Returns collection of nameservice Ids from the configuration.
430 * @param conf configuration
431 * @return collection of nameservice Ids, or null if not specified
432 */
433 public static Collection<String> getNameServiceIds(Configuration conf) {
434 return conf.getTrimmedStringCollection(DFS_NAMESERVICES);
435 }
436
437 /**
438 * @return <code>coll</code> if it is non-null and non-empty. Otherwise,
439 * returns a list with a single null value.
440 */
441 private static Collection<String> emptyAsSingletonNull(Collection<String> coll) {
442 if (coll == null || coll.isEmpty()) {
443 return Collections.singletonList(null);
444 } else {
445 return coll;
446 }
447 }
448
449 /**
450 * Namenode HighAvailability related configuration.
451 * Returns collection of namenode Ids from the configuration. One logical id
452 * for each namenode in the in the HA setup.
453 *
454 * @param conf configuration
455 * @param nsId the nameservice ID to look at, or null for non-federated
456 * @return collection of namenode Ids
457 */
458 public static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
459 String key = addSuffix(DFS_HA_NAMENODES_KEY_PREFIX, nsId);
460 return conf.getTrimmedStringCollection(key);
461 }
462
463 /**
464 * Given a list of keys in the order of preference, returns a value
465 * for the key in the given order from the configuration.
466 * @param defaultValue default value to return, when key was not found
467 * @param keySuffix suffix to add to the key, if it is not null
468 * @param conf Configuration
469 * @param keys list of keys in the order of preference
470 * @return value of the key or default if a key was not found in configuration
471 */
472 private static String getConfValue(String defaultValue, String keySuffix,
473 Configuration conf, String... keys) {
474 String value = null;
475 for (String key : keys) {
476 key = addSuffix(key, keySuffix);
477 value = conf.get(key);
478 if (value != null) {
479 break;
480 }
481 }
482 if (value == null) {
483 value = defaultValue;
484 }
485 return value;
486 }
487
488 /** Add non empty and non null suffix to a key */
489 private static String addSuffix(String key, String suffix) {
490 if (suffix == null || suffix.isEmpty()) {
491 return key;
492 }
493 assert !suffix.startsWith(".") :
494 "suffix '" + suffix + "' should not already have '.' prepended.";
495 return key + "." + suffix;
496 }
497
498 /** Concatenate list of suffix strings '.' separated */
499 private static String concatSuffixes(String... suffixes) {
500 if (suffixes == null) {
501 return null;
502 }
503 return Joiner.on(".").skipNulls().join(suffixes);
504 }
505
506 /**
507 * Return configuration key of format key.suffix1.suffix2...suffixN
508 */
509 public static String addKeySuffixes(String key, String... suffixes) {
510 String keySuffix = concatSuffixes(suffixes);
511 return addSuffix(key, keySuffix);
512 }
513
514 /**
515 * Returns the configured address for all NameNodes in the cluster.
516 * @param conf configuration
517 * @param defaultAddress default address to return in case key is not found.
518 * @param keys Set of keys to look for in the order of preference
519 * @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
520 */
521 private static Map<String, Map<String, InetSocketAddress>>
522 getAddresses(Configuration conf,
523 String defaultAddress, String... keys) {
524 Collection<String> nameserviceIds = getNameServiceIds(conf);
525
526 // Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>]
527 // across all of the configured nameservices and namenodes.
528 Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap();
529 for (String nsId : emptyAsSingletonNull(nameserviceIds)) {
530 Map<String, InetSocketAddress> isas =
531 getAddressesForNameserviceId(conf, nsId, defaultAddress, keys);
532 if (!isas.isEmpty()) {
533 ret.put(nsId, isas);
534 }
535 }
536 return ret;
537 }
538
539 private static Map<String, InetSocketAddress> getAddressesForNameserviceId(
540 Configuration conf, String nsId, String defaultValue,
541 String[] keys) {
542 Collection<String> nnIds = getNameNodeIds(conf, nsId);
543 Map<String, InetSocketAddress> ret = Maps.newHashMap();
544 for (String nnId : emptyAsSingletonNull(nnIds)) {
545 String suffix = concatSuffixes(nsId, nnId);
546 String address = getConfValue(defaultValue, suffix, conf, keys);
547 if (address != null) {
548 InetSocketAddress isa = NetUtils.createSocketAddr(address);
549 ret.put(nnId, isa);
550 }
551 }
552 return ret;
553 }
554
555 /**
556 * @return a collection of all configured NN Kerberos principals.
557 */
558 public static Set<String> getAllNnPrincipals(Configuration conf) throws IOException {
559 Set<String> principals = new HashSet<String>();
560 for (String nsId : DFSUtil.getNameServiceIds(conf)) {
561 if (HAUtil.isHAEnabled(conf, nsId)) {
562 for (String nnId : DFSUtil.getNameNodeIds(conf, nsId)) {
563 Configuration confForNn = new Configuration(conf);
564 NameNode.initializeGenericKeys(confForNn, nsId, nnId);
565 String principal = SecurityUtil.getServerPrincipal(confForNn
566 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
567 NameNode.getAddress(confForNn).getHostName());
568 principals.add(principal);
569 }
570 } else {
571 Configuration confForNn = new Configuration(conf);
572 NameNode.initializeGenericKeys(confForNn, nsId, null);
573 String principal = SecurityUtil.getServerPrincipal(confForNn
574 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
575 NameNode.getAddress(confForNn).getHostName());
576 principals.add(principal);
577 }
578 }
579
580 return principals;
581 }
582
583 /**
584 * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from
585 * the configuration.
586 *
587 * @param conf configuration
588 * @return list of InetSocketAddresses
589 */
590 public static Map<String, Map<String, InetSocketAddress>> getHaNnRpcAddresses(
591 Configuration conf) {
592 return getAddresses(conf, null, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
593 }
594
595 /**
596 * Returns list of InetSocketAddress corresponding to backup node rpc
597 * addresses from the configuration.
598 *
599 * @param conf configuration
600 * @return list of InetSocketAddresses
601 * @throws IOException on error
602 */
603 public static Map<String, Map<String, InetSocketAddress>> getBackupNodeAddresses(
604 Configuration conf) throws IOException {
605 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf,
606 null, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
607 if (addressList.isEmpty()) {
608 throw new IOException("Incorrect configuration: backup node address "
609 + DFS_NAMENODE_BACKUP_ADDRESS_KEY + " is not configured.");
610 }
611 return addressList;
612 }
613
614 /**
615 * Returns list of InetSocketAddresses of corresponding to secondary namenode
616 * http addresses from the configuration.
617 *
618 * @param conf configuration
619 * @return list of InetSocketAddresses
620 * @throws IOException on error
621 */
622 public static Map<String, Map<String, InetSocketAddress>> getSecondaryNameNodeAddresses(
623 Configuration conf) throws IOException {
624 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf, null,
625 DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
626 if (addressList.isEmpty()) {
627 throw new IOException("Incorrect configuration: secondary namenode address "
628 + DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY + " is not configured.");
629 }
630 return addressList;
631 }
632
633 /**
634 * Returns list of InetSocketAddresses corresponding to namenodes from the
635 * configuration. Note this is to be used by datanodes to get the list of
636 * namenode addresses to talk to.
637 *
638 * Returns namenode address specifically configured for datanodes (using
639 * service ports), if found. If not, regular RPC address configured for other
640 * clients is returned.
641 *
642 * @param conf configuration
643 * @return list of InetSocketAddress
644 * @throws IOException on error
645 */
646 public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddresses(
647 Configuration conf) throws IOException {
648 // Use default address as fall back
649 String defaultAddress;
650 try {
651 defaultAddress = NetUtils.getHostPortString(NameNode.getAddress(conf));
652 } catch (IllegalArgumentException e) {
653 defaultAddress = null;
654 }
655
656 Map<String, Map<String, InetSocketAddress>> addressList =
657 getAddresses(conf, defaultAddress,
658 DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);
659 if (addressList.isEmpty()) {
660 throw new IOException("Incorrect configuration: namenode address "
661 + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or "
662 + DFS_NAMENODE_RPC_ADDRESS_KEY
663 + " is not configured.");
664 }
665 return addressList;
666 }
667
668 /**
669 * Flatten the given map, as returned by other functions in this class,
670 * into a flat list of {@link ConfiguredNNAddress} instances.
671 */
672 public static List<ConfiguredNNAddress> flattenAddressMap(
673 Map<String, Map<String, InetSocketAddress>> map) {
674 List<ConfiguredNNAddress> ret = Lists.newArrayList();
675
676 for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
677 map.entrySet()) {
678 String nsId = entry.getKey();
679 Map<String, InetSocketAddress> nnMap = entry.getValue();
680 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
681 String nnId = e2.getKey();
682 InetSocketAddress addr = e2.getValue();
683
684 ret.add(new ConfiguredNNAddress(nsId, nnId, addr));
685 }
686 }
687 return ret;
688 }
689
690 /**
691 * Format the given map, as returned by other functions in this class,
692 * into a string suitable for debugging display. The format of this string
693 * should not be considered an interface, and is liable to change.
694 */
695 public static String addressMapToString(
696 Map<String, Map<String, InetSocketAddress>> map) {
697 StringBuilder b = new StringBuilder();
698 for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
699 map.entrySet()) {
700 String nsId = entry.getKey();
701 Map<String, InetSocketAddress> nnMap = entry.getValue();
702 b.append("Nameservice <").append(nsId).append(">:").append("\n");
703 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
704 b.append(" NN ID ").append(e2.getKey())
705 .append(" => ").append(e2.getValue()).append("\n");
706 }
707 }
708 return b.toString();
709 }
710
711 public static String nnAddressesAsString(Configuration conf) {
712 Map<String, Map<String, InetSocketAddress>> addresses =
713 getHaNnRpcAddresses(conf);
714 return addressMapToString(addresses);
715 }
716
717 /**
718 * Represent one of the NameNodes configured in the cluster.
719 */
720 public static class ConfiguredNNAddress {
721 private final String nameserviceId;
722 private final String namenodeId;
723 private final InetSocketAddress addr;
724
725 private ConfiguredNNAddress(String nameserviceId, String namenodeId,
726 InetSocketAddress addr) {
727 this.nameserviceId = nameserviceId;
728 this.namenodeId = namenodeId;
729 this.addr = addr;
730 }
731
732 public String getNameserviceId() {
733 return nameserviceId;
734 }
735
736 public String getNamenodeId() {
737 return namenodeId;
738 }
739
740 public InetSocketAddress getAddress() {
741 return addr;
742 }
743
744 @Override
745 public String toString() {
746 return "ConfiguredNNAddress[nsId=" + nameserviceId + ";" +
747 "nnId=" + namenodeId + ";addr=" + addr + "]";
748 }
749 }
750
751 /**
752 * Get a URI for each configured nameservice. If a nameservice is
753 * HA-enabled, then the logical URI of the nameservice is returned. If the
754 * nameservice is not HA-enabled, then a URI corresponding to an RPC address
755 * of the single NN for that nameservice is returned, preferring the service
756 * RPC address over the client RPC address.
757 *
758 * @param conf configuration
759 * @return a collection of all configured NN URIs, preferring service
760 * addresses
761 */
762 public static Collection<URI> getNsServiceRpcUris(Configuration conf) {
763 return getNameServiceUris(conf,
764 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
765 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
766 }
767
768 /**
769 * Get a URI for each configured nameservice. If a nameservice is
770 * HA-enabled, then the logical URI of the nameservice is returned. If the
771 * nameservice is not HA-enabled, then a URI corresponding to the address of
772 * the single NN for that nameservice is returned.
773 *
774 * @param conf configuration
775 * @param keys configuration keys to try in order to get the URI for non-HA
776 * nameservices
777 * @return a collection of all configured NN URIs
778 */
779 public static Collection<URI> getNameServiceUris(Configuration conf,
780 String... keys) {
781 Set<URI> ret = new HashSet<URI>();
782
783 // We're passed multiple possible configuration keys for any given NN or HA
784 // nameservice, and search the config in order of these keys. In order to
785 // make sure that a later config lookup (e.g. fs.defaultFS) doesn't add a
786 // URI for a config key for which we've already found a preferred entry, we
787 // keep track of non-preferred keys here.
788 Set<URI> nonPreferredUris = new HashSet<URI>();
789
790 for (String nsId : getNameServiceIds(conf)) {
791 if (HAUtil.isHAEnabled(conf, nsId)) {
792 // Add the logical URI of the nameservice.
793 try {
794 ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId));
795 } catch (URISyntaxException ue) {
796 throw new IllegalArgumentException(ue);
797 }
798 } else {
799 // Add the URI corresponding to the address of the NN.
800 boolean uriFound = false;
801 for (String key : keys) {
802 String addr = conf.get(concatSuffixes(key, nsId));
803 if (addr != null) {
804 URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME,
805 NetUtils.createSocketAddr(addr));
806 if (!uriFound) {
807 uriFound = true;
808 ret.add(uri);
809 } else {
810 nonPreferredUris.add(uri);
811 }
812 }
813 }
814 }
815 }
816
817 // Add the generic configuration keys.
818 boolean uriFound = false;
819 for (String key : keys) {
820 String addr = conf.get(key);
821 if (addr != null) {
822 URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr));
823 if (!uriFound) {
824 uriFound = true;
825 ret.add(uri);
826 } else {
827 nonPreferredUris.add(uri);
828 }
829 }
830 }
831
832 // Add the default URI if it is an HDFS URI.
833 URI defaultUri = FileSystem.getDefaultUri(conf);
834 // checks if defaultUri is ip:port format
835 // and convert it to hostname:port format
836 if (defaultUri != null && (defaultUri.getPort() != -1)) {
837 defaultUri = createUri(defaultUri.getScheme(),
838 NetUtils.createSocketAddr(defaultUri.getHost(),
839 defaultUri.getPort()));
840 }
841 if (defaultUri != null &&
842 HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) &&
843 !nonPreferredUris.contains(defaultUri)) {
844 ret.add(defaultUri);
845 }
846
847 return ret;
848 }
849
850 /**
851 * Given the InetSocketAddress this method returns the nameservice Id
852 * corresponding to the key with matching address, by doing a reverse
853 * lookup on the list of nameservices until it finds a match.
854 *
855 * Since the process of resolving URIs to Addresses is slightly expensive,
856 * this utility method should not be used in performance-critical routines.
857 *
858 * @param conf - configuration
859 * @param address - InetSocketAddress for configured communication with NN.
860 * Configured addresses are typically given as URIs, but we may have to
861 * compare against a URI typed in by a human, or the server name may be
862 * aliased, so we compare unambiguous InetSocketAddresses instead of just
863 * comparing URI substrings.
864 * @param keys - list of configured communication parameters that should
865 * be checked for matches. For example, to compare against RPC addresses,
866 * provide the list DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
867 * DFS_NAMENODE_RPC_ADDRESS_KEY. Use the generic parameter keys,
868 * not the NameServiceId-suffixed keys.
869 * @return nameserviceId, or null if no match found
870 */
871 public static String getNameServiceIdFromAddress(final Configuration conf,
872 final InetSocketAddress address, String... keys) {
873 // Configuration with a single namenode and no nameserviceId
874 String[] ids = getSuffixIDs(conf, address, keys);
875 return (ids != null) ? ids[0] : null;
876 }
877
878 /**
879 * return server http or https address from the configuration for a
880 * given namenode rpc address.
881 * @param conf
882 * @param namenodeAddr - namenode RPC address
883 * @param httpsAddress -If true, and if security is enabled, returns server
884 * https address. If false, returns server http address.
885 * @return server http or https address
886 * @throws IOException
887 */
888 public static String getInfoServer(InetSocketAddress namenodeAddr,
889 Configuration conf, boolean httpsAddress) throws IOException {
890 boolean securityOn = UserGroupInformation.isSecurityEnabled();
891 String httpAddressKey = (securityOn && httpsAddress) ?
892 DFS_NAMENODE_HTTPS_ADDRESS_KEY : DFS_NAMENODE_HTTP_ADDRESS_KEY;
893 String httpAddressDefault = (securityOn && httpsAddress) ?
894 DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT : DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
895
896 String suffixes[];
897 if (namenodeAddr != null) {
898 // if non-default namenode, try reverse look up
899 // the nameServiceID if it is available
900 suffixes = getSuffixIDs(conf, namenodeAddr,
901 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
902 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
903 } else {
904 suffixes = new String[2];
905 }
906 String configuredInfoAddr = getSuffixedConf(conf, httpAddressKey,
907 httpAddressDefault, suffixes);
908 if (namenodeAddr != null) {
909 return substituteForWildcardAddress(configuredInfoAddr,
910 namenodeAddr.getHostName());
911 } else {
912 return configuredInfoAddr;
913 }
914 }
915
916
917 /**
918 * Substitute a default host in the case that an address has been configured
919 * with a wildcard. This is used, for example, when determining the HTTP
920 * address of the NN -- if it's configured to bind to 0.0.0.0, we want to
921 * substitute the hostname from the filesystem URI rather than trying to
922 * connect to 0.0.0.0.
923 * @param configuredAddress the address found in the configuration
924 * @param defaultHost the host to substitute with, if configuredAddress
925 * is a local/wildcard address.
926 * @return the substituted address
927 * @throws IOException if it is a wildcard address and security is enabled
928 */
929 public static String substituteForWildcardAddress(String configuredAddress,
930 String defaultHost) throws IOException {
931 InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress);
932 InetSocketAddress defaultSockAddr = NetUtils.createSocketAddr(defaultHost
933 + ":0");
934 if (sockAddr.getAddress().isAnyLocalAddress()) {
935 if (UserGroupInformation.isSecurityEnabled() &&
936 defaultSockAddr.getAddress().isAnyLocalAddress()) {
937 throw new IOException("Cannot use a wildcard address with security. " +
938 "Must explicitly set bind address for Kerberos");
939 }
940 return defaultHost + ":" + sockAddr.getPort();
941 } else {
942 return configuredAddress;
943 }
944 }
945
946 private static String getSuffixedConf(Configuration conf,
947 String key, String defaultVal, String[] suffixes) {
948 String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes));
949 if (ret != null) {
950 return ret;
951 }
952 return conf.get(key, defaultVal);
953 }
954
955 /**
956 * Sets the node specific setting into generic configuration key. Looks up
957 * value of "key.nameserviceId.namenodeId" and if found sets that value into
958 * generic key in the conf. If this is not found, falls back to
959 * "key.nameserviceId" and then the unmodified key.
960 *
961 * Note that this only modifies the runtime conf.
962 *
963 * @param conf
964 * Configuration object to lookup specific key and to set the value
965 * to the key passed. Note the conf object is modified.
966 * @param nameserviceId
967 * nameservice Id to construct the node specific key. Pass null if
968 * federation is not configuration.
969 * @param nnId
970 * namenode Id to construct the node specific key. Pass null if
971 * HA is not configured.
972 * @param keys
973 * The key for which node specific value is looked up
974 */
975 public static void setGenericConf(Configuration conf,
976 String nameserviceId, String nnId, String... keys) {
977 for (String key : keys) {
978 String value = conf.get(addKeySuffixes(key, nameserviceId, nnId));
979 if (value != null) {
980 conf.set(key, value);
981 continue;
982 }
983 value = conf.get(addKeySuffixes(key, nameserviceId));
984 if (value != null) {
985 conf.set(key, value);
986 }
987 }
988 }
989
990 /** Return used as percentage of capacity */
991 public static float getPercentUsed(long used, long capacity) {
992 return capacity <= 0 ? 100 : (used * 100.0f)/capacity;
993 }
994
995 /** Return remaining as percentage of capacity */
996 public static float getPercentRemaining(long remaining, long capacity) {
997 return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity;
998 }
999
1000 /** Convert percentage to a string. */
1001 public static String percent2String(double percentage) {
1002 return StringUtils.format("%.2f%%", percentage);
1003 }
1004
1005 /**
1006 * Round bytes to GiB (gibibyte)
1007 * @param bytes number of bytes
1008 * @return number of GiB
1009 */
1010 public static int roundBytesToGB(long bytes) {
1011 return Math.round((float)bytes/ 1024 / 1024 / 1024);
1012 }
1013
1014 /** Create a {@link ClientDatanodeProtocol} proxy */
1015 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1016 DatanodeID datanodeid, Configuration conf, int socketTimeout,
1017 boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
1018 return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout,
1019 connectToDnViaHostname, locatedBlock);
1020 }
1021
1022 /** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */
1023 static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1024 DatanodeID datanodeid, Configuration conf, int socketTimeout,
1025 boolean connectToDnViaHostname) throws IOException {
1026 return new ClientDatanodeProtocolTranslatorPB(
1027 datanodeid, conf, socketTimeout, connectToDnViaHostname);
1028 }
1029
1030 /** Create a {@link ClientDatanodeProtocol} proxy */
1031 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1032 InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
1033 SocketFactory factory) throws IOException {
1034 return new ClientDatanodeProtocolTranslatorPB(addr, ticket, conf, factory);
1035 }
1036
1037 /**
1038 * Get nameservice Id for the {@link NameNode} based on namenode RPC address
1039 * matching the local node address.
1040 */
1041 public static String getNamenodeNameServiceId(Configuration conf) {
1042 return getNameServiceId(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
1043 }
1044
1045 /**
1046 * Get nameservice Id for the BackupNode based on backup node RPC address
1047 * matching the local node address.
1048 */
1049 public static String getBackupNameServiceId(Configuration conf) {
1050 return getNameServiceId(conf, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
1051 }
1052
1053 /**
1054 * Get nameservice Id for the secondary node based on secondary http address
1055 * matching the local node address.
1056 */
1057 public static String getSecondaryNameServiceId(Configuration conf) {
1058 return getNameServiceId(conf, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
1059 }
1060
1061 /**
1062 * Get the nameservice Id by matching the {@code addressKey} with the
1063 * the address of the local node.
1064 *
1065 * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically
1066 * configured, and more than one nameservice Id is configured, this method
1067 * determines the nameservice Id by matching the local node's address with the
1068 * configured addresses. When a match is found, it returns the nameservice Id
1069 * from the corresponding configuration key.
1070 *
1071 * @param conf Configuration
1072 * @param addressKey configuration key to get the address.
1073 * @return nameservice Id on success, null if federation is not configured.
1074 * @throws HadoopIllegalArgumentException on error
1075 */
1076 private static String getNameServiceId(Configuration conf, String addressKey) {
1077 String nameserviceId = conf.get(DFS_NAMESERVICE_ID);
1078 if (nameserviceId != null) {
1079 return nameserviceId;
1080 }
1081 Collection<String> nsIds = getNameServiceIds(conf);
1082 if (1 == nsIds.size()) {
1083 return nsIds.toArray(new String[1])[0];
1084 }
1085 String nnId = conf.get(DFS_HA_NAMENODE_ID_KEY);
1086
1087 return getSuffixIDs(conf, addressKey, null, nnId, LOCAL_ADDRESS_MATCHER)[0];
1088 }
1089
1090 /**
1091 * Returns nameservice Id and namenode Id when the local host matches the
1092 * configuration parameter {@code addressKey}.<nameservice Id>.<namenode Id>
1093 *
1094 * @param conf Configuration
1095 * @param addressKey configuration key corresponding to the address.
1096 * @param knownNsId only look at configs for the given nameservice, if not-null
1097 * @param knownNNId only look at configs for the given namenode, if not null
1098 * @param matcher matching criteria for matching the address
1099 * @return Array with nameservice Id and namenode Id on success. First element
1100 * in the array is nameservice Id and second element is namenode Id.
1101 * Null value indicates that the configuration does not have the the
1102 * Id.
1103 * @throws HadoopIllegalArgumentException on error
1104 */
1105 static String[] getSuffixIDs(final Configuration conf, final String addressKey,
1106 String knownNsId, String knownNNId,
1107 final AddressMatcher matcher) {
1108 String nameserviceId = null;
1109 String namenodeId = null;
1110 int found = 0;
1111
1112 Collection<String> nsIds = getNameServiceIds(conf);
1113 for (String nsId : emptyAsSingletonNull(nsIds)) {
1114 if (knownNsId != null && !knownNsId.equals(nsId)) {
1115 continue;
1116 }
1117
1118 Collection<String> nnIds = getNameNodeIds(conf, nsId);
1119 for (String nnId : emptyAsSingletonNull(nnIds)) {
1120 if (LOG.isTraceEnabled()) {
1121 LOG.trace(String.format("addressKey: %s nsId: %s nnId: %s",
1122 addressKey, nsId, nnId));
1123 }
1124 if (knownNNId != null && !knownNNId.equals(nnId)) {
1125 continue;
1126 }
1127 String key = addKeySuffixes(addressKey, nsId, nnId);
1128 String addr = conf.get(key);
1129 if (addr == null) {
1130 continue;
1131 }
1132 InetSocketAddress s = null;
1133 try {
1134 s = NetUtils.createSocketAddr(addr);
1135 } catch (Exception e) {
1136 LOG.warn("Exception in creating socket address " + addr, e);
1137 continue;
1138 }
1139 if (!s.isUnresolved() && matcher.match(s)) {
1140 nameserviceId = nsId;
1141 namenodeId = nnId;
1142 found++;
1143 }
1144 }
1145 }
1146 if (found > 1) { // Only one address must match the local address
1147 String msg = "Configuration has multiple addresses that match "
1148 + "local node's address. Please configure the system with "
1149 + DFS_NAMESERVICE_ID + " and "
1150 + DFS_HA_NAMENODE_ID_KEY;
1151 throw new HadoopIllegalArgumentException(msg);
1152 }
1153 return new String[] { nameserviceId, namenodeId };
1154 }
1155
1156 /**
1157 * For given set of {@code keys} adds nameservice Id and or namenode Id
1158 * and returns {nameserviceId, namenodeId} when address match is found.
1159 * @see #getSuffixIDs(Configuration, String, AddressMatcher)
1160 */
1161 static String[] getSuffixIDs(final Configuration conf,
1162 final InetSocketAddress address, final String... keys) {
1163 AddressMatcher matcher = new AddressMatcher() {
1164 @Override
1165 public boolean match(InetSocketAddress s) {
1166 return address.equals(s);
1167 }
1168 };
1169
1170 for (String key : keys) {
1171 String[] ids = getSuffixIDs(conf, key, null, null, matcher);
1172 if (ids != null && (ids [0] != null || ids[1] != null)) {
1173 return ids;
1174 }
1175 }
1176 return null;
1177 }
1178
1179 private interface AddressMatcher {
1180 public boolean match(InetSocketAddress s);
1181 }
1182
1183 /** Create a URI from the scheme and address */
1184 public static URI createUri(String scheme, InetSocketAddress address) {
1185 try {
1186 return new URI(scheme, null, address.getHostName(), address.getPort(),
1187 null, null, null);
1188 } catch (URISyntaxException ue) {
1189 throw new IllegalArgumentException(ue);
1190 }
1191 }
1192
1193 /**
1194 * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}
1195 * @param conf configuration
1196 * @param protocol Protocol interface
1197 * @param service service that implements the protocol
1198 * @param server RPC server to which the protocol & implementation is added to
1199 * @throws IOException
1200 */
1201 public static void addPBProtocol(Configuration conf, Class<?> protocol,
1202 BlockingService service, RPC.Server server) throws IOException {
1203 RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
1204 server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
1205 }
1206
1207 /**
1208 * Map a logical namenode ID to its service address. Use the given
1209 * nameservice if specified, or the configured one if none is given.
1210 *
1211 * @param conf Configuration
1212 * @param nsId which nameservice nnId is a part of, optional
1213 * @param nnId the namenode ID to get the service addr for
1214 * @return the service addr, null if it could not be determined
1215 */
1216 public static String getNamenodeServiceAddr(final Configuration conf,
1217 String nsId, String nnId) {
1218
1219 if (nsId == null) {
1220 nsId = getOnlyNameServiceIdOrNull(conf);
1221 }
1222
1223 String serviceAddrKey = concatSuffixes(
1224 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId);
1225
1226 String addrKey = concatSuffixes(
1227 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId);
1228
1229 String serviceRpcAddr = conf.get(serviceAddrKey);
1230 if (serviceRpcAddr == null) {
1231 serviceRpcAddr = conf.get(addrKey);
1232 }
1233 return serviceRpcAddr;
1234 }
1235
1236 /**
1237 * If the configuration refers to only a single nameservice, return the
1238 * name of that nameservice. If it refers to 0 or more than 1, return null.
1239 */
1240 public static String getOnlyNameServiceIdOrNull(Configuration conf) {
1241 Collection<String> nsIds = getNameServiceIds(conf);
1242 if (1 == nsIds.size()) {
1243 return nsIds.toArray(new String[1])[0];
1244 } else {
1245 // No nameservice ID was given and more than one is configured
1246 return null;
1247 }
1248 }
1249
1250 public static Options helpOptions = new Options();
1251 public static Option helpOpt = new Option("h", "help", false,
1252 "get help information");
1253
1254 static {
1255 helpOptions.addOption(helpOpt);
1256 }
1257
1258 /**
1259 * Parse the arguments for commands
1260 *
1261 * @param args the argument to be parsed
1262 * @param helpDescription help information to be printed out
1263 * @param out Printer
1264 * @param printGenericCommandUsage whether to print the
1265 * generic command usage defined in ToolRunner
1266 * @return true when the argument matches help option, false if not
1267 */
1268 public static boolean parseHelpArgument(String[] args,
1269 String helpDescription, PrintStream out, boolean printGenericCommandUsage) {
1270 if (args.length == 1) {
1271 try {
1272 CommandLineParser parser = new PosixParser();
1273 CommandLine cmdLine = parser.parse(helpOptions, args);
1274 if (cmdLine.hasOption(helpOpt.getOpt())
1275 || cmdLine.hasOption(helpOpt.getLongOpt())) {
1276 // should print out the help information
1277 out.println(helpDescription + "\n");
1278 if (printGenericCommandUsage) {
1279 ToolRunner.printGenericCommandUsage(out);
1280 }
1281 return true;
1282 }
1283 } catch (ParseException pe) {
1284 return false;
1285 }
1286 }
1287 return false;
1288 }
1289
1290 /**
1291 * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration.
1292 *
1293 * @param conf Configuration
1294 * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION
1295 */
1296 public static float getInvalidateWorkPctPerIteration(Configuration conf) {
1297 float blocksInvalidateWorkPct = conf.getFloat(
1298 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
1299 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
1300 Preconditions.checkArgument(
1301 (blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f),
1302 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
1303 " = '" + blocksInvalidateWorkPct + "' is invalid. " +
1304 "It should be a positive, non-zero float value, not greater than 1.0f, " +
1305 "to indicate a percentage.");
1306 return blocksInvalidateWorkPct;
1307 }
1308
1309 /**
1310 * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from
1311 * configuration.
1312 *
1313 * @param conf Configuration
1314 * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
1315 */
1316 public static int getReplWorkMultiplier(Configuration conf) {
1317 int blocksReplWorkMultiplier = conf.getInt(
1318 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
1319 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
1320 Preconditions.checkArgument(
1321 (blocksReplWorkMultiplier > 0),
1322 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
1323 " = '" + blocksReplWorkMultiplier + "' is invalid. " +
1324 "It should be a positive, non-zero integer value.");
1325 return blocksReplWorkMultiplier;
1326 }
1327
1328 /**
1329 * Get SPNEGO keytab Key from configuration
1330 *
1331 * @param conf
1332 * Configuration
1333 * @param defaultKey
1334 * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty
1335 * else return defaultKey
1336 */
1337 public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) {
1338 String value =
1339 conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
1340 return (value == null || value.isEmpty()) ?
1341 defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY;
1342 }
1343 }