001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs;
019
020 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY;
024 import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HA_DT_SERVICE_PREFIX;
025
026 import java.io.IOException;
027 import java.net.InetSocketAddress;
028 import java.net.URI;
029 import java.util.ArrayList;
030 import java.util.Collection;
031 import java.util.List;
032 import java.util.Map;
033
034 import org.apache.commons.logging.Log;
035 import org.apache.commons.logging.LogFactory;
036 import org.apache.hadoop.HadoopIllegalArgumentException;
037 import org.apache.hadoop.conf.Configuration;
038 import org.apache.hadoop.fs.FileSystem;
039 import org.apache.hadoop.fs.Path;
040 import org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo;
041 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
042 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
043 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
044 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
045 import org.apache.hadoop.hdfs.server.namenode.NameNode;
046 import org.apache.hadoop.hdfs.server.namenode.ha.AbstractNNFailoverProxyProvider;
047 import org.apache.hadoop.io.Text;
048 import org.apache.hadoop.ipc.RPC;
049 import org.apache.hadoop.ipc.RemoteException;
050 import org.apache.hadoop.ipc.StandbyException;
051 import org.apache.hadoop.security.SecurityUtil;
052 import org.apache.hadoop.security.UserGroupInformation;
053 import org.apache.hadoop.security.token.Token;
054
055 import com.google.common.base.Joiner;
056 import com.google.common.base.Preconditions;
057 import com.google.common.collect.Lists;
058
059 public class HAUtil {
060
061 private static final Log LOG =
062 LogFactory.getLog(HAUtil.class);
063
064 private static final DelegationTokenSelector tokenSelector =
065 new DelegationTokenSelector();
066
067 private HAUtil() { /* Hidden constructor */ }
068
069 /**
070 * Returns true if HA for namenode is configured for the given nameservice
071 *
072 * @param conf Configuration
073 * @param nsId nameservice, or null if no federated NS is configured
074 * @return true if HA is configured in the configuration; else false.
075 */
076 public static boolean isHAEnabled(Configuration conf, String nsId) {
077 Map<String, Map<String, InetSocketAddress>> addresses =
078 DFSUtil.getHaNnRpcAddresses(conf);
079 if (addresses == null) return false;
080 Map<String, InetSocketAddress> nnMap = addresses.get(nsId);
081 return nnMap != null && nnMap.size() > 1;
082 }
083
084 /**
085 * Returns true if HA is using a shared edits directory.
086 *
087 * @param conf Configuration
088 * @return true if HA config is using a shared edits dir, false otherwise.
089 */
090 public static boolean usesSharedEditsDir(Configuration conf) {
091 return null != conf.get(DFS_NAMENODE_SHARED_EDITS_DIR_KEY);
092 }
093
094 /**
095 * Get the namenode Id by matching the {@code addressKey}
096 * with the the address of the local node.
097 *
098 * If {@link DFSConfigKeys#DFS_HA_NAMENODE_ID_KEY} is not specifically
099 * configured, this method determines the namenode Id by matching the local
100 * node's address with the configured addresses. When a match is found, it
101 * returns the namenode Id from the corresponding configuration key.
102 *
103 * @param conf Configuration
104 * @return namenode Id on success, null on failure.
105 * @throws HadoopIllegalArgumentException on error
106 */
107 public static String getNameNodeId(Configuration conf, String nsId) {
108 String namenodeId = conf.getTrimmed(DFS_HA_NAMENODE_ID_KEY);
109 if (namenodeId != null) {
110 return namenodeId;
111 }
112
113 String suffixes[] = DFSUtil.getSuffixIDs(conf, DFS_NAMENODE_RPC_ADDRESS_KEY,
114 nsId, null, DFSUtil.LOCAL_ADDRESS_MATCHER);
115 if (suffixes == null) {
116 String msg = "Configuration " + DFS_NAMENODE_RPC_ADDRESS_KEY +
117 " must be suffixed with nameservice and namenode ID for HA " +
118 "configuration.";
119 throw new HadoopIllegalArgumentException(msg);
120 }
121
122 return suffixes[1];
123 }
124
125 /**
126 * Similar to
127 * {@link DFSUtil#getNameServiceIdFromAddress(Configuration,
128 * InetSocketAddress, String...)}
129 */
130 public static String getNameNodeIdFromAddress(final Configuration conf,
131 final InetSocketAddress address, String... keys) {
132 // Configuration with a single namenode and no nameserviceId
133 String[] ids = DFSUtil.getSuffixIDs(conf, address, keys);
134 if (ids != null && ids.length > 1) {
135 return ids[1];
136 }
137 return null;
138 }
139
140 /**
141 * Get the NN ID of the other node in an HA setup.
142 *
143 * @param conf the configuration of this node
144 * @return the NN ID of the other node in this nameservice
145 */
146 public static String getNameNodeIdOfOtherNode(Configuration conf, String nsId) {
147 Preconditions.checkArgument(nsId != null,
148 "Could not determine namespace id. Please ensure that this " +
149 "machine is one of the machines listed as a NN RPC address, " +
150 "or configure " + DFSConfigKeys.DFS_NAMESERVICE_ID);
151
152 Collection<String> nnIds = DFSUtil.getNameNodeIds(conf, nsId);
153 String myNNId = conf.get(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY);
154 Preconditions.checkArgument(nnIds != null,
155 "Could not determine namenode ids in namespace '%s'. " +
156 "Please configure " +
157 DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
158 nsId),
159 nsId);
160 Preconditions.checkArgument(nnIds.size() == 2,
161 "Expected exactly 2 NameNodes in namespace '%s'. " +
162 "Instead, got only %s (NN ids were '%s'",
163 nsId, nnIds.size(), Joiner.on("','").join(nnIds));
164 Preconditions.checkState(myNNId != null && !myNNId.isEmpty(),
165 "Could not determine own NN ID in namespace '%s'. Please " +
166 "ensure that this node is one of the machines listed as an " +
167 "NN RPC address, or configure " + DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY,
168 nsId);
169
170 ArrayList<String> nnSet = Lists.newArrayList(nnIds);
171 nnSet.remove(myNNId);
172 assert nnSet.size() == 1;
173 return nnSet.get(0);
174 }
175
176 /**
177 * Given the configuration for this node, return a Configuration object for
178 * the other node in an HA setup.
179 *
180 * @param myConf the configuration of this node
181 * @return the configuration of the other node in an HA setup
182 */
183 public static Configuration getConfForOtherNode(
184 Configuration myConf) {
185
186 String nsId = DFSUtil.getNamenodeNameServiceId(myConf);
187 String otherNn = getNameNodeIdOfOtherNode(myConf, nsId);
188
189 // Look up the address of the active NN.
190 Configuration confForOtherNode = new Configuration(myConf);
191 NameNode.initializeGenericKeys(confForOtherNode, nsId, otherNn);
192 return confForOtherNode;
193 }
194
195 /**
196 * This is used only by tests at the moment.
197 * @return true if the NN should allow read operations while in standby mode.
198 */
199 public static boolean shouldAllowStandbyReads(Configuration conf) {
200 return conf.getBoolean("dfs.ha.allow.stale.reads", false);
201 }
202
203 public static void setAllowStandbyReads(Configuration conf, boolean val) {
204 conf.setBoolean("dfs.ha.allow.stale.reads", val);
205 }
206
207 /**
208 * @return true if the given nameNodeUri appears to be a logical URI.
209 */
210 public static boolean isLogicalUri(
211 Configuration conf, URI nameNodeUri) {
212 String host = nameNodeUri.getHost();
213 // A logical name must be one of the service IDs.
214 return DFSUtil.getNameServiceIds(conf).contains(host);
215 }
216
217 /**
218 * Check whether the client has a failover proxy provider configured
219 * for the namenode/nameservice.
220 *
221 * @param conf Configuration
222 * @param nameNodeUri The URI of namenode
223 * @return true if failover is configured.
224 */
225 public static boolean isClientFailoverConfigured(
226 Configuration conf, URI nameNodeUri) {
227 String host = nameNodeUri.getHost();
228 String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
229 + host;
230 return conf.get(configKey) != null;
231 }
232
233 /**
234 * Check whether logical URI is needed for the namenode and
235 * the corresponding failover proxy provider in the config.
236 *
237 * @param conf Configuration
238 * @param nameNodeUri The URI of namenode
239 * @return true if logical URI is needed. false, if not needed.
240 * @throws IOException most likely due to misconfiguration.
241 */
242 public static boolean useLogicalUri(Configuration conf, URI nameNodeUri)
243 throws IOException {
244 // Create the proxy provider. Actual proxy is not created.
245 AbstractNNFailoverProxyProvider<ClientProtocol> provider = NameNodeProxies
246 .createFailoverProxyProvider(conf, nameNodeUri, ClientProtocol.class,
247 false, null);
248
249 // No need to use logical URI since failover is not configured.
250 if (provider == null) {
251 return false;
252 }
253 // Check whether the failover proxy provider uses logical URI.
254 return provider.useLogicalURI();
255 }
256
257 /**
258 * Parse the file system URI out of the provided token.
259 */
260 public static URI getServiceUriFromToken(final String scheme, Token<?> token) {
261 String tokStr = token.getService().toString();
262 final String prefix = buildTokenServicePrefixForLogicalUri(scheme);
263 if (tokStr.startsWith(prefix)) {
264 tokStr = tokStr.replaceFirst(prefix, "");
265 }
266 return URI.create(scheme + "://" + tokStr);
267 }
268
269 /**
270 * Get the service name used in the delegation token for the given logical
271 * HA service.
272 * @param uri the logical URI of the cluster
273 * @param scheme the scheme of the corresponding FileSystem
274 * @return the service name
275 */
276 public static Text buildTokenServiceForLogicalUri(final URI uri,
277 final String scheme) {
278 return new Text(buildTokenServicePrefixForLogicalUri(scheme)
279 + uri.getHost());
280 }
281
282 /**
283 * @return true if this token corresponds to a logical nameservice
284 * rather than a specific namenode.
285 */
286 public static boolean isTokenForLogicalUri(Token<?> token) {
287 return token.getService().toString().startsWith(HA_DT_SERVICE_PREFIX);
288 }
289
290 public static String buildTokenServicePrefixForLogicalUri(String scheme) {
291 return HA_DT_SERVICE_PREFIX + scheme + ":";
292 }
293
294 /**
295 * Locate a delegation token associated with the given HA cluster URI, and if
296 * one is found, clone it to also represent the underlying namenode address.
297 * @param ugi the UGI to modify
298 * @param haUri the logical URI for the cluster
299 * @param nnAddrs collection of NNs in the cluster to which the token
300 * applies
301 */
302 public static void cloneDelegationTokenForLogicalUri(
303 UserGroupInformation ugi, URI haUri,
304 Collection<InetSocketAddress> nnAddrs) {
305 // this cloning logic is only used by hdfs
306 Text haService = HAUtil.buildTokenServiceForLogicalUri(haUri,
307 HdfsConstants.HDFS_URI_SCHEME);
308 Token<DelegationTokenIdentifier> haToken =
309 tokenSelector.selectToken(haService, ugi.getTokens());
310 if (haToken != null) {
311 for (InetSocketAddress singleNNAddr : nnAddrs) {
312 // this is a minor hack to prevent physical HA tokens from being
313 // exposed to the user via UGI.getCredentials(), otherwise these
314 // cloned tokens may be inadvertently propagated to jobs
315 Token<DelegationTokenIdentifier> specificToken =
316 new Token.PrivateToken<DelegationTokenIdentifier>(haToken);
317 SecurityUtil.setTokenService(specificToken, singleNNAddr);
318 Text alias = new Text(
319 buildTokenServicePrefixForLogicalUri(HdfsConstants.HDFS_URI_SCHEME)
320 + "//" + specificToken.getService());
321 ugi.addToken(alias, specificToken);
322 LOG.debug("Mapped HA service delegation token for logical URI " +
323 haUri + " to namenode " + singleNNAddr);
324 }
325 } else {
326 LOG.debug("No HA service delegation token found for logical URI " +
327 haUri);
328 }
329 }
330
331 /**
332 * Get the internet address of the currently-active NN. This should rarely be
333 * used, since callers of this method who connect directly to the NN using the
334 * resulting InetSocketAddress will not be able to connect to the active NN if
335 * a failover were to occur after this method has been called.
336 *
337 * @param fs the file system to get the active address of.
338 * @return the internet address of the currently-active NN.
339 * @throws IOException if an error occurs while resolving the active NN.
340 */
341 public static InetSocketAddress getAddressOfActive(FileSystem fs)
342 throws IOException {
343 if (!(fs instanceof DistributedFileSystem)) {
344 throw new IllegalArgumentException("FileSystem " + fs + " is not a DFS.");
345 }
346 // force client address resolution.
347 fs.exists(new Path("/"));
348 DistributedFileSystem dfs = (DistributedFileSystem) fs;
349 DFSClient dfsClient = dfs.getClient();
350 return RPC.getServerAddress(dfsClient.getNamenode());
351 }
352
353 /**
354 * Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
355 * call should be made on every NN in an HA nameservice, not just the active.
356 *
357 * @param conf configuration
358 * @param nsId the nameservice to get all of the proxies for.
359 * @return a list of RPC proxies for each NN in the nameservice.
360 * @throws IOException in the event of error.
361 */
362 public static List<ClientProtocol> getProxiesForAllNameNodesInNameservice(
363 Configuration conf, String nsId) throws IOException {
364 List<ProxyAndInfo<ClientProtocol>> proxies =
365 getProxiesForAllNameNodesInNameservice(conf, nsId, ClientProtocol.class);
366
367 List<ClientProtocol> namenodes = new ArrayList<ClientProtocol>(
368 proxies.size());
369 for (ProxyAndInfo<ClientProtocol> proxy : proxies) {
370 namenodes.add(proxy.getProxy());
371 }
372 return namenodes;
373 }
374
375 /**
376 * Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
377 * call should be made on every NN in an HA nameservice, not just the active.
378 *
379 * @param conf configuration
380 * @param nsId the nameservice to get all of the proxies for.
381 * @param xface the protocol class.
382 * @return a list of RPC proxies for each NN in the nameservice.
383 * @throws IOException in the event of error.
384 */
385 public static <T> List<ProxyAndInfo<T>> getProxiesForAllNameNodesInNameservice(
386 Configuration conf, String nsId, Class<T> xface) throws IOException {
387 Map<String, InetSocketAddress> nnAddresses =
388 DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null);
389
390 List<ProxyAndInfo<T>> proxies = new ArrayList<ProxyAndInfo<T>>(
391 nnAddresses.size());
392 for (InetSocketAddress nnAddress : nnAddresses.values()) {
393 NameNodeProxies.ProxyAndInfo<T> proxyInfo = null;
394 proxyInfo = NameNodeProxies.createNonHAProxy(conf,
395 nnAddress, xface,
396 UserGroupInformation.getCurrentUser(), false);
397 proxies.add(proxyInfo);
398 }
399 return proxies;
400 }
401
402 /**
403 * Used to ensure that at least one of the given HA NNs is currently in the
404 * active state..
405 *
406 * @param namenodes list of RPC proxies for each NN to check.
407 * @return true if at least one NN is active, false if all are in the standby state.
408 * @throws IOException in the event of error.
409 */
410 public static boolean isAtLeastOneActive(List<ClientProtocol> namenodes)
411 throws IOException {
412 for (ClientProtocol namenode : namenodes) {
413 try {
414 namenode.getFileInfo("/");
415 return true;
416 } catch (RemoteException re) {
417 IOException cause = re.unwrapRemoteException();
418 if (cause instanceof StandbyException) {
419 // This is expected to happen for a standby NN.
420 } else {
421 throw re;
422 }
423 }
424 }
425 return false;
426 }
427 }