001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs;
019
020 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
027 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY;
028 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT;
029
030 import java.io.IOException;
031 import java.lang.reflect.Constructor;
032 import java.lang.reflect.InvocationHandler;
033 import java.lang.reflect.Proxy;
034 import java.net.InetSocketAddress;
035 import java.net.URI;
036 import java.util.HashMap;
037 import java.util.Map;
038 import java.util.concurrent.TimeUnit;
039
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042 import org.apache.hadoop.conf.Configuration;
043 import org.apache.hadoop.hdfs.DFSClient.Conf;
044 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
045 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
046 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
047 import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
048 import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
049 import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
050 import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
051 import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
052 import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
053 import org.apache.hadoop.hdfs.server.namenode.NameNode;
054 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
055 import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
056 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
057 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
058 import org.apache.hadoop.io.Text;
059 import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
060 import org.apache.hadoop.io.retry.FailoverProxyProvider;
061 import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
062 import org.apache.hadoop.io.retry.RetryPolicies;
063 import org.apache.hadoop.io.retry.RetryPolicy;
064 import org.apache.hadoop.io.retry.RetryProxy;
065 import org.apache.hadoop.io.retry.RetryUtils;
066 import org.apache.hadoop.ipc.ProtobufRpcEngine;
067 import org.apache.hadoop.ipc.RPC;
068 import org.apache.hadoop.ipc.RemoteException;
069 import org.apache.hadoop.net.NetUtils;
070 import org.apache.hadoop.security.RefreshUserMappingsProtocol;
071 import org.apache.hadoop.security.SecurityUtil;
072 import org.apache.hadoop.security.UserGroupInformation;
073 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
074 import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
075 import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
076 import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
077 import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
078 import org.apache.hadoop.ipc.RefreshCallQueueProtocol;
079 import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
080 import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolClientSideTranslatorPB;
081 import org.apache.hadoop.tools.GetUserMappingsProtocol;
082 import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
083 import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
084
085 import com.google.common.annotations.VisibleForTesting;
086 import com.google.common.base.Preconditions;
087
088 /**
089 * Create proxy objects to communicate with a remote NN. All remote access to an
090 * NN should be funneled through this class. Most of the time you'll want to use
091 * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
092 * create either an HA- or non-HA-enabled client proxy as appropriate.
093 */
094 public class NameNodeProxies {
095
096 private static final Log LOG = LogFactory.getLog(NameNodeProxies.class);
097
098 /**
099 * Wrapper for a client proxy as well as its associated service ID.
100 * This is simply used as a tuple-like return type for
101 * {@link NameNodeProxies#createProxy} and
102 * {@link NameNodeProxies#createNonHAProxy}.
103 */
104 public static class ProxyAndInfo<PROXYTYPE> {
105 private final PROXYTYPE proxy;
106 private final Text dtService;
107
108 public ProxyAndInfo(PROXYTYPE proxy, Text dtService) {
109 this.proxy = proxy;
110 this.dtService = dtService;
111 }
112
113 public PROXYTYPE getProxy() {
114 return proxy;
115 }
116
117 public Text getDelegationTokenService() {
118 return dtService;
119 }
120 }
121
122 /**
123 * Creates the namenode proxy with the passed protocol. This will handle
124 * creation of either HA- or non-HA-enabled proxy objects, depending upon
125 * if the provided URI is a configured logical URI.
126 *
127 * @param conf the configuration containing the required IPC
128 * properties, client failover configurations, etc.
129 * @param nameNodeUri the URI pointing either to a specific NameNode
130 * or to a logical nameservice.
131 * @param xface the IPC interface which should be created
132 * @return an object containing both the proxy and the associated
133 * delegation token service it corresponds to
134 * @throws IOException if there is an error creating the proxy
135 **/
136 @SuppressWarnings("unchecked")
137 public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
138 URI nameNodeUri, Class<T> xface) throws IOException {
139 Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
140 getFailoverProxyProviderClass(conf, nameNodeUri, xface);
141
142 if (failoverProxyProviderClass == null) {
143 // Non-HA case
144 return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
145 UserGroupInformation.getCurrentUser(), true);
146 } else {
147 // HA case
148 FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies
149 .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface,
150 nameNodeUri);
151 Conf config = new Conf(conf);
152 T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
153 RetryPolicies.failoverOnNetworkException(
154 RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
155 config.maxRetryAttempts, config.failoverSleepBaseMillis,
156 config.failoverSleepMaxMillis));
157
158 Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
159 return new ProxyAndInfo<T>(proxy, dtService);
160 }
161 }
162
163 /**
164 * Generate a dummy namenode proxy instance that utilizes our hacked
165 * {@link LossyRetryInvocationHandler}. Proxy instance generated using this
166 * method will proactively drop RPC responses. Currently this method only
167 * support HA setup. null will be returned if the given configuration is not
168 * for HA.
169 *
170 * @param config the configuration containing the required IPC
171 * properties, client failover configurations, etc.
172 * @param nameNodeUri the URI pointing either to a specific NameNode
173 * or to a logical nameservice.
174 * @param xface the IPC interface which should be created
175 * @param numResponseToDrop The number of responses to drop for each RPC call
176 * @return an object containing both the proxy and the associated
177 * delegation token service it corresponds to. Will return null of the
178 * given configuration does not support HA.
179 * @throws IOException if there is an error creating the proxy
180 */
181 @SuppressWarnings("unchecked")
182 public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
183 Configuration config, URI nameNodeUri, Class<T> xface,
184 int numResponseToDrop) throws IOException {
185 Preconditions.checkArgument(numResponseToDrop > 0);
186 Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
187 getFailoverProxyProviderClass(config, nameNodeUri, xface);
188 if (failoverProxyProviderClass != null) { // HA case
189 FailoverProxyProvider<T> failoverProxyProvider =
190 createFailoverProxyProvider(config, failoverProxyProviderClass,
191 xface, nameNodeUri);
192 int delay = config.getInt(
193 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
194 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
195 int maxCap = config.getInt(
196 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
197 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
198 int maxFailoverAttempts = config.getInt(
199 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
200 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
201 int maxRetryAttempts = config.getInt(
202 DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
203 DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
204 InvocationHandler dummyHandler = new LossyRetryInvocationHandler<T>(
205 numResponseToDrop, failoverProxyProvider,
206 RetryPolicies.failoverOnNetworkException(
207 RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
208 Math.max(numResponseToDrop + 1, maxRetryAttempts), delay,
209 maxCap));
210
211 T proxy = (T) Proxy.newProxyInstance(
212 failoverProxyProvider.getInterface().getClassLoader(),
213 new Class[] { xface }, dummyHandler);
214 Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
215 return new ProxyAndInfo<T>(proxy, dtService);
216 } else {
217 LOG.warn("Currently creating proxy using " +
218 "LossyRetryInvocationHandler requires NN HA setup");
219 return null;
220 }
221 }
222
223 /**
224 * Creates an explicitly non-HA-enabled proxy object. Most of the time you
225 * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
226 *
227 * @param conf the configuration object
228 * @param nnAddr address of the remote NN to connect to
229 * @param xface the IPC interface which should be created
230 * @param ugi the user who is making the calls on the proxy object
231 * @param withRetries certain interfaces have a non-standard retry policy
232 * @return an object containing both the proxy and the associated
233 * delegation token service it corresponds to
234 * @throws IOException
235 */
236 @SuppressWarnings("unchecked")
237 public static <T> ProxyAndInfo<T> createNonHAProxy(
238 Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
239 UserGroupInformation ugi, boolean withRetries) throws IOException {
240 Text dtService = SecurityUtil.buildTokenService(nnAddr);
241
242 T proxy;
243 if (xface == ClientProtocol.class) {
244 proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
245 withRetries);
246 } else if (xface == JournalProtocol.class) {
247 proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
248 } else if (xface == NamenodeProtocol.class) {
249 proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
250 withRetries);
251 } else if (xface == GetUserMappingsProtocol.class) {
252 proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
253 } else if (xface == RefreshUserMappingsProtocol.class) {
254 proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
255 } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
256 proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
257 conf, ugi);
258 } else if (xface == RefreshCallQueueProtocol.class) {
259 proxy = (T) createNNProxyWithRefreshCallQueueProtocol(nnAddr, conf, ugi);
260 } else {
261 String message = "Unsupported protocol found when creating the proxy " +
262 "connection to NameNode: " +
263 ((xface != null) ? xface.getClass().getName() : "null");
264 LOG.error(message);
265 throw new IllegalStateException(message);
266 }
267
268 return new ProxyAndInfo<T>(proxy, dtService);
269 }
270
271 private static JournalProtocol createNNProxyWithJournalProtocol(
272 InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
273 throws IOException {
274 JournalProtocolPB proxy = (JournalProtocolPB) createNameNodeProxy(address,
275 conf, ugi, JournalProtocolPB.class);
276 return new JournalProtocolTranslatorPB(proxy);
277 }
278
279 private static RefreshAuthorizationPolicyProtocol
280 createNNProxyWithRefreshAuthorizationPolicyProtocol(InetSocketAddress address,
281 Configuration conf, UserGroupInformation ugi) throws IOException {
282 RefreshAuthorizationPolicyProtocolPB proxy = (RefreshAuthorizationPolicyProtocolPB)
283 createNameNodeProxy(address, conf, ugi, RefreshAuthorizationPolicyProtocolPB.class);
284 return new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(proxy);
285 }
286
287 private static RefreshUserMappingsProtocol
288 createNNProxyWithRefreshUserMappingsProtocol(InetSocketAddress address,
289 Configuration conf, UserGroupInformation ugi) throws IOException {
290 RefreshUserMappingsProtocolPB proxy = (RefreshUserMappingsProtocolPB)
291 createNameNodeProxy(address, conf, ugi, RefreshUserMappingsProtocolPB.class);
292 return new RefreshUserMappingsProtocolClientSideTranslatorPB(proxy);
293 }
294
295 private static RefreshCallQueueProtocol
296 createNNProxyWithRefreshCallQueueProtocol(InetSocketAddress address,
297 Configuration conf, UserGroupInformation ugi) throws IOException {
298 RefreshCallQueueProtocolPB proxy = (RefreshCallQueueProtocolPB)
299 createNameNodeProxy(address, conf, ugi, RefreshCallQueueProtocolPB.class);
300 return new RefreshCallQueueProtocolClientSideTranslatorPB(proxy);
301 }
302
303 private static GetUserMappingsProtocol createNNProxyWithGetUserMappingsProtocol(
304 InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
305 throws IOException {
306 GetUserMappingsProtocolPB proxy = (GetUserMappingsProtocolPB)
307 createNameNodeProxy(address, conf, ugi, GetUserMappingsProtocolPB.class);
308 return new GetUserMappingsProtocolClientSideTranslatorPB(proxy);
309 }
310
311 private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
312 InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
313 boolean withRetries) throws IOException {
314 NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
315 address, conf, ugi, NamenodeProtocolPB.class);
316 if (withRetries) { // create the proxy with retries
317 RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
318 TimeUnit.MILLISECONDS);
319 Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
320 = new HashMap<Class<? extends Exception>, RetryPolicy>();
321 RetryPolicy methodPolicy = RetryPolicies.retryByException(timeoutPolicy,
322 exceptionToPolicyMap);
323 Map<String, RetryPolicy> methodNameToPolicyMap
324 = new HashMap<String, RetryPolicy>();
325 methodNameToPolicyMap.put("getBlocks", methodPolicy);
326 methodNameToPolicyMap.put("getAccessKeys", methodPolicy);
327 proxy = (NamenodeProtocolPB) RetryProxy.create(NamenodeProtocolPB.class,
328 proxy, methodNameToPolicyMap);
329 }
330 return new NamenodeProtocolTranslatorPB(proxy);
331 }
332
333 private static ClientProtocol createNNProxyWithClientProtocol(
334 InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
335 boolean withRetries) throws IOException {
336 RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
337
338 final RetryPolicy defaultPolicy =
339 RetryUtils.getDefaultRetryPolicy(
340 conf,
341 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
342 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
343 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
344 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
345 SafeModeException.class);
346
347 final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
348 ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
349 ClientNamenodeProtocolPB.class, version, address, ugi, conf,
350 NetUtils.getDefaultSocketFactory(conf),
351 org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy)
352 .getProxy();
353
354 if (withRetries) { // create the proxy with retries
355
356 RetryPolicy createPolicy = RetryPolicies
357 .retryUpToMaximumCountWithFixedSleep(5,
358 HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
359
360 Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap
361 = new HashMap<Class<? extends Exception>, RetryPolicy>();
362 remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
363 createPolicy);
364
365 Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
366 = new HashMap<Class<? extends Exception>, RetryPolicy>();
367 exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
368 .retryByRemoteException(defaultPolicy,
369 remoteExceptionToPolicyMap));
370 RetryPolicy methodPolicy = RetryPolicies.retryByException(
371 defaultPolicy, exceptionToPolicyMap);
372 Map<String, RetryPolicy> methodNameToPolicyMap
373 = new HashMap<String, RetryPolicy>();
374
375 methodNameToPolicyMap.put("create", methodPolicy);
376
377 proxy = (ClientNamenodeProtocolPB) RetryProxy.create(
378 ClientNamenodeProtocolPB.class,
379 new DefaultFailoverProxyProvider<ClientNamenodeProtocolPB>(
380 ClientNamenodeProtocolPB.class, proxy),
381 methodNameToPolicyMap,
382 defaultPolicy);
383 }
384 return new ClientNamenodeProtocolTranslatorPB(proxy);
385 }
386
387 private static Object createNameNodeProxy(InetSocketAddress address,
388 Configuration conf, UserGroupInformation ugi, Class<?> xface)
389 throws IOException {
390 RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
391 Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
392 ugi, conf, NetUtils.getDefaultSocketFactory(conf));
393 return proxy;
394 }
395
396 /** Gets the configured Failover proxy provider's class */
397 @VisibleForTesting
398 public static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass(
399 Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException {
400 if (nameNodeUri == null) {
401 return null;
402 }
403 String host = nameNodeUri.getHost();
404
405 String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
406 + host;
407 try {
408 @SuppressWarnings("unchecked")
409 Class<FailoverProxyProvider<T>> ret = (Class<FailoverProxyProvider<T>>) conf
410 .getClass(configKey, null, FailoverProxyProvider.class);
411 if (ret != null) {
412 // If we found a proxy provider, then this URI should be a logical NN.
413 // Given that, it shouldn't have a non-default port number.
414 int port = nameNodeUri.getPort();
415 if (port > 0 && port != NameNode.DEFAULT_PORT) {
416 throw new IOException("Port " + port + " specified in URI "
417 + nameNodeUri + " but host '" + host
418 + "' is a logical (HA) namenode"
419 + " and does not use port information.");
420 }
421 }
422 return ret;
423 } catch (RuntimeException e) {
424 if (e.getCause() instanceof ClassNotFoundException) {
425 throw new IOException("Could not load failover proxy provider class "
426 + conf.get(configKey) + " which is configured for authority "
427 + nameNodeUri, e);
428 } else {
429 throw e;
430 }
431 }
432 }
433
434 /** Creates the Failover proxy provider instance*/
435 @VisibleForTesting
436 public static <T> FailoverProxyProvider<T> createFailoverProxyProvider(
437 Configuration conf, Class<FailoverProxyProvider<T>> failoverProxyProviderClass,
438 Class<T> xface, URI nameNodeUri) throws IOException {
439 Preconditions.checkArgument(
440 xface.isAssignableFrom(NamenodeProtocols.class),
441 "Interface %s is not a NameNode protocol", xface);
442 try {
443 Constructor<FailoverProxyProvider<T>> ctor = failoverProxyProviderClass
444 .getConstructor(Configuration.class, URI.class, Class.class);
445 FailoverProxyProvider<T> provider = ctor.newInstance(conf, nameNodeUri,
446 xface);
447 return provider;
448 } catch (Exception e) {
449 String message = "Couldn't create proxy provider " + failoverProxyProviderClass;
450 if (LOG.isDebugEnabled()) {
451 LOG.debug(message, e);
452 }
453 if (e.getCause() instanceof IOException) {
454 throw (IOException) e.getCause();
455 } else {
456 throw new IOException(message, e);
457 }
458 }
459 }
460
461 }