001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.hdfs;
019
020 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
021 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
022 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
023 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
024 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
025 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
026 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
027
028 import java.io.IOException;
029 import java.lang.reflect.Constructor;
030 import java.lang.reflect.InvocationHandler;
031 import java.lang.reflect.Proxy;
032 import java.net.InetSocketAddress;
033 import java.net.URI;
034 import java.util.HashMap;
035 import java.util.Map;
036 import java.util.concurrent.TimeUnit;
037
038 import org.apache.commons.logging.Log;
039 import org.apache.commons.logging.LogFactory;
040 import org.apache.hadoop.conf.Configuration;
041 import org.apache.hadoop.hdfs.DFSClient.Conf;
042 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
043 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
044 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
045 import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
046 import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
047 import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
048 import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
049 import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
050 import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
051 import org.apache.hadoop.hdfs.server.namenode.NameNode;
052 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
053 import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
054 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
055 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
056 import org.apache.hadoop.io.Text;
057 import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
058 import org.apache.hadoop.io.retry.FailoverProxyProvider;
059 import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
060 import org.apache.hadoop.io.retry.RetryPolicies;
061 import org.apache.hadoop.io.retry.RetryPolicy;
062 import org.apache.hadoop.io.retry.RetryProxy;
063 import org.apache.hadoop.io.retry.RetryUtils;
064 import org.apache.hadoop.ipc.ProtobufRpcEngine;
065 import org.apache.hadoop.ipc.RPC;
066 import org.apache.hadoop.ipc.RemoteException;
067 import org.apache.hadoop.net.NetUtils;
068 import org.apache.hadoop.security.RefreshUserMappingsProtocol;
069 import org.apache.hadoop.security.SecurityUtil;
070 import org.apache.hadoop.security.UserGroupInformation;
071 import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
072 import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
073 import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
074 import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
075 import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
076 import org.apache.hadoop.tools.GetUserMappingsProtocol;
077 import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
078 import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
079
080 import com.google.common.annotations.VisibleForTesting;
081 import com.google.common.base.Preconditions;
082
083 /**
084 * Create proxy objects to communicate with a remote NN. All remote access to an
085 * NN should be funneled through this class. Most of the time you'll want to use
086 * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
087 * create either an HA- or non-HA-enabled client proxy as appropriate.
088 */
089 public class NameNodeProxies {
090
091 private static final Log LOG = LogFactory.getLog(NameNodeProxies.class);
092
093 /**
094 * Wrapper for a client proxy as well as its associated service ID.
095 * This is simply used as a tuple-like return type for
096 * {@link NameNodeProxies#createProxy} and
097 * {@link NameNodeProxies#createNonHAProxy}.
098 */
099 public static class ProxyAndInfo<PROXYTYPE> {
100 private final PROXYTYPE proxy;
101 private final Text dtService;
102
103 public ProxyAndInfo(PROXYTYPE proxy, Text dtService) {
104 this.proxy = proxy;
105 this.dtService = dtService;
106 }
107
108 public PROXYTYPE getProxy() {
109 return proxy;
110 }
111
112 public Text getDelegationTokenService() {
113 return dtService;
114 }
115 }
116
117 /**
118 * Creates the namenode proxy with the passed protocol. This will handle
119 * creation of either HA- or non-HA-enabled proxy objects, depending upon
120 * if the provided URI is a configured logical URI.
121 *
122 * @param conf the configuration containing the required IPC
123 * properties, client failover configurations, etc.
124 * @param nameNodeUri the URI pointing either to a specific NameNode
125 * or to a logical nameservice.
126 * @param xface the IPC interface which should be created
127 * @return an object containing both the proxy and the associated
128 * delegation token service it corresponds to
129 * @throws IOException if there is an error creating the proxy
130 **/
131 @SuppressWarnings("unchecked")
132 public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
133 URI nameNodeUri, Class<T> xface) throws IOException {
134 Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
135 getFailoverProxyProviderClass(conf, nameNodeUri, xface);
136
137 if (failoverProxyProviderClass == null) {
138 // Non-HA case
139 return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
140 UserGroupInformation.getCurrentUser(), true);
141 } else {
142 // HA case
143 FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies
144 .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface,
145 nameNodeUri);
146 Conf config = new Conf(conf);
147 T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies
148 .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
149 config.maxFailoverAttempts, config.failoverSleepBaseMillis,
150 config.failoverSleepMaxMillis));
151
152 Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
153 return new ProxyAndInfo<T>(proxy, dtService);
154 }
155 }
156
157 /**
158 * Generate a dummy namenode proxy instance that utilizes our hacked
159 * {@link LossyRetryInvocationHandler}. Proxy instance generated using this
160 * method will proactively drop RPC responses. Currently this method only
161 * support HA setup. null will be returned if the given configuration is not
162 * for HA.
163 *
164 * @param config the configuration containing the required IPC
165 * properties, client failover configurations, etc.
166 * @param nameNodeUri the URI pointing either to a specific NameNode
167 * or to a logical nameservice.
168 * @param xface the IPC interface which should be created
169 * @param numResponseToDrop The number of responses to drop for each RPC call
170 * @return an object containing both the proxy and the associated
171 * delegation token service it corresponds to. Will return null of the
172 * given configuration does not support HA.
173 * @throws IOException if there is an error creating the proxy
174 */
175 @SuppressWarnings("unchecked")
176 public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
177 Configuration config, URI nameNodeUri, Class<T> xface,
178 int numResponseToDrop) throws IOException {
179 Preconditions.checkArgument(numResponseToDrop > 0);
180 Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
181 getFailoverProxyProviderClass(config, nameNodeUri, xface);
182 if (failoverProxyProviderClass != null) { // HA case
183 FailoverProxyProvider<T> failoverProxyProvider =
184 createFailoverProxyProvider(config, failoverProxyProviderClass,
185 xface, nameNodeUri);
186 int delay = config.getInt(
187 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
188 DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
189 int maxCap = config.getInt(
190 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
191 DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
192 int maxFailoverAttempts = config.getInt(
193 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
194 DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
195 InvocationHandler dummyHandler = new LossyRetryInvocationHandler<T>(
196 numResponseToDrop, failoverProxyProvider,
197 RetryPolicies.failoverOnNetworkException(
198 RetryPolicies.TRY_ONCE_THEN_FAIL,
199 Math.max(numResponseToDrop + 1, maxFailoverAttempts), delay,
200 maxCap));
201
202 T proxy = (T) Proxy.newProxyInstance(
203 failoverProxyProvider.getInterface().getClassLoader(),
204 new Class[] { xface }, dummyHandler);
205 Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
206 return new ProxyAndInfo<T>(proxy, dtService);
207 } else {
208 LOG.warn("Currently creating proxy using " +
209 "LossyRetryInvocationHandler requires NN HA setup");
210 return null;
211 }
212 }
213
214 /**
215 * Creates an explicitly non-HA-enabled proxy object. Most of the time you
216 * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
217 *
218 * @param conf the configuration object
219 * @param nnAddr address of the remote NN to connect to
220 * @param xface the IPC interface which should be created
221 * @param ugi the user who is making the calls on the proxy object
222 * @param withRetries certain interfaces have a non-standard retry policy
223 * @return an object containing both the proxy and the associated
224 * delegation token service it corresponds to
225 * @throws IOException
226 */
227 @SuppressWarnings("unchecked")
228 public static <T> ProxyAndInfo<T> createNonHAProxy(
229 Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
230 UserGroupInformation ugi, boolean withRetries) throws IOException {
231 Text dtService = SecurityUtil.buildTokenService(nnAddr);
232
233 T proxy;
234 if (xface == ClientProtocol.class) {
235 proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
236 withRetries);
237 } else if (xface == JournalProtocol.class) {
238 proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
239 } else if (xface == NamenodeProtocol.class) {
240 proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
241 withRetries);
242 } else if (xface == GetUserMappingsProtocol.class) {
243 proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
244 } else if (xface == RefreshUserMappingsProtocol.class) {
245 proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
246 } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
247 proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
248 conf, ugi);
249 } else {
250 String message = "Upsupported protocol found when creating the proxy " +
251 "connection to NameNode: " +
252 ((xface != null) ? xface.getClass().getName() : "null");
253 LOG.error(message);
254 throw new IllegalStateException(message);
255 }
256 return new ProxyAndInfo<T>(proxy, dtService);
257 }
258
259 private static JournalProtocol createNNProxyWithJournalProtocol(
260 InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
261 throws IOException {
262 JournalProtocolPB proxy = (JournalProtocolPB) createNameNodeProxy(address,
263 conf, ugi, JournalProtocolPB.class);
264 return new JournalProtocolTranslatorPB(proxy);
265 }
266
267 private static RefreshAuthorizationPolicyProtocol
268 createNNProxyWithRefreshAuthorizationPolicyProtocol(InetSocketAddress address,
269 Configuration conf, UserGroupInformation ugi) throws IOException {
270 RefreshAuthorizationPolicyProtocolPB proxy = (RefreshAuthorizationPolicyProtocolPB)
271 createNameNodeProxy(address, conf, ugi, RefreshAuthorizationPolicyProtocolPB.class);
272 return new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(proxy);
273 }
274
275 private static RefreshUserMappingsProtocol
276 createNNProxyWithRefreshUserMappingsProtocol(InetSocketAddress address,
277 Configuration conf, UserGroupInformation ugi) throws IOException {
278 RefreshUserMappingsProtocolPB proxy = (RefreshUserMappingsProtocolPB)
279 createNameNodeProxy(address, conf, ugi, RefreshUserMappingsProtocolPB.class);
280 return new RefreshUserMappingsProtocolClientSideTranslatorPB(proxy);
281 }
282
283 private static GetUserMappingsProtocol createNNProxyWithGetUserMappingsProtocol(
284 InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
285 throws IOException {
286 GetUserMappingsProtocolPB proxy = (GetUserMappingsProtocolPB)
287 createNameNodeProxy(address, conf, ugi, GetUserMappingsProtocolPB.class);
288 return new GetUserMappingsProtocolClientSideTranslatorPB(proxy);
289 }
290
291 private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
292 InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
293 boolean withRetries) throws IOException {
294 NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
295 address, conf, ugi, NamenodeProtocolPB.class);
296 if (withRetries) { // create the proxy with retries
297 RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
298 TimeUnit.MILLISECONDS);
299 Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
300 = new HashMap<Class<? extends Exception>, RetryPolicy>();
301 RetryPolicy methodPolicy = RetryPolicies.retryByException(timeoutPolicy,
302 exceptionToPolicyMap);
303 Map<String, RetryPolicy> methodNameToPolicyMap
304 = new HashMap<String, RetryPolicy>();
305 methodNameToPolicyMap.put("getBlocks", methodPolicy);
306 methodNameToPolicyMap.put("getAccessKeys", methodPolicy);
307 proxy = (NamenodeProtocolPB) RetryProxy.create(NamenodeProtocolPB.class,
308 proxy, methodNameToPolicyMap);
309 }
310 return new NamenodeProtocolTranslatorPB(proxy);
311 }
312
313 private static ClientProtocol createNNProxyWithClientProtocol(
314 InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
315 boolean withRetries) throws IOException {
316 RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
317
318 final RetryPolicy defaultPolicy =
319 RetryUtils.getDefaultRetryPolicy(
320 conf,
321 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
322 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
323 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
324 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
325 SafeModeException.class);
326
327 final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
328 ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
329 ClientNamenodeProtocolPB.class, version, address, ugi, conf,
330 NetUtils.getDefaultSocketFactory(conf),
331 org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy)
332 .getProxy();
333
334 if (withRetries) { // create the proxy with retries
335
336 RetryPolicy createPolicy = RetryPolicies
337 .retryUpToMaximumCountWithFixedSleep(5,
338 HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
339
340 Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap
341 = new HashMap<Class<? extends Exception>, RetryPolicy>();
342 remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
343 createPolicy);
344
345 Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
346 = new HashMap<Class<? extends Exception>, RetryPolicy>();
347 exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
348 .retryByRemoteException(defaultPolicy,
349 remoteExceptionToPolicyMap));
350 RetryPolicy methodPolicy = RetryPolicies.retryByException(
351 defaultPolicy, exceptionToPolicyMap);
352 Map<String, RetryPolicy> methodNameToPolicyMap
353 = new HashMap<String, RetryPolicy>();
354
355 methodNameToPolicyMap.put("create", methodPolicy);
356
357 proxy = (ClientNamenodeProtocolPB) RetryProxy.create(
358 ClientNamenodeProtocolPB.class,
359 new DefaultFailoverProxyProvider<ClientNamenodeProtocolPB>(
360 ClientNamenodeProtocolPB.class, proxy),
361 methodNameToPolicyMap,
362 defaultPolicy);
363 }
364 return new ClientNamenodeProtocolTranslatorPB(proxy);
365 }
366
367 private static Object createNameNodeProxy(InetSocketAddress address,
368 Configuration conf, UserGroupInformation ugi, Class<?> xface)
369 throws IOException {
370 RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
371 Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
372 ugi, conf, NetUtils.getDefaultSocketFactory(conf));
373 return proxy;
374 }
375
376 /** Gets the configured Failover proxy provider's class */
377 @VisibleForTesting
378 public static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass(
379 Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException {
380 if (nameNodeUri == null) {
381 return null;
382 }
383 String host = nameNodeUri.getHost();
384
385 String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
386 + host;
387 try {
388 @SuppressWarnings("unchecked")
389 Class<FailoverProxyProvider<T>> ret = (Class<FailoverProxyProvider<T>>) conf
390 .getClass(configKey, null, FailoverProxyProvider.class);
391 if (ret != null) {
392 // If we found a proxy provider, then this URI should be a logical NN.
393 // Given that, it shouldn't have a non-default port number.
394 int port = nameNodeUri.getPort();
395 if (port > 0 && port != NameNode.DEFAULT_PORT) {
396 throw new IOException("Port " + port + " specified in URI "
397 + nameNodeUri + " but host '" + host
398 + "' is a logical (HA) namenode"
399 + " and does not use port information.");
400 }
401 }
402 return ret;
403 } catch (RuntimeException e) {
404 if (e.getCause() instanceof ClassNotFoundException) {
405 throw new IOException("Could not load failover proxy provider class "
406 + conf.get(configKey) + " which is configured for authority "
407 + nameNodeUri, e);
408 } else {
409 throw e;
410 }
411 }
412 }
413
414 /** Creates the Failover proxy provider instance*/
415 @VisibleForTesting
416 public static <T> FailoverProxyProvider<T> createFailoverProxyProvider(
417 Configuration conf, Class<FailoverProxyProvider<T>> failoverProxyProviderClass,
418 Class<T> xface, URI nameNodeUri) throws IOException {
419 Preconditions.checkArgument(
420 xface.isAssignableFrom(NamenodeProtocols.class),
421 "Interface %s is not a NameNode protocol", xface);
422 try {
423 Constructor<FailoverProxyProvider<T>> ctor = failoverProxyProviderClass
424 .getConstructor(Configuration.class, URI.class, Class.class);
425 FailoverProxyProvider<T> provider = ctor.newInstance(conf, nameNodeUri,
426 xface);
427 return provider;
428 } catch (Exception e) {
429 String message = "Couldn't create proxy provider " + failoverProxyProviderClass;
430 if (LOG.isDebugEnabled()) {
431 LOG.debug(message, e);
432 }
433 if (e.getCause() instanceof IOException) {
434 throw (IOException) e.getCause();
435 } else {
436 throw new IOException(message, e);
437 }
438 }
439 }
440
441 }