001/**
002res * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs.web;
020
021import java.io.BufferedOutputStream;
022import java.io.FileNotFoundException;
023import java.io.IOException;
024import java.io.InputStream;
025import java.io.InputStreamReader;
026import java.net.HttpURLConnection;
027import java.net.InetSocketAddress;
028import java.net.MalformedURLException;
029import java.net.URI;
030import java.net.URL;
031import java.security.PrivilegedExceptionAction;
032import java.util.ArrayList;
033import java.util.EnumSet;
034import java.util.List;
035import java.util.Map;
036import java.util.StringTokenizer;
037
038import javax.ws.rs.core.MediaType;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.fs.BlockLocation;
044import org.apache.hadoop.fs.CommonConfigurationKeys;
045import org.apache.hadoop.fs.ContentSummary;
046import org.apache.hadoop.fs.DelegationTokenRenewer;
047import org.apache.hadoop.fs.FSDataInputStream;
048import org.apache.hadoop.fs.FSDataOutputStream;
049import org.apache.hadoop.fs.FileStatus;
050import org.apache.hadoop.fs.FileSystem;
051import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
052import org.apache.hadoop.fs.Options;
053import org.apache.hadoop.fs.Path;
054import org.apache.hadoop.fs.XAttrCodec;
055import org.apache.hadoop.fs.XAttrSetFlag;
056import org.apache.hadoop.fs.permission.AclEntry;
057import org.apache.hadoop.fs.permission.AclStatus;
058import org.apache.hadoop.fs.permission.FsAction;
059import org.apache.hadoop.fs.permission.FsPermission;
060import org.apache.hadoop.hdfs.DFSConfigKeys;
061import org.apache.hadoop.hdfs.DFSUtil;
062import org.apache.hadoop.hdfs.HAUtil;
063import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
064import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
065import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
066import org.apache.hadoop.hdfs.web.resources.*;
067import org.apache.hadoop.hdfs.web.resources.HttpOpParam.Op;
068import org.apache.hadoop.io.Text;
069import org.apache.hadoop.io.retry.RetryPolicies;
070import org.apache.hadoop.io.retry.RetryPolicy;
071import org.apache.hadoop.io.retry.RetryUtils;
072import org.apache.hadoop.ipc.RemoteException;
073import org.apache.hadoop.net.NetUtils;
074import org.apache.hadoop.security.AccessControlException;
075import org.apache.hadoop.security.SecurityUtil;
076import org.apache.hadoop.security.UserGroupInformation;
077import org.apache.hadoop.security.token.SecretManager.InvalidToken;
078import org.apache.hadoop.security.token.Token;
079import org.apache.hadoop.security.token.TokenIdentifier;
080import org.apache.hadoop.security.token.TokenSelector;
081import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
082import org.apache.hadoop.util.Progressable;
083import org.mortbay.util.ajax.JSON;
084
085import com.google.common.annotations.VisibleForTesting;
086import com.google.common.base.Charsets;
087import com.google.common.base.Preconditions;
088import com.google.common.collect.Lists;
089
090/** A FileSystem for HDFS over the web. */
091public class WebHdfsFileSystem extends FileSystem
092    implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
093  public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
094  /** File System URI: {SCHEME}://namenode:port/path/to/file */
095  public static final String SCHEME = "webhdfs";
096  /** WebHdfs version. */
097  public static final int VERSION = 1;
098  /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
099  public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
100
101  /** Default connection factory may be overridden in tests to use smaller timeout values */
102  protected URLConnectionFactory connectionFactory;
103
104  /** Delegation token kind */
105  public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
106
107  @VisibleForTesting
108  public static final String CANT_FALLBACK_TO_INSECURE_MSG =
109      "The client is configured to only allow connecting to secure cluster";
110
111  private boolean canRefreshDelegationToken;
112
113  private UserGroupInformation ugi;
114  private URI uri;
115  private Token<?> delegationToken;
116  protected Text tokenServiceName;
117  private RetryPolicy retryPolicy = null;
118  private Path workingDir;
119  private InetSocketAddress nnAddrs[];
120  private int currentNNAddrIndex;
121  private boolean disallowFallbackToInsecureCluster;
122
123  /**
124   * Return the protocol scheme for the FileSystem.
125   * <p/>
126   *
127   * @return <code>webhdfs</code>
128   */
129  @Override
130  public String getScheme() {
131    return SCHEME;
132  }
133
134  /**
135   * return the underlying transport protocol (http / https).
136   */
137  protected String getTransportScheme() {
138    return "http";
139  }
140
141  protected Text getTokenKind() {
142    return TOKEN_KIND;
143  }
144
145  @Override
146  public synchronized void initialize(URI uri, Configuration conf
147      ) throws IOException {
148    super.initialize(uri, conf);
149    setConf(conf);
150    /** set user pattern based on configuration file */
151    UserParam.setUserPattern(conf.get(
152        DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
153        DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
154
155    connectionFactory = URLConnectionFactory
156        .newDefaultURLConnectionFactory(conf);
157
158    ugi = UserGroupInformation.getCurrentUser();
159    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
160    this.nnAddrs = resolveNNAddr();
161
162    boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.uri);
163    boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.uri);
164    // In non-HA or non-logical URI case, the code needs to call
165    // getCanonicalUri() in order to handle the case where no port is
166    // specified in the URI
167    this.tokenServiceName = isLogicalUri ?
168        HAUtil.buildTokenServiceForLogicalUri(uri, getScheme())
169        : SecurityUtil.buildTokenService(getCanonicalUri());
170
171    if (!isHA) {
172      this.retryPolicy =
173          RetryUtils.getDefaultRetryPolicy(
174              conf,
175              DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
176              DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
177              DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
178              DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
179              SafeModeException.class);
180    } else {
181
182      int maxFailoverAttempts = conf.getInt(
183          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
184          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
185      int maxRetryAttempts = conf.getInt(
186          DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
187          DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
188      int failoverSleepBaseMillis = conf.getInt(
189          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
190          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
191      int failoverSleepMaxMillis = conf.getInt(
192          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
193          DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
194
195      this.retryPolicy = RetryPolicies
196          .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
197              maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis,
198              failoverSleepMaxMillis);
199    }
200
201    this.workingDir = getHomeDirectory();
202    this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
203    this.disallowFallbackToInsecureCluster = !conf.getBoolean(
204        CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
205        CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
206    this.delegationToken = null;
207  }
208
209  @Override
210  public URI getCanonicalUri() {
211    return super.getCanonicalUri();
212  }
213
214  /** Is WebHDFS enabled in conf? */
215  public static boolean isEnabled(final Configuration conf, final Log log) {
216    final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
217        DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
218    return b;
219  }
220
221  TokenSelector<DelegationTokenIdentifier> tokenSelector =
222      new AbstractDelegationTokenSelector<DelegationTokenIdentifier>(getTokenKind()){};
223
224  // the first getAuthParams() for a non-token op will either get the
225  // internal token from the ugi or lazy fetch one
226  protected synchronized Token<?> getDelegationToken() throws IOException {
227    if (canRefreshDelegationToken && delegationToken == null) {
228      Token<?> token = tokenSelector.selectToken(
229          new Text(getCanonicalServiceName()), ugi.getTokens());
230      // ugi tokens are usually indicative of a task which can't
231      // refetch tokens.  even if ugi has credentials, don't attempt
232      // to get another token to match hdfs/rpc behavior
233      if (token != null) {
234        LOG.debug("Using UGI token: " + token);
235        canRefreshDelegationToken = false; 
236      } else {
237        token = getDelegationToken(null);
238        if (token != null) {
239          LOG.debug("Fetched new token: " + token);
240        } else { // security is disabled
241          canRefreshDelegationToken = false;
242        }
243      }
244      setDelegationToken(token);
245    }
246    return delegationToken;
247  }
248
249  @VisibleForTesting
250  synchronized boolean replaceExpiredDelegationToken() throws IOException {
251    boolean replaced = false;
252    if (canRefreshDelegationToken) {
253      Token<?> token = getDelegationToken(null);
254      LOG.debug("Replaced expired token: " + token);
255      setDelegationToken(token);
256      replaced = (token != null);
257    }
258    return replaced;
259  }
260
261  @Override
262  @VisibleForTesting
263  public int getDefaultPort() {
264    return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
265        DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
266  }
267
268  @Override
269  public URI getUri() {
270    return this.uri;
271  }
272  
273  @Override
274  protected URI canonicalizeUri(URI uri) {
275    return NetUtils.getCanonicalUri(uri, getDefaultPort());
276  }
277
278  /** @return the home directory. */
279  public static String getHomeDirectoryString(final UserGroupInformation ugi) {
280    return "/user/" + ugi.getShortUserName();
281  }
282
283  @Override
284  public Path getHomeDirectory() {
285    return makeQualified(new Path(getHomeDirectoryString(ugi)));
286  }
287
288  @Override
289  public synchronized Path getWorkingDirectory() {
290    return workingDir;
291  }
292
293  @Override
294  public synchronized void setWorkingDirectory(final Path dir) {
295    String result = makeAbsolute(dir).toUri().getPath();
296    if (!DFSUtil.isValidName(result)) {
297      throw new IllegalArgumentException("Invalid DFS directory name " + 
298                                         result);
299    }
300    workingDir = makeAbsolute(dir);
301  }
302
303  private Path makeAbsolute(Path f) {
304    return f.isAbsolute()? f: new Path(workingDir, f);
305  }
306
307  static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
308      ) throws IOException {
309    if (c.getContentLength() == 0) {
310      return null;
311    }
312    final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
313    if (in == null) {
314      throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
315    }
316    final String contentType = c.getContentType();
317    if (contentType != null) {
318      final MediaType parsed = MediaType.valueOf(contentType);
319      if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
320        throw new IOException("Content-Type \"" + contentType
321            + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
322            + "\" (parsed=\"" + parsed + "\")");
323      }
324    }
325    return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
326  }
327
328  private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
329      final HttpURLConnection conn, boolean unwrapException) throws IOException {
330    final int code = conn.getResponseCode();
331    // server is demanding an authentication we don't support
332    if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
333      // match hdfs/rpc exception
334      throw new AccessControlException(conn.getResponseMessage());
335    }
336    if (code != op.getExpectedHttpResponseCode()) {
337      final Map<?, ?> m;
338      try {
339        m = jsonParse(conn, true);
340      } catch(Exception e) {
341        throw new IOException("Unexpected HTTP response: code=" + code + " != "
342            + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
343            + ", message=" + conn.getResponseMessage(), e);
344      }
345
346      if (m == null) {
347        throw new IOException("Unexpected HTTP response: code=" + code + " != "
348            + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
349            + ", message=" + conn.getResponseMessage());
350      } else if (m.get(RemoteException.class.getSimpleName()) == null) {
351        return m;
352      }
353
354      IOException re = JsonUtil.toRemoteException(m);
355      // extract UGI-related exceptions and unwrap InvalidToken
356      // the NN mangles these exceptions but the DN does not and may need
357      // to re-fetch a token if either report the token is expired
358      if (re.getMessage() != null && re.getMessage().startsWith("Failed to obtain user group information:")) {
359        String[] parts = re.getMessage().split(":\\s+", 3);
360        re = new RemoteException(parts[1], parts[2]);
361        re = ((RemoteException)re).unwrapRemoteException(InvalidToken.class);
362      }
363      throw unwrapException? toIOException(re): re;
364    }
365    return null;
366  }
367
368  /**
369   * Covert an exception to an IOException.
370   * 
371   * For a non-IOException, wrap it with IOException.
372   * For a RemoteException, unwrap it.
373   * For an IOException which is not a RemoteException, return it. 
374   */
375  private static IOException toIOException(Exception e) {
376    if (!(e instanceof IOException)) {
377      return new IOException(e);
378    }
379
380    final IOException ioe = (IOException)e;
381    if (!(ioe instanceof RemoteException)) {
382      return ioe;
383    }
384
385    return ((RemoteException)ioe).unwrapRemoteException();
386  }
387
388  private synchronized InetSocketAddress getCurrentNNAddr() {
389    return nnAddrs[currentNNAddrIndex];
390  }
391
392  /**
393   * Reset the appropriate state to gracefully fail over to another name node
394   */
395  private synchronized void resetStateToFailOver() {
396    currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length;
397  }
398
399  /**
400   * Return a URL pointing to given path on the namenode.
401   *
402   * @param path to obtain the URL for
403   * @param query string to append to the path
404   * @return namenode URL referring to the given path
405   * @throws IOException on error constructing the URL
406   */
407  private URL getNamenodeURL(String path, String query) throws IOException {
408    InetSocketAddress nnAddr = getCurrentNNAddr();
409    final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
410          nnAddr.getPort(), path + '?' + query);
411    if (LOG.isTraceEnabled()) {
412      LOG.trace("url=" + url);
413    }
414    return url;
415  }
416  
417  Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
418    List<Param<?,?>> authParams = Lists.newArrayList();    
419    // Skip adding delegation token for token operations because these
420    // operations require authentication.
421    Token<?> token = null;
422    if (!op.getRequireAuth()) {
423      token = getDelegationToken();
424    }
425    if (token != null) {
426      authParams.add(new DelegationParam(token.encodeToUrlString()));
427    } else {
428      UserGroupInformation userUgi = ugi;
429      UserGroupInformation realUgi = userUgi.getRealUser();
430      if (realUgi != null) { // proxy user
431        authParams.add(new DoAsParam(userUgi.getShortUserName()));
432        userUgi = realUgi;
433      }
434      authParams.add(new UserParam(userUgi.getShortUserName()));
435    }
436    return authParams.toArray(new Param<?,?>[0]);
437  }
438
439  URL toUrl(final HttpOpParam.Op op, final Path fspath,
440      final Param<?,?>... parameters) throws IOException {
441    //initialize URI path and query
442    final String path = PATH_PREFIX
443        + (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
444    final String query = op.toQueryString()
445        + Param.toSortedString("&", getAuthParameters(op))
446        + Param.toSortedString("&", parameters);
447    final URL url = getNamenodeURL(path, query);
448    if (LOG.isTraceEnabled()) {
449      LOG.trace("url=" + url);
450    }
451    return url;
452  }
453
454  /**
455   * This class is for initialing a HTTP connection, connecting to server,
456   * obtaining a response, and also handling retry on failures.
457   */
458  abstract class AbstractRunner<T> {
459    abstract protected URL getUrl() throws IOException;
460
461    protected final HttpOpParam.Op op;
462    private final boolean redirected;
463    protected ExcludeDatanodesParam excludeDatanodes = new ExcludeDatanodesParam("");
464
465    private boolean checkRetry;
466
467    protected AbstractRunner(final HttpOpParam.Op op, boolean redirected) {
468      this.op = op;
469      this.redirected = redirected;
470    }
471
472    T run() throws IOException {
473      UserGroupInformation connectUgi = ugi.getRealUser();
474      if (connectUgi == null) {
475        connectUgi = ugi;
476      }
477      if (op.getRequireAuth()) {
478        connectUgi.checkTGTAndReloginFromKeytab();
479      }
480      try {
481        // the entire lifecycle of the connection must be run inside the
482        // doAs to ensure authentication is performed correctly
483        return connectUgi.doAs(
484            new PrivilegedExceptionAction<T>() {
485              @Override
486              public T run() throws IOException {
487                return runWithRetry();
488              }
489            });
490      } catch (InterruptedException e) {
491        throw new IOException(e);
492      }
493    }
494
495    /**
496     * Two-step requests redirected to a DN
497     * 
498     * Create/Append:
499     * Step 1) Submit a Http request with neither auto-redirect nor data. 
500     * Step 2) Submit another Http request with the URL from the Location header with data.
501     * 
502     * The reason of having two-step create/append is for preventing clients to
503     * send out the data before the redirect. This issue is addressed by the
504     * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
505     * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
506     * and Java 6 http client), which do not correctly implement "Expect:
507     * 100-continue". The two-step create/append is a temporary workaround for
508     * the software library bugs.
509     * 
510     * Open/Checksum
511     * Also implements two-step connects for other operations redirected to
512     * a DN such as open and checksum
513     */
514    private HttpURLConnection connect(URL url) throws IOException {
515      //redirect hostname and port
516      String redirectHost = null;
517
518      
519      // resolve redirects for a DN operation unless already resolved
520      if (op.getRedirect() && !redirected) {
521        final HttpOpParam.Op redirectOp =
522            HttpOpParam.TemporaryRedirectOp.valueOf(op);
523        final HttpURLConnection conn = connect(redirectOp, url);
524        // application level proxy like httpfs might not issue a redirect
525        if (conn.getResponseCode() == op.getExpectedHttpResponseCode()) {
526          return conn;
527        }
528        try {
529          validateResponse(redirectOp, conn, false);
530          url = new URL(conn.getHeaderField("Location"));
531          redirectHost = url.getHost() + ":" + url.getPort();
532        } finally {
533          conn.disconnect();
534        }
535      }
536      try {
537        return connect(op, url);
538      } catch (IOException ioe) {
539        if (redirectHost != null) {
540          if (excludeDatanodes.getValue() != null) {
541            excludeDatanodes = new ExcludeDatanodesParam(redirectHost + ","
542                + excludeDatanodes.getValue());
543          } else {
544            excludeDatanodes = new ExcludeDatanodesParam(redirectHost);
545          }
546        }
547        throw ioe;
548      }      
549    }
550
551    private HttpURLConnection connect(final HttpOpParam.Op op, final URL url)
552        throws IOException {
553      final HttpURLConnection conn =
554          (HttpURLConnection)connectionFactory.openConnection(url);
555      final boolean doOutput = op.getDoOutput();
556      conn.setRequestMethod(op.getType().toString());
557      conn.setInstanceFollowRedirects(false);
558      switch (op.getType()) {
559        // if not sending a message body for a POST or PUT operation, need
560        // to ensure the server/proxy knows this 
561        case POST:
562        case PUT: {
563          conn.setDoOutput(true);
564          if (!doOutput) {
565            // explicitly setting content-length to 0 won't do spnego!!
566            // opening and closing the stream will send "Content-Length: 0"
567            conn.getOutputStream().close();
568          } else {
569            conn.setRequestProperty("Content-Type",
570                MediaType.APPLICATION_OCTET_STREAM);
571            conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
572          }
573          break;
574        }
575        default: {
576          conn.setDoOutput(doOutput);
577          break;
578        }
579      }
580      conn.connect();
581      return conn;
582    }
583
584    private T runWithRetry() throws IOException {
585      /**
586       * Do the real work.
587       *
588       * There are three cases that the code inside the loop can throw an
589       * IOException:
590       *
591       * <ul>
592       * <li>The connection has failed (e.g., ConnectException,
593       * @see FailoverOnNetworkExceptionRetry for more details)</li>
594       * <li>The namenode enters the standby state (i.e., StandbyException).</li>
595       * <li>The server returns errors for the command (i.e., RemoteException)</li>
596       * </ul>
597       *
598       * The call to shouldRetry() will conduct the retry policy. The policy
599       * examines the exception and swallows it if it decides to rerun the work.
600       */
601      for(int retry = 0; ; retry++) {
602        checkRetry = !redirected;
603        final URL url = getUrl();
604        try {
605          final HttpURLConnection conn = connect(url);
606          // output streams will validate on close
607          if (!op.getDoOutput()) {
608            validateResponse(op, conn, false);
609          }
610          return getResponse(conn);
611        } catch (AccessControlException ace) {
612          // no retries for auth failures
613          throw ace;
614        } catch (InvalidToken it) {
615          // try to replace the expired token with a new one.  the attempt
616          // to acquire a new token must be outside this operation's retry
617          // so if it fails after its own retries, this operation fails too.
618          if (op.getRequireAuth() || !replaceExpiredDelegationToken()) {
619            throw it;
620          }
621        } catch (IOException ioe) {
622          shouldRetry(ioe, retry);
623        }
624      }
625    }
626
627    private void shouldRetry(final IOException ioe, final int retry
628        ) throws IOException {
629      InetSocketAddress nnAddr = getCurrentNNAddr();
630      if (checkRetry) {
631        try {
632          final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
633              ioe, retry, 0, true);
634
635          boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
636          boolean isFailoverAndRetry =
637              a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
638
639          if (isRetry || isFailoverAndRetry) {
640            LOG.info("Retrying connect to namenode: " + nnAddr
641                + ". Already tried " + retry + " time(s); retry policy is "
642                + retryPolicy + ", delay " + a.delayMillis + "ms.");
643
644            if (isFailoverAndRetry) {
645              resetStateToFailOver();
646            }
647
648            Thread.sleep(a.delayMillis);
649            return;
650          }
651        } catch(Exception e) {
652          LOG.warn("Original exception is ", ioe);
653          throw toIOException(e);
654        }
655      }
656      throw toIOException(ioe);
657    }
658
659    abstract T getResponse(HttpURLConnection conn) throws IOException;
660  }
661
662  /**
663   * Abstract base class to handle path-based operations with params
664   */
665  abstract class AbstractFsPathRunner<T> extends AbstractRunner<T> {
666    private final Path fspath;
667    private final Param<?,?>[] parameters;
668    
669    AbstractFsPathRunner(final HttpOpParam.Op op, final Path fspath,
670        Param<?,?>... parameters) {
671      super(op, false);
672      this.fspath = fspath;
673      this.parameters = parameters;
674    }
675    
676    AbstractFsPathRunner(final HttpOpParam.Op op, Param<?,?>[] parameters,
677        final Path fspath) {
678      super(op, false);
679      this.fspath = fspath;
680      this.parameters = parameters;
681    }
682    
683    @Override
684    protected URL getUrl() throws IOException {
685      if (excludeDatanodes.getValue() != null) {
686        Param<?, ?>[] tmpParam = new Param<?, ?>[parameters.length + 1];
687        System.arraycopy(parameters, 0, tmpParam, 0, parameters.length);
688        tmpParam[parameters.length] = excludeDatanodes;
689        return toUrl(op, fspath, tmpParam);
690      } else {
691        return toUrl(op, fspath, parameters);
692      }
693    }
694  }
695
696  /**
697   * Default path-based implementation expects no json response
698   */
699  class FsPathRunner extends AbstractFsPathRunner<Void> {
700    FsPathRunner(Op op, Path fspath, Param<?,?>... parameters) {
701      super(op, fspath, parameters);
702    }
703    
704    @Override
705    Void getResponse(HttpURLConnection conn) throws IOException {
706      return null;
707    }
708  }
709
710  /**
711   * Handle path-based operations with a json response
712   */
713  abstract class FsPathResponseRunner<T> extends AbstractFsPathRunner<T> {
714    FsPathResponseRunner(final HttpOpParam.Op op, final Path fspath,
715        Param<?,?>... parameters) {
716      super(op, fspath, parameters);
717    }
718    
719    FsPathResponseRunner(final HttpOpParam.Op op, Param<?,?>[] parameters,
720        final Path fspath) {
721      super(op, parameters, fspath);
722    }
723    
724    @Override
725    final T getResponse(HttpURLConnection conn) throws IOException {
726      try {
727        final Map<?,?> json = jsonParse(conn, false);
728        if (json == null) {
729          // match exception class thrown by parser
730          throw new IllegalStateException("Missing response");
731        }
732        return decodeResponse(json);
733      } catch (IOException ioe) {
734        throw ioe;
735      } catch (Exception e) { // catch json parser errors
736        final IOException ioe =
737            new IOException("Response decoding failure: "+e.toString(), e);
738        if (LOG.isDebugEnabled()) {
739          LOG.debug(ioe);
740        }
741        throw ioe;
742      } finally {
743        conn.disconnect();
744      }
745    }
746    
747    abstract T decodeResponse(Map<?,?> json) throws IOException;
748  }
749
750  /**
751   * Handle path-based operations with json boolean response
752   */
753  class FsPathBooleanRunner extends FsPathResponseRunner<Boolean> {
754    FsPathBooleanRunner(Op op, Path fspath, Param<?,?>... parameters) {
755      super(op, fspath, parameters);
756    }
757    
758    @Override
759    Boolean decodeResponse(Map<?,?> json) throws IOException {
760      return (Boolean)json.get("boolean");
761    }
762  }
763
764  /**
765   * Handle create/append output streams
766   */
767  class FsPathOutputStreamRunner extends AbstractFsPathRunner<FSDataOutputStream> {
768    private final int bufferSize;
769    
770    FsPathOutputStreamRunner(Op op, Path fspath, int bufferSize,
771        Param<?,?>... parameters) {
772      super(op, fspath, parameters);
773      this.bufferSize = bufferSize;
774    }
775    
776    @Override
777    FSDataOutputStream getResponse(final HttpURLConnection conn)
778        throws IOException {
779      return new FSDataOutputStream(new BufferedOutputStream(
780          conn.getOutputStream(), bufferSize), statistics) {
781        @Override
782        public void close() throws IOException {
783          try {
784            super.close();
785          } finally {
786            try {
787              validateResponse(op, conn, true);
788            } finally {
789              conn.disconnect();
790            }
791          }
792        }
793      };
794    }
795  }
796
797  class FsPathConnectionRunner extends AbstractFsPathRunner<HttpURLConnection> {
798    FsPathConnectionRunner(Op op, Path fspath, Param<?,?>... parameters) {
799      super(op, fspath, parameters);
800    }
801    @Override
802    HttpURLConnection getResponse(final HttpURLConnection conn)
803        throws IOException {
804      return conn;
805    }
806  }
807  
808  /**
809   * Used by open() which tracks the resolved url itself
810   */
811  final class URLRunner extends AbstractRunner<HttpURLConnection> {
812    private final URL url;
813    @Override
814    protected URL getUrl() {
815      return url;
816    }
817
818    protected URLRunner(final HttpOpParam.Op op, final URL url, boolean redirected) {
819      super(op, redirected);
820      this.url = url;
821    }
822
823    @Override
824    HttpURLConnection getResponse(HttpURLConnection conn) throws IOException {
825      return conn;
826    }
827  }
828
829  private FsPermission applyUMask(FsPermission permission) {
830    if (permission == null) {
831      permission = FsPermission.getDefault();
832    }
833    return permission.applyUMask(FsPermission.getUMask(getConf()));
834  }
835
836  private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
837    final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
838    HdfsFileStatus status = new FsPathResponseRunner<HdfsFileStatus>(op, f) {
839      @Override
840      HdfsFileStatus decodeResponse(Map<?,?> json) {
841        return JsonUtil.toFileStatus(json, true);
842      }
843    }.run();
844    if (status == null) {
845      throw new FileNotFoundException("File does not exist: " + f);
846    }
847    return status;
848  }
849
850  @Override
851  public FileStatus getFileStatus(Path f) throws IOException {
852    statistics.incrementReadOps(1);
853    return makeQualified(getHdfsFileStatus(f), f);
854  }
855
856  private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
857    return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
858        f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
859        f.getPermission(), f.getOwner(), f.getGroup(),
860        f.isSymlink() ? new Path(f.getSymlink()) : null,
861        f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
862  }
863
864  @Override
865  public AclStatus getAclStatus(Path f) throws IOException {
866    final HttpOpParam.Op op = GetOpParam.Op.GETACLSTATUS;
867    AclStatus status = new FsPathResponseRunner<AclStatus>(op, f) {
868      @Override
869      AclStatus decodeResponse(Map<?,?> json) {
870        return JsonUtil.toAclStatus(json);
871      }
872    }.run();
873    if (status == null) {
874      throw new FileNotFoundException("File does not exist: " + f);
875    }
876    return status;
877  }
878
879  @Override
880  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
881    statistics.incrementWriteOps(1);
882    final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
883    return new FsPathBooleanRunner(op, f,
884        new PermissionParam(applyUMask(permission))
885    ).run();
886  }
887
888  /**
889   * Create a symlink pointing to the destination path.
890   * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean) 
891   */
892  public void createSymlink(Path destination, Path f, boolean createParent
893      ) throws IOException {
894    statistics.incrementWriteOps(1);
895    final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
896    new FsPathRunner(op, f,
897        new DestinationParam(makeQualified(destination).toUri().getPath()),
898        new CreateParentParam(createParent)
899    ).run();
900  }
901
902  @Override
903  public boolean rename(final Path src, final Path dst) throws IOException {
904    statistics.incrementWriteOps(1);
905    final HttpOpParam.Op op = PutOpParam.Op.RENAME;
906    return new FsPathBooleanRunner(op, src,
907        new DestinationParam(makeQualified(dst).toUri().getPath())
908    ).run();
909  }
910
911  @SuppressWarnings("deprecation")
912  @Override
913  public void rename(final Path src, final Path dst,
914      final Options.Rename... options) throws IOException {
915    statistics.incrementWriteOps(1);
916    final HttpOpParam.Op op = PutOpParam.Op.RENAME;
917    new FsPathRunner(op, src,
918        new DestinationParam(makeQualified(dst).toUri().getPath()),
919        new RenameOptionSetParam(options)
920    ).run();
921  }
922  
923  @Override
924  public void setXAttr(Path p, String name, byte[] value, 
925      EnumSet<XAttrSetFlag> flag) throws IOException {
926    statistics.incrementWriteOps(1);
927    final HttpOpParam.Op op = PutOpParam.Op.SETXATTR;
928    if (value != null) {
929      new FsPathRunner(op, p, new XAttrNameParam(name), new XAttrValueParam(
930          XAttrCodec.encodeValue(value, XAttrCodec.HEX)), 
931          new XAttrSetFlagParam(flag)).run();
932    } else {
933      new FsPathRunner(op, p, new XAttrNameParam(name), 
934          new XAttrSetFlagParam(flag)).run();
935    }
936  }
937  
938  @Override
939  public byte[] getXAttr(Path p, final String name) throws IOException {
940    final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
941    return new FsPathResponseRunner<byte[]>(op, p, new XAttrNameParam(name), 
942        new XAttrEncodingParam(XAttrCodec.HEX)) {
943      @Override
944      byte[] decodeResponse(Map<?, ?> json) throws IOException {
945        return JsonUtil.getXAttr(json, name);
946      }
947    }.run();
948  }
949  
950  @Override
951  public Map<String, byte[]> getXAttrs(Path p) throws IOException {
952    final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
953    return new FsPathResponseRunner<Map<String, byte[]>>(op, p, 
954        new XAttrEncodingParam(XAttrCodec.HEX)) {
955      @Override
956      Map<String, byte[]> decodeResponse(Map<?, ?> json) throws IOException {
957        return JsonUtil.toXAttrs(json);
958      }
959    }.run();
960  }
961  
962  @Override
963  public Map<String, byte[]> getXAttrs(Path p, final List<String> names) 
964      throws IOException {
965    Preconditions.checkArgument(names != null && !names.isEmpty(), 
966        "XAttr names cannot be null or empty.");
967    Param<?,?>[] parameters = new Param<?,?>[names.size() + 1];
968    for (int i = 0; i < parameters.length - 1; i++) {
969      parameters[i] = new XAttrNameParam(names.get(i));
970    }
971    parameters[parameters.length - 1] = new XAttrEncodingParam(XAttrCodec.HEX);
972    
973    final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
974    return new FsPathResponseRunner<Map<String, byte[]>>(op, parameters, p) {
975      @Override
976      Map<String, byte[]> decodeResponse(Map<?, ?> json) throws IOException {
977        return JsonUtil.toXAttrs(json);
978      }
979    }.run();
980  }
981  
982  @Override
983  public List<String> listXAttrs(Path p) throws IOException {
984    final HttpOpParam.Op op = GetOpParam.Op.LISTXATTRS;
985    return new FsPathResponseRunner<List<String>>(op, p) {
986      @Override
987      List<String> decodeResponse(Map<?, ?> json) throws IOException {
988        return JsonUtil.toXAttrNames(json);
989      }
990    }.run();
991  }
992
993  @Override
994  public void removeXAttr(Path p, String name) throws IOException {
995    statistics.incrementWriteOps(1);
996    final HttpOpParam.Op op = PutOpParam.Op.REMOVEXATTR;
997    new FsPathRunner(op, p, new XAttrNameParam(name)).run();
998  }
999
1000  @Override
1001  public void setOwner(final Path p, final String owner, final String group
1002      ) throws IOException {
1003    if (owner == null && group == null) {
1004      throw new IOException("owner == null && group == null");
1005    }
1006
1007    statistics.incrementWriteOps(1);
1008    final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
1009    new FsPathRunner(op, p,
1010        new OwnerParam(owner), new GroupParam(group)
1011    ).run();
1012  }
1013
1014  @Override
1015  public void setPermission(final Path p, final FsPermission permission
1016      ) throws IOException {
1017    statistics.incrementWriteOps(1);
1018    final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
1019    new FsPathRunner(op, p,new PermissionParam(permission)).run();
1020  }
1021
1022  @Override
1023  public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
1024      throws IOException {
1025    statistics.incrementWriteOps(1);
1026    final HttpOpParam.Op op = PutOpParam.Op.MODIFYACLENTRIES;
1027    new FsPathRunner(op, path, new AclPermissionParam(aclSpec)).run();
1028  }
1029
1030  @Override
1031  public void removeAclEntries(Path path, List<AclEntry> aclSpec)
1032      throws IOException {
1033    statistics.incrementWriteOps(1);
1034    final HttpOpParam.Op op = PutOpParam.Op.REMOVEACLENTRIES;
1035    new FsPathRunner(op, path, new AclPermissionParam(aclSpec)).run();
1036  }
1037
1038  @Override
1039  public void removeDefaultAcl(Path path) throws IOException {
1040    statistics.incrementWriteOps(1);
1041    final HttpOpParam.Op op = PutOpParam.Op.REMOVEDEFAULTACL;
1042    new FsPathRunner(op, path).run();
1043  }
1044
1045  @Override
1046  public void removeAcl(Path path) throws IOException {
1047    statistics.incrementWriteOps(1);
1048    final HttpOpParam.Op op = PutOpParam.Op.REMOVEACL;
1049    new FsPathRunner(op, path).run();
1050  }
1051
1052  @Override
1053  public void setAcl(final Path p, final List<AclEntry> aclSpec)
1054      throws IOException {
1055    statistics.incrementWriteOps(1);
1056    final HttpOpParam.Op op = PutOpParam.Op.SETACL;
1057    new FsPathRunner(op, p, new AclPermissionParam(aclSpec)).run();
1058  }
1059
1060  @Override
1061  public Path createSnapshot(final Path path, final String snapshotName) 
1062      throws IOException {
1063    statistics.incrementWriteOps(1);
1064    final HttpOpParam.Op op = PutOpParam.Op.CREATESNAPSHOT;
1065    Path spath = new FsPathResponseRunner<Path>(op, path,
1066        new SnapshotNameParam(snapshotName)) {
1067      @Override
1068      Path decodeResponse(Map<?,?> json) {
1069        return new Path((String) json.get(Path.class.getSimpleName()));
1070      }
1071    }.run();
1072    return spath;
1073  }
1074
1075  @Override
1076  public void deleteSnapshot(final Path path, final String snapshotName)
1077      throws IOException {
1078    statistics.incrementWriteOps(1);
1079    final HttpOpParam.Op op = DeleteOpParam.Op.DELETESNAPSHOT;
1080    new FsPathRunner(op, path, new SnapshotNameParam(snapshotName)).run();
1081  }
1082
1083  @Override
1084  public void renameSnapshot(final Path path, final String snapshotOldName,
1085      final String snapshotNewName) throws IOException {
1086    statistics.incrementWriteOps(1);
1087    final HttpOpParam.Op op = PutOpParam.Op.RENAMESNAPSHOT;
1088    new FsPathRunner(op, path, new OldSnapshotNameParam(snapshotOldName),
1089        new SnapshotNameParam(snapshotNewName)).run();
1090  }
1091
1092  @Override
1093  public boolean setReplication(final Path p, final short replication
1094     ) throws IOException {
1095    statistics.incrementWriteOps(1);
1096    final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
1097    return new FsPathBooleanRunner(op, p,
1098        new ReplicationParam(replication)
1099    ).run();
1100  }
1101
1102  @Override
1103  public void setTimes(final Path p, final long mtime, final long atime
1104      ) throws IOException {
1105    statistics.incrementWriteOps(1);
1106    final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
1107    new FsPathRunner(op, p,
1108        new ModificationTimeParam(mtime),
1109        new AccessTimeParam(atime)
1110    ).run();
1111  }
1112
1113  @Override
1114  public long getDefaultBlockSize() {
1115    return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
1116        DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
1117  }
1118
1119  @Override
1120  public short getDefaultReplication() {
1121    return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
1122        DFSConfigKeys.DFS_REPLICATION_DEFAULT);
1123  }
1124
1125  @Override
1126  public void concat(final Path trg, final Path [] srcs) throws IOException {
1127    statistics.incrementWriteOps(1);
1128    final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
1129    new FsPathRunner(op, trg, new ConcatSourcesParam(srcs)).run();
1130  }
1131
1132  @Override
1133  public FSDataOutputStream create(final Path f, final FsPermission permission,
1134      final boolean overwrite, final int bufferSize, final short replication,
1135      final long blockSize, final Progressable progress) throws IOException {
1136    statistics.incrementWriteOps(1);
1137
1138    final HttpOpParam.Op op = PutOpParam.Op.CREATE;
1139    return new FsPathOutputStreamRunner(op, f, bufferSize,
1140        new PermissionParam(applyUMask(permission)),
1141        new OverwriteParam(overwrite),
1142        new BufferSizeParam(bufferSize),
1143        new ReplicationParam(replication),
1144        new BlockSizeParam(blockSize)
1145    ).run();
1146  }
1147
1148  @Override
1149  public FSDataOutputStream append(final Path f, final int bufferSize,
1150      final Progressable progress) throws IOException {
1151    statistics.incrementWriteOps(1);
1152
1153    final HttpOpParam.Op op = PostOpParam.Op.APPEND;
1154    return new FsPathOutputStreamRunner(op, f, bufferSize,
1155        new BufferSizeParam(bufferSize)
1156    ).run();
1157  }
1158
1159  @Override
1160  public boolean delete(Path f, boolean recursive) throws IOException {
1161    final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
1162    return new FsPathBooleanRunner(op, f,
1163        new RecursiveParam(recursive)
1164    ).run();
1165  }
1166
1167  @Override
1168  public FSDataInputStream open(final Path f, final int buffersize
1169      ) throws IOException {
1170    statistics.incrementReadOps(1);
1171    final HttpOpParam.Op op = GetOpParam.Op.OPEN;
1172    // use a runner so the open can recover from an invalid token
1173    FsPathConnectionRunner runner =
1174        new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize));
1175    return new FSDataInputStream(new OffsetUrlInputStream(
1176        new UnresolvedUrlOpener(runner), new OffsetUrlOpener(null)));
1177  }
1178
1179  @Override
1180  public synchronized void close() throws IOException {
1181    try {
1182      if (canRefreshDelegationToken && delegationToken != null) {
1183        cancelDelegationToken(delegationToken);
1184      }
1185    } catch (IOException ioe) {
1186      LOG.debug("Token cancel failed: "+ioe);
1187    } finally {
1188      super.close();
1189    }
1190  }
1191
1192  // use FsPathConnectionRunner to ensure retries for InvalidTokens
1193  class UnresolvedUrlOpener extends ByteRangeInputStream.URLOpener {
1194    private final FsPathConnectionRunner runner;
1195    UnresolvedUrlOpener(FsPathConnectionRunner runner) {
1196      super(null);
1197      this.runner = runner;
1198    }
1199
1200    @Override
1201    protected HttpURLConnection connect(long offset, boolean resolved)
1202        throws IOException {
1203      assert offset == 0;
1204      HttpURLConnection conn = runner.run();
1205      setURL(conn.getURL());
1206      return conn;
1207    }
1208  }
1209
1210  class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
1211    OffsetUrlOpener(final URL url) {
1212      super(url);
1213    }
1214
1215    /** Setup offset url and connect. */
1216    @Override
1217    protected HttpURLConnection connect(final long offset,
1218        final boolean resolved) throws IOException {
1219      final URL offsetUrl = offset == 0L? url
1220          : new URL(url + "&" + new OffsetParam(offset));
1221      return new URLRunner(GetOpParam.Op.OPEN, offsetUrl, resolved).run();
1222    }  
1223  }
1224
1225  private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
1226
1227  /** Remove offset parameter, if there is any, from the url */
1228  static URL removeOffsetParam(final URL url) throws MalformedURLException {
1229    String query = url.getQuery();
1230    if (query == null) {
1231      return url;
1232    }
1233    final String lower = query.toLowerCase();
1234    if (!lower.startsWith(OFFSET_PARAM_PREFIX)
1235        && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
1236      return url;
1237    }
1238
1239    //rebuild query
1240    StringBuilder b = null;
1241    for(final StringTokenizer st = new StringTokenizer(query, "&");
1242        st.hasMoreTokens();) {
1243      final String token = st.nextToken();
1244      if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
1245        if (b == null) {
1246          b = new StringBuilder("?").append(token);
1247        } else {
1248          b.append('&').append(token);
1249        }
1250      }
1251    }
1252    query = b == null? "": b.toString();
1253
1254    final String urlStr = url.toString();
1255    return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
1256  }
1257
1258  static class OffsetUrlInputStream extends ByteRangeInputStream {
1259    OffsetUrlInputStream(UnresolvedUrlOpener o, OffsetUrlOpener r)
1260        throws IOException {
1261      super(o, r);
1262    }
1263
1264    /** Remove offset parameter before returning the resolved url. */
1265    @Override
1266    protected URL getResolvedUrl(final HttpURLConnection connection
1267        ) throws MalformedURLException {
1268      return removeOffsetParam(connection.getURL());
1269    }
1270  }
1271
1272  @Override
1273  public FileStatus[] listStatus(final Path f) throws IOException {
1274    statistics.incrementReadOps(1);
1275
1276    final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
1277    return new FsPathResponseRunner<FileStatus[]>(op, f) {
1278      @Override
1279      FileStatus[] decodeResponse(Map<?,?> json) {
1280        final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
1281        final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
1282
1283        //convert FileStatus
1284        final FileStatus[] statuses = new FileStatus[array.length];
1285        for (int i = 0; i < array.length; i++) {
1286          final Map<?, ?> m = (Map<?, ?>)array[i];
1287          statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
1288        }
1289        return statuses;
1290      }
1291    }.run();
1292  }
1293
1294  @Override
1295  public Token<DelegationTokenIdentifier> getDelegationToken(
1296      final String renewer) throws IOException {
1297    final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
1298    Token<DelegationTokenIdentifier> token =
1299        new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
1300            op, null, new RenewerParam(renewer)) {
1301      @Override
1302      Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
1303          throws IOException {
1304        return JsonUtil.toDelegationToken(json);
1305      }
1306    }.run();
1307    if (token != null) {
1308      token.setService(tokenServiceName);
1309    } else {
1310      if (disallowFallbackToInsecureCluster) {
1311        throw new AccessControlException(CANT_FALLBACK_TO_INSECURE_MSG);
1312      }
1313    }
1314    return token;
1315  }
1316
1317  @Override
1318  public synchronized Token<?> getRenewToken() {
1319    return delegationToken;
1320  }
1321
1322  @Override
1323  public <T extends TokenIdentifier> void setDelegationToken(
1324      final Token<T> token) {
1325    synchronized (this) {
1326      delegationToken = token;
1327    }
1328  }
1329
1330  @Override
1331  public synchronized long renewDelegationToken(final Token<?> token
1332      ) throws IOException {
1333    final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
1334    return new FsPathResponseRunner<Long>(op, null,
1335        new TokenArgumentParam(token.encodeToUrlString())) {
1336      @Override
1337      Long decodeResponse(Map<?,?> json) throws IOException {
1338        return (Long) json.get("long");
1339      }
1340    }.run();
1341  }
1342
1343  @Override
1344  public synchronized void cancelDelegationToken(final Token<?> token
1345      ) throws IOException {
1346    final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
1347    new FsPathRunner(op, null,
1348        new TokenArgumentParam(token.encodeToUrlString())
1349    ).run();
1350  }
1351  
1352  @Override
1353  public BlockLocation[] getFileBlockLocations(final FileStatus status,
1354      final long offset, final long length) throws IOException {
1355    if (status == null) {
1356      return null;
1357    }
1358    return getFileBlockLocations(status.getPath(), offset, length);
1359  }
1360
1361  @Override
1362  public BlockLocation[] getFileBlockLocations(final Path p, 
1363      final long offset, final long length) throws IOException {
1364    statistics.incrementReadOps(1);
1365
1366    final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
1367    return new FsPathResponseRunner<BlockLocation[]>(op, p,
1368        new OffsetParam(offset), new LengthParam(length)) {
1369      @Override
1370      BlockLocation[] decodeResponse(Map<?,?> json) throws IOException {
1371        return DFSUtil.locatedBlocks2Locations(
1372            JsonUtil.toLocatedBlocks(json));
1373      }
1374    }.run();
1375  }
1376
1377  @Override
1378  public void access(final Path path, final FsAction mode) throws IOException {
1379    final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
1380    new FsPathRunner(op, path, new FsActionParam(mode)).run();
1381  }
1382
1383  @Override
1384  public ContentSummary getContentSummary(final Path p) throws IOException {
1385    statistics.incrementReadOps(1);
1386
1387    final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
1388    return new FsPathResponseRunner<ContentSummary>(op, p) {
1389      @Override
1390      ContentSummary decodeResponse(Map<?,?> json) {
1391        return JsonUtil.toContentSummary(json);        
1392      }
1393    }.run();
1394  }
1395
1396  @Override
1397  public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
1398      ) throws IOException {
1399    statistics.incrementReadOps(1);
1400  
1401    final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
1402    return new FsPathResponseRunner<MD5MD5CRC32FileChecksum>(op, p) {
1403      @Override
1404      MD5MD5CRC32FileChecksum decodeResponse(Map<?,?> json) throws IOException {
1405        return JsonUtil.toMD5MD5CRC32FileChecksum(json);
1406      }
1407    }.run();
1408  }
1409
1410  /**
1411   * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS
1412   * resolver when the URL points to an non-HA cluster. When the URL points to
1413   * an HA cluster with its logical name, the resolver further resolves the
1414   * logical name(i.e., the authority in the URL) into real namenode addresses.
1415   */
1416  private InetSocketAddress[] resolveNNAddr() throws IOException {
1417    Configuration conf = getConf();
1418    final String scheme = uri.getScheme();
1419
1420    ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>();
1421
1422    if (!HAUtil.isLogicalUri(conf, uri)) {
1423      InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(),
1424          getDefaultPort());
1425      ret.add(addr);
1426
1427    } else {
1428      Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil
1429          .getHaNnWebHdfsAddresses(conf, scheme);
1430
1431      // Extract the entry corresponding to the logical name.
1432      Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost());
1433      for (InetSocketAddress addr : addrs.values()) {
1434        ret.add(addr);
1435      }
1436    }
1437
1438    InetSocketAddress[] r = new InetSocketAddress[ret.size()];
1439    return ret.toArray(r);
1440  }
1441
1442  @Override
1443  public String getCanonicalServiceName() {
1444    return tokenServiceName == null ? super.getCanonicalServiceName()
1445        : tokenServiceName.toString();
1446  }
1447
1448  @VisibleForTesting
1449  InetSocketAddress[] getResolvedNNAddr() {
1450    return nnAddrs;
1451  }
1452}