001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URL;
031 import java.security.PrivilegedExceptionAction;
032 import java.util.List;
033 import java.util.Map;
034 import java.util.StringTokenizer;
035
036 import javax.ws.rs.core.MediaType;
037
038 import org.apache.commons.logging.Log;
039 import org.apache.commons.logging.LogFactory;
040 import org.apache.hadoop.conf.Configuration;
041 import org.apache.hadoop.fs.BlockLocation;
042 import org.apache.hadoop.fs.ContentSummary;
043 import org.apache.hadoop.fs.DelegationTokenRenewer;
044 import org.apache.hadoop.fs.FSDataInputStream;
045 import org.apache.hadoop.fs.FSDataOutputStream;
046 import org.apache.hadoop.fs.FileStatus;
047 import org.apache.hadoop.fs.FileSystem;
048 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
049 import org.apache.hadoop.fs.Options;
050 import org.apache.hadoop.fs.Path;
051 import org.apache.hadoop.fs.permission.AclEntry;
052 import org.apache.hadoop.fs.permission.AclStatus;
053 import org.apache.hadoop.fs.permission.FsPermission;
054 import org.apache.hadoop.hdfs.DFSConfigKeys;
055 import org.apache.hadoop.hdfs.DFSUtil;
056 import org.apache.hadoop.hdfs.HAUtil;
057 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
058 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
059 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
060 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
061 import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
062 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
063 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
064 import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
065 import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
066 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
067 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
068 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
069 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
070 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
071 import org.apache.hadoop.hdfs.web.resources.GroupParam;
072 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
073 import org.apache.hadoop.hdfs.web.resources.LengthParam;
074 import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
075 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
076 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
077 import org.apache.hadoop.hdfs.web.resources.OwnerParam;
078 import org.apache.hadoop.hdfs.web.resources.Param;
079 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
080 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
081 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
082 import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
083 import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
084 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
085 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
086 import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
087 import org.apache.hadoop.hdfs.web.resources.UserParam;
088 import org.apache.hadoop.io.Text;
089 import org.apache.hadoop.io.retry.RetryPolicies;
090 import org.apache.hadoop.io.retry.RetryPolicy;
091 import org.apache.hadoop.io.retry.RetryUtils;
092 import org.apache.hadoop.ipc.RemoteException;
093 import org.apache.hadoop.net.NetUtils;
094 import org.apache.hadoop.security.SecurityUtil;
095 import org.apache.hadoop.security.UserGroupInformation;
096 import org.apache.hadoop.security.authentication.client.AuthenticationException;
097 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
098 import org.apache.hadoop.security.token.Token;
099 import org.apache.hadoop.security.token.TokenIdentifier;
100 import org.apache.hadoop.util.Progressable;
101 import org.mortbay.util.ajax.JSON;
102
103 import com.google.common.annotations.VisibleForTesting;
104 import com.google.common.base.Charsets;
105 import com.google.common.collect.Lists;
106
107 /** A FileSystem for HDFS over the web. */
108 public class WebHdfsFileSystem extends FileSystem
109 implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
110 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
111 /** File System URI: {SCHEME}://namenode:port/path/to/file */
112 public static final String SCHEME = "webhdfs";
113 /** WebHdfs version. */
114 public static final int VERSION = 1;
115 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
116 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
117
118 /** Default connection factory may be overridden in tests to use smaller timeout values */
119 protected URLConnectionFactory connectionFactory;
120
121 /** Delegation token kind */
122 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
123 protected TokenAspect<? extends WebHdfsFileSystem> tokenAspect;
124
125 private UserGroupInformation ugi;
126 private URI uri;
127 private Token<?> delegationToken;
128 protected Text tokenServiceName;
129 private RetryPolicy retryPolicy = null;
130 private Path workingDir;
131 private InetSocketAddress nnAddrs[];
132 private int currentNNAddrIndex;
133
134 /**
135 * Return the protocol scheme for the FileSystem.
136 * <p/>
137 *
138 * @return <code>webhdfs</code>
139 */
140 @Override
141 public String getScheme() {
142 return SCHEME;
143 }
144
145 /**
146 * return the underlying transport protocol (http / https).
147 */
148 protected String getTransportScheme() {
149 return "http";
150 }
151
152 /**
153 * Initialize tokenAspect. This function is intended to
154 * be overridden by SWebHdfsFileSystem.
155 */
156 protected synchronized void initializeTokenAspect() {
157 tokenAspect = new TokenAspect<WebHdfsFileSystem>(this, tokenServiceName,
158 TOKEN_KIND);
159 }
160
161 @Override
162 public synchronized void initialize(URI uri, Configuration conf
163 ) throws IOException {
164 super.initialize(uri, conf);
165 setConf(conf);
166 /** set user pattern based on configuration file */
167 UserParam.setUserPattern(conf.get(
168 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
169 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
170
171 connectionFactory = URLConnectionFactory
172 .newDefaultURLConnectionFactory(conf);
173
174 ugi = UserGroupInformation.getCurrentUser();
175 this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
176 this.nnAddrs = DFSUtil.resolveWebHdfsUri(this.uri, conf);
177
178 boolean isHA = HAUtil.isLogicalUri(conf, this.uri);
179 // In non-HA case, the code needs to call getCanonicalUri() in order to
180 // handle the case where no port is specified in the URI
181 this.tokenServiceName = isHA ? HAUtil.buildTokenServiceForLogicalUri(uri)
182 : SecurityUtil.buildTokenService(getCanonicalUri());
183 initializeTokenAspect();
184
185 if (!isHA) {
186 this.retryPolicy =
187 RetryUtils.getDefaultRetryPolicy(
188 conf,
189 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
190 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
191 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
192 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
193 SafeModeException.class);
194 } else {
195
196 int maxFailoverAttempts = conf.getInt(
197 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
198 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
199 int maxRetryAttempts = conf.getInt(
200 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
201 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
202 int failoverSleepBaseMillis = conf.getInt(
203 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
204 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
205 int failoverSleepMaxMillis = conf.getInt(
206 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
207 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
208
209 this.retryPolicy = RetryPolicies
210 .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
211 maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis,
212 failoverSleepMaxMillis);
213 }
214
215 this.workingDir = getHomeDirectory();
216
217 if (UserGroupInformation.isSecurityEnabled()) {
218 tokenAspect.initDelegationToken(ugi);
219 }
220 }
221
222 @Override
223 public URI getCanonicalUri() {
224 return super.getCanonicalUri();
225 }
226
227 /** Is WebHDFS enabled in conf? */
228 public static boolean isEnabled(final Configuration conf, final Log log) {
229 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
230 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
231 return b;
232 }
233
234 protected synchronized Token<?> getDelegationToken() throws IOException {
235 tokenAspect.ensureTokenInitialized();
236 return delegationToken;
237 }
238
239 @Override
240 protected int getDefaultPort() {
241 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
242 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
243 }
244
245 @Override
246 public URI getUri() {
247 return this.uri;
248 }
249
250 @Override
251 protected URI canonicalizeUri(URI uri) {
252 return NetUtils.getCanonicalUri(uri, getDefaultPort());
253 }
254
255 /** @return the home directory. */
256 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
257 return "/user/" + ugi.getShortUserName();
258 }
259
260 @Override
261 public Path getHomeDirectory() {
262 return makeQualified(new Path(getHomeDirectoryString(ugi)));
263 }
264
265 @Override
266 public synchronized Path getWorkingDirectory() {
267 return workingDir;
268 }
269
270 @Override
271 public synchronized void setWorkingDirectory(final Path dir) {
272 String result = makeAbsolute(dir).toUri().getPath();
273 if (!DFSUtil.isValidName(result)) {
274 throw new IllegalArgumentException("Invalid DFS directory name " +
275 result);
276 }
277 workingDir = makeAbsolute(dir);
278 }
279
280 private Path makeAbsolute(Path f) {
281 return f.isAbsolute()? f: new Path(workingDir, f);
282 }
283
284 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
285 ) throws IOException {
286 if (c.getContentLength() == 0) {
287 return null;
288 }
289 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
290 if (in == null) {
291 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
292 }
293 final String contentType = c.getContentType();
294 if (contentType != null) {
295 final MediaType parsed = MediaType.valueOf(contentType);
296 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
297 throw new IOException("Content-Type \"" + contentType
298 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
299 + "\" (parsed=\"" + parsed + "\")");
300 }
301 }
302 return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
303 }
304
305 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
306 final HttpURLConnection conn, boolean unwrapException) throws IOException {
307 final int code = conn.getResponseCode();
308 // server is demanding an authentication we don't support
309 if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
310 throw new IOException(
311 new AuthenticationException(conn.getResponseMessage()));
312 }
313 if (code != op.getExpectedHttpResponseCode()) {
314 final Map<?, ?> m;
315 try {
316 m = jsonParse(conn, true);
317 } catch(Exception e) {
318 throw new IOException("Unexpected HTTP response: code=" + code + " != "
319 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
320 + ", message=" + conn.getResponseMessage(), e);
321 }
322
323 if (m == null) {
324 throw new IOException("Unexpected HTTP response: code=" + code + " != "
325 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
326 + ", message=" + conn.getResponseMessage());
327 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
328 return m;
329 }
330
331 final RemoteException re = JsonUtil.toRemoteException(m);
332 throw unwrapException? toIOException(re): re;
333 }
334 return null;
335 }
336
337 /**
338 * Covert an exception to an IOException.
339 *
340 * For a non-IOException, wrap it with IOException.
341 * For a RemoteException, unwrap it.
342 * For an IOException which is not a RemoteException, return it.
343 */
344 private static IOException toIOException(Exception e) {
345 if (!(e instanceof IOException)) {
346 return new IOException(e);
347 }
348
349 final IOException ioe = (IOException)e;
350 if (!(ioe instanceof RemoteException)) {
351 return ioe;
352 }
353
354 return ((RemoteException)ioe).unwrapRemoteException();
355 }
356
357 private synchronized InetSocketAddress getCurrentNNAddr() {
358 return nnAddrs[currentNNAddrIndex];
359 }
360
361 /**
362 * Reset the appropriate state to gracefully fail over to another name node
363 */
364 private synchronized void resetStateToFailOver() {
365 currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length;
366 delegationToken = null;
367 tokenAspect.reset();
368 }
369
370 /**
371 * Return a URL pointing to given path on the namenode.
372 *
373 * @param path to obtain the URL for
374 * @param query string to append to the path
375 * @return namenode URL referring to the given path
376 * @throws IOException on error constructing the URL
377 */
378 private URL getNamenodeURL(String path, String query) throws IOException {
379 InetSocketAddress nnAddr = getCurrentNNAddr();
380 final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
381 nnAddr.getPort(), path + '?' + query);
382 if (LOG.isTraceEnabled()) {
383 LOG.trace("url=" + url);
384 }
385 return url;
386 }
387
388 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
389 List<Param<?,?>> authParams = Lists.newArrayList();
390 // Skip adding delegation token for token operations because these
391 // operations require authentication.
392 Token<?> token = null;
393 if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) {
394 token = getDelegationToken();
395 }
396 if (token != null) {
397 authParams.add(new DelegationParam(token.encodeToUrlString()));
398 } else {
399 UserGroupInformation userUgi = ugi;
400 UserGroupInformation realUgi = userUgi.getRealUser();
401 if (realUgi != null) { // proxy user
402 authParams.add(new DoAsParam(userUgi.getShortUserName()));
403 userUgi = realUgi;
404 }
405 authParams.add(new UserParam(userUgi.getShortUserName()));
406 }
407 return authParams.toArray(new Param<?,?>[0]);
408 }
409
410 URL toUrl(final HttpOpParam.Op op, final Path fspath,
411 final Param<?,?>... parameters) throws IOException {
412 //initialize URI path and query
413 final String path = PATH_PREFIX
414 + (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
415 final String query = op.toQueryString()
416 + Param.toSortedString("&", getAuthParameters(op))
417 + Param.toSortedString("&", parameters);
418 final URL url = getNamenodeURL(path, query);
419 if (LOG.isTraceEnabled()) {
420 LOG.trace("url=" + url);
421 }
422 return url;
423 }
424
425 /**
426 * Run a http operation.
427 * Connect to the http server, validate response, and obtain the JSON output.
428 *
429 * @param op http operation
430 * @param fspath file system path
431 * @param parameters parameters for the operation
432 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
433 * @throws IOException
434 */
435 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
436 final Param<?,?>... parameters) throws IOException {
437 return new FsPathRunner(op, fspath, parameters).run().json;
438 }
439
440 /**
441 * This class is for initialing a HTTP connection, connecting to server,
442 * obtaining a response, and also handling retry on failures.
443 */
444 abstract class AbstractRunner {
445 abstract protected URL getUrl() throws IOException;
446
447 protected final HttpOpParam.Op op;
448 private final boolean redirected;
449
450 private boolean checkRetry;
451 protected HttpURLConnection conn = null;
452 private Map<?, ?> json = null;
453
454 protected AbstractRunner(final HttpOpParam.Op op, boolean redirected) {
455 this.op = op;
456 this.redirected = redirected;
457 }
458
459 AbstractRunner run() throws IOException {
460 UserGroupInformation connectUgi = ugi.getRealUser();
461 if (connectUgi == null) {
462 connectUgi = ugi;
463 }
464 if (op.getRequireAuth()) {
465 connectUgi.checkTGTAndReloginFromKeytab();
466 }
467 try {
468 // the entire lifecycle of the connection must be run inside the
469 // doAs to ensure authentication is performed correctly
470 return connectUgi.doAs(
471 new PrivilegedExceptionAction<AbstractRunner>() {
472 @Override
473 public AbstractRunner run() throws IOException {
474 return runWithRetry();
475 }
476 });
477 } catch (InterruptedException e) {
478 throw new IOException(e);
479 }
480 }
481
482 private void init() throws IOException {
483 checkRetry = !redirected;
484 URL url = getUrl();
485 conn = (HttpURLConnection) connectionFactory.openConnection(url);
486 }
487
488 private void connect() throws IOException {
489 connect(op.getDoOutput());
490 }
491
492 private void connect(boolean doOutput) throws IOException {
493 conn.setRequestMethod(op.getType().toString());
494 conn.setDoOutput(doOutput);
495 conn.setInstanceFollowRedirects(false);
496 conn.connect();
497 }
498
499 private void disconnect() {
500 if (conn != null) {
501 conn.disconnect();
502 conn = null;
503 }
504 }
505
506 private AbstractRunner runWithRetry() throws IOException {
507 /**
508 * Do the real work.
509 *
510 * There are three cases that the code inside the loop can throw an
511 * IOException:
512 *
513 * <ul>
514 * <li>The connection has failed (e.g., ConnectException,
515 * @see FailoverOnNetworkExceptionRetry for more details)</li>
516 * <li>The namenode enters the standby state (i.e., StandbyException).</li>
517 * <li>The server returns errors for the command (i.e., RemoteException)</li>
518 * </ul>
519 *
520 * The call to shouldRetry() will conduct the retry policy. The policy
521 * examines the exception and swallows it if it decides to rerun the work.
522 */
523 for(int retry = 0; ; retry++) {
524 try {
525 init();
526 if (op.getDoOutput()) {
527 twoStepWrite();
528 } else {
529 getResponse(op != GetOpParam.Op.OPEN);
530 }
531 return this;
532 } catch(IOException ioe) {
533 Throwable cause = ioe.getCause();
534 if (cause != null && cause instanceof AuthenticationException) {
535 throw ioe; // no retries for auth failures
536 }
537 shouldRetry(ioe, retry);
538 }
539 }
540 }
541
542 private void shouldRetry(final IOException ioe, final int retry
543 ) throws IOException {
544 InetSocketAddress nnAddr = getCurrentNNAddr();
545 if (checkRetry) {
546 try {
547 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
548 ioe, retry, 0, true);
549
550 boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
551 boolean isFailoverAndRetry =
552 a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
553
554 if (isRetry || isFailoverAndRetry) {
555 LOG.info("Retrying connect to namenode: " + nnAddr
556 + ". Already tried " + retry + " time(s); retry policy is "
557 + retryPolicy + ", delay " + a.delayMillis + "ms.");
558
559 if (isFailoverAndRetry) {
560 resetStateToFailOver();
561 }
562
563 Thread.sleep(a.delayMillis);
564 return;
565 }
566 } catch(Exception e) {
567 LOG.warn("Original exception is ", ioe);
568 throw toIOException(e);
569 }
570 }
571 throw toIOException(ioe);
572 }
573
574 /**
575 * Two-step Create/Append:
576 * Step 1) Submit a Http request with neither auto-redirect nor data.
577 * Step 2) Submit another Http request with the URL from the Location header with data.
578 *
579 * The reason of having two-step create/append is for preventing clients to
580 * send out the data before the redirect. This issue is addressed by the
581 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
582 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
583 * and Java 6 http client), which do not correctly implement "Expect:
584 * 100-continue". The two-step create/append is a temporary workaround for
585 * the software library bugs.
586 */
587 HttpURLConnection twoStepWrite() throws IOException {
588 //Step 1) Submit a Http request with neither auto-redirect nor data.
589 connect(false);
590 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false);
591 final String redirect = conn.getHeaderField("Location");
592 disconnect();
593 checkRetry = false;
594
595 //Step 2) Submit another Http request with the URL from the Location header with data.
596 conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
597 redirect));
598 conn.setRequestProperty("Content-Type",
599 MediaType.APPLICATION_OCTET_STREAM);
600 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
601 connect();
602 return conn;
603 }
604
605 FSDataOutputStream write(final int bufferSize) throws IOException {
606 return WebHdfsFileSystem.this.write(op, conn, bufferSize);
607 }
608
609 void getResponse(boolean getJsonAndDisconnect) throws IOException {
610 try {
611 connect();
612 final int code = conn.getResponseCode();
613 if (!redirected && op.getRedirect()
614 && code != op.getExpectedHttpResponseCode()) {
615 final String redirect = conn.getHeaderField("Location");
616 json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op),
617 conn, false);
618 disconnect();
619
620 checkRetry = false;
621 conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
622 redirect));
623 connect();
624 }
625
626 json = validateResponse(op, conn, false);
627 if (json == null && getJsonAndDisconnect) {
628 json = jsonParse(conn, false);
629 }
630 } finally {
631 if (getJsonAndDisconnect) {
632 disconnect();
633 }
634 }
635 }
636 }
637
638 final class FsPathRunner extends AbstractRunner {
639 private final Path fspath;
640 private final Param<?, ?>[] parameters;
641
642 FsPathRunner(final HttpOpParam.Op op, final Path fspath, final Param<?,?>... parameters) {
643 super(op, false);
644 this.fspath = fspath;
645 this.parameters = parameters;
646 }
647
648 @Override
649 protected URL getUrl() throws IOException {
650 return toUrl(op, fspath, parameters);
651 }
652 }
653
654 final class URLRunner extends AbstractRunner {
655 private final URL url;
656 @Override
657 protected URL getUrl() {
658 return url;
659 }
660
661 protected URLRunner(final HttpOpParam.Op op, final URL url, boolean redirected) {
662 super(op, redirected);
663 this.url = url;
664 }
665 }
666
667 private FsPermission applyUMask(FsPermission permission) {
668 if (permission == null) {
669 permission = FsPermission.getDefault();
670 }
671 return permission.applyUMask(FsPermission.getUMask(getConf()));
672 }
673
674 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
675 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
676 final Map<?, ?> json = run(op, f);
677 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
678 if (status == null) {
679 throw new FileNotFoundException("File does not exist: " + f);
680 }
681 return status;
682 }
683
684 @Override
685 public FileStatus getFileStatus(Path f) throws IOException {
686 statistics.incrementReadOps(1);
687 return makeQualified(getHdfsFileStatus(f), f);
688 }
689
690 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
691 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
692 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
693 f.getPermission(), f.getOwner(), f.getGroup(),
694 f.isSymlink() ? new Path(f.getSymlink()) : null,
695 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
696 }
697
698 @Override
699 public AclStatus getAclStatus(Path f) throws IOException {
700 final HttpOpParam.Op op = GetOpParam.Op.GETACLSTATUS;
701 final Map<?, ?> json = run(op, f);
702 AclStatus status = JsonUtil.toAclStatus(json);
703 if (status == null) {
704 throw new FileNotFoundException("File does not exist: " + f);
705 }
706 return status;
707 }
708
709 @Override
710 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
711 statistics.incrementWriteOps(1);
712 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
713 final Map<?, ?> json = run(op, f,
714 new PermissionParam(applyUMask(permission)));
715 return (Boolean)json.get("boolean");
716 }
717
718 /**
719 * Create a symlink pointing to the destination path.
720 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
721 */
722 public void createSymlink(Path destination, Path f, boolean createParent
723 ) throws IOException {
724 statistics.incrementWriteOps(1);
725 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
726 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
727 new CreateParentParam(createParent));
728 }
729
730 @Override
731 public boolean rename(final Path src, final Path dst) throws IOException {
732 statistics.incrementWriteOps(1);
733 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
734 final Map<?, ?> json = run(op, src,
735 new DestinationParam(makeQualified(dst).toUri().getPath()));
736 return (Boolean)json.get("boolean");
737 }
738
739 @SuppressWarnings("deprecation")
740 @Override
741 public void rename(final Path src, final Path dst,
742 final Options.Rename... options) throws IOException {
743 statistics.incrementWriteOps(1);
744 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
745 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
746 new RenameOptionSetParam(options));
747 }
748
749 @Override
750 public void setOwner(final Path p, final String owner, final String group
751 ) throws IOException {
752 if (owner == null && group == null) {
753 throw new IOException("owner == null && group == null");
754 }
755
756 statistics.incrementWriteOps(1);
757 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
758 run(op, p, new OwnerParam(owner), new GroupParam(group));
759 }
760
761 @Override
762 public void setPermission(final Path p, final FsPermission permission
763 ) throws IOException {
764 statistics.incrementWriteOps(1);
765 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
766 run(op, p, new PermissionParam(permission));
767 }
768
769 @Override
770 public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
771 throws IOException {
772 statistics.incrementWriteOps(1);
773 final HttpOpParam.Op op = PutOpParam.Op.MODIFYACLENTRIES;
774 run(op, path, new AclPermissionParam(aclSpec));
775 }
776
777 @Override
778 public void removeAclEntries(Path path, List<AclEntry> aclSpec)
779 throws IOException {
780 statistics.incrementWriteOps(1);
781 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACLENTRIES;
782 run(op, path, new AclPermissionParam(aclSpec));
783 }
784
785 @Override
786 public void removeDefaultAcl(Path path) throws IOException {
787 statistics.incrementWriteOps(1);
788 final HttpOpParam.Op op = PutOpParam.Op.REMOVEDEFAULTACL;
789 run(op, path);
790 }
791
792 @Override
793 public void removeAcl(Path path) throws IOException {
794 statistics.incrementWriteOps(1);
795 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACL;
796 run(op, path);
797 }
798
799 @Override
800 public void setAcl(final Path p, final List<AclEntry> aclSpec)
801 throws IOException {
802 statistics.incrementWriteOps(1);
803 final HttpOpParam.Op op = PutOpParam.Op.SETACL;
804 run(op, p, new AclPermissionParam(aclSpec));
805 }
806
807 @Override
808 public boolean setReplication(final Path p, final short replication
809 ) throws IOException {
810 statistics.incrementWriteOps(1);
811 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
812 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
813 return (Boolean)json.get("boolean");
814 }
815
816 @Override
817 public void setTimes(final Path p, final long mtime, final long atime
818 ) throws IOException {
819 statistics.incrementWriteOps(1);
820 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
821 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
822 }
823
824 @Override
825 public long getDefaultBlockSize() {
826 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
827 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
828 }
829
830 @Override
831 public short getDefaultReplication() {
832 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
833 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
834 }
835
836 FSDataOutputStream write(final HttpOpParam.Op op,
837 final HttpURLConnection conn, final int bufferSize) throws IOException {
838 return new FSDataOutputStream(new BufferedOutputStream(
839 conn.getOutputStream(), bufferSize), statistics) {
840 @Override
841 public void close() throws IOException {
842 try {
843 super.close();
844 } finally {
845 try {
846 validateResponse(op, conn, true);
847 } finally {
848 conn.disconnect();
849 }
850 }
851 }
852 };
853 }
854
855 @Override
856 public void concat(final Path trg, final Path [] srcs) throws IOException {
857 statistics.incrementWriteOps(1);
858 final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
859
860 ConcatSourcesParam param = new ConcatSourcesParam(srcs);
861 run(op, trg, param);
862 }
863
864 @Override
865 public FSDataOutputStream create(final Path f, final FsPermission permission,
866 final boolean overwrite, final int bufferSize, final short replication,
867 final long blockSize, final Progressable progress) throws IOException {
868 statistics.incrementWriteOps(1);
869
870 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
871 return new FsPathRunner(op, f,
872 new PermissionParam(applyUMask(permission)),
873 new OverwriteParam(overwrite),
874 new BufferSizeParam(bufferSize),
875 new ReplicationParam(replication),
876 new BlockSizeParam(blockSize))
877 .run()
878 .write(bufferSize);
879 }
880
881 @Override
882 public FSDataOutputStream append(final Path f, final int bufferSize,
883 final Progressable progress) throws IOException {
884 statistics.incrementWriteOps(1);
885
886 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
887 return new FsPathRunner(op, f, new BufferSizeParam(bufferSize))
888 .run()
889 .write(bufferSize);
890 }
891
892 @Override
893 public boolean delete(Path f, boolean recursive) throws IOException {
894 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
895 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
896 return (Boolean)json.get("boolean");
897 }
898
899 @Override
900 public FSDataInputStream open(final Path f, final int buffersize
901 ) throws IOException {
902 statistics.incrementReadOps(1);
903 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
904 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
905 return new FSDataInputStream(new OffsetUrlInputStream(
906 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
907 }
908
909 @Override
910 public void close() throws IOException {
911 super.close();
912 synchronized (this) {
913 tokenAspect.removeRenewAction();
914 }
915 }
916
917 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
918 OffsetUrlOpener(final URL url) {
919 super(url);
920 }
921
922 /** Setup offset url and connect. */
923 @Override
924 protected HttpURLConnection connect(final long offset,
925 final boolean resolved) throws IOException {
926 final URL offsetUrl = offset == 0L? url
927 : new URL(url + "&" + new OffsetParam(offset));
928 return new URLRunner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn;
929 }
930 }
931
932 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
933
934 /** Remove offset parameter, if there is any, from the url */
935 static URL removeOffsetParam(final URL url) throws MalformedURLException {
936 String query = url.getQuery();
937 if (query == null) {
938 return url;
939 }
940 final String lower = query.toLowerCase();
941 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
942 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
943 return url;
944 }
945
946 //rebuild query
947 StringBuilder b = null;
948 for(final StringTokenizer st = new StringTokenizer(query, "&");
949 st.hasMoreTokens();) {
950 final String token = st.nextToken();
951 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
952 if (b == null) {
953 b = new StringBuilder("?").append(token);
954 } else {
955 b.append('&').append(token);
956 }
957 }
958 }
959 query = b == null? "": b.toString();
960
961 final String urlStr = url.toString();
962 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
963 }
964
965 static class OffsetUrlInputStream extends ByteRangeInputStream {
966 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
967 super(o, r);
968 }
969
970 /** Remove offset parameter before returning the resolved url. */
971 @Override
972 protected URL getResolvedUrl(final HttpURLConnection connection
973 ) throws MalformedURLException {
974 return removeOffsetParam(connection.getURL());
975 }
976 }
977
978 @Override
979 public FileStatus[] listStatus(final Path f) throws IOException {
980 statistics.incrementReadOps(1);
981
982 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
983 final Map<?, ?> json = run(op, f);
984 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
985 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
986
987 //convert FileStatus
988 final FileStatus[] statuses = new FileStatus[array.length];
989 for(int i = 0; i < array.length; i++) {
990 final Map<?, ?> m = (Map<?, ?>)array[i];
991 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
992 }
993 return statuses;
994 }
995
996 @Override
997 public Token<DelegationTokenIdentifier> getDelegationToken(
998 final String renewer) throws IOException {
999 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
1000 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
1001 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
1002 token.setService(tokenServiceName);
1003 return token;
1004 }
1005
1006 @Override
1007 public synchronized Token<?> getRenewToken() {
1008 return delegationToken;
1009 }
1010
1011 @Override
1012 public <T extends TokenIdentifier> void setDelegationToken(
1013 final Token<T> token) {
1014 synchronized (this) {
1015 delegationToken = token;
1016 }
1017 }
1018
1019 @Override
1020 public synchronized long renewDelegationToken(final Token<?> token
1021 ) throws IOException {
1022 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
1023 TokenArgumentParam dtargParam = new TokenArgumentParam(
1024 token.encodeToUrlString());
1025 final Map<?, ?> m = run(op, null, dtargParam);
1026 return (Long) m.get("long");
1027 }
1028
1029 @Override
1030 public synchronized void cancelDelegationToken(final Token<?> token
1031 ) throws IOException {
1032 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
1033 TokenArgumentParam dtargParam = new TokenArgumentParam(
1034 token.encodeToUrlString());
1035 run(op, null, dtargParam);
1036 }
1037
1038 @Override
1039 public BlockLocation[] getFileBlockLocations(final FileStatus status,
1040 final long offset, final long length) throws IOException {
1041 if (status == null) {
1042 return null;
1043 }
1044 return getFileBlockLocations(status.getPath(), offset, length);
1045 }
1046
1047 @Override
1048 public BlockLocation[] getFileBlockLocations(final Path p,
1049 final long offset, final long length) throws IOException {
1050 statistics.incrementReadOps(1);
1051
1052 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
1053 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
1054 new LengthParam(length));
1055 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
1056 }
1057
1058 @Override
1059 public ContentSummary getContentSummary(final Path p) throws IOException {
1060 statistics.incrementReadOps(1);
1061
1062 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
1063 final Map<?, ?> m = run(op, p);
1064 return JsonUtil.toContentSummary(m);
1065 }
1066
1067 @Override
1068 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
1069 ) throws IOException {
1070 statistics.incrementReadOps(1);
1071
1072 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
1073 final Map<?, ?> m = run(op, p);
1074 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
1075 }
1076
1077 @Override
1078 public String getCanonicalServiceName() {
1079 return tokenServiceName == null ? super.getCanonicalServiceName()
1080 : tokenServiceName.toString();
1081 }
1082
1083 @VisibleForTesting
1084 InetSocketAddress[] getResolvedNNAddr() {
1085 return nnAddrs;
1086 }
1087 }