001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URL;
031 import java.security.PrivilegedExceptionAction;
032 import java.util.List;
033 import java.util.Map;
034 import java.util.StringTokenizer;
035
036 import javax.ws.rs.core.MediaType;
037
038 import org.apache.commons.logging.Log;
039 import org.apache.commons.logging.LogFactory;
040 import org.apache.hadoop.conf.Configuration;
041 import org.apache.hadoop.fs.BlockLocation;
042 import org.apache.hadoop.fs.ContentSummary;
043 import org.apache.hadoop.fs.DelegationTokenRenewer;
044 import org.apache.hadoop.fs.FSDataInputStream;
045 import org.apache.hadoop.fs.FSDataOutputStream;
046 import org.apache.hadoop.fs.FileStatus;
047 import org.apache.hadoop.fs.FileSystem;
048 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
049 import org.apache.hadoop.fs.Options;
050 import org.apache.hadoop.fs.Path;
051 import org.apache.hadoop.fs.permission.AclEntry;
052 import org.apache.hadoop.fs.permission.AclStatus;
053 import org.apache.hadoop.fs.permission.FsPermission;
054 import org.apache.hadoop.hdfs.DFSConfigKeys;
055 import org.apache.hadoop.hdfs.DFSUtil;
056 import org.apache.hadoop.hdfs.HAUtil;
057 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
058 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
059 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
060 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
061 import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
062 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
063 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
064 import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
065 import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
066 import org.apache.hadoop.hdfs.web.resources.DelegationParam;
067 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
068 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
069 import org.apache.hadoop.hdfs.web.resources.DoAsParam;
070 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
071 import org.apache.hadoop.hdfs.web.resources.GroupParam;
072 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
073 import org.apache.hadoop.hdfs.web.resources.LengthParam;
074 import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
075 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
076 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
077 import org.apache.hadoop.hdfs.web.resources.OwnerParam;
078 import org.apache.hadoop.hdfs.web.resources.Param;
079 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
080 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
081 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
082 import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
083 import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
084 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
085 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
086 import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
087 import org.apache.hadoop.hdfs.web.resources.UserParam;
088 import org.apache.hadoop.io.Text;
089 import org.apache.hadoop.io.retry.RetryPolicies;
090 import org.apache.hadoop.io.retry.RetryPolicy;
091 import org.apache.hadoop.io.retry.RetryUtils;
092 import org.apache.hadoop.ipc.RemoteException;
093 import org.apache.hadoop.net.NetUtils;
094 import org.apache.hadoop.security.SecurityUtil;
095 import org.apache.hadoop.security.UserGroupInformation;
096 import org.apache.hadoop.security.authentication.client.AuthenticationException;
097 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
098 import org.apache.hadoop.security.token.Token;
099 import org.apache.hadoop.security.token.TokenIdentifier;
100 import org.apache.hadoop.util.Progressable;
101 import org.mortbay.util.ajax.JSON;
102
103 import com.google.common.base.Charsets;
104 import com.google.common.collect.Lists;
105
106 /** A FileSystem for HDFS over the web. */
107 public class WebHdfsFileSystem extends FileSystem
108 implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
109 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
110 /** File System URI: {SCHEME}://namenode:port/path/to/file */
111 public static final String SCHEME = "webhdfs";
112 /** WebHdfs version. */
113 public static final int VERSION = 1;
114 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
115 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
116
117 /** Default connection factory may be overridden in tests to use smaller timeout values */
118 protected URLConnectionFactory connectionFactory;
119
120 /** Delegation token kind */
121 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
122 protected TokenAspect<? extends WebHdfsFileSystem> tokenAspect;
123
124 private UserGroupInformation ugi;
125 private URI uri;
126 private Token<?> delegationToken;
127 protected Text tokenServiceName;
128 private RetryPolicy retryPolicy = null;
129 private Path workingDir;
130 private InetSocketAddress nnAddrs[];
131 private int currentNNAddrIndex;
132
133 /**
134 * Return the protocol scheme for the FileSystem.
135 * <p/>
136 *
137 * @return <code>webhdfs</code>
138 */
139 @Override
140 public String getScheme() {
141 return SCHEME;
142 }
143
144 /**
145 * return the underlying transport protocol (http / https).
146 */
147 protected String getTransportScheme() {
148 return "http";
149 }
150
151 /**
152 * Initialize tokenAspect. This function is intended to
153 * be overridden by SWebHdfsFileSystem.
154 */
155 protected synchronized void initializeTokenAspect() {
156 tokenAspect = new TokenAspect<WebHdfsFileSystem>(this, tokenServiceName,
157 TOKEN_KIND);
158 }
159
160 @Override
161 public synchronized void initialize(URI uri, Configuration conf
162 ) throws IOException {
163 super.initialize(uri, conf);
164 setConf(conf);
165 /** set user pattern based on configuration file */
166 UserParam.setUserPattern(conf.get(
167 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
168 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
169
170 connectionFactory = URLConnectionFactory
171 .newDefaultURLConnectionFactory(conf);
172
173 ugi = UserGroupInformation.getCurrentUser();
174 this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
175 this.nnAddrs = DFSUtil.resolveWebHdfsUri(this.uri, conf);
176
177 boolean isHA = HAUtil.isLogicalUri(conf, this.uri);
178 // In non-HA case, the code needs to call getCanonicalUri() in order to
179 // handle the case where no port is specified in the URI
180 this.tokenServiceName = isHA ? HAUtil.buildTokenServiceForLogicalUri(uri)
181 : SecurityUtil.buildTokenService(getCanonicalUri());
182 initializeTokenAspect();
183
184 if (!isHA) {
185 this.retryPolicy =
186 RetryUtils.getDefaultRetryPolicy(
187 conf,
188 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
189 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
190 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
191 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
192 SafeModeException.class);
193 } else {
194
195 int maxFailoverAttempts = conf.getInt(
196 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
197 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
198 int maxRetryAttempts = conf.getInt(
199 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
200 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
201 int failoverSleepBaseMillis = conf.getInt(
202 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
203 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
204 int failoverSleepMaxMillis = conf.getInt(
205 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
206 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
207
208 this.retryPolicy = RetryPolicies
209 .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
210 maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis,
211 failoverSleepMaxMillis);
212 }
213
214 this.workingDir = getHomeDirectory();
215
216 if (UserGroupInformation.isSecurityEnabled()) {
217 tokenAspect.initDelegationToken(ugi);
218 }
219 }
220
221 @Override
222 public URI getCanonicalUri() {
223 return super.getCanonicalUri();
224 }
225
226 /** Is WebHDFS enabled in conf? */
227 public static boolean isEnabled(final Configuration conf, final Log log) {
228 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
229 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
230 return b;
231 }
232
233 protected synchronized Token<?> getDelegationToken() throws IOException {
234 tokenAspect.ensureTokenInitialized();
235 return delegationToken;
236 }
237
238 @Override
239 protected int getDefaultPort() {
240 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
241 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
242 }
243
244 @Override
245 public URI getUri() {
246 return this.uri;
247 }
248
249 @Override
250 protected URI canonicalizeUri(URI uri) {
251 return NetUtils.getCanonicalUri(uri, getDefaultPort());
252 }
253
254 /** @return the home directory. */
255 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
256 return "/user/" + ugi.getShortUserName();
257 }
258
259 @Override
260 public Path getHomeDirectory() {
261 return makeQualified(new Path(getHomeDirectoryString(ugi)));
262 }
263
264 @Override
265 public synchronized Path getWorkingDirectory() {
266 return workingDir;
267 }
268
269 @Override
270 public synchronized void setWorkingDirectory(final Path dir) {
271 String result = makeAbsolute(dir).toUri().getPath();
272 if (!DFSUtil.isValidName(result)) {
273 throw new IllegalArgumentException("Invalid DFS directory name " +
274 result);
275 }
276 workingDir = makeAbsolute(dir);
277 }
278
279 private Path makeAbsolute(Path f) {
280 return f.isAbsolute()? f: new Path(workingDir, f);
281 }
282
283 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
284 ) throws IOException {
285 if (c.getContentLength() == 0) {
286 return null;
287 }
288 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
289 if (in == null) {
290 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
291 }
292 final String contentType = c.getContentType();
293 if (contentType != null) {
294 final MediaType parsed = MediaType.valueOf(contentType);
295 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
296 throw new IOException("Content-Type \"" + contentType
297 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
298 + "\" (parsed=\"" + parsed + "\")");
299 }
300 }
301 return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
302 }
303
304 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
305 final HttpURLConnection conn, boolean unwrapException) throws IOException {
306 final int code = conn.getResponseCode();
307 // server is demanding an authentication we don't support
308 if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
309 throw new IOException(
310 new AuthenticationException(conn.getResponseMessage()));
311 }
312 if (code != op.getExpectedHttpResponseCode()) {
313 final Map<?, ?> m;
314 try {
315 m = jsonParse(conn, true);
316 } catch(Exception e) {
317 throw new IOException("Unexpected HTTP response: code=" + code + " != "
318 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
319 + ", message=" + conn.getResponseMessage(), e);
320 }
321
322 if (m == null) {
323 throw new IOException("Unexpected HTTP response: code=" + code + " != "
324 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
325 + ", message=" + conn.getResponseMessage());
326 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
327 return m;
328 }
329
330 final RemoteException re = JsonUtil.toRemoteException(m);
331 throw unwrapException? toIOException(re): re;
332 }
333 return null;
334 }
335
336 /**
337 * Covert an exception to an IOException.
338 *
339 * For a non-IOException, wrap it with IOException.
340 * For a RemoteException, unwrap it.
341 * For an IOException which is not a RemoteException, return it.
342 */
343 private static IOException toIOException(Exception e) {
344 if (!(e instanceof IOException)) {
345 return new IOException(e);
346 }
347
348 final IOException ioe = (IOException)e;
349 if (!(ioe instanceof RemoteException)) {
350 return ioe;
351 }
352
353 return ((RemoteException)ioe).unwrapRemoteException();
354 }
355
356 private synchronized InetSocketAddress getCurrentNNAddr() {
357 return nnAddrs[currentNNAddrIndex];
358 }
359
360 /**
361 * Reset the appropriate state to gracefully fail over to another name node
362 */
363 private synchronized void resetStateToFailOver() {
364 currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length;
365 delegationToken = null;
366 tokenAspect.reset();
367 }
368
369 /**
370 * Return a URL pointing to given path on the namenode.
371 *
372 * @param path to obtain the URL for
373 * @param query string to append to the path
374 * @return namenode URL referring to the given path
375 * @throws IOException on error constructing the URL
376 */
377 private URL getNamenodeURL(String path, String query) throws IOException {
378 InetSocketAddress nnAddr = getCurrentNNAddr();
379 final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
380 nnAddr.getPort(), path + '?' + query);
381 if (LOG.isTraceEnabled()) {
382 LOG.trace("url=" + url);
383 }
384 return url;
385 }
386
387 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
388 List<Param<?,?>> authParams = Lists.newArrayList();
389 // Skip adding delegation token for token operations because these
390 // operations require authentication.
391 Token<?> token = null;
392 if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) {
393 token = getDelegationToken();
394 }
395 if (token != null) {
396 authParams.add(new DelegationParam(token.encodeToUrlString()));
397 } else {
398 UserGroupInformation userUgi = ugi;
399 UserGroupInformation realUgi = userUgi.getRealUser();
400 if (realUgi != null) { // proxy user
401 authParams.add(new DoAsParam(userUgi.getShortUserName()));
402 userUgi = realUgi;
403 }
404 authParams.add(new UserParam(userUgi.getShortUserName()));
405 }
406 return authParams.toArray(new Param<?,?>[0]);
407 }
408
409 URL toUrl(final HttpOpParam.Op op, final Path fspath,
410 final Param<?,?>... parameters) throws IOException {
411 //initialize URI path and query
412 final String path = PATH_PREFIX
413 + (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
414 final String query = op.toQueryString()
415 + Param.toSortedString("&", getAuthParameters(op))
416 + Param.toSortedString("&", parameters);
417 final URL url = getNamenodeURL(path, query);
418 if (LOG.isTraceEnabled()) {
419 LOG.trace("url=" + url);
420 }
421 return url;
422 }
423
424 /**
425 * Run a http operation.
426 * Connect to the http server, validate response, and obtain the JSON output.
427 *
428 * @param op http operation
429 * @param fspath file system path
430 * @param parameters parameters for the operation
431 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
432 * @throws IOException
433 */
434 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
435 final Param<?,?>... parameters) throws IOException {
436 return new FsPathRunner(op, fspath, parameters).run().json;
437 }
438
439 /**
440 * This class is for initialing a HTTP connection, connecting to server,
441 * obtaining a response, and also handling retry on failures.
442 */
443 abstract class AbstractRunner {
444 abstract protected URL getUrl() throws IOException;
445
446 protected final HttpOpParam.Op op;
447 private final boolean redirected;
448
449 private boolean checkRetry;
450 protected HttpURLConnection conn = null;
451 private Map<?, ?> json = null;
452
453 protected AbstractRunner(final HttpOpParam.Op op, boolean redirected) {
454 this.op = op;
455 this.redirected = redirected;
456 }
457
458 AbstractRunner run() throws IOException {
459 UserGroupInformation connectUgi = ugi.getRealUser();
460 if (connectUgi == null) {
461 connectUgi = ugi;
462 }
463 if (op.getRequireAuth()) {
464 connectUgi.checkTGTAndReloginFromKeytab();
465 }
466 try {
467 // the entire lifecycle of the connection must be run inside the
468 // doAs to ensure authentication is performed correctly
469 return connectUgi.doAs(
470 new PrivilegedExceptionAction<AbstractRunner>() {
471 @Override
472 public AbstractRunner run() throws IOException {
473 return runWithRetry();
474 }
475 });
476 } catch (InterruptedException e) {
477 throw new IOException(e);
478 }
479 }
480
481 private void init() throws IOException {
482 checkRetry = !redirected;
483 URL url = getUrl();
484 conn = (HttpURLConnection) connectionFactory.openConnection(url);
485 }
486
487 private void connect() throws IOException {
488 connect(op.getDoOutput());
489 }
490
491 private void connect(boolean doOutput) throws IOException {
492 conn.setRequestMethod(op.getType().toString());
493 conn.setDoOutput(doOutput);
494 conn.setInstanceFollowRedirects(false);
495 conn.connect();
496 }
497
498 private void disconnect() {
499 if (conn != null) {
500 conn.disconnect();
501 conn = null;
502 }
503 }
504
505 private AbstractRunner runWithRetry() throws IOException {
506 /**
507 * Do the real work.
508 *
509 * There are three cases that the code inside the loop can throw an
510 * IOException:
511 *
512 * <ul>
513 * <li>The connection has failed (e.g., ConnectException,
514 * @see FailoverOnNetworkExceptionRetry for more details)</li>
515 * <li>The namenode enters the standby state (i.e., StandbyException).</li>
516 * <li>The server returns errors for the command (i.e., RemoteException)</li>
517 * </ul>
518 *
519 * The call to shouldRetry() will conduct the retry policy. The policy
520 * examines the exception and swallows it if it decides to rerun the work.
521 */
522 for(int retry = 0; ; retry++) {
523 try {
524 init();
525 if (op.getDoOutput()) {
526 twoStepWrite();
527 } else {
528 getResponse(op != GetOpParam.Op.OPEN);
529 }
530 return this;
531 } catch(IOException ioe) {
532 Throwable cause = ioe.getCause();
533 if (cause != null && cause instanceof AuthenticationException) {
534 throw ioe; // no retries for auth failures
535 }
536 shouldRetry(ioe, retry);
537 }
538 }
539 }
540
541 private void shouldRetry(final IOException ioe, final int retry
542 ) throws IOException {
543 InetSocketAddress nnAddr = getCurrentNNAddr();
544 if (checkRetry) {
545 try {
546 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
547 ioe, retry, 0, true);
548
549 boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
550 boolean isFailoverAndRetry =
551 a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
552
553 if (isRetry || isFailoverAndRetry) {
554 LOG.info("Retrying connect to namenode: " + nnAddr
555 + ". Already tried " + retry + " time(s); retry policy is "
556 + retryPolicy + ", delay " + a.delayMillis + "ms.");
557
558 if (isFailoverAndRetry) {
559 resetStateToFailOver();
560 }
561
562 Thread.sleep(a.delayMillis);
563 return;
564 }
565 } catch(Exception e) {
566 LOG.warn("Original exception is ", ioe);
567 throw toIOException(e);
568 }
569 }
570 throw toIOException(ioe);
571 }
572
573 /**
574 * Two-step Create/Append:
575 * Step 1) Submit a Http request with neither auto-redirect nor data.
576 * Step 2) Submit another Http request with the URL from the Location header with data.
577 *
578 * The reason of having two-step create/append is for preventing clients to
579 * send out the data before the redirect. This issue is addressed by the
580 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
581 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
582 * and Java 6 http client), which do not correctly implement "Expect:
583 * 100-continue". The two-step create/append is a temporary workaround for
584 * the software library bugs.
585 */
586 HttpURLConnection twoStepWrite() throws IOException {
587 //Step 1) Submit a Http request with neither auto-redirect nor data.
588 connect(false);
589 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false);
590 final String redirect = conn.getHeaderField("Location");
591 disconnect();
592 checkRetry = false;
593
594 //Step 2) Submit another Http request with the URL from the Location header with data.
595 conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
596 redirect));
597 conn.setRequestProperty("Content-Type",
598 MediaType.APPLICATION_OCTET_STREAM);
599 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
600 connect();
601 return conn;
602 }
603
604 FSDataOutputStream write(final int bufferSize) throws IOException {
605 return WebHdfsFileSystem.this.write(op, conn, bufferSize);
606 }
607
608 void getResponse(boolean getJsonAndDisconnect) throws IOException {
609 try {
610 connect();
611 final int code = conn.getResponseCode();
612 if (!redirected && op.getRedirect()
613 && code != op.getExpectedHttpResponseCode()) {
614 final String redirect = conn.getHeaderField("Location");
615 json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op),
616 conn, false);
617 disconnect();
618
619 checkRetry = false;
620 conn = (HttpURLConnection) connectionFactory.openConnection(new URL(
621 redirect));
622 connect();
623 }
624
625 json = validateResponse(op, conn, false);
626 if (json == null && getJsonAndDisconnect) {
627 json = jsonParse(conn, false);
628 }
629 } finally {
630 if (getJsonAndDisconnect) {
631 disconnect();
632 }
633 }
634 }
635 }
636
637 final class FsPathRunner extends AbstractRunner {
638 private final Path fspath;
639 private final Param<?, ?>[] parameters;
640
641 FsPathRunner(final HttpOpParam.Op op, final Path fspath, final Param<?,?>... parameters) {
642 super(op, false);
643 this.fspath = fspath;
644 this.parameters = parameters;
645 }
646
647 @Override
648 protected URL getUrl() throws IOException {
649 return toUrl(op, fspath, parameters);
650 }
651 }
652
653 final class URLRunner extends AbstractRunner {
654 private final URL url;
655 @Override
656 protected URL getUrl() {
657 return url;
658 }
659
660 protected URLRunner(final HttpOpParam.Op op, final URL url, boolean redirected) {
661 super(op, redirected);
662 this.url = url;
663 }
664 }
665
666 private FsPermission applyUMask(FsPermission permission) {
667 if (permission == null) {
668 permission = FsPermission.getDefault();
669 }
670 return permission.applyUMask(FsPermission.getUMask(getConf()));
671 }
672
673 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
674 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
675 final Map<?, ?> json = run(op, f);
676 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
677 if (status == null) {
678 throw new FileNotFoundException("File does not exist: " + f);
679 }
680 return status;
681 }
682
683 @Override
684 public FileStatus getFileStatus(Path f) throws IOException {
685 statistics.incrementReadOps(1);
686 return makeQualified(getHdfsFileStatus(f), f);
687 }
688
689 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
690 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
691 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
692 f.getPermission(), f.getOwner(), f.getGroup(),
693 f.isSymlink() ? new Path(f.getSymlink()) : null,
694 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
695 }
696
697 @Override
698 public AclStatus getAclStatus(Path f) throws IOException {
699 final HttpOpParam.Op op = GetOpParam.Op.GETACLSTATUS;
700 final Map<?, ?> json = run(op, f);
701 AclStatus status = JsonUtil.toAclStatus(json);
702 if (status == null) {
703 throw new FileNotFoundException("File does not exist: " + f);
704 }
705 return status;
706 }
707
708 @Override
709 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
710 statistics.incrementWriteOps(1);
711 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
712 final Map<?, ?> json = run(op, f,
713 new PermissionParam(applyUMask(permission)));
714 return (Boolean)json.get("boolean");
715 }
716
717 /**
718 * Create a symlink pointing to the destination path.
719 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
720 */
721 public void createSymlink(Path destination, Path f, boolean createParent
722 ) throws IOException {
723 statistics.incrementWriteOps(1);
724 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
725 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
726 new CreateParentParam(createParent));
727 }
728
729 @Override
730 public boolean rename(final Path src, final Path dst) throws IOException {
731 statistics.incrementWriteOps(1);
732 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
733 final Map<?, ?> json = run(op, src,
734 new DestinationParam(makeQualified(dst).toUri().getPath()));
735 return (Boolean)json.get("boolean");
736 }
737
738 @SuppressWarnings("deprecation")
739 @Override
740 public void rename(final Path src, final Path dst,
741 final Options.Rename... options) throws IOException {
742 statistics.incrementWriteOps(1);
743 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
744 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
745 new RenameOptionSetParam(options));
746 }
747
748 @Override
749 public void setOwner(final Path p, final String owner, final String group
750 ) throws IOException {
751 if (owner == null && group == null) {
752 throw new IOException("owner == null && group == null");
753 }
754
755 statistics.incrementWriteOps(1);
756 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
757 run(op, p, new OwnerParam(owner), new GroupParam(group));
758 }
759
760 @Override
761 public void setPermission(final Path p, final FsPermission permission
762 ) throws IOException {
763 statistics.incrementWriteOps(1);
764 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
765 run(op, p, new PermissionParam(permission));
766 }
767
768 @Override
769 public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
770 throws IOException {
771 statistics.incrementWriteOps(1);
772 final HttpOpParam.Op op = PutOpParam.Op.MODIFYACLENTRIES;
773 run(op, path, new AclPermissionParam(aclSpec));
774 }
775
776 @Override
777 public void removeAclEntries(Path path, List<AclEntry> aclSpec)
778 throws IOException {
779 statistics.incrementWriteOps(1);
780 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACLENTRIES;
781 run(op, path, new AclPermissionParam(aclSpec));
782 }
783
784 @Override
785 public void removeDefaultAcl(Path path) throws IOException {
786 statistics.incrementWriteOps(1);
787 final HttpOpParam.Op op = PutOpParam.Op.REMOVEDEFAULTACL;
788 run(op, path);
789 }
790
791 @Override
792 public void removeAcl(Path path) throws IOException {
793 statistics.incrementWriteOps(1);
794 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACL;
795 run(op, path);
796 }
797
798 @Override
799 public void setAcl(final Path p, final List<AclEntry> aclSpec)
800 throws IOException {
801 statistics.incrementWriteOps(1);
802 final HttpOpParam.Op op = PutOpParam.Op.SETACL;
803 run(op, p, new AclPermissionParam(aclSpec));
804 }
805
806 @Override
807 public boolean setReplication(final Path p, final short replication
808 ) throws IOException {
809 statistics.incrementWriteOps(1);
810 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
811 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
812 return (Boolean)json.get("boolean");
813 }
814
815 @Override
816 public void setTimes(final Path p, final long mtime, final long atime
817 ) throws IOException {
818 statistics.incrementWriteOps(1);
819 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
820 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
821 }
822
823 @Override
824 public long getDefaultBlockSize() {
825 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
826 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
827 }
828
829 @Override
830 public short getDefaultReplication() {
831 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
832 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
833 }
834
835 FSDataOutputStream write(final HttpOpParam.Op op,
836 final HttpURLConnection conn, final int bufferSize) throws IOException {
837 return new FSDataOutputStream(new BufferedOutputStream(
838 conn.getOutputStream(), bufferSize), statistics) {
839 @Override
840 public void close() throws IOException {
841 try {
842 super.close();
843 } finally {
844 try {
845 validateResponse(op, conn, true);
846 } finally {
847 conn.disconnect();
848 }
849 }
850 }
851 };
852 }
853
854 @Override
855 public void concat(final Path trg, final Path [] srcs) throws IOException {
856 statistics.incrementWriteOps(1);
857 final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
858
859 ConcatSourcesParam param = new ConcatSourcesParam(srcs);
860 run(op, trg, param);
861 }
862
863 @Override
864 public FSDataOutputStream create(final Path f, final FsPermission permission,
865 final boolean overwrite, final int bufferSize, final short replication,
866 final long blockSize, final Progressable progress) throws IOException {
867 statistics.incrementWriteOps(1);
868
869 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
870 return new FsPathRunner(op, f,
871 new PermissionParam(applyUMask(permission)),
872 new OverwriteParam(overwrite),
873 new BufferSizeParam(bufferSize),
874 new ReplicationParam(replication),
875 new BlockSizeParam(blockSize))
876 .run()
877 .write(bufferSize);
878 }
879
880 @Override
881 public FSDataOutputStream append(final Path f, final int bufferSize,
882 final Progressable progress) throws IOException {
883 statistics.incrementWriteOps(1);
884
885 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
886 return new FsPathRunner(op, f, new BufferSizeParam(bufferSize))
887 .run()
888 .write(bufferSize);
889 }
890
891 @Override
892 public boolean delete(Path f, boolean recursive) throws IOException {
893 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
894 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
895 return (Boolean)json.get("boolean");
896 }
897
898 @Override
899 public FSDataInputStream open(final Path f, final int buffersize
900 ) throws IOException {
901 statistics.incrementReadOps(1);
902 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
903 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
904 return new FSDataInputStream(new OffsetUrlInputStream(
905 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
906 }
907
908 @Override
909 public void close() throws IOException {
910 super.close();
911 synchronized (this) {
912 tokenAspect.removeRenewAction();
913 }
914 }
915
916 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
917 OffsetUrlOpener(final URL url) {
918 super(url);
919 }
920
921 /** Setup offset url and connect. */
922 @Override
923 protected HttpURLConnection connect(final long offset,
924 final boolean resolved) throws IOException {
925 final URL offsetUrl = offset == 0L? url
926 : new URL(url + "&" + new OffsetParam(offset));
927 return new URLRunner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn;
928 }
929 }
930
931 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
932
933 /** Remove offset parameter, if there is any, from the url */
934 static URL removeOffsetParam(final URL url) throws MalformedURLException {
935 String query = url.getQuery();
936 if (query == null) {
937 return url;
938 }
939 final String lower = query.toLowerCase();
940 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
941 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
942 return url;
943 }
944
945 //rebuild query
946 StringBuilder b = null;
947 for(final StringTokenizer st = new StringTokenizer(query, "&");
948 st.hasMoreTokens();) {
949 final String token = st.nextToken();
950 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
951 if (b == null) {
952 b = new StringBuilder("?").append(token);
953 } else {
954 b.append('&').append(token);
955 }
956 }
957 }
958 query = b == null? "": b.toString();
959
960 final String urlStr = url.toString();
961 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
962 }
963
964 static class OffsetUrlInputStream extends ByteRangeInputStream {
965 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
966 super(o, r);
967 }
968
969 /** Remove offset parameter before returning the resolved url. */
970 @Override
971 protected URL getResolvedUrl(final HttpURLConnection connection
972 ) throws MalformedURLException {
973 return removeOffsetParam(connection.getURL());
974 }
975 }
976
977 @Override
978 public FileStatus[] listStatus(final Path f) throws IOException {
979 statistics.incrementReadOps(1);
980
981 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
982 final Map<?, ?> json = run(op, f);
983 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
984 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
985
986 //convert FileStatus
987 final FileStatus[] statuses = new FileStatus[array.length];
988 for(int i = 0; i < array.length; i++) {
989 final Map<?, ?> m = (Map<?, ?>)array[i];
990 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
991 }
992 return statuses;
993 }
994
995 @Override
996 public Token<DelegationTokenIdentifier> getDelegationToken(
997 final String renewer) throws IOException {
998 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
999 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
1000 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
1001 token.setService(tokenServiceName);
1002 return token;
1003 }
1004
1005 @Override
1006 public synchronized Token<?> getRenewToken() {
1007 return delegationToken;
1008 }
1009
1010 @Override
1011 public <T extends TokenIdentifier> void setDelegationToken(
1012 final Token<T> token) {
1013 synchronized (this) {
1014 delegationToken = token;
1015 }
1016 }
1017
1018 @Override
1019 public synchronized long renewDelegationToken(final Token<?> token
1020 ) throws IOException {
1021 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
1022 TokenArgumentParam dtargParam = new TokenArgumentParam(
1023 token.encodeToUrlString());
1024 final Map<?, ?> m = run(op, null, dtargParam);
1025 return (Long) m.get("long");
1026 }
1027
1028 @Override
1029 public synchronized void cancelDelegationToken(final Token<?> token
1030 ) throws IOException {
1031 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
1032 TokenArgumentParam dtargParam = new TokenArgumentParam(
1033 token.encodeToUrlString());
1034 run(op, null, dtargParam);
1035 }
1036
1037 @Override
1038 public BlockLocation[] getFileBlockLocations(final FileStatus status,
1039 final long offset, final long length) throws IOException {
1040 if (status == null) {
1041 return null;
1042 }
1043 return getFileBlockLocations(status.getPath(), offset, length);
1044 }
1045
1046 @Override
1047 public BlockLocation[] getFileBlockLocations(final Path p,
1048 final long offset, final long length) throws IOException {
1049 statistics.incrementReadOps(1);
1050
1051 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
1052 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
1053 new LengthParam(length));
1054 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
1055 }
1056
1057 @Override
1058 public ContentSummary getContentSummary(final Path p) throws IOException {
1059 statistics.incrementReadOps(1);
1060
1061 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
1062 final Map<?, ?> m = run(op, p);
1063 return JsonUtil.toContentSummary(m);
1064 }
1065
1066 @Override
1067 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
1068 ) throws IOException {
1069 statistics.incrementReadOps(1);
1070
1071 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
1072 final Map<?, ?> m = run(op, p);
1073 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
1074 }
1075
1076 @Override
1077 public String getCanonicalServiceName() {
1078 return tokenServiceName == null ? super.getCanonicalServiceName()
1079 : tokenServiceName.toString();
1080 }
1081 }