001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.security.PrivilegedExceptionAction;
022
023 import java.io.BufferedOutputStream;
024 import java.io.FileNotFoundException;
025 import java.io.IOException;
026 import java.io.InputStream;
027 import java.io.InputStreamReader;
028 import java.net.HttpURLConnection;
029 import java.net.InetSocketAddress;
030 import java.net.MalformedURLException;
031 import java.net.URI;
032 import java.net.URISyntaxException;
033 import java.net.URL;
034 import java.util.Collection;
035 import java.util.List;
036 import java.util.Map;
037 import java.util.StringTokenizer;
038
039 import javax.ws.rs.core.MediaType;
040
041 import org.apache.commons.logging.Log;
042 import org.apache.commons.logging.LogFactory;
043 import org.apache.hadoop.conf.Configuration;
044 import org.apache.hadoop.fs.BlockLocation;
045 import org.apache.hadoop.fs.ContentSummary;
046 import org.apache.hadoop.fs.FSDataInputStream;
047 import org.apache.hadoop.fs.FSDataOutputStream;
048 import org.apache.hadoop.fs.FileAlreadyExistsException;
049 import org.apache.hadoop.fs.FileStatus;
050 import org.apache.hadoop.fs.FileSystem;
051 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
052 import org.apache.hadoop.fs.Options;
053 import org.apache.hadoop.fs.ParentNotDirectoryException;
054 import org.apache.hadoop.fs.Path;
055 import org.apache.hadoop.fs.permission.FsPermission;
056 import org.apache.hadoop.hdfs.ByteRangeInputStream;
057 import org.apache.hadoop.hdfs.DFSConfigKeys;
058 import org.apache.hadoop.hdfs.DFSUtil;
059 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
060 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
061 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
062 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
063 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
064 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer;
065 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
066 import org.apache.hadoop.hdfs.server.common.JspHelper;
067 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
068 import org.apache.hadoop.hdfs.web.resources.*;
069 import org.apache.hadoop.io.Text;
070 import org.apache.hadoop.ipc.RemoteException;
071 import org.apache.hadoop.net.NetUtils;
072 import org.apache.hadoop.security.AccessControlException;
073 import org.apache.hadoop.security.SecurityUtil;
074 import org.apache.hadoop.security.UserGroupInformation;
075 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
076 import org.apache.hadoop.security.authentication.client.AuthenticationException;
077 import org.apache.hadoop.security.authorize.AuthorizationException;
078 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
079 import org.apache.hadoop.security.token.Token;
080 import org.apache.hadoop.security.token.TokenIdentifier;
081 import org.apache.hadoop.security.token.TokenRenewer;
082 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
083 import org.apache.hadoop.util.Progressable;
084 import org.mortbay.util.ajax.JSON;
085
086 import com.google.common.annotations.VisibleForTesting;
087 import com.google.common.collect.Lists;
088
089 /** A FileSystem for HDFS over the web. */
090 public class WebHdfsFileSystem extends FileSystem
091 implements DelegationTokenRenewer.Renewable {
092 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
093 /** File System URI: {SCHEME}://namenode:port/path/to/file */
094 public static final String SCHEME = "webhdfs";
095 /** WebHdfs version. */
096 public static final int VERSION = 1;
097 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
098 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
099
100 /** SPNEGO authenticator */
101 private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
102 /** Delegation token kind */
103 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
104 /** Token selector */
105 public static final WebHdfsDelegationTokenSelector DT_SELECTOR
106 = new WebHdfsDelegationTokenSelector();
107
108 private DelegationTokenRenewer dtRenewer = null;
109 @VisibleForTesting
110 DelegationTokenRenewer.RenewAction<?> action;
111
112 @VisibleForTesting
113 protected synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
114 if (dtRenewer == null) {
115 dtRenewer = DelegationTokenRenewer.getInstance();
116 }
117
118 action = dtRenewer.addRenewAction(webhdfs);
119 }
120
121 /** Is WebHDFS enabled in conf? */
122 public static boolean isEnabled(final Configuration conf, final Log log) {
123 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
124 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
125 log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
126 return b;
127 }
128
129 private UserGroupInformation ugi;
130 private InetSocketAddress nnAddr;
131 private URI uri;
132 private boolean hasInitedToken;
133 private Token<?> delegationToken;
134 private Path workingDir;
135
136 @Override
137 public synchronized void initialize(URI uri, Configuration conf
138 ) throws IOException {
139 super.initialize(uri, conf);
140 setConf(conf);
141 ugi = UserGroupInformation.getCurrentUser();
142 try {
143 this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
144 } catch (URISyntaxException e) {
145 throw new IllegalArgumentException(e);
146 }
147 this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
148 this.workingDir = getHomeDirectory();
149
150 if (UserGroupInformation.isSecurityEnabled()) {
151 initDelegationToken();
152 }
153 }
154
155 protected void initDelegationToken() throws IOException {
156 // look for webhdfs token, then try hdfs
157 Token<?> token = selectDelegationToken(ugi);
158 if (token != null) {
159 LOG.debug("Found existing DT for " + token.getService());
160 setDelegationToken(token);
161 hasInitedToken = true;
162 }
163 }
164
165 protected synchronized Token<?> getDelegationToken() throws IOException {
166 // we haven't inited yet, or we used to have a token but it expired
167 if (!hasInitedToken || (action != null && !action.isValid())) {
168 //since we don't already have a token, go get one
169 Token<?> token = getDelegationToken(null);
170 // security might be disabled
171 if (token != null) {
172 setDelegationToken(token);
173 addRenewAction(this);
174 LOG.debug("Created new DT for " + token.getService());
175 }
176 hasInitedToken = true;
177 }
178 return delegationToken;
179 }
180
181 protected Token<DelegationTokenIdentifier> selectDelegationToken(
182 UserGroupInformation ugi) {
183 return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
184 }
185
186 @Override
187 protected int getDefaultPort() {
188 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
189 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
190 }
191
192 @Override
193 public URI getUri() {
194 return this.uri;
195 }
196
197 /** @return the home directory. */
198 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
199 return "/user/" + ugi.getShortUserName();
200 }
201
202 @Override
203 public Path getHomeDirectory() {
204 return makeQualified(new Path(getHomeDirectoryString(ugi)));
205 }
206
207 @Override
208 public synchronized Path getWorkingDirectory() {
209 return workingDir;
210 }
211
212 @Override
213 public synchronized void setWorkingDirectory(final Path dir) {
214 String result = makeAbsolute(dir).toUri().getPath();
215 if (!DFSUtil.isValidName(result)) {
216 throw new IllegalArgumentException("Invalid DFS directory name " +
217 result);
218 }
219 workingDir = makeAbsolute(dir);
220 }
221
222 private Path makeAbsolute(Path f) {
223 return f.isAbsolute()? f: new Path(workingDir, f);
224 }
225
226 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
227 ) throws IOException {
228 if (c.getContentLength() == 0) {
229 return null;
230 }
231 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
232 if (in == null) {
233 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
234 }
235 final String contentType = c.getContentType();
236 if (contentType != null) {
237 final MediaType parsed = MediaType.valueOf(contentType);
238 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
239 throw new IOException("Content-Type \"" + contentType
240 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
241 + "\" (parsed=\"" + parsed + "\")");
242 }
243 }
244 return (Map<?, ?>)JSON.parse(new InputStreamReader(in));
245 }
246
247 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
248 final HttpURLConnection conn) throws IOException {
249 final int code = conn.getResponseCode();
250 if (code != op.getExpectedHttpResponseCode()) {
251 final Map<?, ?> m;
252 try {
253 m = jsonParse(conn, true);
254 } catch(IOException e) {
255 throw new IOException("Unexpected HTTP response: code=" + code + " != "
256 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
257 + ", message=" + conn.getResponseMessage(), e);
258 }
259
260 if (m == null) {
261 throw new IOException("Unexpected HTTP response: code=" + code + " != "
262 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
263 + ", message=" + conn.getResponseMessage());
264 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
265 return m;
266 }
267
268 final RemoteException re = JsonUtil.toRemoteException(m);
269 throw re.unwrapRemoteException(AccessControlException.class,
270 InvalidToken.class,
271 AuthenticationException.class,
272 AuthorizationException.class,
273 FileAlreadyExistsException.class,
274 FileNotFoundException.class,
275 ParentNotDirectoryException.class,
276 UnresolvedPathException.class,
277 SafeModeException.class,
278 DSQuotaExceededException.class,
279 NSQuotaExceededException.class);
280 }
281 return null;
282 }
283
284 /**
285 * Return a URL pointing to given path on the namenode.
286 *
287 * @param path to obtain the URL for
288 * @param query string to append to the path
289 * @return namenode URL referring to the given path
290 * @throws IOException on error constructing the URL
291 */
292 private URL getNamenodeURL(String path, String query) throws IOException {
293 final URL url = new URL("http", nnAddr.getHostName(),
294 nnAddr.getPort(), path + '?' + query);
295 if (LOG.isTraceEnabled()) {
296 LOG.trace("url=" + url);
297 }
298 return url;
299 }
300
301 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
302 List<Param<?,?>> authParams = Lists.newArrayList();
303 // Skip adding delegation token for token operations because these
304 // operations require authentication.
305 Token<?> token = null;
306 if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) {
307 token = getDelegationToken();
308 }
309 if (token != null) {
310 authParams.add(new DelegationParam(token.encodeToUrlString()));
311 } else {
312 UserGroupInformation userUgi = ugi;
313 UserGroupInformation realUgi = userUgi.getRealUser();
314 if (realUgi != null) { // proxy user
315 authParams.add(new DoAsParam(userUgi.getShortUserName()));
316 userUgi = realUgi;
317 }
318 authParams.add(new UserParam(userUgi.getShortUserName()));
319 }
320 return authParams.toArray(new Param<?,?>[0]);
321 }
322
323 URL toUrl(final HttpOpParam.Op op, final Path fspath,
324 final Param<?,?>... parameters) throws IOException {
325 //initialize URI path and query
326 final String path = PATH_PREFIX
327 + (fspath == null? "/": makeQualified(fspath).toUri().getPath());
328 final String query = op.toQueryString()
329 + Param.toSortedString("&", getAuthParameters(op))
330 + Param.toSortedString("&", parameters);
331 final URL url = getNamenodeURL(path, query);
332 if (LOG.isTraceEnabled()) {
333 LOG.trace("url=" + url);
334 }
335 return url;
336 }
337
338 private HttpURLConnection getHttpUrlConnection(final URL url,
339 final boolean requireAuth)
340 throws IOException {
341 UserGroupInformation connectUgi = ugi.getRealUser();
342 if (connectUgi == null) {
343 connectUgi = ugi;
344 }
345 try {
346 return connectUgi.doAs(
347 new PrivilegedExceptionAction<HttpURLConnection>() {
348 @Override
349 public HttpURLConnection run() throws IOException {
350 return openHttpUrlConnection(url, requireAuth);
351 }
352 });
353 } catch (InterruptedException e) {
354 throw new IOException(e);
355 }
356 }
357
358 private HttpURLConnection openHttpUrlConnection(URL url, boolean requireAuth)
359 throws IOException {
360 final HttpURLConnection conn;
361 try {
362 if (requireAuth) {
363 LOG.debug("open AuthenticatedURL connection");
364 UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
365 final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
366 conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
367 } else {
368 LOG.debug("open URL connection");
369 conn = (HttpURLConnection)url.openConnection();
370 }
371 } catch (AuthenticationException e) {
372 throw new IOException(e);
373 }
374 return conn;
375 }
376
377 private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
378 final Param<?,?>... parameters) throws IOException {
379 final URL url = toUrl(op, fspath, parameters);
380
381 //connect and get response
382 HttpURLConnection conn = getHttpUrlConnection(url, op.getRequireAuth());
383 try {
384 conn.setRequestMethod(op.getType().toString());
385 if (op.getDoOutput()) {
386 conn = twoStepWrite(conn, op);
387 conn.setRequestProperty("Content-Type", "application/octet-stream");
388 }
389 conn.setDoOutput(op.getDoOutput());
390 conn.connect();
391 return conn;
392 } catch (IOException e) {
393 conn.disconnect();
394 throw e;
395 }
396 }
397
398 /**
399 * Two-step Create/Append:
400 * Step 1) Submit a Http request with neither auto-redirect nor data.
401 * Step 2) Submit another Http request with the URL from the Location header with data.
402 *
403 * The reason of having two-step create/append is for preventing clients to
404 * send out the data before the redirect. This issue is addressed by the
405 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
406 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
407 * and Java 6 http client), which do not correctly implement "Expect:
408 * 100-continue". The two-step create/append is a temporary workaround for
409 * the software library bugs.
410 */
411 static HttpURLConnection twoStepWrite(HttpURLConnection conn,
412 final HttpOpParam.Op op) throws IOException {
413 //Step 1) Submit a Http request with neither auto-redirect nor data.
414 conn.setInstanceFollowRedirects(false);
415 conn.setDoOutput(false);
416 conn.connect();
417 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn);
418 final String redirect = conn.getHeaderField("Location");
419 conn.disconnect();
420
421 //Step 2) Submit another Http request with the URL from the Location header with data.
422 conn = (HttpURLConnection)new URL(redirect).openConnection();
423 conn.setRequestMethod(op.getType().toString());
424 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
425 return conn;
426 }
427
428 /**
429 * Run a http operation.
430 * Connect to the http server, validate response, and obtain the JSON output.
431 *
432 * @param op http operation
433 * @param fspath file system path
434 * @param parameters parameters for the operation
435 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
436 * @throws IOException
437 */
438 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
439 final Param<?,?>... parameters) throws IOException {
440 final HttpURLConnection conn = httpConnect(op, fspath, parameters);
441 try {
442 final Map<?, ?> m = validateResponse(op, conn);
443 return m != null? m: jsonParse(conn, false);
444 } finally {
445 conn.disconnect();
446 }
447 }
448
449 private FsPermission applyUMask(FsPermission permission) {
450 if (permission == null) {
451 permission = FsPermission.getDefault();
452 }
453 return permission.applyUMask(FsPermission.getUMask(getConf()));
454 }
455
456 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
457 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
458 final Map<?, ?> json = run(op, f);
459 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
460 if (status == null) {
461 throw new FileNotFoundException("File does not exist: " + f);
462 }
463 return status;
464 }
465
466 @Override
467 public FileStatus getFileStatus(Path f) throws IOException {
468 statistics.incrementReadOps(1);
469 return makeQualified(getHdfsFileStatus(f), f);
470 }
471
472 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
473 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
474 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
475 f.getPermission(), f.getOwner(), f.getGroup(),
476 f.isSymlink() ? new Path(f.getSymlink()) : null,
477 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
478 }
479
480 @Override
481 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
482 statistics.incrementWriteOps(1);
483 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
484 final Map<?, ?> json = run(op, f,
485 new PermissionParam(applyUMask(permission)));
486 return (Boolean)json.get("boolean");
487 }
488
489 /**
490 * Create a symlink pointing to the destination path.
491 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
492 */
493 public void createSymlink(Path destination, Path f, boolean createParent
494 ) throws IOException {
495 statistics.incrementWriteOps(1);
496 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
497 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
498 new CreateParentParam(createParent));
499 }
500
501 @Override
502 public boolean rename(final Path src, final Path dst) throws IOException {
503 statistics.incrementWriteOps(1);
504 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
505 final Map<?, ?> json = run(op, src,
506 new DestinationParam(makeQualified(dst).toUri().getPath()));
507 return (Boolean)json.get("boolean");
508 }
509
510 @SuppressWarnings("deprecation")
511 @Override
512 public void rename(final Path src, final Path dst,
513 final Options.Rename... options) throws IOException {
514 statistics.incrementWriteOps(1);
515 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
516 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
517 new RenameOptionSetParam(options));
518 }
519
520 @Override
521 public void setOwner(final Path p, final String owner, final String group
522 ) throws IOException {
523 if (owner == null && group == null) {
524 throw new IOException("owner == null && group == null");
525 }
526
527 statistics.incrementWriteOps(1);
528 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
529 run(op, p, new OwnerParam(owner), new GroupParam(group));
530 }
531
532 @Override
533 public void setPermission(final Path p, final FsPermission permission
534 ) throws IOException {
535 statistics.incrementWriteOps(1);
536 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
537 run(op, p, new PermissionParam(permission));
538 }
539
540 @Override
541 public boolean setReplication(final Path p, final short replication
542 ) throws IOException {
543 statistics.incrementWriteOps(1);
544 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
545 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
546 return (Boolean)json.get("boolean");
547 }
548
549 @Override
550 public void setTimes(final Path p, final long mtime, final long atime
551 ) throws IOException {
552 statistics.incrementWriteOps(1);
553 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
554 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
555 }
556
557 @Override
558 public long getDefaultBlockSize() {
559 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
560 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
561 }
562
563 @Override
564 public short getDefaultReplication() {
565 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
566 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
567 }
568
569 FSDataOutputStream write(final HttpOpParam.Op op,
570 final HttpURLConnection conn, final int bufferSize) throws IOException {
571 return new FSDataOutputStream(new BufferedOutputStream(
572 conn.getOutputStream(), bufferSize), statistics) {
573 @Override
574 public void close() throws IOException {
575 try {
576 super.close();
577 } finally {
578 try {
579 validateResponse(op, conn);
580 } finally {
581 conn.disconnect();
582 }
583 }
584 }
585 };
586 }
587
588 @Override
589 public FSDataOutputStream create(final Path f, final FsPermission permission,
590 final boolean overwrite, final int bufferSize, final short replication,
591 final long blockSize, final Progressable progress) throws IOException {
592 statistics.incrementWriteOps(1);
593
594 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
595 final HttpURLConnection conn = httpConnect(op, f,
596 new PermissionParam(applyUMask(permission)),
597 new OverwriteParam(overwrite),
598 new BufferSizeParam(bufferSize),
599 new ReplicationParam(replication),
600 new BlockSizeParam(blockSize));
601 return write(op, conn, bufferSize);
602 }
603
604 @Override
605 public FSDataOutputStream append(final Path f, final int bufferSize,
606 final Progressable progress) throws IOException {
607 statistics.incrementWriteOps(1);
608
609 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
610 final HttpURLConnection conn = httpConnect(op, f,
611 new BufferSizeParam(bufferSize));
612 return write(op, conn, bufferSize);
613 }
614
615 @SuppressWarnings("deprecation")
616 @Override
617 public boolean delete(final Path f) throws IOException {
618 return delete(f, true);
619 }
620
621 @Override
622 public boolean delete(Path f, boolean recursive) throws IOException {
623 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
624 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
625 return (Boolean)json.get("boolean");
626 }
627
628 @Override
629 public FSDataInputStream open(final Path f, final int buffersize
630 ) throws IOException {
631 statistics.incrementReadOps(1);
632 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
633 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
634 return new FSDataInputStream(new OffsetUrlInputStream(
635 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
636 }
637
638 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
639 /** The url with offset parameter */
640 private URL offsetUrl;
641
642 OffsetUrlOpener(final URL url) {
643 super(url);
644 }
645
646 /** Open connection with offset url. */
647 @Override
648 protected HttpURLConnection openConnection() throws IOException {
649 return getHttpUrlConnection(offsetUrl, false);
650 }
651
652 /** Setup offset url before open connection. */
653 @Override
654 protected HttpURLConnection openConnection(final long offset) throws IOException {
655 offsetUrl = offset == 0L? url: new URL(url + "&" + new OffsetParam(offset));
656 final HttpURLConnection conn = openConnection();
657 conn.setRequestMethod("GET");
658 return conn;
659 }
660 }
661
662 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
663
664 /** Remove offset parameter, if there is any, from the url */
665 static URL removeOffsetParam(final URL url) throws MalformedURLException {
666 String query = url.getQuery();
667 if (query == null) {
668 return url;
669 }
670 final String lower = query.toLowerCase();
671 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
672 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
673 return url;
674 }
675
676 //rebuild query
677 StringBuilder b = null;
678 for(final StringTokenizer st = new StringTokenizer(query, "&");
679 st.hasMoreTokens();) {
680 final String token = st.nextToken();
681 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
682 if (b == null) {
683 b = new StringBuilder("?").append(token);
684 } else {
685 b.append('&').append(token);
686 }
687 }
688 }
689 query = b == null? "": b.toString();
690
691 final String urlStr = url.toString();
692 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
693 }
694
695 static class OffsetUrlInputStream extends ByteRangeInputStream {
696 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
697 super(o, r);
698 }
699
700 @Override
701 protected void checkResponseCode(final HttpURLConnection connection
702 ) throws IOException {
703 validateResponse(GetOpParam.Op.OPEN, connection);
704 }
705
706 /** Remove offset parameter before returning the resolved url. */
707 @Override
708 protected URL getResolvedUrl(final HttpURLConnection connection
709 ) throws MalformedURLException {
710 return removeOffsetParam(connection.getURL());
711 }
712 }
713
714 @Override
715 public FileStatus[] listStatus(final Path f) throws IOException {
716 statistics.incrementReadOps(1);
717
718 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
719 final Map<?, ?> json = run(op, f);
720 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
721 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
722
723 //convert FileStatus
724 final FileStatus[] statuses = new FileStatus[array.length];
725 for(int i = 0; i < array.length; i++) {
726 final Map<?, ?> m = (Map<?, ?>)array[i];
727 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
728 }
729 return statuses;
730 }
731
732 @Override
733 public Token<DelegationTokenIdentifier> getDelegationToken(
734 final String renewer) throws IOException {
735 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
736 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
737 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
738 SecurityUtil.setTokenService(token, nnAddr);
739 return token;
740 }
741
742 @Override
743 public Token<?> getRenewToken() {
744 return delegationToken;
745 }
746
747 @Override
748 public <T extends TokenIdentifier> void setDelegationToken(
749 final Token<T> token) {
750 synchronized(this) {
751 delegationToken = token;
752 }
753 }
754
755 private synchronized long renewDelegationToken(final Token<?> token
756 ) throws IOException {
757 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
758 TokenArgumentParam dtargParam = new TokenArgumentParam(
759 token.encodeToUrlString());
760 final Map<?, ?> m = run(op, null, dtargParam);
761 return (Long) m.get("long");
762 }
763
764 private synchronized void cancelDelegationToken(final Token<?> token
765 ) throws IOException {
766 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
767 TokenArgumentParam dtargParam = new TokenArgumentParam(
768 token.encodeToUrlString());
769 run(op, null, dtargParam);
770 }
771
772 @Override
773 public BlockLocation[] getFileBlockLocations(final FileStatus status,
774 final long offset, final long length) throws IOException {
775 if (status == null) {
776 return null;
777 }
778 return getFileBlockLocations(status.getPath(), offset, length);
779 }
780
781 @Override
782 public BlockLocation[] getFileBlockLocations(final Path p,
783 final long offset, final long length) throws IOException {
784 statistics.incrementReadOps(1);
785
786 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
787 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
788 new LengthParam(length));
789 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
790 }
791
792 @Override
793 public ContentSummary getContentSummary(final Path p) throws IOException {
794 statistics.incrementReadOps(1);
795
796 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
797 final Map<?, ?> m = run(op, p);
798 return JsonUtil.toContentSummary(m);
799 }
800
801 @Override
802 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
803 ) throws IOException {
804 statistics.incrementReadOps(1);
805
806 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
807 final Map<?, ?> m = run(op, p);
808 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
809 }
810
811 /** Delegation token renewer. */
812 public static class DtRenewer extends TokenRenewer {
813 @Override
814 public boolean handleKind(Text kind) {
815 return kind.equals(TOKEN_KIND);
816 }
817
818 @Override
819 public boolean isManaged(Token<?> token) throws IOException {
820 return true;
821 }
822
823 private static WebHdfsFileSystem getWebHdfs(
824 final Token<?> token, final Configuration conf) throws IOException {
825
826 final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
827 final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
828 return (WebHdfsFileSystem)FileSystem.get(uri, conf);
829 }
830
831 @Override
832 public long renew(final Token<?> token, final Configuration conf
833 ) throws IOException, InterruptedException {
834 return getWebHdfs(token, conf).renewDelegationToken(token);
835 }
836
837 @Override
838 public void cancel(final Token<?> token, final Configuration conf
839 ) throws IOException, InterruptedException {
840 getWebHdfs(token, conf).cancelDelegationToken(token);
841 }
842 }
843
844 private static class WebHdfsDelegationTokenSelector
845 extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
846 private static final DelegationTokenSelector hdfsTokenSelector =
847 new DelegationTokenSelector();
848
849 public WebHdfsDelegationTokenSelector() {
850 super(TOKEN_KIND);
851 }
852
853 Token<DelegationTokenIdentifier> selectToken(URI nnUri,
854 Collection<Token<?>> tokens, Configuration conf) {
855 Token<DelegationTokenIdentifier> token =
856 selectToken(SecurityUtil.buildTokenService(nnUri), tokens);
857 if (token == null) {
858 token = hdfsTokenSelector.selectToken(nnUri, tokens, conf);
859 }
860 return token;
861 }
862 }
863 }