001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.security.PrivilegedExceptionAction;
022
023 import java.io.BufferedOutputStream;
024 import java.io.FileNotFoundException;
025 import java.io.IOException;
026 import java.io.InputStream;
027 import java.io.InputStreamReader;
028 import java.net.HttpURLConnection;
029 import java.net.InetSocketAddress;
030 import java.net.MalformedURLException;
031 import java.net.URI;
032 import java.net.URISyntaxException;
033 import java.net.URL;
034 import java.util.Collection;
035 import java.util.List;
036 import java.util.Map;
037 import java.util.StringTokenizer;
038
039 import javax.ws.rs.core.MediaType;
040
041 import org.apache.commons.logging.Log;
042 import org.apache.commons.logging.LogFactory;
043 import org.apache.hadoop.conf.Configuration;
044 import org.apache.hadoop.fs.BlockLocation;
045 import org.apache.hadoop.fs.ContentSummary;
046 import org.apache.hadoop.fs.FSDataInputStream;
047 import org.apache.hadoop.fs.FSDataOutputStream;
048 import org.apache.hadoop.fs.FileAlreadyExistsException;
049 import org.apache.hadoop.fs.FileStatus;
050 import org.apache.hadoop.fs.FileSystem;
051 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
052 import org.apache.hadoop.fs.Options;
053 import org.apache.hadoop.fs.ParentNotDirectoryException;
054 import org.apache.hadoop.fs.Path;
055 import org.apache.hadoop.fs.permission.FsPermission;
056 import org.apache.hadoop.hdfs.ByteRangeInputStream;
057 import org.apache.hadoop.hdfs.DFSConfigKeys;
058 import org.apache.hadoop.hdfs.DFSUtil;
059 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
060 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
061 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
062 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
063 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
064 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer;
065 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
066 import org.apache.hadoop.hdfs.server.common.JspHelper;
067 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
068 import org.apache.hadoop.hdfs.web.resources.*;
069 import org.apache.hadoop.io.Text;
070 import org.apache.hadoop.ipc.RemoteException;
071 import org.apache.hadoop.net.NetUtils;
072 import org.apache.hadoop.security.AccessControlException;
073 import org.apache.hadoop.security.SecurityUtil;
074 import org.apache.hadoop.security.UserGroupInformation;
075 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
076 import org.apache.hadoop.security.authentication.client.AuthenticationException;
077 import org.apache.hadoop.security.authorize.AuthorizationException;
078 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
079 import org.apache.hadoop.security.token.Token;
080 import org.apache.hadoop.security.token.TokenIdentifier;
081 import org.apache.hadoop.security.token.TokenRenewer;
082 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
083 import org.apache.hadoop.util.Progressable;
084 import org.mortbay.util.ajax.JSON;
085
086 import com.google.common.annotations.VisibleForTesting;
087 import com.google.common.collect.Lists;
088
089 /** A FileSystem for HDFS over the web. */
090 public class WebHdfsFileSystem extends FileSystem
091 implements DelegationTokenRenewer.Renewable {
092 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
093 /** File System URI: {SCHEME}://namenode:port/path/to/file */
094 public static final String SCHEME = "webhdfs";
095 /** WebHdfs version. */
096 public static final int VERSION = 1;
097 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
098 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
099
100 /** SPNEGO authenticator */
101 private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
102 /** Delegation token kind */
103 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
104 /** Token selector */
105 public static final WebHdfsDelegationTokenSelector DT_SELECTOR
106 = new WebHdfsDelegationTokenSelector();
107
108 private static DelegationTokenRenewer<WebHdfsFileSystem> DT_RENEWER = null;
109
110 @VisibleForTesting
111 protected synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
112 if (DT_RENEWER == null) {
113 DT_RENEWER = new DelegationTokenRenewer<WebHdfsFileSystem>(WebHdfsFileSystem.class);
114 DT_RENEWER.start();
115 }
116
117 DT_RENEWER.addRenewAction(webhdfs);
118 }
119
120 /** Is WebHDFS enabled in conf? */
121 public static boolean isEnabled(final Configuration conf, final Log log) {
122 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
123 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
124 log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
125 return b;
126 }
127
128 private UserGroupInformation ugi;
129 private InetSocketAddress nnAddr;
130 private URI uri;
131 private boolean hasInitedToken;
132 private Token<?> delegationToken;
133 private Path workingDir;
134
135 @Override
136 public synchronized void initialize(URI uri, Configuration conf
137 ) throws IOException {
138 super.initialize(uri, conf);
139 setConf(conf);
140 ugi = UserGroupInformation.getCurrentUser();
141 try {
142 this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
143 } catch (URISyntaxException e) {
144 throw new IllegalArgumentException(e);
145 }
146 this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
147 this.workingDir = getHomeDirectory();
148
149 if (UserGroupInformation.isSecurityEnabled()) {
150 initDelegationToken();
151 }
152 }
153
154 protected void initDelegationToken() throws IOException {
155 // look for webhdfs token, then try hdfs
156 Token<?> token = selectDelegationToken(ugi);
157 if (token != null) {
158 LOG.debug("Found existing DT for " + token.getService());
159 setDelegationToken(token);
160 hasInitedToken = true;
161 }
162 }
163
164 protected synchronized Token<?> getDelegationToken() throws IOException {
165 if (!hasInitedToken) {
166 //since we don't already have a token, go get one
167 Token<?> token = getDelegationToken(null);
168 // security might be disabled
169 if (token != null) {
170 setDelegationToken(token);
171 addRenewAction(this);
172 LOG.debug("Created new DT for " + token.getService());
173 }
174 hasInitedToken = true;
175 }
176 return delegationToken;
177 }
178
179 protected Token<DelegationTokenIdentifier> selectDelegationToken(
180 UserGroupInformation ugi) {
181 return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
182 }
183
184 @Override
185 protected int getDefaultPort() {
186 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
187 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
188 }
189
190 @Override
191 public URI getUri() {
192 return this.uri;
193 }
194
195 /** @return the home directory. */
196 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
197 return "/user/" + ugi.getShortUserName();
198 }
199
200 @Override
201 public Path getHomeDirectory() {
202 return makeQualified(new Path(getHomeDirectoryString(ugi)));
203 }
204
205 @Override
206 public synchronized Path getWorkingDirectory() {
207 return workingDir;
208 }
209
210 @Override
211 public synchronized void setWorkingDirectory(final Path dir) {
212 String result = makeAbsolute(dir).toUri().getPath();
213 if (!DFSUtil.isValidName(result)) {
214 throw new IllegalArgumentException("Invalid DFS directory name " +
215 result);
216 }
217 workingDir = makeAbsolute(dir);
218 }
219
220 private Path makeAbsolute(Path f) {
221 return f.isAbsolute()? f: new Path(workingDir, f);
222 }
223
224 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
225 ) throws IOException {
226 if (c.getContentLength() == 0) {
227 return null;
228 }
229 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
230 if (in == null) {
231 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
232 }
233 final String contentType = c.getContentType();
234 if (contentType != null) {
235 final MediaType parsed = MediaType.valueOf(contentType);
236 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
237 throw new IOException("Content-Type \"" + contentType
238 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
239 + "\" (parsed=\"" + parsed + "\")");
240 }
241 }
242 return (Map<?, ?>)JSON.parse(new InputStreamReader(in));
243 }
244
245 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
246 final HttpURLConnection conn) throws IOException {
247 final int code = conn.getResponseCode();
248 if (code != op.getExpectedHttpResponseCode()) {
249 final Map<?, ?> m;
250 try {
251 m = jsonParse(conn, true);
252 } catch(IOException e) {
253 throw new IOException("Unexpected HTTP response: code=" + code + " != "
254 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
255 + ", message=" + conn.getResponseMessage(), e);
256 }
257
258 if (m == null) {
259 throw new IOException("Unexpected HTTP response: code=" + code + " != "
260 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
261 + ", message=" + conn.getResponseMessage());
262 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
263 return m;
264 }
265
266 final RemoteException re = JsonUtil.toRemoteException(m);
267 throw re.unwrapRemoteException(AccessControlException.class,
268 InvalidToken.class,
269 AuthenticationException.class,
270 AuthorizationException.class,
271 FileAlreadyExistsException.class,
272 FileNotFoundException.class,
273 ParentNotDirectoryException.class,
274 UnresolvedPathException.class,
275 SafeModeException.class,
276 DSQuotaExceededException.class,
277 NSQuotaExceededException.class);
278 }
279 return null;
280 }
281
282 /**
283 * Return a URL pointing to given path on the namenode.
284 *
285 * @param path to obtain the URL for
286 * @param query string to append to the path
287 * @return namenode URL referring to the given path
288 * @throws IOException on error constructing the URL
289 */
290 private URL getNamenodeURL(String path, String query) throws IOException {
291 final URL url = new URL("http", nnAddr.getHostName(),
292 nnAddr.getPort(), path + '?' + query);
293 if (LOG.isTraceEnabled()) {
294 LOG.trace("url=" + url);
295 }
296 return url;
297 }
298
299 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
300 List<Param<?,?>> authParams = Lists.newArrayList();
301 // Skip adding delegation token for token operations because these
302 // operations require authentication.
303 Token<?> token = null;
304 if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) {
305 token = getDelegationToken();
306 }
307 if (token != null) {
308 authParams.add(new DelegationParam(token.encodeToUrlString()));
309 } else {
310 UserGroupInformation userUgi = ugi;
311 UserGroupInformation realUgi = userUgi.getRealUser();
312 if (realUgi != null) { // proxy user
313 authParams.add(new DoAsParam(userUgi.getShortUserName()));
314 userUgi = realUgi;
315 }
316 authParams.add(new UserParam(userUgi.getShortUserName()));
317 }
318 return authParams.toArray(new Param<?,?>[0]);
319 }
320
321 URL toUrl(final HttpOpParam.Op op, final Path fspath,
322 final Param<?,?>... parameters) throws IOException {
323 //initialize URI path and query
324 final String path = PATH_PREFIX
325 + (fspath == null? "/": makeQualified(fspath).toUri().getPath());
326 final String query = op.toQueryString()
327 + Param.toSortedString("&", getAuthParameters(op))
328 + Param.toSortedString("&", parameters);
329 final URL url = getNamenodeURL(path, query);
330 if (LOG.isTraceEnabled()) {
331 LOG.trace("url=" + url);
332 }
333 return url;
334 }
335
336 private HttpURLConnection getHttpUrlConnection(final URL url,
337 final boolean requireAuth)
338 throws IOException {
339 UserGroupInformation connectUgi = ugi.getRealUser();
340 if (connectUgi == null) {
341 connectUgi = ugi;
342 }
343 try {
344 return connectUgi.doAs(
345 new PrivilegedExceptionAction<HttpURLConnection>() {
346 @Override
347 public HttpURLConnection run() throws IOException {
348 return openHttpUrlConnection(url, requireAuth);
349 }
350 });
351 } catch (InterruptedException e) {
352 throw new IOException(e);
353 }
354 }
355
356 private HttpURLConnection openHttpUrlConnection(URL url, boolean requireAuth)
357 throws IOException {
358 final HttpURLConnection conn;
359 try {
360 if (requireAuth) {
361 LOG.debug("open AuthenticatedURL connection");
362 UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
363 final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
364 conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
365 } else {
366 LOG.debug("open URL connection");
367 conn = (HttpURLConnection)url.openConnection();
368 }
369 } catch (AuthenticationException e) {
370 throw new IOException(e);
371 }
372 return conn;
373 }
374
375 private HttpURLConnection httpConnect(final HttpOpParam.Op op, final Path fspath,
376 final Param<?,?>... parameters) throws IOException {
377 final URL url = toUrl(op, fspath, parameters);
378
379 //connect and get response
380 HttpURLConnection conn = getHttpUrlConnection(url, op.getRequireAuth());
381 try {
382 conn.setRequestMethod(op.getType().toString());
383 if (op.getDoOutput()) {
384 conn = twoStepWrite(conn, op);
385 conn.setRequestProperty("Content-Type", "application/octet-stream");
386 }
387 conn.setDoOutput(op.getDoOutput());
388 conn.connect();
389 return conn;
390 } catch (IOException e) {
391 conn.disconnect();
392 throw e;
393 }
394 }
395
396 /**
397 * Two-step Create/Append:
398 * Step 1) Submit a Http request with neither auto-redirect nor data.
399 * Step 2) Submit another Http request with the URL from the Location header with data.
400 *
401 * The reason of having two-step create/append is for preventing clients to
402 * send out the data before the redirect. This issue is addressed by the
403 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
404 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
405 * and Java 6 http client), which do not correctly implement "Expect:
406 * 100-continue". The two-step create/append is a temporary workaround for
407 * the software library bugs.
408 */
409 static HttpURLConnection twoStepWrite(HttpURLConnection conn,
410 final HttpOpParam.Op op) throws IOException {
411 //Step 1) Submit a Http request with neither auto-redirect nor data.
412 conn.setInstanceFollowRedirects(false);
413 conn.setDoOutput(false);
414 conn.connect();
415 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn);
416 final String redirect = conn.getHeaderField("Location");
417 conn.disconnect();
418
419 //Step 2) Submit another Http request with the URL from the Location header with data.
420 conn = (HttpURLConnection)new URL(redirect).openConnection();
421 conn.setRequestMethod(op.getType().toString());
422 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
423 return conn;
424 }
425
426 /**
427 * Run a http operation.
428 * Connect to the http server, validate response, and obtain the JSON output.
429 *
430 * @param op http operation
431 * @param fspath file system path
432 * @param parameters parameters for the operation
433 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
434 * @throws IOException
435 */
436 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
437 final Param<?,?>... parameters) throws IOException {
438 final HttpURLConnection conn = httpConnect(op, fspath, parameters);
439 try {
440 final Map<?, ?> m = validateResponse(op, conn);
441 return m != null? m: jsonParse(conn, false);
442 } finally {
443 conn.disconnect();
444 }
445 }
446
447 private FsPermission applyUMask(FsPermission permission) {
448 if (permission == null) {
449 permission = FsPermission.getDefault();
450 }
451 return permission.applyUMask(FsPermission.getUMask(getConf()));
452 }
453
454 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
455 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
456 final Map<?, ?> json = run(op, f);
457 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
458 if (status == null) {
459 throw new FileNotFoundException("File does not exist: " + f);
460 }
461 return status;
462 }
463
464 @Override
465 public FileStatus getFileStatus(Path f) throws IOException {
466 statistics.incrementReadOps(1);
467 return makeQualified(getHdfsFileStatus(f), f);
468 }
469
470 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
471 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
472 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
473 f.getPermission(), f.getOwner(), f.getGroup(),
474 f.isSymlink() ? new Path(f.getSymlink()) : null,
475 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
476 }
477
478 @Override
479 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
480 statistics.incrementWriteOps(1);
481 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
482 final Map<?, ?> json = run(op, f,
483 new PermissionParam(applyUMask(permission)));
484 return (Boolean)json.get("boolean");
485 }
486
487 /**
488 * Create a symlink pointing to the destination path.
489 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
490 */
491 public void createSymlink(Path destination, Path f, boolean createParent
492 ) throws IOException {
493 statistics.incrementWriteOps(1);
494 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
495 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
496 new CreateParentParam(createParent));
497 }
498
499 @Override
500 public boolean rename(final Path src, final Path dst) throws IOException {
501 statistics.incrementWriteOps(1);
502 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
503 final Map<?, ?> json = run(op, src,
504 new DestinationParam(makeQualified(dst).toUri().getPath()));
505 return (Boolean)json.get("boolean");
506 }
507
508 @SuppressWarnings("deprecation")
509 @Override
510 public void rename(final Path src, final Path dst,
511 final Options.Rename... options) throws IOException {
512 statistics.incrementWriteOps(1);
513 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
514 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
515 new RenameOptionSetParam(options));
516 }
517
518 @Override
519 public void setOwner(final Path p, final String owner, final String group
520 ) throws IOException {
521 if (owner == null && group == null) {
522 throw new IOException("owner == null && group == null");
523 }
524
525 statistics.incrementWriteOps(1);
526 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
527 run(op, p, new OwnerParam(owner), new GroupParam(group));
528 }
529
530 @Override
531 public void setPermission(final Path p, final FsPermission permission
532 ) throws IOException {
533 statistics.incrementWriteOps(1);
534 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
535 run(op, p, new PermissionParam(permission));
536 }
537
538 @Override
539 public boolean setReplication(final Path p, final short replication
540 ) throws IOException {
541 statistics.incrementWriteOps(1);
542 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
543 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
544 return (Boolean)json.get("boolean");
545 }
546
547 @Override
548 public void setTimes(final Path p, final long mtime, final long atime
549 ) throws IOException {
550 statistics.incrementWriteOps(1);
551 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
552 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
553 }
554
555 @Override
556 public long getDefaultBlockSize() {
557 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
558 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
559 }
560
561 @Override
562 public short getDefaultReplication() {
563 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
564 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
565 }
566
567 FSDataOutputStream write(final HttpOpParam.Op op,
568 final HttpURLConnection conn, final int bufferSize) throws IOException {
569 return new FSDataOutputStream(new BufferedOutputStream(
570 conn.getOutputStream(), bufferSize), statistics) {
571 @Override
572 public void close() throws IOException {
573 try {
574 super.close();
575 } finally {
576 try {
577 validateResponse(op, conn);
578 } finally {
579 conn.disconnect();
580 }
581 }
582 }
583 };
584 }
585
586 @Override
587 public FSDataOutputStream create(final Path f, final FsPermission permission,
588 final boolean overwrite, final int bufferSize, final short replication,
589 final long blockSize, final Progressable progress) throws IOException {
590 statistics.incrementWriteOps(1);
591
592 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
593 final HttpURLConnection conn = httpConnect(op, f,
594 new PermissionParam(applyUMask(permission)),
595 new OverwriteParam(overwrite),
596 new BufferSizeParam(bufferSize),
597 new ReplicationParam(replication),
598 new BlockSizeParam(blockSize));
599 return write(op, conn, bufferSize);
600 }
601
602 @Override
603 public FSDataOutputStream append(final Path f, final int bufferSize,
604 final Progressable progress) throws IOException {
605 statistics.incrementWriteOps(1);
606
607 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
608 final HttpURLConnection conn = httpConnect(op, f,
609 new BufferSizeParam(bufferSize));
610 return write(op, conn, bufferSize);
611 }
612
613 @SuppressWarnings("deprecation")
614 @Override
615 public boolean delete(final Path f) throws IOException {
616 return delete(f, true);
617 }
618
619 @Override
620 public boolean delete(Path f, boolean recursive) throws IOException {
621 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
622 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
623 return (Boolean)json.get("boolean");
624 }
625
626 @Override
627 public FSDataInputStream open(final Path f, final int buffersize
628 ) throws IOException {
629 statistics.incrementReadOps(1);
630 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
631 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
632 return new FSDataInputStream(new OffsetUrlInputStream(
633 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
634 }
635
636 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
637 /** The url with offset parameter */
638 private URL offsetUrl;
639
640 OffsetUrlOpener(final URL url) {
641 super(url);
642 }
643
644 /** Open connection with offset url. */
645 @Override
646 protected HttpURLConnection openConnection() throws IOException {
647 return getHttpUrlConnection(offsetUrl, false);
648 }
649
650 /** Setup offset url before open connection. */
651 @Override
652 protected HttpURLConnection openConnection(final long offset) throws IOException {
653 offsetUrl = offset == 0L? url: new URL(url + "&" + new OffsetParam(offset));
654 final HttpURLConnection conn = openConnection();
655 conn.setRequestMethod("GET");
656 return conn;
657 }
658 }
659
660 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
661
662 /** Remove offset parameter, if there is any, from the url */
663 static URL removeOffsetParam(final URL url) throws MalformedURLException {
664 String query = url.getQuery();
665 if (query == null) {
666 return url;
667 }
668 final String lower = query.toLowerCase();
669 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
670 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
671 return url;
672 }
673
674 //rebuild query
675 StringBuilder b = null;
676 for(final StringTokenizer st = new StringTokenizer(query, "&");
677 st.hasMoreTokens();) {
678 final String token = st.nextToken();
679 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
680 if (b == null) {
681 b = new StringBuilder("?").append(token);
682 } else {
683 b.append('&').append(token);
684 }
685 }
686 }
687 query = b == null? "": b.toString();
688
689 final String urlStr = url.toString();
690 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
691 }
692
693 static class OffsetUrlInputStream extends ByteRangeInputStream {
694 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
695 super(o, r);
696 }
697
698 @Override
699 protected void checkResponseCode(final HttpURLConnection connection
700 ) throws IOException {
701 validateResponse(GetOpParam.Op.OPEN, connection);
702 }
703
704 /** Remove offset parameter before returning the resolved url. */
705 @Override
706 protected URL getResolvedUrl(final HttpURLConnection connection
707 ) throws MalformedURLException {
708 return removeOffsetParam(connection.getURL());
709 }
710 }
711
712 @Override
713 public FileStatus[] listStatus(final Path f) throws IOException {
714 statistics.incrementReadOps(1);
715
716 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
717 final Map<?, ?> json = run(op, f);
718 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
719 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
720
721 //convert FileStatus
722 final FileStatus[] statuses = new FileStatus[array.length];
723 for(int i = 0; i < array.length; i++) {
724 final Map<?, ?> m = (Map<?, ?>)array[i];
725 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
726 }
727 return statuses;
728 }
729
730 @Override
731 public Token<DelegationTokenIdentifier> getDelegationToken(
732 final String renewer) throws IOException {
733 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
734 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
735 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
736 SecurityUtil.setTokenService(token, nnAddr);
737 return token;
738 }
739
740 @Override
741 public Token<?> getRenewToken() {
742 return delegationToken;
743 }
744
745 @Override
746 public <T extends TokenIdentifier> void setDelegationToken(
747 final Token<T> token) {
748 synchronized(this) {
749 delegationToken = token;
750 }
751 }
752
753 private synchronized long renewDelegationToken(final Token<?> token
754 ) throws IOException {
755 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
756 TokenArgumentParam dtargParam = new TokenArgumentParam(
757 token.encodeToUrlString());
758 final Map<?, ?> m = run(op, null, dtargParam);
759 return (Long) m.get("long");
760 }
761
762 private synchronized void cancelDelegationToken(final Token<?> token
763 ) throws IOException {
764 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
765 TokenArgumentParam dtargParam = new TokenArgumentParam(
766 token.encodeToUrlString());
767 run(op, null, dtargParam);
768 }
769
770 @Override
771 public BlockLocation[] getFileBlockLocations(final FileStatus status,
772 final long offset, final long length) throws IOException {
773 if (status == null) {
774 return null;
775 }
776 return getFileBlockLocations(status.getPath(), offset, length);
777 }
778
779 @Override
780 public BlockLocation[] getFileBlockLocations(final Path p,
781 final long offset, final long length) throws IOException {
782 statistics.incrementReadOps(1);
783
784 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
785 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
786 new LengthParam(length));
787 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
788 }
789
790 @Override
791 public ContentSummary getContentSummary(final Path p) throws IOException {
792 statistics.incrementReadOps(1);
793
794 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
795 final Map<?, ?> m = run(op, p);
796 return JsonUtil.toContentSummary(m);
797 }
798
799 @Override
800 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
801 ) throws IOException {
802 statistics.incrementReadOps(1);
803
804 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
805 final Map<?, ?> m = run(op, p);
806 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
807 }
808
809 /** Delegation token renewer. */
810 public static class DtRenewer extends TokenRenewer {
811 @Override
812 public boolean handleKind(Text kind) {
813 return kind.equals(TOKEN_KIND);
814 }
815
816 @Override
817 public boolean isManaged(Token<?> token) throws IOException {
818 return true;
819 }
820
821 private static WebHdfsFileSystem getWebHdfs(
822 final Token<?> token, final Configuration conf) throws IOException {
823
824 final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
825 final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
826 return (WebHdfsFileSystem)FileSystem.get(uri, conf);
827 }
828
829 @Override
830 public long renew(final Token<?> token, final Configuration conf
831 ) throws IOException, InterruptedException {
832 return getWebHdfs(token, conf).renewDelegationToken(token);
833 }
834
835 @Override
836 public void cancel(final Token<?> token, final Configuration conf
837 ) throws IOException, InterruptedException {
838 getWebHdfs(token, conf).cancelDelegationToken(token);
839 }
840 }
841
842 private static class WebHdfsDelegationTokenSelector
843 extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
844 private static final DelegationTokenSelector hdfsTokenSelector =
845 new DelegationTokenSelector();
846
847 public WebHdfsDelegationTokenSelector() {
848 super(TOKEN_KIND);
849 }
850
851 Token<DelegationTokenIdentifier> selectToken(URI nnUri,
852 Collection<Token<?>> tokens, Configuration conf) {
853 Token<DelegationTokenIdentifier> token =
854 selectToken(SecurityUtil.buildTokenService(nnUri), tokens);
855 if (token == null) {
856 token = hdfsTokenSelector.selectToken(nnUri, tokens, conf);
857 }
858 return token;
859 }
860 }
861 }