001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URISyntaxException;
031 import java.net.URL;
032 import java.util.Collection;
033 import java.util.Map;
034 import java.util.StringTokenizer;
035
036 import javax.ws.rs.core.MediaType;
037
038 import org.apache.commons.logging.Log;
039 import org.apache.commons.logging.LogFactory;
040 import org.apache.hadoop.conf.Configuration;
041 import org.apache.hadoop.fs.BlockLocation;
042 import org.apache.hadoop.fs.ContentSummary;
043 import org.apache.hadoop.fs.DelegationTokenRenewer;
044 import org.apache.hadoop.fs.FSDataInputStream;
045 import org.apache.hadoop.fs.FSDataOutputStream;
046 import org.apache.hadoop.fs.FileAlreadyExistsException;
047 import org.apache.hadoop.fs.FileStatus;
048 import org.apache.hadoop.fs.FileSystem;
049 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
050 import org.apache.hadoop.fs.Options;
051 import org.apache.hadoop.fs.ParentNotDirectoryException;
052 import org.apache.hadoop.fs.Path;
053 import org.apache.hadoop.fs.permission.FsPermission;
054 import org.apache.hadoop.hdfs.ByteRangeInputStream;
055 import org.apache.hadoop.hdfs.DFSConfigKeys;
056 import org.apache.hadoop.hdfs.DFSUtil;
057 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
058 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
059 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
060 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
061 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
062 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
063 import org.apache.hadoop.hdfs.server.common.JspHelper;
064 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
065 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
066 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
067 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
068 import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
069 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
070 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
071 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
072 import org.apache.hadoop.hdfs.web.resources.GroupParam;
073 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
074 import org.apache.hadoop.hdfs.web.resources.LengthParam;
075 import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
076 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
077 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
078 import org.apache.hadoop.hdfs.web.resources.OwnerParam;
079 import org.apache.hadoop.hdfs.web.resources.Param;
080 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
081 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
082 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
083 import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
084 import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
085 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
086 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
087 import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
088 import org.apache.hadoop.hdfs.web.resources.UserParam;
089 import org.apache.hadoop.io.Text;
090 import org.apache.hadoop.io.retry.RetryPolicy;
091 import org.apache.hadoop.io.retry.RetryUtils;
092 import org.apache.hadoop.ipc.RemoteException;
093 import org.apache.hadoop.net.NetUtils;
094 import org.apache.hadoop.security.AccessControlException;
095 import org.apache.hadoop.security.SecurityUtil;
096 import org.apache.hadoop.security.UserGroupInformation;
097 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
098 import org.apache.hadoop.security.authentication.client.AuthenticationException;
099 import org.apache.hadoop.security.authorize.AuthorizationException;
100 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
101 import org.apache.hadoop.security.token.Token;
102 import org.apache.hadoop.security.token.TokenIdentifier;
103 import org.apache.hadoop.security.token.TokenRenewer;
104 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
105 import org.apache.hadoop.util.Progressable;
106 import org.mortbay.util.ajax.JSON;
107
108 /** A FileSystem for HDFS over the web. */
109 public class WebHdfsFileSystem extends FileSystem
110 implements DelegationTokenRenewer.Renewable {
111 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
112 /** File System URI: {SCHEME}://namenode:port/path/to/file */
113 public static final String SCHEME = "webhdfs";
114 /** WebHdfs version. */
115 public static final int VERSION = 1;
116 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
117 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
118
119 /** SPNEGO authenticator */
120 private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
121 /** Delegation token kind */
122 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
123 /** Token selector */
124 public static final WebHdfsDelegationTokenSelector DT_SELECTOR
125 = new WebHdfsDelegationTokenSelector();
126
127 private static DelegationTokenRenewer<WebHdfsFileSystem> DT_RENEWER = null;
128
129 private static synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
130 if (DT_RENEWER == null) {
131 DT_RENEWER = new DelegationTokenRenewer<WebHdfsFileSystem>(WebHdfsFileSystem.class);
132 DT_RENEWER.start();
133 }
134
135 DT_RENEWER.addRenewAction(webhdfs);
136 }
137
138 /** Is WebHDFS enabled in conf? */
139 public static boolean isEnabled(final Configuration conf, final Log log) {
140 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
141 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
142 log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
143 return b;
144 }
145
146 private final UserGroupInformation ugi;
147 private InetSocketAddress nnAddr;
148 private URI uri;
149 private Token<?> delegationToken;
150 private final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
151 private RetryPolicy retryPolicy = null;
152 private Path workingDir;
153
154 {
155 try {
156 ugi = UserGroupInformation.getCurrentUser();
157 } catch (IOException e) {
158 throw new RuntimeException(e);
159 }
160 }
161
162 /**
163 * Return the protocol scheme for the FileSystem.
164 * <p/>
165 *
166 * @return <code>webhdfs</code>
167 */
168 @Override
169 public String getScheme() {
170 return "webhdfs";
171 }
172
173 @Override
174 public synchronized void initialize(URI uri, Configuration conf
175 ) throws IOException {
176 super.initialize(uri, conf);
177 setConf(conf);
178 try {
179 this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
180 } catch (URISyntaxException e) {
181 throw new IllegalArgumentException(e);
182 }
183 this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
184 this.retryPolicy =
185 RetryUtils.getDefaultRetryPolicy(
186 conf,
187 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
188 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
189 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
190 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
191 SafeModeException.class);
192 this.workingDir = getHomeDirectory();
193
194 if (UserGroupInformation.isSecurityEnabled()) {
195 initDelegationToken();
196 }
197 }
198
199 protected void initDelegationToken() throws IOException {
200 // look for webhdfs token, then try hdfs
201 Token<?> token = selectDelegationToken(ugi);
202
203 //since we don't already have a token, go get one
204 boolean createdToken = false;
205 if (token == null) {
206 token = getDelegationToken(null);
207 createdToken = (token != null);
208 }
209
210 // security might be disabled
211 if (token != null) {
212 setDelegationToken(token);
213 if (createdToken) {
214 addRenewAction(this);
215 LOG.debug("Created new DT for " + token.getService());
216 } else {
217 LOG.debug("Found existing DT for " + token.getService());
218 }
219 }
220 }
221
222 protected Token<DelegationTokenIdentifier> selectDelegationToken(
223 UserGroupInformation ugi) {
224 return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
225 }
226
227 @Override
228 protected int getDefaultPort() {
229 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
230 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
231 }
232
233 @Override
234 public URI getUri() {
235 return this.uri;
236 }
237
238 /** @return the home directory. */
239 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
240 return "/user/" + ugi.getShortUserName();
241 }
242
243 @Override
244 public Path getHomeDirectory() {
245 return makeQualified(new Path(getHomeDirectoryString(ugi)));
246 }
247
248 @Override
249 public synchronized Path getWorkingDirectory() {
250 return workingDir;
251 }
252
253 @Override
254 public synchronized void setWorkingDirectory(final Path dir) {
255 String result = makeAbsolute(dir).toUri().getPath();
256 if (!DFSUtil.isValidName(result)) {
257 throw new IllegalArgumentException("Invalid DFS directory name " +
258 result);
259 }
260 workingDir = makeAbsolute(dir);
261 }
262
263 private Path makeAbsolute(Path f) {
264 return f.isAbsolute()? f: new Path(workingDir, f);
265 }
266
267 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
268 ) throws IOException {
269 if (c.getContentLength() == 0) {
270 return null;
271 }
272 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
273 if (in == null) {
274 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
275 }
276 final String contentType = c.getContentType();
277 if (contentType != null) {
278 final MediaType parsed = MediaType.valueOf(contentType);
279 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
280 throw new IOException("Content-Type \"" + contentType
281 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
282 + "\" (parsed=\"" + parsed + "\")");
283 }
284 }
285 return (Map<?, ?>)JSON.parse(new InputStreamReader(in));
286 }
287
288 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
289 final HttpURLConnection conn, boolean unwrapException) throws IOException {
290 final int code = conn.getResponseCode();
291 if (code != op.getExpectedHttpResponseCode()) {
292 final Map<?, ?> m;
293 try {
294 m = jsonParse(conn, true);
295 } catch(Exception e) {
296 throw new IOException("Unexpected HTTP response: code=" + code + " != "
297 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
298 + ", message=" + conn.getResponseMessage(), e);
299 }
300
301 if (m == null) {
302 throw new IOException("Unexpected HTTP response: code=" + code + " != "
303 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
304 + ", message=" + conn.getResponseMessage());
305 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
306 return m;
307 }
308
309 final RemoteException re = JsonUtil.toRemoteException(m);
310 throw unwrapException? toIOException(re): re;
311 }
312 return null;
313 }
314
315 /**
316 * Covert an exception to an IOException.
317 *
318 * For a non-IOException, wrap it with IOException.
319 * For a RemoteException, unwrap it.
320 * For an IOException which is not a RemoteException, return it.
321 */
322 private static IOException toIOException(Exception e) {
323 if (!(e instanceof IOException)) {
324 return new IOException(e);
325 }
326
327 final IOException ioe = (IOException)e;
328 if (!(ioe instanceof RemoteException)) {
329 return ioe;
330 }
331
332 final RemoteException re = (RemoteException)ioe;
333 return re.unwrapRemoteException(AccessControlException.class,
334 InvalidToken.class,
335 AuthenticationException.class,
336 AuthorizationException.class,
337 FileAlreadyExistsException.class,
338 FileNotFoundException.class,
339 ParentNotDirectoryException.class,
340 UnresolvedPathException.class,
341 SafeModeException.class,
342 DSQuotaExceededException.class,
343 NSQuotaExceededException.class);
344 }
345
346 /**
347 * Return a URL pointing to given path on the namenode.
348 *
349 * @param path to obtain the URL for
350 * @param query string to append to the path
351 * @return namenode URL referring to the given path
352 * @throws IOException on error constructing the URL
353 */
354 private URL getNamenodeURL(String path, String query) throws IOException {
355 final URL url = new URL("http", nnAddr.getHostName(),
356 nnAddr.getPort(), path + '?' + query);
357 if (LOG.isTraceEnabled()) {
358 LOG.trace("url=" + url);
359 }
360 return url;
361 }
362
363 private String addDt2Query(String query) throws IOException {
364 if (UserGroupInformation.isSecurityEnabled()) {
365 synchronized (this) {
366 if (delegationToken != null) {
367 final String encoded = delegationToken.encodeToUrlString();
368 return query + JspHelper.getDelegationTokenUrlParam(encoded);
369 } // else we are talking to an insecure cluster
370 }
371 }
372 return query;
373 }
374
375 URL toUrl(final HttpOpParam.Op op, final Path fspath,
376 final Param<?,?>... parameters) throws IOException {
377 //initialize URI path and query
378 final String path = PATH_PREFIX
379 + (fspath == null? "/": makeQualified(fspath).toUri().getPath());
380 final String query = op.toQueryString()
381 + '&' + new UserParam(ugi)
382 + Param.toSortedString("&", parameters);
383 final URL url;
384 if (op == PutOpParam.Op.RENEWDELEGATIONTOKEN
385 || op == GetOpParam.Op.GETDELEGATIONTOKEN) {
386 // Skip adding delegation token for getting or renewing delegation token,
387 // because these operations require kerberos authentication.
388 url = getNamenodeURL(path, query);
389 } else {
390 url = getNamenodeURL(path, addDt2Query(query));
391 }
392 if (LOG.isTraceEnabled()) {
393 LOG.trace("url=" + url);
394 }
395 return url;
396 }
397
398 private HttpURLConnection getHttpUrlConnection(URL url)
399 throws IOException, AuthenticationException {
400 final HttpURLConnection conn;
401 if (ugi.hasKerberosCredentials()) {
402 conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
403 } else {
404 conn = (HttpURLConnection)url.openConnection();
405 }
406 return conn;
407 }
408
409 /**
410 * Run a http operation.
411 * Connect to the http server, validate response, and obtain the JSON output.
412 *
413 * @param op http operation
414 * @param fspath file system path
415 * @param parameters parameters for the operation
416 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
417 * @throws IOException
418 */
419 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
420 final Param<?,?>... parameters) throws IOException {
421 return new Runner(op, fspath, parameters).run().json;
422 }
423
424 /**
425 * This class is for initialing a HTTP connection, connecting to server,
426 * obtaining a response, and also handling retry on failures.
427 */
428 class Runner {
429 private final HttpOpParam.Op op;
430 private final URL url;
431 private final boolean redirected;
432
433 private boolean checkRetry;
434 private HttpURLConnection conn = null;
435 private Map<?, ?> json = null;
436
437 Runner(final HttpOpParam.Op op, final URL url, final boolean redirected) {
438 this.op = op;
439 this.url = url;
440 this.redirected = redirected;
441 }
442
443 Runner(final HttpOpParam.Op op, final Path fspath,
444 final Param<?,?>... parameters) throws IOException {
445 this(op, toUrl(op, fspath, parameters), false);
446 }
447
448 Runner(final HttpOpParam.Op op, final HttpURLConnection conn) {
449 this(op, null, false);
450 this.conn = conn;
451 }
452
453 private void init() throws IOException {
454 checkRetry = !redirected;
455 try {
456 conn = getHttpUrlConnection(url);
457 } catch(AuthenticationException ae) {
458 checkRetry = false;
459 throw new IOException("Authentication failed, url=" + url, ae);
460 }
461 }
462
463 private void connect() throws IOException {
464 connect(op.getDoOutput());
465 }
466
467 private void connect(boolean doOutput) throws IOException {
468 conn.setRequestMethod(op.getType().toString());
469 conn.setDoOutput(doOutput);
470 conn.setInstanceFollowRedirects(false);
471 conn.connect();
472 }
473
474 private void disconnect() {
475 if (conn != null) {
476 conn.disconnect();
477 conn = null;
478 }
479 }
480
481 Runner run() throws IOException {
482 for(int retry = 0; ; retry++) {
483 try {
484 init();
485 if (op.getDoOutput()) {
486 twoStepWrite();
487 } else {
488 getResponse(op != GetOpParam.Op.OPEN);
489 }
490 return this;
491 } catch(IOException ioe) {
492 shouldRetry(ioe, retry);
493 }
494 }
495 }
496
497 private void shouldRetry(final IOException ioe, final int retry
498 ) throws IOException {
499 if (checkRetry) {
500 try {
501 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
502 ioe, retry, 0, true);
503 if (a.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
504 LOG.info("Retrying connect to namenode: " + nnAddr
505 + ". Already tried " + retry + " time(s); retry policy is "
506 + retryPolicy + ", delay " + a.delayMillis + "ms.");
507 Thread.sleep(a.delayMillis);
508 return;
509 }
510 } catch(Exception e) {
511 LOG.warn("Original exception is ", ioe);
512 throw toIOException(e);
513 }
514 }
515 throw toIOException(ioe);
516 }
517
518 /**
519 * Two-step Create/Append:
520 * Step 1) Submit a Http request with neither auto-redirect nor data.
521 * Step 2) Submit another Http request with the URL from the Location header with data.
522 *
523 * The reason of having two-step create/append is for preventing clients to
524 * send out the data before the redirect. This issue is addressed by the
525 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
526 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
527 * and Java 6 http client), which do not correctly implement "Expect:
528 * 100-continue". The two-step create/append is a temporary workaround for
529 * the software library bugs.
530 */
531 HttpURLConnection twoStepWrite() throws IOException {
532 //Step 1) Submit a Http request with neither auto-redirect nor data.
533 connect(false);
534 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false);
535 final String redirect = conn.getHeaderField("Location");
536 disconnect();
537 checkRetry = false;
538
539 //Step 2) Submit another Http request with the URL from the Location header with data.
540 conn = (HttpURLConnection)new URL(redirect).openConnection();
541 conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
542 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
543 connect();
544 return conn;
545 }
546
547 FSDataOutputStream write(final int bufferSize) throws IOException {
548 return WebHdfsFileSystem.this.write(op, conn, bufferSize);
549 }
550
551 void getResponse(boolean getJsonAndDisconnect) throws IOException {
552 try {
553 connect();
554 final int code = conn.getResponseCode();
555 if (!redirected && op.getRedirect()
556 && code != op.getExpectedHttpResponseCode()) {
557 final String redirect = conn.getHeaderField("Location");
558 json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op),
559 conn, false);
560 disconnect();
561
562 checkRetry = false;
563 conn = (HttpURLConnection)new URL(redirect).openConnection();
564 connect();
565 }
566
567 json = validateResponse(op, conn, false);
568 if (json == null && getJsonAndDisconnect) {
569 json = jsonParse(conn, false);
570 }
571 } finally {
572 if (getJsonAndDisconnect) {
573 disconnect();
574 }
575 }
576 }
577 }
578
579 private FsPermission applyUMask(FsPermission permission) {
580 if (permission == null) {
581 permission = FsPermission.getDefault();
582 }
583 return permission.applyUMask(FsPermission.getUMask(getConf()));
584 }
585
586 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
587 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
588 final Map<?, ?> json = run(op, f);
589 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
590 if (status == null) {
591 throw new FileNotFoundException("File does not exist: " + f);
592 }
593 return status;
594 }
595
596 @Override
597 public FileStatus getFileStatus(Path f) throws IOException {
598 statistics.incrementReadOps(1);
599 return makeQualified(getHdfsFileStatus(f), f);
600 }
601
602 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
603 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
604 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
605 f.getPermission(), f.getOwner(), f.getGroup(),
606 f.isSymlink() ? new Path(f.getSymlink()) : null,
607 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
608 }
609
610 @Override
611 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
612 statistics.incrementWriteOps(1);
613 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
614 final Map<?, ?> json = run(op, f,
615 new PermissionParam(applyUMask(permission)));
616 return (Boolean)json.get("boolean");
617 }
618
619 /**
620 * Create a symlink pointing to the destination path.
621 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
622 */
623 public void createSymlink(Path destination, Path f, boolean createParent
624 ) throws IOException {
625 statistics.incrementWriteOps(1);
626 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
627 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
628 new CreateParentParam(createParent));
629 }
630
631 @Override
632 public boolean rename(final Path src, final Path dst) throws IOException {
633 statistics.incrementWriteOps(1);
634 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
635 final Map<?, ?> json = run(op, src,
636 new DestinationParam(makeQualified(dst).toUri().getPath()));
637 return (Boolean)json.get("boolean");
638 }
639
640 @SuppressWarnings("deprecation")
641 @Override
642 public void rename(final Path src, final Path dst,
643 final Options.Rename... options) throws IOException {
644 statistics.incrementWriteOps(1);
645 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
646 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
647 new RenameOptionSetParam(options));
648 }
649
650 @Override
651 public void setOwner(final Path p, final String owner, final String group
652 ) throws IOException {
653 if (owner == null && group == null) {
654 throw new IOException("owner == null && group == null");
655 }
656
657 statistics.incrementWriteOps(1);
658 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
659 run(op, p, new OwnerParam(owner), new GroupParam(group));
660 }
661
662 @Override
663 public void setPermission(final Path p, final FsPermission permission
664 ) throws IOException {
665 statistics.incrementWriteOps(1);
666 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
667 run(op, p, new PermissionParam(permission));
668 }
669
670 @Override
671 public boolean setReplication(final Path p, final short replication
672 ) throws IOException {
673 statistics.incrementWriteOps(1);
674 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
675 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
676 return (Boolean)json.get("boolean");
677 }
678
679 @Override
680 public void setTimes(final Path p, final long mtime, final long atime
681 ) throws IOException {
682 statistics.incrementWriteOps(1);
683 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
684 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
685 }
686
687 @Override
688 public long getDefaultBlockSize() {
689 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
690 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
691 }
692
693 @Override
694 public short getDefaultReplication() {
695 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
696 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
697 }
698
699 FSDataOutputStream write(final HttpOpParam.Op op,
700 final HttpURLConnection conn, final int bufferSize) throws IOException {
701 return new FSDataOutputStream(new BufferedOutputStream(
702 conn.getOutputStream(), bufferSize), statistics) {
703 @Override
704 public void close() throws IOException {
705 try {
706 super.close();
707 } finally {
708 try {
709 validateResponse(op, conn, true);
710 } finally {
711 conn.disconnect();
712 }
713 }
714 }
715 };
716 }
717
718 @Override
719 public FSDataOutputStream create(final Path f, final FsPermission permission,
720 final boolean overwrite, final int bufferSize, final short replication,
721 final long blockSize, final Progressable progress) throws IOException {
722 statistics.incrementWriteOps(1);
723
724 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
725 return new Runner(op, f,
726 new PermissionParam(applyUMask(permission)),
727 new OverwriteParam(overwrite),
728 new BufferSizeParam(bufferSize),
729 new ReplicationParam(replication),
730 new BlockSizeParam(blockSize))
731 .run()
732 .write(bufferSize);
733 }
734
735 @Override
736 public FSDataOutputStream append(final Path f, final int bufferSize,
737 final Progressable progress) throws IOException {
738 statistics.incrementWriteOps(1);
739
740 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
741 return new Runner(op, f, new BufferSizeParam(bufferSize))
742 .run()
743 .write(bufferSize);
744 }
745
746 @SuppressWarnings("deprecation")
747 @Override
748 public boolean delete(final Path f) throws IOException {
749 return delete(f, true);
750 }
751
752 @Override
753 public boolean delete(Path f, boolean recursive) throws IOException {
754 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
755 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
756 return (Boolean)json.get("boolean");
757 }
758
759 @Override
760 public FSDataInputStream open(final Path f, final int buffersize
761 ) throws IOException {
762 statistics.incrementReadOps(1);
763 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
764 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
765 return new FSDataInputStream(new OffsetUrlInputStream(
766 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
767 }
768
769 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
770 OffsetUrlOpener(final URL url) {
771 super(url);
772 }
773
774 /** Setup offset url and connect. */
775 @Override
776 protected HttpURLConnection connect(final long offset,
777 final boolean resolved) throws IOException {
778 final URL offsetUrl = offset == 0L? url
779 : new URL(url + "&" + new OffsetParam(offset));
780 return new Runner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn;
781 }
782 }
783
784 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
785
786 /** Remove offset parameter, if there is any, from the url */
787 static URL removeOffsetParam(final URL url) throws MalformedURLException {
788 String query = url.getQuery();
789 if (query == null) {
790 return url;
791 }
792 final String lower = query.toLowerCase();
793 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
794 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
795 return url;
796 }
797
798 //rebuild query
799 StringBuilder b = null;
800 for(final StringTokenizer st = new StringTokenizer(query, "&");
801 st.hasMoreTokens();) {
802 final String token = st.nextToken();
803 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
804 if (b == null) {
805 b = new StringBuilder("?").append(token);
806 } else {
807 b.append('&').append(token);
808 }
809 }
810 }
811 query = b == null? "": b.toString();
812
813 final String urlStr = url.toString();
814 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
815 }
816
817 static class OffsetUrlInputStream extends ByteRangeInputStream {
818 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
819 super(o, r);
820 }
821
822 /** Remove offset parameter before returning the resolved url. */
823 @Override
824 protected URL getResolvedUrl(final HttpURLConnection connection
825 ) throws MalformedURLException {
826 return removeOffsetParam(connection.getURL());
827 }
828 }
829
830 @Override
831 public FileStatus[] listStatus(final Path f) throws IOException {
832 statistics.incrementReadOps(1);
833
834 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
835 final Map<?, ?> json = run(op, f);
836 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
837 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
838
839 //convert FileStatus
840 final FileStatus[] statuses = new FileStatus[array.length];
841 for(int i = 0; i < array.length; i++) {
842 final Map<?, ?> m = (Map<?, ?>)array[i];
843 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
844 }
845 return statuses;
846 }
847
848 @Override
849 public Token<DelegationTokenIdentifier> getDelegationToken(
850 final String renewer) throws IOException {
851 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
852 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
853 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
854 SecurityUtil.setTokenService(token, nnAddr);
855 return token;
856 }
857
858 @Override
859 public Token<?> getRenewToken() {
860 return delegationToken;
861 }
862
863 @Override
864 public <T extends TokenIdentifier> void setDelegationToken(
865 final Token<T> token) {
866 synchronized(this) {
867 delegationToken = token;
868 }
869 }
870
871 private synchronized long renewDelegationToken(final Token<?> token
872 ) throws IOException {
873 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
874 TokenArgumentParam dtargParam = new TokenArgumentParam(
875 token.encodeToUrlString());
876 final Map<?, ?> m = run(op, null, dtargParam);
877 return (Long) m.get("long");
878 }
879
880 private synchronized void cancelDelegationToken(final Token<?> token
881 ) throws IOException {
882 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
883 TokenArgumentParam dtargParam = new TokenArgumentParam(
884 token.encodeToUrlString());
885 run(op, null, dtargParam);
886 }
887
888 @Override
889 public BlockLocation[] getFileBlockLocations(final FileStatus status,
890 final long offset, final long length) throws IOException {
891 if (status == null) {
892 return null;
893 }
894 return getFileBlockLocations(status.getPath(), offset, length);
895 }
896
897 @Override
898 public BlockLocation[] getFileBlockLocations(final Path p,
899 final long offset, final long length) throws IOException {
900 statistics.incrementReadOps(1);
901
902 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
903 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
904 new LengthParam(length));
905 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
906 }
907
908 @Override
909 public ContentSummary getContentSummary(final Path p) throws IOException {
910 statistics.incrementReadOps(1);
911
912 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
913 final Map<?, ?> m = run(op, p);
914 return JsonUtil.toContentSummary(m);
915 }
916
917 @Override
918 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
919 ) throws IOException {
920 statistics.incrementReadOps(1);
921
922 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
923 final Map<?, ?> m = run(op, p);
924 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
925 }
926
927 /** Delegation token renewer. */
928 public static class DtRenewer extends TokenRenewer {
929 @Override
930 public boolean handleKind(Text kind) {
931 return kind.equals(TOKEN_KIND);
932 }
933
934 @Override
935 public boolean isManaged(Token<?> token) throws IOException {
936 return true;
937 }
938
939 private static WebHdfsFileSystem getWebHdfs(
940 final Token<?> token, final Configuration conf) throws IOException {
941
942 final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
943 final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
944 return (WebHdfsFileSystem)FileSystem.get(uri, conf);
945 }
946
947 @Override
948 public long renew(final Token<?> token, final Configuration conf
949 ) throws IOException, InterruptedException {
950 final UserGroupInformation ugi = UserGroupInformation.getLoginUser();
951 // update the kerberos credentials, if they are coming from a keytab
952 ugi.reloginFromKeytab();
953
954 return getWebHdfs(token, conf).renewDelegationToken(token);
955 }
956
957 @Override
958 public void cancel(final Token<?> token, final Configuration conf
959 ) throws IOException, InterruptedException {
960 final UserGroupInformation ugi = UserGroupInformation.getLoginUser();
961 // update the kerberos credentials, if they are coming from a keytab
962 ugi.checkTGTAndReloginFromKeytab();
963
964 getWebHdfs(token, conf).cancelDelegationToken(token);
965 }
966 }
967
968 private static class WebHdfsDelegationTokenSelector
969 extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
970 private static final DelegationTokenSelector hdfsTokenSelector =
971 new DelegationTokenSelector();
972
973 public WebHdfsDelegationTokenSelector() {
974 super(TOKEN_KIND);
975 }
976
977 Token<DelegationTokenIdentifier> selectToken(URI nnUri,
978 Collection<Token<?>> tokens, Configuration conf) {
979 Token<DelegationTokenIdentifier> token =
980 selectToken(SecurityUtil.buildTokenService(nnUri), tokens);
981 if (token == null) {
982 token = hdfsTokenSelector.selectToken(nnUri, tokens, conf);
983 }
984 return token;
985 }
986 }
987 }