001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URISyntaxException;
031 import java.net.URL;
032 import java.util.ArrayList;
033 import java.util.Collection;
034 import java.util.List;
035 import java.util.Map;
036 import java.util.StringTokenizer;
037
038 import javax.ws.rs.core.MediaType;
039
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042 import org.apache.hadoop.conf.Configuration;
043 import org.apache.hadoop.fs.BlockLocation;
044 import org.apache.hadoop.fs.ContentSummary;
045 import org.apache.hadoop.fs.DelegationTokenRenewer;
046 import org.apache.hadoop.fs.FSDataInputStream;
047 import org.apache.hadoop.fs.FSDataOutputStream;
048 import org.apache.hadoop.fs.FileAlreadyExistsException;
049 import org.apache.hadoop.fs.FileStatus;
050 import org.apache.hadoop.fs.FileSystem;
051 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
052 import org.apache.hadoop.fs.Options;
053 import org.apache.hadoop.fs.ParentNotDirectoryException;
054 import org.apache.hadoop.fs.Path;
055 import org.apache.hadoop.fs.permission.FsPermission;
056 import org.apache.hadoop.hdfs.ByteRangeInputStream;
057 import org.apache.hadoop.hdfs.DFSConfigKeys;
058 import org.apache.hadoop.hdfs.DFSUtil;
059 import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
060 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
061 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
062 import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
063 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
064 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
065 import org.apache.hadoop.hdfs.server.common.JspHelper;
066 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
067 import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
068 import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
069 import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
070 import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
071 import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
072 import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
073 import org.apache.hadoop.hdfs.web.resources.DestinationParam;
074 import org.apache.hadoop.hdfs.web.resources.GetOpParam;
075 import org.apache.hadoop.hdfs.web.resources.GroupParam;
076 import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
077 import org.apache.hadoop.hdfs.web.resources.LengthParam;
078 import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
079 import org.apache.hadoop.hdfs.web.resources.OffsetParam;
080 import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
081 import org.apache.hadoop.hdfs.web.resources.OwnerParam;
082 import org.apache.hadoop.hdfs.web.resources.Param;
083 import org.apache.hadoop.hdfs.web.resources.PermissionParam;
084 import org.apache.hadoop.hdfs.web.resources.PostOpParam;
085 import org.apache.hadoop.hdfs.web.resources.PutOpParam;
086 import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
087 import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
088 import org.apache.hadoop.hdfs.web.resources.RenewerParam;
089 import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
090 import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
091 import org.apache.hadoop.hdfs.web.resources.UserParam;
092 import org.apache.hadoop.io.Text;
093 import org.apache.hadoop.io.retry.RetryPolicy;
094 import org.apache.hadoop.io.retry.RetryUtils;
095 import org.apache.hadoop.ipc.RemoteException;
096 import org.apache.hadoop.net.NetUtils;
097 import org.apache.hadoop.security.AccessControlException;
098 import org.apache.hadoop.security.SecurityUtil;
099 import org.apache.hadoop.security.UserGroupInformation;
100 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
101 import org.apache.hadoop.security.authentication.client.AuthenticationException;
102 import org.apache.hadoop.security.authorize.AuthorizationException;
103 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
104 import org.apache.hadoop.security.token.Token;
105 import org.apache.hadoop.security.token.TokenIdentifier;
106 import org.apache.hadoop.security.token.TokenRenewer;
107 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
108 import org.apache.hadoop.util.Progressable;
109 import org.apache.hadoop.util.StringUtils;
110 import org.mortbay.util.ajax.JSON;
111
112 import com.google.common.base.Charsets;
113
114 /** A FileSystem for HDFS over the web. */
115 public class WebHdfsFileSystem extends FileSystem
116 implements DelegationTokenRenewer.Renewable {
117 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
118 /** File System URI: {SCHEME}://namenode:port/path/to/file */
119 public static final String SCHEME = "webhdfs";
120 /** WebHdfs version. */
121 public static final int VERSION = 1;
122 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
123 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
124
125 /** SPNEGO authenticator */
126 private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
127 /** Delegation token kind */
128 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
129 /** Token selector */
130 public static final WebHdfsDelegationTokenSelector DT_SELECTOR
131 = new WebHdfsDelegationTokenSelector();
132
133 private DelegationTokenRenewer dtRenewer = null;
134
135 private synchronized void addRenewAction(final WebHdfsFileSystem webhdfs) {
136 if (dtRenewer == null) {
137 dtRenewer = DelegationTokenRenewer.getInstance();
138 }
139
140 dtRenewer.addRenewAction(webhdfs);
141 }
142
143 /** Is WebHDFS enabled in conf? */
144 public static boolean isEnabled(final Configuration conf, final Log log) {
145 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
146 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
147 log.info(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY + " = " + b);
148 return b;
149 }
150
151 private final UserGroupInformation ugi;
152 private InetSocketAddress nnAddr;
153 private URI uri;
154 private Token<?> delegationToken;
155 private final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
156 private RetryPolicy retryPolicy = null;
157 private Path workingDir;
158
159 {
160 try {
161 ugi = UserGroupInformation.getCurrentUser();
162 } catch (IOException e) {
163 throw new RuntimeException(e);
164 }
165 }
166
167 /**
168 * Return the protocol scheme for the FileSystem.
169 * <p/>
170 *
171 * @return <code>webhdfs</code>
172 */
173 @Override
174 public String getScheme() {
175 return "webhdfs";
176 }
177
178 @Override
179 public synchronized void initialize(URI uri, Configuration conf
180 ) throws IOException {
181 super.initialize(uri, conf);
182 setConf(conf);
183 try {
184 this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null);
185 } catch (URISyntaxException e) {
186 throw new IllegalArgumentException(e);
187 }
188 this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
189 this.retryPolicy =
190 RetryUtils.getDefaultRetryPolicy(
191 conf,
192 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
193 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
194 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
195 DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
196 SafeModeException.class);
197 this.workingDir = getHomeDirectory();
198
199 if (UserGroupInformation.isSecurityEnabled()) {
200 initDelegationToken();
201 }
202 }
203
204 protected void initDelegationToken() throws IOException {
205 // look for webhdfs token, then try hdfs
206 Token<?> token = selectDelegationToken(ugi);
207
208 //since we don't already have a token, go get one
209 boolean createdToken = false;
210 if (token == null) {
211 token = getDelegationToken(null);
212 createdToken = (token != null);
213 }
214
215 // security might be disabled
216 if (token != null) {
217 setDelegationToken(token);
218 if (createdToken) {
219 addRenewAction(this);
220 LOG.debug("Created new DT for " + token.getService());
221 } else {
222 LOG.debug("Found existing DT for " + token.getService());
223 }
224 }
225 }
226
227 protected Token<DelegationTokenIdentifier> selectDelegationToken(
228 UserGroupInformation ugi) {
229 return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
230 }
231
232 @Override
233 protected int getDefaultPort() {
234 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
235 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
236 }
237
238 @Override
239 public URI getUri() {
240 return this.uri;
241 }
242
243 /** @return the home directory. */
244 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
245 return "/user/" + ugi.getShortUserName();
246 }
247
248 @Override
249 public Path getHomeDirectory() {
250 return makeQualified(new Path(getHomeDirectoryString(ugi)));
251 }
252
253 @Override
254 public synchronized Path getWorkingDirectory() {
255 return workingDir;
256 }
257
258 @Override
259 public synchronized void setWorkingDirectory(final Path dir) {
260 String result = makeAbsolute(dir).toUri().getPath();
261 if (!DFSUtil.isValidName(result)) {
262 throw new IllegalArgumentException("Invalid DFS directory name " +
263 result);
264 }
265 workingDir = makeAbsolute(dir);
266 }
267
268 private Path makeAbsolute(Path f) {
269 return f.isAbsolute()? f: new Path(workingDir, f);
270 }
271
272 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
273 ) throws IOException {
274 if (c.getContentLength() == 0) {
275 return null;
276 }
277 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
278 if (in == null) {
279 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
280 }
281 final String contentType = c.getContentType();
282 if (contentType != null) {
283 final MediaType parsed = MediaType.valueOf(contentType);
284 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
285 throw new IOException("Content-Type \"" + contentType
286 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
287 + "\" (parsed=\"" + parsed + "\")");
288 }
289 }
290 return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
291 }
292
293 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
294 final HttpURLConnection conn, boolean unwrapException) throws IOException {
295 final int code = conn.getResponseCode();
296 if (code != op.getExpectedHttpResponseCode()) {
297 final Map<?, ?> m;
298 try {
299 m = jsonParse(conn, true);
300 } catch(Exception e) {
301 throw new IOException("Unexpected HTTP response: code=" + code + " != "
302 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
303 + ", message=" + conn.getResponseMessage(), e);
304 }
305
306 if (m == null) {
307 throw new IOException("Unexpected HTTP response: code=" + code + " != "
308 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
309 + ", message=" + conn.getResponseMessage());
310 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
311 return m;
312 }
313
314 final RemoteException re = JsonUtil.toRemoteException(m);
315 throw unwrapException? toIOException(re): re;
316 }
317 return null;
318 }
319
320 /**
321 * Covert an exception to an IOException.
322 *
323 * For a non-IOException, wrap it with IOException.
324 * For a RemoteException, unwrap it.
325 * For an IOException which is not a RemoteException, return it.
326 */
327 private static IOException toIOException(Exception e) {
328 if (!(e instanceof IOException)) {
329 return new IOException(e);
330 }
331
332 final IOException ioe = (IOException)e;
333 if (!(ioe instanceof RemoteException)) {
334 return ioe;
335 }
336
337 final RemoteException re = (RemoteException)ioe;
338 return re.unwrapRemoteException(AccessControlException.class,
339 InvalidToken.class,
340 AuthenticationException.class,
341 AuthorizationException.class,
342 FileAlreadyExistsException.class,
343 FileNotFoundException.class,
344 ParentNotDirectoryException.class,
345 UnresolvedPathException.class,
346 SafeModeException.class,
347 DSQuotaExceededException.class,
348 NSQuotaExceededException.class);
349 }
350
351 /**
352 * Return a URL pointing to given path on the namenode.
353 *
354 * @param path to obtain the URL for
355 * @param query string to append to the path
356 * @return namenode URL referring to the given path
357 * @throws IOException on error constructing the URL
358 */
359 private URL getNamenodeURL(String path, String query) throws IOException {
360 final URL url = new URL("http", nnAddr.getHostName(),
361 nnAddr.getPort(), path + '?' + query);
362 if (LOG.isTraceEnabled()) {
363 LOG.trace("url=" + url);
364 }
365 return url;
366 }
367
368 private String addDt2Query(String query) throws IOException {
369 if (UserGroupInformation.isSecurityEnabled()) {
370 synchronized (this) {
371 if (delegationToken != null) {
372 final String encoded = delegationToken.encodeToUrlString();
373 return query + JspHelper.getDelegationTokenUrlParam(encoded);
374 } // else we are talking to an insecure cluster
375 }
376 }
377 return query;
378 }
379
380 URL toUrl(final HttpOpParam.Op op, final Path fspath,
381 final Param<?,?>... parameters) throws IOException {
382 //initialize URI path and query
383 final String path = PATH_PREFIX
384 + (fspath == null? "/": makeQualified(fspath).toUri().getPath());
385 final String query = op.toQueryString()
386 + '&' + new UserParam(ugi)
387 + Param.toSortedString("&", parameters);
388 final URL url;
389 if (op == PutOpParam.Op.RENEWDELEGATIONTOKEN
390 || op == GetOpParam.Op.GETDELEGATIONTOKEN) {
391 // Skip adding delegation token for getting or renewing delegation token,
392 // because these operations require kerberos authentication.
393 url = getNamenodeURL(path, query);
394 } else {
395 url = getNamenodeURL(path, addDt2Query(query));
396 }
397 if (LOG.isTraceEnabled()) {
398 LOG.trace("url=" + url);
399 }
400 return url;
401 }
402
403 private HttpURLConnection getHttpUrlConnection(URL url)
404 throws IOException, AuthenticationException {
405 final HttpURLConnection conn;
406 if (ugi.hasKerberosCredentials()) {
407 conn = new AuthenticatedURL(AUTH).openConnection(url, authToken);
408 } else {
409 conn = (HttpURLConnection)url.openConnection();
410 }
411 return conn;
412 }
413
414 /**
415 * Run a http operation.
416 * Connect to the http server, validate response, and obtain the JSON output.
417 *
418 * @param op http operation
419 * @param fspath file system path
420 * @param parameters parameters for the operation
421 * @return a JSON object, e.g. Object[], Map<?, ?>, etc.
422 * @throws IOException
423 */
424 private Map<?, ?> run(final HttpOpParam.Op op, final Path fspath,
425 final Param<?,?>... parameters) throws IOException {
426 return new Runner(op, fspath, parameters).run().json;
427 }
428
429 /**
430 * This class is for initialing a HTTP connection, connecting to server,
431 * obtaining a response, and also handling retry on failures.
432 */
433 class Runner {
434 private final HttpOpParam.Op op;
435 private final URL url;
436 private final boolean redirected;
437
438 private boolean checkRetry;
439 private HttpURLConnection conn = null;
440 private Map<?, ?> json = null;
441
442 Runner(final HttpOpParam.Op op, final URL url, final boolean redirected) {
443 this.op = op;
444 this.url = url;
445 this.redirected = redirected;
446 }
447
448 Runner(final HttpOpParam.Op op, final Path fspath,
449 final Param<?,?>... parameters) throws IOException {
450 this(op, toUrl(op, fspath, parameters), false);
451 }
452
453 Runner(final HttpOpParam.Op op, final HttpURLConnection conn) {
454 this(op, null, false);
455 this.conn = conn;
456 }
457
458 private void init() throws IOException {
459 checkRetry = !redirected;
460 try {
461 conn = getHttpUrlConnection(url);
462 } catch(AuthenticationException ae) {
463 checkRetry = false;
464 throw new IOException("Authentication failed, url=" + url, ae);
465 }
466 }
467
468 private void connect() throws IOException {
469 connect(op.getDoOutput());
470 }
471
472 private void connect(boolean doOutput) throws IOException {
473 conn.setRequestMethod(op.getType().toString());
474 conn.setDoOutput(doOutput);
475 conn.setInstanceFollowRedirects(false);
476 conn.connect();
477 }
478
479 private void disconnect() {
480 if (conn != null) {
481 conn.disconnect();
482 conn = null;
483 }
484 }
485
486 Runner run() throws IOException {
487 for(int retry = 0; ; retry++) {
488 try {
489 init();
490 if (op.getDoOutput()) {
491 twoStepWrite();
492 } else {
493 getResponse(op != GetOpParam.Op.OPEN);
494 }
495 return this;
496 } catch(IOException ioe) {
497 shouldRetry(ioe, retry);
498 }
499 }
500 }
501
502 private void shouldRetry(final IOException ioe, final int retry
503 ) throws IOException {
504 if (checkRetry) {
505 try {
506 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
507 ioe, retry, 0, true);
508 if (a.action == RetryPolicy.RetryAction.RetryDecision.RETRY) {
509 LOG.info("Retrying connect to namenode: " + nnAddr
510 + ". Already tried " + retry + " time(s); retry policy is "
511 + retryPolicy + ", delay " + a.delayMillis + "ms.");
512 Thread.sleep(a.delayMillis);
513 return;
514 }
515 } catch(Exception e) {
516 LOG.warn("Original exception is ", ioe);
517 throw toIOException(e);
518 }
519 }
520 throw toIOException(ioe);
521 }
522
523 /**
524 * Two-step Create/Append:
525 * Step 1) Submit a Http request with neither auto-redirect nor data.
526 * Step 2) Submit another Http request with the URL from the Location header with data.
527 *
528 * The reason of having two-step create/append is for preventing clients to
529 * send out the data before the redirect. This issue is addressed by the
530 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
531 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
532 * and Java 6 http client), which do not correctly implement "Expect:
533 * 100-continue". The two-step create/append is a temporary workaround for
534 * the software library bugs.
535 */
536 HttpURLConnection twoStepWrite() throws IOException {
537 //Step 1) Submit a Http request with neither auto-redirect nor data.
538 connect(false);
539 validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op), conn, false);
540 final String redirect = conn.getHeaderField("Location");
541 disconnect();
542 checkRetry = false;
543
544 //Step 2) Submit another Http request with the URL from the Location header with data.
545 conn = (HttpURLConnection)new URL(redirect).openConnection();
546 conn.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
547 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
548 connect();
549 return conn;
550 }
551
552 FSDataOutputStream write(final int bufferSize) throws IOException {
553 return WebHdfsFileSystem.this.write(op, conn, bufferSize);
554 }
555
556 void getResponse(boolean getJsonAndDisconnect) throws IOException {
557 try {
558 connect();
559 final int code = conn.getResponseCode();
560 if (!redirected && op.getRedirect()
561 && code != op.getExpectedHttpResponseCode()) {
562 final String redirect = conn.getHeaderField("Location");
563 json = validateResponse(HttpOpParam.TemporaryRedirectOp.valueOf(op),
564 conn, false);
565 disconnect();
566
567 checkRetry = false;
568 conn = (HttpURLConnection)new URL(redirect).openConnection();
569 connect();
570 }
571
572 json = validateResponse(op, conn, false);
573 if (json == null && getJsonAndDisconnect) {
574 json = jsonParse(conn, false);
575 }
576 } finally {
577 if (getJsonAndDisconnect) {
578 disconnect();
579 }
580 }
581 }
582 }
583
584 private FsPermission applyUMask(FsPermission permission) {
585 if (permission == null) {
586 permission = FsPermission.getDefault();
587 }
588 return permission.applyUMask(FsPermission.getUMask(getConf()));
589 }
590
591 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
592 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
593 final Map<?, ?> json = run(op, f);
594 final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
595 if (status == null) {
596 throw new FileNotFoundException("File does not exist: " + f);
597 }
598 return status;
599 }
600
601 @Override
602 public FileStatus getFileStatus(Path f) throws IOException {
603 statistics.incrementReadOps(1);
604 return makeQualified(getHdfsFileStatus(f), f);
605 }
606
607 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
608 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
609 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
610 f.getPermission(), f.getOwner(), f.getGroup(),
611 f.isSymlink() ? new Path(f.getSymlink()) : null,
612 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
613 }
614
615 @Override
616 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
617 statistics.incrementWriteOps(1);
618 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
619 final Map<?, ?> json = run(op, f,
620 new PermissionParam(applyUMask(permission)));
621 return (Boolean)json.get("boolean");
622 }
623
624 /**
625 * Create a symlink pointing to the destination path.
626 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
627 */
628 public void createSymlink(Path destination, Path f, boolean createParent
629 ) throws IOException {
630 statistics.incrementWriteOps(1);
631 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
632 run(op, f, new DestinationParam(makeQualified(destination).toUri().getPath()),
633 new CreateParentParam(createParent));
634 }
635
636 @Override
637 public boolean rename(final Path src, final Path dst) throws IOException {
638 statistics.incrementWriteOps(1);
639 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
640 final Map<?, ?> json = run(op, src,
641 new DestinationParam(makeQualified(dst).toUri().getPath()));
642 return (Boolean)json.get("boolean");
643 }
644
645 @SuppressWarnings("deprecation")
646 @Override
647 public void rename(final Path src, final Path dst,
648 final Options.Rename... options) throws IOException {
649 statistics.incrementWriteOps(1);
650 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
651 run(op, src, new DestinationParam(makeQualified(dst).toUri().getPath()),
652 new RenameOptionSetParam(options));
653 }
654
655 @Override
656 public void setOwner(final Path p, final String owner, final String group
657 ) throws IOException {
658 if (owner == null && group == null) {
659 throw new IOException("owner == null && group == null");
660 }
661
662 statistics.incrementWriteOps(1);
663 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
664 run(op, p, new OwnerParam(owner), new GroupParam(group));
665 }
666
667 @Override
668 public void setPermission(final Path p, final FsPermission permission
669 ) throws IOException {
670 statistics.incrementWriteOps(1);
671 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
672 run(op, p, new PermissionParam(permission));
673 }
674
675 @Override
676 public boolean setReplication(final Path p, final short replication
677 ) throws IOException {
678 statistics.incrementWriteOps(1);
679 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
680 final Map<?, ?> json = run(op, p, new ReplicationParam(replication));
681 return (Boolean)json.get("boolean");
682 }
683
684 @Override
685 public void setTimes(final Path p, final long mtime, final long atime
686 ) throws IOException {
687 statistics.incrementWriteOps(1);
688 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
689 run(op, p, new ModificationTimeParam(mtime), new AccessTimeParam(atime));
690 }
691
692 @Override
693 public long getDefaultBlockSize() {
694 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
695 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
696 }
697
698 @Override
699 public short getDefaultReplication() {
700 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
701 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
702 }
703
704 FSDataOutputStream write(final HttpOpParam.Op op,
705 final HttpURLConnection conn, final int bufferSize) throws IOException {
706 return new FSDataOutputStream(new BufferedOutputStream(
707 conn.getOutputStream(), bufferSize), statistics) {
708 @Override
709 public void close() throws IOException {
710 try {
711 super.close();
712 } finally {
713 try {
714 validateResponse(op, conn, true);
715 } finally {
716 conn.disconnect();
717 }
718 }
719 }
720 };
721 }
722
723 @Override
724 public void concat(final Path trg, final Path [] psrcs) throws IOException {
725 statistics.incrementWriteOps(1);
726 final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
727
728 List<String> strPaths = new ArrayList<String>(psrcs.length);
729 for(Path psrc : psrcs) {
730 strPaths.add(psrc.toUri().getPath());
731 }
732
733 String srcs = StringUtils.join(",", strPaths);
734
735 ConcatSourcesParam param = new ConcatSourcesParam(srcs);
736 run(op, trg, param);
737 }
738
739 @Override
740 public FSDataOutputStream create(final Path f, final FsPermission permission,
741 final boolean overwrite, final int bufferSize, final short replication,
742 final long blockSize, final Progressable progress) throws IOException {
743 statistics.incrementWriteOps(1);
744
745 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
746 return new Runner(op, f,
747 new PermissionParam(applyUMask(permission)),
748 new OverwriteParam(overwrite),
749 new BufferSizeParam(bufferSize),
750 new ReplicationParam(replication),
751 new BlockSizeParam(blockSize))
752 .run()
753 .write(bufferSize);
754 }
755
756 @Override
757 public FSDataOutputStream append(final Path f, final int bufferSize,
758 final Progressable progress) throws IOException {
759 statistics.incrementWriteOps(1);
760
761 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
762 return new Runner(op, f, new BufferSizeParam(bufferSize))
763 .run()
764 .write(bufferSize);
765 }
766
767 @SuppressWarnings("deprecation")
768 @Override
769 public boolean delete(final Path f) throws IOException {
770 return delete(f, true);
771 }
772
773 @Override
774 public boolean delete(Path f, boolean recursive) throws IOException {
775 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
776 final Map<?, ?> json = run(op, f, new RecursiveParam(recursive));
777 return (Boolean)json.get("boolean");
778 }
779
780 @Override
781 public FSDataInputStream open(final Path f, final int buffersize
782 ) throws IOException {
783 statistics.incrementReadOps(1);
784 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
785 final URL url = toUrl(op, f, new BufferSizeParam(buffersize));
786 return new FSDataInputStream(new OffsetUrlInputStream(
787 new OffsetUrlOpener(url), new OffsetUrlOpener(null)));
788 }
789
790 @Override
791 public void close() throws IOException {
792 super.close();
793 if (dtRenewer != null) {
794 dtRenewer.removeRenewAction(this); // blocks
795 }
796 }
797
798 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
799 OffsetUrlOpener(final URL url) {
800 super(url);
801 }
802
803 /** Setup offset url and connect. */
804 @Override
805 protected HttpURLConnection connect(final long offset,
806 final boolean resolved) throws IOException {
807 final URL offsetUrl = offset == 0L? url
808 : new URL(url + "&" + new OffsetParam(offset));
809 return new Runner(GetOpParam.Op.OPEN, offsetUrl, resolved).run().conn;
810 }
811 }
812
813 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
814
815 /** Remove offset parameter, if there is any, from the url */
816 static URL removeOffsetParam(final URL url) throws MalformedURLException {
817 String query = url.getQuery();
818 if (query == null) {
819 return url;
820 }
821 final String lower = query.toLowerCase();
822 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
823 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
824 return url;
825 }
826
827 //rebuild query
828 StringBuilder b = null;
829 for(final StringTokenizer st = new StringTokenizer(query, "&");
830 st.hasMoreTokens();) {
831 final String token = st.nextToken();
832 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
833 if (b == null) {
834 b = new StringBuilder("?").append(token);
835 } else {
836 b.append('&').append(token);
837 }
838 }
839 }
840 query = b == null? "": b.toString();
841
842 final String urlStr = url.toString();
843 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
844 }
845
846 static class OffsetUrlInputStream extends ByteRangeInputStream {
847 OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) {
848 super(o, r);
849 }
850
851 /** Remove offset parameter before returning the resolved url. */
852 @Override
853 protected URL getResolvedUrl(final HttpURLConnection connection
854 ) throws MalformedURLException {
855 return removeOffsetParam(connection.getURL());
856 }
857 }
858
859 @Override
860 public FileStatus[] listStatus(final Path f) throws IOException {
861 statistics.incrementReadOps(1);
862
863 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
864 final Map<?, ?> json = run(op, f);
865 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
866 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
867
868 //convert FileStatus
869 final FileStatus[] statuses = new FileStatus[array.length];
870 for(int i = 0; i < array.length; i++) {
871 final Map<?, ?> m = (Map<?, ?>)array[i];
872 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
873 }
874 return statuses;
875 }
876
877 @Override
878 public Token<DelegationTokenIdentifier> getDelegationToken(
879 final String renewer) throws IOException {
880 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
881 final Map<?, ?> m = run(op, null, new RenewerParam(renewer));
882 final Token<DelegationTokenIdentifier> token = JsonUtil.toDelegationToken(m);
883 SecurityUtil.setTokenService(token, nnAddr);
884 return token;
885 }
886
887 @Override
888 public Token<?> getRenewToken() {
889 return delegationToken;
890 }
891
892 @Override
893 public <T extends TokenIdentifier> void setDelegationToken(
894 final Token<T> token) {
895 synchronized(this) {
896 delegationToken = token;
897 }
898 }
899
900 private synchronized long renewDelegationToken(final Token<?> token
901 ) throws IOException {
902 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
903 TokenArgumentParam dtargParam = new TokenArgumentParam(
904 token.encodeToUrlString());
905 final Map<?, ?> m = run(op, null, dtargParam);
906 return (Long) m.get("long");
907 }
908
909 private synchronized void cancelDelegationToken(final Token<?> token
910 ) throws IOException {
911 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
912 TokenArgumentParam dtargParam = new TokenArgumentParam(
913 token.encodeToUrlString());
914 run(op, null, dtargParam);
915 }
916
917 @Override
918 public BlockLocation[] getFileBlockLocations(final FileStatus status,
919 final long offset, final long length) throws IOException {
920 if (status == null) {
921 return null;
922 }
923 return getFileBlockLocations(status.getPath(), offset, length);
924 }
925
926 @Override
927 public BlockLocation[] getFileBlockLocations(final Path p,
928 final long offset, final long length) throws IOException {
929 statistics.incrementReadOps(1);
930
931 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
932 final Map<?, ?> m = run(op, p, new OffsetParam(offset),
933 new LengthParam(length));
934 return DFSUtil.locatedBlocks2Locations(JsonUtil.toLocatedBlocks(m));
935 }
936
937 @Override
938 public ContentSummary getContentSummary(final Path p) throws IOException {
939 statistics.incrementReadOps(1);
940
941 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
942 final Map<?, ?> m = run(op, p);
943 return JsonUtil.toContentSummary(m);
944 }
945
946 @Override
947 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
948 ) throws IOException {
949 statistics.incrementReadOps(1);
950
951 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
952 final Map<?, ?> m = run(op, p);
953 return JsonUtil.toMD5MD5CRC32FileChecksum(m);
954 }
955
956 /** Delegation token renewer. */
957 public static class DtRenewer extends TokenRenewer {
958 @Override
959 public boolean handleKind(Text kind) {
960 return kind.equals(TOKEN_KIND);
961 }
962
963 @Override
964 public boolean isManaged(Token<?> token) throws IOException {
965 return true;
966 }
967
968 private static WebHdfsFileSystem getWebHdfs(
969 final Token<?> token, final Configuration conf) throws IOException {
970
971 final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
972 final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
973 return (WebHdfsFileSystem)FileSystem.get(uri, conf);
974 }
975
976 @Override
977 public long renew(final Token<?> token, final Configuration conf
978 ) throws IOException, InterruptedException {
979 final UserGroupInformation ugi = UserGroupInformation.getLoginUser();
980 // update the kerberos credentials, if they are coming from a keytab
981 ugi.reloginFromKeytab();
982
983 return getWebHdfs(token, conf).renewDelegationToken(token);
984 }
985
986 @Override
987 public void cancel(final Token<?> token, final Configuration conf
988 ) throws IOException, InterruptedException {
989 final UserGroupInformation ugi = UserGroupInformation.getLoginUser();
990 // update the kerberos credentials, if they are coming from a keytab
991 ugi.checkTGTAndReloginFromKeytab();
992
993 getWebHdfs(token, conf).cancelDelegationToken(token);
994 }
995 }
996
997 private static class WebHdfsDelegationTokenSelector
998 extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
999 private static final DelegationTokenSelector hdfsTokenSelector =
1000 new DelegationTokenSelector();
1001
1002 public WebHdfsDelegationTokenSelector() {
1003 super(TOKEN_KIND);
1004 }
1005
1006 Token<DelegationTokenIdentifier> selectToken(URI nnUri,
1007 Collection<Token<?>> tokens, Configuration conf) {
1008 Token<DelegationTokenIdentifier> token =
1009 selectToken(SecurityUtil.buildTokenService(nnUri), tokens);
1010 if (token == null) {
1011 token = hdfsTokenSelector.selectToken(nnUri, tokens, conf);
1012 }
1013 return token;
1014 }
1015 }
1016 }