org.apache.hadoop.hdfs.web

Class WebHdfsFileSystem

java.lang.Object

org.apache.hadoop.conf.Configured

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.hdfs.web.WebHdfsFileSystem

All Implemented Interfaces:

Closeable, AutoCloseable, org.apache.hadoop.conf.Configurable, org.apache.hadoop.fs.DelegationTokenRenewer.Renewable

Direct Known Subclasses:

SWebHdfsFileSystem

public class WebHdfsFileSystem
extends org.apache.hadoop.fs.FileSystem
implements org.apache.hadoop.fs.DelegationTokenRenewer.Renewable

A FileSystem for HDFS over the web.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.FileSystem.Statistics

Field Summary

Fields

Modifier and Type	Field and Description
static String	CANT_FALLBACK_TO_INSECURE_MSG
protected org.apache.hadoop.hdfs.web.URLConnectionFactory	connectionFactory Default connection factory may be overridden in tests to use smaller timeout values
static org.apache.commons.logging.Log	LOG
static String	PATH_PREFIX Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file
static String	SCHEME File System URI: {SCHEME}://namenode:port/path/to/file
static org.apache.hadoop.io.Text	TOKEN_KIND Delegation token kind
protected org.apache.hadoop.io.Text	tokenServiceName
static int	VERSION WebHdfs version.

Fields inherited from class org.apache.hadoop.fs.FileSystem

DEFAULT_FS, FS_DEFAULT_NAME_KEY, SHUTDOWN_HOOK_PRIORITY, statistics

Constructor Summary

Constructors

Constructor and Description
WebHdfsFileSystem()

Method Summary

Methods

Modifier and Type	Method and Description
void	access(org.apache.hadoop.fs.Path path, org.apache.hadoop.fs.permission.FsAction mode)
org.apache.hadoop.fs.FSDataOutputStream	append(org.apache.hadoop.fs.Path f, int bufferSize, org.apache.hadoop.util.Progressable progress)
void	cancelDelegationToken(org.apache.hadoop.security.token.Token<?> token)
protected URI	canonicalizeUri(URI uri)
void	close()
void	concat(org.apache.hadoop.fs.Path trg, org.apache.hadoop.fs.Path[] srcs)
org.apache.hadoop.fs.FSDataOutputStream	create(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, org.apache.hadoop.util.Progressable progress)
org.apache.hadoop.fs.Path	createSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
void	createSymlink(org.apache.hadoop.fs.Path destination, org.apache.hadoop.fs.Path f, boolean createParent) Create a symlink pointing to the destination path.
boolean	delete(org.apache.hadoop.fs.Path f, boolean recursive)
void	deleteSnapshot(org.apache.hadoop.fs.Path path, String snapshotName)
org.apache.hadoop.fs.permission.AclStatus	getAclStatus(org.apache.hadoop.fs.Path f)
String	getCanonicalServiceName()
URI	getCanonicalUri()
org.apache.hadoop.fs.ContentSummary	getContentSummary(org.apache.hadoop.fs.Path p)
long	getDefaultBlockSize()
int	getDefaultPort()
short	getDefaultReplication()
protected org.apache.hadoop.security.token.Token<?>	getDelegationToken()
org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier>	getDelegationToken(String renewer)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.FileStatus status, long offset, long length)
org.apache.hadoop.fs.BlockLocation[]	getFileBlockLocations(org.apache.hadoop.fs.Path p, long offset, long length)
org.apache.hadoop.fs.MD5MD5CRC32FileChecksum	getFileChecksum(org.apache.hadoop.fs.Path p)
org.apache.hadoop.fs.FileStatus	getFileStatus(org.apache.hadoop.fs.Path f)
org.apache.hadoop.fs.Path	getHomeDirectory()
static String	getHomeDirectoryString(org.apache.hadoop.security.UserGroupInformation ugi)
org.apache.hadoop.security.token.Token<?>	getRenewToken()
String	getScheme() Return the protocol scheme for the FileSystem.
protected org.apache.hadoop.io.Text	getTokenKind()
protected String	getTransportScheme() return the underlying transport protocol (http / https).
URI	getUri()
org.apache.hadoop.fs.Path	getWorkingDirectory()
byte[]	getXAttr(org.apache.hadoop.fs.Path p, String name)
Map<String,byte[]>	getXAttrs(org.apache.hadoop.fs.Path p)
Map<String,byte[]>	getXAttrs(org.apache.hadoop.fs.Path p, List<String> names)
void	initialize(URI uri, org.apache.hadoop.conf.Configuration conf)
static boolean	isEnabled(org.apache.hadoop.conf.Configuration conf, org.apache.commons.logging.Log log) Is WebHDFS enabled in conf?
org.apache.hadoop.fs.FileStatus[]	listStatus(org.apache.hadoop.fs.Path f)
List<String>	listXAttrs(org.apache.hadoop.fs.Path p)
boolean	mkdirs(org.apache.hadoop.fs.Path f, org.apache.hadoop.fs.permission.FsPermission permission)
void	modifyAclEntries(org.apache.hadoop.fs.Path path, List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
org.apache.hadoop.fs.FSDataInputStream	open(org.apache.hadoop.fs.Path f, int buffersize)
void	removeAcl(org.apache.hadoop.fs.Path path)
void	removeAclEntries(org.apache.hadoop.fs.Path path, List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
void	removeDefaultAcl(org.apache.hadoop.fs.Path path)
void	removeXAttr(org.apache.hadoop.fs.Path p, String name)
boolean	rename(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst)
void	rename(org.apache.hadoop.fs.Path src, org.apache.hadoop.fs.Path dst, org.apache.hadoop.fs.Options.Rename... options)
void	renameSnapshot(org.apache.hadoop.fs.Path path, String snapshotOldName, String snapshotNewName)
long	renewDelegationToken(org.apache.hadoop.security.token.Token<?> token)
void	setAcl(org.apache.hadoop.fs.Path p, List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
<T extends org.apache.hadoop.security.token.TokenIdentifier> void	setDelegationToken(org.apache.hadoop.security.token.Token<T> token)
void	setOwner(org.apache.hadoop.fs.Path p, String owner, String group)
void	setPermission(org.apache.hadoop.fs.Path p, org.apache.hadoop.fs.permission.FsPermission permission)
boolean	setReplication(org.apache.hadoop.fs.Path p, short replication)
void	setTimes(org.apache.hadoop.fs.Path p, long mtime, long atime)
void	setWorkingDirectory(org.apache.hadoop.fs.Path dir)
void	setXAttr(org.apache.hadoop.fs.Path p, String name, byte[] value, EnumSet<org.apache.hadoop.fs.XAttrSetFlag> flag)

Methods inherited from class org.apache.hadoop.fs.FileSystem

append, append, areSymlinksEnabled, cancelDeleteOnExit, checkPath, clearStatistics, closeAll, closeAllForUGI, completeLocalOutput, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyFromLocalFile, copyToLocalFile, copyToLocalFile, copyToLocalFile, create, create, create, create, create, create, create, create, create, create, create, create, createNewFile, createNonRecursive, createNonRecursive, createNonRecursive, createSnapshot, delete, deleteOnExit, enableSymlinks, exists, fixRelativePart, get, get, get, getAllStatistics, getBlockSize, getDefaultBlockSize, getDefaultReplication, getDefaultUri, getFileChecksum, getFileLinkStatus, getFileSystemClass, getFSofPath, getInitialWorkingDirectory, getLength, getLinkTarget, getLocal, getName, getNamed, getReplication, getServerDefaults, getServerDefaults, getStatistics, getStatistics, getStatus, getStatus, getUsed, globStatus, globStatus, isDirectory, isFile, listCorruptFileBlocks, listFiles, listLocatedStatus, listLocatedStatus, listStatus, listStatus, listStatus, makeQualified, mkdirs, mkdirs, moveFromLocalFile, moveFromLocalFile, moveToLocalFile, newInstance, newInstance, newInstance, newInstanceLocal, open, primitiveCreate, primitiveMkdir, primitiveMkdir, printStatistics, processDeleteOnExit, resolveLink, resolvePath, setDefaultUri, setDefaultUri, setVerifyChecksum, setWriteChecksum, setXAttr, startLocalOutput, supportsSymlinks

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.apache.commons.logging.Log LOG

SCHEME

public static final String SCHEME

File System URI: {SCHEME}://namenode:port/path/to/file

See Also:

Constant Field Values

VERSION

public static final int VERSION

WebHdfs version.

See Also:

Constant Field Values

PATH_PREFIX

public static final String PATH_PREFIX

Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file

See Also:

Constant Field Values

connectionFactory

protected org.apache.hadoop.hdfs.web.URLConnectionFactory connectionFactory

Default connection factory may be overridden in tests to use smaller timeout values

TOKEN_KIND

public static final org.apache.hadoop.io.Text TOKEN_KIND

Delegation token kind

CANT_FALLBACK_TO_INSECURE_MSG

public static final String CANT_FALLBACK_TO_INSECURE_MSG

See Also:

Constant Field Values

tokenServiceName

protected org.apache.hadoop.io.Text tokenServiceName

Constructor Detail

WebHdfsFileSystem

public WebHdfsFileSystem()

Method Detail

getScheme

public String getScheme()

Return the protocol scheme for the FileSystem.

Overrides:

getScheme in class org.apache.hadoop.fs.FileSystem

Returns:

webhdfs

getTransportScheme

protected String getTransportScheme()

return the underlying transport protocol (http / https).

getTokenKind

protected org.apache.hadoop.io.Text getTokenKind()

initialize

public void initialize(URI uri,
              org.apache.hadoop.conf.Configuration conf)
                throws IOException

Overrides:

initialize in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getCanonicalUri

public URI getCanonicalUri()

Overrides:

getCanonicalUri in class org.apache.hadoop.fs.FileSystem

isEnabled

public static boolean isEnabled(org.apache.hadoop.conf.Configuration conf,
                org.apache.commons.logging.Log log)

Is WebHDFS enabled in conf?

getDelegationToken

protected org.apache.hadoop.security.token.Token<?> getDelegationToken()
                                                                throws IOException

Throws:

IOException

getDefaultPort

public int getDefaultPort()

Overrides:

getDefaultPort in class org.apache.hadoop.fs.FileSystem

getUri

public URI getUri()

Specified by:

getUri in class org.apache.hadoop.fs.FileSystem

canonicalizeUri

protected URI canonicalizeUri(URI uri)

Overrides:

canonicalizeUri in class org.apache.hadoop.fs.FileSystem

getHomeDirectoryString

public static String getHomeDirectoryString(org.apache.hadoop.security.UserGroupInformation ugi)

Returns:

the home directory.

getHomeDirectory

public org.apache.hadoop.fs.Path getHomeDirectory()

Overrides:

getHomeDirectory in class org.apache.hadoop.fs.FileSystem

getWorkingDirectory

public org.apache.hadoop.fs.Path getWorkingDirectory()

Specified by:

getWorkingDirectory in class org.apache.hadoop.fs.FileSystem

setWorkingDirectory

public void setWorkingDirectory(org.apache.hadoop.fs.Path dir)

Specified by:

setWorkingDirectory in class org.apache.hadoop.fs.FileSystem

getFileStatus

public org.apache.hadoop.fs.FileStatus getFileStatus(org.apache.hadoop.fs.Path f)
                                              throws IOException

Specified by:

getFileStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getAclStatus

public org.apache.hadoop.fs.permission.AclStatus getAclStatus(org.apache.hadoop.fs.Path f)
                                                       throws IOException

Overrides:

getAclStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

mkdirs

public boolean mkdirs(org.apache.hadoop.fs.Path f,
             org.apache.hadoop.fs.permission.FsPermission permission)
               throws IOException

Specified by:

mkdirs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSymlink

public void createSymlink(org.apache.hadoop.fs.Path destination,
                 org.apache.hadoop.fs.Path f,
                 boolean createParent)
                   throws IOException

Create a symlink pointing to the destination path.

Overrides:

createSymlink in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

See Also:

Hdfs.createSymlink(Path, Path, boolean)

rename

public boolean rename(org.apache.hadoop.fs.Path src,
             org.apache.hadoop.fs.Path dst)
               throws IOException

Specified by:

rename in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

rename

public void rename(org.apache.hadoop.fs.Path src,
          org.apache.hadoop.fs.Path dst,
          org.apache.hadoop.fs.Options.Rename... options)
            throws IOException

Overrides:

rename in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setXAttr

public void setXAttr(org.apache.hadoop.fs.Path p,
            String name,
            byte[] value,
            EnumSet<org.apache.hadoop.fs.XAttrSetFlag> flag)
              throws IOException

Overrides:

setXAttr in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getXAttr

public byte[] getXAttr(org.apache.hadoop.fs.Path p,
              String name)
                throws IOException

Overrides:

getXAttr in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getXAttrs

public Map<String,byte[]> getXAttrs(org.apache.hadoop.fs.Path p)
                             throws IOException

Overrides:

getXAttrs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getXAttrs

public Map<String,byte[]> getXAttrs(org.apache.hadoop.fs.Path p,
                           List<String> names)
                             throws IOException

Overrides:

getXAttrs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listXAttrs

public List<String> listXAttrs(org.apache.hadoop.fs.Path p)
                        throws IOException

Overrides:

listXAttrs in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

removeXAttr

public void removeXAttr(org.apache.hadoop.fs.Path p,
               String name)
                 throws IOException

Overrides:

removeXAttr in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setOwner

public void setOwner(org.apache.hadoop.fs.Path p,
            String owner,
            String group)
              throws IOException

Overrides:

setOwner in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setPermission

public void setPermission(org.apache.hadoop.fs.Path p,
                 org.apache.hadoop.fs.permission.FsPermission permission)
                   throws IOException

Overrides:

setPermission in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

modifyAclEntries

public void modifyAclEntries(org.apache.hadoop.fs.Path path,
                    List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
                      throws IOException

Overrides:

modifyAclEntries in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

removeAclEntries

public void removeAclEntries(org.apache.hadoop.fs.Path path,
                    List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
                      throws IOException

Overrides:

removeAclEntries in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

removeDefaultAcl

public void removeDefaultAcl(org.apache.hadoop.fs.Path path)
                      throws IOException

Overrides:

removeDefaultAcl in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

removeAcl

public void removeAcl(org.apache.hadoop.fs.Path path)
               throws IOException

Overrides:

removeAcl in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setAcl

public void setAcl(org.apache.hadoop.fs.Path p,
          List<org.apache.hadoop.fs.permission.AclEntry> aclSpec)
            throws IOException

Overrides:

setAcl in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

createSnapshot

public org.apache.hadoop.fs.Path createSnapshot(org.apache.hadoop.fs.Path path,
                                       String snapshotName)
                                         throws IOException

Overrides:

createSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

deleteSnapshot

public void deleteSnapshot(org.apache.hadoop.fs.Path path,
                  String snapshotName)
                    throws IOException

Overrides:

deleteSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

renameSnapshot

public void renameSnapshot(org.apache.hadoop.fs.Path path,
                  String snapshotOldName,
                  String snapshotNewName)
                    throws IOException

Overrides:

renameSnapshot in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setReplication

public boolean setReplication(org.apache.hadoop.fs.Path p,
                     short replication)
                       throws IOException

Overrides:

setReplication in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

setTimes

public void setTimes(org.apache.hadoop.fs.Path p,
            long mtime,
            long atime)
              throws IOException

Overrides:

setTimes in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDefaultBlockSize

public long getDefaultBlockSize()

Overrides:

getDefaultBlockSize in class org.apache.hadoop.fs.FileSystem

getDefaultReplication

public short getDefaultReplication()

Overrides:

getDefaultReplication in class org.apache.hadoop.fs.FileSystem

concat

public void concat(org.apache.hadoop.fs.Path trg,
          org.apache.hadoop.fs.Path[] srcs)
            throws IOException

Overrides:

concat in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

create

public org.apache.hadoop.fs.FSDataOutputStream create(org.apache.hadoop.fs.Path f,
                                             org.apache.hadoop.fs.permission.FsPermission permission,
                                             boolean overwrite,
                                             int bufferSize,
                                             short replication,
                                             long blockSize,
                                             org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Specified by:

create in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

append

public org.apache.hadoop.fs.FSDataOutputStream append(org.apache.hadoop.fs.Path f,
                                             int bufferSize,
                                             org.apache.hadoop.util.Progressable progress)
                                               throws IOException

Specified by:

append in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

delete

public boolean delete(org.apache.hadoop.fs.Path f,
             boolean recursive)
               throws IOException

Specified by:

delete in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

open

public org.apache.hadoop.fs.FSDataInputStream open(org.apache.hadoop.fs.Path f,
                                          int buffersize)
                                            throws IOException

Specified by:

open in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

close

public void close()
           throws IOException

Specified by:

close in interface Closeable

Specified by:

close in interface AutoCloseable

Overrides:

close in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

listStatus

public org.apache.hadoop.fs.FileStatus[] listStatus(org.apache.hadoop.fs.Path f)
                                             throws IOException

Specified by:

listStatus in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getDelegationToken

public org.apache.hadoop.security.token.Token<org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier> getDelegationToken(String renewer)
                                                                                                                                      throws IOException

Overrides:

getDelegationToken in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getRenewToken

public org.apache.hadoop.security.token.Token<?> getRenewToken()

Specified by:

getRenewToken in interface org.apache.hadoop.fs.DelegationTokenRenewer.Renewable

setDelegationToken

public <T extends org.apache.hadoop.security.token.TokenIdentifier> void setDelegationToken(org.apache.hadoop.security.token.Token<T> token)

Specified by:

setDelegationToken in interface org.apache.hadoop.fs.DelegationTokenRenewer.Renewable

renewDelegationToken

public long renewDelegationToken(org.apache.hadoop.security.token.Token<?> token)
                          throws IOException

Throws:

IOException

cancelDelegationToken

public void cancelDelegationToken(org.apache.hadoop.security.token.Token<?> token)
                           throws IOException

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.FileStatus status,
                                                         long offset,
                                                         long length)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileBlockLocations

public org.apache.hadoop.fs.BlockLocation[] getFileBlockLocations(org.apache.hadoop.fs.Path p,
                                                         long offset,
                                                         long length)
                                                           throws IOException

Overrides:

getFileBlockLocations in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

access

public void access(org.apache.hadoop.fs.Path path,
          org.apache.hadoop.fs.permission.FsAction mode)
            throws IOException

Overrides:

access in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getContentSummary

public org.apache.hadoop.fs.ContentSummary getContentSummary(org.apache.hadoop.fs.Path p)
                                                      throws IOException

Overrides:

getContentSummary in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getFileChecksum

public org.apache.hadoop.fs.MD5MD5CRC32FileChecksum getFileChecksum(org.apache.hadoop.fs.Path p)
                                                             throws IOException

Overrides:

getFileChecksum in class org.apache.hadoop.fs.FileSystem

Throws:

IOException

getCanonicalServiceName

public String getCanonicalServiceName()

Overrides:

getCanonicalServiceName in class org.apache.hadoop.fs.FileSystem