class
HDFSMetadataLog[T] extends MetadataLog[T] with Logging
Instance Constructors
-
new
HDFSMetadataLog(sparkSession: SparkSession, path: String)(implicit arg0: ClassTag[T])
Value Members
-
final
def
!=(arg0: AnyRef): Boolean
-
final
def
!=(arg0: Any): Boolean
-
final
def
##(): Int
-
final
def
==(arg0: AnyRef): Boolean
-
final
def
==(arg0: Any): Boolean
-
def
add(batchId: Long, metadata: T): Boolean
-
final
def
asInstanceOf[T0]: T0
-
def
batchIdToPath(batchId: Long): Path
-
def
clone(): AnyRef
-
def
deserialize(bytes: Array[Byte]): T
-
final
def
eq(arg0: AnyRef): Boolean
-
def
equals(arg0: Any): Boolean
-
-
def
finalize(): Unit
-
def
get(startId: Option[Long], endId: Option[Long]): Array[(Long, T)]
-
def
get(batchId: Long): Option[T]
-
final
def
getClass(): Class[_]
-
def
getLatest(): Option[(Long, T)]
-
def
hashCode(): Int
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
-
def
isBatchFile(path: Path): Boolean
-
final
def
isInstanceOf[T0]: Boolean
-
def
isTraceEnabled(): Boolean
-
def
log: Logger
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
-
def
logDebug(msg: ⇒ String): Unit
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
-
def
logError(msg: ⇒ String): Unit
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
-
def
logInfo(msg: ⇒ String): Unit
-
def
logName: String
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
-
def
logTrace(msg: ⇒ String): Unit
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
-
def
logWarning(msg: ⇒ String): Unit
-
val
metadataPath: Path
-
final
def
ne(arg0: AnyRef): Boolean
-
final
def
notify(): Unit
-
final
def
notifyAll(): Unit
-
def
pathToBatchId(path: Path): Long
-
def
serialize(metadata: T): Array[Byte]
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
-
def
toString(): String
-
final
def
wait(): Unit
-
final
def
wait(arg0: Long, arg1: Int): Unit
-
final
def
wait(arg0: Long): Unit
Inherited from Logging
Inherited from AnyRef
Inherited from Any
A MetadataLog implementation based on HDFS. HDFSMetadataLog uses the specified
path
as the metadata storage.When writing a new batch, HDFSMetadataLog will firstly write to a temp file and then rename it to the final batch file. If the rename step fails, there must be multiple writers and only one of them will succeed and the others will fail.
Note: HDFSMetadataLog doesn't support S3-like file systems as they don't guarantee listing files in a directory always shows the latest files.