DailySuffixParquetThrift

When Using these sources or creating subclasses of them, you can provide a filter predicate and / or a set of fields (columns) to keep (project).

The filter predicate will be pushed down to the input format, potentially making the filter significantly more efficient than a filter applied to a TypedPipe (parquet push-down filters can skip reading entire chunks of data off disk).

For data with a large schema (many fields / columns), providing the set of columns you intend to use can also make your job significantly more efficient (parquet column projection push-down will skip reading unused columns from disk). The columns are specified in the format described here: https://github.com/apache/incubator-parquet-mr/blob/master/parquet_cascading.md#21-projection-pushdown-with-thriftscrooge-records

These settings are defined in the traits com.twitter.scalding.parquet.HasFilterPredicate and com.twitter.scalding.parquet.HasColumnProjection

Here are two ways you can use these in a parquet source:

class MyParquetSource(dr: DateRange) extends DailySuffixParquetThrift("/a/path", dr)

val mySourceFilteredAndProjected = new MyParquetSource(dr) {
  override val withFilter: Option[FilterPredicate] = Some(myFp)
  override val withColumns: Set[String] = Set("a/b/c", "x/y")
}

The other way is to add these as constructor arguments:

class MyParquetSource(
  dr: DateRange,
  override val withFilter: Option[FilterPredicate] = None
  override val withColumns: Set[String] = Set()
) extends DailySuffixParquetThrift("/a/path", dr)

val mySourceFilteredAndProjected = new MyParquetSource(dr, Some(myFp), Set("a/b/c", "x/y"))

Linear Supertypes

ParquetThrift[T], ParquetThriftBase[T], HasColumnProjection, HasFilterPredicate, LocalTapSource, typed.TypedSink[T], SingleMappable[T], Mappable[T], typed.TypedSource[T], DailySuffixSource, TimePathedSource, TimeSeqPathedSource, FileSource, LocalSourceOverride, SchemedSource, Source, Serializable, AnyRef, Any

Instance Constructors

new DailySuffixParquetThrift(path: String, dateRange: DateRange)(implicit mf: Manifest[T])

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def allPaths: Iterable[String]

Definition Classes
TimeSeqPathedSource
def allPathsFor(pattern: String): Iterable[String]

Attributes
protected
Definition Classes
TimeSeqPathedSource
def andThen[U](fn: (T) ⇒ U): typed.TypedSource[U]

Definition Classes
TypedSource
final def asInstanceOf[T0]: T0

Definition Classes
Any
def checkFlowDefNotNull(implicit flowDef: FlowDef, mode: Mode): Unit

Attributes
protected
Definition Classes
Source
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def columnGlobs: Set[ColumnProjectionGlob]

Attributes
protected[com.twitter.scalding.parquet]
Definition Classes
HasColumnProjection
def config: parquet.cascading.ParquetValueScheme.Config[T]

Definition Classes
ParquetThriftBase
def contraMap[U](fn: (U) ⇒ T): typed.TypedSink[U]

Definition Classes
TypedSink
def converter[U >: T]: TupleConverter[U]

Definition Classes
SingleMappable → TypedSource
def createHdfsReadTap(hdfsMode: Hdfs): Tap[JobConf, _, _]

Attributes
protected
Definition Classes
FileSource
def createLocalTap(sinkMode: SinkMode): Tap[_, _, _]

Definition Classes
LocalTapSource → LocalSourceOverride
def createTap(readOrWrite: AccessMode)(implicit mode: Mode): Tap[_, _, _]

Definition Classes
FileSource → Source
val dateRange: DateRange

Definition Classes
TimeSeqPathedSource
def defaultDurationFor(pattern: String): Option[Duration]

Attributes
protected
Definition Classes
TimeSeqPathedSource
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(that: Any): Boolean

Definition Classes
TimeSeqPathedSource → AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def flatMapTo[U](out: Fields)(mf: (T) ⇒ TraversableOnce[U])(implicit flowDef: FlowDef, mode: Mode, setter: TupleSetter[U]): Pipe

Definition Classes
Mappable
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getPathStatuses(conf: Configuration): Iterable[(String, Boolean)]

Definition Classes
TimeSeqPathedSource
final def globsInParquetStringFormat: Option[String]

Parquet accepts globs separated by the ; character
Parquet accepts globs separated by the ; character

Attributes
protected[com.twitter.scalding.parquet]
Definition Classes
HasColumnProjection
def goodHdfsPaths(hdfsMode: Hdfs): Iterable[String]

Attributes
protected
Definition Classes
FileSource
def hashCode(): Int

Definition Classes
TimeSeqPathedSource → AnyRef → Any
def hdfsPaths: Iterable[String]

Definition Classes
TimeSeqPathedSource → FileSource
def hdfsReadPathsAreGood(conf: Configuration): Boolean

Definition Classes
TimeSeqPathedSource → FileSource
def hdfsScheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], _, _]

Definition Classes
ParquetThrift → SchemedSource
def hdfsWritePath: String

Definition Classes
TimePathedSource → FileSource
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def localPath: String

Definition Classes
TimePathedSource → LocalSourceOverride
def localScheme: Scheme[Properties, InputStream, OutputStream, _, _]

Definition Classes
SchemedSource
final def mapTo[U](out: Fields)(mf: (T) ⇒ U)(implicit flowDef: FlowDef, mode: Mode, setter: TupleSetter[U]): Pipe

Definition Classes
Mappable
implicit val mf: Manifest[T]

Definition Classes
DailySuffixParquetThrift → ParquetThriftBase
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def pathIsGood(p: String, conf: Configuration): Boolean

Attributes
protected
Definition Classes
FileSource
val pattern: String

Definition Classes
TimePathedSource
val patterns: Seq[String]

Definition Classes
TimeSeqPathedSource
def read(implicit flowDef: FlowDef, mode: Mode): Pipe

Definition Classes
Source
def setter[U <: T]: TupleSetter[U]

Definition Classes
ParquetThriftBase → TypedSink
def sinkFields: Fields

Definition Classes
TypedSink
val sinkMode: SinkMode

Definition Classes
SchemedSource
def sourceFields: Fields

Definition Classes
TypedSource
def sourceId: String

Definition Classes
Source
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toIterator(implicit config: Config, mode: Mode): Iterator[T]

Definition Classes
Mappable
def toString(): String

Definition Classes
TimeSeqPathedSource → AnyRef → Any
def transformForRead(pipe: Pipe): Pipe

Attributes
protected
Definition Classes
Source
def transformForWrite(pipe: Pipe): Pipe

Attributes
protected
Definition Classes
Source
def transformInTest: Boolean

Definition Classes
Source
val tz: TimeZone

Definition Classes
TimeSeqPathedSource
def validateTaps(mode: Mode): Unit

Definition Classes
FileSource → Source
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def withColumns: Set[String]

The format for specifying columns is described here: https://github.com/apache/incubator-parquet-mr/blob/master/parquet_cascading.md#21-projection-pushdown-with-thriftscrooge-records
The format for specifying columns is described here: https://github.com/apache/incubator-parquet-mr/blob/master/parquet_cascading.md#21-projection-pushdown-with-thriftscrooge-records
Note that the format described there says that multiple globs can be combined with a ; character. Instead, we use a Set() here and will eventually join the set on the ; character for you.

Definition Classes
HasColumnProjection
def withFilter: Option[FilterPredicate]

Definition Classes
HasFilterPredicate
def writeFrom(pipe: Pipe)(implicit flowDef: FlowDef, mode: Mode): Pipe

Definition Classes
Source

Deprecated Value Members

def readAtSubmitter[T](implicit mode: Mode, conv: TupleConverter[T]): Stream[T]

Definition Classes
Source
Annotations
@deprecated
Deprecated
(Since version 0.9.0) replace with Mappable.toIterator

Related Doc: package thrift

class DailySuffixParquetThrift[T <: ThriftBase] extends DailySuffixSource with ParquetThrift[T]

Instance Constructors

new DailySuffixParquetThrift(path: String, dateRange: DateRange)(implicit mf: Manifest[T])

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def allPaths: Iterable[String]

def allPathsFor(pattern: String): Iterable[String]

def andThen[U](fn: (T) ⇒ U): typed.TypedSource[U]

final def asInstanceOf[T0]: T0

def checkFlowDefNotNull(implicit flowDef: FlowDef, mode: Mode): Unit

def clone(): AnyRef

final def columnGlobs: Set[ColumnProjectionGlob]

def config: parquet.cascading.ParquetValueScheme.Config[T]

def contraMap[U](fn: (U) ⇒ T): typed.TypedSink[U]

def converter[U >: T]: TupleConverter[U]

def createHdfsReadTap(hdfsMode: Hdfs): Tap[JobConf, _, _]

def createLocalTap(sinkMode: SinkMode): Tap[_, _, _]

def createTap(readOrWrite: AccessMode)(implicit mode: Mode): Tap[_, _, _]

val dateRange: DateRange

def defaultDurationFor(pattern: String): Option[Duration]

final def eq(arg0: AnyRef): Boolean

def equals(that: Any): Boolean

def finalize(): Unit

final def flatMapTo[U](out: Fields)(mf: (T) ⇒ TraversableOnce[U])(implicit flowDef: FlowDef, mode: Mode, setter: TupleSetter[U]): Pipe

final def getClass(): Class[_]

def getPathStatuses(conf: Configuration): Iterable[(String, Boolean)]

final def globsInParquetStringFormat: Option[String]

def goodHdfsPaths(hdfsMode: Hdfs): Iterable[String]

def hashCode(): Int

def hdfsPaths: Iterable[String]

def hdfsReadPathsAreGood(conf: Configuration): Boolean

def hdfsScheme: Scheme[JobConf, RecordReader[_, _], OutputCollector[_, _], _, _]

def hdfsWritePath: String

final def isInstanceOf[T0]: Boolean

def localPath: String

def localScheme: Scheme[Properties, InputStream, OutputStream, _, _]

final def mapTo[U](out: Fields)(mf: (T) ⇒ U)(implicit flowDef: FlowDef, mode: Mode, setter: TupleSetter[U]): Pipe

implicit val mf: Manifest[T]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def pathIsGood(p: String, conf: Configuration): Boolean

val pattern: String

val patterns: Seq[String]

def read(implicit flowDef: FlowDef, mode: Mode): Pipe

def setter[U <: T]: TupleSetter[U]

def sinkFields: Fields

val sinkMode: SinkMode

def sourceFields: Fields

def sourceId: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toIterator(implicit config: Config, mode: Mode): Iterator[T]

def toString(): String

def transformForRead(pipe: Pipe): Pipe

def transformForWrite(pipe: Pipe): Pipe

def transformInTest: Boolean

val tz: TimeZone

def validateTaps(mode: Mode): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def withColumns: Set[String]

def withFilter: Option[FilterPredicate]

def writeFrom(pipe: Pipe)(implicit flowDef: FlowDef, mode: Mode): Pipe

Deprecated Value Members

def readAtSubmitter[T](implicit mode: Mode, conv: TupleConverter[T]): Stream[T]

Inherited from ParquetThrift[T]

Inherited from ParquetThriftBase[T]

Inherited from HasColumnProjection

Inherited from HasFilterPredicate

Inherited from LocalTapSource

Inherited from typed.TypedSink[T]

Inherited from SingleMappable[T]

Inherited from Mappable[T]

Inherited from typed.TypedSource[T]

Inherited from DailySuffixSource