SparkHdfs

Type Members

type Backend[A] = EitherT[[β$1$]WriterT[[β$0$]Kleisli[M, Config, β$0$], Vector[PhaseResult], β$1$], FileSystemError, A]

Definition Classes
BackendModule
type BackendT[F[_], A] = EitherT[[β$1$]WriterT[[β$0$]Kleisli[F, Config, β$0$], Vector[PhaseResult], β$1$], FileSystemError, A]

Definition Classes
BackendModule
type Config = HdfsConfig

Definition Classes
SparkHdfs → BackendModule
type Configured[A] = Kleisli[M, Config, A]

Definition Classes
BackendModule
type ConfiguredT[F[_], A] = Kleisli[F, Config, A]

Definition Classes
BackendModule
type Eff[A] = Coproduct[[β$4$]Read[FileSystem, β$4$], Eff7, A]

Definition Classes
SparkHdfs → SparkCore
type Eff1[A] = Coproduct[[γ$0$]KeyValueStore[ResultHandle, SparkCursor, γ$0$], [β$1$]Read[SparkContext, β$1$], A]
type Eff2[A] = Coproduct[[γ$2$]KeyValueStore[ReadHandle, SparkCursor, γ$2$], Eff1, A]
type Eff3[A] = Coproduct[[γ$3$]KeyValueStore[WriteHandle, HdfsWriteCursor, γ$3$], Eff2, A]
type Eff4[A] = Coproduct[Task, Eff3, A]
type Eff5[A] = Coproduct[PhysErr, Eff4, A]
type Eff6[A] = Coproduct[MonotonicSeq, Eff5, A]
type Eff7[A] = Coproduct[SparkConnectorDetails, Eff6, A]
implicit class LiftBackend[A] extends AnyRef

Definition Classes
BackendModule
implicit class LiftBackendConfigured[A] extends AnyRef

Definition Classes
BackendModule
type LowerLevel[A] = Coproduct[Task, PhysErr, A]

Definition Classes
SparkCore
type M[A] = Free[Eff, A]

Definition Classes
SparkCore → BackendModule
trait ManageFileModule extends AnyRef

Definition Classes
BackendModule
type QS[T[_[_]]] = :\:[[β$6$]QScriptCore[T, β$6$], :/:[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$]]]

Definition Classes
SparkCore → BackendModule
type QSM[T[_[_]], A] = fp.:\:.M[A]

Definition Classes
BackendModule
trait QueryFileModule extends AnyRef

Definition Classes
BackendModule
trait ReadFileModule extends AnyRef

Definition Classes
BackendModule
type Repr = RDD[Data]

Definition Classes
SparkCore → BackendModule
abstract class SparkCoreManageFileModule extends ManageFileModule

Definition Classes
SparkCore
trait WriteFileModule extends AnyRef

Definition Classes
BackendModule
class details[F[_]] extends AnyRef

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def DelayRenderTreeQSM[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Delay[RenderTree, [β$14$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$14$]]

Definition Classes
SparkCore → BackendModule
def ExtractPathQSM[T[_[_]]](implicit arg0: RecursiveT[T]): ExtractPath[[β$15$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$15$], APath]

Definition Classes
SparkCore → BackendModule
def FunctorQSM[T[_[_]]]: Functor[[β$13$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$13$]]

Definition Classes
SparkCore → BackendModule
object HdfsManageFileModule extends SparkCoreManageFileModule
object HdfsWriteFileModule extends WriteFileModule
def ManageFileModule: ManageFileModule

Definition Classes
SparkHdfs → BackendModule
def MonadM: Monad[M]

Definition Classes
SparkCore → BackendModule
def MonotonicSeqInj: Inject[MonotonicSeq, Eff]

Definition Classes
SparkHdfs → SparkCore
def QFKeyValueStoreInj: Inject[[γ$12$]KeyValueStore[ResultHandle, SparkCursor, γ$12$], Eff]

Definition Classes
SparkHdfs → SparkCore
def QSCoreInject[T[_[_]]]: :<:[[β$16$]QScriptCore[T, β$16$], [β$17$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$17$]]

Definition Classes
SparkCore → BackendModule
def QueryFileModule: QueryFileModule

Definition Classes
SparkCore → BackendModule
def RFKeyValueStoreInj: Inject[[γ$11$]KeyValueStore[ReadHandle, SparkCursor, γ$11$], Eff]

Definition Classes
SparkHdfs → SparkCore
def ReadFileModule: SparkReadFileModule

Definition Classes
SparkCore → BackendModule
def ReadSparkContextInj: Inject[[β$10$]Read[SparkContext, β$10$], Eff]

Definition Classes
SparkHdfs → SparkCore
def SparkConnectorDetailsInj: Inject[SparkConnectorDetails, Eff]

Definition Classes
SparkHdfs → SparkCore
object SparkQueryFileModule extends QueryFileModule

Definition Classes
SparkCore
object SparkReadFileModule extends ReadFileModule

Definition Classes
SparkCore
def TaskInj: Inject[Task, Eff]

Definition Classes
SparkHdfs → SparkCore
val Type: FileSystemType

Definition Classes
SparkHdfs → BackendModule
def UnicoalesceCap[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Capture[T, QS[T]]

Definition Classes
SparkCore → BackendModule
def UnirewriteT[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Unirewrite[T, QS[T]]

Definition Classes
SparkCore → BackendModule
def WriteFileModule: HdfsWriteFileModule

Definition Classes
SparkHdfs → BackendModule
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compile(cfg: Config): DefErrT[Task, (~>[M, Task], Task[slamdata.Predef.Unit])]

Definition Classes
SparkCore → BackendModule
final def config[F[_]](implicit C: MonadReader_[F, Config]): F[Config]

Definition Classes
BackendModule
final val definition: BackendDef[Task]

Definition Classes
BackendModule
object details
def detailsOps: Ops[Eff]

Definition Classes
SparkCore
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def first: (RDD[Data]) ⇒ M[Data]

Definition Classes
SparkCore
def generateHdfsFS(sfsConf: HdfsConfig): Task[FileSystem]

Annotations
@SuppressWarnings()
def generateSC: (HdfsConfig) ⇒ DefErrT[Task, SparkContext]

Definition Classes
SparkHdfs → SparkCore
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def hdfsFSOps: Ops[FileSystem, Eff]
def hdfsPathStr(sfsConf: HdfsConfig): (AFile) ⇒ ReaderT[Task, SparkContext, slamdata.Predef.String]
def interpreter(cfg: Config): DefErrT[Task, (~>[BackendEffect, Task], Task[slamdata.Predef.Unit])]

Definition Classes
ChrootedInterpreter → BackendModule
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def lowerToTask: ~>[LowerLevel, Task]

Definition Classes
SparkCore
final def lpToQScript[T[_[_]], M[_]](lp: T[LogicalPlan], lc: ListContents[M])(implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T], arg4: Monad[M], arg5: MonadFsErr[M], arg6: PhaseResultTell[M]): M[T[[β$13$]fp.:\:.M[β$13$]]]

Definition Classes
BackendModule
final def lpToRepr[T[_[_]]](lp: T[LogicalPlan])(implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Backend[PhysicalPlan[Repr]]

Definition Classes
BackendModule
def msOps: Ops[Eff]

Definition Classes
SparkCore
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def optimize[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T]): (QSM[T, T[[β$10$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$10$]]]) ⇒ QSM[T, T[[β$11$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$11$]]]

Definition Classes
SparkCore → BackendModule
def parseConfig(connUri: ConnectionUri): DefErrT[Task, HdfsConfig]

Definition Classes
SparkHdfs → BackendModule
def plan[T[_[_]]](cp: T[[β$20$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$20$]])(implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Backend[Repr]

Definition Classes
SparkCore → BackendModule
implicit def qScriptToQScriptTotal[T[_[_]]]: Aux[[β$5$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$5$], [β$6$]Coproduct[[β$0$]QScriptCore[T, β$0$], [β$1$]Coproduct[[β$2$]ProjectBucket[T, β$2$], [β$3$]Coproduct[[β$4$]ThetaJoin[T, β$4$], [β$5$]Coproduct[[β$6$]EquiJoin[T, β$6$], [β$7$]Coproduct[[β$8$]Const[ShiftedRead[Path[Abs, Dir, Sandboxed]], β$8$], [β$9$]Coproduct[[β$10$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$10$], [β$11$]Coproduct[[β$12$]Const[Read[Path[Abs, Dir, Sandboxed]], β$12$], [β$13$]Coproduct[[β$14$]Const[Read[Path[Abs, File, Sandboxed]], β$14$], [β$15$]Const[DeadEnd, β$15$], β$13$], β$11$], β$9$], β$7$], β$5$], β$3$], β$1$], β$6$]]

Definition Classes
SparkHdfs → BackendModule
def qfKvsOps: Ops[ResultHandle, SparkCursor, Eff]

Definition Classes
SparkCore
def rddFrom: (AFile) ⇒ Configured[RDD[Data]]

Definition Classes
SparkCore
def readScOps: Ops[SparkContext, Eff]

Definition Classes
SparkCore
def rootPrefix(cfg: HdfsConfig): ADir

Definition Classes
SparkHdfs → ChrootedInterpreter
def sequenceOps: Ops[Eff]
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toLowerLevel[S[_]](sc: SparkContext, config: HdfsConfig)(implicit S0: :<:[Task, S], S1: :<:[PhysErr, S]): Task[~>[[β$13$]Free[[A]Coproduct[[β$4$]Read[FileSystem, β$4$], [A]Coproduct[SparkConnectorDetails, [A]Coproduct[MonotonicSeq, [A]Coproduct[[A]Failure[PhysicalError, A], [A]Coproduct[Task, [A]Coproduct[[γ$3$]KeyValueStore[WriteHandle, HdfsWriteCursor, γ$3$], [A]Coproduct[[γ$2$]KeyValueStore[ReadHandle, SparkCursor, γ$2$], [A]Coproduct[[γ$0$]KeyValueStore[ResultHandle, SparkCursor, γ$0$], [β$1$]Read[SparkContext, β$1$], A], A], A], A], A], A], A], A], β$13$], [β$14$]Free[S, β$14$]]]

Definition Classes
SparkHdfs → SparkCore
def toString(): String

Definition Classes
AnyRef → Any
def toTask(sc: SparkContext, config: Config): Task[~>[M, Task]]

Definition Classes
SparkCore
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def writersOps: Ops[WriteHandle, HdfsWriteCursor, Eff]

Related Doc: package hdfs

object SparkHdfs extends SparkCore with ChrootedInterpreter

Type Members

type Backend[A] = EitherT[[β$1$]WriterT[[β$0$]Kleisli[M, Config, β$0$], Vector[PhaseResult], β$1$], FileSystemError, A]

type BackendT[F[_], A] = EitherT[[β$1$]WriterT[[β$0$]Kleisli[F, Config, β$0$], Vector[PhaseResult], β$1$], FileSystemError, A]

type Config = HdfsConfig

type Configured[A] = Kleisli[M, Config, A]

type ConfiguredT[F[_], A] = Kleisli[F, Config, A]

type Eff[A] = Coproduct[[β$4$]Read[FileSystem, β$4$], Eff7, A]

type Eff1[A] = Coproduct[[γ$0$]KeyValueStore[ResultHandle, SparkCursor, γ$0$], [β$1$]Read[SparkContext, β$1$], A]

type Eff2[A] = Coproduct[[γ$2$]KeyValueStore[ReadHandle, SparkCursor, γ$2$], Eff1, A]

type Eff3[A] = Coproduct[[γ$3$]KeyValueStore[WriteHandle, HdfsWriteCursor, γ$3$], Eff2, A]

type Eff4[A] = Coproduct[Task, Eff3, A]

type Eff5[A] = Coproduct[PhysErr, Eff4, A]

type Eff6[A] = Coproduct[MonotonicSeq, Eff5, A]

type Eff7[A] = Coproduct[SparkConnectorDetails, Eff6, A]

implicit class LiftBackend[A] extends AnyRef

implicit class LiftBackendConfigured[A] extends AnyRef

type LowerLevel[A] = Coproduct[Task, PhysErr, A]

type M[A] = Free[Eff, A]

trait ManageFileModule extends AnyRef

type QS[T[_[_]]] = :\:[[β$6$]QScriptCore[T, β$6$], :/:[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$]]]

type QSM[T[_[_]], A] = fp.:\:.M[A]

trait QueryFileModule extends AnyRef

trait ReadFileModule extends AnyRef

type Repr = RDD[Data]

abstract class SparkCoreManageFileModule extends ManageFileModule

trait WriteFileModule extends AnyRef

class details[F[_]] extends AnyRef

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def ExtractPathQSM[T[_[_]]](implicit arg0: RecursiveT[T]): ExtractPath[[β$15$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$15$], APath]

def FunctorQSM[T[_[_]]]: Functor[[β$13$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$13$]]

object HdfsManageFileModule extends SparkCoreManageFileModule

object HdfsWriteFileModule extends WriteFileModule

def ManageFileModule: ManageFileModule

def MonadM: Monad[M]

def MonotonicSeqInj: Inject[MonotonicSeq, Eff]

def QFKeyValueStoreInj: Inject[[γ$12$]KeyValueStore[ResultHandle, SparkCursor, γ$12$], Eff]

def QSCoreInject[T[_[_]]]: :<:[[β$16$]QScriptCore[T, β$16$], [β$17$]Coproduct[[β$6$]QScriptCore[T, β$6$], [A]Coproduct[[β$4$]EquiJoin[T, β$4$], [β$5$]Const[ShiftedRead[Path[Abs, File, Sandboxed]], β$5$], A], β$17$]]

def QueryFileModule: QueryFileModule

def RFKeyValueStoreInj: Inject[[γ$11$]KeyValueStore[ReadHandle, SparkCursor, γ$11$], Eff]

def ReadFileModule: SparkReadFileModule

def ReadSparkContextInj: Inject[[β$10$]Read[SparkContext, β$10$], Eff]

def SparkConnectorDetailsInj: Inject[SparkConnectorDetails, Eff]

object SparkQueryFileModule extends QueryFileModule

object SparkReadFileModule extends ReadFileModule

def TaskInj: Inject[Task, Eff]

val Type: FileSystemType

def UnicoalesceCap[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Capture[T, QS[T]]

def UnirewriteT[T[_[_]]](implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Unirewrite[T, QS[T]]

def WriteFileModule: HdfsWriteFileModule

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def compile(cfg: Config): DefErrT[Task, (~>[M, Task], Task[slamdata.Predef.Unit])]

final def config[F[_]](implicit C: MonadReader_[F, Config]): F[Config]

final val definition: BackendDef[Task]

object details

def detailsOps: Ops[Eff]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def first: (RDD[Data]) ⇒ M[Data]

def generateHdfsFS(sfsConf: HdfsConfig): Task[FileSystem]

def generateSC: (HdfsConfig) ⇒ DefErrT[Task, SparkContext]

final def getClass(): Class[_]

def hashCode(): Int

def hdfsFSOps: Ops[FileSystem, Eff]

def hdfsPathStr(sfsConf: HdfsConfig): (AFile) ⇒ ReaderT[Task, SparkContext, slamdata.Predef.String]

def interpreter(cfg: Config): DefErrT[Task, (~>[BackendEffect, Task], Task[slamdata.Predef.Unit])]

final def isInstanceOf[T0]: Boolean

def lowerToTask: ~>[LowerLevel, Task]

final def lpToQScript[T[_[_]], M[_]](lp: T[LogicalPlan], lc: ListContents[M])(implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T], arg4: Monad[M], arg5: MonadFsErr[M], arg6: PhaseResultTell[M]): M[T[[β$13$]fp.:\:.M[β$13$]]]

final def lpToRepr[T[_[_]]](lp: T[LogicalPlan])(implicit arg0: BirecursiveT[T], arg1: EqualT[T], arg2: ShowT[T], arg3: RenderTreeT[T]): Backend[PhysicalPlan[Repr]]

def msOps: Ops[Eff]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit