quasar
.
physical
.
sparkcore
.
fs
.
hdfs
SparkHdfs
Related Doc:
package hdfs
object
SparkHdfs
extends
SparkCore
with
ChrootedInterpreter
Linear Supertypes
ChrootedInterpreter
,
SparkCore
,
DefaultAnalyzeModule
,
BackendModule
,
AnyRef
,
Any
Ordering
Alphabetic
By Inheritance
Inherited
SparkHdfs
ChrootedInterpreter
SparkCore
DefaultAnalyzeModule
BackendModule
AnyRef
Any
Hide All
Show All
Visibility
Public
All
Type Members
trait
AnalyzeModule
extends
AnyRef
Definition Classes
BackendModule
type
Backend
[
A
]
=
EitherT
[[β$1$]
WriterT
[[β$0$]
Kleisli
[
M
,
Config
,
β$0$
],
Vector
[
PhaseResult
],
β$1$
],
FileSystemError
,
A
]
Definition Classes
BackendModule
type
BackendT
[
F
[
_
]
,
A
]
=
EitherT
[[β$1$]
WriterT
[[β$0$]
Kleisli
[
F
,
Config
,
β$0$
],
Vector
[
PhaseResult
],
β$1$
],
FileSystemError
,
A
]
Definition Classes
BackendModule
type
Config
=
HdfsConfig
Definition Classes
SparkHdfs
→ BackendModule
type
Configured
[
A
]
=
Kleisli
[
M
,
Config
,
A
]
Definition Classes
BackendModule
type
ConfiguredT
[
F
[
_
]
,
A
]
=
Kleisli
[
F
,
Config
,
A
]
Definition Classes
BackendModule
type
Eff
[
A
]
=
Coproduct
[[β$4$]
Read
[
FileSystem
,
β$4$
],
Eff7
,
A
]
Definition Classes
SparkHdfs
→
SparkCore
type
Eff1
[
A
]
=
Coproduct
[[γ$0$]
KeyValueStore
[
ResultHandle
,
SparkCursor
,
γ$0$
], [β$1$]
Read
[
SparkContext
,
β$1$
],
A
]
type
Eff2
[
A
]
=
Coproduct
[[γ$2$]
KeyValueStore
[
ReadHandle
,
SparkCursor
,
γ$2$
],
Eff1
,
A
]
type
Eff3
[
A
]
=
Coproduct
[[γ$3$]
KeyValueStore
[
WriteHandle
,
HdfsWriteCursor
,
γ$3$
],
Eff2
,
A
]
type
Eff4
[
A
]
=
Coproduct
[
Task
,
Eff3
,
A
]
type
Eff5
[
A
]
=
Coproduct
[
PhysErr
,
Eff4
,
A
]
type
Eff6
[
A
]
=
Coproduct
[
MonotonicSeq
,
Eff5
,
A
]
type
Eff7
[
A
]
=
Coproduct
[
SparkConnectorDetails
,
Eff6
,
A
]
implicit
class
LiftBackend
[
A
]
extends
AnyRef
Definition Classes
BackendModule
implicit
class
LiftBackendConfigured
[
A
]
extends
AnyRef
Definition Classes
BackendModule
type
LowerLevel
[
A
]
=
Coproduct
[
Task
,
PhysErr
,
A
]
Definition Classes
SparkCore
type
M
[
A
]
=
Free
[
Eff
,
A
]
Definition Classes
SparkCore
→ BackendModule
trait
ManageFileModule
extends
AnyRef
Definition Classes
BackendModule
type
QS
[
T
[
_
[
_
]
]
]
=
:\:
[[β$2$]
QScriptCore
[
T
,
β$2$
],
:/:
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
]]]
Definition Classes
SparkCore
→ BackendModule
type
QSM
[
T
[
_
[
_
]
]
,
A
]
=
fp.:\:.M
[
A
]
Definition Classes
BackendModule
trait
QueryFileModule
extends
AnyRef
Definition Classes
BackendModule
trait
ReadFileModule
extends
AnyRef
Definition Classes
BackendModule
type
Repr
=
RDD
[
Data
]
Definition Classes
SparkCore
→ BackendModule
abstract
class
SparkCoreManageFileModule
extends
ManageFileModule
Definition Classes
SparkCore
trait
WriteFileModule
extends
AnyRef
Definition Classes
BackendModule
class
details
[
F
[
_
]
]
extends
AnyRef
Value Members
final
def
!=
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
final
def
##
()
:
Int
Definition Classes
AnyRef → Any
final
def
==
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
def
CardinalityQSM
:
Cardinality
[[β$12$]
Coproduct
[[β$2$]
QScriptCore
[
Fix
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
Fix
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$12$
]]
Definition Classes
SparkCore
→ DefaultAnalyzeModule
def
CostQSM
:
Cost
[[β$14$]
Coproduct
[[β$2$]
QScriptCore
[
Fix
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
Fix
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$14$
]]
Definition Classes
SparkCore
→ DefaultAnalyzeModule
def
DelayRenderTreeQSM
[
T
[
_
[
_
]
]
]
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
)
:
Delay
[
RenderTree
, [β$18$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$18$
]]
Definition Classes
SparkCore
→ BackendModule
def
ExtractPathQSM
[
T
[
_
[
_
]
]
]
(
implicit
arg0:
RecursiveT
[
T
]
)
:
ExtractPath
[[β$19$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$19$
],
APath
]
Definition Classes
SparkCore
→ BackendModule
def
FunctorQSM
[
T
[
_
[
_
]
]
]
:
Functor
[[β$16$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$16$
]]
Definition Classes
SparkCore
→ BackendModule
object
HdfsManageFileModule
extends
SparkCoreManageFileModule
object
HdfsWriteFileModule
extends
WriteFileModule
def
ManageFileModule
:
ManageFileModule
Definition Classes
SparkHdfs
→ BackendModule
def
MonadM
:
Monad
[
M
]
Definition Classes
SparkCore
→ BackendModule
def
MonotonicSeqInj
:
Inject
[
MonotonicSeq
,
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
def
QFKeyValueStoreInj
:
Inject
[[γ$7$]
KeyValueStore
[
ResultHandle
,
SparkCursor
,
γ$7$
],
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
def
QSCoreInject
[
T
[
_
[
_
]
]
]
:
:<:
[[β$20$]
QScriptCore
[
T
,
β$20$
], [β$21$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$21$
]]
Definition Classes
SparkCore
→ BackendModule
def
QueryFileModule
:
QueryFileModule
Definition Classes
SparkCore
→ BackendModule
def
RFKeyValueStoreInj
:
Inject
[[γ$6$]
KeyValueStore
[
ReadHandle
,
SparkCursor
,
γ$6$
],
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
def
ReadFileModule
:
SparkReadFileModule
Definition Classes
SparkCore
→ BackendModule
def
ReadSparkContextInj
:
Inject
[[β$5$]
Read
[
SparkContext
,
β$5$
],
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
def
SparkConnectorDetailsInj
:
Inject
[
SparkConnectorDetails
,
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
object
SparkQueryFileModule
extends
QueryFileModule
Definition Classes
SparkCore
object
SparkReadFileModule
extends
ReadFileModule
Definition Classes
SparkCore
def
TaskInj
:
Inject
[
Task
,
Eff
]
Definition Classes
SparkHdfs
→
SparkCore
def
TraverseQSM
[
T
[
_
[
_
]
]
]
:
Traverse
[[β$17$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$17$
]]
Definition Classes
SparkCore
→ DefaultAnalyzeModule
val
Type
:
FileSystemType
Definition Classes
SparkHdfs
→ BackendModule
def
UnicoalesceCap
[
T
[
_
[
_
]
]
]
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
)
:
Capture
[
T
,
QS
[
T
]]
Definition Classes
SparkCore
→ BackendModule
def
UnirewriteT
[
T
[
_
[
_
]
]
]
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
)
:
Unirewrite
[
T
,
QS
[
T
]]
Definition Classes
SparkCore
→ BackendModule
def
WriteFileModule
:
HdfsWriteFileModule
Definition Classes
SparkHdfs
→ BackendModule
final
def
asInstanceOf
[
T0
]
:
T0
Definition Classes
Any
def
clone
()
:
AnyRef
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
...
)
def
compile
(
cfg:
Config
)
:
DefErrT
[
Task
, (
~>
[
M
,
Task
],
Task
[
slamdata.Predef.Unit
])]
Definition Classes
SparkCore
→ BackendModule
final
def
config
[
F
[
_
]
]
(
implicit
C:
MonadReader_
[
F
,
Config
]
)
:
F
[
Config
]
Definition Classes
BackendModule
final
val
definition
:
BackendDef
[
Task
]
Definition Classes
BackendModule
object
details
def
detailsOps
:
Ops
[
Eff
]
Definition Classes
SparkCore
final
def
eq
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
def
equals
(
arg0:
Any
)
:
Boolean
Definition Classes
AnyRef → Any
def
finalize
()
:
Unit
Attributes
protected[
java.lang
]
Definition Classes
AnyRef
Annotations
@throws
(
classOf[java.lang.Throwable]
)
def
first
: (
RDD
[
Data
]) ⇒
M
[
Data
]
Definition Classes
SparkCore
def
generateHdfsFS
(
sfsConf:
HdfsConfig
)
:
Task
[
FileSystem
]
Annotations
@SuppressWarnings
()
def
generateSC
: (
APath
,
HdfsConfig
) ⇒
DefErrT
[
Task
,
SparkContext
]
Definition Classes
SparkHdfs
→
SparkCore
final
def
getClass
()
:
Class
[_]
Definition Classes
AnyRef → Any
def
getSparkConf
: (
Config
) ⇒
SparkConf
Definition Classes
SparkHdfs
→
SparkCore
def
hashCode
()
:
Int
Definition Classes
AnyRef → Any
def
hdfsFSOps
:
Ops
[
FileSystem
,
Eff
]
def
hdfsPathStr
(
sfsConf:
HdfsConfig
)
: (
AFile
) ⇒
ReaderT
[
Task
,
SparkContext
,
slamdata.Predef.String
]
def
initSC
: (
Config
) ⇒
DefErrT
[
Task
,
SparkContext
]
Definition Classes
SparkCore
def
interpreter
(
cfg:
Config
)
:
DefErrT
[
Task
, (
~>
[
BackendEffect
,
Task
],
Task
[
slamdata.Predef.Unit
])]
Definition Classes
ChrootedInterpreter → BackendModule
final
def
isInstanceOf
[
T0
]
:
Boolean
Definition Classes
Any
def
lowerToTask
:
~>
[
LowerLevel
,
Task
]
Definition Classes
SparkCore
final
def
lpToQScript
[
T
[
_
[
_
]
]
,
M
[
_
]
]
(
lp:
T
[
LogicalPlan
]
,
lc:
ListContents
[
M
]
)
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
,
arg4:
Monad
[
M
]
,
arg5:
MonadFsErr
[
M
]
,
arg6:
PhaseResultTell
[
M
]
)
:
M
[
T
[[β$16$]
fp.:\:.M
[
β$16$
]]]
Definition Classes
BackendModule
final
def
lpToRepr
[
T
[
_
[
_
]
]
]
(
lp:
T
[
LogicalPlan
]
)
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
)
:
Backend
[
PhysicalPlan
[
Repr
]]
Definition Classes
BackendModule
def
msOps
:
Ops
[
Eff
]
Definition Classes
SparkCore
final
def
ne
(
arg0:
AnyRef
)
:
Boolean
Definition Classes
AnyRef
final
def
notify
()
:
Unit
Definition Classes
AnyRef
final
def
notifyAll
()
:
Unit
Definition Classes
AnyRef
def
optimize
[
T
[
_
[
_
]
]
]
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
)
: (
QSM
[
T
,
T
[[β$9$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$9$
]]]) ⇒
QSM
[
T
,
T
[[β$10$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$10$
]]]
Definition Classes
SparkCore
→ BackendModule
def
parseConfig
(
connUri:
ConnectionUri
)
:
DefErrT
[
Task
,
HdfsConfig
]
Definition Classes
SparkHdfs
→ BackendModule
def
plan
[
T
[
_
[
_
]
]
]
(
cp:
T
[[β$23$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$23$
]]
)
(
implicit
arg0:
BirecursiveT
[
T
]
,
arg1:
EqualT
[
T
]
,
arg2:
ShowT
[
T
]
,
arg3:
RenderTreeT
[
T
]
)
:
Backend
[
Repr
]
Definition Classes
SparkCore
→ BackendModule
implicit
def
qScriptToQScriptTotal
[
T
[
_
[
_
]
]
]
:
Aux
[[β$4$]
Coproduct
[[β$2$]
QScriptCore
[
T
,
β$2$
], [A]
Coproduct
[[β$0$]
EquiJoin
[
T
,
β$0$
], [β$1$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$1$
],
A
],
β$4$
], [β$5$]
Coproduct
[[β$0$]
QScriptCore
[
T
,
β$0$
], [β$1$]
Coproduct
[[β$2$]
ProjectBucket
[
T
,
β$2$
], [β$3$]
Coproduct
[[β$4$]
ThetaJoin
[
T
,
β$4$
], [β$5$]
Coproduct
[[β$6$]
EquiJoin
[
T
,
β$6$
], [β$7$]
Coproduct
[[β$8$]
Const
[
ShiftedRead
[
Path
[
Abs
,
Dir
,
Sandboxed
]],
β$8$
], [β$9$]
Coproduct
[[β$10$]
Const
[
ShiftedRead
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$10$
], [β$11$]
Coproduct
[[β$12$]
Const
[
Read
[
Path
[
Abs
,
Dir
,
Sandboxed
]],
β$12$
], [β$13$]
Coproduct
[[β$14$]
Const
[
Read
[
Path
[
Abs
,
File
,
Sandboxed
]],
β$14$
], [β$15$]
Const
[
DeadEnd
,
β$15$
],
β$13$
],
β$11$
],
β$9$
],
β$7$
],
β$5$
],
β$3$
],
β$1$
],
β$5$
]]
Definition Classes
SparkCore
→ BackendModule
def
qfKvsOps
:
Ops
[
ResultHandle
,
SparkCursor
,
Eff
]
Definition Classes
SparkCore
def
rddFrom
: (
AFile
) ⇒
Configured
[
RDD
[
Data
]]
Definition Classes
SparkCore
def
readScOps
:
Ops
[
SparkContext
,
Eff
]
Definition Classes
SparkCore
def
rootPrefix
(
cfg:
HdfsConfig
)
:
ADir
Definition Classes
SparkHdfs
→ ChrootedInterpreter
def
sequenceOps
:
Ops
[
Eff
]
def
sparkCoreJar
:
DefErrT
[
Task
,
APath
]
Definition Classes
SparkCore
final
def
synchronized
[
T0
]
(
arg0: ⇒
T0
)
:
T0
Definition Classes
AnyRef
def
toLowerLevel
[
S
[
_
]
]
(
sc:
SparkContext
,
config:
HdfsConfig
)
(
implicit
S0:
:<:
[
Task
,
S
]
,
S1:
:<:
[
PhysErr
,
S
]
)
:
Task
[
~>
[[β$8$]
Free
[[A]
Coproduct
[[β$4$]
Read
[
FileSystem
,
β$4$
], [A]
Coproduct
[
SparkConnectorDetails
, [A]
Coproduct
[
MonotonicSeq
, [A]
Coproduct
[[A]
Failure
[
PhysicalError
,
A
], [A]
Coproduct
[
Task
, [A]
Coproduct
[[γ$3$]
KeyValueStore
[
WriteHandle
,
HdfsWriteCursor
,
γ$3$
], [A]
Coproduct
[[γ$2$]
KeyValueStore
[
ReadHandle
,
SparkCursor
,
γ$2$
], [A]
Coproduct
[[γ$0$]
KeyValueStore
[
ResultHandle
,
SparkCursor
,
γ$0$
], [β$1$]
Read
[
SparkContext
,
β$1$
],
A
],
A
],
A
],
A
],
A
],
A
],
A
],
A
],
β$8$
], [β$9$]
Free
[
S
,
β$9$
]]]
Definition Classes
SparkHdfs
→
SparkCore
def
toString
()
:
String
Definition Classes
AnyRef → Any
def
toTask
(
sc:
SparkContext
,
config:
Config
)
:
Task
[
~>
[
M
,
Task
]]
Definition Classes
SparkCore
final
def
wait
()
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
,
arg1:
Int
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
final
def
wait
(
arg0:
Long
)
:
Unit
Definition Classes
AnyRef
Annotations
@throws
(
...
)
def
writersOps
:
Ops
[
WriteHandle
,
HdfsWriteCursor
,
Eff
]
Inherited from
ChrootedInterpreter
Inherited from
SparkCore
Inherited from
DefaultAnalyzeModule
Inherited from
BackendModule
Inherited from
AnyRef
Inherited from
Any
Ungrouped