PySparkInterpreter

Instance Constructors

new PySparkInterpreter(_compiler: ScalaCompiler, jepInstance: Jep, jepExecutor: Executor, jepThread: AtomicReference[Thread], jepBlockingService: Blocking, runtime: Runtime[Any], pyApi: PythonAPI, venvPath: Option[Path])

Type Members

case class PythonState extends State with Product with Serializable

Definition Classes
PythonInterpreter

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def compile(parsed: PyObject, cell: String): Task[PyObject]

Attributes
protected
Definition Classes
PythonInterpreter
val compiler: ScalaCompiler

Definition Classes
PythonInterpreter
def completionsAt(code: String, pos: Int, state: State): Task[List[Completion]]

Definition Classes
PythonInterpreter → Interpreter
def convertFromPython(jep: Jep): PartialFunction[(String, PyObject), (scala.tools.nsc.interactive.Global.Type, Any)]

Attributes
protected
Definition Classes
PySparkInterpreter → PythonInterpreter
def convertToPython(jep: Jep): PartialFunction[(String, Any), AnyRef]

Attributes
protected
Definition Classes
PySparkInterpreter → PythonInterpreter
def defaultConvertToPython(nv: (String, Any)): AnyRef

Attributes
protected
Definition Classes
PythonInterpreter
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def errorCause(get: PyCallable): Option[Throwable]

Attributes
protected
Definition Classes
PySparkInterpreter → PythonInterpreter
def eval[T](code: String)(implicit arg0: ClassTag[T]): Task[T]

Attributes
protected[polynote.kernel.interpreter.python]
Definition Classes
PythonInterpreter
def exec(code: String): Task[Unit]

Attributes
protected[polynote.kernel.interpreter.python]
Definition Classes
PythonInterpreter
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
val gatewayRef: AtomicReference[GatewayServer]
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getValue(name: String): Task[PyObject]

Attributes
protected[polynote.kernel.interpreter.python]
Definition Classes
PythonInterpreter
def handlePyError(get: PyCallable, trace: ArrayList[AnyRef]): Throwable

Definition Classes
PythonInterpreter
def hashCode(): Int

Definition Classes
AnyRef → Any
def importFutureAnnotations: String

Attributes
protected
Definition Classes
PythonInterpreter
def init(state: State): RIO[InterpreterEnv, State]

Definition Classes
PySparkInterpreter → PythonInterpreter → Interpreter
def injectGlobals(globals: PyObject): RIO[CurrentRuntime, Unit]

Attributes
protected
Definition Classes
PySparkInterpreter → PythonInterpreter
def isFutureAnnotationsSupported: Task[Boolean]

Attributes
protected
Definition Classes
PythonInterpreter
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def jep[T](fn: (Jep) ⇒ T): Task[T]

Attributes
protected[polynote.kernel.interpreter.python]
Definition Classes
PythonInterpreter
def matplotlib: String

Attributes
protected
Definition Classes
PythonInterpreter
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def parametersAt(code: String, pos: Int, state: State): Task[Option[Signatures]]

Definition Classes
PythonInterpreter → Interpreter
def parse(code: String, cell: String): Task[PyObject]

Attributes
protected
Definition Classes
PythonInterpreter
def populateGlobals(state: State): Task[PyObject]

Attributes
protected
Definition Classes
PythonInterpreter
def pysparkImports: String

Handle setting up PySpark.
Handle setting up PySpark.
First, we need to pick the python interpreter. Unfortunately this means we need to re-implement Spark's interpreter configuration logic, because that's only implemented inside SparkSubmit (and only when you use pyspark-shell actually).
Here's the order we follow for the driver python executable (from org.apache.spark.launcher.SparkSubmitCommandBuilder):
1. conf spark.pyspark.driver.python 2. conf spark.pyspark.python 3. environment variable PYSPARK_DRIVER_PYTHON 4. environment variable PYSPARK_PYTHON
For the executors we just omit the driver python - so it's just:
1. conf spark.pyspark.python 2. environment variable PYSPARK_PYTHON
Additionally, to load pyspark itself we try to grab the its location from the Spark distribution. This ensures that all the versions match up.
WARNING: Using pyspark from pip install pyspark, could break things - don't use it!
Attributes
protected
def run(compiled: PyObject, globals: PyObject, state: State): RIO[CurrentRuntime, State]

Attributes
protected
Definition Classes
PythonInterpreter
def run(code: String, state: State): RIO[InterpreterEnv, State]

Definition Classes
PythonInterpreter → Interpreter
def setValue(name: String, value: AnyRef): Task[Unit]

Attributes
protected[polynote.kernel.interpreter.python]
Definition Classes
PythonInterpreter
def setup(isFutureAnnotationsSupported: Boolean): String

Attributes
protected
Definition Classes
PythonInterpreter
def shutdown(): Task[Unit]

Definition Classes
PythonInterpreter → Interpreter
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Docs: object PySparkInterpreter | package python

class PySparkInterpreter extends PythonInterpreter

Instance Constructors

new PySparkInterpreter(_compiler: ScalaCompiler, jepInstance: Jep, jepExecutor: Executor, jepThread: AtomicReference[Thread], jepBlockingService: Blocking, runtime: Runtime[Any], pyApi: PythonAPI, venvPath: Option[Path])

Type Members

case class PythonState extends State with Product with Serializable

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def compile(parsed: PyObject, cell: String): Task[PyObject]

val compiler: ScalaCompiler

def completionsAt(code: String, pos: Int, state: State): Task[List[Completion]]

def convertFromPython(jep: Jep): PartialFunction[(String, PyObject), (scala.tools.nsc.interactive.Global.Type, Any)]

def convertToPython(jep: Jep): PartialFunction[(String, Any), AnyRef]

def defaultConvertToPython(nv: (String, Any)): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def errorCause(get: PyCallable): Option[Throwable]

def eval[T](code: String)(implicit arg0: ClassTag[T]): Task[T]

def exec(code: String): Task[Unit]

def finalize(): Unit

val gatewayRef: AtomicReference[GatewayServer]

final def getClass(): Class[_]

def getValue(name: String): Task[PyObject]

def handlePyError(get: PyCallable, trace: ArrayList[AnyRef]): Throwable

def hashCode(): Int

def importFutureAnnotations: String

def init(state: State): RIO[InterpreterEnv, State]

def injectGlobals(globals: PyObject): RIO[CurrentRuntime, Unit]

def isFutureAnnotationsSupported: Task[Boolean]

final def isInstanceOf[T0]: Boolean

def jep[T](fn: (Jep) ⇒ T): Task[T]

def matplotlib: String

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def parametersAt(code: String, pos: Int, state: State): Task[Option[Signatures]]

def parse(code: String, cell: String): Task[PyObject]

def populateGlobals(state: State): Task[PyObject]

def pysparkImports: String

def run(compiled: PyObject, globals: PyObject, state: State): RIO[CurrentRuntime, State]

def run(code: String, state: State): RIO[InterpreterEnv, State]

def setValue(name: String, value: AnyRef): Task[Unit]

def setup(isFutureAnnotationsSupported: Boolean): String

def shutdown(): Task[Unit]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from PythonInterpreter

Inherited from Interpreter

Inherited from AnyRef

Inherited from Any

Ungrouped