JobManagerActor

The JobManager actor supervises jobs running in a single SparkContext, as well as shared metadata. It creates a SparkContext (or a StreamingContext etc. depending on the factory class) It also creates and supervises a JobResultActor and JobStatusActor, although an existing JobResultActor can be passed in as well.

contextConfig

num-cpu-cores = 4         # Total # of CPU cores to allocate across the cluster
memory-per-node = 512m    # -Xmx style memory string for total memory to use for executor on one node
dependent-jar-uris = ["local://opt/foo/my-foo-lib.jar"]
                          # URIs for dependent jars to load for entire context
context-factory = "spark.jobserver.context.DefaultSparkContextFactory"
spark.mesos.coarse = true  # per-context, rather than per-job, resource allocation
rdd-ttl = 24 h            # time-to-live for RDDs in a SparkContext.  Don't specify = forever
is-adhoc = false          # true if context is ad-hoc context
context.name = "sql"      # Name of context

global configuration

spark {
  jobserver {
    max-jobs-per-context = 16      # Number of jobs that can be run simultaneously per context
  }
}

Linear Supertypes

InstrumentedActor, ActorMetrics, Slf4jLogging, ActorStack, Actor, AnyRef, Any

Instance Constructors

new JobManagerActor(contextConfig: Config)

Type Members

type Receive = PartialFunction[Any, Unit]

Definition Classes
Actor

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def aroundPostRestart(reason: Throwable): Unit

Attributes
protected[akka]
Definition Classes
Actor
def aroundPostStop(): Unit

Attributes
protected[akka]
Definition Classes
Actor
def aroundPreRestart(reason: Throwable, message: Option[Any]): Unit

Attributes
protected[akka]
Definition Classes
Actor
def aroundPreStart(): Unit

Attributes
protected[akka]
Definition Classes
Actor
def aroundReceive(receive: akka.actor.Actor.Receive, msg: Any): Unit

Attributes
protected[akka]
Definition Classes
Actor
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val config: Config
implicit val context: ActorContext

Definition Classes
Actor
def createContextFromConfig(contextName: String = contextName): ContextLike
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
val executionContext: ExecutionContextExecutorService
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getRootCause(t: Throwable): Throwable

Attributes
protected
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
var jobContext: ContextLike
val logger: Logger

Definition Classes
Slf4jLogging
val metricReceiveTimer: Timer

Definition Classes
ActorMetrics
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def postRestart(reason: Throwable): Unit

Definition Classes
Actor
Annotations
@throws( classOf[java.lang.Exception] )
def postStop(): Unit

postStop() is called when actor is stopped or restarted due to Exceptions *
postStop() is called when actor is stopped or restarted due to Exceptions *

Definition Classes
JobManagerActor → InstrumentedActor → Actor
def preRestart(reason: Throwable, message: Option[Any]): Unit

preRestart() is called when actor is killed due to exception, and will be restarted.
preRestart() is called when actor is killed due to exception, and will be restarted. It is run on the current actor instance that is about to be killed. We just log errors. The super (original) method should call postStop() and shut down children as well.

Definition Classes
InstrumentedActor → Actor
def preStart(): Unit

Definition Classes
Actor
Annotations
@throws( classOf[java.lang.Exception] )
def receive: Receive

Stackable traits should override and call super.receive(x) for stacking functionality
Stackable traits should override and call super.receive(x) for stacking functionality

Definition Classes
ActorMetrics → ActorStack → Actor
var resultActor: ActorRef

Attributes
protected
implicit final val self: ActorRef

Definition Classes
Actor
final def sender(): ActorRef

Definition Classes
Actor
var sparkEnv: SparkEnv
def startJobInternal(appName: String, classPath: String, jobConfig: Config, events: Set[Class[_]], jobContext: ContextLike, sparkEnv: SparkEnv): Option[Future[Any]]
def supervisorStrategy: SupervisorStrategy

Definition Classes
Actor
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def unhandled(message: Any): Unit

Definition Classes
Actor
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def wrapInRuntimeException(t: Throwable): RuntimeException

Attributes
protected
def wrappedReceive: Receive

Actor classes should implement this partialFunction for standard actor message handling
Actor classes should implement this partialFunction for standard actor message handling

Definition Classes
JobManagerActor → ActorStack

Related Docs: object JobManagerActor | package jobserver

class JobManagerActor extends InstrumentedActor

contextConfig

global configuration

Instance Constructors

new JobManagerActor(contextConfig: Config)

Type Members

type Receive = PartialFunction[Any, Unit]

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def aroundPostRestart(reason: Throwable): Unit

def aroundPostStop(): Unit

def aroundPreRestart(reason: Throwable, message: Option[Any]): Unit

def aroundPreStart(): Unit

def aroundReceive(receive: akka.actor.Actor.Receive, msg: Any): Unit

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val config: Config

implicit val context: ActorContext

def createContextFromConfig(contextName: String = contextName): ContextLike

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

val executionContext: ExecutionContextExecutorService

def finalize(): Unit

final def getClass(): Class[_]

def getRootCause(t: Throwable): Throwable

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

var jobContext: ContextLike

val logger: Logger

val metricReceiveTimer: Timer

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def postRestart(reason: Throwable): Unit

def postStop(): Unit

def preRestart(reason: Throwable, message: Option[Any]): Unit

def preStart(): Unit

def receive: Receive

var resultActor: ActorRef

implicit final val self: ActorRef

final def sender(): ActorRef

var sparkEnv: SparkEnv

def startJobInternal(appName: String, classPath: String, jobConfig: Config, events: Set[Class[_]], jobContext: ContextLike, sparkEnv: SparkEnv): Option[Future[Any]]

def supervisorStrategy: SupervisorStrategy

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def unhandled(message: Any): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def wrapInRuntimeException(t: Throwable): RuntimeException

def wrappedReceive: Receive

Inherited from InstrumentedActor

Inherited from ActorMetrics

Inherited from Slf4jLogging

Inherited from ActorStack

Inherited from Actor

Inherited from AnyRef

Inherited from Any

Ungrouped