Packages

p

org.apache.spark.sql

executionmetrics

package executionmetrics

data schema is in snake case spark doesn't gives us a native converter from snake case to camel case, we have to do that in this class.

Linear Supertypes
AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. executionmetrics
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. case class ComponentRuns(uid: String, component_uri: String, pipeline_uri: String, pipeline_run_uid: String, fabric_uid: String, component_name: String, interim_component_name: String, component_type: String, interim_subgraph_name: String, interim_process_id: String, interim_out_port: String, created_at: Option[Timestamp] = None, created_by: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, expired: Option[Boolean] = Some(false), run_type: Option[String], job_uri: Option[String], branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable
  2. case class ComponentRunsWithRunDates(componentRunsWithStatusAndInterims: Option[List[ComponentRunsWithStatusAndInterims]] = None, runDates: Option[List[RunDates]] = None) extends Product with Serializable
  3. case class ComponentRunsWithStatus(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], job_uri: Option[String] = None, run_type: Option[String] = None, branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable
  4. case class ComponentRunsWithStatusAndInterims(uid: String, component_uri: String, pipeline_run_uid: String, pipeline_uri: String, fabric_uid: String, component_name: String, interim_component_name: String, records: Option[Long] = None, bytes: Option[Long] = None, partitions: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, component_type: String, interim_out_port: String, interim_subgraph_name: String, interim_process_id: String, expired: Option[Boolean] = Some(false), status: Option[String], interim: Option[String] = None, job_uri: Option[String] = None, run_type: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable
  5. case class DatasetRunsResponseCamelCase(uid: String, datasetUID: String, datasetName: Option[String] = None, pipelineUID: String, pipelineName: Option[String] = None, fabric: Option[Fabric], submissionTime: Option[Timestamp] = None, pipelineRun: PipelineRun, componentName: String, componentType: String, recordsProcessed: Option[Long] = None, status: Option[String] = None, interims: Option[String] = None, runDates: Option[Seq[RunDates]] = None, runType: Option[String] = None, jobUID: Option[String] = None, jobName: Option[String] = None, bytes: Option[Long], partition: Option[Long], branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable
  6. trait ExecutionMetricsDAO[A <: ExecutionMetricsEntity] extends AnyRef
  7. sealed trait ExecutionMetricsEntity extends AnyRef

    TODO Atm we hopefully should have something related to spark which helps in expressionEncoder to keep cases in camelCase rather than snake_case.

  8. case class Fabric(uid: String, name: Option[String] = None) extends Product with Serializable
  9. case class FileContent(path: String, content: String) extends Product with Serializable
  10. case class HistoricalViewCodeResponse(pipelineId: String, pipelineRun: PipelineRunsResponseCamelCase) extends Product with Serializable
  11. case class InMemoryStore(uuid: String, sparkSession: SparkSession)(implicit ec: ExecutionContext) extends LazyLogging with Product with Serializable
  12. case class InterimResponse(uid: String, interim_component_name: String, interim_out_port: String, interim_process_id: String, interim: String) extends Product with Serializable
  13. case class InterimResponseCamelCase(uid: String, interimComponentName: String, interimOutPort: String, interimProcessId: String, interim: String, submissionTime: Option[Timestamp] = None) extends ResponseCamelCase with Product with Serializable
  14. implicit class LoggingSparkSession extends LazyLogging
  15. case class NextFilters(lastSubmissionTimeInMs: Timestamp, lastUid: String) extends Product with Serializable
  16. case class PipelineRun(uid: String) extends Product with Serializable
  17. case class PipelineRuns(uid: String, pipeline_uri: String, job_uri: Option[String] = None, job_run_uid: String, task_run_uid: String, status: String, fabric_uid: String, time_taken: Option[Long] = None, rows_read: Option[Long] = None, rows_written: Option[Long] = None, created_at: Option[Timestamp] = None, created_by: String, run_type: String, input_datasets: Option[Array[String]] = None, output_datasets: Option[Array[String]] = None, workflow_code: Option[Map[String, String]] = None, expired: Option[Boolean] = Some(false), branch: Option[String] = None) extends ExecutionMetricsEntity with Product with Serializable
  18. case class PipelineRunsResponseCamelCase(pipelineRunId: String, pipelineUID: String, jobUID: Option[String] = None, jobName: Option[String] = None, fabric: Option[Fabric] = None, fabricId: Option[Long] = None, submissionTime: Option[Timestamp], status: String, timeTaken: Option[Long] = None, rowsRead: Option[Long] = None, rowsWritten: Option[Long] = None, runType: String, code: Option[Map[String, String]] = None, branch: Option[String] = None) extends ResponseCamelCase with Product with Serializable
  19. sealed trait ResponseCamelCase extends AnyRef
  20. implicit class ResponseWrapperAsList[T <: ResponseCamelCase] extends AnyRef
  21. case class ResponsesAsList[T <: ResponseCamelCase](rows: List[T], limit: Int, next_filters: Option[NextFilters]) extends Product with Serializable
  22. case class RunDates(uid: String, runId: String, submissionTime: Timestamp) extends Product with Serializable
  23. sealed trait RunningMode extends EnumEntry

Value Members

  1. val Branch: String
  2. val Bytes: String
  3. val ComponentName: String
  4. val ComponentType: String
  5. val ComponentUri: String
  6. val CreatedAt: String
  7. val CreatedBy: String
  8. val Expired: String
  9. val FabricUID: String
  10. val InputDatasets: String
  11. val Interim: String
  12. val InterimComponentName: String
  13. val InterimOutPort: String
  14. val InterimProcessId: String
  15. val InterimSubgraphName: String
  16. val JobRunUid: String
  17. val JobUri: String
  18. val OutputDatasets: String
  19. val Partitions: String
  20. val PipelineRunUid: String
  21. val PipelineUri: String
  22. val Records: String
  23. val RowsRead: String
  24. val RowsWritten: String
  25. val RunTypeColumn: String
  26. val Status: String
  27. val TaskRunUid: String
  28. val TimeTaken: String
  29. val UID: String
  30. val WorkflowCode: String
  31. val WorkflowJson: String
  32. def checkExpiredRowPf[T <: ExecutionMetricsEntity](uid: String): PartialFunction[T, T]
  33. def error(msg: String, cause: Option[Throwable] = None): Nothing
  34. def now(): Timestamp
  35. def onFail(uid: String): Nothing
  36. implicit lazy val timestampFormat: Format[Timestamp]
  37. object ComponentRuns extends Serializable
  38. object ComponentRunsWithStatus extends Serializable
  39. object ComponentRunsWithStatusAndInterims extends Serializable
  40. object DatasetRunsResponseCamelCase extends Serializable
  41. object Fabric extends Serializable
  42. object FileContent extends Serializable
  43. object HistoricalViewCodeResponse extends Serializable
  44. object InterimResponse extends Serializable
  45. object InterimResponseCamelCase extends Serializable
  46. object NextFilters extends Serializable
  47. object PipelineRun extends Serializable
  48. object PipelineRuns extends Serializable
  49. object PipelineRunsResponseCamelCase extends Serializable
  50. object ResponseCamelCase
  51. object ResponsesAsList extends Serializable
  52. object RunDates extends Serializable
  53. object RunningMode extends Enum[RunningMode] with PlayJsonEnum[RunningMode]
  54. object ZipFileExtractor extends LazyLogging

Inherited from AnyRef

Inherited from Any

Ungrouped