case class ActionPipelineContext(feed: String, application: String, executionId: SDLExecutionId, instanceRegistry: InstanceRegistry, referenceTimestamp: Option[LocalDateTime] = None, appConfig: SmartDataLakeBuilderConfig, runStartTime: LocalDateTime = LocalDateTime.now(), attemptStartTime: LocalDateTime = LocalDateTime.now(), simulation: Boolean = false, phase: ExecutionPhase = ExecutionPhase.Prepare, dataFrameReuseStatistics: Map[(DataObjectId, Seq[PartitionValues]), Seq[ActionId]] = mutable.Map(), actionsSelected: Seq[ActionId] = Seq(), actionsSkipped: Seq[ActionId] = Seq(), serializableHadoopConf: SerializableHadoopConfiguration, globalConfig: GlobalConfig) extends SmartDataLakeLogger with Product with Serializable
ActionPipelineContext contains start and runtime information about a SmartDataLake run.
- feed
feed selector of the run
- application
application name of the run
- executionId
SDLExecutionId of this runs. Contains runId and attemptId. Both stay 1 if state is not enabled.
- instanceRegistry
registry of all SmartDataLake objects parsed from the config
- referenceTimestamp
timestamp used as reference in certain actions (e.g. HistorizeAction)
- appConfig
the command line parameters parsed into a SmartDataLakeBuilderConfig object
- runStartTime
start time of the run
- attemptStartTime
start time of attempt
- simulation
true if this is a simulation run
- phase
current execution phase
- dataFrameReuseStatistics
Counter how many times a DataFrame of a SparkSubFeed is reused by an Action later in the pipeline. The counter is increased during ExecutionPhase.Init when preparing the SubFeeds for an Action and it is decreased in ExecutionPhase.Exec to unpersist the DataFrame after there is no need for it anymore.
- actionsSelected
actions selected for execution by command line parameter --feed-sel
- actionsSkipped
actions selected but skipped in current attempt because they already succeeded in a previous attempt.
- Annotations
- @Scaladoc() @DeveloperApi()
- Alphabetic
- By Inheritance
- ActionPipelineContext
- Serializable
- Serializable
- Product
- Equals
- SmartDataLakeLogger
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Instance Constructors
-
new
ActionPipelineContext(feed: String, application: String, executionId: SDLExecutionId, instanceRegistry: InstanceRegistry, referenceTimestamp: Option[LocalDateTime] = None, appConfig: SmartDataLakeBuilderConfig, runStartTime: LocalDateTime = LocalDateTime.now(), attemptStartTime: LocalDateTime = LocalDateTime.now(), simulation: Boolean = false, phase: ExecutionPhase = ExecutionPhase.Prepare, dataFrameReuseStatistics: Map[(DataObjectId, Seq[PartitionValues]), Seq[ActionId]] = mutable.Map(), actionsSelected: Seq[ActionId] = Seq(), actionsSkipped: Seq[ActionId] = Seq(), serializableHadoopConf: SerializableHadoopConfiguration, globalConfig: GlobalConfig)
- feed
feed selector of the run
- application
application name of the run
- executionId
SDLExecutionId of this runs. Contains runId and attemptId. Both stay 1 if state is not enabled.
- instanceRegistry
registry of all SmartDataLake objects parsed from the config
- referenceTimestamp
timestamp used as reference in certain actions (e.g. HistorizeAction)
- appConfig
the command line parameters parsed into a SmartDataLakeBuilderConfig object
- runStartTime
start time of the run
- attemptStartTime
start time of attempt
- simulation
true if this is a simulation run
- phase
current execution phase
- dataFrameReuseStatistics
Counter how many times a DataFrame of a SparkSubFeed is reused by an Action later in the pipeline. The counter is increased during ExecutionPhase.Init when preparing the SubFeeds for an Action and it is decreased in ExecutionPhase.Exec to unpersist the DataFrame after there is no need for it anymore.
- actionsSelected
actions selected for execution by command line parameter --feed-sel
- actionsSkipped
actions selected but skipped in current attempt because they already succeeded in a previous attempt.
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- val actionsSelected: Seq[ActionId]
- val actionsSkipped: Seq[ActionId]
- val appConfig: SmartDataLakeBuilderConfig
- val application: String
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- val attemptStartTime: LocalDateTime
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native() @HotSpotIntrinsicCandidate()
- val dataFrameReuseStatistics: Map[(DataObjectId, Seq[PartitionValues]), Seq[ActionId]]
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- val executionId: SDLExecutionId
- val feed: String
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- val globalConfig: GlobalConfig
-
def
hadoopConf: Configuration
helper method to access hadoop configuration
helper method to access hadoop configuration
- Annotations
- @Scaladoc()
-
def
hasSparkSession: Boolean
True if a SparkSession has been created in this job
True if a SparkSession has been created in this job
- Annotations
- @Scaladoc()
- val instanceRegistry: InstanceRegistry
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
lazy val
logger: Logger
- Attributes
- protected
- Definition Classes
- SmartDataLakeLogger
- Annotations
- @transient()
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
- var phase: ExecutionPhase
- val referenceTimestamp: Option[LocalDateTime]
- val runStartTime: LocalDateTime
- val serializableHadoopConf: SerializableHadoopConfiguration
- val simulation: Boolean
-
def
sparkSession: SparkSession
helper method to access spark session
helper method to access spark session
- Annotations
- @Scaladoc()
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
Deprecated Value Members
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] ) @Deprecated
- Deprecated