case class FileSubFeed(fileRefs: Option[Seq[FileRef]], dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues], isDAGStart: Boolean = false, isSkipped: Boolean = false, fileRefMapping: Option[Seq[FileRefMapping]] = None) extends SubFeed with Product with Serializable
A FileSubFeed is used to transport references to files between Actions.
- fileRefs
path to files to be processed
- dataObjectId
id of the DataObject this SubFeed corresponds to
- partitionValues
Values of Partitions transported by this SubFeed
- isDAGStart
true if this subfeed is a start node of the dag
- isSkipped
true if this subfeed is the result of a skipped action
- fileRefMapping
store mapping of input to output file references. This is also used for post processing (e.g. delete after read).
- Annotations
- @Scaladoc()
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- FileSubFeed
- Serializable
- Serializable
- Product
- Equals
- SubFeed
- SmartDataLakeLogger
- DAGResult
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Instance Constructors
-
new
FileSubFeed(fileRefs: Option[Seq[FileRef]], dataObjectId: DataObjectId, partitionValues: Seq[PartitionValues], isDAGStart: Boolean = false, isSkipped: Boolean = false, fileRefMapping: Option[Seq[FileRefMapping]] = None)
- fileRefs
path to files to be processed
- dataObjectId
id of the DataObject this SubFeed corresponds to
- partitionValues
Values of Partitions transported by this SubFeed
- isDAGStart
true if this subfeed is a start node of the dag
- isSkipped
true if this subfeed is the result of a skipped action
- fileRefMapping
store mapping of input to output file references. This is also used for post processing (e.g. delete after read).
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
applyExecutionModeResultForInput(result: ExecutionModeResult, mainInputId: DataObjectId)(implicit context: ActionPipelineContext): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
applyExecutionModeResultForOutput(result: ExecutionModeResult)(implicit context: ActionPipelineContext): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
breakLineage(implicit context: ActionPipelineContext): FileSubFeed
Break lineage.
Break lineage. This means to discard an existing DataFrame or List of FileRefs, so that it is requested again from the DataObject. On one side this is usable to break long DataFrame Lineages over multiple Actions and instead reread the data from an intermediate table. On the other side it is needed if partition values or filter condition are changed.
- Definition Classes
- FileSubFeed → SubFeed
- def checkPartitionValuesColsExisting(partitions: Set[String]): Boolean
-
def
clearDAGStart(): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
clearPartitionValues(breakLineageOnChange: Boolean = true)(implicit context: ActionPipelineContext): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
clearSkipped(): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native() @HotSpotIntrinsicCandidate()
-
val
dataObjectId: DataObjectId
- Definition Classes
- FileSubFeed → SubFeed
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- val fileRefMapping: Option[Seq[FileRefMapping]]
- val fileRefs: Option[Seq[FileRef]]
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
val
isDAGStart: Boolean
- Definition Classes
- FileSubFeed → SubFeed
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
val
isSkipped: Boolean
- Definition Classes
- FileSubFeed → SubFeed
-
lazy val
logger: Logger
- Attributes
- protected
- Definition Classes
- SmartDataLakeLogger
- Annotations
- @transient()
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native() @HotSpotIntrinsicCandidate()
-
val
partitionValues: Seq[PartitionValues]
- Definition Classes
- FileSubFeed → SubFeed
-
def
resultId: String
- Definition Classes
- SubFeed → DAGResult
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toOutput(dataObjectId: DataObjectId): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
union(other: SubFeed)(implicit context: ActionPipelineContext): SubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
def
unionPartitionValues(otherPartitionValues: Seq[PartitionValues]): Seq[PartitionValues]
- Definition Classes
- SubFeed
-
def
updatePartitionValues(partitions: Seq[String], breakLineageOnChange: Boolean = true, newPartitionValues: Option[Seq[PartitionValues]] = None)(implicit context: ActionPipelineContext): FileSubFeed
- Definition Classes
- FileSubFeed → SubFeed
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
Deprecated Value Members
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] ) @Deprecated
- Deprecated