c

org.apache.spark.sql.delta.stats

DeltaTaskStatisticsTracker

class DeltaTaskStatisticsTracker extends WriteTaskStatsTracker

A per-task (i.e. one instance per executor) WriteTaskStatsTracker that collects the statistics defined by StatisticsCollection for files that are being written into a delta table.

Linear Supertypes
WriteTaskStatsTracker, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. DeltaTaskStatisticsTracker
  2. WriteTaskStatsTracker
  3. AnyRef
  4. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Instance Constructors

  1. new DeltaTaskStatisticsTracker(dataCols: Seq[Attribute], statsColExpr: Expression, rootPath: Path, hadoopConf: Configuration)

    dataCols

    Resolved data (i.e. non-partitionBy) columns of the dataframe to be written.

    statsColExpr

    Resolved expression for computing all the statistics that we want to gather.

    rootPath

    The Reservoir's root path.

    hadoopConf

    Hadoop Config for being able to instantiate a FileSystem.

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val aggBufferAttrs: Seq[Attribute]
    Attributes
    protected
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @native()
  7. def closeFile(filePath: String): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  8. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  9. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  10. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable])
  11. def flattenStructCols(field: StructType, customExprId: String): Array[(String, DataType)]

    Flattens the struct nested columns and for each column returns its data type and an unique custom name created by appending the inner field column name to the given customExprId (i.e.

    Flattens the struct nested columns and for each column returns its data type and an unique custom name created by appending the inner field column name to the given customExprId (i.e. an accumulator for so far created custom name, which initially is equal to the column exprId).

  12. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  13. def getFinalStats(taskCommitTime: Long): DeltaFileStatistics
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  14. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  15. def initializeAggBuf(buffer: SpecificInternalRow): InternalRow
    Attributes
    protected
  16. val initializeStats: MutableProjection
    Attributes
    protected
  17. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  18. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  19. def newFile(newFilePath: String): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  20. def newPartition(partitionValues: InternalRow): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  21. def newRow(filePath: String, currentRow: InternalRow): Unit
    Definition Classes
    DeltaTaskStatisticsTracker → WriteTaskStatsTracker
  22. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  23. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  24. val results: HashMap[String, String]
    Attributes
    protected
  25. val submittedFiles: HashMap[String, InternalRow]
    Attributes
    protected[this]
  26. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  27. def toString(): String
    Definition Classes
    AnyRef → Any
  28. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  29. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  30. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()

Inherited from WriteTaskStatsTracker

Inherited from AnyRef

Inherited from Any

Ungrouped