trait SparkShims extends AnyRef
- Alphabetic
- By Inheritance
- SparkShims
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Abstract Value Members
-
abstract
def
ansiCastRule: ExprRule[_ <: Expression]
Return the replacement rule for AnsiCast.
Return the replacement rule for AnsiCast. 'AnsiCast' is removed from Spark 3.4.0, so need to handle it separately.
- abstract def aqeShuffleReaderExec: ExecRule[_ <: SparkPlan]
-
abstract
def
attachTreeIfSupported[TreeType <: TreeNode[_], A](tree: TreeType, msg: String = "")(f: ⇒ A): A
dropped by SPARK-34234
- abstract def avroRebaseReadKey: String
- abstract def avroRebaseWriteKey: String
- abstract def broadcastModeTransform(mode: BroadcastMode, toArray: Array[InternalRow]): Any
- abstract def columnarAdaptivePlan(a: AdaptiveSparkPlanExec, goal: CoalesceSizeGoal): SparkPlan
- abstract def filesFromFileIndex(fileCatalog: PartitioningAwareFileIndex): Seq[FileStatus]
-
abstract
def
findOperators(plan: SparkPlan, predicate: (SparkPlan) ⇒ Boolean): Seq[SparkPlan]
Walk the plan recursively and return a list of operators that match the predicate
- abstract def getAdaptiveInputPlan(adaptivePlan: AdaptiveSparkPlanExec): SparkPlan
- abstract def getDataWriteCmds: Map[Class[_ <: DataWritingCommand], DataWritingCommandRule[_ <: DataWritingCommand]]
- abstract def getDateFormatter(): DateFormatter
- abstract def getExecs: Map[Class[_ <: SparkPlan], ExecRule[_ <: SparkPlan]]
- abstract def getExprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]]
- abstract def getFileScanRDD(sparkSession: SparkSession, readFunction: (PartitionedFile) ⇒ Iterator[InternalRow], filePartitions: Seq[FilePartition], readDataSchema: StructType, metadataColumns: Seq[AttributeReference] = Seq.empty, fileFormat: Option[FileFormat] = None): RDD[InternalRow]
- abstract def getParquetFilters(schema: MessageType, pushDownDate: Boolean, pushDownTimestamp: Boolean, pushDownDecimal: Boolean, pushDownStartWith: Boolean, pushDownInFilterThreshold: Int, caseSensitive: Boolean, lookupFileMeta: (String) ⇒ String, dateTimeRebaseModeFromConf: String): ParquetFilters
- abstract def getRunnableCmds: Map[Class[_ <: RunnableCommand], RunnableCommandRule[_ <: RunnableCommand]]
- abstract def getScans: Map[Class[_ <: Scan], ScanRule[_ <: Scan]]
- abstract def hasAliasQuoteFix: Boolean
- abstract def hasCastFloatTimestampUpcast: Boolean
- abstract def int96ParquetRebaseRead(conf: SQLConf): String
- abstract def int96ParquetRebaseReadKey: String
- abstract def int96ParquetRebaseWrite(conf: SQLConf): String
- abstract def int96ParquetRebaseWriteKey: String
- abstract def isAqePlan(p: SparkPlan): Boolean
- abstract def isCustomReaderExec(x: SparkPlan): Boolean
- abstract def isEmptyRelation(relation: Any): Boolean
- abstract def isExchangeOp(plan: SparkPlanMeta[_]): Boolean
- abstract def isWindowFunctionExec(plan: SparkPlan): Boolean
- abstract def leafNodeDefaultParallelism(ss: SparkSession): Int
- abstract def neverReplaceShowCurrentNamespaceCommand: ExecRule[_ <: SparkPlan]
- abstract def newBroadcastQueryStageExec(old: BroadcastQueryStageExec, newPlan: SparkPlan): BroadcastQueryStageExec
- abstract def parquetRebaseRead(conf: SQLConf): String
- abstract def parquetRebaseReadKey: String
- abstract def parquetRebaseWrite(conf: SQLConf): String
- abstract def parquetRebaseWriteKey: String
-
abstract
def
reproduceEmptyStringBug: Boolean
Handle regexp_replace inconsistency from https://issues.apache.org/jira/browse/SPARK-39107
- abstract def reusedExchangeExecPfn: PartialFunction[SparkPlan, ReusedExchangeExec]
- abstract def sessionFromPlan(plan: SparkPlan): SparkSession
- abstract def shouldFailDivOverflow: Boolean
-
abstract
def
skipAssertIsOnTheGpu(plan: SparkPlan): Boolean
Our tests, by default, will check that all operators are running on the GPU, but there are some operators that we do not translate to GPU plans, so we need a way to bypass the check for those.
-
abstract
def
supportsColumnarAdaptivePlans: Boolean
Determine if the Spark version allows the supportsColumnar flag to be overridden in AdaptiveSparkPlanExec.
Determine if the Spark version allows the supportsColumnar flag to be overridden in AdaptiveSparkPlanExec. This feature was introduced in Spark 3.2 as part of SPARK-35881.
-
abstract
def
tryTransformIfEmptyRelation(mode: BroadcastMode): Option[Any]
This call can produce an
EmptyHashedRelationor an empty array, allowing the AQE ruleEliminateJoinToEmptyRelationin Spark 3.1.x to optimize certain joins.This call can produce an
EmptyHashedRelationor an empty array, allowing the AQE ruleEliminateJoinToEmptyRelationin Spark 3.1.x to optimize certain joins.In Spark 3.2.0, the optimization is still performed (under
AQEPropagateEmptyRelation), but the AQE optimizer is looking at the metrics for the query stage to determine if numRows == 0, and if so it can eliminate certain joins.The call is implemented only for Spark 3.1.x+. It is disabled in Databricks because it requires a task context to perform the
BroadcastMode.transformcall, but we'd like to call this from the driver. - abstract def v1RepairTableCommand(tableName: TableIdentifier): RunnableCommand
Concrete Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
addExecBroadcastShuffle(p: SparkPlan): SparkPlan
If the shim doesn't support executor broadcast, just return the plan passed in
-
def
addRowShuffleToQueryStageTransitionIfNeeded(c2r: ColumnarToRowTransition, sqse: ShuffleQueryStageExec): SparkPlan
Adds a row-based shuffle to the transititonal shuffle query stage if needed.
Adds a row-based shuffle to the transititonal shuffle query stage if needed. This is needed when AQE plans a GPU shuffleexchange to be reused by a parent plan exec that consumes rows
- def applyPostShimPlanRules(plan: SparkPlan): SparkPlan
- def applyShimPlanRules(plan: SparkPlan, conf: RapidsConf): SparkPlan
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
- def checkCToRWithExecBroadcastAQECoalPart(p: SparkPlan, parent: Option[SparkPlan]): Boolean
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def getShuffleFromCToRWithExecBroadcastAQECoalPart(p: SparkPlan): Option[SparkPlan]
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def isCastingStringToNegDecimalScaleSupported: Boolean
- def isExecutorBroadcastShuffle(shuffle: ShuffleExchangeLike): Boolean
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- def shuffleParentReadsShuffleData(shuffle: ShuffleExchangeLike, parent: SparkPlan): Boolean
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
def
toString(): String
- Definition Classes
- AnyRef → Any
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()