Pfthb

I am using spark sql optimizer to optimize two ranges join, it is optimized to calculate two range intersection to avoid join

test("SparkTest") 
 object RangeIntersectRule extends Rule[LogicalPlan] 
 override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown 
 case Join(Project(_, Range(start1, end1, _, _, _, _)), Project(_, Range(start2, end2, _, _, _, _)), _, _) => 
 val start = start1 max start2
 val end = end1 min end2
 if (start1 > end2 
 
 

 val spark = SparkSession.builder().master("local").appName("SparkTest").enableHiveSupport().getOrCreate()
 spark.experimental.extraOptimizations = Seq(RangeIntersectRule)
 spark.range(10, 40).toDF("x").createOrReplaceTempView("t1")
 spark.range(20, 50).toDF("y").createOrReplaceTempView("t2")
 val df = spark.sql("select t1.x from t1 join t2 on t1.x = t2.y")
 df.explain(true)
 df.show(truncate = false)
 

But when I run it, an exception throws, could someone help where the problem is?Thanks

The optimized logical plan and physical plan is:

== Optimized Logical Plan ==
Project [x#2L]
+- !Project [id#0L AS x#2L]
 +- Range (20, 40, step=1, splits=Some(1))

== Physical Plan ==
Project [x#2L]
+- !Project [id#0L AS x#2L]
 +- Range (20, 40, step=1, splits=1)

The exception is:

Caused by: java.lang.RuntimeException: Couldn't find id#0L in [id#14L]
 at scala.sys.package$.error(package.scala:27)
 at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:106)
 at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:100)
 at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:59)
 ... 47 more

object RangeIntersectRule extends Rule[LogicalPlan] 
 override def apply(plan: LogicalPlan): LogicalPlan = plan transformDown 
 case Join(Range(start1, end1, 1, Some(1), output1, false), Range(start2, end2, 1, Some(1), output2, false), Inner, _) => end1 < start2) Project(output1, Range(0, 0, 1, Some(1), output1, false))
 else Project(output1, Range(start, end, 1, Some(1), output1, false))