Why Scala's stack is got overflow there for this Spark API calls?-CodePudding

In Appache Zeppelin it is uneasy to find what is wrong with this expression:

val dailySummariesDf =
      getDFFromJdbcSource(SparkSession.builder().appName("test").master("local").getOrCreate(), 
      s"SELECT *  FROM mf_joined WHERE (timestamp > '2022-01-24 08:00:00.000')")
      .withColumn("DiffInSeconds", to_timestamp(date_format(col("writetime"), "HH:mm:ss.SSS")).cast(LongType) - 
         to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")).cast(LongType))
         .withColumn("time", to_timestamp(date_format(col("timestamp"), "HH:mm:ss.SSS")))
  //.withColumn("hour",date_trunc("hour",($"unit_ts")))
  .withColumn("hour",date_trunc("minute",($"writetime")))
  .withColumn("max",max("DiffInSeconds").over(Window.partitionBy($"hour")))
  .withColumn("min",min("DiffInSeconds").over(Window.partitionBy($"hour")))
  .withColumn("mean",mean("DiffInSeconds").over(Window.partitionBy($"hour")))
  .withColumn("count",count("DiffInSeconds").over(Window.partitionBy($"hour")))
  .withColumn("max2",when(col("max") > 100, 101).otherwise(col("max")))
  .withColumn("min2",when(col("min") < -100, -101).otherwise(col("min")))
  .withColumn("mean2",when(col("mean") < -100, -101).otherwise(col("mean")))
  .withColumn("rnb",row_number().over(Window.partitionBy($"hour").orderBy($"timestamp".desc)))
  .where($"rnb"===1)
  //.select($"hour",$"max",$"min",$"mean",$"max2",$"min2",$"mean2",$"count")
  .orderBy($"hour".desc)

        .persist(StorageLevel.MEMORY_ONLY_SER)
       //orderBy("windowstart")

and I'm gettng the error message

java.lang.StackOverflowError
    at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
    at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
    at scala.reflect.internal.Types$TypeRef.isTrivial(Types.scala:2081)
    at scala.reflect.internal.Types$MethodType.isTrivialResult(Types.scala:2476)
    at scala.reflect.internal.Types$MethodType.isTrivial(Types.scala:2471)
    at scala.reflect.internal.Types$Type.asSeenFrom(Types.scala:698)
    at scala.reflect.internal.Types$Type.computeMemberType(Types.scala:740)
    at scala.reflect.internal.Symbols$MethodSymbol.typeAsMemberOf(Symbols.scala:3008)
    at scala.reflect.internal.Types$Type.memberType(Types.scala:731)
    at scala.tools.nsc.typechecker.Infer$Inferencer.checkAccessible(Infer.scala:282)
    at scala.tools.nsc.typechecker.Typers$Typer.makeAccessible(Typers.scala:592)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$59(Typers.scala:5081)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$silent$2(Typers.scala:727)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:727)
    at scala.tools.nsc.typechecker.Typers$Typer.typedSelect$1(Typers.scala:5081)
    at scala.tools.nsc.typechecker.Typers$Typer.typedSelectOrSuperCall$1(Typers.scala:5138)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5671)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$39(Typers.scala:4837)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
    at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4839)
    at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4867)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$39(Typers.scala:4837)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
    at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4839)
    at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4867)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$39(Typers.scala:4837)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
    at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4839)
    at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4867)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$39(Typers.scala:4837)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
    at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4839)
    at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4867)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.typechecker.Typers$Typer.$anonfun$typed1$39(Typers.scala:4837)
    at scala.tools.nsc.typechecker.Typers$Typer.silent(Typers.scala:714)
    at scala.tools.nsc.typechecker.Typers$Typer.normalTypedApply$1(Typers.scala:4839)
    at scala.tools.nsc.typechecker.Typers$Typer.typedApply$1(Typers.scala:4867)
    at scala.tools.nsc.typechecker.Typers$Typer.typed1(Typers.scala:5670)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)
    at scala.tools.nsc.typechecker.Typers$Typer.typed(Typers.scala:5716)
    at scala.tools.nsc.transform.Erasure$Eraser.adaptMember(Erasure.scala:734)
    at scala.tools.nsc.transform.Erasure$Eraser.typed1(Erasure.scala:809)

CodePudding user response：

For Appache Zeppelin if was enough to restart the interpreter in Databook's Settings and after that the problem has gone. But the origin is still unclear