refactor(LNG-147): Rewrite list to tree conversion tail recusive (#711)

* Implemented ListToTreeConverter

parser/src/main/scala/aqua/parser/Expr.scala

@@ -15,6 +15,8 @@ import cats.parse.{Parser as P, Parser0 as P0}
 import cats.syntax.comonad.*
 import cats.{~>, Comonad, Eval}
 import scribe.Logging
+import aqua.parser.Ast.Tree
+import aqua.parser.ListToTreeConverter
 
 abstract class Expr[F[_]](val companion: Expr.Companion, val token: Token[F]) {
 
@@ -92,143 +94,19 @@ object Expr {
   abstract class AndIndented extends Block with Logging {
     def validChildren: List[Lexem]
 
-    private def leaf[F[_]](expr: Expr[F]): Ast.Tree[F] =
-      Cofree[Chain, Expr[F]](
-        expr,
-        Eval.now(Chain.empty)
-      )
-
-    private def last[F[_]](tree: Ast.Tree[F]): Expr[F] =
-      tree.tailForced.lastOption.fold(tree.head)(last)
-
-    private def setLeafs[F[_]](tree: Ast.Tree[F], children: Chain[Tree[F]]): Tree[F] =
-      tree.copy(tail = tree.tail.map {
-        case pref :== last =>
-          pref :+ setLeafs(last, children)
-        case _ =>
-          children
-      })
-
-    // Check if expression can be added in current block
-    private def canAddToBlock[F[_]](block: Tree[F], expr: Expr[F]): Boolean = {
-      block.head.companion match {
-        case b: AndIndented =>
-          b.validChildren.map {
-            case ll: LazyLexem => ll.c
-            case vc => vc
-          }.contains(expr.companion)
-
-        case _: Prefix =>
-          block.tail.value.headOption.exists(t => canAddToBlock(t, expr))
-        case _ => false
-      }
-    }
-
-    // Generate error if expression (child) cannot be added to a block
-    private def wrongChildError[F[_]](indent: F[String], expr: Expr[F]): ParserError[F] = {
-      val msg = expr match {
-        case ReturnExpr(_) =>
-          "Return expression must be on the top indentation level and at the end of function body"
-        // could there be other expressions?
-        case _ => "This expression is on the wrong indentation level"
-      }
-      BlockIndentError(indent, msg)
-    }
-
-    private def headIsBlock[F[_]](tree: Tree[F]): Boolean = {
-      tree.tail.value.headOption match {
-        case Some(t) => t.head.isBlock
-        case _ => tree.head.isBlock
-      }
-    }
-
-    private case class Acc[F[_]](
-      block: Tree[F],
-      initialIndentF: F[String],
-      tail: Chain[(F[String], Ast.Tree[F])] = Chain.empty[(F[String], Ast.Tree[F])],
-      window: Chain[Tree[F]] = Chain.empty[Tree[F]],
-      errors: Chain[ParserError[F]] = Chain.empty[ParserError[F]]
-    )
-
-    // converts list of expressions to a tree of tokens
-    private def listToTree[F[_]: Comonad: LiftParser](
-      acc: Acc[F]
-    ): ValidatedNec[ParserError[F], Acc[F]] = {
-      val initialIndent = acc.initialIndentF.extract.length
-
-      acc.tail.uncons match {
-        case Some(((currentIndent, currentExpr), tail)) =>
-          val current = last(currentExpr)
-
-          // if current indent is bigger then block indentation
-          // then add current expression to this block
-          if (currentIndent.extract.length > initialIndent) {
-            // if current expression is a block, create tree of this block and return remaining tail
-            if (headIsBlock(currentExpr)) {
-              listToTree(Acc(currentExpr, currentIndent, tail, errors = acc.errors)).andThen {
-                case a@Acc(innerTree, _, newTail, window, errors) =>
-                  if (window.nonEmpty) {
-                    logger.warn("Internal: Window cannot be empty after converting list of expressions to a tree.")
-                    logger.warn("Current state: " + a)
-                  }
-
-                  listToTree(
-                    acc.copy(
-                      window = acc.window :+ innerTree,
-                      tail = newTail,
-                      errors = acc.errors ++ errors
-                    )
-                  )
-              }
-            } else {
-              // if expression not a block, add it to a window until we meet the end of the block
-              if (canAddToBlock(acc.block, current)) {
-                listToTree(acc.copy(window = acc.window :+ currentExpr, tail = tail))
-              } else {
-                val error = wrongChildError(currentIndent, current)
-                validNec(acc.copy(tail = tail, errors = acc.errors :+ error))
-              }
-
-            }
-          } else {
-            val errors = if (acc.window.isEmpty) {
-              // error if a block is empty
-              val error = BlockIndentError(acc.initialIndentF, "Block expression has no body")
-              acc.errors :+ error
-            } else acc.errors
-
-            // if current indentation less or equal to block indentation,
-            // add all expressions in window to a head
-            validNec(
-              Acc(
-                setLeafs(acc.block, acc.window),
-                acc.initialIndentF,
-                (currentIndent, currentExpr) +: tail,
-                errors = errors
-              )
-            )
-
-          }
-
-        case None =>
-          // end of top-level block
-          NonEmptyChain
-            .fromChain(acc.errors)
-            .map(invalid)
-            .getOrElse(validNec(Acc(setLeafs(acc.block, acc.window), acc.initialIndentF)))
-      }
-    }
-
     override lazy val ast: P[ValidatedNec[ParserError[Span.S], Ast.Tree[Span.S]]] =
       (readLine ~ (` \n+` *>
         (P.repSep(
           ` `.lift ~ P.oneOf(validChildren.map(_.readLine.backtrack)),
           ` \n+`
-        ) <* ` \n`.?))).map { t =>
+        ) <* ` \n`.?))).map { case (open, lines) =>
-        val startIndent = t._1.head.token.as("")
+        lines
-        listToTree(Acc(t._1, startIndent, Chain.fromSeq(t._2.toList))).map { res =>
+          .foldLeft(
-          res._1
+            ListToTreeConverter(open)
-        }
+          ) { case (acc, (indent, line)) =>
+            acc.next(indent, line)
+          }
+          .result
       }
   }
 }

parser/src/main/scala/aqua/parser/ListToTreeConverter.scala

@@ -0,0 +1,188 @@
+package aqua.parser
+
+import aqua.parser.Ast.Tree
+import aqua.parser.Expr.{AndIndented, LazyLexem, Prefix}
+import aqua.parser.expr.func.ReturnExpr
+import aqua.parser.lift.LiftParser
+
+import cats.Comonad
+import cats.data.Chain
+import cats.data.Chain.:==
+import cats.syntax.comonad.*
+import cats.data.{NonEmptyChain, ValidatedNec}
+import cats.data.Validated.*
+import aqua.parser.Expr.LazyLexem
+
+/**
+ * Converts a list of lines to a tree
+ */
+final case class ListToTreeConverter[F[_]](
+  currentBlock: ListToTreeConverter.Block[F], // Current block data
+  stack: List[ListToTreeConverter.Block[F]] = Nil, // Stack of opened blocks
+  errors: Chain[ParserError[F]] = Chain.empty[ParserError[F]] // Errors
+)(using Comonad[F]) {
+  // Import helper functions
+  import ListToTreeConverter.*
+
+  private def addError(error: ParserError[F]): ListToTreeConverter[F] =
+    copy(errors = errors :+ error)
+
+  private def pushBlock(indent: F[String], line: Tree[F]): ListToTreeConverter[F] =
+    copy(currentBlock = Block(indent, line), stack = currentBlock :: stack)
+
+  private def addToCurrentBlock(line: Tree[F]): ListToTreeConverter[F] =
+    copy(currentBlock = currentBlock.add(line))
+
+  private def popBlock: Option[ListToTreeConverter[F]] =
+    stack match {
+      case Nil => None
+      case prevBlock :: tail =>
+        Some(copy(currentBlock = prevBlock.add(currentBlock.close), stack = tail))
+    }
+
+  /**
+   * Method to call on each new line
+   */
+  @scala.annotation.tailrec
+  def next(indent: F[String], line: Tree[F]): ListToTreeConverter[F] =
+    if (indentValue(indent) > indentValue(currentBlock.indent)) {
+      if (isBlock(line)) {
+        pushBlock(indent, line)
+      } else {
+        val expr = lastExpr(line)
+
+        if (currentBlock.canAdd(expr)) {
+          addToCurrentBlock(line)
+        } else {
+          addError(wrongChildError(indent, expr))
+        }
+      }
+    } else {
+      val emptyChecked = if (currentBlock.isEmpty) {
+        addError(emptyBlockError(currentBlock.indent))
+      } else this
+
+      emptyChecked.popBlock match {
+        case Some(blockPopped) => blockPopped.next(indent, line)
+        // This should not happen because of the way of parsing
+        case _ => emptyChecked.addError(unexpectedIndentError(indent))
+      }
+    }
+
+  /**
+   * Produce the result of the conversion
+   */
+  @scala.annotation.tailrec
+  def result: ValidatedNec[ParserError[F], Tree[F]] =
+    popBlock match {
+      case Some(blockPopped) => blockPopped.result
+      case _ =>
+        NonEmptyChain
+          .fromChain(errors)
+          .map(invalid)
+          .getOrElse(
+            valid(currentBlock.close)
+          )
+    }
+}
+
+object ListToTreeConverter {
+
+  /**
+   * Constructs a converter from block opening line
+   */
+  def apply[F[_]](open: Tree[F])(using Comonad[F]): ListToTreeConverter[F] =
+    ListToTreeConverter(Block(open.head.token.as(""), open))
+
+  /**
+   * Data associated with a block
+   */
+  final case class Block[F[_]](
+    indent: F[String], // Indentation of the block opening line
+    block: Tree[F], // Block opening line
+    content: Chain[Tree[F]] = Chain.empty[Tree[F]] // Children of the block
+  ) {
+
+    /**
+     * Check if expr can be added to this block
+     */
+    def canAdd(expr: Expr[F]): Boolean = {
+      def checkFor(tree: Tree[F]): Boolean =
+        tree.head.companion match {
+          case indented: AndIndented =>
+            indented.validChildren.map {
+              case ll: LazyLexem => ll.c
+              case vc => vc
+            }.contains(expr.companion)
+          case _: Prefix =>
+            tree.tail.value.headOption.exists(checkFor)
+          case _ => false
+        }
+
+      checkFor(block)
+    }
+
+    /**
+     * Add child to the block
+     */
+    def add(child: Tree[F]): Block[F] =
+      copy(content = content :+ child)
+
+    /**
+     * Check if the block has no children
+     */
+    def isEmpty: Boolean = content.isEmpty
+
+    /**
+     * Create a tree corresponding to the block
+     */
+    def close: Tree[F] = {
+
+      /**
+       * Set children of the rightmost expression in tree
+       */
+      def setLast(tree: Tree[F], children: Chain[Tree[F]]): Tree[F] =
+        tree.copy(tail = tree.tail.map {
+          case init :== last =>
+            init :+ setLast(last, children)
+          case _ =>
+            children
+        })
+
+      setLast(block, content)
+    }
+  }
+
+  // TODO(LNG-150): This way of counting indent does not play way with mixed tabs and spaces
+  private def indentValue[F[_]](indent: F[String])(using Comonad[F]): Int =
+    indent.extract.length
+
+  private def isBlock[F[_]](line: Tree[F]): Boolean =
+    line.tail.value.headOption
+      .map(_.head.isBlock)
+      .getOrElse(line.head.isBlock)
+
+  @scala.annotation.tailrec
+  private def lastExpr[F[_]](tree: Tree[F]): Expr[F] =
+    tree.tailForced.lastOption match {
+      case Some(t) => lastExpr(t)
+      case _ => tree.head
+    }
+
+  def wrongChildError[F[_]](indent: F[String], expr: Expr[F]): ParserError[F] = {
+    val msg = expr match {
+      case ReturnExpr(_) =>
+        "Return expression must be on the top indentation level and at the end of function body"
+      // could there be other expressions?
+      case _ => "This expression is on the wrong indentation level"
+    }
+    BlockIndentError(indent, msg)
+  }
+
+  def emptyBlockError[F[_]](indent: F[String]): ParserError[F] =
+    BlockIndentError(indent, "Block expression has no body")
+
+  def unexpectedIndentError[F[_]](indent: F[String]): ParserError[F] =
+    BlockIndentError(indent, "Unexpected indentation")
+
+}