From e3716f6f3139f935b4a2a0df3eca0f4ed283f9ba Mon Sep 17 00:00:00 2001 From: Dmitry Kurinskiy Date: Tue, 3 Aug 2021 17:23:28 +0300 Subject: [PATCH] Converting compiler to a pure function (#217) * Trying to make the compiler a pure function * Pure compiler WIP * Compiler should be working now * printlns * printlns * printlns * fix, delete printlns * more logs * fix import resolving, more tests * fix imports, add tests for imports resolving * fix test * correct paths to targets, correct output * refactoring, create directories to file if not exist * small changes * fix test * Tiny fixes WIP * Tiny fixes * Incrementing base version, as host_peer_id is added (fixes #218) * render error messages, WIP * small fix * get src for lexer error * wrap parser error * add list of errors * Handle file write errors accurately * Use show syntax * fix test * fix test * fix test * println Co-authored-by: DieMyst --- README.md | 4 +- .../scala/aqua/backend/air/AirBackend.scala | 8 +- .../scala/aqua/backend/air/FuncAirGen.scala | 4 +- .../aqua/backend/js/JavaScriptBackend.scala | 28 ++- .../aqua/backend/js/JavaScriptFile.scala | 4 +- .../aqua/backend/js/JavaScriptFunc.scala | 4 +- .../src/main/scala/aqua/backend/Backend.scala | 21 +- .../main/scala/aqua/backend/Generated.scala | 9 + .../aqua/backend/ts/TypeScriptBackend.scala | 28 ++- .../aqua/backend/ts/TypeScriptFile.scala | 4 +- .../aqua/backend/ts/TypeScriptFunc.scala | 4 +- build.sbt | 2 +- cli/src/main/scala/aqua/AppOps.scala | 9 +- cli/src/main/scala/aqua/AquaCli.scala | 24 +- .../src/main/scala/aqua}/AquaIO.scala | 6 +- .../main/scala/aqua/AquaPathCompiler.scala | 39 +++ .../main/scala/aqua/CustomLogFormatter.scala | 13 + cli/src/main/scala/aqua/ErrorRendering.scala | 72 ++++++ cli/src/main/scala/aqua/Test.scala | 14 +- .../scala/aqua/files/AquaFileSources.scala | 124 ++++++++++ .../scala/aqua/{ => files}/AquaFilesIO.scala | 84 ++++--- .../main/scala/aqua/files/FileModuleId.scala | 11 + .../main/scala/aqua}/io/AquaFileError.scala | 33 ++- cli/src/test/scala/SourcesSpec.scala | 157 ++++++++++++ cli/src/test/scala/WriteFileSpec.scala | 14 +- .../test/test-dir/broken-import/broken.aqua | 4 + cli/src/test/test-dir/importNear.aqua | 5 + cli/src/test/test-dir/imports/import.aqua | 5 + cli/src/test/test-dir/index.aqua | 4 + .../test/test-dir/path-test/importNear.aqua | 5 + cli/src/test/test-dir/path-test/index.aqua | 4 + .../scala/aqua/compiler/AquaCompiled.scala | 5 + .../scala/aqua/compiler/AquaCompiler.scala | 227 ++++++------------ .../main/scala/aqua/compiler/AquaError.scala | 49 +--- .../main/scala/aqua/compiler/AquaParser.scala | 111 +++++++++ .../scala/aqua/compiler/AquaProcessed.scala | 7 + .../scala/aqua/compiler/AquaSources.scala | 14 ++ .../main/scala/aqua/compiler/Prepared.scala | 54 ----- .../scala/aqua/compiler/io/AquaFile.scala | 114 --------- .../scala/aqua/compiler/io/AquaFiles.scala | 90 ------- .../scala/aqua/compiler/io/FileModuleId.scala | 5 - .../main/scala/aqua/linker/AquaModule.scala | 12 +- .../src/main/scala/aqua/linker/Linker.scala | 22 +- .../src/main/scala/aqua/linker/Modules.scala | 9 + .../test/scala/aqua/linker/LinkerSpec.scala | 12 +- ...odyConfig.scala => GenerationConfig.scala} | 25 +- .../aqua/model/transform/Transform.scala | 2 +- model/test-kit/src/main/scala/aqua/Node.scala | 21 +- .../aqua/model/transform/TransformSpec.scala | 6 +- parser/src/main/scala/aqua/parser/Ast.scala | 3 +- .../main/scala/aqua/parser/ParserError.scala | 2 +- .../scala/aqua/parser/lift/FileSpan.scala | 18 +- .../scala/aqua/parser/lift/LiftParser.scala | 7 + .../main/scala/aqua/parser/lift/Span.scala | 3 + .../scala/aqua/semantics/SemanticsSpec.scala | 2 +- 55 files changed, 924 insertions(+), 643 deletions(-) create mode 100644 backend/src/main/scala/aqua/backend/Generated.scala rename {compiler/src/main/scala/aqua/compiler => cli/src/main/scala/aqua}/AquaIO.scala (80%) create mode 100644 cli/src/main/scala/aqua/AquaPathCompiler.scala create mode 100644 cli/src/main/scala/aqua/CustomLogFormatter.scala create mode 100644 cli/src/main/scala/aqua/ErrorRendering.scala create mode 100644 cli/src/main/scala/aqua/files/AquaFileSources.scala rename cli/src/main/scala/aqua/{ => files}/AquaFilesIO.scala (70%) create mode 100644 cli/src/main/scala/aqua/files/FileModuleId.scala rename {compiler/src/main/scala/aqua/compiler => cli/src/main/scala/aqua}/io/AquaFileError.scala (52%) create mode 100644 cli/src/test/scala/SourcesSpec.scala create mode 100644 cli/src/test/test-dir/broken-import/broken.aqua create mode 100644 cli/src/test/test-dir/importNear.aqua create mode 100644 cli/src/test/test-dir/imports/import.aqua create mode 100644 cli/src/test/test-dir/index.aqua create mode 100644 cli/src/test/test-dir/path-test/importNear.aqua create mode 100644 cli/src/test/test-dir/path-test/index.aqua create mode 100644 compiler/src/main/scala/aqua/compiler/AquaCompiled.scala create mode 100644 compiler/src/main/scala/aqua/compiler/AquaParser.scala create mode 100644 compiler/src/main/scala/aqua/compiler/AquaProcessed.scala create mode 100644 compiler/src/main/scala/aqua/compiler/AquaSources.scala delete mode 100644 compiler/src/main/scala/aqua/compiler/Prepared.scala delete mode 100644 compiler/src/main/scala/aqua/compiler/io/AquaFile.scala delete mode 100644 compiler/src/main/scala/aqua/compiler/io/AquaFiles.scala delete mode 100644 compiler/src/main/scala/aqua/compiler/io/FileModuleId.scala rename model/src/main/scala/aqua/model/transform/{BodyConfig.scala => GenerationConfig.scala} (60%) diff --git a/README.md b/README.md index 8a3c35bb..e508a61c 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Therefore, Aqua scripts are compiled into several targets at once, with AIR and ## Using Aqua -Please refer to [aqua-playground](https://github.com/fluencelabs/aqua-playground) to learn how to use Aqua. +Please refer to [Aqua Book](https://doc.fluence.dev/aqua-book/) to learn how to use Aqua. ## Compiler CLI @@ -34,6 +34,8 @@ Input directory should contain files with `aqua` scripts. - **[model](./model)** - middle-end, internal representation of the code, optimizations and transfromations - **[semantics](./semantics)** - rules to convert source AST into the model - **[linker](./linker)** - checks dependencies between modules, builds and combines an abstract dependencies tree +- **[backend](./backend)** - compilation backend interface +- **[compiler](./compiler)** - compiler as a pure function made from _linker_, _semantics_ and _backend_ - **[backend/air](./backend/air)** – generates AIR code from the middle-end model - **[backend/ts](./backend/ts)** - generates AIR code and Typescript wrappers for use with Fluence JS SDK - **[cli](./cli)** - CLI interface diff --git a/backend/air/src/main/scala/aqua/backend/air/AirBackend.scala b/backend/air/src/main/scala/aqua/backend/air/AirBackend.scala index d91e0fed..0bb02782 100644 --- a/backend/air/src/main/scala/aqua/backend/air/AirBackend.scala +++ b/backend/air/src/main/scala/aqua/backend/air/AirBackend.scala @@ -1,17 +1,17 @@ package aqua.backend.air -import aqua.backend.{Backend, Compiled} +import aqua.backend.{Backend, Generated} import aqua.model.AquaContext -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import cats.implicits.toShow object AirBackend extends Backend { val ext = ".air" - override def generate(context: AquaContext, bc: BodyConfig): Seq[Compiled] = { + override def generate(context: AquaContext, genConf: GenerationConfig): Seq[Generated] = { context.funcs.values.toList.map(fc => - Compiled("." + fc.funcName + ext, FuncAirGen(fc).generateAir(bc).show) + Generated("." + fc.funcName + ext, FuncAirGen(fc).generateAir(genConf).show) ) } } diff --git a/backend/air/src/main/scala/aqua/backend/air/FuncAirGen.scala b/backend/air/src/main/scala/aqua/backend/air/FuncAirGen.scala index 0022f131..3f5656b4 100644 --- a/backend/air/src/main/scala/aqua/backend/air/FuncAirGen.scala +++ b/backend/air/src/main/scala/aqua/backend/air/FuncAirGen.scala @@ -1,14 +1,14 @@ package aqua.backend.air import aqua.model.func.FuncCallable -import aqua.model.transform.{BodyConfig, Transform} +import aqua.model.transform.{GenerationConfig, Transform} case class FuncAirGen(func: FuncCallable) { /** * Generates AIR from the function body */ - def generateAir(conf: BodyConfig = BodyConfig()): Air = + def generateAir(conf: GenerationConfig = GenerationConfig()): Air = AirGen( Transform.forClient(func, conf) ).generate diff --git a/backend/js/src/main/scala/aqua/backend/js/JavaScriptBackend.scala b/backend/js/src/main/scala/aqua/backend/js/JavaScriptBackend.scala index 79db25ba..f03d4984 100644 --- a/backend/js/src/main/scala/aqua/backend/js/JavaScriptBackend.scala +++ b/backend/js/src/main/scala/aqua/backend/js/JavaScriptBackend.scala @@ -1,21 +1,29 @@ package aqua.backend.js -import aqua.backend.{Backend, Compiled} +import aqua.backend.{Backend, Generated} import aqua.model.AquaContext -import aqua.model.transform.BodyConfig -import cats.data.Chain +import aqua.model.transform.GenerationConfig +import cats.data.NonEmptyChain object JavaScriptBackend extends Backend { val ext = ".js" - override def generate(context: AquaContext, bc: BodyConfig): Seq[Compiled] = { - val funcs = Chain.fromSeq(context.funcs.values.toSeq).map(JavaScriptFunc(_)) - Seq( - Compiled( - ext, - JavaScriptFile.Header + "\n\n" + funcs.map(_.generateJavascript(bc)).toList.mkString("\n\n") + override def generate(context: AquaContext, genConf: GenerationConfig): Seq[Generated] = { + val funcs = NonEmptyChain.fromSeq(context.funcs.values.toSeq).map(_.map(JavaScriptFunc(_))) + funcs + .map(fs => + Seq( + Generated( + ext, + JavaScriptFile.Header + "\n\n" + fs + .map(_.generateJavascript(genConf)) + .toChain + .toList + .mkString("\n\n") + ) + ) ) - ) + .getOrElse(Seq.empty) } } diff --git a/backend/js/src/main/scala/aqua/backend/js/JavaScriptFile.scala b/backend/js/src/main/scala/aqua/backend/js/JavaScriptFile.scala index ea98b232..d6e05884 100644 --- a/backend/js/src/main/scala/aqua/backend/js/JavaScriptFile.scala +++ b/backend/js/src/main/scala/aqua/backend/js/JavaScriptFile.scala @@ -1,7 +1,7 @@ package aqua.backend.js import aqua.model.AquaContext -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import cats.data.Chain case class JavaScriptFile(context: AquaContext) { @@ -9,7 +9,7 @@ case class JavaScriptFile(context: AquaContext) { def funcs: Chain[JavaScriptFunc] = Chain.fromSeq(context.funcs.values.toSeq).map(JavaScriptFunc(_)) - def generateJS(conf: BodyConfig = BodyConfig()): String = + def generateJS(conf: GenerationConfig = GenerationConfig()): String = JavaScriptFile.Header + "\n\n" + funcs.map(_.generateJavascript(conf)).toList.mkString("\n\n") } diff --git a/backend/js/src/main/scala/aqua/backend/js/JavaScriptFunc.scala b/backend/js/src/main/scala/aqua/backend/js/JavaScriptFunc.scala index 85cc608d..a8c3f2bc 100644 --- a/backend/js/src/main/scala/aqua/backend/js/JavaScriptFunc.scala +++ b/backend/js/src/main/scala/aqua/backend/js/JavaScriptFunc.scala @@ -2,7 +2,7 @@ package aqua.backend.js import aqua.backend.air.FuncAirGen import aqua.model.func.{ArgDef, FuncCallable} -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import aqua.types._ import cats.syntax.show._ @@ -38,7 +38,7 @@ case class JavaScriptFunc(func: FuncCallable) { |""".stripMargin } - def generateJavascript(conf: BodyConfig = BodyConfig()): String = { + def generateJavascript(conf: GenerationConfig = GenerationConfig()): String = { val tsAir = FuncAirGen(func).generateAir(conf) diff --git a/backend/src/main/scala/aqua/backend/Backend.scala b/backend/src/main/scala/aqua/backend/Backend.scala index 7b08e220..8eeb365e 100644 --- a/backend/src/main/scala/aqua/backend/Backend.scala +++ b/backend/src/main/scala/aqua/backend/Backend.scala @@ -1,18 +1,19 @@ package aqua.backend import aqua.model.AquaContext -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig /** - * Compilation result - * @param suffix extension or another info that will be added to a resulted file - * @param content a code that is used as an output - */ -case class Compiled(suffix: String, content: String) - -/** - * Describes how context can be finalized + * Compiler backend generates output based on the processed model */ trait Backend { - def generate(context: AquaContext, bc: BodyConfig): Seq[Compiled] + + /** + * Generate the result based on the given [[AquaContext]] and [[GenerationConfig]] + * + * @param context Source file context, processed, transformed + * @param genConf Generation configuration + * @return Zero or more [[Generated]] objects, based on arguments + */ + def generate(context: AquaContext, genConf: GenerationConfig): Seq[Generated] } diff --git a/backend/src/main/scala/aqua/backend/Generated.scala b/backend/src/main/scala/aqua/backend/Generated.scala new file mode 100644 index 00000000..0392657f --- /dev/null +++ b/backend/src/main/scala/aqua/backend/Generated.scala @@ -0,0 +1,9 @@ +package aqua.backend + +/** + * Compilation result + * + * @param suffix extension or another info that will be added to a resulted file + * @param content compiled code + */ +case class Generated(suffix: String, content: String) diff --git a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptBackend.scala b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptBackend.scala index 73ff1feb..f1c6ecd0 100644 --- a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptBackend.scala +++ b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptBackend.scala @@ -1,21 +1,29 @@ package aqua.backend.ts -import aqua.backend.{Backend, Compiled} +import aqua.backend.{Backend, Generated} import aqua.model.AquaContext -import aqua.model.transform.BodyConfig -import cats.data.Chain +import aqua.model.transform.GenerationConfig +import cats.data.NonEmptyChain object TypeScriptBackend extends Backend { val ext = ".ts" - override def generate(context: AquaContext, bc: BodyConfig): Seq[Compiled] = { - val funcs = Chain.fromSeq(context.funcs.values.toSeq).map(TypeScriptFunc(_)) - Seq( - Compiled( - ext, - TypeScriptFile.Header + "\n\n" + funcs.map(_.generateTypescript(bc)).toList.mkString("\n\n") + override def generate(context: AquaContext, genConf: GenerationConfig): Seq[Generated] = { + val funcs = NonEmptyChain.fromSeq(context.funcs.values.toSeq).map(_.map(TypeScriptFunc(_))) + funcs + .map(fs => + Seq( + Generated( + ext, + TypeScriptFile.Header + "\n\n" + fs + .map(_.generateTypescript(genConf)) + .toChain + .toList + .mkString("\n\n") + ) + ) ) - ) + .getOrElse(Seq.empty) } } diff --git a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFile.scala b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFile.scala index 53f8447e..39af2908 100644 --- a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFile.scala +++ b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFile.scala @@ -1,7 +1,7 @@ package aqua.backend.ts import aqua.model.AquaContext -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import cats.data.Chain case class TypeScriptFile(context: AquaContext) { @@ -9,7 +9,7 @@ case class TypeScriptFile(context: AquaContext) { def funcs: Chain[TypeScriptFunc] = Chain.fromSeq(context.funcs.values.toSeq).map(TypeScriptFunc(_)) - def generateTS(conf: BodyConfig = BodyConfig()): String = + def generateTS(conf: GenerationConfig = GenerationConfig()): String = TypeScriptFile.Header + "\n\n" + funcs.map(_.generateTypescript(conf)).toList.mkString("\n\n") } diff --git a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFunc.scala b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFunc.scala index 577e80b6..1144c0cf 100644 --- a/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFunc.scala +++ b/backend/ts/src/main/scala/aqua/backend/ts/TypeScriptFunc.scala @@ -2,7 +2,7 @@ package aqua.backend.ts import aqua.backend.air.FuncAirGen import aqua.model.func.{ArgDef, FuncCallable} -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import aqua.types._ import cats.syntax.show._ @@ -46,7 +46,7 @@ case class TypeScriptFunc(func: FuncCallable) { |""".stripMargin } - def generateTypescript(conf: BodyConfig = BodyConfig()): String = { + def generateTypescript(conf: GenerationConfig = GenerationConfig()): String = { val tsAir = FuncAirGen(func).generateAir(conf) diff --git a/build.sbt b/build.sbt index ea872c33..bcc342fa 100644 --- a/build.sbt +++ b/build.sbt @@ -28,7 +28,7 @@ val cats = "org.typelevel" %% "cats-core" % catsV name := "aqua-hll" val commons = Seq( - baseAquaVersion := "0.1.10", + baseAquaVersion := "0.1.11", version := baseAquaVersion.value + "-" + sys.env.getOrElse("BUILD_NUMBER", "SNAPSHOT"), scalaVersion := dottyVersion, libraryDependencies ++= Seq( diff --git a/cli/src/main/scala/aqua/AppOps.scala b/cli/src/main/scala/aqua/AppOps.scala index 6a734b27..32a36510 100644 --- a/cli/src/main/scala/aqua/AppOps.scala +++ b/cli/src/main/scala/aqua/AppOps.scala @@ -1,7 +1,7 @@ package aqua import aqua.model.LiteralModel -import aqua.model.transform.Constant +import aqua.model.transform.GenerationConfig import aqua.parser.expr.ConstantExpr import aqua.parser.lift.LiftParser import cats.data.Validated.{Invalid, Valid} @@ -95,7 +95,7 @@ object AppOps { } .withDefault(List.empty) - def constantOpts[F[_]: LiftParser: Comonad]: Opts[List[Constant]] = + def constantOpts[F[_]: LiftParser: Comonad]: Opts[List[GenerationConfig.Const]] = Opts .options[String]("const", "Constant that will be used in an aqua code", "c") .mapValidated { strs => @@ -108,8 +108,9 @@ object AppOps { NonEmptyList .fromList(errors) .fold( - Validated.validNel[String, List[Constant]](parsed.collect { case Right(v) => - Constant(v._1.value, LiteralModel(v._2.value, v._2.ts)) + Validated.validNel[String, List[GenerationConfig.Const]](parsed.collect { + case Right(v) => + GenerationConfig.Const(v._1.value, LiteralModel(v._2.value, v._2.ts)) }) ) { errors => Validated.invalid(errors.map(_.toString)) diff --git a/cli/src/main/scala/aqua/AquaCli.scala b/cli/src/main/scala/aqua/AquaCli.scala index 91a8568c..46aa8ec8 100644 --- a/cli/src/main/scala/aqua/AquaCli.scala +++ b/cli/src/main/scala/aqua/AquaCli.scala @@ -4,8 +4,8 @@ import aqua.backend.Backend import aqua.backend.air.AirBackend import aqua.backend.js.JavaScriptBackend import aqua.backend.ts.TypeScriptBackend -import aqua.compiler.{AquaCompiler, AquaIO} -import aqua.model.transform.BodyConfig +import aqua.files.AquaFilesIO +import aqua.model.transform.GenerationConfig import aqua.parser.lift.LiftParser.Implicits.idLiftParser import cats.Id import cats.data.Validated @@ -18,17 +18,7 @@ import com.monovore.decline.effect.CommandIOApp import fs2.io.file.Files import org.typelevel.log4cats.slf4j.Slf4jLogger import org.typelevel.log4cats.{Logger, SelfAwareStructuredLogger} -import wvlet.log.LogFormatter.{appendStackTrace, highlightLog} -import wvlet.log.{LogFormatter, LogRecord, LogSupport, Logger => WLogger} - -object CustomLogFormatter extends LogFormatter { - - override def formatLog(r: LogRecord): String = { - val log = - s"[${highlightLog(r.level, r.level.name)}] ${highlightLog(r.level, r.getMessage)}" - appendStackTrace(log, r) - } -} +import wvlet.log.{LogSupport, Logger => WLogger} object AquaCli extends IOApp with LogSupport { import AppOps._ @@ -82,11 +72,11 @@ object AquaCli extends IOApp with LogSupport { else if (toJs) JavaScriptTarget else TypescriptTarget val bc = { - val bc = BodyConfig(wrapWithXor = !noXor, constants = constants) + val bc = GenerationConfig(wrapWithXor = !noXor, constants = constants) bc.copy(relayVarName = bc.relayVarName.filterNot(_ => noRelay)) } info(s"Aqua Compiler ${versionStr}") - AquaCompiler + AquaPathCompiler .compileFilesTo[F]( input, imports, @@ -96,10 +86,10 @@ object AquaCli extends IOApp with LogSupport { ) .map { case Validated.Invalid(errs) => - errs.map(println) + errs.map(System.out.println) ExitCode.Error case Validated.Valid(results) => - results.map(println) + results.map(info(_)) ExitCode.Success } } diff --git a/compiler/src/main/scala/aqua/compiler/AquaIO.scala b/cli/src/main/scala/aqua/AquaIO.scala similarity index 80% rename from compiler/src/main/scala/aqua/compiler/AquaIO.scala rename to cli/src/main/scala/aqua/AquaIO.scala index 4816e735..a92681e6 100644 --- a/compiler/src/main/scala/aqua/compiler/AquaIO.scala +++ b/cli/src/main/scala/aqua/AquaIO.scala @@ -1,7 +1,6 @@ -package aqua.compiler +package aqua -import aqua.compiler.io.AquaFileError -import aqua.parser.lift.FileSpan +import aqua.io.AquaFileError import cats.data.{Chain, EitherT, ValidatedNec} import java.nio.file.Path @@ -10,7 +9,6 @@ trait AquaIO[F[_]] { def readFile(file: Path): EitherT[F, AquaFileError, String] def resolve( - focus: FileSpan.Focus, src: Path, imports: List[Path] ): EitherT[F, AquaFileError, Path] diff --git a/cli/src/main/scala/aqua/AquaPathCompiler.scala b/cli/src/main/scala/aqua/AquaPathCompiler.scala new file mode 100644 index 00000000..c895f54b --- /dev/null +++ b/cli/src/main/scala/aqua/AquaPathCompiler.scala @@ -0,0 +1,39 @@ +package aqua + +import aqua.backend.Backend +import aqua.compiler.{AquaCompiler, AquaError} +import aqua.files.{AquaFileSources, FileModuleId} +import aqua.io._ +import aqua.model.transform.GenerationConfig +import aqua.parser.lift.FileSpan +import cats.data._ +import cats.syntax.functor._ +import cats.syntax.show._ +import cats.{Monad, Show} +import wvlet.log.LogSupport + +import java.nio.file.Path + +object AquaPathCompiler extends LogSupport { + + def compileFilesTo[F[_]: AquaIO: Monad]( + srcPath: Path, + imports: List[Path], + targetPath: Path, + backend: Backend, + bodyConfig: GenerationConfig + ): F[ValidatedNec[String, Chain[String]]] = { + import ErrorRendering.showError + val sources = new AquaFileSources[F](srcPath, imports) + AquaCompiler + .compileTo[F, AquaFileError, FileModuleId, FileSpan.F, String]( + sources, + (fmid, src) => FileSpan.fileSpanLiftParser(fmid.file.toString, src), + backend, + bodyConfig, + sources.write(targetPath) + ) + .map(_.leftMap(_.map(_.show))) + } + +} diff --git a/cli/src/main/scala/aqua/CustomLogFormatter.scala b/cli/src/main/scala/aqua/CustomLogFormatter.scala new file mode 100644 index 00000000..793ed777 --- /dev/null +++ b/cli/src/main/scala/aqua/CustomLogFormatter.scala @@ -0,0 +1,13 @@ +package aqua + +import wvlet.log.LogFormatter.{appendStackTrace, highlightLog} +import wvlet.log.{LogFormatter, LogRecord} + +object CustomLogFormatter extends LogFormatter { + + override def formatLog(r: LogRecord): String = { + val log = + s"[${highlightLog(r.level, r.level.name)}] ${highlightLog(r.level, r.getMessage)}" + appendStackTrace(log, r) + } +} diff --git a/cli/src/main/scala/aqua/ErrorRendering.scala b/cli/src/main/scala/aqua/ErrorRendering.scala new file mode 100644 index 00000000..1ecc5b29 --- /dev/null +++ b/cli/src/main/scala/aqua/ErrorRendering.scala @@ -0,0 +1,72 @@ +package aqua + +import aqua.compiler._ +import aqua.files.FileModuleId +import aqua.io.AquaFileError +import aqua.parser.lift.FileSpan +import aqua.parser.{BlockIndentError, FuncReturnError, LexerError} +import aqua.semantics.{RulesViolated, WrongAST} +import cats.Show + +object ErrorRendering { + + def showForConsole(span: FileSpan, message: String): String = + span + .focus(3) + .map( + _.toConsoleStr( + message, + Console.RED + ) + ) + .getOrElse( + "(offset is beyond the script, syntax errors) Error: " + Console.RED + message + .mkString(", ") + ) + Console.RESET + "\n" + + implicit val showError: Show[AquaError[FileModuleId, AquaFileError, FileSpan.F]] = Show.show { + case ParserErr(err) => + err match { + case BlockIndentError(indent, message) => showForConsole(indent._1, message) + case FuncReturnError(point, message) => showForConsole(point._1, message) + case LexerError((span, e)) => + span + .focus(3) + .map(spanFocus => + spanFocus.toConsoleStr( + s"Syntax error, expected: ${e.expected.toList.mkString(", ")}", + Console.RED + ) + ) + .getOrElse( + "(offset is beyond the script, syntax errors) " + Console.RED + e.expected.toList + .mkString(", ") + ) + Console.RESET + "\n" + } + case SourcesErr(err) => + Console.RED + err.showForConsole + Console.RESET + case ResolveImportsErr(_, token, err) => + val span = token.unit._1 + showForConsole(span, s"Cannot resolve imports: ${err.showForConsole}") + + case ImportErr(token) => + val span = token.unit._1 + showForConsole(span, s"Cannot resolve import") + case CycleError(modules) => + s"Cycle loops detected in imports: ${modules.map(_.file.getFileName)}" + case CompileError(err) => + err match { + case RulesViolated(token, message) => + token.unit._1 + .focus(2) + .map(_.toConsoleStr(message, Console.CYAN)) + .getOrElse("(Dup error, but offset is beyond the script)") + "\n" + case WrongAST(ast) => + s"Semantic error" + + } + + case OutputError(_, err) => + Console.RED + err.showForConsole + Console.RESET + } +} diff --git a/cli/src/main/scala/aqua/Test.scala b/cli/src/main/scala/aqua/Test.scala index 43be5032..1b09e271 100644 --- a/cli/src/main/scala/aqua/Test.scala +++ b/cli/src/main/scala/aqua/Test.scala @@ -1,8 +1,8 @@ package aqua import aqua.backend.ts.TypeScriptBackend -import aqua.compiler.{AquaCompiler, AquaIO} -import aqua.model.transform.BodyConfig +import aqua.files.AquaFilesIO +import aqua.model.transform.GenerationConfig import cats.data.Validated import cats.effect.{IO, IOApp, Sync} import org.typelevel.log4cats.SelfAwareStructuredLogger @@ -18,19 +18,19 @@ object Test extends IOApp.Simple { implicit val aio: AquaIO[IO] = new AquaFilesIO[IO] override def run: IO[Unit] = - AquaCompiler + AquaPathCompiler .compileFilesTo[IO]( Paths.get("./aqua-src"), List(Paths.get("./aqua")), Paths.get("./target"), TypeScriptBackend, - BodyConfig() + GenerationConfig() ) .map { case Validated.Invalid(errs) => - errs.map(println) - case Validated.Valid(_) => - + errs.map(System.err.println) + case Validated.Valid(res) => + res.map(println) } } diff --git a/cli/src/main/scala/aqua/files/AquaFileSources.scala b/cli/src/main/scala/aqua/files/AquaFileSources.scala new file mode 100644 index 00000000..8a284131 --- /dev/null +++ b/cli/src/main/scala/aqua/files/AquaFileSources.scala @@ -0,0 +1,124 @@ +package aqua.files + +import aqua.AquaIO +import aqua.compiler.{AquaCompiled, AquaSources} +import aqua.io.{AquaFileError, FileSystemError, ListAquaErrors} +import cats.Monad +import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec} +import cats.implicits.catsSyntaxApplicativeId +import cats.syntax.either._ +import cats.syntax.flatMap._ +import cats.syntax.functor._ +import cats.syntax.traverse._ + +import java.nio.file.{Path, Paths} +import scala.util.Try + +class AquaFileSources[F[_]: AquaIO: Monad](sourcesPath: Path, importFrom: List[Path]) + extends AquaSources[F, AquaFileError, FileModuleId] { + private val filesIO = implicitly[AquaIO[F]] + + override def sources: F[ValidatedNec[AquaFileError, Chain[(FileModuleId, String)]]] = + filesIO.listAqua(sourcesPath).flatMap { + case Validated.Valid(files) => + files + .map(f => + filesIO + .readFile(f) + .value + .map[ValidatedNec[AquaFileError, Chain[(FileModuleId, String)]]] { + case Left(err) => Validated.invalidNec(err) + case Right(content) => Validated.validNec(Chain.one(FileModuleId(f) -> content)) + } + ) + .traverse(identity) + .map( + _.foldLeft[ValidatedNec[AquaFileError, Chain[(FileModuleId, String)]]]( + Validated.validNec(Chain.nil) + )(_ combine _) + ) + case Validated.Invalid(e) => + Validated + .invalidNec[AquaFileError, Chain[(FileModuleId, String)]](ListAquaErrors(e)) + .pure[F] + } + + // Resolve an import that was written in a 'from' file + // Try to find it in a list of given imports or near 'from' file + override def resolveImport( + from: FileModuleId, + imp: String + ): F[ValidatedNec[AquaFileError, FileModuleId]] = { + Validated.fromEither(Try(Paths.get(imp)).toEither.leftMap(FileSystemError)) match { + case Validated.Valid(importP) => + filesIO + .resolve(importP, from.file.getParent +: importFrom) + .bimap(NonEmptyChain.one, FileModuleId(_)) + .value + .map(Validated.fromEither) + case Validated.Invalid(err) => Validated.invalidNec[AquaFileError, FileModuleId](err).pure[F] + } + + } + + override def load(file: FileModuleId): F[ValidatedNec[AquaFileError, String]] = + filesIO.readFile(file.file).leftMap(NonEmptyChain.one).value.map(Validated.fromEither) + + /** + * @param srcFile aqua source + * @param targetPath a main path where all output files will be written + * @param suffix `.aqua` will be replaced with this suffix + * @return + */ + def resolveTargetPath( + srcFile: Path, + targetPath: Path, + suffix: String + ): Validated[Throwable, Path] = + Validated.catchNonFatal { + val srcDir = if (sourcesPath.toFile.isDirectory) sourcesPath else sourcesPath.getParent + val srcFilePath = srcDir.toAbsolutePath + .normalize() + .relativize(srcFile.toAbsolutePath.normalize()) + + val targetDir = + targetPath.toAbsolutePath + .normalize() + .resolve( + srcFilePath + ) + + targetDir.getParent.resolve(srcFile.getFileName.toString.stripSuffix(".aqua") + suffix) + } + + def write( + targetPath: Path + )(ac: AquaCompiled[FileModuleId]): F[Seq[Validated[AquaFileError, String]]] = + if (ac.compiled.isEmpty) + Seq( + Validated.valid[AquaFileError, String]( + s"Source ${ac.sourceId.file}: compilation OK (nothing to emit)" + ) + ).pure[F] + else + ac.compiled.map { compiled => + resolveTargetPath( + ac.sourceId.file, + targetPath, + compiled.suffix + ).leftMap(FileSystemError) + .map { target => + filesIO + .writeFile( + target, + compiled.content + ) + .as(s"Result $target: compilation OK (${ac.compiled.size} functions)") + .value + .map(Validated.fromEither) + } + // TODO: we use both EitherT and F[Validated] to handle errors, that's why so weird + .traverse(identity) + }.traverse(identity) + .map(_.map(_.andThen(identity))) +} diff --git a/cli/src/main/scala/aqua/AquaFilesIO.scala b/cli/src/main/scala/aqua/files/AquaFilesIO.scala similarity index 70% rename from cli/src/main/scala/aqua/AquaFilesIO.scala rename to cli/src/main/scala/aqua/files/AquaFilesIO.scala index bd322b71..544dca2e 100644 --- a/cli/src/main/scala/aqua/AquaFilesIO.scala +++ b/cli/src/main/scala/aqua/files/AquaFilesIO.scala @@ -1,24 +1,18 @@ -package aqua +package aqua.files -import aqua.compiler.AquaIO -import aqua.compiler.io.{ - AquaFileError, - EmptyFileError, - FileNotFound, - FileSystemError, - FileWriteError -} -import aqua.parser.lift.FileSpan +import aqua.AquaIO +import aqua.io._ import cats.data.Validated.{Invalid, Valid} -import cats.data.{Chain, EitherT, NonEmptyChain, Validated, ValidatedNec} -import cats.syntax.functor._ -import cats.syntax.either._ +import cats.data._ import cats.effect.kernel.Concurrent +import cats.syntax.applicative._ +import cats.syntax.apply._ +import cats.syntax.either._ +import cats.syntax.flatMap._ +import cats.syntax.functor._ +import cats.syntax.applicativeError._ import fs2.io.file.Files import fs2.text -import cats.syntax.applicative._ -import cats.syntax.flatMap._ -import cats.syntax.apply._ import java.nio.file.Path import scala.util.Try @@ -43,6 +37,10 @@ class AquaFilesIO[F[_]: Files: Concurrent] extends AquaIO[F] { ) ) + /** + * Find the first file that exists in the given list of paths + * If there is no such file - error + */ private def findFirstF( in: List[Path], notFound: EitherT[F, AquaFileError, Path] @@ -67,29 +65,29 @@ class AquaFilesIO[F[_]: Files: Concurrent] extends AquaIO[F] { * Checks if a file exists in the list of possible paths */ def resolve( - focus: FileSpan.Focus, src: Path, imports: List[Path] ): EitherT[F, AquaFileError, Path] = findFirstF( imports .map(_.resolve(src)), - EitherT.leftT(FileNotFound(focus, src, imports)) + EitherT.leftT(FileNotFound(src, imports)) ) override def listAqua(folder: Path): F[ValidatedNec[AquaFileError, Chain[Path]]] = Validated - .fromTry( + .fromEither( Try { val f = folder.toFile - if (f.isDirectory) { - f.listFiles().toList + if (!f.exists()) { + Left(FileNotFound(folder, Nil)) + } else if (f.isDirectory) { + Right(f.listFiles().toList) } else { - f :: Nil + Right(f :: Nil) } - } + }.toEither.leftMap[AquaFileError](FileSystemError).flatMap(identity) ) - .leftMap[AquaFileError](FileSystemError) .leftMap(NonEmptyChain.one) .pure[F] .flatMap { @@ -113,27 +111,27 @@ class AquaFilesIO[F[_]: Files: Concurrent] extends AquaIO[F] { Validated.invalid[NonEmptyChain[AquaFileError], Chain[Path]](errs).pure[F] } + private def deleteIfExists(file: Path): EitherT[F, AquaFileError, Boolean] = + Files[F].deleteIfExists(file).attemptT.leftMap(FileSystemError) + + private def createDirectories(path: Path): EitherT[F, AquaFileError, Path] = + Files[F].createDirectories(path).attemptT.leftMap(FileSystemError) + + // Writes to a file, creates directories if they do not exist override def writeFile(file: Path, content: String): EitherT[F, AquaFileError, Unit] = - EitherT - .right[AquaFileError](Files[F].deleteIfExists(file)) - .flatMap(_ => - EitherT[F, AquaFileError, Unit]( - fs2.Stream - .emit( - content - ) - .through(text.utf8Encode) - .through(Files[F].writeAll(file)) - .attempt - .map { e => - e.left - .map(t => FileWriteError(file, t)) - } - .compile - .drain - .map(_ => Right(())) - ) + deleteIfExists(file) >> createDirectories(file.getParent) >> + EitherT( + fs2.Stream + .emit(content) + .through(text.utf8Encode) + .through(Files[F].writeAll(file)) + .attempt + .compile + .last + .map(_.getOrElse(Right())) ) + .leftMap(FileWriteError(file, _)) + } object AquaFilesIO { diff --git a/cli/src/main/scala/aqua/files/FileModuleId.scala b/cli/src/main/scala/aqua/files/FileModuleId.scala new file mode 100644 index 00000000..dbb882d7 --- /dev/null +++ b/cli/src/main/scala/aqua/files/FileModuleId.scala @@ -0,0 +1,11 @@ +package aqua.files + +import java.nio.file.Path + +case class FileModuleId private (file: Path) + +object FileModuleId { + + def apply(file: Path): FileModuleId = + new FileModuleId(file.toAbsolutePath.normalize()) +} diff --git a/compiler/src/main/scala/aqua/compiler/io/AquaFileError.scala b/cli/src/main/scala/aqua/io/AquaFileError.scala similarity index 52% rename from compiler/src/main/scala/aqua/compiler/io/AquaFileError.scala rename to cli/src/main/scala/aqua/io/AquaFileError.scala index 8d5aa27f..1a93d6ed 100644 --- a/compiler/src/main/scala/aqua/compiler/io/AquaFileError.scala +++ b/cli/src/main/scala/aqua/io/AquaFileError.scala @@ -1,7 +1,5 @@ -package aqua.compiler.io +package aqua.io -import aqua.compiler.AquaError -import aqua.parser.lift.FileSpan import cats.data.NonEmptyChain import java.nio.file.Path @@ -12,13 +10,19 @@ sealed trait AquaFileError { override def toString: String = showForConsole } -case class FileNotFound(focus: FileSpan.Focus, name: Path, imports: Seq[Path]) - extends AquaFileError { +case class ListAquaErrors(errors: NonEmptyChain[AquaFileError]) extends AquaFileError { - override def showForConsole: String = focus.toConsoleStr( - s"File not found at $name, looking in ${imports.mkString(", ")}", - Console.YELLOW - ) + override def showForConsole: String = + s"Cannot read '*.aqua' files:\n" + errors.map(_.showForConsole) +} + +case class FileNotFound(name: Path, imports: Seq[Path]) extends AquaFileError { + + override def showForConsole: String = + if (imports.nonEmpty) + s"File '$name' not found, looking in ${imports.mkString(", ")}" + else + s"File '$name' not found" } case class EmptyFileError(path: Path) extends AquaFileError { @@ -26,20 +30,13 @@ case class EmptyFileError(path: Path) extends AquaFileError { } case class FileSystemError(err: Throwable) extends Exception(err) with AquaFileError { - override def showForConsole: String = s"File system error: ${err.getMessage}" + override def showForConsole: String = s"File system error: $err" } case class FileWriteError(file: Path, err: Throwable) extends Exception(err) with AquaFileError { - override def showForConsole: String = s"Cannot write a file $file: ${err.getMessage}" + override def showForConsole: String = s"Cannot write a file $file: $err" } case class Unresolvable(msg: String) extends AquaFileError { override def showForConsole: String = s"Unresolvable: $msg" } - -// TODO there should be no AquaErrors, as they does not fit -case class AquaScriptErrors(errors: NonEmptyChain[AquaError]) extends AquaFileError { - - override def showForConsole: String = - errors.map(_.showForConsole).toChain.toList.mkString("\n") -} diff --git a/cli/src/test/scala/SourcesSpec.scala b/cli/src/test/scala/SourcesSpec.scala new file mode 100644 index 00000000..bea13f00 --- /dev/null +++ b/cli/src/test/scala/SourcesSpec.scala @@ -0,0 +1,157 @@ +import aqua.AquaIO +import aqua.backend.Generated +import aqua.compiler.AquaCompiled +import aqua.files.{AquaFileSources, AquaFilesIO, FileModuleId} +import cats.data.Chain +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import fs2.io.file.Files +import fs2.text +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.Paths + +class SourcesSpec extends AnyFlatSpec with Matchers { + implicit val aquaIO: AquaIO[IO] = AquaFilesIO.summon[IO] + + "AquaFileSources" should "generate correct fileId with imports" in { + val path = Paths.get("cli/src/test/test-dir/path-test") + val importPath = path.resolve("imports") + + val sourceGen = new AquaFileSources[IO](path, importPath :: Nil) + + val result = sourceGen.sources.unsafeRunSync() + result.isValid shouldBe true + + val listResult = result + .getOrElse(Chain.empty) + .toList + .map { case (fid, s) => + (fid.file.toString.split("/").last, s) + } + .sortBy(_._1) // sort cause different systems have different order of file reading + + val (id, importFile) = listResult(1) + id shouldBe "index.aqua" + importFile.nonEmpty shouldBe true + + val (importNearId, importFileNear) = listResult.head + importNearId shouldBe "importNear.aqua" + importFileNear.nonEmpty shouldBe true + } + + "AquaFileSources" should "throw an error if a source file doesn't exist" in { + val path = Paths.get("some/random/path") + + val sourceGen = new AquaFileSources[IO](path, Nil) + + val result = sourceGen.sources.unsafeRunSync() + result.isInvalid shouldBe true + } + + "AquaFileSources" should "throw an error if there is no import that is indicated in a source" in { + val path = Paths.get("cli/src/test/test-dir") + val importPath = path.resolve("random/import/path") + + val sourceGen = new AquaFileSources[IO](path, importPath :: Nil) + val result = + sourceGen.resolveImport(FileModuleId(path.resolve("no-file.aqua")), "no/file").unsafeRunSync() + result.isInvalid shouldBe true + } + + "AquaFileSources" should "find correct imports" in { + val srcPath = Paths.get("cli/src/test/test-dir/index.aqua") + val importPath = srcPath.resolve("imports") + + val sourceGen = new AquaFileSources[IO](srcPath, importPath :: Nil) + + // should be found in importPath + val result = + sourceGen + .resolveImport(FileModuleId(srcPath), "imports/import.aqua") + .unsafeRunSync() + + result.isValid shouldBe true + result.getOrElse(FileModuleId(Paths.get("/some/random"))).file.toFile.exists() shouldBe true + + // should be found near src file + val result2 = + sourceGen + .resolveImport(FileModuleId(srcPath), "importNear.aqua") + .unsafeRunSync() + + result2.isValid shouldBe true + result2.getOrElse(FileModuleId(Paths.get("/some/random"))).file.toFile.exists() shouldBe true + + // near src file but in another directory + val sourceGen2 = new AquaFileSources[IO](srcPath, Nil) + val result3 = + sourceGen2 + .resolveImport(FileModuleId(srcPath), "imports/import.aqua") + .unsafeRunSync() + + result3.isValid shouldBe true + result3.getOrElse(FileModuleId(Paths.get("/some/random"))).file.toFile.exists() shouldBe true + } + + "AquaFileSources" should "resolve correct path for target" in { + val path = Paths.get("cli/src/test/test-dir") + val filePath = path.resolve("some-dir/file.aqua") + + val targetPath = Paths.get("/target/dir/") + + val sourceGen = new AquaFileSources[IO](path, Nil) + + val suffix = "_custom.super" + + val resolved = sourceGen.resolveTargetPath(filePath, targetPath, suffix) + resolved.isValid shouldBe true + + val targetFilePath = resolved.toOption.get + targetFilePath.toString shouldBe "/target/dir/some-dir/file_custom.super" + } + + "AquaFileSources" should "write correct file with correct path" in { + val path = Paths.get("cli/src/test/test-dir") + val filePath = path.resolve("imports/import.aqua") + + val targetPath = path.resolve("target/") + + // clean up + val resultPath = Paths.get("cli/src/test/test-dir/target/imports/import_hey.custom") + Files[IO].deleteIfExists(resultPath).unsafeRunSync() + + val sourceGen = new AquaFileSources[IO](path, Nil) + val content = "some random content" + val compiled = AquaCompiled[FileModuleId]( + FileModuleId(filePath), + Seq(Generated("_hey.custom", content)) + ) + + val resolved = sourceGen.write(targetPath)(compiled).unsafeRunSync() + resolved.size shouldBe 1 + resolved.head.isValid shouldBe true + + Files[IO].exists(resultPath).unsafeRunSync() shouldBe true + + val resultText = Files[IO] + .readAll(resultPath, 1000) + .fold( + Vector + .empty[Byte] + )((acc, b) => acc :+ b) + .flatMap(fs2.Stream.emits) + .through(text.utf8Decode) + .attempt + .compile + .last + .unsafeRunSync() + .get + .right + .get + resultText shouldBe content + + Files[IO].deleteIfExists(resultPath).unsafeRunSync() + } +} diff --git a/cli/src/test/scala/WriteFileSpec.scala b/cli/src/test/scala/WriteFileSpec.scala index 0330375f..6aefc227 100644 --- a/cli/src/test/scala/WriteFileSpec.scala +++ b/cli/src/test/scala/WriteFileSpec.scala @@ -1,8 +1,8 @@ +import aqua.AquaPathCompiler import aqua.backend.air.AirBackend import aqua.backend.js.JavaScriptBackend import aqua.backend.ts.TypeScriptBackend -import aqua.compiler.AquaCompiler -import aqua.model.transform.BodyConfig +import aqua.model.transform.GenerationConfig import cats.effect.IO import cats.effect.unsafe.implicits.global import org.scalatest.flatspec.AnyFlatSpec @@ -17,10 +17,10 @@ class WriteFileSpec extends AnyFlatSpec with Matchers { val targetJs = Files.createTempDirectory("js") val targetAir = Files.createTempDirectory("air") - import aqua.AquaFilesIO.summon + import aqua.files.AquaFilesIO.summon - val bc = BodyConfig() - AquaCompiler + val bc = GenerationConfig() + AquaPathCompiler .compileFilesTo[IO](src, List.empty, targetTs, TypeScriptBackend, bc) .unsafeRunSync() .leftMap { err => @@ -32,7 +32,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers { targetTsFile.toFile.exists() should be(true) Files.deleteIfExists(targetTsFile) - AquaCompiler + AquaPathCompiler .compileFilesTo[IO](src, List.empty, targetJs, JavaScriptBackend, bc) .unsafeRunSync() .leftMap { err => @@ -44,7 +44,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers { targetJsFile.toFile.exists() should be(true) Files.deleteIfExists(targetJsFile) - AquaCompiler + AquaPathCompiler .compileFilesTo[IO](src, List.empty, targetAir, AirBackend, bc) .unsafeRunSync() .leftMap { err => diff --git a/cli/src/test/test-dir/broken-import/broken.aqua b/cli/src/test/test-dir/broken-import/broken.aqua new file mode 100644 index 00000000..919b4bdb --- /dev/null +++ b/cli/src/test/test-dir/broken-import/broken.aqua @@ -0,0 +1,4 @@ +import "random/import/import.aqua" + +func indexCall(): + Println.print("it is true") \ No newline at end of file diff --git a/cli/src/test/test-dir/importNear.aqua b/cli/src/test/test-dir/importNear.aqua new file mode 100644 index 00000000..1a5f47af --- /dev/null +++ b/cli/src/test/test-dir/importNear.aqua @@ -0,0 +1,5 @@ +service Println("println-service-id"): + print: string -> () + +func print(str: string): + Println.print(str) diff --git a/cli/src/test/test-dir/imports/import.aqua b/cli/src/test/test-dir/imports/import.aqua new file mode 100644 index 00000000..1a5f47af --- /dev/null +++ b/cli/src/test/test-dir/imports/import.aqua @@ -0,0 +1,5 @@ +service Println("println-service-id"): + print: string -> () + +func print(str: string): + Println.print(str) diff --git a/cli/src/test/test-dir/index.aqua b/cli/src/test/test-dir/index.aqua new file mode 100644 index 00000000..2adf2e62 --- /dev/null +++ b/cli/src/test/test-dir/index.aqua @@ -0,0 +1,4 @@ +import "imports/import.aqua" + +func indexCall(): + Println.print("it is true") \ No newline at end of file diff --git a/cli/src/test/test-dir/path-test/importNear.aqua b/cli/src/test/test-dir/path-test/importNear.aqua new file mode 100644 index 00000000..1a5f47af --- /dev/null +++ b/cli/src/test/test-dir/path-test/importNear.aqua @@ -0,0 +1,5 @@ +service Println("println-service-id"): + print: string -> () + +func print(str: string): + Println.print(str) diff --git a/cli/src/test/test-dir/path-test/index.aqua b/cli/src/test/test-dir/path-test/index.aqua new file mode 100644 index 00000000..2adf2e62 --- /dev/null +++ b/cli/src/test/test-dir/path-test/index.aqua @@ -0,0 +1,4 @@ +import "imports/import.aqua" + +func indexCall(): + Println.print("it is true") \ No newline at end of file diff --git a/compiler/src/main/scala/aqua/compiler/AquaCompiled.scala b/compiler/src/main/scala/aqua/compiler/AquaCompiled.scala new file mode 100644 index 00000000..e7a91690 --- /dev/null +++ b/compiler/src/main/scala/aqua/compiler/AquaCompiled.scala @@ -0,0 +1,5 @@ +package aqua.compiler + +import aqua.backend.Generated + +case class AquaCompiled[I](sourceId: I, compiled: Seq[Generated]) diff --git a/compiler/src/main/scala/aqua/compiler/AquaCompiler.scala b/compiler/src/main/scala/aqua/compiler/AquaCompiler.scala index ed62e7b5..f27ae591 100644 --- a/compiler/src/main/scala/aqua/compiler/AquaCompiler.scala +++ b/compiler/src/main/scala/aqua/compiler/AquaCompiler.scala @@ -1,167 +1,90 @@ package aqua.compiler import aqua.backend.Backend -import aqua.compiler.io._ import aqua.linker.Linker import aqua.model.AquaContext -import aqua.model.transform.BodyConfig -import aqua.parser.lift.FileSpan -import aqua.semantics.{RulesViolated, SemanticError, Semantics, WrongAST} -import cats.data._ -import cats.kernel.Monoid +import aqua.model.transform.GenerationConfig +import aqua.parser.lift.LiftParser +import aqua.semantics.Semantics +import cats.data.Validated.{validNec, Invalid, Valid} +import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec} +import cats.syntax.applicative._ import cats.syntax.flatMap._ import cats.syntax.functor._ -import cats.{Applicative, Monad} -import wvlet.log.LogSupport +import cats.syntax.traverse._ +import cats.{Comonad, Monad} -import java.nio.file.Path +object AquaCompiler { -object AquaCompiler extends LogSupport { - - private def gatherPreparedFiles( - srcPath: Path, - targetPath: Path, - files: Map[FileModuleId, ValidatedNec[SemanticError[FileSpan.F], AquaContext]] - ): ValidatedNec[String, Chain[Prepared]] = { - val (errs, _, preps) = files.toSeq.foldLeft[(Chain[String], Set[String], Chain[Prepared])]( - (Chain.empty, Set.empty, Chain.empty) - ) { case ((errs, errsSet, preps), (modId, proc)) => - proc.fold( - es => { - val newErrs = showProcErrors(es.toChain).filterNot(errsSet.contains) - (errs ++ newErrs, errsSet ++ newErrs.iterator, preps) - }, - c => { - Prepared(modId.file, srcPath, targetPath, c) match { - case Validated.Valid(p) ⇒ - (errs, errsSet, preps :+ p) - case Validated.Invalid(err) ⇒ - (errs :+ err.getMessage, errsSet, preps) - } - - } - ) - } - NonEmptyChain - .fromChain(errs) - .fold(Validated.validNec[String, Chain[Prepared]](preps))(Validated.invalid) - } - - /** - * Create a structure that will be used to create output by a backend - */ - def prepareFiles[F[_]: AquaIO: Monad]( - srcPath: Path, - imports: List[Path], - targetPath: Path - )(implicit aqum: Monoid[AquaContext]): F[ValidatedNec[String, Chain[Prepared]]] = - AquaFiles - .readAndResolve[F, ValidatedNec[SemanticError[FileSpan.F], AquaContext]]( - srcPath, - imports, - ast => context => context.andThen(ctx => Semantics.process(ast, ctx)) - ) - .value - .map { - case Left(fileErrors) => - Validated.invalid(fileErrors.map(_.showForConsole)) - - case Right(modules) => - Linker[FileModuleId, AquaFileError, ValidatedNec[SemanticError[FileSpan.F], AquaContext]]( - modules, - ids => Unresolvable(ids.map(_.id.file.toString).mkString(" -> ")) - ) match { - case Validated.Valid(files) ⇒ - gatherPreparedFiles(srcPath, targetPath, files) - - case Validated.Invalid(errs) ⇒ - Validated.invalid( - errs - .map(_.showForConsole) - ) - } - } - - def showProcErrors( - errors: Chain[SemanticError[FileSpan.F]] - ): Chain[String] = - errors.map { - case RulesViolated(token, hint) => - token.unit._1 - .focus(2) - .map(_.toConsoleStr(hint, Console.CYAN)) - .getOrElse("(Dup error, but offset is beyond the script)") + "\n" - case WrongAST(_) => - "Semantic error" - } - - private def gatherResults[F[_]: Monad]( - results: List[EitherT[F, String, Unit]] - ): F[ValidatedNec[String, Chain[String]]] = { - results - .foldLeft( - EitherT.rightT[F, NonEmptyChain[String]](Chain.empty[String]) - ) { case (accET, writeET) => - EitherT(for { - acc <- accET.value - writeResult <- writeET.value - } yield (acc, writeResult) match { - case (Left(errs), Left(err)) => Left(errs :+ err) - case (Right(res), Right(_)) => Right(res) - case (Left(errs), _) => Left(errs) - case (_, Left(err)) => Left(NonEmptyChain.of(err)) - }) - } - .value - .map(Validated.fromEither) - } - - def compileFilesTo[F[_]: AquaIO: Monad]( - srcPath: Path, - imports: List[Path], - targetPath: Path, + def compile[F[_]: Monad, E, I, S[_]: Comonad]( + sources: AquaSources[F, E, I], + liftI: (I, String) => LiftParser[S], backend: Backend, - bodyConfig: BodyConfig - ): F[ValidatedNec[String, Chain[String]]] = { - import bodyConfig.aquaContextMonoid - prepareFiles(srcPath, imports, targetPath) - .map(_.map(_.filter { p => - val hasOutput = p.hasOutput - if (!hasOutput) info(s"Source ${p.srcFile}: compilation OK (nothing to emit)") - hasOutput - })) - .flatMap[ValidatedNec[String, Chain[String]]] { - case Validated.Invalid(e) => - Applicative[F].pure(Validated.invalid(e)) - case Validated.Valid(preps) => - val results = preps.toList - .flatMap(p => - backend.generate(p.context, bodyConfig).map { compiled => - val targetPath = p.targetPath( - p.srcFile.getFileName.toString.stripSuffix(".aqua") + compiled.suffix + config: GenerationConfig + ): F[ValidatedNec[AquaError[I, E, S], Chain[AquaCompiled[I]]]] = { + import config.aquaContextMonoid + type Err = AquaError[I, E, S] + new AquaParser[F, E, I, S](sources, liftI) + .resolve[ValidatedNec[Err, AquaContext]](ast => + context => + context.andThen(ctx => Semantics.process(ast, ctx).leftMap(_.map[Err](CompileError(_)))) + ) + .map { + case Valid(modules) => + Linker.link[I, AquaError[I, E, S], ValidatedNec[Err, AquaContext]]( + modules, + cycle => CycleError[I, E, S](cycle.map(_.id)) + ) match { + case Valid(filesWithContext) => + filesWithContext + .foldLeft[ValidatedNec[Err, Chain[AquaProcessed[I]]]]( + validNec(Chain.nil) + ) { + case (acc, (i, Valid(context))) => + acc combine validNec(Chain.one(AquaProcessed(i, context))) + case (acc, (_, Invalid(errs))) => + acc combine Invalid(errs) + } + .map( + _.map { ap => + val compiled = backend.generate(ap.context, config) + AquaCompiled(ap.id, compiled) + } ) - - targetPath.fold( - t => EitherT.leftT[F, Unit](t.getMessage), - tp => - AquaIO[F] - .writeFile(tp, compiled.content) - .flatTap { _ => - EitherT.pure( - Validated.catchNonFatal( - info( - s"Result ${tp.toAbsolutePath}: compilation OK (${p.context.funcs.size} functions)" - ) - ) - ) - } - .leftMap(_.showForConsole) - ) - } - ) - - gatherResults(results) + case i @ Invalid(_) => i + } + case i @ Invalid(_) => i } } + def compileTo[F[_]: Monad, E, I, S[_]: Comonad, T]( + sources: AquaSources[F, E, I], + liftI: (I, String) => LiftParser[S], + backend: Backend, + config: GenerationConfig, + write: AquaCompiled[I] => F[Seq[Validated[E, T]]] + ): F[ValidatedNec[AquaError[I, E, S], Chain[T]]] = + compile[F, E, I, S](sources, liftI, backend, config).flatMap { + case Valid(compiled) => + compiled.map { ac => + write(ac).map( + _.map( + _.bimap[NonEmptyChain[AquaError[I, E, S]], Chain[T]]( + e => NonEmptyChain.one(OutputError(ac, e)), + Chain.one + ) + ) + ) + }.toList + .traverse(identity) + .map( + _.flatten + .foldLeft[ValidatedNec[AquaError[I, E, S], Chain[T]]](validNec(Chain.nil))( + _ combine _ + ) + ) + + case Validated.Invalid(errs) => + Validated.invalid[NonEmptyChain[AquaError[I, E, S]], Chain[T]](errs).pure[F] + } } diff --git a/compiler/src/main/scala/aqua/compiler/AquaError.scala b/compiler/src/main/scala/aqua/compiler/AquaError.scala index 21328361..a72fc22c 100644 --- a/compiler/src/main/scala/aqua/compiler/AquaError.scala +++ b/compiler/src/main/scala/aqua/compiler/AquaError.scala @@ -1,43 +1,18 @@ package aqua.compiler -import aqua.parser.lift.FileSpan -import cats.data.NonEmptyList -import cats.parse.Parser.Expectation +import aqua.parser.ParserError +import aqua.parser.lexer.Token +import aqua.semantics.SemanticError -sealed trait AquaError { - def showForConsole: String -} +trait AquaError[I, E, S[_]] +case class SourcesErr[I, E, S[_]](err: E) extends AquaError[I, E, S] +case class ParserErr[I, E, S[_]](err: ParserError[S]) extends AquaError[I, E, S] -case class CustomSyntaxError(span: FileSpan, message: String) extends AquaError { +case class ResolveImportsErr[I, E, S[_]](fromFile: I, token: Token[S], err: E) + extends AquaError[I, E, S] +case class ImportErr[I, E, S[_]](token: Token[S]) extends AquaError[I, E, S] - override def showForConsole: String = - span - .focus(3) - .map( - _.toConsoleStr( - message, - Console.RED - ) - ) - .getOrElse( - "(offset is beyond the script, syntax errors) Error: " + Console.RED + message - .mkString(", ") - ) + Console.RESET + "\n" -} +case class CycleError[I, E, S[_]](modules: List[I]) extends AquaError[I, E, S] -case class SyntaxError(span: FileSpan, expectations: NonEmptyList[Expectation]) extends AquaError { - - override def showForConsole: String = - span - .focus(3) - .map(spanFocus => - spanFocus.toConsoleStr( - s"Syntax error, expected: ${expectations.toList.mkString(", ")}", - Console.RED - ) - ) - .getOrElse( - "(offset is beyond the script, syntax errors) " + Console.RED + expectations.toList - .mkString(", ") - ) + Console.RESET + "\n" -} +case class CompileError[I, E, S[_]](err: SemanticError[S]) extends AquaError[I, E, S] +case class OutputError[I, E, S[_]](compiled: AquaCompiled[I], err: E) extends AquaError[I, E, S] diff --git a/compiler/src/main/scala/aqua/compiler/AquaParser.scala b/compiler/src/main/scala/aqua/compiler/AquaParser.scala new file mode 100644 index 00000000..b1ea9c0b --- /dev/null +++ b/compiler/src/main/scala/aqua/compiler/AquaParser.scala @@ -0,0 +1,111 @@ +package aqua.compiler + +import aqua.linker.{AquaModule, Modules} +import aqua.parser.Ast +import aqua.parser.head.ImportExpr +import aqua.parser.lift.LiftParser +import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec} +import cats.syntax.applicative._ +import cats.syntax.flatMap._ +import cats.syntax.functor._ +import cats.syntax.traverse._ +import cats.{Comonad, Monad} + +// TODO: add tests +class AquaParser[F[_]: Monad, E, I, S[_]: Comonad]( + sources: AquaSources[F, E, I], + liftI: (I, String) => LiftParser[S] +) { + + type Body = Ast[S] + type Err = AquaError[I, E, S] + + // Parse all the source files + def parseSources: F[ValidatedNec[Err, Chain[(I, Body)]]] = + sources.sources + .map( + _.leftMap(_.map[Err](SourcesErr(_))).andThen(_.map { case (i, s) => + implicit val lift: LiftParser[S] = liftI(i, s) + Ast.fromString[S](s).bimap(_.map[Err](ParserErr(_)), ast => Chain.one(i -> ast)) + }.foldLeft(Validated.validNec[Err, Chain[(I, Body)]](Chain.nil))(_ combine _)) + ) + + // Resolve imports (not parse, just resolve) of the given file + def resolveImports(id: I, ast: Ast[S]): F[ValidatedNec[Err, Map[I, Err]]] = + ast.head.tailForced + .map(_.head) + .collect { case ImportExpr(filename) => + sources + .resolveImport(id, filename.value.drop(1).dropRight(1)) + .map( + _.bimap( + _.map(ResolveImportsErr(id, filename, _)), + importId => Chain.one[(I, Err)](importId -> ImportErr(filename)) + ) + ) + } + .traverse(identity) + .map( + _.foldLeft(Validated.validNec[Err, Chain[(I, Err)]](Chain.nil))(_ combine _) + .map(_.toList.toMap) + ) + + // Parse sources, convert to modules + def sourceModules: F[ValidatedNec[Err, Modules[I, Err, Body]]] = + parseSources.flatMap { + case Validated.Valid(srcs) => + srcs.traverse { case (id, ast) => + resolveImports(id, ast).map(_.map(AquaModule(id, _, ast)).map(Chain.one)) + }.map( + _.foldLeft(Validated.validNec[Err, Chain[AquaModule[I, Err, Body]]](Chain.empty))( + _ combine _ + ) + ) + case Validated.Invalid(errs) => + Validated.invalid[NonEmptyChain[Err], Chain[AquaModule[I, Err, Body]]](errs).pure[F] + }.map(_.map(_.foldLeft(Modules[I, Err, Body]())(_.add(_, export = true)))) + + def loadModule(imp: I): F[ValidatedNec[Err, AquaModule[I, Err, Ast[S]]]] = + sources + .load(imp) + .map(_.leftMap(_.map[Err](SourcesErr(_))).andThen { src => + implicit val lift: LiftParser[S] = liftI(imp, src) + Ast.fromString[S](src).leftMap(_.map[Err](ParserErr(_))) + }) + .flatMap { + case Validated.Valid(ast) => + resolveImports(imp, ast).map(_.map(AquaModule(imp, _, ast))) + case Validated.Invalid(errs) => + Validated.invalid[NonEmptyChain[Err], AquaModule[I, Err, Ast[S]]](errs).pure[F] + } + + def resolveModules( + modules: Modules[I, Err, Body] + ): F[ValidatedNec[Err, Modules[I, Err, Ast[S]]]] = + modules.dependsOn.map { case (moduleId, unresolvedErrors) => + loadModule(moduleId).map(_.leftMap(_ ++ unresolvedErrors)) + }.toList + .traverse(identity) + .map(_.foldLeft[ValidatedNec[Err, Modules[I, Err, Ast[S]]]](Validated.validNec(modules)) { + case (mods, m) => + mods.andThen(ms => m.map(ms.add(_))) + }) + .flatMap { + case Validated.Valid(ms) if ms.isResolved => + Validated.validNec[Err, Modules[I, Err, Ast[S]]](ms).pure[F] + case Validated.Valid(ms) => + resolveModules(ms) + case err => + err.pure[F] + } + + def resolveSources: F[ValidatedNec[Err, Modules[I, Err, Ast[S]]]] = + sourceModules.flatMap { + case Validated.Valid(ms) => resolveModules(ms) + case err => err.pure[F] + } + + def resolve[T](transpile: Ast[S] => T => T): F[ValidatedNec[Err, Modules[I, Err, T => T]]] = + resolveSources.map(_.map(_.map(transpile))) + +} diff --git a/compiler/src/main/scala/aqua/compiler/AquaProcessed.scala b/compiler/src/main/scala/aqua/compiler/AquaProcessed.scala new file mode 100644 index 00000000..5cdd6e41 --- /dev/null +++ b/compiler/src/main/scala/aqua/compiler/AquaProcessed.scala @@ -0,0 +1,7 @@ +package aqua.compiler + +import aqua.model.AquaContext + +case class AquaProcessed[I](id: I, context: AquaContext) { + def hasOutput: Boolean = context.funcs.nonEmpty +} diff --git a/compiler/src/main/scala/aqua/compiler/AquaSources.scala b/compiler/src/main/scala/aqua/compiler/AquaSources.scala new file mode 100644 index 00000000..8ea979ce --- /dev/null +++ b/compiler/src/main/scala/aqua/compiler/AquaSources.scala @@ -0,0 +1,14 @@ +package aqua.compiler + +import cats.data.{Chain, ValidatedNec} + +trait AquaSources[F[_], Err, I] { + // Read the sources in the sources directory as (Id, String Content) pairs + def sources: F[ValidatedNec[Err, Chain[(I, String)]]] + + // Resolve id of the imported imp string from I file + def resolveImport(from: I, imp: String): F[ValidatedNec[Err, I]] + + // Load file by its resolved I + def load(file: I): F[ValidatedNec[Err, String]] +} diff --git a/compiler/src/main/scala/aqua/compiler/Prepared.scala b/compiler/src/main/scala/aqua/compiler/Prepared.scala deleted file mode 100644 index abd32afd..00000000 --- a/compiler/src/main/scala/aqua/compiler/Prepared.scala +++ /dev/null @@ -1,54 +0,0 @@ -package aqua.compiler - -import aqua.model.AquaContext -import cats.data.Validated - -import java.nio.file.Path - -object Prepared { - - /** - * @param srcFile aqua source - * @param srcPath a main source path with all aqua files - * @param targetPath a main path where all output files will be written - * @param context processed aqua code - * @return - */ - def apply( - srcFile: Path, - srcPath: Path, - targetPath: Path, - context: AquaContext - ): Validated[Throwable, Prepared] = - Validated.catchNonFatal { - val srcDir = if (srcPath.toFile.isDirectory) srcPath else srcPath.getParent - val srcFilePath = srcDir.toAbsolutePath - .normalize() - .relativize(srcFile.toAbsolutePath.normalize()) - - val targetDir = - targetPath.toAbsolutePath - .normalize() - .resolve( - srcFilePath - ) - - new Prepared(targetDir, srcFile, context) - } -} - -/** - * All info that can be used to write a final output. - * @param targetDir a directory to write to - * @param srcFile file with a source (aqua code) - * @param context processed code - */ -case class Prepared private (targetDir: Path, srcFile: Path, context: AquaContext) { - - def hasOutput: Boolean = context.funcs.nonEmpty - - def targetPath(fileName: String): Validated[Throwable, Path] = - Validated.catchNonFatal { - targetDir.getParent.resolve(fileName) - } -} diff --git a/compiler/src/main/scala/aqua/compiler/io/AquaFile.scala b/compiler/src/main/scala/aqua/compiler/io/AquaFile.scala deleted file mode 100644 index 1729eee9..00000000 --- a/compiler/src/main/scala/aqua/compiler/io/AquaFile.scala +++ /dev/null @@ -1,114 +0,0 @@ -package aqua.compiler.io - -import aqua.compiler.io.AquaFiles.ETC -import aqua.compiler.{AquaIO, CustomSyntaxError, SyntaxError} -import aqua.linker.AquaModule -import aqua.parser.head.ImportExpr -import aqua.parser.lift.FileSpan.F -import aqua.parser.lift.{FileSpan, LiftParser, Span} -import aqua.parser.{Ast, BlockIndentError, FuncReturnError, LexerError} -import cats.{Eval, Monad} -import cats.data.{EitherT, NonEmptyChain} -import cats.parse.LocationMap -import cats.syntax.apply._ -import cats.syntax.functor._ - -import java.nio.file.{Path, Paths} -import scala.collection.immutable - -case class AquaFile( - id: FileModuleId, - imports: Map[String, FileSpan.Focus], - source: String, - ast: Ast[FileSpan.F] -) { - - /** - * Gathers all errors and results - */ - private def gatherResolvedResults[F[_]: Monad]( - results: immutable.Iterable[EitherT[F, AquaFileError, (FileModuleId, FileNotFound)]] - ): ETC[F, Map[FileModuleId, AquaFileError]] = - results - .foldLeft[AquaFiles.ETC[F, Map[FileModuleId, AquaFileError]]](EitherT.rightT(Map())) { - case (files, nextFile) => - EitherT((files.value, nextFile.value).mapN { - case (files, Right(resolvedImport)) => - files.map(_ + resolvedImport) - case (Right(_), Left(err)) => - Left(NonEmptyChain(err)) - case (Left(errs), Left(err)) => - Left(errs.append(err)) - }) - } - - def createModule[F[_]: AquaIO: Monad, T]( - transpile: Ast[FileSpan.F] => T => T, - importFrom: List[Path] - ): AquaFiles.ETC[F, AquaModule[FileModuleId, AquaFileError, T]] = { - val resolvedImports = imports.map { case (pathString, focus) => - AquaIO[F] - .resolve(focus, Paths.get(pathString), id.file.getParent +: importFrom) - .map(FileModuleId) - // 'FileNotFound' will be used later if there will be problems in compilation - .map(id => (id -> FileNotFound(focus, id.file, importFrom))) - } - - for { - importsWithInfo <- gatherResolvedResults(resolvedImports) - } yield AquaModule( - id, - importsWithInfo, - transpile(ast) - ) - } -} - -object AquaFile { - - def parseAst(name: String, input: String): Either[AquaFileError, Ast[F]] = { - implicit val fileLift: LiftParser[FileSpan.F] = FileSpan.fileSpanLiftParser(name, input) - Ast - .fromString[FileSpan.F](input) - .leftMap(_.map { - case BlockIndentError(indent, message) => CustomSyntaxError(indent._1, message) - case FuncReturnError(point, message) => CustomSyntaxError(point._1, message) - case LexerError(pe) => - val fileSpan = - FileSpan( - name, - input, - Eval.later(LocationMap(input)), - Span(pe.failedAtOffset, pe.failedAtOffset + 1) - ) - SyntaxError(fileSpan, pe.expected) - }) - .toEither - .left - .map(AquaScriptErrors(_)) - } - - def read[F[_]: AquaIO: Monad](file: Path): EitherT[F, AquaFileError, AquaFile] = - for { - source <- AquaIO[F].readFile(file) - _ <- EitherT.cond[F](source.nonEmpty, (), EmptyFileError(file)) - ast <- EitherT.fromEither(parseAst(file.toString, source)) - imports = ast.head.tailForced - .map(_.head) - .collect { case ImportExpr(filename) => - val path = filename.value.drop(1).dropRight(1) - val focus = filename.unit._1.focus(1) - path -> focus - } - .collect { case (path, Some(focus)) => - path -> focus - } - .toList - .toMap - } yield AquaFile( - FileModuleId(file.toAbsolutePath.normalize()), - imports, - source, - ast - ) -} diff --git a/compiler/src/main/scala/aqua/compiler/io/AquaFiles.scala b/compiler/src/main/scala/aqua/compiler/io/AquaFiles.scala deleted file mode 100644 index f605e2c8..00000000 --- a/compiler/src/main/scala/aqua/compiler/io/AquaFiles.scala +++ /dev/null @@ -1,90 +0,0 @@ -package aqua.compiler.io - -import aqua.compiler.AquaIO -import aqua.linker.Modules -import aqua.parser.Ast -import aqua.parser.lift.FileSpan -import cats.Monad -import cats.data.{Chain, EitherT, NonEmptyChain, Validated, ValidatedNec} -import cats.syntax.functor._ -import cats.syntax.traverse._ -import cats.syntax.flatMap._ -import cats.syntax.applicative._ - -import java.nio.file.Path - -object AquaFiles { - type Mods[T] = Modules[FileModuleId, AquaFileError, T] - type ETC[F[_], T] = EitherT[F, NonEmptyChain[AquaFileError], T] - - def readSources[F[_]: AquaIO: Monad]( - sourcePath: Path - ): ETC[F, Chain[AquaFile]] = - EitherT( - AquaIO[F] - .listAqua(sourcePath) - .flatMap[ValidatedNec[AquaFileError, Chain[AquaFile]]] { - case Validated.Invalid(e) => - Validated.invalid[NonEmptyChain[AquaFileError], Chain[AquaFile]](e).pure[F] - case Validated.Valid(paths) => - paths - .traverse(AquaFile.read(_)) - .leftMap(NonEmptyChain.one) - .value - .map(Validated.fromEither) - } - .map(_.toEither) - ) - - def createModules[F[_]: AquaIO: Monad, T]( - sources: Chain[AquaFile], - importFromPaths: List[Path], - transpile: Ast[FileSpan.F] => T => T - ): ETC[F, Mods[T]] = - sources - .map(_.createModule(transpile, importFromPaths)) - .foldLeft[ETC[F, Mods[T]]]( - EitherT.rightT(Modules()) - ) { case (modulesF, modF) => - for { - ms <- modulesF - m <- modF - } yield ms.add(m, export = true) - } - - def resolveModules[F[_]: AquaIO: Monad, T]( - modules: Modules[FileModuleId, AquaFileError, T], - importFromPaths: List[Path], - transpile: Ast[FileSpan.F] => T => T - ): ETC[F, Mods[T]] = - modules.dependsOn.map { case (moduleId, unresolvedErrors) => - AquaFile - .read[F](moduleId.file) - .leftMap(unresolvedErrors.prepend) - .flatMap(_.createModule(transpile, importFromPaths)) - - }.foldLeft[ETC[F, Mods[T]]]( - EitherT.rightT(modules) - ) { case (modulesF, modF) => - for { - ms <- modulesF - m <- modF - } yield ms.add(m) - }.flatMap { - case ms if ms.isResolved => - EitherT.rightT(ms) - case ms => resolveModules(ms, importFromPaths, transpile) - } - - def readAndResolve[F[_]: AquaIO: Monad, T]( - sourcePath: Path, - importFromPaths: List[Path], - transpile: Ast[FileSpan.F] => T => T - ): ETC[F, Mods[T]] = - for { - sources <- readSources(sourcePath) - sourceModules <- createModules(sources, importFromPaths, transpile) - resolvedModules <- resolveModules(sourceModules, importFromPaths, transpile) - } yield resolvedModules - -} diff --git a/compiler/src/main/scala/aqua/compiler/io/FileModuleId.scala b/compiler/src/main/scala/aqua/compiler/io/FileModuleId.scala deleted file mode 100644 index f1e8c05a..00000000 --- a/compiler/src/main/scala/aqua/compiler/io/FileModuleId.scala +++ /dev/null @@ -1,5 +0,0 @@ -package aqua.compiler.io - -import java.nio.file.Path - -case class FileModuleId(file: Path) diff --git a/linker/src/main/scala/aqua/linker/AquaModule.scala b/linker/src/main/scala/aqua/linker/AquaModule.scala index b29c2a29..9f58c831 100644 --- a/linker/src/main/scala/aqua/linker/AquaModule.scala +++ b/linker/src/main/scala/aqua/linker/AquaModule.scala @@ -1,8 +1,8 @@ package aqua.linker -// HACK: here E is a FileNotFound error with Focus that the code will 'throw' -// if not found it in the list of loaded modules in `Modules` class. -// Essentially this error is a container with import information -// and a future error if the file for this import is not found -// TODO: fix it -case class AquaModule[I, E, T](id: I, dependsOn: Map[I, E], body: T => T) +case class AquaModule[I, E, T](id: I, dependsOn: Map[I, E], body: T) { + def map[TT](f: T => TT): AquaModule[I, E, TT] = copy(body = f(body)) + + def mapErr[EE](f: E => EE): AquaModule[I, EE, T] = + copy(dependsOn = dependsOn.view.mapValues(f).toMap) +} diff --git a/linker/src/main/scala/aqua/linker/Linker.scala b/linker/src/main/scala/aqua/linker/Linker.scala index 2239579a..8c34cdf2 100644 --- a/linker/src/main/scala/aqua/linker/Linker.scala +++ b/linker/src/main/scala/aqua/linker/Linker.scala @@ -11,15 +11,18 @@ object Linker extends LogSupport { @tailrec def iter[I, E, T: Semigroup]( - mods: List[AquaModule[I, E, T]], + mods: List[AquaModule[I, E, T => T]], proc: Map[I, T => T], - cycleError: List[AquaModule[I, E, T]] => E + cycleError: List[AquaModule[I, E, T => T]] => E ): Either[E, Map[I, T => T]] = mods match { - case Nil => Right(proc) + case Nil => + Right(proc) case _ => val (canHandle, postpone) = mods.partition(_.dependsOn.keySet.forall(proc.contains)) debug("ITERATE, can handle: " + canHandle.map(_.id)) + debug(s"dependsOn = ${mods.map(_.dependsOn.keySet)}") + debug(s"postpone = ${postpone.map(_.id)}") debug(s"proc = ${proc.keySet}") if (canHandle.isEmpty && postpone.nonEmpty) @@ -47,17 +50,20 @@ object Linker extends LogSupport { } } - def apply[I, E, T: Monoid]( - modules: Modules[I, E, T], - cycleError: List[AquaModule[I, E, T]] => E + def link[I, E, T: Monoid]( + modules: Modules[I, E, T => T], + cycleError: List[AquaModule[I, E, T => T]] => E ): ValidatedNec[E, Map[I, T]] = if (modules.dependsOn.nonEmpty) Validated.invalid(modules.dependsOn.values.reduce(_ ++ _)) - else + else { + val result = iter(modules.loaded.values.toList, Map.empty[I, T => T], cycleError) + Validated.fromEither( - iter(modules.loaded.values.toList, Map.empty[I, T => T], cycleError) + result .map(_.view.filterKeys(modules.exports).mapValues(_.apply(Monoid[T].empty)).toMap) .left .map(NonEmptyChain.one) ) + } } diff --git a/linker/src/main/scala/aqua/linker/Modules.scala b/linker/src/main/scala/aqua/linker/Modules.scala index e3dcfd18..b5ffa64b 100644 --- a/linker/src/main/scala/aqua/linker/Modules.scala +++ b/linker/src/main/scala/aqua/linker/Modules.scala @@ -24,4 +24,13 @@ case class Modules[I, E, T]( ) def isResolved: Boolean = dependsOn.isEmpty + + def map[TT](f: T => TT): Modules[I, E, TT] = + copy(loaded = loaded.view.mapValues(_.map(f)).toMap) + + def mapErr[EE](f: E => EE): Modules[I, EE, T] = + copy( + loaded = loaded.view.mapValues(_.mapErr(f)).toMap, + dependsOn = dependsOn.view.mapValues(_.map(f)).toMap + ) } diff --git a/linker/src/test/scala/aqua/linker/LinkerSpec.scala b/linker/src/test/scala/aqua/linker/LinkerSpec.scala index bf455945..34ff96d9 100644 --- a/linker/src/test/scala/aqua/linker/LinkerSpec.scala +++ b/linker/src/test/scala/aqua/linker/LinkerSpec.scala @@ -8,17 +8,21 @@ class LinkerSpec extends AnyFlatSpec with Matchers { "linker" should "resolve dependencies" in { - val empty = Modules[String, String, String]() + val empty = Modules[String, String, String => String]() val withMod1 = empty .add( - AquaModule("mod1", Map("mod2" -> "unresolved mod2 in mod1"), _ ++ " | mod1"), + AquaModule[String, String, String => String]( + "mod1", + Map("mod2" -> "unresolved mod2 in mod1"), + _ ++ " | mod1" + ), export = true ) withMod1.isResolved should be(false) - Linker[String, String, String]( + Linker.link[String, String, String]( withMod1, cycle => cycle.map(_.id).mkString(" -> ") ) should be(Validated.invalidNec("unresolved mod2 in mod1")) @@ -28,7 +32,7 @@ class LinkerSpec extends AnyFlatSpec with Matchers { withMod2.isResolved should be(true) - Linker[String, String, String]( + Linker.link[String, String, String]( withMod2, cycle => cycle.map(_.id + "?").mkString(" -> ") ) should be(Validated.validNec(Map("mod1" -> " | mod2 | mod1"))) diff --git a/model/src/main/scala/aqua/model/transform/BodyConfig.scala b/model/src/main/scala/aqua/model/transform/GenerationConfig.scala similarity index 60% rename from model/src/main/scala/aqua/model/transform/BodyConfig.scala rename to model/src/main/scala/aqua/model/transform/GenerationConfig.scala index 8176bfcf..5e575f5f 100644 --- a/model/src/main/scala/aqua/model/transform/BodyConfig.scala +++ b/model/src/main/scala/aqua/model/transform/GenerationConfig.scala @@ -1,12 +1,10 @@ package aqua.model.transform import aqua.model.{AquaContext, LiteralModel, ValueModel, VarModel} -import aqua.types.{DataType, OptionType} +import aqua.types.ScalarType import cats.kernel.Monoid -case class Constant(name: String, value: ValueModel) - -case class BodyConfig( +case class GenerationConfig( getDataService: String = "getDataSrv", callbackService: String = "callbackSrv", errorHandlingService: String = "errorHandlingSrv", @@ -14,7 +12,7 @@ case class BodyConfig( respFuncName: String = "response", relayVarName: Option[String] = Some("-relay-"), wrapWithXor: Boolean = true, - constants: List[Constant] = Nil + constants: List[GenerationConfig.Const] = Nil ) { val errorId: ValueModel = LiteralModel.quote(errorFuncName) @@ -22,8 +20,16 @@ case class BodyConfig( val callbackSrvId: ValueModel = LiteralModel.quote(callbackService) val dataSrvId: ValueModel = LiteralModel.quote(getDataService) + // Host peer id holds %init_peer_id% in case Aqua is not compiled to be executed behind a relay, + // or relay's variable otherwise + val hostPeerId: GenerationConfig.Const = + GenerationConfig.Const( + "host_peer_id", + relayVarName.fold[ValueModel](LiteralModel.initPeerId)(r => VarModel(r, ScalarType.string)) + ) + implicit val aquaContextMonoid: Monoid[AquaContext] = { - val constantsMap = constants.map(c => c.name -> c.value).toMap + val constantsMap = (hostPeerId :: constants).map(c => c.name -> c.value).toMap AquaContext .implicits( AquaContext.blank @@ -38,3 +44,10 @@ case class BodyConfig( } } + +object GenerationConfig { + case class Const(name: String, value: ValueModel) + + def forHost: GenerationConfig = + GenerationConfig(wrapWithXor = false, relayVarName = None) +} diff --git a/model/src/main/scala/aqua/model/transform/Transform.scala b/model/src/main/scala/aqua/model/transform/Transform.scala index ca2e4b5d..bbc734df 100644 --- a/model/src/main/scala/aqua/model/transform/Transform.scala +++ b/model/src/main/scala/aqua/model/transform/Transform.scala @@ -22,7 +22,7 @@ object Transform extends LogSupport { ): Cofree[Chain, ResolvedOp] = tree.copy(tail = tree.tail.map(_.filter(t => filter(t.head)).map(clear(_, filter)))) - def forClient(func: FuncCallable, conf: BodyConfig): Cofree[Chain, ResolvedOp] = { + def forClient(func: FuncCallable, conf: GenerationConfig): Cofree[Chain, ResolvedOp] = { val initCallable: InitPeerCallable = InitViaRelayCallable( Chain.fromOption(conf.relayVarName).map(VarModel(_, ScalarType.string)) ) diff --git a/model/test-kit/src/main/scala/aqua/Node.scala b/model/test-kit/src/main/scala/aqua/Node.scala index 47b66163..d528f29e 100644 --- a/model/test-kit/src/main/scala/aqua/Node.scala +++ b/model/test-kit/src/main/scala/aqua/Node.scala @@ -3,7 +3,7 @@ package aqua import aqua.model.func.Call import aqua.model.func.raw._ import aqua.model.func.resolved.{CallServiceRes, MakeRes, MatchMismatchRes, ResolvedOp} -import aqua.model.transform.{BodyConfig, ErrorsCatcher} +import aqua.model.transform.{ErrorsCatcher, GenerationConfig} import aqua.model.{LiteralModel, ValueModel, VarModel} import aqua.types.{ArrayType, LiteralType, ScalarType} import cats.Eval @@ -88,7 +88,7 @@ object Node { ) ) - def errorCall(bc: BodyConfig, i: Int, on: ValueModel = initPeer): Res = Node[ResolvedOp]( + def errorCall(bc: GenerationConfig, i: Int, on: ValueModel = initPeer): Res = Node[ResolvedOp]( CallServiceRes( bc.errorHandlingCallback, bc.errorFuncName, @@ -103,7 +103,7 @@ object Node { ) ) - def respCall(bc: BodyConfig, value: ValueModel, on: ValueModel = initPeer): Res = + def respCall(bc: GenerationConfig, value: ValueModel, on: ValueModel = initPeer): Res = Node[ResolvedOp]( CallServiceRes( bc.callbackSrvId, @@ -113,14 +113,15 @@ object Node { ) ) - def dataCall(bc: BodyConfig, name: String, on: ValueModel = initPeer): Res = Node[ResolvedOp]( - CallServiceRes( - bc.dataSrvId, - name, - Call(Nil, Some(Call.Export(name, ScalarType.string))), - on + def dataCall(bc: GenerationConfig, name: String, on: ValueModel = initPeer): Res = + Node[ResolvedOp]( + CallServiceRes( + bc.dataSrvId, + name, + Call(Nil, Some(Call.Export(name, ScalarType.string))), + on + ) ) - ) def on(peer: ValueModel, via: List[ValueModel], body: Raw*) = Node( diff --git a/model/test-kit/src/test/scala/aqua/model/transform/TransformSpec.scala b/model/test-kit/src/test/scala/aqua/model/transform/TransformSpec.scala index 0b49c7c2..6b0890a1 100644 --- a/model/test-kit/src/test/scala/aqua/model/transform/TransformSpec.scala +++ b/model/test-kit/src/test/scala/aqua/model/transform/TransformSpec.scala @@ -26,7 +26,7 @@ class TransformSpec extends AnyFlatSpec with Matchers { Map.empty ) - val bc = BodyConfig() + val bc = GenerationConfig() val fc = Transform.forClient(func, bc) @@ -76,7 +76,7 @@ class TransformSpec extends AnyFlatSpec with Matchers { Map.empty ) - val bc = BodyConfig(wrapWithXor = false) + val bc = GenerationConfig(wrapWithXor = false) val fc = Transform.forClient(func, bc) @@ -137,7 +137,7 @@ class TransformSpec extends AnyFlatSpec with Matchers { Map.empty ) - val bc = BodyConfig(wrapWithXor = false) + val bc = GenerationConfig(wrapWithXor = false) val res = Transform.forClient(f2, bc): Node.Res diff --git a/parser/src/main/scala/aqua/parser/Ast.scala b/parser/src/main/scala/aqua/parser/Ast.scala index 3e9bcfbd..3428e442 100644 --- a/parser/src/main/scala/aqua/parser/Ast.scala +++ b/parser/src/main/scala/aqua/parser/Ast.scala @@ -3,6 +3,7 @@ package aqua.parser import aqua.parser.expr._ import aqua.parser.head.{HeadExpr, HeaderExpr} import aqua.parser.lift.LiftParser +import aqua.parser.lift.LiftParser._ import cats.data.{Chain, Validated, ValidatedNec} import cats.free.Cofree import cats.parse.{Parser0 => P0} @@ -27,7 +28,7 @@ object Ast { parser[F]() .parseAll(script) match { case Right(value) => value - case Left(e) => Validated.invalidNec(LexerError[F](e)) + case Left(e) => Validated.invalidNec(LexerError[F](e.wrapErr)) } } diff --git a/parser/src/main/scala/aqua/parser/ParserError.scala b/parser/src/main/scala/aqua/parser/ParserError.scala index d3e98546..675395d6 100644 --- a/parser/src/main/scala/aqua/parser/ParserError.scala +++ b/parser/src/main/scala/aqua/parser/ParserError.scala @@ -4,6 +4,6 @@ import cats.parse.Parser trait ParserError[F[_]] -case class LexerError[F[_]](err: Parser.Error) extends ParserError[F] +case class LexerError[F[_]](err: F[Parser.Error]) extends ParserError[F] case class BlockIndentError[F[_]](indent: F[String], message: String) extends ParserError[F] case class FuncReturnError[F[_]](point: F[Unit], message: String) extends ParserError[F] diff --git a/parser/src/main/scala/aqua/parser/lift/FileSpan.scala b/parser/src/main/scala/aqua/parser/lift/FileSpan.scala index 666c443f..6fad0ad0 100644 --- a/parser/src/main/scala/aqua/parser/lift/FileSpan.scala +++ b/parser/src/main/scala/aqua/parser/lift/FileSpan.scala @@ -6,7 +6,8 @@ import cats.{Comonad, Eval} import scala.language.implicitConversions // TODO: rewrite FileSpan and Span under one trait -case class FileSpan(name: String, source: String, locationMap: Eval[LocationMap], span: Span) { +// TODO: move FileSpan to another package? +case class FileSpan(name: String, locationMap: Eval[LocationMap], span: Span) { def focus(ctx: Int): Option[FileSpan.Focus] = span.focus(locationMap, ctx).map(FileSpan.Focus(name, locationMap, ctx, _)) @@ -37,18 +38,27 @@ object FileSpan { def fileSpanLiftParser(name: String, source: String): LiftParser[F] = new LiftParser[F] { - val memoizedLocationMap = Eval.later(LocationMap(source)).memoize + private val memoizedLocationMap = Eval.later(LocationMap(source)).memoize override def lift[T](p: P[T]): P[F[T]] = { implicitly[LiftParser[Span.F]].lift(p).map { case (span, value) => - (FileSpan(name, source, memoizedLocationMap, span), value) + (FileSpan(name, memoizedLocationMap, span), value) } } override def lift0[T](p0: Parser0[T]): Parser0[(FileSpan, T)] = { implicitly[LiftParser[Span.F]].lift0(p0).map { case (span, value) => - (FileSpan(name, source, memoizedLocationMap, span), value) + (FileSpan(name, memoizedLocationMap, span), value) } } + + override def wrapErr(e: P.Error): (FileSpan, P.Error) = ( + FileSpan( + name, + memoizedLocationMap, + Span(e.failedAtOffset, e.failedAtOffset + 1) + ), + e + ) } } diff --git a/parser/src/main/scala/aqua/parser/lift/LiftParser.scala b/parser/src/main/scala/aqua/parser/lift/LiftParser.scala index e15b6706..19efb77e 100644 --- a/parser/src/main/scala/aqua/parser/lift/LiftParser.scala +++ b/parser/src/main/scala/aqua/parser/lift/LiftParser.scala @@ -7,10 +7,16 @@ trait LiftParser[F[_]] { def lift[T](p: Parser[T]): Parser[F[T]] def lift0[T](p0: Parser0[T]): Parser0[F[T]] + + def wrapErr(e: Parser.Error): F[Parser.Error] } object LiftParser { + implicit class LiftErrorOps[F[_]: LiftParser, T](e: Parser.Error) { + def wrapErr: F[Parser.Error] = implicitly[LiftParser[F]].wrapErr(e) + } + implicit class LiftParserOps[F[_]: LiftParser, T](parser: Parser[T]) { def lift: Parser[F[T]] = implicitly[LiftParser[F]].lift(parser) } @@ -24,6 +30,7 @@ object LiftParser { implicit object idLiftParser extends LiftParser[Id] { override def lift[T](p: Parser[T]): Parser[Id[T]] = p override def lift0[T](p0: Parser0[T]): Parser0[Id[T]] = p0 + override def wrapErr(e: Parser.Error): Id[Parser.Error] = e } } diff --git a/parser/src/main/scala/aqua/parser/lift/Span.scala b/parser/src/main/scala/aqua/parser/lift/Span.scala index 83801c55..ee6992b1 100644 --- a/parser/src/main/scala/aqua/parser/lift/Span.scala +++ b/parser/src/main/scala/aqua/parser/lift/Span.scala @@ -105,6 +105,9 @@ object Span { (P.index ~ p0).map { case (i, v) ⇒ (Span(i, i), v) } + + override def wrapErr(e: P.Error): (Span, P.Error) = + (Span(e.failedAtOffset, e.failedAtOffset + 1), e) } } diff --git a/semantics/src/test/scala/aqua/semantics/SemanticsSpec.scala b/semantics/src/test/scala/aqua/semantics/SemanticsSpec.scala index f4d1917b..7e0da597 100644 --- a/semantics/src/test/scala/aqua/semantics/SemanticsSpec.scala +++ b/semantics/src/test/scala/aqua/semantics/SemanticsSpec.scala @@ -29,7 +29,7 @@ class SemanticsSpec extends AnyFlatSpec with Matchers { val ast = Ast.fromString(script).toList.head val ctx = AquaContext.blank - val bc = BodyConfig() + val bc = GenerationConfig() import bc.aquaContextMonoid val p = Semantics.process(ast, ctx)