Cli/compiler refactoring (#184)

This commit is contained in:
Dima 2021-06-29 16:31:20 +03:00 committed by GitHub
parent 5e1ef6e227
commit f15bd0558b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 230 additions and 202 deletions

View File

@ -20,6 +20,8 @@ val declineV = "2.0.0-RC1" // Scala3 issue: https://github.com/bkirwi/decline/is
val declineEnumV = "1.3.0"
val airframeLog = "org.wvlet.airframe" %% "airframe-log" % airframeLogV
val catsEffect = "org.typelevel" %% "cats-effect" % catsEffectV
val fs2Io = "co.fs2" %% "fs2-io" % fs2V
name := "aqua-hll"
@ -46,16 +48,14 @@ lazy val cli = project
libraryDependencies ++= Seq(
"com.monovore" %% "decline" % declineV,
"com.monovore" %% "decline-effect" % declineV,
"org.typelevel" %% "cats-effect" % catsEffectV,
"co.fs2" %% "fs2-core" % fs2V,
"co.fs2" %% "fs2-io" % fs2V,
catsEffect,
"org.typelevel" %% "log4cats-slf4j" % log4catsV,
"com.beachape" %% "enumeratum" % enumeratumV,
"org.slf4j" % "slf4j-jdk14" % slf4jV,
"com.monovore" %% "decline-enumeratum" % declineEnumV
)
)
.dependsOn(semantics, `backend-air`, `backend-ts`, `backend-js`, linker, backend)
.dependsOn(compiler, `backend-air`, `backend-ts`, `backend-js`)
lazy val types = project
.settings(commons)
@ -108,6 +108,17 @@ lazy val semantics = project
)
.dependsOn(model, `test-kit` % Test, parser)
lazy val compiler = project
.in(file("compiler"))
.settings(commons: _*)
.settings(
libraryDependencies ++= Seq(
catsEffect,
fs2Io
)
)
.dependsOn(model, semantics, linker, backend)
lazy val backend = project
.in(file("backend"))
.settings(commons: _*)

View File

@ -61,7 +61,7 @@ object AppOps {
val outputOpts: Opts[Path] =
Opts.option[Path]("output", "Path to the output directory", "o").mapValidated(checkPath)
val importOpts: Opts[LazyList[Path]] =
val importOpts: Opts[List[Path]] =
Opts
.options[Path]("import", "Path to the directory to import from", "m")
.mapValidated { ps =>
@ -91,9 +91,9 @@ object AppOps {
}.traverse {
case Valid(a) => Validated.validNel(a)
case Invalid(e) => Validated.invalidNel(e)
}.map(_.to(LazyList))
}
.withDefault(LazyList.empty)
}
.withDefault(List.empty)
def constantOpts[F[_]: LiftParser: Comonad]: Opts[List[Constant]] =
Opts

View File

@ -1,30 +0,0 @@
package aqua
import aqua.parser.lift.{FileSpan, LiftParser, Span}
import aqua.parser.{Ast, BlockIndentError, FuncReturnError, LexerError}
import cats.Eval
import cats.data.ValidatedNec
import cats.parse.LocationMap
object Aqua {
def parseFileString(name: String, input: String): ValidatedNec[AquaError, Ast[FileSpan.F]] = {
implicit val fileLift: LiftParser[FileSpan.F] = FileSpan.fileSpanLiftParser(name, input)
Ast
.fromString[FileSpan.F](input)
.leftMap(_.map {
case BlockIndentError(indent, message) => CustomSyntaxError(indent._1, message)
case FuncReturnError(point, message) => CustomSyntaxError(point._1, message)
case LexerError(pe) =>
val fileSpan =
FileSpan(
name,
input,
Eval.later(LocationMap(input)),
Span(pe.failedAtOffset, pe.failedAtOffset + 1)
)
SyntaxError(fileSpan, pe.expected)
})
}
}

View File

@ -1,5 +1,11 @@
package aqua
import aqua.backend.Backend
import aqua.backend.air.AirBackend
import aqua.backend.js.JavaScriptBackend
import aqua.backend.ts.TypeScriptBackend
import aqua.compiler.AquaCompiler
import aqua.compiler.AquaCompiler.{AirTarget, CompileTarget, JavaScriptTarget, TypescriptTarget}
import aqua.model.transform.BodyConfig
import aqua.parser.lift.LiftParser.Implicits.idLiftParser
import cats.Id
@ -28,6 +34,17 @@ object CustomLogFormatter extends LogFormatter {
object AquaCli extends IOApp with LogSupport {
import AppOps._
def targetToBackend(target: CompileTarget): Backend = {
target match {
case TypescriptTarget =>
TypeScriptBackend
case JavaScriptTarget =>
JavaScriptBackend
case AirTarget =>
AirBackend
}
}
def main[F[_]: Concurrent: Files: ConsoleEff: Logger]: Opts[F[ExitCode]] = {
versionOpt
.as(
@ -68,7 +85,7 @@ object AquaCli extends IOApp with LogSupport {
input,
imports,
output,
target,
targetToBackend(target),
bc
)
.map {

View File

@ -1,5 +1,7 @@
package aqua
import aqua.backend.ts.TypeScriptBackend
import aqua.compiler.AquaCompiler
import aqua.model.transform.BodyConfig
import cats.data.Validated
import cats.effect.{IO, IOApp, Sync}
@ -17,9 +19,9 @@ object Test extends IOApp.Simple {
AquaCompiler
.compileFilesTo[IO](
Paths.get("./aqua-src"),
LazyList(Paths.get("./aqua")),
List(Paths.get("./aqua")),
Paths.get("./target"),
AquaCompiler.TypescriptTarget,
TypeScriptBackend,
BodyConfig()
)
.map {

View File

@ -1,98 +0,0 @@
package aqua.io
import aqua.Aqua
import aqua.linker.AquaModule
import aqua.parser.Ast
import aqua.parser.head.ImportExpr
import aqua.parser.lift.FileSpan
import cats.data.{EitherT, NonEmptyChain}
import cats.effect.Concurrent
import cats.syntax.apply._
import cats.syntax.functor._
import fs2.io.file.Files
import java.nio.file.{Path, Paths}
case class AquaFile(
id: FileModuleId,
imports: Map[String, FileSpan.Focus],
source: String,
ast: Ast[FileSpan.F]
) {
def module[F[_]: Concurrent, T](
transpile: Ast[FileSpan.F] => T => T,
importFrom: LazyList[Path]
): AquaFiles.ETC[F, AquaModule[FileModuleId, AquaFileError, T]] =
imports.map { case (k, v) =>
FileModuleId.resolve(v, Paths.get(k), id.file.getParent +: importFrom).map(_ -> v)
}.foldLeft[AquaFiles.ETC[F, AquaModule[FileModuleId, AquaFileError, T]]](
EitherT.rightT(
AquaModule(
id,
Map(),
transpile(ast)
)
)
) { case (modF, nextF) =>
EitherT((modF.value, nextF.value).mapN {
case (moduleV, Right(dependency)) =>
moduleV.map(m =>
m.copy(dependsOn =
m.dependsOn + dependency.map(FileNotFound(_, dependency._1.file, importFrom))
)
)
case (Right(_), Left(err)) =>
Left(NonEmptyChain(err))
case (Left(errs), Left(err)) =>
Left(errs.append(err))
})
}
}
object AquaFile {
def readAst[F[_]: Files: Concurrent](
file: Path
): fs2.Stream[F, Either[AquaFileError, (String, Ast[FileSpan.F])]] =
FileOps
.readSourceText[F](file)
.map {
_.left
.map(t => FileSystemError(t))
}
.map(
_.flatMap(source =>
Aqua
.parseFileString(file.toString, source)
.map(source -> _)
.toEither
.left
.map(AquaScriptErrors(_))
)
)
def read[F[_]: Files: Concurrent](file: Path): EitherT[F, AquaFileError, AquaFile] =
EitherT(readAst[F](file).compile.last.map(_.getOrElse(Left(EmptyFileError(file))))).map {
case (source, ast) =>
AquaFile(
FileModuleId(file.toAbsolutePath.normalize()),
ast.head.tailForced
.map(_.head)
.collect { case ImportExpr(filename) =>
val fn = filename.value.drop(1).dropRight(1)
val focus = filename.unit._1.focus(1)
fn -> focus
}
.collect { case (a, Some(b)) =>
a -> b
}
.toList
.toMap,
source,
ast
)
}
}

View File

@ -1,4 +1,7 @@
import aqua.AquaCompiler
import aqua.backend.air.AirBackend
import aqua.backend.js.JavaScriptBackend
import aqua.backend.ts.TypeScriptBackend
import aqua.compiler.AquaCompiler
import aqua.model.transform.BodyConfig
import cats.effect.IO
import cats.effect.unsafe.implicits.global
@ -16,7 +19,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
val bc = BodyConfig()
AquaCompiler
.compileFilesTo[IO](src, LazyList.empty, targetTs, AquaCompiler.TypescriptTarget, bc)
.compileFilesTo[IO](src, List.empty, targetTs, TypeScriptBackend, bc)
.unsafeRunSync()
.leftMap { err =>
println(err)
@ -28,7 +31,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
Files.deleteIfExists(targetTsFile)
AquaCompiler
.compileFilesTo[IO](src, LazyList.empty, targetJs, AquaCompiler.JavaScriptTarget, bc)
.compileFilesTo[IO](src, List.empty, targetJs, JavaScriptBackend, bc)
.unsafeRunSync()
.leftMap { err =>
println(err)
@ -40,7 +43,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
Files.deleteIfExists(targetJsFile)
AquaCompiler
.compileFilesTo[IO](src, LazyList.empty, targetAir, AquaCompiler.AirTarget, bc)
.compileFilesTo[IO](src, List.empty, targetAir, AirBackend, bc)
.unsafeRunSync()
.leftMap { err =>
println(err)

View File

@ -1,10 +1,7 @@
package aqua
package aqua.compiler
import aqua.backend.Backend
import aqua.backend.air.AirBackend
import aqua.backend.js.JavaScriptBackend
import aqua.backend.ts.TypeScriptBackend
import aqua.io._
import aqua.compiler.io._
import aqua.linker.Linker
import aqua.model.AquaContext
import aqua.model.transform.BodyConfig
@ -61,14 +58,14 @@ object AquaCompiler extends LogSupport {
*/
def prepareFiles[F[_]: Files: Concurrent](
srcPath: Path,
imports: LazyList[Path],
imports: List[Path],
targetPath: Path
)(implicit aqum: Monoid[AquaContext]): F[ValidatedNec[String, Chain[Prepared]]] =
AquaFiles
.readAndResolve[F, ValidatedNec[SemanticError[FileSpan.F], AquaContext]](
srcPath,
imports,
ast => _.andThen(ctx => Semantics.process(ast, ctx))
ast => context => context.andThen(ctx => Semantics.process(ast, ctx))
)
.value
.map {
@ -104,17 +101,6 @@ object AquaCompiler extends LogSupport {
"Semantic error"
}
def targetToBackend(target: CompileTarget): Backend = {
target match {
case TypescriptTarget =>
TypeScriptBackend
case JavaScriptTarget =>
JavaScriptBackend
case AirTarget =>
AirBackend
}
}
private def gatherResults[F[_]: Monad](
results: List[EitherT[F, String, Unit]]
): F[Validated[NonEmptyChain[String], Chain[String]]] = {
@ -138,15 +124,15 @@ object AquaCompiler extends LogSupport {
def compileFilesTo[F[_]: Files: Concurrent](
srcPath: Path,
imports: LazyList[Path],
imports: List[Path],
targetPath: Path,
compileTo: CompileTarget,
backend: Backend,
bodyConfig: BodyConfig
): F[ValidatedNec[String, Chain[String]]] = {
import bodyConfig.aquaContextMonoid
prepareFiles(srcPath, imports, targetPath)
.map(_.map(_.filter { p =>
val hasOutput = p.hasOutput(compileTo)
val hasOutput = p.hasOutput
if (!hasOutput) info(s"Source ${p.srcFile}: compilation OK (nothing to emit)")
hasOutput
}))
@ -154,7 +140,6 @@ object AquaCompiler extends LogSupport {
case Validated.Invalid(e) =>
Applicative[F].pure(Validated.invalid(e))
case Validated.Valid(preps) =>
val backend = targetToBackend(compileTo)
val results = preps.toList
.flatMap(p =>
backend.generate(p.context, bodyConfig).map { compiled =>

View File

@ -1,4 +1,4 @@
package aqua
package aqua.compiler
import aqua.parser.lift.FileSpan
import cats.data.NonEmptyList

View File

@ -1,6 +1,5 @@
package aqua
package aqua.compiler
import aqua.AquaCompiler.CompileTarget
import aqua.model.AquaContext
import cats.data.Validated
@ -46,9 +45,7 @@ object Prepared {
*/
case class Prepared private (targetDir: Path, srcFile: Path, context: AquaContext) {
def hasOutput(target: CompileTarget): Boolean = target match {
case _ => context.funcs.nonEmpty
}
def hasOutput: Boolean = context.funcs.nonEmpty
def targetPath(fileName: String): Validated[Throwable, Path] =
Validated.catchNonFatal {

View File

@ -0,0 +1,131 @@
package aqua.compiler.io
import aqua.compiler.io.AquaFiles.ETC
import aqua.compiler.{CustomSyntaxError, SyntaxError}
import aqua.linker.AquaModule
import aqua.parser.head.ImportExpr
import aqua.parser.lift.FileSpan.F
import aqua.parser.lift.{FileSpan, LiftParser, Span}
import aqua.parser.{Ast, BlockIndentError, FuncReturnError, LexerError}
import cats.Eval
import cats.data.{EitherT, NonEmptyChain}
import cats.effect.Concurrent
import cats.parse.LocationMap
import cats.syntax.apply._
import cats.syntax.functor._
import fs2.io.file.Files
import java.nio.file.{Path, Paths}
import scala.collection.immutable
case class AquaFile(
id: FileModuleId,
imports: Map[String, FileSpan.Focus],
source: String,
ast: Ast[FileSpan.F]
) {
/**
* Gathers all errors and results
*/
private def gatherResolvedResults[F[_]: Concurrent](
results: immutable.Iterable[EitherT[F, AquaFileError, (FileModuleId, FileNotFound)]]
): ETC[F, Map[FileModuleId, AquaFileError]] = {
results
.foldLeft[AquaFiles.ETC[F, Map[FileModuleId, AquaFileError]]](EitherT.rightT(Map())) {
case (files, nextFile) =>
EitherT((files.value, nextFile.value).mapN {
case (files, Right(resolvedImport)) =>
files.map(_ + resolvedImport)
case (Right(_), Left(err)) =>
Left(NonEmptyChain(err))
case (Left(errs), Left(err)) =>
Left(errs.append(err))
})
}
}
def createModule[F[_]: Concurrent, T](
transpile: Ast[FileSpan.F] => T => T,
importFrom: List[Path]
): AquaFiles.ETC[F, AquaModule[FileModuleId, AquaFileError, T]] = {
val resolvedImports = imports.map { case (pathString, focus) =>
FileModuleId
.resolve(focus, Paths.get(pathString), id.file.getParent +: importFrom)
// 'FileNotFound' will be used later if there will be problems in compilation
.map(id => (id -> FileNotFound(focus, id.file, importFrom)))
}
for {
importsWithInfo <- gatherResolvedResults(resolvedImports)
} yield AquaModule(
id,
importsWithInfo,
transpile(ast)
)
}
}
object AquaFile {
def parseAst(name: String, input: String): Either[AquaFileError, Ast[F]] = {
implicit val fileLift: LiftParser[FileSpan.F] = FileSpan.fileSpanLiftParser(name, input)
Ast
.fromString[FileSpan.F](input)
.leftMap(_.map {
case BlockIndentError(indent, message) => CustomSyntaxError(indent._1, message)
case FuncReturnError(point, message) => CustomSyntaxError(point._1, message)
case LexerError(pe) =>
val fileSpan =
FileSpan(
name,
input,
Eval.later(LocationMap(input)),
Span(pe.failedAtOffset, pe.failedAtOffset + 1)
)
SyntaxError(fileSpan, pe.expected)
})
.toEither
.left
.map(AquaScriptErrors(_))
}
def read[F[_]: Files: Concurrent](file: Path): EitherT[F, AquaFileError, AquaFile] = {
for {
sourceOp <- EitherT.right(
FileOps
.readSourceText[F](file)
.map {
_.left
.map(t => FileSystemError(t))
}
.compile
.last
)
source <- EitherT.fromEither(sourceOp.getOrElse(Left(EmptyFileError(file))))
_ <- EitherT.fromEither(
if (source.isEmpty) Left(EmptyFileError(file): AquaFileError) else Right(())
)
ast <- EitherT.fromEither(parseAst(file.toString, source))
imports = ast.head.tailForced
.map(_.head)
.collect { case ImportExpr(filename) =>
val path = filename.value.drop(1).dropRight(1)
val focus = filename.unit._1.focus(1)
path -> focus
}
.collect { case (path, Some(focus)) =>
path -> focus
}
.toList
.toMap
} yield {
AquaFile(
FileModuleId(file.toAbsolutePath.normalize()),
imports,
source,
ast
)
}
}
}

View File

@ -1,6 +1,6 @@
package aqua.io
package aqua.compiler.io
import aqua.AquaError
import aqua.compiler.AquaError
import aqua.parser.lift.FileSpan
import cats.data.NonEmptyChain

View File

@ -1,4 +1,4 @@
package aqua.io
package aqua.compiler.io
import aqua.linker.Modules
import aqua.parser.Ast
@ -58,13 +58,13 @@ object AquaFiles {
}
)
def sourceModules[F[_]: Concurrent, T](
def createModules[F[_]: Concurrent, T](
sources: Chain[AquaFile],
importFromPaths: LazyList[Path],
importFromPaths: List[Path],
transpile: Ast[FileSpan.F] => T => T
): ETC[F, Mods[T]] =
sources
.map(_.module(transpile, importFromPaths))
.map(_.createModule(transpile, importFromPaths))
.foldLeft[ETC[F, Mods[T]]](
EitherT.rightT(Modules())
) { case (modulesF, modF) =>
@ -76,14 +76,14 @@ object AquaFiles {
def resolveModules[F[_]: Files: Concurrent, T](
modules: Modules[FileModuleId, AquaFileError, T],
importFromPaths: LazyList[Path],
importFromPaths: List[Path],
transpile: Ast[FileSpan.F] => T => T
): ETC[F, Mods[T]] =
modules.dependsOn.map { case (moduleId, unresolvedErrors) =>
AquaFile
.read[F](moduleId.file)
.leftMap(unresolvedErrors.prepend)
.flatMap(_.module(transpile, importFromPaths))
.flatMap(_.createModule(transpile, importFromPaths))
}.foldLeft[ETC[F, Mods[T]]](
EitherT.rightT(modules)
@ -100,13 +100,13 @@ object AquaFiles {
def readAndResolve[F[_]: Files: Concurrent, T](
sourcePath: Path,
importFromPaths: LazyList[Path],
importFromPaths: List[Path],
transpile: Ast[FileSpan.F] => T => T
): ETC[F, Mods[T]] =
for {
srcs <- readSources(sourcePath)
srcMods <- sourceModules(srcs, importFromPaths, transpile)
resMods <- resolveModules(srcMods, importFromPaths, transpile)
} yield resMods
sources <- readSources(sourcePath)
sourceModules <- createModules(sources, importFromPaths, transpile)
resolvedModules <- resolveModules(sourceModules, importFromPaths, transpile)
} yield resolvedModules
}

View File

@ -1,4 +1,4 @@
package aqua.io
package aqua.compiler.io
import aqua.parser.lift.FileSpan
import cats.data.EitherT
@ -7,12 +7,12 @@ import cats.syntax.applicative._
import java.nio.file.Path
case class FileModuleId(file: Path) {}
case class FileModuleId(file: Path)
object FileModuleId {
private def findFirstF[F[_]: Concurrent](
in: LazyList[Path],
in: List[Path],
notFound: EitherT[F, AquaFileError, FileModuleId]
): EitherT[F, AquaFileError, FileModuleId] =
in.headOption.fold(notFound)(p =>
@ -31,10 +31,13 @@ object FileModuleId {
}
)
/**
* Checks if a file existed in the list of possible paths
*/
def resolve[F[_]: Concurrent](
focus: FileSpan.Focus,
src: Path,
imports: LazyList[Path]
imports: List[Path]
): EitherT[F, AquaFileError, FileModuleId] =
findFirstF(
imports

View File

@ -1,4 +1,4 @@
package aqua.io
package aqua.compiler.io
import cats.data.EitherT
import cats.effect.Concurrent

View File

@ -1,3 +1,8 @@
package aqua.linker
// HACK: here E is a FileNotFound error with Focus that the code will 'throw'
// if not found it in the list of loaded modules in `Modules` class.
// Essentially this error is a container with import information
// and a future error if the file for this import is not found
// TODO: fix it
case class AquaModule[I, E, T](id: I, dependsOn: Map[I, E], body: T => T)

View File

@ -26,12 +26,13 @@ object Linker extends LogSupport {
Left(cycleError(postpone))
else {
val folded = canHandle.foldLeft(proc) { case (acc, m) =>
debug(m.id + " dependsOn " + m.dependsOn.keySet)
val importKeys = m.dependsOn.keySet
debug(m.id + " dependsOn " + importKeys)
val deps: T => T =
m.dependsOn.keySet.map(acc).foldLeft[T => T](identity) { case (fAcc, f) =>
importKeys.map(acc).foldLeft[T => T](identity) { case (fAcc, f) =>
debug("COMBINING ONE TIME ")
t => {
debug(s"call combine ${t}")
debug(s"call combine $t")
fAcc(t) |+| f(t)
}
}

View File

@ -9,17 +9,18 @@ case class Modules[I, E, T](
exports: Set[I] = Set.empty[I]
) {
def add(m: AquaModule[I, E, T], export: Boolean = false): Modules[I, E, T] =
if (loaded.contains(m.id)) this
def add(aquaModule: AquaModule[I, E, T], export: Boolean = false): Modules[I, E, T] =
if (loaded.contains(aquaModule.id)) this
else
copy(
loaded = loaded + (m.id -> m),
dependsOn = m.dependsOn.foldLeft(dependsOn - m.id) {
case (deps, (mId, _)) if loaded.contains(mId) || mId == m.id => deps
case (deps, (mId, err)) =>
deps.updatedWith(mId)(_.fold(NonEmptyChain.one(err))(_.append(err)).some)
loaded = loaded + (aquaModule.id -> aquaModule),
dependsOn = aquaModule.dependsOn.foldLeft(dependsOn - aquaModule.id) {
case (deps, (moduleId, _)) if loaded.contains(moduleId) || moduleId == aquaModule.id =>
deps
case (deps, (moduleId, err)) =>
deps.updatedWith(moduleId)(_.fold(NonEmptyChain.one(err))(_.append(err)).some)
},
exports = if (export) exports + m.id else exports
exports = if (export) exports + aquaModule.id else exports
)
def isResolved: Boolean = dependsOn.isEmpty

View File

@ -1 +1 @@
sbt.version=1.5.1
sbt.version=1.5.2