feat(compiler): Allow redeclaring services (and abilities) [LNG-360] (#1135)

* Reverse ability search order

* Gather arrows from imported services

* Add test

* Refactor AquaContext

* Add test for abilities

* Update cache
This commit is contained in:
InversionSpaces 2024-05-10 12:43:52 +02:00 committed by GitHub
parent 6cc068ac36
commit faf5b8071f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 470 additions and 181 deletions

View File

@ -27,15 +27,11 @@ object CompilerAPI extends Logging {
filesWithContext.toList
// Process all contexts maintaining Cache
.traverse { case (i, rawContext) =>
for {
cache <- State.get[AquaContext.Cache]
_ = logger.trace(s"Going to prepare exports for $i...")
(exp, expCache) = AquaContext.exportsFromRaw(rawContext, cache)
_ = logger.trace(s"AquaProcessed prepared for $i")
_ <- State.set(expCache)
} yield AquaProcessed(i, exp)
AquaContext
.exportsFromRaw(rawContext)
.map(exp => AquaProcessed(i, exp))
}
.runA(AquaContext.Cache())
.runA(AquaContext.Cache.empty)
// Convert result List to Chain
.map(Chain.fromSeq)
.value

View File

@ -20,10 +20,13 @@ import aqua.res.ResBuilder
import aqua.semantics.FileId
import aqua.types.{ArrayType, CanonStreamType, LiteralType, ScalarType, StreamType, Type}
import cats.Eval
import cats.Id
import cats.data.{Chain, NonEmptyChain, NonEmptyMap, Validated, ValidatedNec}
import cats.free.Cofree
import cats.instances.string.*
import cats.syntax.either.*
import cats.syntax.flatMap.*
import cats.syntax.option.*
import cats.syntax.show.*
import org.scalatest.Inside
@ -889,6 +892,77 @@ class AquaCompilerSpec extends AnyFlatSpec with Matchers with Inside {
}
}
extension [A](l: List[List[A]]) {
def rotate: List[List[A]] =
l.foldLeft(List.empty[List[A]]) { case (acc, next) =>
if (acc.isEmpty) next.map(List(_))
else
for {
elem <- next
prev <- acc
} yield elem +: prev
}
}
type NameRename = (String, Option[String])
def testImportsHierarchy(test: List[NameRename] => Any) = {
// Simple
(1 to 10).foreach { i =>
val names = (1 to i).map(n => s"Imp$n").toList
withClue(s"Testing ${names.mkString(" -> ")}") {
test(names.map(_ -> none))
}
}
// // With subpaths
(1 to 4).foreach { i =>
(1 to i)
.map(idx =>
paths(
List("Imp", "Sub", "Path")
.map(p => s"$p$idx")
)
)
.toList
.rotate
.foreach(names =>
withClue(s"Testing ${names.mkString(" -> ")}") {
test(names.map(_ -> none))
}
)
}
// // With renames
(1 to 3).foreach { i =>
(1 to i)
.map(idx =>
for {
name <- paths(
List("Imp", "Sub", "Path")
.map(p => s"$p$idx")
)
rename <- None :: paths(
List("Rename", "To", "Other")
.map(p => s"$p$idx")
).map(_.some)
} yield name -> rename
)
.toList
.rotate
.foreach(names =>
val message = names.map { case (n, r) =>
s"$n${r.fold("")(n => s" as $n")}"
}.mkString(" -> ")
withClue(s"Testing $message:") {
test(names)
}
)
}
}
it should "import redeclared functions" in {
final case class Imp(
@ -920,8 +994,6 @@ class AquaCompilerSpec extends AnyFlatSpec with Matchers with Inside {
lazy val access: String = rename.getOrElse(name)
}
type NameRename = (String, Option[String])
def test(imps: List[NameRename]) = {
(imps.length > 0) should be(true)
@ -976,72 +1048,251 @@ class AquaCompilerSpec extends AnyFlatSpec with Matchers with Inside {
}
}
// Simple
(1 to 10).foreach { i =>
val names = (1 to i).map(n => s"Imp$n").toList
withClue(s"Testing ${names.mkString(" -> ")}") {
test(names.map(_ -> none))
testImportsHierarchy(test)
}
it should "import redeclared services" in {
final case class Imp(
idx: Int,
name: String,
rename: Option[String] = None,
use: Option[Imp] = None
) {
def withUse(other: Imp): Imp = copy(use = Some(other))
lazy val path: String = s"import$idx.aqua"
lazy val declares: List[String] = use
.map(u => u.declares.map(n => s"${u.access}.$n"))
.getOrElse(Nil)
.prepended(s"TestSrv$idx")
lazy val code: String =
s"""|aqua $name declares ${declares.mkString(", ")}
|
|${use.fold("")(_.usage)}
|
|service TestSrv$idx("test-srv-$idx"):
| call()
|""".stripMargin
lazy val usage: String = s"use \"$path\"" + rename.fold("")(n => s" as $n")
lazy val access: String = rename.getOrElse(name)
}
def test(imps: List[NameRename]) = {
(imps.length > 0) should be(true)
val top = imps.zipWithIndex.map { case ((name, rename), idx) =>
Imp(idx, name, rename)
}.reduceRight { case (cur, prev) =>
cur.withUse(prev)
}
val lines = top.declares.zipWithIndex.flatMap { case (decl, idx) =>
val call: List[String] = List(
s"${top.access}.$decl.call()",
s"doCallTwice{${top.access}.$decl}(${top.access}.$decl)"
)
val resolve = s"${top.access}.$decl \"test-srv-$idx-resolved\""
def capture(n: Int): List[String] = {
val cName = s"c${idx}n$n"
List(
s"$cName = ${top.access}.$decl.call",
s"$cName()",
s"callCapture($cName)"
)
}
call ++ capture(0) ++ List(resolve) ++ call ++ capture(1)
}
val main =
s"""|aqua Main
|
|export main
|
|${top.usage}
|
|ability TestAb:
| call()
|
|func doCallTwice{TestAb}(ab: TestAb):
| TestAb.call()
| ab.call()
|
|func callCapture(capture: -> ()):
| capture()
|
|func main():
| ${lines.mkString("\n ")}
|""".stripMargin
val allImps = List.unfold(top.some)(_.map(i => i -> i.use))
val imports = allImps.map(i => i.path -> i.code).toMap
val src = Map("main.aqua" -> main)
val transformCfg = TransformConfig(relayVarName = None, noEmptyResponse = true)
insideRes(src, imports, transformCfg)(
"main"
) { case main :: _ =>
def serviceCalls(idx: Int): List[CallServiceRes] = {
val default = CallServiceRes(
LiteralModel.quote(s"test-srv-$idx"),
"call",
CallRes(Nil, None),
initPeer
)
val resolved = default.copy(
serviceId = LiteralModel.quote(s"test-srv-$idx-resolved")
)
List.fill(5)(default) ++ List.fill(5)(resolved)
}
val expected = XorRes.wrap(
SeqRes.wrap(
allImps.flatMap(i => serviceCalls(i.idx)).map(_.leaf)
),
errorCall(transformCfg, 0, initPeer)
)
main.body.equalsOrShowDiff(expected) shouldBe (true)
}
}
extension [A](l: List[List[A]]) {
def rotate: List[List[A]] =
l.foldLeft(List.empty[List[A]]) { case (acc, next) =>
if (acc.isEmpty) next.map(List(_))
else
for {
elem <- next
prev <- acc
} yield elem +: prev
}
testImportsHierarchy(test)
}
it should "import redeclared abilities" in {
final case class Imp(
idx: Int,
name: String,
rename: Option[String] = None,
use: Option[Imp] = None
) {
def withUse(other: Imp): Imp = copy(use = Some(other))
lazy val path: String = s"import$idx.aqua"
lazy val declares: List[String] = use
.map(u => u.declares.map(n => s"${u.access}.$n"))
.getOrElse(Nil)
.prepended(s"TestAb$idx")
lazy val code: String =
s"""|aqua $name declares ${declares.mkString(", ")}
|
|${use.fold("")(_.usage)}
|
|ability TestAb$idx:
| call(x: i32) -> i32
| value: i32
|""".stripMargin
lazy val usage: String = s"use \"$path\"" + rename.fold("")(n => s" as $n")
lazy val access: String = rename.getOrElse(name)
}
// With subpaths
(1 to 4).foreach { i =>
(1 to i)
.map(idx =>
paths(
List("Imp", "Sub", "Path")
.map(p => s"$p$idx")
)
)
.toList
.rotate
.foreach(names =>
withClue(s"Testing ${names.mkString(" -> ")}") {
test(names.map(_ -> none))
def test(imps: List[NameRename]) = {
(imps.length > 0) should be(true)
val top = imps.zipWithIndex.map { case ((name, rename), idx) =>
Imp(idx, name, rename)
}.reduceRight { case (cur, prev) =>
cur.withUse(prev)
}
val mainAb = top.declares.zipWithIndex.map { case (decl, idx) =>
s"ab$idx: ${top.access}.$decl"
}.prepended("ability MainAb:").mkString("\n ")
val funcs = top.declares.zipWithIndex.map { case (decl, idx) =>
val full = s"${top.access}.$decl"
s"""|func f$idx{$full}() -> i32:
| call = $full.call
| val = $full.value
| <- call(val) + $full.call($full.value)
|""".stripMargin
}.mkString("\n")
val (abs, definitions) = top.declares.zipWithIndex.map { case (decl, idx) =>
val ab = s"ab$idx"
val definition = s"$ab = ${top.access}.$decl(call, value)"
ab -> definition
}.unzip
val (results, calls) = top.declares.indices.map { idx =>
val result = s"v$idx"
val call = s"$result <- f$idx{ab$idx}()"
result -> call
}.unzip
val mainDef = s"mainAb = MainAb(${abs.mkString(", ")})"
val (mainResults, mainCalls) = abs.zipWithIndex.map { case (ab, idx) =>
val result = s"vM$idx"
val call = s"$result <- mainAb.$ab.call(mainAb.$ab.value)"
result -> call
}.unzip
val main =
s"""|aqua Main
|
|export main
|
|${top.usage}
|
|$mainAb
|
|$funcs
|
|func main(value: i32) -> i32:
| call = (x: i32) -> i32:
| <- x + 1
| ${definitions.mkString("\n ")}
| ${calls.mkString("\n ")}
| $mainDef
| ${mainCalls.mkString("\n ")}
| <- ${(results ++ mainResults).mkString(" + ")}
|""".stripMargin
val allImps = List.unfold(top.some)(_.map(i => i -> i.use))
val imports = allImps.map(i => i.path -> i.code).toMap
val src = Map("main.aqua" -> main)
val transformCfg = TransformConfig(relayVarName = None, noEmptyResponse = true)
insideRes(src, imports, transformCfg)(
"main"
) { case main :: _ =>
val adds = Cofree
.cata(main.body) { (parent, children: Chain[Chain[CallServiceRes]]) =>
parent match {
case p @ CallServiceRes(LiteralModel.String("\"math\""), "add", _, _) =>
Eval.now(p +: children.flatten)
case _ => Eval.now(children.flatten)
}
}
)
}
// With renames
(1 to 3).foreach { i =>
(1 to i)
.map(idx =>
for {
name <- paths(
List("Imp", "Sub", "Path")
.map(p => s"$p$idx")
)
rename <- None :: paths(
List("Rename", "To", "Other")
.map(p => s"$p$idx")
).map(_.some)
} yield name -> rename
)
.toList
.rotate
.foreach(names =>
val message = names.map { case (n, r) =>
s"$n${r.fold("")(n => s" as $n")}"
}.mkString(" -> ")
withClue(s"Testing $message") {
test(names)
}
)
.value
/**
* 3 * n for calls to `func`s
* n for for calls through `mainAb`
* 2 * n - 1 for final sum
*/
adds.size should be(abs.length * 6 - 1)
}
}
testImportsHierarchy(test)
}
it should "not generate error propagation in `if` with `noXor = true`" in {

View File

@ -552,8 +552,9 @@ object ArrowInliner extends Logging {
for {
// Process renamings, prepare environment
fnCanon <- ArrowInliner.prelude(arrow, call, exports, arrows)
inlineResult <- ArrowInliner.inline(fnCanon._1, call, streams)
} yield inlineResult.copy(tree = SeqModel.wrap(fnCanon._2, inlineResult.tree))
(fn, canons) = fnCanon
inlineResult <- ArrowInliner.inline(fn, call, streams)
} yield inlineResult.copy(tree = SeqModel.wrap(canons, inlineResult.tree))
)
)

View File

@ -343,7 +343,8 @@ object TagInliner extends Logging {
for {
_ <- Exports[S].resolved(name, VarModel(name, t))
} yield TagInlined.Empty()
case _ => internalError(s"Cannot declare $value as stream, because it is not a stream type")
case _ =>
internalError(s"Cannot declare $value as stream, because it is not a stream type")
case ServiceIdTag(id, serviceType, name) =>
for {

View File

@ -8,10 +8,11 @@ import aqua.raw.{ConstantRaw, RawContext, RawPart, ServiceRaw, TypeRaw}
import aqua.types.{AbilityType, StructType, Type}
import cats.Monoid
import cats.data.Chain
import cats.data.NonEmptyMap
import cats.data.{Chain, NonEmptyMap, State}
import cats.kernel.Semigroup
import cats.syntax.applicative.*
import cats.syntax.bifunctor.*
import cats.syntax.flatMap.*
import cats.syntax.foldable.*
import cats.syntax.functor.*
import cats.syntax.monoid.*
@ -40,13 +41,16 @@ case class AquaContext(
}
).map(_.leftMap(prefix + _)).toMap
lazy val allServices: Map[String, ServiceModel] =
all(_.services)
lazy val allValues: Map[String, ValueModel] =
all(_.values) ++
/**
* Add values from services that have default ID
* So that they will be available in functions.
*/
services.flatMap { case (srvName, srv) =>
allServices.flatMap { case (srvName, srv) =>
srv.defaultId.toList.flatMap(_ =>
srv.`type`.arrows.map { case (arrowName, arrowType) =>
val fullName = AbilityType.fullName(srvName, arrowName)
@ -71,7 +75,7 @@ case class AquaContext(
* Add functions from services that have default ID
* So that they will be available in functions.
*/
services.flatMap { case (srvName, srv) =>
allServices.flatMap { case (srvName, srv) =>
srv.defaultId.toList.flatMap(id =>
srv.`type`.arrows.map { case (arrowName, arrowType) =>
val fullName = AbilityType.fullName(srvName, arrowName)
@ -109,17 +113,69 @@ case class AquaContext(
pickOne(name, newName, abilities, (ctx, el) => ctx.copy(abilities = el)) |+|
pickOne(name, newName, services, (ctx, el) => ctx.copy(services = el))
}
def withModule(newModule: Option[String]): AquaContext =
copy(module = newModule)
def withAbilities(newAbilities: Map[String, AquaContext]): AquaContext =
copy(abilities = newAbilities)
def withServices(newServices: Map[String, ServiceModel]): AquaContext =
copy(services = newServices)
def withValues(newValues: Map[String, ValueModel]): AquaContext =
copy(values = newValues)
def withFuncs(newFuncs: Map[String, FuncArrow]): AquaContext =
copy(funcs = newFuncs)
def withTypes(newTypes: Map[String, Type]): AquaContext =
copy(types = newTypes)
override def toString(): String =
s"AquaContext(" +
s"module=$module, " +
s"funcs=${funcs.keys}, " +
s"types=${types.keys}, " +
s"values=${values.keys}, " +
s"abilities=${abilities.keys}, " +
s"services=${services.keys})"
}
object AquaContext extends Logging {
case class Cache(private val data: Chain[(RawContext, AquaContext)] = Chain.empty) {
case class Cache private (
private val data: Map[Cache.RefKey[RawContext], AquaContext]
) {
lazy val size: Long = data.size
def get(ctx: RawContext): Option[AquaContext] =
data.collectFirst { case (rawCtx, aquaCtx) if rawCtx eq ctx => aquaCtx }
private def get(ctx: RawContext): Option[AquaContext] =
data.get(Cache.RefKey(ctx))
def updated(ctx: RawContext, aCtx: AquaContext): Cache = copy(data :+ (ctx -> aCtx))
private def updated(ctx: RawContext, aCtx: AquaContext): Cache =
copy(data = data.updated(Cache.RefKey(ctx), aCtx))
}
type Cached[A] = State[Cache, A]
object Cache {
val empty: Cache = Cache(Map.empty)
def get(ctx: RawContext): Cached[Option[AquaContext]] =
State.inspect(_.get(ctx))
def updated(ctx: RawContext, aCtx: AquaContext): Cached[Unit] =
State.modify(_.updated(ctx, aCtx))
private class RefKey[T <: AnyRef](val ref: T) extends AnyVal {
override def equals(other: Any): Boolean = other match {
case that: RefKey[_] => that.ref eq ref
}
override def hashCode(): Int = System.identityHashCode(ref)
}
}
val blank: AquaContext =
@ -141,9 +197,9 @@ object AquaContext extends Logging {
)
def fromService(sm: ServiceRaw, serviceId: ValueRaw): AquaContext =
blank.copy(
module = Some(sm.name),
funcs = sm.`type`.arrows.map { case (fnName, arrowType) =>
blank
.withModule(Some(sm.name))
.withFuncs(sm.`type`.arrows.map { case (fnName, arrowType) =>
fnName -> FuncArrow.fromServiceMethod(
fnName,
sm.name,
@ -151,108 +207,80 @@ object AquaContext extends Logging {
arrowType,
serviceId
)
}
)
})
// Convert RawContext into AquaContext, with exports handled
def exportsFromRaw(rawContext: RawContext, cache: Cache): (AquaContext, Cache) = {
logger.trace(s"ExportsFromRaw ${rawContext.module}")
val (ctx, newCache) = fromRawContext(rawContext, cache)
logger.trace("raw: " + rawContext)
logger.trace("ctx: " + ctx)
rawContext.exports
.foldLeft(
// Module name is what persists
blank.copy(
module = ctx.module
)
) { case (acc, (k, v)) =>
// Pick exported things, accumulate
acc |+| ctx.pick(k, v)
} -> newCache
}
def exportsFromRaw(raw: RawContext): Cached[AquaContext] = for {
ctx <- fromRawContext(raw)
handled = raw.exports.toList
.foldMap(ctx.pick.tupled)
.withModule(ctx.module)
} yield handled
// Convert RawContext into AquaContext, with no exports handled
private def fromRawContext(rawContext: RawContext, cache: Cache): (AquaContext, Cache) =
cache
.get(rawContext)
.fold {
logger.trace(s"Compiling ${rawContext.module}, cache has ${cache.size} entries")
val (newCtx, newCache) = rawContext.parts
.foldLeft[(AquaContext, Cache)] {
// Laziness unefficiency happens here
logger.trace(s"raw: ${rawContext.module}")
val (abs, absCache) =
rawContext.abilities.foldLeft[(Map[String, AquaContext], Cache)]((Map.empty, cache)) {
case ((acc, cAcc), (k, v)) =>
val (abCtx, abCache) = fromRawContext(v, cAcc)
(acc + (k -> abCtx), abCache)
}
blank.copy(abilities = abs) -> absCache
} {
case ((ctx, ctxCache), (partContext, c: ConstantRaw)) =>
logger.trace("Adding constant " + c.name)
// Just saving a constant
// Actually this should have no effect, as constants are resolved by semantics
val (pctx, pcache) = fromRawContext(partContext, ctxCache)
logger.trace("Got " + c.name + " from raw")
val add =
blank
.copy(values =
if (c.allowOverrides && pctx.values.contains(c.name)) Map.empty
else Map(c.name -> ValueModel.fromRaw(c.value).resolveWith(pctx.allValues))
)
(ctx |+| add, pcache)
case ((ctx, ctxCache), (partContext, func: FuncRaw)) =>
// To add a function, we have to know its scope
logger.trace("Adding func " + func.name)
val (pctx, pcache) = fromRawContext(partContext, ctxCache)
logger.trace("Got " + func.name + " from raw")
val fr = FuncArrow.fromRaw(func, pctx.allFuncs, pctx.allValues, None)
logger.trace("Captured recursively for " + func.name)
val add = blank.copy(funcs = Map(func.name -> fr))
(ctx |+| add, pcache)
case ((ctx, ctxCache), (_, t: TypeRaw)) =>
// Just remember the type (why? it's can't be exported, so seems useless)
val add = blank.copy(types = Map(t.name -> t.`type`))
(ctx |+| add, ctxCache)
case ((ctx, ctxCache), (partContext, m: ServiceRaw)) =>
// To add a service, we need to resolve its ID, if any
logger.trace("Adding service " + m.name)
val (pctx, pcache) = fromRawContext(partContext, ctxCache)
logger.trace("Got " + m.name + " from raw")
val id = m.defaultId
.map(ValueModel.fromRaw)
.map(_.resolveWith(pctx.allValues))
val srv = ServiceModel(m.name, m.`type`, id)
val add =
blank
.copy(
abilities = m.defaultId
.map(id => Map(m.name -> fromService(m, id)))
.orEmpty,
services = Map(m.name -> srv)
)
(ctx |+| add, pcache)
case (ctxAndCache, _) => ctxAndCache
}
(newCtx, newCache.updated(rawContext, newCtx))
} { ac =>
logger.trace("Got from cache")
ac -> cache
private def fromRawContext(raw: RawContext): Cached[AquaContext] =
Cache
.get(raw)
.flatMap {
case Some(aCtx) => aCtx.pure
case None =>
for {
init <- raw.abilities.toList.traverse { case (name, ab) =>
fromRawContext(ab).map(name -> _)
}.map(abs => blank.withAbilities(abs.toMap))
parts <- raw.parts.foldMapM(handlePart.tupled)
} yield init |+| parts
}
.flatTap(aCtx => Cache.updated(raw, aCtx))
private def handlePart(raw: RawContext, part: RawPart): Cached[AquaContext] =
part match {
case c: ConstantRaw =>
// Just saving a constant
// Actually this should have no effect, as constants are resolved by semantics
fromRawContext(raw).map(pctx =>
blank.withValues(
if (c.allowOverrides && pctx.values.contains(c.name)) Map.empty
else Map(c.name -> ValueModel.fromRaw(c.value).resolveWith(pctx.allValues))
)
)
case func: FuncRaw =>
fromRawContext(raw).map(pctx =>
blank.withFuncs(
Map(
func.name -> FuncArrow.fromRaw(
raw = func,
arrows = pctx.allFuncs,
constants = pctx.allValues,
topology = None
)
)
)
)
case t: TypeRaw =>
// Just remember the type (why? it's can't be exported, so seems useless)
blank.withTypes(Map(t.name -> t.`type`)).pure
case m: ServiceRaw =>
// To add a service, we need to resolve its ID, if any
fromRawContext(raw).map { pctx =>
val id = m.defaultId
.map(ValueModel.fromRaw)
.map(_.resolveWith(pctx.allValues))
val srv = ServiceModel(m.name, m.`type`, id)
blank
.withAbilities(
m.defaultId
.map(id => Map(m.name -> fromService(m, id)))
.orEmpty
)
.withServices(Map(m.name -> srv))
}
case _ => blank.pure
}
}

View File

@ -123,6 +123,18 @@ object LiteralModel {
}
}
/*
* Used to match string literals in pattern matching
*/
object String {
def unapply(lm: LiteralModel): Option[String] =
lm match {
case LiteralModel(value, ScalarType.string | LiteralType.string) => value.some
case _ => none
}
}
// AquaVM will return 0 for
// :error:.$.error_code if there is no :error:
val emptyErrorCode = number(0)

View File

@ -143,7 +143,7 @@ class ValuesAlgebra[S[_], Alg[_]: Monad](using
.widen[ValueToken[S]]
val ability = OptionT(
prop.toAbility.findM { case (ab, _) =>
prop.toAbility.reverse.findM { case (ab, _) =>
// Test if name is an import
A.isDefinedAbility(ab)
}