[Engine] Load stable packages from ressources

sdk/compiler/scenario-service/server/BUILD.bazel

@@ -35,6 +35,7 @@ da_scala_binary(
         "//daml-lf/interpreter",
         "//daml-lf/language",
         "//daml-lf/scenario-interpreter",
+        "//daml-lf/stable-packages",
         "//daml-lf/transaction",
         "//daml-lf/validation",
         "//daml-script/converter",

sdk/compiler/scenario-service/server/src/main/scala/com/daml/lf/scenario/Context.scala

@@ -26,14 +26,15 @@ import com.daml.lf.validation.Validation
 import com.daml.script.converter
 import com.google.protobuf.ByteString
 import com.daml.lf.engine.script.{Runner, Script}
+import com.daml.lf.stablepackages.StablePackagesV2
 import com.daml.logging.LoggingContext
 import org.slf4j.LoggerFactory
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 import scala.collection.immutable.HashMap
 import scala.concurrent.duration._
-import scala.util.{Failure, Success}
+import scala.util.{Success, Failure}
 
 /** Scenario interpretation context: maintains a set of modules and external packages, with which
   * scenarios can be interpreted.
@@ -76,8 +77,9 @@ class Context(
     */
   val homePackageId: PackageId = PackageId.assertFromString("-homePackageId-")
 
-  private var extSignatures: Map[PackageId, Ast.PackageSignature] = HashMap.empty
-  private var extDefns: Map[SDefinitionRef, SDefinition] = HashMap.empty
+  private var extSignatures: Map[PackageId, Ast.PackageSignature] =
+    StablePackagesV2.allPackageSignatures
+  private var extDefns: Map[SDefinitionRef, SDefinition] = Compiler.stablePackageDefs.toMap
   private var modules: Map[ModuleName, Ast.Module] = HashMap.empty
   private var modDefns: Map[ModuleName, Map[SDefinitionRef, SDefinition]] = HashMap.empty
   private var defns: Map[SDefinitionRef, SDefinition] = HashMap.empty

sdk/daml-lf/engine/src/main/scala/com/daml/lf/engine/ConcurrentCompiledPackages.scala

@@ -10,6 +10,7 @@ import com.daml.lf.engine.ConcurrentCompiledPackages.AddPackageState
 import com.daml.lf.language.Ast.{Package, PackageSignature}
 import com.daml.lf.language.{Util => AstUtil}
 import com.daml.lf.speedy.Compiler
+import com.daml.lf.stablepackages.StablePackagesV2
 import com.daml.nameof.NameOf
 import com.daml.scalautil.Statement.discard
 
@@ -28,6 +29,7 @@ private[lf] final class ConcurrentCompiledPackages(compilerConfig: Compiler.Conf
     new ConcurrentHashMap().asScala
   private[this] val packageDeps: ConcurrentMap[PackageId, Set[PackageId]] =
     new ConcurrentHashMap().asScala
+  clear()
 
   /** Might ask for a package if the package you're trying to add references it.
     *
@@ -152,10 +154,16 @@ private[lf] final class ConcurrentCompiledPackages(compilerConfig: Compiler.Conf
       ResultDone.Unit
     }
 
+  private lazy val stablePkgSig =
+    StablePackagesV2.packageSignatures(compilerConfig.allowedLanguageVersions.max)
+
   def clear(): Unit = this.synchronized[Unit] {
     signatures.clear()
+    discard(signatures ++= stablePkgSig)
     definitions.clear()
+    discard(definitions ++= Compiler.stablePackageDefs)
     packageDeps.clear()
+    discard(packageDeps ++= stablePkgSig.mapValues(_.directDeps))
   }
 
   def getPackageDependencies(pkgId: PackageId): Option[Set[PackageId]] =

sdk/daml-lf/engine/src/main/scala/com/daml/lf/engine/Engine.scala

@@ -25,7 +25,7 @@ import com.daml.lf.value.Value
 import com.daml.lf.value.Value.ContractId
 import com.daml.lf.language.{LanguageMajorVersion, LanguageVersion, LookupError, PackageInterface}
 import com.daml.lf.speedy.Speedy.Machine.newTraceLog
-import com.daml.lf.stablepackages.StablePackages
+import com.daml.lf.stablepackages.StablePackagesV2
 import com.daml.lf.validation.Validation
 import com.daml.logging.LoggingContext
 import com.daml.nameof.NameOf
@@ -88,7 +88,8 @@ class Engine(val config: EngineConfig) {
 
   private[this] val compiledPackages = ConcurrentCompiledPackages(config.getCompilerConfig)
 
-  private[this] val stablePackageIds = StablePackages.ids(config.allowedLanguageVersions)
+  private[this] val stablePackageIds =
+    StablePackagesV2.packageSignatures(config.allowedLanguageVersions.max).keySet
 
   private[engine] val preprocessor =
     new preprocessing.Preprocessor(

sdk/daml-lf/engine/src/test/scala/com/daml/lf/engine/EngineTest.scala

@@ -2337,29 +2337,24 @@ class EngineTest(majorLanguageVersion: LanguageMajorVersion)
       )
 
     val devVersion = majorLanguageVersion.dev
-    val (_, _, allPackagesDev) = new EngineTestHelpers(majorLanguageVersion).loadAndAddPackage(
-      s"daml-lf/engine/BasicTests-v${majorLanguageVersion.pretty}dev.dar"
-    )
     val compatibleLanguageVersions = LanguageVersion.AllV2
     val stablePackages = StablePackages(majorLanguageVersion).allPackages
 
     s"accept stable packages from ${devVersion} even if version is smaller than min version" in {
       for {
         lv <- compatibleLanguageVersions.filter(_ <= devVersion)
         eng = engine(min = lv, max = devVersion)
-        pkg <- stablePackages
-        pkgId = pkg.packageId
-        pkg <- allPackagesDev.get(pkgId).toList
+        entry <- stablePackages
+        (pkgId, pkg) = entry
       } yield eng.preloadPackage(pkgId, pkg) shouldBe a[ResultDone[_]]
     }
 
     s"reject stable packages from ${devVersion} if version is greater than max version" in {
       for {
         lv <- compatibleLanguageVersions
         eng = engine(min = compatibleLanguageVersions.min, max = lv)
-        pkg <- stablePackages
-        pkgId = pkg.packageId
-        pkg <- allPackagesDev.get(pkgId).toList
+        entry <- stablePackages
+        (pkgId, pkg) = entry
       } yield inside(eng.preloadPackage(pkgId, pkg)) {
         case ResultDone(_) => pkg.languageVersion shouldBe <=(lv)
         case ResultError(_) => pkg.languageVersion shouldBe >(lv)

sdk/daml-lf/interpreter/src/main/scala/com/daml/lf/CompiledPackages.scala

@@ -8,6 +8,7 @@ import com.daml.lf.language.Ast.{Package, PackageSignature}
 import com.daml.lf.language.{PackageInterface, Util}
 import com.daml.lf.speedy.SExpr.SDefinitionRef
 import com.daml.lf.speedy.{Compiler, SDefinition}
+import com.daml.lf.stablepackages.StablePackagesV2
 
 /** Trait to abstract over a collection holding onto Daml-LF package definitions + the
   * compiled speedy expressions.
@@ -49,9 +50,13 @@ private[lf] object PureCompiledPackages {
       packages: Map[PackageId, Package],
       compilerConfig: Compiler.Config,
   ): Either[String, PureCompiledPackages] = {
+    val stablePkgSigs =
+      StablePackagesV2.packageSignatures(compilerConfig.allowedLanguageVersions.max)
+    val pkgsToCompile = packages.view.filterKeys(x => !stablePkgSigs.contains(x))
+    val pkgSignatures = Util.toSignatures(packages) ++ stablePkgSigs
     Compiler
-      .compilePackages(PackageInterface(packages), packages, compilerConfig)
-      .map(apply(Util.toSignatures(packages), _, compilerConfig))
+      .compilePackages(new PackageInterface(pkgSignatures), pkgsToCompile, compilerConfig)
+      .map(compiled => apply(pkgSignatures, stableDefs ++ compiled, compilerConfig))
   }
 
   def assertBuild(
@@ -60,7 +65,13 @@ private[lf] object PureCompiledPackages {
   ): PureCompiledPackages =
     data.assertRight(build(packages, compilerConfig))
 
-  def Empty(compilerConfig: Compiler.Config): PureCompiledPackages =
-    PureCompiledPackages(Map.empty, Map.empty, compilerConfig)
+  private[this] val stableDefs = Compiler.stablePackageDefs.to(collection.immutable.HashMap)
+
+  // Contains only stable packages
+  def Empty(compilerConfig: Compiler.Config): PureCompiledPackages = {
+    val stablePkgSigs =
+      StablePackagesV2.packageSignatures(compilerConfig.allowedLanguageVersions.max)
+    PureCompiledPackages.apply(stablePkgSigs.toMap, stableDefs, compilerConfig)
+  }
 
 }

sdk/daml-lf/interpreter/src/main/scala/com/daml/lf/speedy/Compiler.scala

@@ -16,7 +16,7 @@ import com.daml.lf.speedy.SBuiltinFun._
 import com.daml.lf.speedy.SValue._
 import com.daml.lf.speedy.{SExpr => t}
 import com.daml.lf.speedy.{SExpr0 => s}
-import com.daml.lf.stablepackages.StablePackages
+import com.daml.lf.stablepackages.StablePackagesV2
 import com.daml.lf.validation.{Validation, ValidationError}
 import com.daml.scalautil.Statement.discard
 import org.slf4j.LoggerFactory
@@ -89,6 +89,21 @@ private[lf] object Compiler {
     )
   }
 
+  val stablePackageDefs: List[(SExpr.SDefinitionRef, SDefinition)] =
+    compilePackages(
+      new PackageInterface(StablePackagesV2.allPackageSignatures),
+      StablePackagesV2.allPackages,
+      Compiler.Config(
+        allowedLanguageVersions = LanguageVersion.AllVersions(LanguageVersion.Major.V2),
+        packageValidation = FullPackageValidation,
+        profiling = NoProfile,
+        stacktracing = NoStackTrace,
+      ),
+    ) match {
+      case Left(err) => throw new Error(s"Cannot compile stable packages: $err")
+      case Right(defs) => defs.toList
+    }
+
   /** Validates and Compiles all the definitions in the packages provided. Returns them in a Map.
     *
     * The packages do not need to be in any specific order, as long as they and all the packages
@@ -97,15 +112,13 @@ private[lf] object Compiler {
 
   private[lf] def compilePackages(
       pkgInterface: PackageInterface,
-      packages: Map[PackageId, Package],
+      packages: Iterable[(PackageId, Package)],
       compilerConfig: Compiler.Config,
-  ): Either[String, Map[t.SDefinitionRef, SDefinition]] = {
+  ): Either[String, Iterable[(t.SDefinitionRef, SDefinition)]] = {
     val compiler = new Compiler(pkgInterface, compilerConfig)
     try {
       Right(
-        packages.foldLeft(Map.empty[t.SDefinitionRef, SDefinition]) { case (acc, (pkgId, pkg)) =>
-          acc ++ compiler.compilePackage(pkgId, pkg)
-        }
+        packages.view.flatMap { case (pkgId, pkg) => compiler.compilePackage(pkgId, pkg) }
       )
     } catch {
       case CompilationError(msg) => Left(s"Compilation Error: $msg")
@@ -172,8 +185,6 @@ private[lf] final class Compiler(
     pipeline(e0)
   }
 
-  private[this] val stablePackageIds = StablePackages.ids(config.allowedLanguageVersions)
-
   private[this] val logger = LoggerFactory.getLogger(this.getClass)
 
   // We add labels before and after flattening
@@ -437,15 +448,8 @@ private[lf] final class Compiler(
 
     val t0 = Time.Timestamp.now()
 
-    pkgInterface.lookupPackage(pkgId) match {
-      case Right(pkg) =>
-        if (
-          !stablePackageIds.contains(pkgId) && !config.allowedLanguageVersions
-            .contains(pkg.languageVersion)
-        )
-          throw LanguageVersionError(pkgId, pkg.languageVersion, config.allowedLanguageVersions)
-      case _ =>
-    }
+    if (!config.allowedLanguageVersions.contains(pkg.languageVersion))
+      throw LanguageVersionError(pkgId, pkg.languageVersion, config.allowedLanguageVersions)
 
     config.packageValidation match {
       case Compiler.NoPackageValidation =>

sdk/daml-lf/repl/src/main/scala/com/daml/lf/speedy/testing/Main.scala

@@ -361,7 +361,7 @@ class Repl(majorLanguageVersion: LanguageMajorVersion) {
         state.scenarioRunner.compilerConfig,
       )
     )
-    defs.get(idToRef(state, args(0))) match {
+    defs.collectFirst { case (pkgId, pkg) if pkgId == idToRef(state, args(0)) => pkg } match {
       case None =>
         println("Error: definition '" + args(0) + "' not found. Try :list.")
         usage()

sdk/daml-lf/stable-packages/BUILD.bazel

@@ -25,6 +25,7 @@ da_scala_library(
         "//compiler/damlc/stable-packages:gen-stable-packages-manifest-v2",
     ],
     scala_deps = [
+        "@maven//:org_scalaz_scalaz_core",
     ],
     scalacopts = lf_scalacopts_stricter,
     tags = ["maven_coordinates=com.daml:daml-lf-stable-packages:__VERSION__"],

sdk/daml-lf/stable-packages/src/main/scala/com/daml/lf/stablepackages/StablePackages.scala

@@ -4,11 +4,13 @@
 package com.daml.lf
 package stablepackages
 
-import com.daml.lf.VersionRange
-import com.daml.lf.archive
 import com.daml.lf.archive.ArchiveDecoder
 import com.daml.lf.data.Ref
-import com.daml.lf.language.{Ast, LanguageMajorVersion, LanguageVersion}
+import com.daml.lf.data.Ref.PackageId
+import com.daml.lf.language.Ast.PackageSignature
+import com.daml.lf.language.{Ast, LanguageVersion, LanguageMajorVersion, Util => AstUtil}
+
+import scala.collection.MapView
 
 private[daml] sealed case class StablePackage(
     moduleName: Ref.ModuleName,
@@ -22,10 +24,25 @@ private[daml] sealed case class StablePackage(
   @throws[IllegalArgumentException]
   def assertIdentifier(idName: String): Ref.Identifier =
     identifier(Ref.DottedName.assertFromString(idName))
+
 }
 
 private[daml] sealed abstract class StablePackages {
-  val allPackages: Seq[StablePackage]
+
+  import Ordering.Implicits._
+
+  val allPackages: Map[Ref.PackageId, Ast.Package]
+
+  def packages(maxVersion: LanguageVersion): MapView[Ref.PackageId, Ast.Package] =
+    allPackages.view.filter { case (_, pkg) => pkg.languageVersion <= maxVersion }
+
+  final lazy val allPackageSignatures: Map[PackageId, PackageSignature] =
+    AstUtil.toSignatures(allPackages)
+
+  def packageSignatures(maxVersion: LanguageVersion): MapView[Ref.PackageId, Ast.PackageSignature] =
+    allPackageSignatures.view.filter { case (_, pkg) => pkg.languageVersion <= maxVersion }
+
+  val stablePackagesByName: Map[String, StablePackage]
 
   val ArithmeticError: Ref.TypeConName
   val AnyChoice: Ref.TypeConName
@@ -47,18 +64,6 @@ private[daml] object StablePackages {
       case LanguageMajorVersion.V1 => throw new IllegalArgumentException("LF1 is not supported")
       case LanguageMajorVersion.V2 => StablePackagesV2
     }
-
-  /** The IDs of stable packages compatible with the provided version range. */
-  def ids(allowedLanguageVersions: VersionRange[LanguageVersion]): Set[Ref.PackageId] = {
-    import com.daml.lf.language.LanguageVersionRangeOps.LanguageVersionRange
-
-    import scala.Ordering.Implicits.infixOrderingOps
-
-    StablePackages(allowedLanguageVersions.majorVersion).allPackages.view
-      .filter(_.languageVersion <= allowedLanguageVersions.max)
-      .map(_.packageId)
-      .toSet
-  }
 }
 
 /** @param manifestResourcePath the path of a resource that contains a newline-separated list of
@@ -68,7 +73,12 @@ private[daml] sealed class StablePackagesImpl(
     protected val manifestResourcePath: String
 ) extends StablePackages {
 
-  override lazy val allPackages: Seq[StablePackage] = allPackagesByName.values.toSeq
+  override lazy val allPackages: Map[Ref.PackageId, Ast.Package] =
+    scala.io.Source
+      .fromResource(manifestResourcePath)
+      .getLines()
+      .map(decodeDalfResource)
+      .toMap
 
   override lazy val ArithmeticError: Ref.TypeConName =
     DA_Exception_ArithmeticError.assertIdentifier("ArithmeticError")
@@ -83,18 +93,17 @@ private[daml] sealed class StablePackagesImpl(
   override lazy val Tuple3: Ref.TypeConName = DA_Types.assertIdentifier("Tuple3")
   override lazy val Either: Ref.TypeConName = GHC_Tuple.assertIdentifier("Either")
 
-  private lazy val DA_Exception_ArithmeticError = allPackagesByName("DA.Exception.ArithmeticError")
-  private lazy val DA_Internal_Any = allPackagesByName("DA.Internal.Any")
-  private lazy val DA_NonEmpty_Types = allPackagesByName("DA.NonEmpty.Types")
-  private lazy val DA_Types = allPackagesByName("DA.Types")
-  private lazy val GHC_Tuple = allPackagesByName("GHC.Tuple")
+  private lazy val DA_Exception_ArithmeticError = stablePackagesByName(
+    "DA.Exception.ArithmeticError"
+  )
+  private lazy val DA_Internal_Any = stablePackagesByName("DA.Internal.Any")
+  private lazy val DA_NonEmpty_Types = stablePackagesByName("DA.NonEmpty.Types")
+  private lazy val DA_Types = stablePackagesByName("DA.Types")
+  private lazy val GHC_Tuple = stablePackagesByName("GHC.Tuple")
 
   /** All stable packages, indexed by module name. */
-  private lazy val allPackagesByName: Map[String, StablePackage] =
-    scala.io.Source
-      .fromResource(manifestResourcePath)
-      .getLines()
-      .map(decodeDalfResource)
+  override lazy val stablePackagesByName: Map[String, StablePackage] =
+    allPackages
       .map((toStablePackage _).tupled)
       .map(pkg => pkg.moduleName.dottedName -> pkg)
       .toMap

sdk/daml-lf/stable-packages/src/test/scala/com/daml/lf/stablepackages/StablePackagesTest.scala

@@ -39,19 +39,19 @@ class StablePackageTest(majorLanguageVersion: LanguageMajorVersion)
     lazy val depPkgs = UniversalArchiveDecoder.assertReadFile(darFile).dependencies.toMap
 
     // This should be all stable packages + `daml-prim` + `daml-stdlib`
-    assert(depPkgs.size == stablePackages.allPackages.size + 2)
+    assert(depPkgs.size == stablePackages.stablePackagesByName.size + 2)
 
     "contains all the stable packages generated by the compiler" in {
       val pkgIdsInDar = depPkgs.keySet
-      stablePackages.allPackages.foreach(pkg =>
+      stablePackages.stablePackagesByName.values.foreach(pkg =>
         withClue(s"in ${pkg.name} ${pkg.moduleName}") {
           pkgIdsInDar should contain(pkg.packageId)
         }
       )
     }
 
     "lists the stable packages with their proper version" in {
-      stablePackages.allPackages.foreach(pkg =>
+      stablePackages.stablePackagesByName.values.foreach(pkg =>
         withClue(s"in ${pkg.name} ${pkg.moduleName} ${depPkgs(pkg.packageId).modules}") {
           pkg.languageVersion shouldBe depPkgs(pkg.packageId).languageVersion
         }