diff --git a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala index 668901dcc..b6eba2765 100644 --- a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala +++ b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala @@ -149,13 +149,13 @@ private[sbt] object EvaluateConfigurations { case file => file.id val parsed = parseConfiguration(file, lines, imports, offset) val (importDefs, definitions) = - if (parsed.definitions.isEmpty) (Nil, DefinedSbtValues.empty) - else { - val definitions = - evaluateDefinitions(eval, name, parsed.imports, parsed.definitions, Some(file)) - val imp = BuildUtilLite.importAllRoot(definitions.enclosingModule :: Nil) - (imp, DefinedSbtValues(definitions)) - } + if parsed.definitions.isEmpty then (Nil, DefinedSbtValues.empty) + else + val defValues = + evaluateDefinitions(eval, name, parsed.imports, parsed.definitions) + val imp = + BuildUtilLite.importAllRoot(defValues.sbtFiles.map(_.enclosingModule)) + (imp, defValues) val allImports = importDefs.map(s => (s, -1)) ++ parsed.imports val dslEntries = parsed.settings map { (dslExpression, range) => evaluateDslEntry(eval, name, allImports, dslExpression, range) @@ -317,22 +317,92 @@ private[sbt] object EvaluateConfigurations { classOf[SettingKey[?]] ).map(_.getName) + /** + * Upper bound on total source characters per compiled definitions module. Together with + * `MaxDefinitionsPerChunk`, this keeps generated bytecode under JVM class and method size limits + * (see sbt/sbt#3057). This is a conservative source-size proxy: JVM method bytecode is capped at + * 65535 bytes, and many small definitions still expand to a large ``/initializer; keeping + * each chunk well under typical large-method growth avoids "class file too large" failures when + * definition count alone is still below `MaxDefinitionsPerChunk`. + */ + private val MaxDefinitionCharsPerChunk: Int = 12000 + + /** + * Upper bound on definition count per compiled module. Large .sbt files with hundreds of + * `lazy val` lines can exceed JVM limits when compiled into a single synthetic object. + */ + private val MaxDefinitionsPerChunk: Int = 100 + + /** + * Partitions top-level .sbt definitions before `Eval` compiles them. Used by + * `evaluateDefinitions` (sbt/sbt#3057) and exposed for unit tests. + */ + private[sbt] def partitionDefinitionRanges( + definitions: Seq[(String, LineRange)] + ): Seq[Seq[(String, scala.Range)]] = + chunkDefinitionRanges(definitions.map((s, r) => (s, r.start to r.end))) + + /** + * Partitions top-level .sbt definitions in source order. Later chunks are compiled with + * wildcard imports of earlier modules so each val can see definitions above it in the file. + */ + private def chunkDefinitionRanges( + definitions: Seq[(String, scala.Range)] + ): Seq[Seq[(String, scala.Range)]] = + if definitions.isEmpty then Nil + else + val out = Seq.newBuilder[Seq[(String, scala.Range)]] + var cur = Seq.newBuilder[(String, scala.Range)] + var chars = 0 + var count = 0 + def flush(): Unit = + val c = cur.result() + if c.nonEmpty then + out += c + cur = Seq.newBuilder + chars = 0 + count = 0 + for pair @ (text, _) <- definitions do + val len = text.length + val overChars = count > 0 && chars + len > MaxDefinitionCharsPerChunk + val overCount = count > 0 && count >= MaxDefinitionsPerChunk + if overChars || overCount then flush() + cur += pair + chars += len + count += 1 + flush() + out.result() + private def evaluateDefinitions( eval: Eval, name: String, imports: Seq[(String, Int)], definitions: Seq[(String, LineRange)], - file: Option[VirtualFileRef], - ): EvalDefinitions = { - val convertedRanges = definitions.map { (s, r) => (s, r.start to r.end) } - eval.evalDefinitions( - convertedRanges, - new EvalImports(imports.map(_._1)), // name - name, - // file, - extractedValTypes - ) - } + ): DefinedSbtValues = + val chunks = partitionDefinitionRanges(definitions) + if chunks.isEmpty then DefinedSbtValues.empty + else + val acc = Seq.newBuilder[EvalDefinitions] + var priorModuleNames = Seq.empty[String] + for chunk <- chunks do + val importSuffix = + if priorModuleNames.isEmpty then Nil + else + BuildUtilLite + .importAllRoot(priorModuleNames) + .map: x => + (x, -1) + val combinedImports = imports ++ importSuffix + val ed = eval.evalDefinitions( + chunk, + new EvalImports(combinedImports.map(_._1)), + name, + extractedValTypes, + extraHash = priorModuleNames.mkString("|"), + ) + acc += ed + priorModuleNames = priorModuleNames :+ ed.enclosingModule + new DefinedSbtValues(acc.result()) } object BuildUtilLite: diff --git a/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala new file mode 100644 index 000000000..ba622625b --- /dev/null +++ b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala @@ -0,0 +1,53 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + +package sbt.internal + +import sbt.internal.util.LineRange + +object EvaluateConfigurationsChunkingSpec extends verify.BasicTestSuite: + + test("partitions by definition count (sbt/sbt#3057)"): + val defs = + ((0 until 105) + .map: i => + (s"lazy val x$i = ()", LineRange(i, i))) + .toList + val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) + assert(parts.size == 2) + assert(parts.head.size == 100) + assert(parts(1).size == 5) + assert(parts.map(_.size).sum == 105) + + test("small definition lists stay in one partition"): + val defs = + ((0 until 5) + .map: i => + (s"lazy val x$i = ()", LineRange(i, i))) + .toList + val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) + assert(parts.size == 1) + assert(parts.head.size == 5) + + test("partitions when character budget is exceeded before count limit"): + val d0 = ("lazy val a = 1", LineRange(0, 0)) + val padding = " " * 13000 + val d1 = (s"lazy val b = 1$padding", LineRange(1, 1)) + val parts = EvaluateConfigurations.partitionDefinitionRanges(List(d0, d1)) + assert(parts.size == 2) + assert(parts(0).size == 1) + assert(parts(1).size == 1) + + test("a single oversized definition is not split"): + val padding = " " * 20000 + val d0 = (s"lazy val huge = 1$padding", LineRange(0, 0)) + val parts = EvaluateConfigurations.partitionDefinitionRanges(List(d0)) + assert(parts.size == 1) + assert(parts.head.size == 1) + +end EvaluateConfigurationsChunkingSpec diff --git a/sbt-app/src/sbt-test/tests/many-values/build.sbt b/sbt-app/src/sbt-test/tests/many-values/build.sbt index ff51438b9..59c413c6b 100644 --- a/sbt-app/src/sbt-test/tests/many-values/build.sbt +++ b/sbt-app/src/sbt-test/tests/many-values/build.sbt @@ -1,5 +1,5 @@ // https://github.com/sbt/sbt/issues/7768 - +// https://github.com/sbt/sbt/issues/3057 (many top-level vals exercise Eval chunking) val a1 = settingKey[Seq[Int]]("") val a2 = settingKey[Int]("") val a3 = settingKey[Int]("") @@ -23,6 +23,84 @@ val a20 = settingKey[Int]("") val a21 = settingKey[Int]("") val a22 = settingKey[Int]("") val a23 = settingKey[Int]("") +val a24 = settingKey[Int]("") +val a25 = settingKey[Int]("") +val a26 = settingKey[Int]("") +val a27 = settingKey[Int]("") +val a28 = settingKey[Int]("") +val a29 = settingKey[Int]("") +val a30 = settingKey[Int]("") +val a31 = settingKey[Int]("") +val a32 = settingKey[Int]("") +val a33 = settingKey[Int]("") +val a34 = settingKey[Int]("") +val a35 = settingKey[Int]("") +val a36 = settingKey[Int]("") +val a37 = settingKey[Int]("") +val a38 = settingKey[Int]("") +val a39 = settingKey[Int]("") +val a40 = settingKey[Int]("") +val a41 = settingKey[Int]("") +val a42 = settingKey[Int]("") +val a43 = settingKey[Int]("") +val a44 = settingKey[Int]("") +val a45 = settingKey[Int]("") +val a46 = settingKey[Int]("") +val a47 = settingKey[Int]("") +val a48 = settingKey[Int]("") +val a49 = settingKey[Int]("") +val a50 = settingKey[Int]("") +val a51 = settingKey[Int]("") +val a52 = settingKey[Int]("") +val a53 = settingKey[Int]("") +val a54 = settingKey[Int]("") +val a55 = settingKey[Int]("") +val a56 = settingKey[Int]("") +val a57 = settingKey[Int]("") +val a58 = settingKey[Int]("") +val a59 = settingKey[Int]("") +val a60 = settingKey[Int]("") +val a61 = settingKey[Int]("") +val a62 = settingKey[Int]("") +val a63 = settingKey[Int]("") +val a64 = settingKey[Int]("") +val a65 = settingKey[Int]("") +val a66 = settingKey[Int]("") +val a67 = settingKey[Int]("") +val a68 = settingKey[Int]("") +val a69 = settingKey[Int]("") +val a70 = settingKey[Int]("") +val a71 = settingKey[Int]("") +val a72 = settingKey[Int]("") +val a73 = settingKey[Int]("") +val a74 = settingKey[Int]("") +val a75 = settingKey[Int]("") +val a76 = settingKey[Int]("") +val a77 = settingKey[Int]("") +val a78 = settingKey[Int]("") +val a79 = settingKey[Int]("") +val a80 = settingKey[Int]("") +val a81 = settingKey[Int]("") +val a82 = settingKey[Int]("") +val a83 = settingKey[Int]("") +val a84 = settingKey[Int]("") +val a85 = settingKey[Int]("") +val a86 = settingKey[Int]("") +val a87 = settingKey[Int]("") +val a88 = settingKey[Int]("") +val a89 = settingKey[Int]("") +val a90 = settingKey[Int]("") +val a91 = settingKey[Int]("") +val a92 = settingKey[Int]("") +val a93 = settingKey[Int]("") +val a94 = settingKey[Int]("") +val a95 = settingKey[Int]("") +val a96 = settingKey[Int]("") +val a97 = settingKey[Int]("") +val a98 = settingKey[Int]("") +val a99 = settingKey[Int]("") +val a100 = settingKey[Int]("") +val a101 = settingKey[Int]("") a1 := List(1) a2 := 2 @@ -47,34 +125,88 @@ a20 := 20 a21 := 21 a22 := 22 a23 := 23 +a24 := 24 +a25 := 25 +a26 := 26 +a27 := 27 +a28 := 28 +a29 := 29 +a30 := 30 +a31 := 31 +a32 := 32 +a33 := 33 +a34 := 34 +a35 := 35 +a36 := 36 +a37 := 37 +a38 := 38 +a39 := 39 +a40 := 40 +a41 := 41 +a42 := 42 +a43 := 43 +a44 := 44 +a45 := 45 +a46 := 46 +a47 := 47 +a48 := 48 +a49 := 49 +a50 := 50 +a51 := 51 +a52 := 52 +a53 := 53 +a54 := 54 +a55 := 55 +a56 := 56 +a57 := 57 +a58 := 58 +a59 := 59 +a60 := 60 +a61 := 61 +a62 := 62 +a63 := 63 +a64 := 64 +a65 := 65 +a66 := 66 +a67 := 67 +a68 := 68 +a69 := 69 +a70 := 70 +a71 := 71 +a72 := 72 +a73 := 73 +a74 := 74 +a75 := 75 +a76 := 76 +a77 := 77 +a78 := 78 +a79 := 79 +a80 := 80 +a81 := 81 +a82 := 82 +a83 := 83 +a84 := 84 +a85 := 85 +a86 := 86 +a87 := 87 +a88 := 88 +a89 := 89 +a90 := 90 +a91 := 91 +a92 := 92 +a93 := 93 +a94 := 94 +a95 := 95 +a96 := 96 +a97 := 97 +a98 := 98 +a99 := 99 +a100 := 100 +a101 := 101 TaskKey[Unit]("check") := Def.uncached { - val sum = ( - a1.value ++ List( - a2.value, - a3.value, - a4.value, - a5.value, - a6.value, - a7.value, - a8.value, - a9.value, - a10.value, - a11.value, - a12.value, - a13.value, - a14.value, - a15.value, - a16.value, - a17.value, - a18.value, - a19.value, - a20.value, - a21.value, - a22.value, - a23.value, - ) - ).sum - assert(sum == 276, sum) + assert(a1.value == List(1), a1.value) + assert(a100.value == 100, a100.value) + assert(a101.value == 101, a101.value) () }