From a1a546456b6dc81aaa58243e78f1067c645fa1d6 Mon Sep 17 00:00:00 2001 From: bitloi Date: Fri, 10 Apr 2026 06:46:20 +0200 Subject: [PATCH 1/2] Split large .sbt definitions across Eval modules Fixes JVM class/method size failures when many top-level vals are compiled into one synthetic object (sbt/sbt#3057). Later chunks import prior modules so definitions can still see vals above them in the file. Partitioning is bounded by definition count and source character budget. --- .../sbt/internal/EvaluateConfigurations.scala | 90 ++++++++++++++++--- .../EvaluateConfigurationsChunkingSpec.scala | 41 +++++++++ 2 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala diff --git a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala index 668901dcc..ad19b51ec 100644 --- a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala +++ b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala @@ -151,10 +151,11 @@ private[sbt] object EvaluateConfigurations { val (importDefs, definitions) = if (parsed.definitions.isEmpty) (Nil, DefinedSbtValues.empty) else { - val definitions = + val defValues = evaluateDefinitions(eval, name, parsed.imports, parsed.definitions, Some(file)) - val imp = BuildUtilLite.importAllRoot(definitions.enclosingModule :: Nil) - (imp, DefinedSbtValues(definitions)) + val imp = + BuildUtilLite.importAllRoot(defValues.sbtFiles.map(_.enclosingModule)) + (imp, defValues) } val allImports = importDefs.map(s => (s, -1)) ++ parsed.imports val dslEntries = parsed.settings map { (dslExpression, range) => @@ -317,21 +318,86 @@ private[sbt] object EvaluateConfigurations { classOf[SettingKey[?]] ).map(_.getName) + /** + * Upper bound on total source characters per compiled definitions module. Together with + * `MaxDefinitionsPerChunk`, this keeps generated bytecode under JVM class and method size limits + * (see sbt/sbt#3057). + */ + private val MaxDefinitionCharsPerChunk: Int = 12000 + + /** + * Upper bound on definition count per compiled module. Large .sbt files with hundreds of + * `lazy val` lines can exceed JVM limits when compiled into a single synthetic object. + */ + private val MaxDefinitionsPerChunk: Int = 100 + + /** + * Partitions top-level .sbt definitions before `Eval` compiles them. Used by + * `evaluateDefinitions` (sbt/sbt#3057) and exposed for unit tests. + */ + private[sbt] def partitionDefinitionRanges( + definitions: Seq[(String, LineRange)] + ): Seq[Seq[(String, scala.Range)]] = + chunkDefinitionRanges(definitions.map { (s, r) => (s, r.start to r.end) }) + + /** + * Partitions top-level .sbt definitions in source order. Later chunks are compiled with + * wildcard imports of earlier modules so each val can see definitions above it in the file. + */ + private def chunkDefinitionRanges( + definitions: Seq[(String, scala.Range)] + ): Seq[Seq[(String, scala.Range)]] = + if definitions.isEmpty then Nil + else + val out = Seq.newBuilder[Seq[(String, scala.Range)]] + var cur = Seq.newBuilder[(String, scala.Range)] + var chars = 0 + var count = 0 + def flush(): Unit = + val c = cur.result() + if c.nonEmpty then + out += c + cur = Seq.newBuilder + chars = 0 + count = 0 + for pair @ (text, _) <- definitions do + val len = text.length + val overChars = count > 0 && chars + len > MaxDefinitionCharsPerChunk + val overCount = count > 0 && count >= MaxDefinitionsPerChunk + if overChars || overCount then flush() + cur += pair + chars += len + count += 1 + flush() + out.result() + private def evaluateDefinitions( eval: Eval, name: String, imports: Seq[(String, Int)], definitions: Seq[(String, LineRange)], file: Option[VirtualFileRef], - ): EvalDefinitions = { - val convertedRanges = definitions.map { (s, r) => (s, r.start to r.end) } - eval.evalDefinitions( - convertedRanges, - new EvalImports(imports.map(_._1)), // name - name, - // file, - extractedValTypes - ) + ): DefinedSbtValues = { + val chunks = partitionDefinitionRanges(definitions) + if chunks.isEmpty then DefinedSbtValues.empty + else + val acc = Seq.newBuilder[EvalDefinitions] + var priorModuleNames = Seq.empty[String] + for chunk <- chunks do + val importSuffix = + if priorModuleNames.isEmpty then Nil + else BuildUtilLite.importAllRoot(priorModuleNames).map(s => (s, -1)) + val combinedImports = imports ++ importSuffix + val ed = eval.evalDefinitions( + chunk, + new EvalImports(combinedImports.map(_._1)), + name, + extractedValTypes, + extraHash = priorModuleNames.mkString("|"), + ) + acc += ed + priorModuleNames = priorModuleNames :+ ed.enclosingModule + new DefinedSbtValues(acc.result()) } } diff --git a/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala new file mode 100644 index 000000000..5b22d9d9f --- /dev/null +++ b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala @@ -0,0 +1,41 @@ +package sbt.internal + +import sbt.internal.util.LineRange + +object EvaluateConfigurationsChunkingSpec extends verify.BasicTestSuite: + + test("partitions by definition count (sbt/sbt#3057)") { + val defs = (0 until 105).map(i => (s"lazy val x$i = ()", LineRange(i, i))).toList + val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) + assert(parts.size == 2) + assert(parts.head.size == 100) + assert(parts(1).size == 5) + assert(parts.map(_.size).sum == 105) + } + + test("small definition lists stay in one partition") { + val defs = (0 until 5).map(i => (s"lazy val x$i = ()", LineRange(i, i))).toList + val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) + assert(parts.size == 1) + assert(parts.head.size == 5) + } + + test("partitions when character budget is exceeded before count limit") { + val d0 = ("lazy val a = 1", LineRange(0, 0)) + val padding = " " * 13000 + val d1 = (s"lazy val b = 1$padding", LineRange(1, 1)) + val parts = EvaluateConfigurations.partitionDefinitionRanges(List(d0, d1)) + assert(parts.size == 2) + assert(parts(0).size == 1) + assert(parts(1).size == 1) + } + + test("a single oversized definition is not split") { + val padding = " " * 20000 + val d0 = (s"lazy val huge = 1$padding", LineRange(0, 0)) + val parts = EvaluateConfigurations.partitionDefinitionRanges(List(d0)) + assert(parts.size == 1) + assert(parts.head.size == 1) + } + +end EvaluateConfigurationsChunkingSpec From 463c006699be2d7c2f8305b42b461ff110394234 Mon Sep 17 00:00:00 2001 From: bitloi Date: Fri, 10 Apr 2026 07:15:08 +0200 Subject: [PATCH 2/2] [2.x] fix: Harden large .sbt definition eval (chunking) **Problem** Very large build.sbt files compiled all top-level vals into one synthetic object, which could exceed JVM class and method size limits (sbt/sbt#3057). Review feedback also asked for integration coverage, license header, and clearer documentation of heuristics. **Solution** Keep definition chunking; extend scripted tests/many-values to 101 top-level vals so the second chunk is exercised end-to-end. Add the standard Apache file header to EvaluateConfigurationsChunkingSpec. Expand the MaxDefinitionCharsPerChunk comment to relate source size to JVM limits. Drop the unused file parameter from evaluateDefinitions. Generated-by: Cursor agent (human reviewed) --- .../sbt/internal/EvaluateConfigurations.scala | 24 ++- .../EvaluateConfigurationsChunkingSpec.scala | 32 ++- .../src/sbt-test/tests/many-values/build.sbt | 188 +++++++++++++++--- 3 files changed, 196 insertions(+), 48 deletions(-) diff --git a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala index ad19b51ec..b6eba2765 100644 --- a/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala +++ b/buildfile/src/main/scala/sbt/internal/EvaluateConfigurations.scala @@ -149,14 +149,13 @@ private[sbt] object EvaluateConfigurations { case file => file.id val parsed = parseConfiguration(file, lines, imports, offset) val (importDefs, definitions) = - if (parsed.definitions.isEmpty) (Nil, DefinedSbtValues.empty) - else { + if parsed.definitions.isEmpty then (Nil, DefinedSbtValues.empty) + else val defValues = - evaluateDefinitions(eval, name, parsed.imports, parsed.definitions, Some(file)) + evaluateDefinitions(eval, name, parsed.imports, parsed.definitions) val imp = BuildUtilLite.importAllRoot(defValues.sbtFiles.map(_.enclosingModule)) (imp, defValues) - } val allImports = importDefs.map(s => (s, -1)) ++ parsed.imports val dslEntries = parsed.settings map { (dslExpression, range) => evaluateDslEntry(eval, name, allImports, dslExpression, range) @@ -321,7 +320,10 @@ private[sbt] object EvaluateConfigurations { /** * Upper bound on total source characters per compiled definitions module. Together with * `MaxDefinitionsPerChunk`, this keeps generated bytecode under JVM class and method size limits - * (see sbt/sbt#3057). + * (see sbt/sbt#3057). This is a conservative source-size proxy: JVM method bytecode is capped at + * 65535 bytes, and many small definitions still expand to a large ``/initializer; keeping + * each chunk well under typical large-method growth avoids "class file too large" failures when + * definition count alone is still below `MaxDefinitionsPerChunk`. */ private val MaxDefinitionCharsPerChunk: Int = 12000 @@ -338,7 +340,7 @@ private[sbt] object EvaluateConfigurations { private[sbt] def partitionDefinitionRanges( definitions: Seq[(String, LineRange)] ): Seq[Seq[(String, scala.Range)]] = - chunkDefinitionRanges(definitions.map { (s, r) => (s, r.start to r.end) }) + chunkDefinitionRanges(definitions.map((s, r) => (s, r.start to r.end))) /** * Partitions top-level .sbt definitions in source order. Later chunks are compiled with @@ -376,8 +378,7 @@ private[sbt] object EvaluateConfigurations { name: String, imports: Seq[(String, Int)], definitions: Seq[(String, LineRange)], - file: Option[VirtualFileRef], - ): DefinedSbtValues = { + ): DefinedSbtValues = val chunks = partitionDefinitionRanges(definitions) if chunks.isEmpty then DefinedSbtValues.empty else @@ -386,7 +387,11 @@ private[sbt] object EvaluateConfigurations { for chunk <- chunks do val importSuffix = if priorModuleNames.isEmpty then Nil - else BuildUtilLite.importAllRoot(priorModuleNames).map(s => (s, -1)) + else + BuildUtilLite + .importAllRoot(priorModuleNames) + .map: x => + (x, -1) val combinedImports = imports ++ importSuffix val ed = eval.evalDefinitions( chunk, @@ -398,7 +403,6 @@ private[sbt] object EvaluateConfigurations { acc += ed priorModuleNames = priorModuleNames :+ ed.enclosingModule new DefinedSbtValues(acc.result()) - } } object BuildUtilLite: diff --git a/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala index 5b22d9d9f..ba622625b 100644 --- a/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala +++ b/buildfile/src/test/scala/sbt/internal/EvaluateConfigurationsChunkingSpec.scala @@ -1,26 +1,40 @@ +/* + * sbt + * Copyright 2023, Scala center + * Copyright 2011 - 2022, Lightbend, Inc. + * Copyright 2008 - 2010, Mark Harrah + * Licensed under Apache License 2.0 (see LICENSE) + */ + package sbt.internal import sbt.internal.util.LineRange object EvaluateConfigurationsChunkingSpec extends verify.BasicTestSuite: - test("partitions by definition count (sbt/sbt#3057)") { - val defs = (0 until 105).map(i => (s"lazy val x$i = ()", LineRange(i, i))).toList + test("partitions by definition count (sbt/sbt#3057)"): + val defs = + ((0 until 105) + .map: i => + (s"lazy val x$i = ()", LineRange(i, i))) + .toList val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) assert(parts.size == 2) assert(parts.head.size == 100) assert(parts(1).size == 5) assert(parts.map(_.size).sum == 105) - } - test("small definition lists stay in one partition") { - val defs = (0 until 5).map(i => (s"lazy val x$i = ()", LineRange(i, i))).toList + test("small definition lists stay in one partition"): + val defs = + ((0 until 5) + .map: i => + (s"lazy val x$i = ()", LineRange(i, i))) + .toList val parts = EvaluateConfigurations.partitionDefinitionRanges(defs) assert(parts.size == 1) assert(parts.head.size == 5) - } - test("partitions when character budget is exceeded before count limit") { + test("partitions when character budget is exceeded before count limit"): val d0 = ("lazy val a = 1", LineRange(0, 0)) val padding = " " * 13000 val d1 = (s"lazy val b = 1$padding", LineRange(1, 1)) @@ -28,14 +42,12 @@ object EvaluateConfigurationsChunkingSpec extends verify.BasicTestSuite: assert(parts.size == 2) assert(parts(0).size == 1) assert(parts(1).size == 1) - } - test("a single oversized definition is not split") { + test("a single oversized definition is not split"): val padding = " " * 20000 val d0 = (s"lazy val huge = 1$padding", LineRange(0, 0)) val parts = EvaluateConfigurations.partitionDefinitionRanges(List(d0)) assert(parts.size == 1) assert(parts.head.size == 1) - } end EvaluateConfigurationsChunkingSpec diff --git a/sbt-app/src/sbt-test/tests/many-values/build.sbt b/sbt-app/src/sbt-test/tests/many-values/build.sbt index ff51438b9..59c413c6b 100644 --- a/sbt-app/src/sbt-test/tests/many-values/build.sbt +++ b/sbt-app/src/sbt-test/tests/many-values/build.sbt @@ -1,5 +1,5 @@ // https://github.com/sbt/sbt/issues/7768 - +// https://github.com/sbt/sbt/issues/3057 (many top-level vals exercise Eval chunking) val a1 = settingKey[Seq[Int]]("") val a2 = settingKey[Int]("") val a3 = settingKey[Int]("") @@ -23,6 +23,84 @@ val a20 = settingKey[Int]("") val a21 = settingKey[Int]("") val a22 = settingKey[Int]("") val a23 = settingKey[Int]("") +val a24 = settingKey[Int]("") +val a25 = settingKey[Int]("") +val a26 = settingKey[Int]("") +val a27 = settingKey[Int]("") +val a28 = settingKey[Int]("") +val a29 = settingKey[Int]("") +val a30 = settingKey[Int]("") +val a31 = settingKey[Int]("") +val a32 = settingKey[Int]("") +val a33 = settingKey[Int]("") +val a34 = settingKey[Int]("") +val a35 = settingKey[Int]("") +val a36 = settingKey[Int]("") +val a37 = settingKey[Int]("") +val a38 = settingKey[Int]("") +val a39 = settingKey[Int]("") +val a40 = settingKey[Int]("") +val a41 = settingKey[Int]("") +val a42 = settingKey[Int]("") +val a43 = settingKey[Int]("") +val a44 = settingKey[Int]("") +val a45 = settingKey[Int]("") +val a46 = settingKey[Int]("") +val a47 = settingKey[Int]("") +val a48 = settingKey[Int]("") +val a49 = settingKey[Int]("") +val a50 = settingKey[Int]("") +val a51 = settingKey[Int]("") +val a52 = settingKey[Int]("") +val a53 = settingKey[Int]("") +val a54 = settingKey[Int]("") +val a55 = settingKey[Int]("") +val a56 = settingKey[Int]("") +val a57 = settingKey[Int]("") +val a58 = settingKey[Int]("") +val a59 = settingKey[Int]("") +val a60 = settingKey[Int]("") +val a61 = settingKey[Int]("") +val a62 = settingKey[Int]("") +val a63 = settingKey[Int]("") +val a64 = settingKey[Int]("") +val a65 = settingKey[Int]("") +val a66 = settingKey[Int]("") +val a67 = settingKey[Int]("") +val a68 = settingKey[Int]("") +val a69 = settingKey[Int]("") +val a70 = settingKey[Int]("") +val a71 = settingKey[Int]("") +val a72 = settingKey[Int]("") +val a73 = settingKey[Int]("") +val a74 = settingKey[Int]("") +val a75 = settingKey[Int]("") +val a76 = settingKey[Int]("") +val a77 = settingKey[Int]("") +val a78 = settingKey[Int]("") +val a79 = settingKey[Int]("") +val a80 = settingKey[Int]("") +val a81 = settingKey[Int]("") +val a82 = settingKey[Int]("") +val a83 = settingKey[Int]("") +val a84 = settingKey[Int]("") +val a85 = settingKey[Int]("") +val a86 = settingKey[Int]("") +val a87 = settingKey[Int]("") +val a88 = settingKey[Int]("") +val a89 = settingKey[Int]("") +val a90 = settingKey[Int]("") +val a91 = settingKey[Int]("") +val a92 = settingKey[Int]("") +val a93 = settingKey[Int]("") +val a94 = settingKey[Int]("") +val a95 = settingKey[Int]("") +val a96 = settingKey[Int]("") +val a97 = settingKey[Int]("") +val a98 = settingKey[Int]("") +val a99 = settingKey[Int]("") +val a100 = settingKey[Int]("") +val a101 = settingKey[Int]("") a1 := List(1) a2 := 2 @@ -47,34 +125,88 @@ a20 := 20 a21 := 21 a22 := 22 a23 := 23 +a24 := 24 +a25 := 25 +a26 := 26 +a27 := 27 +a28 := 28 +a29 := 29 +a30 := 30 +a31 := 31 +a32 := 32 +a33 := 33 +a34 := 34 +a35 := 35 +a36 := 36 +a37 := 37 +a38 := 38 +a39 := 39 +a40 := 40 +a41 := 41 +a42 := 42 +a43 := 43 +a44 := 44 +a45 := 45 +a46 := 46 +a47 := 47 +a48 := 48 +a49 := 49 +a50 := 50 +a51 := 51 +a52 := 52 +a53 := 53 +a54 := 54 +a55 := 55 +a56 := 56 +a57 := 57 +a58 := 58 +a59 := 59 +a60 := 60 +a61 := 61 +a62 := 62 +a63 := 63 +a64 := 64 +a65 := 65 +a66 := 66 +a67 := 67 +a68 := 68 +a69 := 69 +a70 := 70 +a71 := 71 +a72 := 72 +a73 := 73 +a74 := 74 +a75 := 75 +a76 := 76 +a77 := 77 +a78 := 78 +a79 := 79 +a80 := 80 +a81 := 81 +a82 := 82 +a83 := 83 +a84 := 84 +a85 := 85 +a86 := 86 +a87 := 87 +a88 := 88 +a89 := 89 +a90 := 90 +a91 := 91 +a92 := 92 +a93 := 93 +a94 := 94 +a95 := 95 +a96 := 96 +a97 := 97 +a98 := 98 +a99 := 99 +a100 := 100 +a101 := 101 TaskKey[Unit]("check") := Def.uncached { - val sum = ( - a1.value ++ List( - a2.value, - a3.value, - a4.value, - a5.value, - a6.value, - a7.value, - a8.value, - a9.value, - a10.value, - a11.value, - a12.value, - a13.value, - a14.value, - a15.value, - a16.value, - a17.value, - a18.value, - a19.value, - a20.value, - a21.value, - a22.value, - a23.value, - ) - ).sum - assert(sum == 276, sum) + assert(a1.value == List(1), a1.value) + assert(a100.value == 100, a100.value) + assert(a101.value == 101, a101.value) () }