Add parallelism to KeyIndex.aggregate

I looked for serial bottlenecks in sbt project loading and discovered
that KeyIndex.aggregate was relatively easily parallelizable. Before
this time, it took about 1 second to run KeyIndex.aggregate in the akka
project on my computer. After this change, it took 250ms. Given that I
have 4 logical cores, the speedup is roughly linear.
This commit is contained in:
Ethan Atkins 2019-08-21 21:16:39 -07:00
parent d6478c9def
commit 3fc8817974
1 changed files with 19 additions and 3 deletions

View File

@ -28,10 +28,26 @@ object KeyIndex {
extra: BuildUtil[_], extra: BuildUtil[_],
projects: Map[URI, Set[String]], projects: Map[URI, Set[String]],
configurations: Map[String, Seq[Configuration]] configurations: Map[String, Seq[Configuration]]
): ExtendableKeyIndex = ): ExtendableKeyIndex = {
(base(projects, configurations) /: known) { (index, key) => /*
index.addAggregated(key, extra) * Used to be:
* (base(projects, configurations) /: known) { (index, key) =>
* index.addAggregated(key, extra)
* }
* This was a significant serial bottleneck during project loading that we can work around by
* computing the aggregations in parallel and then bulk adding them to the index.
*/
val toAggregate = known.par.map {
case key if validID(key.key.label) =>
Aggregation.aggregate(key, ScopeMask(), extra, reverse = true)
case _ => Nil
} }
(base(projects, configurations) /: toAggregate) {
case (index, Nil) => index
case (index, keys) => (index /: keys)(_ add _)
}
}
private[this] def base( private[this] def base(
projects: Map[URI, Set[String]], projects: Map[URI, Set[String]],
configurations: Map[String, Seq[Configuration]] configurations: Map[String, Seq[Configuration]]