Add parallelism to KeyIndex.aggregate

I looked for serial bottlenecks in sbt project loading and discovered
that KeyIndex.aggregate was relatively easily parallelizable. Before
this time, it took about 1 second to run KeyIndex.aggregate in the akka
project on my computer. After this change, it took 250ms. Given that I
have 4 logical cores, the speedup is roughly linear.
This commit is contained in:
Ethan Atkins 2019-08-21 21:16:39 -07:00
parent d6478c9def
commit 3fc8817974
1 changed files with 19 additions and 3 deletions

View File

@ -28,10 +28,26 @@ object KeyIndex {
extra: BuildUtil[_],
projects: Map[URI, Set[String]],
configurations: Map[String, Seq[Configuration]]
): ExtendableKeyIndex =
(base(projects, configurations) /: known) { (index, key) =>
index.addAggregated(key, extra)
): ExtendableKeyIndex = {
/*
* Used to be:
* (base(projects, configurations) /: known) { (index, key) =>
* index.addAggregated(key, extra)
* }
* This was a significant serial bottleneck during project loading that we can work around by
* computing the aggregations in parallel and then bulk adding them to the index.
*/
val toAggregate = known.par.map {
case key if validID(key.key.label) =>
Aggregation.aggregate(key, ScopeMask(), extra, reverse = true)
case _ => Nil
}
(base(projects, configurations) /: toAggregate) {
case (index, Nil) => index
case (index, keys) => (index /: keys)(_ add _)
}
}
private[this] def base(
projects: Map[URI, Set[String]],
configurations: Map[String, Seq[Configuration]]