mirror of https://github.com/sbt/sbt.git
Add parallelism to KeyIndex.aggregate
I looked for serial bottlenecks in sbt project loading and discovered that KeyIndex.aggregate was relatively easily parallelizable. Before this time, it took about 1 second to run KeyIndex.aggregate in the akka project on my computer. After this change, it took 250ms. Given that I have 4 logical cores, the speedup is roughly linear.
This commit is contained in:
parent
d6478c9def
commit
3fc8817974
|
|
@ -28,10 +28,26 @@ object KeyIndex {
|
||||||
extra: BuildUtil[_],
|
extra: BuildUtil[_],
|
||||||
projects: Map[URI, Set[String]],
|
projects: Map[URI, Set[String]],
|
||||||
configurations: Map[String, Seq[Configuration]]
|
configurations: Map[String, Seq[Configuration]]
|
||||||
): ExtendableKeyIndex =
|
): ExtendableKeyIndex = {
|
||||||
(base(projects, configurations) /: known) { (index, key) =>
|
/*
|
||||||
index.addAggregated(key, extra)
|
* Used to be:
|
||||||
|
* (base(projects, configurations) /: known) { (index, key) =>
|
||||||
|
* index.addAggregated(key, extra)
|
||||||
|
* }
|
||||||
|
* This was a significant serial bottleneck during project loading that we can work around by
|
||||||
|
* computing the aggregations in parallel and then bulk adding them to the index.
|
||||||
|
*/
|
||||||
|
val toAggregate = known.par.map {
|
||||||
|
case key if validID(key.key.label) =>
|
||||||
|
Aggregation.aggregate(key, ScopeMask(), extra, reverse = true)
|
||||||
|
case _ => Nil
|
||||||
}
|
}
|
||||||
|
(base(projects, configurations) /: toAggregate) {
|
||||||
|
case (index, Nil) => index
|
||||||
|
case (index, keys) => (index /: keys)(_ add _)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private[this] def base(
|
private[this] def base(
|
||||||
projects: Map[URI, Set[String]],
|
projects: Map[URI, Set[String]],
|
||||||
configurations: Map[String, Seq[Configuration]]
|
configurations: Map[String, Seq[Configuration]]
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue