From f523b9da0d2673b9117c986d1cbd1a4df173d1fe Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 01:11:47 -0400 Subject: [PATCH 1/9] Add perf logs --- main/src/main/scala/sbt/Plugins.scala | 61 +++++---- main/src/main/scala/sbt/internal/Load.scala | 144 ++++++++++++-------- 2 files changed, 123 insertions(+), 82 deletions(-) diff --git a/main/src/main/scala/sbt/Plugins.scala b/main/src/main/scala/sbt/Plugins.scala index 90a6c8f66..8177aa0ab 100644 --- a/main/src/main/scala/sbt/Plugins.scala +++ b/main/src/main/scala/sbt/Plugins.scala @@ -175,36 +175,38 @@ object Plugins extends PluginsFunctions { // Note: Here is where the function begins. We're given a list of plugins now. (requestedPlugins, log) => { - def explicitlyDisabled(p: AutoPlugin): Boolean = hasExclude(requestedPlugins, p) - val alwaysEnabled: List[AutoPlugin] = defined.filter(_.isAlwaysEnabled).filterNot(explicitlyDisabled) - val knowlege0: Set[Atom] = ((flatten(requestedPlugins) ++ alwaysEnabled) collect { - case x: AutoPlugin => Atom(x.label) - }).toSet - val clauses = Clauses((allRequirementsClause ::: allEnabledByClause) filterNot { _.head subsetOf knowlege0 }) - log.debug(s"deducing auto plugins based on known facts ${knowlege0.toString} and clauses ${clauses.toString}") - Logic.reduce(clauses, (flattenConvert(requestedPlugins) ++ convertAll(alwaysEnabled)).toSet) match { - case Left(problem) => throw AutoPluginException(problem) - case Right(results) => - log.debug(s" :: deduced result: ${results}") - val selectedAtoms: List[Atom] = results.ordered - val selectedPlugins = selectedAtoms map { a => - byAtomMap.getOrElse(a, throw AutoPluginException(s"${a} was not found in atom map.")) - } - val forbidden: Set[AutoPlugin] = (selectedPlugins flatMap { Plugins.asExclusions }).toSet - val c = selectedPlugins.toSet & forbidden - if (c.nonEmpty) { - exlusionConflictError(requestedPlugins, selectedPlugins, c.toSeq sortBy { _.label }) - } - val retval = topologicalSort(selectedPlugins, log) - log.debug(s" :: sorted deduced result: ${retval.toString}") - retval + timed("Plugins.deducer#function", log) { + def explicitlyDisabled(p: AutoPlugin): Boolean = hasExclude(requestedPlugins, p) + val alwaysEnabled: List[AutoPlugin] = defined.filter(_.isAlwaysEnabled).filterNot(explicitlyDisabled) + val knowlege0: Set[Atom] = ((flatten(requestedPlugins) ++ alwaysEnabled) collect { + case x: AutoPlugin => Atom(x.label) + }).toSet + val clauses = Clauses((allRequirementsClause ::: allEnabledByClause) filterNot { _.head subsetOf knowlege0 }) + log.debug(s"deducing auto plugins based on known facts ${knowlege0.toString} and clauses ${clauses.toString}") + Logic.reduce(clauses, (flattenConvert(requestedPlugins) ++ convertAll(alwaysEnabled)).toSet) match { + case Left(problem) => throw AutoPluginException(problem) + case Right(results) => + log.debug(s" :: deduced result: ${results}") + val selectedAtoms: List[Atom] = results.ordered + val selectedPlugins = selectedAtoms map { a => + byAtomMap.getOrElse(a, throw AutoPluginException(s"${a} was not found in atom map.")) + } + val forbidden: Set[AutoPlugin] = (selectedPlugins flatMap { Plugins.asExclusions }).toSet + val c = selectedPlugins.toSet & forbidden + if (c.nonEmpty) { + exlusionConflictError(requestedPlugins, selectedPlugins, c.toSeq sortBy { _.label }) + } + val retval = topologicalSort(selectedPlugins, log) + // log.debug(s" :: sorted deduced result: ${retval.toString}") + retval + } } } } private[sbt] def topologicalSort(ns: List[AutoPlugin], log: Logger): List[AutoPlugin] = { - log.debug(s"sorting: ns: ${ns.toString}") + // log.debug(s"sorting: ns: ${ns.toString}") @tailrec def doSort(found0: List[AutoPlugin], notFound0: List[AutoPlugin], limit0: Int): List[AutoPlugin] = { - log.debug(s" :: sorting:: found: ${found0.toString} not found ${notFound0.toString}") + // log.debug(s" :: sorting:: found: ${found0.toString} not found ${notFound0.toString}") if (limit0 < 0) throw AutoPluginException(s"Failed to sort ${ns} topologically") else if (notFound0.isEmpty) found0 else { @@ -360,4 +362,13 @@ ${listConflicts(conflicting)}""") } hasGetterOpt getOrElse false } + + /** Debugging method to time how long it takes to run various compilation tasks. */ + private[this] def timed[T](label: String, log: Logger)(t: => T): T = { + val start = System.nanoTime + val result = t + val elapsed = System.nanoTime - start + log.debug(label + " took " + (elapsed / 1e6) + " ms") + result + } } diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 4c77a5731..b2b70f1c1 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -50,14 +50,18 @@ private[sbt] object Load { // note that there is State passed in but not pulled out def defaultLoad(state: State, baseDirectory: File, log: Logger, isPlugin: Boolean = false, topLevelExtras: List[URI] = Nil): (() => Eval, BuildStructure) = { - val globalBase = getGlobalBase(state) - val base = baseDirectory.getCanonicalFile - val rawConfig = defaultPreGlobal(state, base, globalBase, log) - val config0 = defaultWithGlobal(state, base, rawConfig, globalBase, log) - val config = if (isPlugin) enableSbtPlugin(config0) else config0.copy(extraBuilds = topLevelExtras) + val (base, config) = timed("Load.defaultLoad until apply", log) { + val globalBase = getGlobalBase(state) + val base = baseDirectory.getCanonicalFile + val rawConfig = defaultPreGlobal(state, base, globalBase, log) + val config0 = defaultWithGlobal(state, base, rawConfig, globalBase, log) + val config = if (isPlugin) enableSbtPlugin(config0) else config0.copy(extraBuilds = topLevelExtras) + (base, config) + } val result = apply(base, state, config) result } + def defaultPreGlobal(state: State, baseDirectory: File, globalBase: File, log: Logger): LoadBuildConfiguration = { val provider = state.configuration.provider @@ -158,16 +162,27 @@ private[sbt] object Load { // 8) Evaluate settings def apply(rootBase: File, s: State, config: LoadBuildConfiguration): (() => Eval, BuildStructure) = { + val log = config.log + // load, which includes some resolution, but can't fill in project IDs yet, so follow with full resolution - val loaded = resolveProjects(load(rootBase, s, config)) + val partBuild = timed("Load.apply: load", log) { load(rootBase, s, config) } + val loaded = timed("Load.apply: resolveProjects", log) { + resolveProjects(partBuild) + } val projects = loaded.units lazy val rootEval = lazyEval(loaded.units(loaded.root).unit) - val settings = finalTransforms(buildConfigurations(loaded, getRootProject(projects), config.injectSettings)) - val delegates = config.delegates(loaded) - val data = Def.make(settings)(delegates, config.scopeLocal, Project.showLoadingKey(loaded)) + val settings = timed("Load.apply: finalTransforms", log) { + finalTransforms(buildConfigurations(loaded, getRootProject(projects), config.injectSettings)) + } + val delegates = timed("Load.apply: config.delegates", log) { config.delegates(loaded) } + val data = timed("Load.apply: Def.make(settings)...", log) { + Def.make(settings)(delegates, config.scopeLocal, Project.showLoadingKey(loaded)) + } Project.checkTargets(data) foreach sys.error - val index = structureIndex(data, settings, loaded.extra(data), projects) - val streams = mkStreams(projects, loaded.root, data) + val index = timed("Load.apply: structureIndex", log) { + structureIndex(data, settings, loaded.extra(data), projects) + } + val streams = timed("Load.apply: mkStreams", log) { mkStreams(projects, loaded.root, data) } (rootEval, new BuildStructure(projects, loaded.root, settings, data, index, streams, delegates, config.scopeLocal)) } @@ -298,13 +313,15 @@ private[sbt] object Load { { val fail = (uri: URI) => sys.error("Invalid build URI (no handler available): " + uri) val resolver = (info: BuildLoader.ResolveInfo) => RetrieveUnit(info) - val build = (info: BuildLoader.BuildInfo) => Some(() => loadUnit(info.uri, info.base, info.state, info.config)) + val build = (info: BuildLoader.BuildInfo) => Some(() => + loadUnit(info.uri, info.base, info.state, info.config)) val components = BuildLoader.components(resolver, build, full = BuildLoader.componentLoader) BuildLoader(components, fail, s, config) } def load(file: File, loaders: BuildLoader, extra: List[URI]): PartBuild = loadURI(IO.directoryURI(file), loaders, extra) def loadURI(uri: URI, loaders: BuildLoader, extra: List[URI]): PartBuild = { + val log = loaders.config.log IO.assertAbsolute(uri) val (referenced, map, newLoaders) = loadAll(uri :: extra, Map.empty, loaders, Map.empty) checkAll(referenced, map) @@ -445,20 +462,23 @@ private[sbt] object Load { def noProject(uri: URI, id: String) = sys.error(s"No project '$id' defined in '$uri'.") def noConfiguration(uri: URI, id: String, conf: String) = sys.error(s"No configuration '$conf' defined in project '$id' in '$uri'") + // Called from builtinLoader def loadUnit(uri: URI, localBase: File, s: State, config: LoadBuildConfiguration): BuildUnit = - { + timed(s"Load.loadUnit($uri, ...)", config.log) { + val log = config.log val normBase = localBase.getCanonicalFile val defDir = projectStandard(normBase) - val plugs = plugins(defDir, s, config.copy(pluginManagement = config.pluginManagement.forPlugin)) - val defsScala = plugs.detected.builds.values - val buildLevelExtraProjects = plugs.detected.autoPlugins flatMap { d => - d.value.extraProjects map { _.setProjectOrigin(ProjectOrigin.ExtraProject) } + val plugs = timed("Load.loadUnit: plugins", log) { + plugins(defDir, s, config.copy(pluginManagement = config.pluginManagement.forPlugin)) + } + val defsScala = timed("Load.loadUnit: defsScala", log) { + plugs.detected.builds.values } // NOTE - because we create an eval here, we need a clean-eval later for this URI. - lazy val eval = mkEval(plugs.classpath, defDir, plugs.pluginData.scalacOptions) - val initialProjects = defsScala.flatMap(b => projectsFromBuild(b, normBase)) ++ buildLevelExtraProjects + lazy val eval = timed("Load.loadUnit: mkEval", log) { mkEval(plugs.classpath, defDir, plugs.pluginData.scalacOptions) } + val initialProjects = defsScala.flatMap(b => projectsFromBuild(b, normBase)) val hasRootAlreadyDefined = defsScala.exists(_.rootProject.isDefined) @@ -467,8 +487,7 @@ private[sbt] object Load { val result = loadTransitive(ps, normBase, plugs, () => eval, config.injectSettings, Nil, memoSettings, config.log, createRoot, uri, config.pluginManagement.context, Nil) result } - - val loadedProjectsRaw = loadProjects(initialProjects, !hasRootAlreadyDefined) + val loadedProjectsRaw = timed("Load.loadUnit: loadedProjectsRaw", log) { loadProjects(initialProjects, !hasRootAlreadyDefined) } // TODO - As of sbt 0.13.6 we should always have a default root project from // here on, so the autogenerated build aggregated can be removed from this code. ( I think) // We may actually want to move it back here and have different flags in loadTransitive... @@ -481,13 +500,14 @@ private[sbt] object Load { val refs = existingIDs.map(id => ProjectRef(uri, id)) val defaultID = autoID(normBase, config.pluginManagement.context, existingIDs) val b = BuildDef.defaultAggregated(defaultID, refs) - val defaultProjects = loadProjects(projectsFromBuild(b, normBase), false) + val defaultProjects = timed("Load.loadUnit: defaultProjects", log) { loadProjects(projectsFromBuild(b, normBase), false) } (defaultProjects.projects ++ loadedProjectsRaw.projects, b, defaultProjects.generatedConfigClassFiles ++ loadedProjectsRaw.generatedConfigClassFiles) } // Now we clean stale class files. // TODO - this may cause issues with multiple sbt clients, but that should be deprecated pending sbt-server anyway - cleanEvalClasses(defDir, keepClassFiles) - + timed("Load.loadUnit: cleanEvalClasses", log) { + cleanEvalClasses(defDir, keepClassFiles) + } val defs = if (defsScala.isEmpty) defaultBuildIfNone :: Nil else defsScala // HERE we pull out the defined vals from memoSettings and unify them all so // we can use them later. @@ -565,7 +585,7 @@ private[sbt] object Load { context: PluginManagement.Context, generatedConfigClassFiles: Seq[File] ): LoadedProjects = - { + /*timed(s"Load.loadTransitive(${ newProjects.map(_.id) })", log)*/ { // load all relevant configuration files (.sbt, as .scala already exists at this point) def discover(auto: AddSettings, base: File): DiscoveredProjects = discoverProjects(auto, base, plugins, eval, memoSettings) @@ -615,7 +635,7 @@ private[sbt] object Load { discover(AddSettings.defaultSbtFiles, buildBase) match { case DiscoveredProjects(Some(root), discovered, files, generated) => log.debug(s"[Loading] Found root project ${root.id} w/ remaining ${discovered.map(_.id).mkString(",")}") - val (finalRoot, projectLevelExtra) = finalizeProject(root, files, true) + val (finalRoot, projectLevelExtra) = timed(s"Load.loadTransitive: finalizeProject($root)", log) { finalizeProject(root, files, true) } loadTransitive(discovered ++ projectLevelExtra, buildBase, plugins, eval, injectSettings, finalRoot +: acc, memoSettings, log, false, buildUri, context, generated ++ generatedConfigClassFiles) // Here we need to create a root project... case DiscoveredProjects(None, discovered, files, generated) => @@ -628,7 +648,7 @@ private[sbt] object Load { val defaultID = autoID(buildBase, context, existingIds) val root0 = if (discovered.isEmpty || java.lang.Boolean.getBoolean("sbt.root.ivyplugin")) BuildDef.defaultAggregatedProject(defaultID, buildBase, refs) else BuildDef.generatedRootWithoutIvyPlugin(defaultID, buildBase, refs) - val (root, _) = finalizeProject(root0, files, false) + val (root, _) = timed(s"Load.loadTransitive: finalizeProject2($root0)", log) { finalizeProject(root0, files, false) } val result = root +: (acc ++ otherProjects.projects) log.debug(s"[Loading] Done in ${buildBase}, returning: ${result.map(_.id).mkString("(", ", ", ")")}") LoadedProjects(result, generated ++ otherGenerated ++ generatedConfigClassFiles) @@ -679,39 +699,40 @@ private[sbt] object Load { globalUserSettings: InjectSettings, memoSettings: mutable.Map[File, LoadedSbtFile], log: Logger - ): Project = { - import AddSettings._ - val autoConfigs = projectPlugins.flatMap(_.projectConfigurations) + ): Project = + timed(s"Load.resolveProject(${p.id})", log) { + import AddSettings._ + val autoConfigs = projectPlugins.flatMap(_.projectConfigurations) - // 3. Use AddSettings instance to order all Setting[_]s appropriately - val allSettings = { - // TODO - This mechanism of applying settings could be off... It's in two places now... - lazy val defaultSbtFiles = configurationSources(p.base) - // Filter the AutoPlugin settings we included based on which ones are - // intended in the AddSettings.AutoPlugins filter. - def autoPluginSettings(f: AutoPlugins) = - projectPlugins.filter(f.include).flatMap(_.projectSettings) - // Grab all the settigns we already loaded from sbt files - def settings(files: Seq[File]): Seq[Setting[_]] = - for { - file <- files - config <- (memoSettings get file).toSeq - setting <- config.settings - } yield setting - // Expand the AddSettings instance into a real Seq[Setting[_]] we'll use on the project - def expandSettings(auto: AddSettings): Seq[Setting[_]] = auto match { - case BuildScalaFiles => p.settings - case User => globalUserSettings.projectLoaded(loadedPlugins.loader) - case sf: SbtFiles => settings(sf.files.map(f => IO.resolve(p.base, f))) - case sf: DefaultSbtFiles => settings(defaultSbtFiles.filter(sf.include)) - case p: AutoPlugins => autoPluginSettings(p) - case q: Sequence => (Seq.empty[Setting[_]] /: q.sequence) { (b, add) => b ++ expandSettings(add) } + // 3. Use AddSettings instance to order all Setting[_]s appropriately + val allSettings = { + // TODO - This mechanism of applying settings could be off... It's in two places now... + lazy val defaultSbtFiles = configurationSources(p.base) + // Filter the AutoPlugin settings we included based on which ones are + // intended in the AddSettings.AutoPlugins filter. + def autoPluginSettings(f: AutoPlugins) = + projectPlugins.filter(f.include).flatMap(_.projectSettings) + // Grab all the settigns we already loaded from sbt files + def settings(files: Seq[File]): Seq[Setting[_]] = + for { + file <- files + config <- (memoSettings get file).toSeq + setting <- config.settings + } yield setting + // Expand the AddSettings instance into a real Seq[Setting[_]] we'll use on the project + def expandSettings(auto: AddSettings): Seq[Setting[_]] = auto match { + case BuildScalaFiles => p.settings + case User => globalUserSettings.projectLoaded(loadedPlugins.loader) + case sf: SbtFiles => settings(sf.files.map(f => IO.resolve(p.base, f))) + case sf: DefaultSbtFiles => settings(defaultSbtFiles.filter(sf.include)) + case p: AutoPlugins => autoPluginSettings(p) + case q: Sequence => (Seq.empty[Setting[_]] /: q.sequence) { (b, add) => b ++ expandSettings(add) } + } + expandSettings(p.auto) } - expandSettings(p.auto) + // Finally, a project we can use in buildStructure. + p.copy(settingsEval = Ev.later(allSettings)).setAutoPlugins(projectPlugins).prefixConfigs(autoConfigs: _*) } - // Finally, a project we can use in buildStructure. - p.copy(settingsEval = Ev.later(allSettings)).setAutoPlugins(projectPlugins).prefixConfigs(autoConfigs: _*) - } /** * This method attempts to discover all Project/settings it can using the configured AddSettings and project base. @@ -921,6 +942,15 @@ private[sbt] object Load { @deprecated("Use BuildUtil.apply", "0.13.0") def buildUtil(root: URI, units: Map[URI, LoadedBuildUnit], keyIndex: KeyIndex, data: Settings[Scope]): BuildUtil[ResolvedProject] = BuildUtil(root, units, keyIndex, data) + + /** Debugging method to time how long it takes to run various compilation tasks. */ + private[sbt] def timed[T](label: String, log: Logger)(t: => T): T = { + val start = System.nanoTime + val result = t + val elapsed = System.nanoTime - start + log.debug(label + " took " + (elapsed / 1e6) + " ms") + result + } } final case class LoadBuildConfiguration( From 3e29a48b5695a712e42b123cde88c4edb7509d57 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 01:12:49 -0400 Subject: [PATCH 2/9] Cache global user settings --- main/src/main/scala/sbt/internal/Load.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index b2b70f1c1..174352e19 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -722,7 +722,7 @@ private[sbt] object Load { // Expand the AddSettings instance into a real Seq[Setting[_]] we'll use on the project def expandSettings(auto: AddSettings): Seq[Setting[_]] = auto match { case BuildScalaFiles => p.settings - case User => globalUserSettings.projectLoaded(loadedPlugins.loader) + case User => globalUserSettings.cachedProjectLoaded(loadedPlugins.loader) case sf: SbtFiles => settings(sf.files.map(f => IO.resolve(p.base, f))) case sf: DefaultSbtFiles => settings(defaultSbtFiles.filter(sf.include)) case p: AutoPlugins => autoPluginSettings(p) @@ -938,7 +938,11 @@ private[sbt] object Load { def referenced[PR <: ProjectReference](definitions: Seq[ProjectDefinition[PR]]): Seq[PR] = definitions flatMap { _.referenced } final class EvaluatedConfigurations(val eval: Eval, val settings: Seq[Setting[_]]) - final case class InjectSettings(global: Seq[Setting[_]], project: Seq[Setting[_]], projectLoaded: ClassLoader => Seq[Setting[_]]) + final case class InjectSettings(global: Seq[Setting[_]], project: Seq[Setting[_]], projectLoaded: ClassLoader => Seq[Setting[_]]) { + private val cache: mutable.Map[Unit, Seq[Setting[_]]] = mutable.Map.empty + def cachedProjectLoaded(cl: ClassLoader): Seq[Setting[_]] = + cache.getOrElseUpdate((), projectLoaded(cl)) + } @deprecated("Use BuildUtil.apply", "0.13.0") def buildUtil(root: URI, units: Map[URI, LoadedBuildUnit], keyIndex: KeyIndex, data: Settings[Scope]): BuildUtil[ResolvedProject] = BuildUtil(root, units, keyIndex, data) From 843f79ffd91a5c01424f884fa262a7d35849dcee Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 01:49:23 -0400 Subject: [PATCH 3/9] Improve structureIndex call by using Vector This call takes around 8035ms for 100 subprojects. I don't think using Vector here had any noticeable effect. --- main/src/main/scala/sbt/internal/Load.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 174352e19..8245ccb00 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -220,10 +220,10 @@ private[sbt] object Load { { val keys = Index.allKeys(settings) val attributeKeys = Index.attributeKeys(data) ++ keys.map(_.key) - val scopedKeys = keys ++ data.allKeys((s, k) => ScopedKey(s, k)) + val scopedKeys = keys ++ data.allKeys((s, k) => ScopedKey(s, k)).toVector val projectsMap = projects.mapValues(_.defined.keySet) - val keyIndex = KeyIndex(scopedKeys, projectsMap) - val aggIndex = KeyIndex.aggregate(scopedKeys, extra(keyIndex), projectsMap) + val keyIndex = KeyIndex(scopedKeys.toVector, projectsMap) + val aggIndex = KeyIndex.aggregate(scopedKeys.toVector, extra(keyIndex), projectsMap) new StructureIndex(Index.stringToKeyMap(attributeKeys), Index.taskToKeyMap(data), Index.triggers(data), keyIndex, aggIndex) } From 7a00ba392517163ec88f90350a715522ea7ad625 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 01:51:07 -0400 Subject: [PATCH 4/9] Display log when sbt loading is going to pause Def.make could take 10099ms for 100 subprojects. This would display logs probably for projects with more than 10 subprojects, which might pause for a few seconds during load. --- main/src/main/scala/sbt/internal/Load.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 8245ccb00..d1ce21864 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -176,6 +176,10 @@ private[sbt] object Load { } val delegates = timed("Load.apply: config.delegates", log) { config.delegates(loaded) } val data = timed("Load.apply: Def.make(settings)...", log) { + // When settings.size is 100000, Def.make takes around 10s. + if (settings.size > 10000) { + log.info(s"Resolving key references (${settings.size} settings) ...") + } Def.make(settings)(delegates, config.scopeLocal, Project.showLoadingKey(loaded)) } Project.checkTargets(data) foreach sys.error From 700017be918ba8875583d8c33b832782ace195ed Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 03:51:12 -0400 Subject: [PATCH 5/9] Cache based on the underlying URLs of the ClassLoader per review --- main/src/main/scala/sbt/internal/Load.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index d1ce21864..59ee46a68 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -943,9 +943,14 @@ private[sbt] object Load { final class EvaluatedConfigurations(val eval: Eval, val settings: Seq[Setting[_]]) final case class InjectSettings(global: Seq[Setting[_]], project: Seq[Setting[_]], projectLoaded: ClassLoader => Seq[Setting[_]]) { - private val cache: mutable.Map[Unit, Seq[Setting[_]]] = mutable.Map.empty + import java.net.URLClassLoader + private val cache: mutable.Map[Set[URL], Seq[Setting[_]]] = mutable.Map.empty + // Cache based on the underlying URL values of the classloader def cachedProjectLoaded(cl: ClassLoader): Seq[Setting[_]] = - cache.getOrElseUpdate((), projectLoaded(cl)) + cl match { + case cl: URLClassLoader => cache.getOrElseUpdate(cl.getURLs.toSet, projectLoaded(cl)) + case _ => projectLoaded(cl) + } } @deprecated("Use BuildUtil.apply", "0.13.0") From 0cfde91a4bd078a89257103ebccd463a50ecedd5 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 03:51:53 -0400 Subject: [PATCH 6/9] Remove unnecessary val --- main/src/main/scala/sbt/internal/Load.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 59ee46a68..0c6ad91d9 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -325,7 +325,6 @@ private[sbt] object Load { def load(file: File, loaders: BuildLoader, extra: List[URI]): PartBuild = loadURI(IO.directoryURI(file), loaders, extra) def loadURI(uri: URI, loaders: BuildLoader, extra: List[URI]): PartBuild = { - val log = loaders.config.log IO.assertAbsolute(uri) val (referenced, map, newLoaders) = loadAll(uri :: extra, Map.empty, loaders, Map.empty) checkAll(referenced, map) From dc71b171dac93d0f00cd3d5a4bec5b32361064e2 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 12:21:35 -0400 Subject: [PATCH 7/9] Cache now takes in account of the parent of the classloader --- main/src/main/scala/sbt/internal/Load.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 0c6ad91d9..0d60029f4 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -943,13 +943,22 @@ private[sbt] object Load { final class EvaluatedConfigurations(val eval: Eval, val settings: Seq[Setting[_]]) final case class InjectSettings(global: Seq[Setting[_]], project: Seq[Setting[_]], projectLoaded: ClassLoader => Seq[Setting[_]]) { import java.net.URLClassLoader - private val cache: mutable.Map[Set[URL], Seq[Setting[_]]] = mutable.Map.empty + private val cache: mutable.Map[List[URL], Seq[Setting[_]]] = mutable.Map.empty // Cache based on the underlying URL values of the classloader def cachedProjectLoaded(cl: ClassLoader): Seq[Setting[_]] = cl match { - case cl: URLClassLoader => cache.getOrElseUpdate(cl.getURLs.toSet, projectLoaded(cl)) + case cl: URLClassLoader => cache.getOrElseUpdate(classLoaderToList(cl), projectLoaded(cl)) case _ => projectLoaded(cl) } + private def classLoaderToList(cl: ClassLoader): List[URL] = + cl match { + case cl: URLClassLoader => + cl.getURLs.toList ::: (Option(cl.getParent) match { + case Some(x) => classLoaderToList(x) + case _ => Nil + }) + case _ => Nil + } } @deprecated("Use BuildUtil.apply", "0.13.0") From 88a644eb4dd9a8806d9a717b9e2f50462fc98fba Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Wed, 25 May 2016 12:57:38 -0400 Subject: [PATCH 8/9] Use parent's toString --- main/src/main/scala/sbt/internal/Load.scala | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index 0d60029f4..e64cd1639 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -943,21 +943,20 @@ private[sbt] object Load { final class EvaluatedConfigurations(val eval: Eval, val settings: Seq[Setting[_]]) final case class InjectSettings(global: Seq[Setting[_]], project: Seq[Setting[_]], projectLoaded: ClassLoader => Seq[Setting[_]]) { import java.net.URLClassLoader - private val cache: mutable.Map[List[URL], Seq[Setting[_]]] = mutable.Map.empty + private val cache: mutable.Map[String, Seq[Setting[_]]] = mutable.Map.empty // Cache based on the underlying URL values of the classloader def cachedProjectLoaded(cl: ClassLoader): Seq[Setting[_]] = cl match { - case cl: URLClassLoader => cache.getOrElseUpdate(classLoaderToList(cl), projectLoaded(cl)) + case cl: URLClassLoader => cache.getOrElseUpdate(classLoaderToHash(Some(cl)), projectLoaded(cl)) case _ => projectLoaded(cl) } - private def classLoaderToList(cl: ClassLoader): List[URL] = - cl match { - case cl: URLClassLoader => - cl.getURLs.toList ::: (Option(cl.getParent) match { - case Some(x) => classLoaderToList(x) - case _ => Nil - }) - case _ => Nil + private def classLoaderToHash(o: Option[ClassLoader]): String = + o match { + case Some(cl: URLClassLoader) => + cl.getURLs.toList.toString + classLoaderToHash(Option(cl.getParent)) + case Some(cl: ClassLoader) => + cl.toString + classLoaderToHash(Option(cl.getParent)) + case _ => "null" } } From e0e9f8d1fcd8a69c60fa72e2e527e564f16300e5 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 14 Jan 2017 07:54:25 -0500 Subject: [PATCH 9/9] Fix patch error --- main/src/main/scala/sbt/internal/Load.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main/src/main/scala/sbt/internal/Load.scala b/main/src/main/scala/sbt/internal/Load.scala index e64cd1639..df99db1b5 100755 --- a/main/src/main/scala/sbt/internal/Load.scala +++ b/main/src/main/scala/sbt/internal/Load.scala @@ -478,10 +478,13 @@ private[sbt] object Load { val defsScala = timed("Load.loadUnit: defsScala", log) { plugs.detected.builds.values } + val buildLevelExtraProjects = plugs.detected.autoPlugins flatMap { d => + d.value.extraProjects map { _.setProjectOrigin(ProjectOrigin.ExtraProject) } + } // NOTE - because we create an eval here, we need a clean-eval later for this URI. lazy val eval = timed("Load.loadUnit: mkEval", log) { mkEval(plugs.classpath, defDir, plugs.pluginData.scalacOptions) } - val initialProjects = defsScala.flatMap(b => projectsFromBuild(b, normBase)) + val initialProjects = defsScala.flatMap(b => projectsFromBuild(b, normBase)) ++ buildLevelExtraProjects val hasRootAlreadyDefined = defsScala.exists(_.rootProject.isDefined)