luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1715 from sbt/fix/1711 

Fixes #1711 and #1704. Cached resolution fixes
This commit is contained in:
Josh Suereth 2014-11-06 07:45:54 -05:00
parent c8f2f04dde af9e520e28
commit aae0f6ef40
4 changed files with 119 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
ivy/src/main/scala/sbt/ivyint/CachedResolutionResolveEngine.scala
View File

@ -11,7 +11,7 @@ import org.apache.ivy.core
import core.resolve._
import core.module.id.{ ModuleRevisionId, ModuleId => IvyModuleId }
import core.report.{ ResolveReport, ConfigurationResolveReport, DownloadReport }
import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule }
import core.module.descriptor.{ DefaultModuleDescriptor, ModuleDescriptor, DefaultDependencyDescriptor, DependencyDescriptor, Configuration => IvyConfiguration, ExcludeRule, IncludeRule }
import core.module.descriptor.OverrideDependencyDescriptorMediator
import core.{ IvyPatternHelper, LogOptions }
import org.apache.ivy.util.Message
@ -29,40 +29,85 @@ private[sbt] class CachedResolutionResolveCache() {
val updateReportCache: concurrent.Map[ModuleRevisionId, Either[ResolveException, UpdateReport]] = concurrent.TrieMap()
val resolveReportCache: concurrent.Map[ModuleRevisionId, ResolveReport] = concurrent.TrieMap()
val resolvePropertiesCache: concurrent.Map[ModuleRevisionId, String] = concurrent.TrieMap()
val directDependencyCache: concurrent.Map[ModuleRevisionId, Vector[DependencyDescriptor]] = concurrent.TrieMap()
val conflictCache: concurrent.Map[(ModuleID, ModuleID), (Vector[ModuleID], Vector[ModuleID], String)] = concurrent.TrieMap()
val maxConflictCacheSize: Int = 10000
def clean(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Unit = {
val mrid0 = md0.getModuleRevisionId
val mds =
if (mrid0.getOrganisation == sbtOrgTemp) Vector(md0)
else buildArtificialModuleDescriptors(md0, prOpt) map { _._1 }
updateReportCache.remove(md0.getModuleRevisionId)
directDependencyCache.remove(md0.getModuleRevisionId)
mds foreach { md =>
updateReportCache.remove(md.getModuleRevisionId)
directDependencyCache.remove(md.getModuleRevisionId)
}
updateReportCache.clear
}
def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] =
directDependencyCache.getOrElseUpdate(md0.getModuleRevisionId, md0.getDependencies.toVector)
def buildArtificialModuleDescriptors(md0: ModuleDescriptor, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] =
md0.getDependencies.toVector
def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[(DefaultModuleDescriptor, Boolean)] =
{
def expandInternalDeps(dep: DependencyDescriptor): Vector[DependencyDescriptor] =
prOpt map {
_.getModuleDescriptor(dep.getDependencyRevisionId) match {
case Some(internal) => directDependencies(internal) flatMap expandInternalDeps
case _ => Vector(dep)
}
} getOrElse Vector(dep)
val expanded = directDependencies(md0) flatMap expandInternalDeps
val rootModuleConfigs = md0.getConfigurations.toVector
val rootModuleConfigs = md0.getConfigurations.toArray.toVector
val expanded = expandInternalDependencies(md0, data, prOpt)
expanded map { buildArtificialModuleDescriptor(_, rootModuleConfigs, md0, prOpt) }
}
// This expands out all internal dependencies and merge them into a single graph that consists
// only of external dependencies.
// The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps.
// Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track
// the configuration chains transitively.
def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver]): Vector[DependencyDescriptor] =
{
val rootModuleConfigs = md0.getConfigurations.toArray.toVector
val rootNode = new IvyNode(data, md0)
def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] =
internalDependency(dep) match {
case Some(internal) =>
val allConfigurations: Vector[String] =
(if (confMap.isEmpty) nextConfMap(dep, confMap)
else confMap).values.flatten.toList.distinct.toVector
directDependencies(internal) filter { dd =>
allConfigurations exists { conf => !dd.getDependencyConfigurations(conf).isEmpty }
} flatMap { dd => expandInternalDeps(dd, nextConfMap(dd, confMap)) }
case _ =>
if (confMap.isEmpty) Vector(dep)
else Vector(remapConfigurations(dep, confMap))
}
def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] =
prOpt match {
case Some(pr) => pr.getModuleDescriptor(dep.getDependencyRevisionId)
case _ => None
}
// This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd.
def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] =
if (previous.isEmpty) {
ListMap(dd.getModuleConfigurations.toList map { conf =>
conf -> (dd.getDependencyConfigurations(conf) flatMap { confName =>
if (confName == "*") Array(confName)
else rootNode.getRealConfs(confName)
})
}: _*)
} else previous map {
case (rootModuleConf, vs) =>
rootModuleConf -> (vs flatMap { conf =>
dd.getDependencyConfigurations(conf) flatMap { confName =>
if (confName == "*") Array(confName)
else rootNode.getRealConfs(confName)
}
})
}
def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]]): DependencyDescriptor =
{
val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId,
dd0.isForce, dd0.isChanging, dd0.isTransitive)
for {
moduleConf <- dd0.getModuleConfigurations
(rootModuleConf, vs) <- confMap
} if (vs contains moduleConf) {
// moduleConf in dd0 maps to rootModuleConf in dd
dd0.getDependencyConfigurations(moduleConf) foreach { conf =>
dd.addDependencyConfiguration(rootModuleConf, conf)
}
dd0.getExcludeRules(moduleConf) foreach { rule =>
dd.addExcludeRule(rootModuleConf, rule)
}
}
dd
}
directDependencies(md0) flatMap { dep => expandInternalDeps(dep, Map()) }
}
def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration], parent: ModuleDescriptor, prOpt: Option[ProjectResolver]): (DefaultModuleDescriptor, Boolean) =
{
def excludeRuleString(rule: ExcludeRule): String =
@ -85,8 +130,10 @@ private[sbt] class CachedResolutionResolveCache() {
case rules => Some(conf + "->(" + (rules map includeRuleString).mkString(",") + ")")
}
})
val mes = parent.getAllExcludeRules.toVector
val mesStr = (mes map excludeRuleString).mkString(",")
val os = extractOverrides(parent)
val moduleLevel = s"""dependencyOverrides=${os.mkString(",")}"""
val moduleLevel = s"""dependencyOverrides=${os.mkString(",")};moduleExclusions=$mesStr"""
val depsString = s"""$mrid;${confMap.mkString(",")};isForce=${dd.isForce};isChanging=${dd.isChanging};isTransitive=${dd.isTransitive};""" +
s"""exclusions=${exclusions.mkString(",")};inclusions=${inclusions.mkString(",")};$moduleLevel;"""
val sha1 = Hash.toHex(Hash(depsString))
@ -100,6 +147,9 @@ private[sbt] class CachedResolutionResolveCache() {
os foreach { ovr =>
md1.addDependencyDescriptorMediator(ovr.moduleId, ovr.pm, ovr.ddm)
}
mes foreach { exclude =>
md1.addExcludeRule(exclude)
}
(md1, IvySbt.isChanging(dd))
}
def extractOverrides(md0: ModuleDescriptor): Vector[IvyOverride] =
@ -228,9 +278,10 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
val miniGraphPath = depDir / "module"
val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
val cache = cachedResolutionResolveCache
cache.directDependencyCache.remove(md0.getModuleRevisionId)
val os = cache.extractOverrides(md0)
val mds = cache.buildArtificialModuleDescriptors(md0, projectResolver)
val options1 = new ResolveOptions(options0)
val data = new ResolveData(this, options1)
val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver)
def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
{
val options1 = new ResolveOptions(options0)

49
sbt/src/sbt-test/dependency-management/cached-resolution-basic/multi.sbt
View File

@ -4,36 +4,43 @@ def commonSettings: Seq[Def.Setting[_]] =
Seq(
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")),
dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency",
scalaVersion := "2.10.4",
resolvers += Resolver.sonatypeRepo("snapshots")
)
lazy val a = project.
settings(commonSettings: _*).
settings(
updateOptions := updateOptions.value.withCachedResolution(true),
artifact in (Compile, packageBin) := Artifact("demo"),
libraryDependencies := Seq(
"net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(),
"com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"),
"net.databinder" %% "unfiltered-uploads" % "0.8.0",
"commons-io" % "commons-io" % "1.3",
"com.typesafe" % "config" % "0.4.9-SNAPSHOT"
),
scalaVersion := "2.10.4",
resolvers += Resolver.sonatypeRepo("snapshots")
)
def consolidatedResolutionSettings: Seq[Def.Setting[_]] =
commonSettings ++ Seq(
updateOptions := updateOptions.value.withConsolidatedResolution(true)
)
lazy val a = project.
settings(consolidatedResolutionSettings: _*).
settings(
artifact in (Compile, packageBin) := Artifact("demo")
"com.typesafe" % "config" % "0.4.9-SNAPSHOT",
"junit" % "junit" % "4.11" % "test"
)
)
lazy val b = project.
settings(commonSettings: _*)
settings(commonSettings: _*).
settings(
libraryDependencies := Seq(
"net.sf.json-lib" % "json-lib" % "2.4" classifier "jdk15" intransitive(),
"com.typesafe.akka" %% "akka-remote" % "2.3.4" exclude("com.typesafe.akka", "akka-actor_2.10"),
"net.databinder" %% "unfiltered-uploads" % "0.8.0",
"commons-io" % "commons-io" % "1.3",
"com.typesafe" % "config" % "0.4.9-SNAPSHOT",
"junit" % "junit" % "4.11" % "test"
)
)
lazy val c = project.
dependsOn(a).
settings(consolidatedResolutionSettings: _*).
settings(commonSettings: _*).
settings(
// libraryDependencies := Seq(organization.value %% "a" % version.value)
updateOptions := updateOptions.value.withCachedResolution(true)
)
lazy val root = (project in file(".")).
@ -43,7 +50,11 @@ lazy val root = (project in file(".")).
check := {
val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName}
val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName}
val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot {_.data.getName == "demo_2.10.jar"}
val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"}
val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"}
if (ctestcp exists { _.data.getName contains "junit-4.11.jar" }) {
sys.error("junit found when it should be excluded: " + ctestcp.toString)
}
if (acp == bcp && acp == ccp) ()
else sys.error("Different classpaths are found:" +
"\n - a (cached) " + acp.toString +

11
sbt/src/sbt-test/dependency-management/cached-resolution-exclude/multi.sbt
View File

@ -17,7 +17,11 @@ def cachedResolutionSettings: Seq[Def.Setting[_]] =
lazy val a = project.
settings(cachedResolutionSettings: _*).
settings(
libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" exclude("commons-io", "commons-io")
libraryDependencies += "net.databinder" %% "unfiltered-uploads" % "0.8.0" exclude("commons-io", "commons-io"),
ivyXML :=
<dependencies>
<exclude module="commons-codec"/>
</dependencies>
)
lazy val b = project.
@ -32,12 +36,15 @@ lazy val root = (project in file(".")).
organization in ThisBuild := "org.example",
version in ThisBuild := "1.0",
check := {
// sys.error(dependencyCacheDirectory.value.toString)
val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName}
val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName}
if (acp exists { _.data.getName contains "commons-io" }) {
sys.error("commons-io found when it should be excluded")
}
if (acp exists { _.data.getName contains "commons-codec" }) {
sys.error("commons-codec found when it should be excluded")
}
// This is checking to make sure excluded graph is not getting picked up
if (!(bcp exists { _.data.getName contains "commons-io" })) {
sys.error("commons-io NOT found when it should NOT be excluded")
}

1
util/collection/src/main/scala/sbt/Settings.scala
View File

@ -85,7 +85,6 @@ trait Init[Scope] {
*/
private[sbt] final def validated[T](key: ScopedKey[T], selfRefOk: Boolean): ValidationCapture[T] = new ValidationCapture(key, selfRefOk)
@deprecated("0.13.7", "Use the version with default arguments and default paramter.")
final def derive[T](s: Setting[T], allowDynamic: Boolean, filter: Scope => Boolean, trigger: AttributeKey[_] => Boolean): Setting[T] =
derive(s, allowDynamic, filter, trigger, false)