mirror of https://github.com/sbt/sbt.git
Merge pull request #1873 from sbt/fix/1730c
Fixes #1711, #1730. Cached resolution: fixes internal project
This commit is contained in:
eugene yokota
commit
5275470525
|
|
@ -17,6 +17,7 @@ import core.{ IvyPatternHelper, LogOptions }
|
|||
import org.apache.ivy.util.{ Message, MessageLogger }
|
||||
import org.apache.ivy.plugins.latest.{ ArtifactInfo => IvyArtifactInfo }
|
||||
import org.apache.ivy.plugins.matcher.{ MapMatcher, PatternMatcher }
|
||||
import Configurations.{ System => _, _ }
|
||||
|
||||
private[sbt] object CachedResolutionResolveCache {
|
||||
def createID(organization: String, name: String, revision: String) =
|
||||
|
|
@ -37,104 +38,25 @@ private[sbt] class CachedResolutionResolveCache() {
|
|||
}
|
||||
def directDependencies(md0: ModuleDescriptor): Vector[DependencyDescriptor] =
|
||||
md0.getDependencies.toVector
|
||||
def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[(DefaultModuleDescriptor, Boolean)] =
|
||||
// Returns a vector of (module descriptor, changing, dd)
|
||||
def buildArtificialModuleDescriptors(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[(DefaultModuleDescriptor, Boolean, DependencyDescriptor)] =
|
||||
{
|
||||
log.debug(s":: building artificial module descriptors from ${md0.getModuleRevisionId}")
|
||||
val expanded = expandInternalDependencies(md0, data, prOpt, log)
|
||||
// val expanded = expandInternalDependencies(md0, data, prOpt, log)
|
||||
val rootModuleConfigs = md0.getConfigurations.toArray.toVector
|
||||
expanded map { dd =>
|
||||
directDependencies(md0) map { dd =>
|
||||
val arts = dd.getAllDependencyArtifacts.toVector map { x => s"""${x.getName}:${x.getType}:${x.getExt}:${x.getExtraAttributes}""" }
|
||||
log.debug(s"::: expanded dd: $dd (artifacts: ${arts.mkString(",")})")
|
||||
log.debug(s"::: dd: $dd (artifacts: ${arts.mkString(",")})")
|
||||
buildArtificialModuleDescriptor(dd, rootModuleConfigs, md0, prOpt, log)
|
||||
}
|
||||
}
|
||||
// This expands out all internal dependencies and merge them into a single graph that consists
|
||||
// only of external dependencies.
|
||||
// The tricky part is the merger of configurations, even though in most cases we will only see compile->compile when it comes to internal deps.
|
||||
// Theoretically, there could be a potential for test->test->runtime kind of situation. nextConfMap and remapConfigurations track
|
||||
// the configuration chains transitively.
|
||||
def expandInternalDependencies(md0: ModuleDescriptor, data: ResolveData, prOpt: Option[ProjectResolver], log: Logger): Vector[DependencyDescriptor] =
|
||||
{
|
||||
log.debug(s"::: expanding internal dependencies of module descriptor ${md0.getModuleRevisionId}")
|
||||
val rootModuleConfigs = md0.getConfigurations.toArray.toVector
|
||||
val rootNode = new IvyNode(data, md0)
|
||||
def expandInternalDeps(dep: DependencyDescriptor, confMap: Map[String, Array[String]]): Vector[DependencyDescriptor] =
|
||||
internalDependency(dep) match {
|
||||
case Some(internal) =>
|
||||
log.debug(s""":::: found internal dependency ${internal.getResolvedModuleRevisionId}""")
|
||||
val allConfigurations: Vector[String] = confMap.values.flatten.toVector.distinct
|
||||
val next = nextConfMap(dep, confMap)
|
||||
// direct dependencies of an internal dependency
|
||||
val directs0 = directDependencies(internal)
|
||||
val directs = directs0 filter { dd =>
|
||||
allConfigurations exists { conf => dd.getDependencyConfigurations(conf).nonEmpty }
|
||||
}
|
||||
directs flatMap { dd => expandInternalDeps(dd, next) }
|
||||
case _ =>
|
||||
Vector(remapConfigurations(dep, confMap, log))
|
||||
}
|
||||
def internalDependency(dep: DependencyDescriptor): Option[ModuleDescriptor] =
|
||||
prOpt match {
|
||||
case Some(pr) => pr.getModuleDescriptor(dep.getDependencyRevisionId)
|
||||
case _ => None
|
||||
}
|
||||
// This creates confMap. The key of the map is rootModuleConf for md0, the value is the dependency configs for dd.
|
||||
def nextConfMap(dd: DependencyDescriptor, previous: Map[String, Array[String]]): Map[String, Array[String]] =
|
||||
previous map {
|
||||
case (rootModuleConf, vs) =>
|
||||
rootModuleConf -> (vs flatMap { conf =>
|
||||
dd.getDependencyConfigurations(conf) flatMap { confName =>
|
||||
if (confName == "*") Array(confName)
|
||||
else rootNode.getRealConfs(confName)
|
||||
}
|
||||
})
|
||||
}
|
||||
// The key of the confMap is rootModuleConf for md0, and the values are modules configuratons of dd0.
|
||||
// For example if project Root depends on project B % "test", and project B depends on junit,
|
||||
// confMap should contain Map("test", Array("compile")).
|
||||
// This remaps junit dependency as junit % "test".
|
||||
def remapConfigurations(dd0: DependencyDescriptor, confMap: Map[String, Array[String]], log: Logger): DependencyDescriptor =
|
||||
{
|
||||
log.debug(s""":::: remapping configuration of ${dd0} with ${confMap.toList map { case (k, v) => (k, v.toList) }}""")
|
||||
val dd = new DefaultDependencyDescriptor(md0, dd0.getDependencyRevisionId, dd0.getDynamicConstraintDependencyRevisionId,
|
||||
dd0.isForce, dd0.isChanging, dd0.isTransitive)
|
||||
val moduleConfigurations = dd0.getModuleConfigurations.toVector
|
||||
for {
|
||||
moduleConf <- moduleConfigurations
|
||||
(rootModuleConf, vs) <- confMap.toSeq
|
||||
} if (vs contains moduleConf) {
|
||||
log.debug(s""":::: ${dd0}: $moduleConf maps to $rootModuleConf""")
|
||||
dd0.getDependencyConfigurations(moduleConf) foreach { conf =>
|
||||
dd.addDependencyConfiguration(rootModuleConf, conf)
|
||||
}
|
||||
dd0.getIncludeRules(moduleConf) foreach { rule =>
|
||||
dd.addIncludeRule(rootModuleConf, rule)
|
||||
}
|
||||
dd0.getExcludeRules(moduleConf) foreach { rule =>
|
||||
dd.addExcludeRule(rootModuleConf, rule)
|
||||
}
|
||||
dd0.getAllDependencyArtifacts foreach { dad0 =>
|
||||
(Option(dad0.getConfigurations) map { confs => confs.isEmpty || confs.contains(moduleConf) || confs.contains("*") }) match {
|
||||
case Some(false) => // do nothing
|
||||
case _ =>
|
||||
val dad = new DefaultDependencyArtifactDescriptor(dd, dad0.getName, dad0.getType, dad0.getExt, dad0.getUrl, dad0.getExtraAttributes)
|
||||
dad.addConfiguration(rootModuleConf)
|
||||
dd.addDependencyArtifact(rootModuleConf, dad)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.debug(s""":::: remapped dd: $dd""")
|
||||
dd
|
||||
}
|
||||
directDependencies(md0) flatMap { dep =>
|
||||
val initialMap = Map(dep.getModuleConfigurations map { rootModuleConf =>
|
||||
(rootModuleConf -> Array(rootModuleConf))
|
||||
}: _*)
|
||||
expandInternalDeps(dep, initialMap)
|
||||
}
|
||||
def internalDependency(dd: DependencyDescriptor, prOpt: Option[ProjectResolver]): Option[ModuleDescriptor] =
|
||||
prOpt match {
|
||||
case Some(pr) => pr.getModuleDescriptor(dd.getDependencyRevisionId)
|
||||
case _ => None
|
||||
}
|
||||
def buildArtificialModuleDescriptor(dd: DependencyDescriptor, rootModuleConfigs: Vector[IvyConfiguration],
|
||||
parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean) =
|
||||
parent: ModuleDescriptor, prOpt: Option[ProjectResolver], log: Logger): (DefaultModuleDescriptor, Boolean, DependencyDescriptor) =
|
||||
{
|
||||
def excludeRuleString(rule: ExcludeRule): String =
|
||||
s"""Exclude(${rule.getId},${rule.getConfigurations.mkString(",")},${rule.getMatcher})"""
|
||||
|
|
@ -179,7 +101,7 @@ private[sbt] class CachedResolutionResolveCache() {
|
|||
mes foreach { exclude =>
|
||||
md1.addExcludeRule(exclude)
|
||||
}
|
||||
(md1, IvySbt.isChanging(dd))
|
||||
(md1, IvySbt.isChanging(dd) || internalDependency(dd, prOpt).isDefined, dd)
|
||||
}
|
||||
def extractOverrides(md0: ModuleDescriptor): Vector[IvyOverride] =
|
||||
{
|
||||
|
|
@ -332,7 +254,20 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
|
|||
val options1 = new ResolveOptions(options0)
|
||||
val data = new ResolveData(this, options1)
|
||||
val mds = cache.buildArtificialModuleDescriptors(md0, data, projectResolver, log)
|
||||
def doWork(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
|
||||
|
||||
def doWork(md: ModuleDescriptor, dd: DependencyDescriptor): Either[ResolveException, UpdateReport] =
|
||||
cache.internalDependency(dd, projectResolver) match {
|
||||
case Some(md1) =>
|
||||
log.debug(s":: call customResolve recursively: $dd")
|
||||
customResolve(md1, missingOk, logicalClock, options0, depDir, log) match {
|
||||
case Right(ur) => Right(remapInternalProject(new IvyNode(data, md1), ur, md0, dd, os, log))
|
||||
case Left(e) => Left(e)
|
||||
}
|
||||
case None =>
|
||||
log.debug(s":: call ivy resolution: $dd")
|
||||
doWorkUsingIvy(md)
|
||||
}
|
||||
def doWorkUsingIvy(md: ModuleDescriptor): Either[ResolveException, UpdateReport] =
|
||||
{
|
||||
val options1 = new ResolveOptions(options0)
|
||||
var rr = withIvy(log) { ivy =>
|
||||
|
|
@ -354,9 +289,9 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
|
|||
}
|
||||
}
|
||||
val results = mds map {
|
||||
case (md, changing) =>
|
||||
case (md, changing, dd) =>
|
||||
cache.getOrElseUpdateMiniGraph(md, changing, logicalClock, miniGraphPath, cachedDescriptor, log) {
|
||||
doWork(md)
|
||||
doWork(md, dd)
|
||||
}
|
||||
}
|
||||
val uReport = mergeResults(md0, results, missingOk, System.currentTimeMillis - start, os, log)
|
||||
|
|
@ -386,6 +321,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
|
|||
}
|
||||
def mergeReports(md0: ModuleDescriptor, reports: Vector[UpdateReport], resolveTime: Long, os: Vector[IvyOverride], log: Logger): UpdateReport =
|
||||
{
|
||||
log.debug(s":: merging update reports")
|
||||
val cachedDescriptor = getSettings.getResolutionCacheManager.getResolvedIvyFileInCache(md0.getModuleRevisionId)
|
||||
val rootModuleConfigs = md0.getConfigurations.toVector
|
||||
val cachedReports = reports filter { !_.stats.cached }
|
||||
|
|
@ -415,6 +351,7 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
|
|||
def mergeOrganizationArtifactReports(rootModuleConf: String, reports0: Vector[OrganizationArtifactReport], os: Vector[IvyOverride], log: Logger): Vector[OrganizationArtifactReport] =
|
||||
(reports0 groupBy { oar => (oar.organization, oar.name) }).toSeq.toVector flatMap {
|
||||
case ((org, name), xs) =>
|
||||
log.debug(s""":::: $rootModuleConf: $org:$name""")
|
||||
if (xs.size < 2) xs
|
||||
else Vector(new OrganizationArtifactReport(org, name, mergeModuleReports(rootModuleConf, xs flatMap { _.modules }, os, log)))
|
||||
}
|
||||
|
|
@ -511,6 +448,46 @@ private[sbt] trait CachedResolutionResolveEngine extends ResolveEngine {
|
|||
(surviving, evicted)
|
||||
}
|
||||
}
|
||||
def remapInternalProject(node: IvyNode, ur: UpdateReport,
|
||||
md0: ModuleDescriptor, dd: DependencyDescriptor,
|
||||
os: Vector[IvyOverride], log: Logger): UpdateReport =
|
||||
{
|
||||
def parentConfigs(c: String): Vector[String] =
|
||||
Option(md0.getConfiguration(c)) match {
|
||||
case Some(config) =>
|
||||
config.getExtends.toVector ++
|
||||
(config.getExtends.toVector flatMap parentConfigs)
|
||||
case None => Vector()
|
||||
}
|
||||
// These are the configurations from the original project we want to resolve.
|
||||
val rootModuleConfs = md0.getConfigurations.toArray.toVector
|
||||
val configurations0 = ur.configurations.toVector
|
||||
// This is how md looks from md0 via dd's mapping.
|
||||
val remappedConfigs0: Map[String, Vector[String]] = Map(rootModuleConfs map { conf0 =>
|
||||
val remapped: Vector[String] = dd.getDependencyConfigurations(conf0.getName).toVector flatMap { conf =>
|
||||
node.getRealConfs(conf).toVector
|
||||
}
|
||||
conf0.getName -> remapped
|
||||
}: _*)
|
||||
// This emulates test-internal extending test configuration etc.
|
||||
val remappedConfigs: Map[String, Vector[String]] =
|
||||
(remappedConfigs0 /: rootModuleConfs) { (acc0, c) =>
|
||||
val ps = parentConfigs(c.getName)
|
||||
(acc0 /: ps) { (acc, parent) =>
|
||||
val vs0 = acc.getOrElse(c.getName, Vector())
|
||||
val vs = acc.getOrElse(parent, Vector())
|
||||
acc.updated(c.getName, (vs0 ++ vs).distinct)
|
||||
}
|
||||
}
|
||||
log.debug(s"::: remapped configs $remappedConfigs")
|
||||
val configurations = rootModuleConfs map { conf0 =>
|
||||
val remappedCRs = configurations0 filter { cr =>
|
||||
remappedConfigs(conf0.getName) contains cr.configuration
|
||||
}
|
||||
mergeConfigurationReports(conf0.getName, remappedCRs, os, log)
|
||||
}
|
||||
new UpdateReport(ur.cachedDescriptor, configurations, ur.stats, ur.stamps)
|
||||
}
|
||||
}
|
||||
|
||||
private[sbt] case class ModuleReportArtifactInfo(moduleReport: ModuleReport) extends IvyArtifactInfo {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
[@cunei]: https://github.com/cunei
|
||||
[@eed3si9n]: https://github.com/eed3si9n
|
||||
[@gkossakowski]: https://github.com/gkossakowski
|
||||
[@jsuereth]: https://github.com/jsuereth
|
||||
[1711]: https://github.com/sbt/sbt/issues/1711
|
||||
[1752]: https://github.com/sbt/sbt/pull/1752
|
||||
|
||||
### Fixes with compatibility implications
|
||||
|
||||
### Improvements
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- Fixes cached resolution handling of internal depdendencies. [#1711][1711] by [@eed3si9n][@eed3si9n]
|
||||
- Fixes cached resolution being too verbose. [#1752][1752] by [@eed3si9n][@eed3si9n]
|
||||
|
|
@ -63,30 +63,30 @@ lazy val root = (project in file(".")).
|
|||
organization in ThisBuild := "org.example",
|
||||
version in ThisBuild := "1.0",
|
||||
check := {
|
||||
val acp = (externalDependencyClasspath in Compile in a).value.sortBy {_.data.getName}
|
||||
val bcp = (externalDependencyClasspath in Compile in b).value.sortBy {_.data.getName}
|
||||
val ccp = (externalDependencyClasspath in Compile in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"}
|
||||
if (!(acp exists { _.data.getName contains "commons-io-1.4-sources.jar" })) {
|
||||
val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}.sorted
|
||||
val bcp = (externalDependencyClasspath in Compile in b).value.map {_.data.getName}.sorted
|
||||
val ccp = (externalDependencyClasspath in Compile in c).value.map {_.data.getName}.sorted filterNot { _ == "demo_2.10.jar"}
|
||||
if (!(acp contains "commons-io-1.4-sources.jar")) {
|
||||
sys.error("commons-io-1.4-sources not found when it should be included: " + acp.toString)
|
||||
}
|
||||
if (!(acp exists { _.data.getName contains "commons-io-1.4.jar" })) {
|
||||
if (!(acp contains "commons-io-1.4.jar")) {
|
||||
sys.error("commons-io-1.4 not found when it should be included: " + acp.toString)
|
||||
}
|
||||
|
||||
// stock Ivy implementation doesn't contain regular (non-source) jar, which probably is a bug
|
||||
val acpWithoutSource = acp filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
val bcpWithoutSource = bcp filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
val ccpWithoutSource = ccp filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
val acpWithoutSource = acp filterNot { _ == "commons-io-1.4.jar"}
|
||||
val bcpWithoutSource = bcp filterNot { _ == "commons-io-1.4.jar"}
|
||||
val ccpWithoutSource = ccp filterNot { _ == "commons-io-1.4.jar"}
|
||||
if (acpWithoutSource == bcpWithoutSource && acpWithoutSource == ccpWithoutSource) ()
|
||||
else sys.error("Different classpaths are found:" +
|
||||
"\n - a (cached) " + acpWithoutSource.toString +
|
||||
"\n - b (plain) " + bcpWithoutSource.toString +
|
||||
"\n - c (inter-project) " + ccpWithoutSource.toString)
|
||||
|
||||
val atestcp = (externalDependencyClasspath in Test in a).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
val btestcp = (externalDependencyClasspath in Test in b).value.sortBy {_.data.getName} filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
val ctestcp = (externalDependencyClasspath in Test in c).value.sortBy {_.data.getName} filterNot { _.data.getName == "demo_2.10.jar"} filterNot { _.data.getName contains "commons-io-1.4"}
|
||||
if (ctestcp exists { _.data.getName contains "junit-4.11.jar" }) {
|
||||
val atestcp = (externalDependencyClasspath in Test in a).value.map {_.data.getName}.sorted filterNot { _ == "commons-io-1.4.jar"}
|
||||
val btestcp = (externalDependencyClasspath in Test in b).value.map {_.data.getName}.sorted filterNot { _ == "commons-io-1.4.jar"}
|
||||
val ctestcp = (externalDependencyClasspath in Test in c).value.map {_.data.getName}.sorted filterNot { _ == "demo_2.10.jar"} filterNot { _ == "commons-io-1.4.jar"}
|
||||
if (ctestcp contains "junit-4.11.jar") {
|
||||
sys.error("junit found when it should be excluded: " + ctestcp.toString)
|
||||
}
|
||||
|
||||
|
|
@ -96,3 +96,4 @@ lazy val root = (project in file(".")).
|
|||
"\n - b test (plain) " + btestcp.toString)
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
> debug
|
||||
|
||||
> a/update
|
||||
|
||||
> a/updateClassifiers
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
// https://github.com/sbt/sbt/issues/1730
|
||||
lazy val check = taskKey[Unit]("Runs the check")
|
||||
|
||||
def commonSettings: Seq[Def.Setting[_]] =
|
||||
Seq(
|
||||
ivyPaths := new IvyPaths( (baseDirectory in ThisBuild).value, Some((baseDirectory in LocalRootProject).value / "ivy-cache")),
|
||||
dependencyCacheDirectory := (baseDirectory in LocalRootProject).value / "dependency",
|
||||
scalaVersion := "2.11.4",
|
||||
resolvers += Resolver.sonatypeRepo("snapshots")
|
||||
)
|
||||
|
||||
def cachedResolutionSettings: Seq[Def.Setting[_]] =
|
||||
commonSettings ++ Seq(
|
||||
updateOptions := updateOptions.value.withCachedResolution(true)
|
||||
)
|
||||
|
||||
lazy val transitiveTest = project.
|
||||
settings(cachedResolutionSettings: _*).
|
||||
settings(
|
||||
libraryDependencies += "junit" % "junit" % "4.11" % Test
|
||||
)
|
||||
|
||||
lazy val transitiveTestDefault = project.
|
||||
settings(cachedResolutionSettings: _*).
|
||||
settings(
|
||||
libraryDependencies += "org.scalatest" %% "scalatest" % "2.2.1"
|
||||
)
|
||||
|
||||
lazy val a = project.
|
||||
dependsOn(transitiveTestDefault % Test, transitiveTest % "test->test").
|
||||
settings(cachedResolutionSettings: _*)
|
||||
|
||||
lazy val root = (project in file(".")).
|
||||
aggregate(a).
|
||||
settings(
|
||||
organization in ThisBuild := "org.example",
|
||||
version in ThisBuild := "1.0",
|
||||
check := {
|
||||
val ur = (update in a).value
|
||||
val acp = (externalDependencyClasspath in Compile in a).value.map {_.data.getName}
|
||||
val atestcp0 = (fullClasspath in Test in a).value
|
||||
val atestcp = (externalDependencyClasspath in Test in a).value.map {_.data.getName}
|
||||
// This is checking to make sure interproject dependency works
|
||||
if (acp exists { _ contains "scalatest" }) {
|
||||
sys.error("scalatest found when it should NOT be included: " + acp.toString)
|
||||
}
|
||||
// This is checking to make sure interproject dependency works
|
||||
if (!(atestcp exists { _ contains "scalatest" })) {
|
||||
sys.error("scalatest NOT found when it should be included: " + atestcp.toString)
|
||||
}
|
||||
// This is checking to make sure interproject dependency works
|
||||
if (!(atestcp exists { _ contains "junit" })) {
|
||||
sys.error("junit NOT found when it should be included: " + atestcp.toString)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
> debug
|
||||
|
||||
> check
|
||||
|
||||
Loading…
Reference in New Issue