Merge pull request #2354 from eed3si9n/wip/internal

Inter-project dependency tracking

ivy/src/main/scala/sbt/TrackLevel.scala

@@ -0,0 +1,33 @@
+package sbt
+
+/**
+ * An enumeration defining the tracking of dependencies.  A level includes all of the levels
+ * with id larger than its own id.  For example, Warn (id=3) includes Error (id=4).
+ */
+object TrackLevel {
+  case object NoTracking extends TrackLevel {
+    override def id: Int = 0
+  }
+  case object TrackIfMissing extends TrackLevel {
+    override def id: Int = 1
+  }
+  case object TrackAlways extends TrackLevel {
+    override def id: Int = 10
+  }
+
+  private[sbt] def apply(x: Int): TrackLevel =
+    x match {
+      case 0  => NoTracking
+      case 1  => TrackIfMissing
+      case 10 => TrackAlways
+    }
+
+  def intersection(a: TrackLevel, b: TrackLevel): TrackLevel =
+    if (a.id < b.id) a
+    else b
+  def intersectionAll(vs: List[TrackLevel]): TrackLevel = vs reduceLeft intersection
+}
+
+sealed trait TrackLevel {
+  def id: Int
+}

main/src/main/scala/sbt/Defaults.scala

@@ -99,6 +99,8 @@ object Defaults extends BuildCommon {
       internalConfigurationMap :== Configurations.internalMap _,
       credentials :== Nil,
       exportJars :== false,
+      trackInternalDependencies :== TrackLevel.TrackAlways,
+      exportToInternal :== TrackLevel.TrackAlways,
       retrieveManaged :== false,
       retrieveManagedSync :== false,
       configurationsToRetrieve :== None,
@@ -1035,6 +1037,9 @@ object Classpaths {
     unmanagedClasspath <<= unmanagedDependencies,
     managedClasspath := managedJars(classpathConfiguration.value, classpathTypes.value, update.value),
     exportedProducts <<= exportProductsTask,
+    exportedProductsAlways <<= trackedExportedProducts(TrackLevel.TrackAlways),
+    exportedProductsIfMissing <<= trackedExportedProducts(TrackLevel.TrackIfMissing),
+    exportedProductsNoTracking <<= trackedExportedProducts(TrackLevel.NoTracking),
     unmanagedJars := findUnmanagedJars(configuration.value, unmanagedBase.value, includeFilter in unmanagedJars value, excludeFilter in unmanagedJars value)
   ).map(exportClasspath)
 
@@ -1569,6 +1574,52 @@ object Classpaths {
     val x2 = copyResources.value
     classDirectory.value :: Nil
   }
+  // This is a variant of exportProductsTask with tracking
+  private[sbt] def trackedExportedProducts(track: TrackLevel): Initialize[Task[Classpath]] = Def.task {
+    val art = (artifact in packageBin).value
+    val module = projectID.value
+    val config = configuration.value
+    for { (f, analysis) <- trackedProductsImplTask(track).value } yield APIMappings.store(analyzed(f, analysis), apiURL.value).put(artifact.key, art).put(moduleID.key, module).put(configuration.key, config)
+  }
+  private[this] def trackedProductsImplTask(track: TrackLevel): Initialize[Task[Seq[(File, Analysis)]]] =
+    Def.taskDyn {
+      val useJars = exportJars.value
+      val jar = (artifactPath in packageBin).value
+      val dirs = productDirectories.value
+      def containsClassFile(fs: List[File]): Boolean =
+        (fs exists { dir =>
+          (dir ** DirectoryFilter).get exists { d =>
+            (d * "*.class").get.nonEmpty
+          }
+        })
+      TrackLevel.intersection(track, exportToInternal.value) match {
+        case TrackLevel.TrackAlways if (useJars) =>
+          Def.task {
+            Seq((packageBin.value, compile.value))
+          }
+        case TrackLevel.TrackAlways =>
+          Def.task {
+            products.value map { (_, compile.value) }
+          }
+        case TrackLevel.TrackIfMissing if (useJars && !jar.exists) =>
+          Def.task {
+            Seq((packageBin.value, compile.value))
+          }
+        case TrackLevel.TrackIfMissing if (!useJars && !containsClassFile(dirs.toList)) =>
+          Def.task {
+            products.value map { (_, compile.value) }
+          }
+        case _ =>
+          Def.task {
+            val analysis = previousCompile.value.analysis
+            (if (useJars) Seq(jar)
+            else dirs) map {
+              (_, analysis)
+            }
+          }
+      }
+    }
+
   def exportProductsTask: Initialize[Task[Classpath]] = Def.task {
     val art = (artifact in packageBin).value
     val module = projectID.value
@@ -1584,7 +1635,7 @@ object Classpaths {
   def constructBuildDependencies: Initialize[BuildDependencies] = loadedBuild(lb => BuildUtil.dependencies(lb.units))
 
   def internalDependencies: Initialize[Task[Classpath]] =
-    (thisProjectRef, classpathConfiguration, configuration, settingsData, buildDependencies) flatMap internalDependencies0
+    (thisProjectRef, classpathConfiguration, configuration, settingsData, buildDependencies, trackInternalDependencies) flatMap internalDependencies0
   def unmanagedDependencies: Initialize[Task[Classpath]] =
     (thisProjectRef, configuration, settingsData, buildDependencies) flatMap unmanagedDependencies0
   def mkIvyConfiguration: Initialize[Task[IvyConfiguration]] =
@@ -1622,17 +1673,25 @@ object Classpaths {
       visited.toSeq
     }
   def unmanagedDependencies0(projectRef: ProjectRef, conf: Configuration, data: Settings[Scope], deps: BuildDependencies): Task[Classpath] =
-    interDependencies(projectRef, deps, conf, conf, data, true, unmanagedLibs)
+    interDependencies(projectRef, deps, conf, conf, data, TrackLevel.TrackAlways, true, unmanagedLibs0)
+  @deprecated("This is no longer public.", "0.13.10")
   def internalDependencies0(projectRef: ProjectRef, conf: Configuration, self: Configuration, data: Settings[Scope], deps: BuildDependencies): Task[Classpath] =
     interDependencies(projectRef, deps, conf, self, data, false, productsTask)
+  private[sbt] def internalDependencies0(projectRef: ProjectRef, conf: Configuration, self: Configuration, data: Settings[Scope], deps: BuildDependencies, track: TrackLevel): Task[Classpath] =
+    interDependencies(projectRef, deps, conf, self, data, track, false, productsTask0)
+  @deprecated("This is no longer public.", "0.13.10")
   def interDependencies(projectRef: ProjectRef, deps: BuildDependencies, conf: Configuration, self: Configuration, data: Settings[Scope], includeSelf: Boolean,
     f: (ProjectRef, String, Settings[Scope]) => Task[Classpath]): Task[Classpath] =
+    interDependencies(projectRef, deps, conf, self, data, TrackLevel.TrackAlways, includeSelf,
+      { (pr: ProjectRef, s: String, sc: Settings[Scope], tl: TrackLevel) => f(pr, s, sc) })
+  private[sbt] def interDependencies(projectRef: ProjectRef, deps: BuildDependencies, conf: Configuration, self: Configuration, data: Settings[Scope],
+    track: TrackLevel, includeSelf: Boolean, f: (ProjectRef, String, Settings[Scope], TrackLevel) => Task[Classpath]): Task[Classpath] =
     {
       val visited = interSort(projectRef, conf, data, deps)
       val tasks = asScalaSet(new LinkedHashSet[Task[Classpath]])
       for ((dep, c) <- visited)
         if (includeSelf || (dep != projectRef) || (conf.name != c && self.name != c))
-          tasks += f(dep, c, data)
+          tasks += f(dep, c, data, track)
 
       (tasks.toSeq.join).map(_.flatten.distinct)
     }
@@ -1676,6 +1735,14 @@ object Classpaths {
     configurations.find(_.name == conf)
   def productsTask(dep: ResolvedReference, conf: String, data: Settings[Scope]): Task[Classpath] =
     getClasspath(exportedProducts, dep, conf, data)
+  def productsTask0(dep: ResolvedReference, conf: String, data: Settings[Scope], track: TrackLevel): Task[Classpath] =
+    track match {
+      case TrackLevel.NoTracking     => getClasspath(exportedProductsNoTracking, dep, conf, data)
+      case TrackLevel.TrackIfMissing => getClasspath(exportedProductsIfMissing, dep, conf, data)
+      case TrackLevel.TrackAlways    => getClasspath(exportedProductsAlways, dep, conf, data)
+    }
+  private[sbt] def unmanagedLibs0(dep: ResolvedReference, conf: String, data: Settings[Scope], track: TrackLevel): Task[Classpath] =
+    unmanagedLibs(dep, conf, data)
   def unmanagedLibs(dep: ResolvedReference, conf: String, data: Settings[Scope]): Task[Classpath] =
     getClasspath(unmanagedJars, dep, conf, data)
   def getClasspath(key: TaskKey[Classpath], dep: ResolvedReference, conf: String, data: Settings[Scope]): Task[Classpath] =
@@ -1741,7 +1808,7 @@ object Classpaths {
 
   private[this] lazy val internalCompilerPluginClasspath: Initialize[Task[Classpath]] =
     (thisProjectRef, settingsData, buildDependencies) flatMap { (ref, data, deps) =>
-      internalDependencies0(ref, CompilerPlugin, CompilerPlugin, data, deps)
+      internalDependencies0(ref, CompilerPlugin, CompilerPlugin, data, deps, TrackLevel.TrackAlways)
     }
 
   lazy val compilerPluginConfig = Seq(

main/src/main/scala/sbt/Keys.scala

@@ -234,6 +234,9 @@ object Keys {
   val productDirectories = TaskKey[Seq[File]]("product-directories", "Base directories of build products.", CTask)
   val exportJars = SettingKey[Boolean]("export-jars", "Determines whether the exported classpath for this project contains classes (false) or a packaged jar (true).", BSetting)
   val exportedProducts = TaskKey[Classpath]("exported-products", "Build products that go on the exported classpath.", CTask)
+  val exportedProductsAlways = TaskKey[Classpath]("exported-products-always", "Build products that go on the exported classpath for other projects.", CTask)
+  val exportedProductsIfMissing = TaskKey[Classpath]("exported-products-if-missing", "Build products that go on the exported classpath if missing.", CTask)
+  val exportedProductsNoTracking = TaskKey[Classpath]("exported-products-no-tracking", "Just the exported classpath without triggering the compilation.", CTask)
   val unmanagedClasspath = TaskKey[Classpath]("unmanaged-classpath", "Classpath entries (deep) that are manually managed.", BPlusTask)
   val unmanagedJars = TaskKey[Classpath]("unmanaged-jars", "Classpath entries for the current project (shallow) that are manually managed.", BPlusTask)
   val managedClasspath = TaskKey[Classpath]("managed-classpath", "The classpath consisting of external, managed library dependencies.", BMinusTask)
@@ -241,6 +244,8 @@ object Keys {
   val externalDependencyClasspath = TaskKey[Classpath]("external-dependency-classpath", "The classpath consisting of library dependencies, both managed and unmanaged.", BMinusTask)
   val dependencyClasspath = TaskKey[Classpath]("dependency-classpath", "The classpath consisting of internal and external, managed and unmanaged dependencies.", BPlusTask)
   val fullClasspath = TaskKey[Classpath]("full-classpath", "The exported classpath, consisting of build products and unmanaged and managed, internal and external dependencies.", BPlusTask)
+  val trackInternalDependencies = SettingKey[TrackLevel]("track-internal-dependencies", "The level of tracking for the internal (inter-project) dependency.", BSetting)
+  val exportToInternal = SettingKey[TrackLevel]("export-to-internal", "The level of tracking for this project by the internal callers.", BSetting)
 
   val internalConfigurationMap = SettingKey[Configuration => Configuration]("internal-configuration-map", "Maps configurations to the actual configuration used to define the classpath.", CSetting)
   val classpathConfiguration = TaskKey[Configuration]("classpath-configuration", "The configuration used to define the classpath.", CTask)

notes/0.13.10/internal-tracking.md

@@ -0,0 +1,32 @@
+
+  [@eed3si9n]: https://github.com/eed3si9n
+  [2266]: https://github.com/sbt/sbt/issues/2266
+  [2354]: https://github.com/sbt/sbt/pull/2354
+
+### Improvements
+
+- Adds `trackInternalDependencies` and `exportToInternal` keys. See below.
+
+### Inter-project dependency tracking
+
+sbt 0.13.10 adds `trackInternalDependencies` and `exportToInternal` settings. These can be used to control whether to trigger compilation of a dependent subprojects when you call `compile`. Both keys will take one of three values: `TrackLevel.NoTracking`, `TrackLevel.TrackIfMissing`, and `TrackLevel.TrackAlways`. By default they are both set to `TrackLevel.TrackAlways`.
+
+When `trackInternalDependencies` is set to `TrackLevel.TrackIfMissing`, sbt will no longer try to compile internal (inter-project) dependencies automatically, unless there are no `*.class` files (or JAR file when `exportJars` is `true`) in the output directory. When the setting is set to `TrackLevel.NoTracking`, the compilation of internal dependencies will be skipped. Note that the classpath will still be appended, and dependency graph will still show them as dependencies. The motivation is to save the I/O overhead of checking for the changes on a build with many subprojects during development. Here's how to set all subprojects to `TrackIfMissing`.
+
+    lazy val root = (project in file(".")).
+      aggregate(....).
+      settings(
+        inThisBuild(Seq(
+          trackInternalDependencies := TrackLevel.TrackIfMissing,
+          exportJars := true
+        ))
+      )
+
+The `exportToInternal` setting allows the dependee subprojects to opt out of the internal tracking, which might be useful if you want to track most subprojects except for a few. The intersection of the `trackInternalDependencies` and `exportToInternal` settings will be used to determine the actual track level. Here's an example to opt-out one project:
+
+    lazy val dontTrackMe = (project in file("dontTrackMe")).
+      settings(
+        exportToInternal := TrackLevel.NoTracking
+      )
+
+[#2266][2266]/[#2354][2354] by [@eed3si9n][@eed3si9n]

sbt/src/sbt-test/project/internal-tracking/a/A.scala

@@ -0,0 +1,3 @@
+package a
+
+object A {}

sbt/src/sbt-test/project/internal-tracking/b/B.scala

@@ -0,0 +1,5 @@
+package b
+
+object B {
+  println(a.A.toString)
+}

sbt/src/sbt-test/project/internal-tracking/build.sbt

@@ -0,0 +1,25 @@
+lazy val root = (project in file(".")).
+  aggregate(a, b, c, d).
+  settings(
+    inThisBuild(Seq(
+      scalaVersion := "2.11.7",
+      trackInternalDependencies := TrackLevel.NoTracking
+    ))
+  )
+
+lazy val a = (project in file("a"))
+
+lazy val b = (project in file("b")).
+  dependsOn(a)
+
+lazy val c = (project in file("c")).
+  settings(
+    exportToInternal := TrackLevel.NoTracking
+  )
+
+lazy val d = (project in file("d")).
+  dependsOn(c).
+  settings(
+    trackInternalDependencies := TrackLevel.TrackIfMissing
+  )
+

sbt/src/sbt-test/project/internal-tracking/c/C.scala

@@ -0,0 +1,3 @@
+package c
+
+object C {}

sbt/src/sbt-test/project/internal-tracking/d/D.scala

@@ -0,0 +1,3 @@
+package d
+
+object D { println(c.C.toString) }

sbt/src/sbt-test/project/internal-tracking/test

@@ -0,0 +1,11 @@
+-> b/compile
+
+> a/compile
+
+> b/compile
+
+-> d/compile
+
+> c/compile
+
+> d/compile