Retrieve compiler-interface sources via Ivy

This commit modifies the build definition so that there's no need
anymore to build precompiled versions of the compiler-interface, and
introduces a new mechanism that allows the `ComponentCompiler` to
retrieve the sources of the compiler-interface using Ivy.

build.sbt

@@ -82,16 +82,6 @@ lazy val bundledLauncherProj =
   )
 
 
-// This is used only for command aggregation
-lazy val allPrecompiled: Project = (project in file("all-precompiled")).
-  aggregate(precompiled282, precompiled292, precompiled293).
-  settings(
-    buildLevelSettings,
-    minimalSettings,
-    publish := {},
-    publishLocal := {}
-  )
-
 /* ** subproject declarations ** */
 
 // defines Java structures used across Scala versions, such as the API structures and relationships extracted by
@@ -317,7 +307,7 @@ lazy val compileInterfaceProj = (project in compilePath / "interface").
   dependsOn(interfaceProj % "compile;test->test", ioProj % "test->test", logProj % "test->test", /*launchProj % "test->test",*/ apiProj % "test->test").
   settings(
     baseSettings,
-    precompiledSettings,
+    libraryDependencies += scalaCompiler.value % "provided",
     name := "Compiler Interface",
     exportJars := true,
     // we need to fork because in unit tests we set usejavacp = true which means
@@ -330,10 +320,6 @@ lazy val compileInterfaceProj = (project in compilePath / "interface").
     artifact in (Compile, packageSrc) := Artifact(srcID).copy(configurations = Compile :: Nil).extra("e:component" -> srcID)
   )
 
-lazy val precompiled282 = precompiled(scala282)
-lazy val precompiled292 = precompiled(scala292)
-lazy val precompiled293 = precompiled(scala293)
-
 // Implements the core functionality of detecting and propagating changes incrementally.
 //   Defines the data structures for representing file fingerprints and relationships and the overall source analysis
 lazy val compileIncrementalProj = (project in compilePath / "inc").
@@ -442,7 +428,7 @@ lazy val mainProj = (project in mainPath).
 //  technically, we need a dependency on all of mainProj's dependencies, but we don't do that since this is strictly an integration project
 //  with the sole purpose of providing certain identifiers without qualification (with a package object)
 lazy val sbtProj = (project in sbtPath).
-  dependsOn(mainProj, compileInterfaceProj, precompiled282, precompiled292, precompiled293, scriptedSbtProj % "test->test").
+  dependsOn(mainProj, compileInterfaceProj, scriptedSbtProj % "test->test").
   settings(
     baseSettings,
     name := "sbt",
@@ -512,7 +498,7 @@ def otherRootSettings = Seq(
   }
 ))
 lazy val docProjects: ScopeFilter = ScopeFilter(
-  inAnyProject -- inProjects(sbtRoot, sbtProj, scriptedBaseProj, scriptedSbtProj, scriptedPluginProj, precompiled282, precompiled292, precompiled293, mavenResolverPluginProj),
+  inAnyProject -- inProjects(sbtRoot, sbtProj, scriptedBaseProj, scriptedSbtProj, scriptedPluginProj, mavenResolverPluginProj),
   inConfigurations(Compile)
 )
 def fullDocSettings = Util.baseScalacOptions ++ Docs.settings ++ Sxr.settings ++ Seq(
@@ -539,36 +525,6 @@ def utilPath   = file("util")
 def compilePath = file("compile")
 def mainPath   = file("main")
 
-def precompiledSettings = Seq(
-  artifact in packageBin <<= (appConfiguration, scalaVersion) { (app, sv) =>
-    val launcher = app.provider.scalaProvider.launcher
-    val bincID = binID + "_" + ScalaInstance(sv, launcher).actualVersion
-    Artifact(binID) extra ("e:component" -> bincID)
-  },
-  target <<= (target, scalaVersion) { (base, sv) => base / ("precompiled_" + sv) },
-  scalacOptions := Nil,
-  ivyScala ~= { _.map(_.copy(checkExplicit = false, overrideScalaVersion = false)) },
-  exportedProducts in Compile := Nil,
-  libraryDependencies += scalaCompiler.value % "provided"
-)
-
-def precompiled(scalav: String): Project = Project(id = normalize("Precompiled " + scalav.replace('.', '_')), base = compilePath / "interface").
-  dependsOn(interfaceProj).
-  settings(
-    baseSettings,
-    precompiledSettings,
-    name := "Precompiled " + scalav.replace('.', '_'),
-    scalaHome := None,
-    scalaVersion <<= (scalaVersion in ThisBuild) { sbtScalaV =>
-      assert(sbtScalaV != scalav, "Precompiled compiler interface cannot have the same Scala version (" + scalav + ") as sbt.")
-      scalav
-    },
-    crossScalaVersions := Seq(scalav),
-    // we disable compiling and running tests in precompiled Projprojects of compiler interface
-    // so we do not need to worry about cross-versioning testing dependencies
-    sources in Test := Nil
-  )
-
 lazy val safeUnitTests = taskKey[Unit]("Known working tests (for both 2.10 and 2.11)")
 lazy val safeProjects: ScopeFilter = ScopeFilter(
   inProjects(mainSettingsProj, mainProj, ivyProj, completeProj,
@@ -616,9 +572,6 @@ def customCommands: Seq[Setting[_]] = Seq(
   },
   commands += Command.command("release-sbt-local") { state =>
     "clean" ::
-    "allPrecompiled/clean" ::
-    "allPrecompiled/compile" ::
-    "allPrecompiled/publishLocal" ::
     "so compile" ::
     "so publishLocal" ::
     "reload" ::
@@ -643,9 +596,6 @@ def customCommands: Seq[Setting[_]] = Seq(
   commands += Command.command("release-sbt") { state =>
     // TODO - Any sort of validation
     "clean" ::
-    "allPrecompiled/clean" ::
-      "allPrecompiled/compile" ::
-      "allPrecompiled/publishSigned" ::
       "conscript-configs" ::
       "so compile" ::
       "so publishSigned" ::
@@ -656,9 +606,6 @@ def customCommands: Seq[Setting[_]] = Seq(
   commands += Command.command("release-nightly") { state =>
     "stamp-version" ::
       "clean" ::
-      "allPrecompiled/clean" ::
-      "allPrecompiled/compile" ::
-      "allPrecompiled/publish" ::
       "compile" ::
       "publish" ::
       "bintrayRelease" ::

compile/ivy/src/main/scala/sbt/compiler/ComponentCompiler.scala

@@ -5,6 +5,7 @@ package sbt
 package compiler
 
 import java.io.File
+import scala.util.Try
 
 object ComponentCompiler {
   val xsbtiID = "xsbti"
@@ -13,7 +14,9 @@ object ComponentCompiler {
   val compilerInterfaceID = "compiler-interface"
   val compilerInterfaceSrcID = compilerInterfaceID + srcExtension
   val javaVersion = System.getProperty("java.class.version")
+  val sbtVersion = "0.13.9-SNAPSHOT" // TODO: Shouldn't be hardcoded...
 
+  @deprecated("Please specify `ivyConfiguration`.", "0.13.10")
   def interfaceProvider(manager: ComponentManager): CompilerInterfaceProvider = new CompilerInterfaceProvider {
     def apply(scalaInstance: xsbti.compile.ScalaInstance, log: Logger): File =
       {
@@ -23,6 +26,33 @@ object ComponentCompiler {
         componentCompiler(compilerInterfaceID)
       }
   }
+
+  def interfaceProvider(manager: ComponentManager, ivyConfiguration: IvyConfiguration): CompilerInterfaceProvider = new CompilerInterfaceProvider {
+    def apply(scalaInstance: xsbti.compile.ScalaInstance, log: Logger): File =
+      {
+        // this is the instance used to compile the interface component
+        val componentCompiler = new NewComponentCompiler(new RawCompiler(scalaInstance, ClasspathOptions.auto, log), manager, ivyConfiguration, log)
+        log.debug("Getting " + compilerInterfaceID + " from component compiler for Scala " + scalaInstance.version)
+        componentCompiler(compilerInterfaceID)
+      }
+  }
+
+  /**
+   * Generates a sequence of version of numbers, from more to less specific from `fullVersion`.
+   */
+  private[compiler] def cascadingSourceModuleVersions(fullVersion: String): Seq[String] = {
+
+    Try {
+      val VersionNumber(numbers @ maj +: min +: _, tags, _) = VersionNumber(fullVersion)
+      Seq(fullVersion,
+        numbers.mkString(".") + (if (tags.nonEmpty) tags.mkString("-", "-", "") else ""),
+        numbers.mkString("."),
+        s"$maj.$min"
+      ).distinct
+    } getOrElse {
+      Seq(fullVersion)
+    }
+  }
 }
 /**
  * This class provides source components compiled with the provided RawCompiler.
@@ -65,3 +95,105 @@ class ComponentCompiler(compiler: RawCompiler, manager: ComponentManager) {
     }
   }
 }
+
+private[compiler] class NewComponentCompiler(compiler: RawCompiler, manager: ComponentManager, ivyConfiguration: IvyConfiguration, log: Logger) {
+  import ComponentCompiler._
+
+  private val ivySbt: IvySbt = new IvySbt(ivyConfiguration)
+
+  def apply(id: String): File = {
+    val binID = binaryID(id)
+    manager.file(binID)(new IfMissing.Define(true, compileAndInstall(id, binID)))
+  }
+
+  private def binaryID(id: String): String = {
+    val base = id + binSeparator + compiler.scalaInstance.actualVersion
+    base + "__" + javaVersion
+  }
+
+  private def compileAndInstall(id: String, binID: String): Unit = {
+    val srcID = id + srcExtension
+    def interfaceSources(moduleVersions: Seq[String]): File =
+      moduleVersions match {
+        case Seq() =>
+          log.debug(s"Fetching default sources: $id")
+          val jarName = s"$srcID-$sbtVersion.jar"
+          update(getModule(id))(_.getName == jarName) getOrElse (throw new InvalidComponent(s"Couldn't retrieve default sources: file '$jarName' in module '$id'"))
+
+        case version +: rest =>
+          log.debug(s"Fetching version-specific sources: ${id}_$version")
+          val moduleName = s"${id}_$version"
+          val jarName = s"${srcID}_$version-$sbtVersion.jar"
+          update(getModule(moduleName))(_.getName == jarName) getOrElse interfaceSources(rest)
+      }
+    IO.withTemporaryDirectory { binaryDirectory =>
+
+      val targetJar = new File(binaryDirectory, s"$binID.jar")
+      val xsbtiJars = manager.files(xsbtiID)(IfMissing.Fail)
+
+      val sourceModuleVersions = cascadingSourceModuleVersions(compiler.scalaInstance.actualVersion)
+      AnalyzingCompiler.compileSources(Seq(interfaceSources(sourceModuleVersions)), targetJar, xsbtiJars, id, compiler, log)
+
+      manager.define(binID, Seq(targetJar))
+
+    }
+  }
+
+  /**
+   * Returns a dummy module that depends on "org.scala-sbt" % `id` % `sbtVersion`.
+   * Note: Sbt's implementation of Ivy requires us to do this, because only the dependencies
+   *       of the specified module will be downloaded.
+   */
+  private def getModule(id: String): ivySbt.Module = {
+    val dummyID = ModuleID(xsbti.ArtifactInfo.SbtOrganization, scala.util.Random.alphanumeric take 20 mkString "", sbtVersion, Some("component"))
+    val moduleID = ModuleID(xsbti.ArtifactInfo.SbtOrganization, id, sbtVersion, Some("component"))
+    getModule(dummyID, Seq(moduleID))
+  }
+
+  private def getModule(moduleID: ModuleID, deps: Seq[ModuleID], uo: UpdateOptions = UpdateOptions()): ivySbt.Module = {
+    val moduleSetting = InlineConfiguration(
+      module = moduleID,
+      moduleInfo = ModuleInfo(moduleID.name),
+      dependencies = deps,
+      configurations = Seq(Configurations.Component),
+      ivyScala = None)
+
+    new ivySbt.Module(moduleSetting)
+  }
+
+  private def dependenciesNames(module: ivySbt.Module): String = module.moduleSettings match {
+    // `module` is a dummy module, we will only fetch its dependencies.
+    case ic: InlineConfiguration =>
+      ic.dependencies map {
+        case mID: ModuleID =>
+          import mID._
+          s"$organization % $name % $revision"
+      } mkString ", "
+    case _ =>
+      s"unknown"
+  }
+
+  private def update(module: ivySbt.Module)(predicate: File => Boolean): Option[File] = {
+
+    val retrieveDirectory = new File(ivyConfiguration.baseDirectory, "component")
+    val retrieveConfiguration = new RetrieveConfiguration(retrieveDirectory, Resolver.defaultRetrievePattern, false)
+    val updateConfiguration = new UpdateConfiguration(Some(retrieveConfiguration), true, UpdateLogging.DownloadOnly)
+
+    log.info(s"Attempting to fetch ${dependenciesNames(module)}. This operation may fail.")
+    IvyActions.updateEither(module, updateConfiguration, UnresolvedWarningConfiguration(), LogicalClock.unknown, None, log) match {
+      case Left(unresolvedWarning) =>
+        None
+
+      case Right(updateReport) =>
+        val files =
+          for {
+            conf <- updateReport.configurations
+            m <- conf.modules
+            (_, f) <- m.artifacts
+            if predicate(f)
+          } yield f
+        files.headOption
+
+    }
+  }
+}

compile/ivy/src/test/sbt/compiler/CascadingVersionsSpecifications.scala

@@ -0,0 +1,50 @@
+package sbt
+package compiler
+
+import org.junit.runner.RunWith
+import org.specs2.mutable.Specification
+import org.specs2.runner.JUnitRunner
+
+@RunWith(classOf[JUnitRunner])
+class CascadingVersionsSpecification extends Specification {
+
+  "Cascading version numbers" should {
+
+    "be correctly generated from Scala release versions (X.Y.Z)" in {
+      val version = "2.11.7"
+      val cascadingVersions = ComponentCompiler.cascadingSourceModuleVersions(version)
+
+      cascadingVersions === Seq("2.11.7", "2.11")
+    }
+
+    "be correctly generated from Scala RC, milestones, ... (X.Y.Z-...)" in {
+      val version = "2.12.0-M1"
+      val cascadingVersions = ComponentCompiler.cascadingSourceModuleVersions(version)
+
+      cascadingVersions === Seq("2.12.0-M1", "2.12.0", "2.12")
+    }
+
+    "be correctly generated from custom Scala builds (X.Y.Z-...)" in {
+      val version = "2.11.8-20150630-134856-c8fbc41631"
+      val cascadingVersions = ComponentCompiler.cascadingSourceModuleVersions(version)
+
+      cascadingVersions === Seq("2.11.8-20150630-134856-c8fbc41631", "2.11.8", "2.11")
+    }
+
+    "be correctly generated for unrecognizable versions" in {
+      val version = "some unintelligible format"
+      val cascadingVersions = ComponentCompiler.cascadingSourceModuleVersions(version)
+
+      cascadingVersions === Seq(version)
+    }
+
+    "not recognize version numbers such as X.Y.Z-" in {
+      val version = "2.11.7-"
+      val cascadingVersions = ComponentCompiler.cascadingSourceModuleVersions(version)
+
+      cascadingVersions === Seq(version)
+    }
+
+  }
+
+}

ivy/src/main/scala/sbt/Configuration.scala

@@ -44,6 +44,7 @@ object Configurations {
 
   lazy val ScalaTool = config("scala-tool") hide
   lazy val CompilerPlugin = config("plugin") hide
+  lazy val Component = config("component") hide
 
   private[sbt] val DefaultMavenConfiguration = defaultConfiguration(true)
   private[sbt] val DefaultIvyConfiguration = defaultConfiguration(false)

main/actions/src/main/scala/sbt/Compiler.scala

@@ -54,6 +54,7 @@ object Compiler {
   def compilers(instance: ScalaInstance, cpOptions: ClasspathOptions)(implicit app: AppConfiguration, log: Logger): Compilers =
     compilers(instance, cpOptions, None)
 
+  @deprecated("Please specify `ivyConfiguration`.", "0.13.10")
   def compilers(instance: ScalaInstance, cpOptions: ClasspathOptions, javaHome: Option[File])(implicit app: AppConfiguration, log: Logger): Compilers =
     {
       val javac =
@@ -68,6 +69,20 @@ object Compiler {
       }
       compilers(instance, cpOptions, CheaterJavaTool(javac2, javac))
     }
+  def compilers(instance: ScalaInstance, cpOptions: ClasspathOptions, javaHome: Option[File], ivyConfiguration: IvyConfiguration)(implicit app: AppConfiguration, log: Logger): Compilers =
+    {
+      val javac =
+        AggressiveCompile.directOrFork(instance, cpOptions, javaHome)
+      val javac2 =
+        JavaTools.directOrFork(instance, cpOptions, javaHome)
+      // Hackery to enable both the new and deprecated APIs to coexist peacefully.
+      case class CheaterJavaTool(newJavac: IncrementalCompilerJavaTools, delegate: JavaTool) extends JavaTool with JavaToolWithNewInterface {
+        def compile(contract: JavacContract, sources: Seq[File], classpath: Seq[File], outputDirectory: File, options: Seq[String])(implicit log: Logger): Unit =
+          javac.compile(contract, sources, classpath, outputDirectory, options)(log)
+        def onArgs(f: Seq[String] => Unit): JavaTool = CheaterJavaTool(newJavac, delegate.onArgs(f))
+      }
+      compilers(instance, cpOptions, CheaterJavaTool(javac2, javac), ivyConfiguration)
+    }
   @deprecated("Deprecated in favor of new sbt.compiler.javac package.", "0.13.8")
   def compilers(instance: ScalaInstance, cpOptions: ClasspathOptions, javac: sbt.compiler.JavaCompiler.Fork)(implicit app: AppConfiguration, log: Logger): Compilers =
     {
@@ -80,6 +95,13 @@ object Compiler {
       val scalac = scalaCompiler(instance, cpOptions)
       new Compilers(scalac, javac)
     }
+  @deprecated("Deprecated in favor of new sbt.compiler.javac package.", "0.13.8")
+  def compilers(instance: ScalaInstance, cpOptions: ClasspathOptions, javac: JavaTool, ivyConfiguration: IvyConfiguration)(implicit app: AppConfiguration, log: Logger): Compilers =
+    {
+      val scalac = scalaCompiler(instance, cpOptions, ivyConfiguration)
+      new Compilers(scalac, javac)
+    }
+  @deprecated("Please provide an IvyConfiguration", "0.13.10")
   def scalaCompiler(instance: ScalaInstance, cpOptions: ClasspathOptions)(implicit app: AppConfiguration, log: Logger): AnalyzingCompiler =
     {
       val launcher = app.provider.scalaProvider.launcher
@@ -87,6 +109,13 @@ object Compiler {
       val provider = ComponentCompiler.interfaceProvider(componentManager)
       new AnalyzingCompiler(instance, provider, cpOptions)
     }
+  def scalaCompiler(instance: ScalaInstance, cpOptions: ClasspathOptions, ivyConfiguration: IvyConfiguration)(implicit app: AppConfiguration, log: Logger): AnalyzingCompiler =
+    {
+      val launcher = app.provider.scalaProvider.launcher
+      val componentManager = new ComponentManager(launcher.globalLock, app.provider.components, Option(launcher.ivyHome), log)
+      val provider = ComponentCompiler.interfaceProvider(componentManager, ivyConfiguration)
+      new AnalyzingCompiler(instance, provider, cpOptions)
+    }
 
   @deprecated("Use the `compile` method instead.", "0.13.8")
   def apply(in: Inputs, log: Logger): Analysis = {

main/src/main/scala/sbt/Defaults.scala

@@ -261,7 +261,7 @@ object Defaults extends BuildCommon {
       if (plugin) scalaBase / ("sbt-" + sbtv) else scalaBase
     }
 
-  def compilersSetting = compilers := Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value)(appConfiguration.value, streams.value.log)
+  def compilersSetting = compilers := Compiler.compilers(scalaInstance.value, classpathOptions.value, javaHome.value, ivyConfiguration.value)(appConfiguration.value, streams.value.log)
 
   lazy val configTasks = docTaskSettings(doc) ++ inTask(compile)(compileInputsSettings) ++ configGlobal ++ compileAnalysisSettings ++ Seq(
     compile <<= compileTask,