Introduce incremental compiler options.

Introduce a way to configure incremental compiler itself instead
of underlying Java/Scala compiler.

Specific list of changes in this commit:
  * Add a method to `xsbti.compile.Setup` that returns incremental
    compiler options as a `java.util.Map<String, String>`. We considered
    statis interface instead of a `Map` but based on mailing
    list feedback we decided that it's not the best way to go because
    static interface is hard to evolve it by adding new options.
  * Since passing `java.util.Map<String, String>` not very convenient
    we convert it immediately to `sbt.inc.IncOptions`
  * Add options argument to various methods/classes that implement
    incremental compilation so in the end options reach
    `sbt.inc.IncOptions` object
  * Add `incOptions` task that allows users to configure incremental
    compiler options in their build files. Default implementation of
    that tasks returns just `IncOptions.DEFAULT`
  * Both system property `xsbt.inc.debug` and `IncOptions.relationsDebug`
    trigger debugging of relations now. In the near future, we should
    deprecate use of `xsbt.inc.debug`.

compile/inc/src/main/scala/sbt/inc/Compile.scala

@@ -16,12 +16,13 @@ object IncrementalCompile
 	    compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit,
 	    previous: Analysis,
 	    forEntry: File => Option[Analysis],
-	    output: Output, log: Logger): (Boolean, Analysis) =
+	    output: Output, log: Logger,
+	    options: IncOptions): (Boolean, Analysis) =
 	{
 		val current = Stamps.initial(Stamp.exists, Stamp.hash, Stamp.lastModified)
 		val internalMap = (f: File) => previous.relations.produced(f).headOption
 		val externalAPI = getExternalAPI(entry, forEntry)
-		Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log)
+		Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log, options)
 	}
 	def doCompile(compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], externalAPI: (File, String) => Option[Source], current: ReadStamps, output: Output) = (srcs: Set[File], changes: DependencyChanges) => {
 		val callback = new AnalysisCallback(internalMap, externalAPI, current, output)

compile/inc/src/main/scala/sbt/inc/IncOptions.scala

@@ -0,0 +1,87 @@
+package sbt.inc
+
+/**
+ * Case class that represents all configuration options for incremental compiler.
+ *
+ * Those are options that configure incremental compiler itself and not underlying
+ * Java/Scala compiler.
+ */
+case class IncOptions(
+    /** After which step include whole transitive closure of invalidated source files. */
+    val transitiveStep: Int,
+    /**
+     * What's the fraction of invalidated source files when we switch to recompiling
+     * all files and giving up incremental compilation altogether. That's useful in
+     * cases when probability that we end up recompiling most of source files but
+     * in multiple steps is high. Multi-step incremental recompilation is slower
+     * than recompiling everything in one step.
+     */
+    val recompileAllFraction: Double,
+    /** Print very detail information about relations (like dependencies between source files). */
+    val relationsDebug: Boolean,
+    /**
+     * Enable tools for debugging API changes. At the moment that option is unused but in the
+     * future it will enable for example:
+     *   - disabling API hashing and API minimization (potentially very memory consuming)
+     *   - dumping textual API representation into files
+     */
+    val apiDebug: Boolean,
+    /**
+     * The directory where we dump textual representation of APIs. This method might be called
+     * only if apiDebug returns true. This is unused option at the moment as the needed functionality
+     * is not implemented yet.
+     */
+    val apiDumpDirectory: Option[java.io.File])
+
+object IncOptions {
+	val Default = IncOptions(
+		transitiveStep = 2,
+		recompileAllFraction = 0.5,
+		relationsDebug = false,
+		apiDebug = false,
+		apiDumpDirectory = None)
+
+	val transitiveStepKey       = "transitiveStep"
+	val recompileAllFractionKey = "recompileAllFraction"
+	val relationsDebugKey       = "relationsDebug"
+	val apiDebugKey             = "apiDebug"
+	val apiDumpDirectoryKey     = "apiDumpDirectory"
+
+	def fromStringMap(m: java.util.Map[String, String]): IncOptions = {
+		// all the code below doesn't look like idiomatic Scala for a good reason: we are working with Java API
+		def getTransitiveStep: Int = {
+			val k = transitiveStepKey
+			if (m.containsKey(k)) m.get(k).toInt else Default.transitiveStep
+		}
+		def getRecompileAllFraction: Double = {
+			val k = recompileAllFractionKey
+			if (m.containsKey(k)) m.get(k).toDouble else Default.recompileAllFraction
+		}
+		def getRelationsDebug: Boolean = {
+			val k = relationsDebugKey
+			if (m.containsKey(k)) m.get(k).toBoolean else Default.relationsDebug
+		}
+		def getApiDebug: Boolean = {
+			val k = apiDebugKey
+			if (m.containsKey(k)) m.get(k).toBoolean else Default.apiDebug
+		}
+		def getApiDumpDirectory: Option[java.io.File] = {
+			val k = apiDumpDirectoryKey
+			if (m.containsKey(k))
+				Some(new java.io.File(m.get(k)))
+			else None
+		}
+
+		IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDumpDirectory)
+	}
+
+	def toStringMap(o: IncOptions): java.util.Map[String, String] = {
+		val m = new java.util.HashMap[String, String]
+		m.put(transitiveStepKey, o.transitiveStep.toString)
+		m.put(recompileAllFractionKey, o.recompileAllFraction.toString)
+		m.put(relationsDebugKey, o.relationsDebug.toString)
+		m.put(apiDebugKey, o.apiDebug.toString)
+		o.apiDumpDirectory.foreach(f => m.put(apiDumpDirectoryKey, f.toString))
+		m
+	}
+}

compile/inc/src/main/scala/sbt/inc/Incremental.scala

@@ -22,9 +22,10 @@ object Incremental
 	    current: ReadStamps,
 	    forEntry: File => Option[Analysis],
 	    doCompile: (Set[File], DependencyChanges) => Analysis,
-	    log: Logger)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) =
+	    log: Logger,
+	    options: IncOptions)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) =
 	{
-		val initialChanges = changedInitial(entry, sources, previous, current, forEntry)
+		val initialChanges = changedInitial(entry, sources, previous, current, forEntry, options)
 		val binaryChanges = new DependencyChanges {
 			val modifiedBinaries = initialChanges.binaryDeps.toArray
 			val modifiedClasses = initialChanges.external.modified.toArray
@@ -32,20 +33,21 @@ object Incremental
 		}
 		val initialInv = invalidateInitial(previous.relations, initialChanges, log)
 		log.debug("Initially invalidated: " + initialInv)
-		val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log)
+		val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log, options)
 		(!initialInv.isEmpty, analysis)
 	}
 
 	val incDebugProp = "xsbt.inc.debug"
+	private def incDebug(options: IncOptions): Boolean = options.relationsDebug || java.lang.Boolean.getBoolean(incDebugProp)
 	// TODO: the Analysis for the last successful compilation should get returned + Boolean indicating success
 	// TODO: full external name changes, scopeInvalidations
 	def cycle(invalidatedRaw: Set[File], allSources: Set[File], binaryChanges: DependencyChanges, previous: Analysis,
-	    doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger): Analysis =
+	    doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger, options: IncOptions): Analysis =
 		if(invalidatedRaw.isEmpty)
 			previous
 		else
 		{
-			def debug(s: => String) = if(java.lang.Boolean.getBoolean(incDebugProp)) log.debug(s) else ()
+			def debug(s: => String) = if (incDebug(options)) log.debug(s) else ()
 			val withPackageObjects = invalidatedRaw ++ invalidatedPackageObjects(invalidatedRaw, previous.relations)
 			val invalidated = expand(withPackageObjects, allSources, log)
 			val pruned = prune(invalidated, previous)
@@ -54,10 +56,10 @@ object Incremental
 			debug("********* Fresh: \n" + fresh.relations + "\n*********")
 			val merged = pruned ++ fresh//.copy(relations = pruned.relations ++ fresh.relations, apis = pruned.apis ++ fresh.apis)
 			debug("********* Merged: \n" + merged.relations + "\n*********")
-			val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _)
+			val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, options)
 			debug("Changes:\n" + incChanges)
 			val incInv = invalidateIncremental(merged.relations, incChanges, invalidated, cycleNum >= TransitiveStep, log)
-			cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log)
+			cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log, options)
 		}
 	private[this] def emptyChanges: DependencyChanges = new DependencyChanges {
 		val modifiedBinaries = new Array[File](0)
@@ -81,7 +83,7 @@ object Incremental
 	* providing the API before and after the last step.  The functions should return
 	* an empty API if the file did not/does not exist.
 	*/
-	def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source): APIChanges[T] =
+	def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source, options: IncOptions): APIChanges[T] =
 	{
 		val oldApis = lastSources.toSeq map oldAPI
 		val newApis = lastSources.toSeq map newAPI
@@ -104,7 +106,8 @@ object Incremental
 		case (co1, co2) => co1.sourceDirectory == co2.sourceDirectory && co1.outputDirectory == co2.outputDirectory
   }
 
-	def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps, forEntry: File => Option[Analysis])(implicit equivS: Equiv[Stamp]): InitialChanges =
+	def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps,
+	    forEntry: File => Option[Analysis], options: IncOptions)(implicit equivS: Equiv[Stamp]): InitialChanges =
 	{
 		val previous = previousAnalysis.stamps
 		val previousAPIs = previousAnalysis.apis
@@ -112,7 +115,7 @@ object Incremental
 		val srcChanges = changes(previous.allInternalSources.toSet, sources,  f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) )
 		val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet
 		val binaryDepChanges = previous.allBinaries.filter( externalBinaryModified(entry, forEntry, previous, current)).toSet
-		val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry))
+		val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry), options)
 
 		InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges )
 	}

compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala

@@ -11,6 +11,7 @@ import inc._
 	import classpath.ClasspathUtilities
 	import classfile.Analyze
 	import inc.Locate.DefinesClass
+	import inc.IncOptions
 	import CompileSetup._
 	import sbinary.DefaultProtocol.{ immutableMapFormat, immutableSetFormat, StringFormat }
 
@@ -21,7 +22,7 @@ import inc._
 
 final class CompileConfiguration(val sources: Seq[File], val classpath: Seq[File],
 	val previousAnalysis: Analysis, val previousSetup: Option[CompileSetup], val currentSetup: CompileSetup, val progress: Option[CompileProgress], val getAnalysis: File => Option[Analysis], val definesClass: DefinesClass,
-	val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache)
+	val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache, val incOptions: IncOptions)
 
 class AggressiveCompile(cacheFile: File)
 {
@@ -37,11 +38,12 @@ class AggressiveCompile(cacheFile: File)
 	    definesClass: DefinesClass = Locate.definesClass _,
 	    reporter: Reporter,
 	    compileOrder: CompileOrder = Mixed,
-	    skip: Boolean = false)(implicit log: Logger): Analysis =
+	    skip: Boolean = false,
+	    incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis =
 	{
 		val setup = new CompileSetup(output, new CompileOptions(options, javacOptions), compiler.scalaInstance.actualVersion, compileOrder)
 		compile1(sources, classpath, setup, progress, store, analysisMap, definesClass,
-		    compiler, javac, reporter, skip, cache)
+		    compiler, javac, reporter, skip, cache, incrementalCompilerOptions)
 	}
 
 	def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] =
@@ -56,14 +58,15 @@ class AggressiveCompile(cacheFile: File)
 		compiler: AnalyzingCompiler,
 		javac: xsbti.compile.JavaCompiler,
 		reporter: Reporter, skip: Boolean,
-		cache: GlobalsCache)(implicit log: Logger): Analysis =
+		cache: GlobalsCache,
+		incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis =
 	{
 		val (previousAnalysis, previousSetup) = extract(store.get())
 		if(skip)
 			previousAnalysis
 		else {
 			val config = new CompileConfiguration(sources, classpath, previousAnalysis, previousSetup, setup,
-			    progress, analysis, definesClass, reporter, compiler, javac, cache)
+			    progress, analysis, definesClass, reporter, compiler, javac, cache, incrementalCompilerOptions)
 			val (modified, result) = compile2(config)
 			if(modified)
 				store.set(result, setup)
@@ -140,7 +143,7 @@ class AggressiveCompile(cacheFile: File)
 			case Some(previous) if equiv.equiv(previous, currentSetup) => previousAnalysis
 			case _ => Incremental.prune(sourcesSet, previousAnalysis)
 		}
-		IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log)
+		IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log, incOptions)
 	}
 	private[this] def outputDirectories(output: Output): Seq[File] = output match {
 		case single: SingleOutput => List(single.outputDirectory)

compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala

@@ -1,7 +1,7 @@
 package sbt.compiler
 
 	import sbt.CompileSetup
-	import sbt.inc.Analysis
+	import sbt.inc.{Analysis, IncOptions}
 	import xsbti.{Logger, Maybe}
 	import xsbti.compile._
 
@@ -17,7 +17,9 @@ object IC extends IncrementalCompiler[Analysis, AnalyzingCompiler]
 		val agg = new AggressiveCompile(setup.cacheFile)
 		val aMap = (f: File) => m2o(analysisMap(f))
 		val defClass = (f: File) => { val dc = definesClass(f); (name: String) => dc.apply(name) }
-		agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap, defClass, reporter, order, skip)(log)
+		val incOptions = IncOptions.fromStringMap(incrementalCompilerOptions)
+		agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap,
+		    defClass, reporter, order, skip, incOptions)(log)
 	}
 
 	private[this] def m2o[S](opt: Maybe[S]): Option[S] = if(opt.isEmpty) None else Some(opt.get)

interface/src/main/java/xsbti/compile/Setup.java

@@ -1,6 +1,8 @@
 package xsbti.compile;
 
 import java.io.File;
+import java.util.Map;
+
 import xsbti.Maybe;
 import xsbti.Reporter;
 
@@ -30,4 +32,16 @@ public interface Setup<Analysis>
 
 	/** The reporter that should be used to report scala compilation to. */
 	Reporter reporter();
+
+	/**
+	 * Returns incremental compiler options.
+	 *
+	 * @see sbt.inc.IncOptions for details
+	 *
+	 * You can get default options by calling <code>sbt.inc.IncOptions.toStringMap(sbt.inc.IncOptions.Default)</code>.
+	 *
+	 * In the future, we'll extend API in <code>xsbti</code> to provide factory methods that would allow to obtain
+	 * defaults values so one can depend on <code>xsbti</code> package only.
+	 **/
+	Map<String, String> incrementalCompilerOptions();
 }

main/actions/src/main/scala/sbt/Compiler.scala

@@ -17,7 +17,7 @@ object Compiler
 
 	final case class Inputs(compilers: Compilers, config: Options, incSetup: IncSetup)
 	final case class Options(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMapper: Position => Position, order: CompileOrder)
-	final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache)
+	final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache, incOptions: IncOptions)
 	final case class Compilers(scalac: AnalyzingCompiler, javac: JavaTool)
 
 	@deprecated("Use the other inputs variant.", "0.12.0")
@@ -27,7 +27,7 @@ object Compiler
 		val classesDirectory = outputDirectory / "classes"
 		val cacheFile = outputDirectory / "cache_old_style"
 		val augClasspath = classesDirectory.asFile +: classpath
-		val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh)
+		val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh, IncOptions.Default)
 		inputs(augClasspath, sources, classesDirectory, options, javacOptions, maxErrors, Nil, order)(compilers, incSetup, log)
 	}
 	def inputs(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMappers: Seq[Position => Option[Position]], order: CompileOrder)(implicit compilers: Compilers, incSetup: IncSetup, log: Logger): Inputs =
@@ -77,7 +77,7 @@ object Compiler
 
 		val agg = new AggressiveCompile(cacheFile)
 		agg(scalac, javac, sources, classpath, CompileOutput(classesDirectory), cache, None, options, javacOptions,
-		    analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip)(log)
+		    analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip, incOptions)(log)
 	}
 
 	private[sbt] def foldMappers[A](mappers: Seq[A => Option[A]]) =

main/src/main/scala/sbt/Defaults.scala

@@ -204,6 +204,7 @@ object Defaults extends BuildCommon
 		compilersSetting,
 		javacOptions in GlobalScope :== Nil,
 		scalacOptions in GlobalScope :== Nil,
+		incOptions in GlobalScope :== sbt.inc.IncOptions.Default,
 		scalaInstance <<= scalaInstanceTask,
 		scalaVersion in GlobalScope := appConfiguration.value.provider.scalaProvider.version,
 		scalaBinaryVersion in GlobalScope := binaryScalaVersion(scalaVersion.value),
@@ -650,8 +651,8 @@ object Defaults extends BuildCommon
 
 	def compileTask = (compileInputs in compile, streams) map { (i,s) => Compiler(i,s.log) }
 	def compileIncSetupTask =
-		(dependencyClasspath, skip in compile, definesClass, compilerCache, streams) map { (cp, skip, definesC, cache, s) =>
+		(dependencyClasspath, skip in compile, definesClass, compilerCache, streams, incOptions) map { (cp, skip, definesC, cache, s, incOptions) =>
-			Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache)
+			Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache, incOptions)
 		}
 	def compileInputsSettings: Seq[Setting[_]] =
 		Seq(compileInputs := {
@@ -1024,7 +1025,7 @@ object Classpaths
 		val si = Defaults.unmanagedScalaInstanceOnly.value.map(si => (si, scalaOrganization.value))
 		val show = Reference.display(thisProjectRef.value)
 		cachedUpdate(s.cacheDirectory, show, ivyModule.value, updateConfiguration.value, si, skip = (skip in update).value, force = isRoot, depsUpdated = depsUpdated, log = s.log)
-	} 
+	}
 
 	def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, scalaInstance: Option[(ScalaInstance, String)], skip: Boolean, force: Boolean, depsUpdated: Boolean, log: Logger): UpdateReport =
 	{

main/src/main/scala/sbt/Keys.scala

@@ -126,6 +126,7 @@ object Keys
 	val maxErrors = SettingKey[Int]("max-errors", "The maximum number of errors, such as compile errors, to list.", ASetting)
 	val scalacOptions = TaskKey[Seq[String]]("scalac-options", "Options for the Scala compiler.", BPlusTask)
 	val javacOptions = TaskKey[Seq[String]]("javac-options", "Options for the Java compiler.", BPlusTask)
+	val incOptions = TaskKey[sbt.inc.IncOptions]("inc-options", "Options for the incremental compiler.", BTask)
 	val compileOrder = SettingKey[CompileOrder]("compile-order", "Configures the order in which Java and sources within a single compilation are compiled.  Valid values are: JavaThenScala, ScalaThenJava, or Mixed.", BPlusSetting)
 	val initialCommands = SettingKey[String]("initial-commands", "Initial commands to execute when starting up the Scala interpreter.", AMinusSetting)
 	val cleanupCommands = SettingKey[String]("cleanup-commands", "Commands to execute before the Scala interpreter exits.", BMinusSetting)