Introduce incremental compiler options.

Introduce a way to configure incremental compiler itself instead
of underlying Java/Scala compiler.

Specific list of changes in this commit:
  * Add a method to `xsbti.compile.Setup` that returns incremental
    compiler options as a `java.util.Map<String, String>`. We considered
    statis interface instead of a `Map` but based on mailing
    list feedback we decided that it's not the best way to go because
    static interface is hard to evolve it by adding new options.
  * Since passing `java.util.Map<String, String>` not very convenient
    we convert it immediately to `sbt.inc.IncOptions`
  * Add options argument to various methods/classes that implement
    incremental compilation so in the end options reach
    `sbt.inc.IncOptions` object
  * Add `incOptions` task that allows users to configure incremental
    compiler options in their build files. Default implementation of
    that tasks returns just `IncOptions.DEFAULT`
  * Both system property `xsbt.inc.debug` and `IncOptions.relationsDebug`
    trigger debugging of relations now. In the near future, we should
    deprecate use of `xsbt.inc.debug`.
This commit is contained in:
Grzegorz Kossakowski 2013-02-19 00:16:51 -08:00 committed by Mark Harrah
parent 4fe0c02536
commit 70036812ab
9 changed files with 138 additions and 26 deletions

View File

@ -16,12 +16,13 @@ object IncrementalCompile
compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit,
previous: Analysis,
forEntry: File => Option[Analysis],
output: Output, log: Logger): (Boolean, Analysis) =
output: Output, log: Logger,
options: IncOptions): (Boolean, Analysis) =
{
val current = Stamps.initial(Stamp.exists, Stamp.hash, Stamp.lastModified)
val internalMap = (f: File) => previous.relations.produced(f).headOption
val externalAPI = getExternalAPI(entry, forEntry)
Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log)
Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log, options)
}
def doCompile(compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], externalAPI: (File, String) => Option[Source], current: ReadStamps, output: Output) = (srcs: Set[File], changes: DependencyChanges) => {
val callback = new AnalysisCallback(internalMap, externalAPI, current, output)

View File

@ -0,0 +1,87 @@
package sbt.inc
/**
* Case class that represents all configuration options for incremental compiler.
*
* Those are options that configure incremental compiler itself and not underlying
* Java/Scala compiler.
*/
case class IncOptions(
/** After which step include whole transitive closure of invalidated source files. */
val transitiveStep: Int,
/**
* What's the fraction of invalidated source files when we switch to recompiling
* all files and giving up incremental compilation altogether. That's useful in
* cases when probability that we end up recompiling most of source files but
* in multiple steps is high. Multi-step incremental recompilation is slower
* than recompiling everything in one step.
*/
val recompileAllFraction: Double,
/** Print very detail information about relations (like dependencies between source files). */
val relationsDebug: Boolean,
/**
* Enable tools for debugging API changes. At the moment that option is unused but in the
* future it will enable for example:
* - disabling API hashing and API minimization (potentially very memory consuming)
* - dumping textual API representation into files
*/
val apiDebug: Boolean,
/**
* The directory where we dump textual representation of APIs. This method might be called
* only if apiDebug returns true. This is unused option at the moment as the needed functionality
* is not implemented yet.
*/
val apiDumpDirectory: Option[java.io.File])
object IncOptions {
val Default = IncOptions(
transitiveStep = 2,
recompileAllFraction = 0.5,
relationsDebug = false,
apiDebug = false,
apiDumpDirectory = None)
val transitiveStepKey = "transitiveStep"
val recompileAllFractionKey = "recompileAllFraction"
val relationsDebugKey = "relationsDebug"
val apiDebugKey = "apiDebug"
val apiDumpDirectoryKey = "apiDumpDirectory"
def fromStringMap(m: java.util.Map[String, String]): IncOptions = {
// all the code below doesn't look like idiomatic Scala for a good reason: we are working with Java API
def getTransitiveStep: Int = {
val k = transitiveStepKey
if (m.containsKey(k)) m.get(k).toInt else Default.transitiveStep
}
def getRecompileAllFraction: Double = {
val k = recompileAllFractionKey
if (m.containsKey(k)) m.get(k).toDouble else Default.recompileAllFraction
}
def getRelationsDebug: Boolean = {
val k = relationsDebugKey
if (m.containsKey(k)) m.get(k).toBoolean else Default.relationsDebug
}
def getApiDebug: Boolean = {
val k = apiDebugKey
if (m.containsKey(k)) m.get(k).toBoolean else Default.apiDebug
}
def getApiDumpDirectory: Option[java.io.File] = {
val k = apiDumpDirectoryKey
if (m.containsKey(k))
Some(new java.io.File(m.get(k)))
else None
}
IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDumpDirectory)
}
def toStringMap(o: IncOptions): java.util.Map[String, String] = {
val m = new java.util.HashMap[String, String]
m.put(transitiveStepKey, o.transitiveStep.toString)
m.put(recompileAllFractionKey, o.recompileAllFraction.toString)
m.put(relationsDebugKey, o.relationsDebug.toString)
m.put(apiDebugKey, o.apiDebug.toString)
o.apiDumpDirectory.foreach(f => m.put(apiDumpDirectoryKey, f.toString))
m
}
}

View File

@ -22,9 +22,10 @@ object Incremental
current: ReadStamps,
forEntry: File => Option[Analysis],
doCompile: (Set[File], DependencyChanges) => Analysis,
log: Logger)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) =
log: Logger,
options: IncOptions)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) =
{
val initialChanges = changedInitial(entry, sources, previous, current, forEntry)
val initialChanges = changedInitial(entry, sources, previous, current, forEntry, options)
val binaryChanges = new DependencyChanges {
val modifiedBinaries = initialChanges.binaryDeps.toArray
val modifiedClasses = initialChanges.external.modified.toArray
@ -32,20 +33,21 @@ object Incremental
}
val initialInv = invalidateInitial(previous.relations, initialChanges, log)
log.debug("Initially invalidated: " + initialInv)
val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log)
val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log, options)
(!initialInv.isEmpty, analysis)
}
val incDebugProp = "xsbt.inc.debug"
private def incDebug(options: IncOptions): Boolean = options.relationsDebug || java.lang.Boolean.getBoolean(incDebugProp)
// TODO: the Analysis for the last successful compilation should get returned + Boolean indicating success
// TODO: full external name changes, scopeInvalidations
def cycle(invalidatedRaw: Set[File], allSources: Set[File], binaryChanges: DependencyChanges, previous: Analysis,
doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger): Analysis =
doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger, options: IncOptions): Analysis =
if(invalidatedRaw.isEmpty)
previous
else
{
def debug(s: => String) = if(java.lang.Boolean.getBoolean(incDebugProp)) log.debug(s) else ()
def debug(s: => String) = if (incDebug(options)) log.debug(s) else ()
val withPackageObjects = invalidatedRaw ++ invalidatedPackageObjects(invalidatedRaw, previous.relations)
val invalidated = expand(withPackageObjects, allSources, log)
val pruned = prune(invalidated, previous)
@ -54,10 +56,10 @@ object Incremental
debug("********* Fresh: \n" + fresh.relations + "\n*********")
val merged = pruned ++ fresh//.copy(relations = pruned.relations ++ fresh.relations, apis = pruned.apis ++ fresh.apis)
debug("********* Merged: \n" + merged.relations + "\n*********")
val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _)
val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, options)
debug("Changes:\n" + incChanges)
val incInv = invalidateIncremental(merged.relations, incChanges, invalidated, cycleNum >= TransitiveStep, log)
cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log)
cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log, options)
}
private[this] def emptyChanges: DependencyChanges = new DependencyChanges {
val modifiedBinaries = new Array[File](0)
@ -81,7 +83,7 @@ object Incremental
* providing the API before and after the last step. The functions should return
* an empty API if the file did not/does not exist.
*/
def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source): APIChanges[T] =
def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source, options: IncOptions): APIChanges[T] =
{
val oldApis = lastSources.toSeq map oldAPI
val newApis = lastSources.toSeq map newAPI
@ -104,7 +106,8 @@ object Incremental
case (co1, co2) => co1.sourceDirectory == co2.sourceDirectory && co1.outputDirectory == co2.outputDirectory
}
def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps, forEntry: File => Option[Analysis])(implicit equivS: Equiv[Stamp]): InitialChanges =
def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps,
forEntry: File => Option[Analysis], options: IncOptions)(implicit equivS: Equiv[Stamp]): InitialChanges =
{
val previous = previousAnalysis.stamps
val previousAPIs = previousAnalysis.apis
@ -112,7 +115,7 @@ object Incremental
val srcChanges = changes(previous.allInternalSources.toSet, sources, f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) )
val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet
val binaryDepChanges = previous.allBinaries.filter( externalBinaryModified(entry, forEntry, previous, current)).toSet
val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry))
val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry), options)
InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges )
}

View File

@ -11,6 +11,7 @@ import inc._
import classpath.ClasspathUtilities
import classfile.Analyze
import inc.Locate.DefinesClass
import inc.IncOptions
import CompileSetup._
import sbinary.DefaultProtocol.{ immutableMapFormat, immutableSetFormat, StringFormat }
@ -21,7 +22,7 @@ import inc._
final class CompileConfiguration(val sources: Seq[File], val classpath: Seq[File],
val previousAnalysis: Analysis, val previousSetup: Option[CompileSetup], val currentSetup: CompileSetup, val progress: Option[CompileProgress], val getAnalysis: File => Option[Analysis], val definesClass: DefinesClass,
val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache)
val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache, val incOptions: IncOptions)
class AggressiveCompile(cacheFile: File)
{
@ -37,11 +38,12 @@ class AggressiveCompile(cacheFile: File)
definesClass: DefinesClass = Locate.definesClass _,
reporter: Reporter,
compileOrder: CompileOrder = Mixed,
skip: Boolean = false)(implicit log: Logger): Analysis =
skip: Boolean = false,
incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis =
{
val setup = new CompileSetup(output, new CompileOptions(options, javacOptions), compiler.scalaInstance.actualVersion, compileOrder)
compile1(sources, classpath, setup, progress, store, analysisMap, definesClass,
compiler, javac, reporter, skip, cache)
compiler, javac, reporter, skip, cache, incrementalCompilerOptions)
}
def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] =
@ -56,14 +58,15 @@ class AggressiveCompile(cacheFile: File)
compiler: AnalyzingCompiler,
javac: xsbti.compile.JavaCompiler,
reporter: Reporter, skip: Boolean,
cache: GlobalsCache)(implicit log: Logger): Analysis =
cache: GlobalsCache,
incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis =
{
val (previousAnalysis, previousSetup) = extract(store.get())
if(skip)
previousAnalysis
else {
val config = new CompileConfiguration(sources, classpath, previousAnalysis, previousSetup, setup,
progress, analysis, definesClass, reporter, compiler, javac, cache)
progress, analysis, definesClass, reporter, compiler, javac, cache, incrementalCompilerOptions)
val (modified, result) = compile2(config)
if(modified)
store.set(result, setup)
@ -140,7 +143,7 @@ class AggressiveCompile(cacheFile: File)
case Some(previous) if equiv.equiv(previous, currentSetup) => previousAnalysis
case _ => Incremental.prune(sourcesSet, previousAnalysis)
}
IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log)
IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log, incOptions)
}
private[this] def outputDirectories(output: Output): Seq[File] = output match {
case single: SingleOutput => List(single.outputDirectory)

View File

@ -1,7 +1,7 @@
package sbt.compiler
import sbt.CompileSetup
import sbt.inc.Analysis
import sbt.inc.{Analysis, IncOptions}
import xsbti.{Logger, Maybe}
import xsbti.compile._
@ -17,7 +17,9 @@ object IC extends IncrementalCompiler[Analysis, AnalyzingCompiler]
val agg = new AggressiveCompile(setup.cacheFile)
val aMap = (f: File) => m2o(analysisMap(f))
val defClass = (f: File) => { val dc = definesClass(f); (name: String) => dc.apply(name) }
agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap, defClass, reporter, order, skip)(log)
val incOptions = IncOptions.fromStringMap(incrementalCompilerOptions)
agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap,
defClass, reporter, order, skip, incOptions)(log)
}
private[this] def m2o[S](opt: Maybe[S]): Option[S] = if(opt.isEmpty) None else Some(opt.get)

View File

@ -1,6 +1,8 @@
package xsbti.compile;
import java.io.File;
import java.util.Map;
import xsbti.Maybe;
import xsbti.Reporter;
@ -30,4 +32,16 @@ public interface Setup<Analysis>
/** The reporter that should be used to report scala compilation to. */
Reporter reporter();
/**
* Returns incremental compiler options.
*
* @see sbt.inc.IncOptions for details
*
* You can get default options by calling <code>sbt.inc.IncOptions.toStringMap(sbt.inc.IncOptions.Default)</code>.
*
* In the future, we'll extend API in <code>xsbti</code> to provide factory methods that would allow to obtain
* defaults values so one can depend on <code>xsbti</code> package only.
**/
Map<String, String> incrementalCompilerOptions();
}

View File

@ -17,7 +17,7 @@ object Compiler
final case class Inputs(compilers: Compilers, config: Options, incSetup: IncSetup)
final case class Options(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMapper: Position => Position, order: CompileOrder)
final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache)
final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache, incOptions: IncOptions)
final case class Compilers(scalac: AnalyzingCompiler, javac: JavaTool)
@deprecated("Use the other inputs variant.", "0.12.0")
@ -27,7 +27,7 @@ object Compiler
val classesDirectory = outputDirectory / "classes"
val cacheFile = outputDirectory / "cache_old_style"
val augClasspath = classesDirectory.asFile +: classpath
val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh)
val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh, IncOptions.Default)
inputs(augClasspath, sources, classesDirectory, options, javacOptions, maxErrors, Nil, order)(compilers, incSetup, log)
}
def inputs(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMappers: Seq[Position => Option[Position]], order: CompileOrder)(implicit compilers: Compilers, incSetup: IncSetup, log: Logger): Inputs =
@ -77,7 +77,7 @@ object Compiler
val agg = new AggressiveCompile(cacheFile)
agg(scalac, javac, sources, classpath, CompileOutput(classesDirectory), cache, None, options, javacOptions,
analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip)(log)
analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip, incOptions)(log)
}
private[sbt] def foldMappers[A](mappers: Seq[A => Option[A]]) =

View File

@ -204,6 +204,7 @@ object Defaults extends BuildCommon
compilersSetting,
javacOptions in GlobalScope :== Nil,
scalacOptions in GlobalScope :== Nil,
incOptions in GlobalScope :== sbt.inc.IncOptions.Default,
scalaInstance <<= scalaInstanceTask,
scalaVersion in GlobalScope := appConfiguration.value.provider.scalaProvider.version,
scalaBinaryVersion in GlobalScope := binaryScalaVersion(scalaVersion.value),
@ -650,8 +651,8 @@ object Defaults extends BuildCommon
def compileTask = (compileInputs in compile, streams) map { (i,s) => Compiler(i,s.log) }
def compileIncSetupTask =
(dependencyClasspath, skip in compile, definesClass, compilerCache, streams) map { (cp, skip, definesC, cache, s) =>
Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache)
(dependencyClasspath, skip in compile, definesClass, compilerCache, streams, incOptions) map { (cp, skip, definesC, cache, s, incOptions) =>
Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache, incOptions)
}
def compileInputsSettings: Seq[Setting[_]] =
Seq(compileInputs := {
@ -1024,7 +1025,7 @@ object Classpaths
val si = Defaults.unmanagedScalaInstanceOnly.value.map(si => (si, scalaOrganization.value))
val show = Reference.display(thisProjectRef.value)
cachedUpdate(s.cacheDirectory, show, ivyModule.value, updateConfiguration.value, si, skip = (skip in update).value, force = isRoot, depsUpdated = depsUpdated, log = s.log)
}
}
def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, scalaInstance: Option[(ScalaInstance, String)], skip: Boolean, force: Boolean, depsUpdated: Boolean, log: Logger): UpdateReport =
{

View File

@ -126,6 +126,7 @@ object Keys
val maxErrors = SettingKey[Int]("max-errors", "The maximum number of errors, such as compile errors, to list.", ASetting)
val scalacOptions = TaskKey[Seq[String]]("scalac-options", "Options for the Scala compiler.", BPlusTask)
val javacOptions = TaskKey[Seq[String]]("javac-options", "Options for the Java compiler.", BPlusTask)
val incOptions = TaskKey[sbt.inc.IncOptions]("inc-options", "Options for the incremental compiler.", BTask)
val compileOrder = SettingKey[CompileOrder]("compile-order", "Configures the order in which Java and sources within a single compilation are compiled. Valid values are: JavaThenScala, ScalaThenJava, or Mixed.", BPlusSetting)
val initialCommands = SettingKey[String]("initial-commands", "Initial commands to execute when starting up the Scala interpreter.", AMinusSetting)
val cleanupCommands = SettingKey[String]("cleanup-commands", "Commands to execute before the Scala interpreter exits.", BMinusSetting)