From 70036812ab22c0b5576cd397c02ce27e86dfd4cc Mon Sep 17 00:00:00 2001 From: Grzegorz Kossakowski Date: Tue, 19 Feb 2013 00:16:51 -0800 Subject: [PATCH] Introduce incremental compiler options. Introduce a way to configure incremental compiler itself instead of underlying Java/Scala compiler. Specific list of changes in this commit: * Add a method to `xsbti.compile.Setup` that returns incremental compiler options as a `java.util.Map`. We considered statis interface instead of a `Map` but based on mailing list feedback we decided that it's not the best way to go because static interface is hard to evolve it by adding new options. * Since passing `java.util.Map` not very convenient we convert it immediately to `sbt.inc.IncOptions` * Add options argument to various methods/classes that implement incremental compilation so in the end options reach `sbt.inc.IncOptions` object * Add `incOptions` task that allows users to configure incremental compiler options in their build files. Default implementation of that tasks returns just `IncOptions.DEFAULT` * Both system property `xsbt.inc.debug` and `IncOptions.relationsDebug` trigger debugging of relations now. In the near future, we should deprecate use of `xsbt.inc.debug`. --- .../inc/src/main/scala/sbt/inc/Compile.scala | 5 +- .../src/main/scala/sbt/inc/IncOptions.scala | 87 +++++++++++++++++++ .../src/main/scala/sbt/inc/Incremental.scala | 23 ++--- .../sbt/compiler/AggressiveCompile.scala | 15 ++-- .../sbt/compiler/IncrementalCompiler.scala | 6 +- .../src/main/java/xsbti/compile/Setup.java | 14 +++ .../actions/src/main/scala/sbt/Compiler.scala | 6 +- main/src/main/scala/sbt/Defaults.scala | 7 +- main/src/main/scala/sbt/Keys.scala | 1 + 9 files changed, 138 insertions(+), 26 deletions(-) create mode 100644 compile/inc/src/main/scala/sbt/inc/IncOptions.scala diff --git a/compile/inc/src/main/scala/sbt/inc/Compile.scala b/compile/inc/src/main/scala/sbt/inc/Compile.scala index 582008cac..bba027cdc 100644 --- a/compile/inc/src/main/scala/sbt/inc/Compile.scala +++ b/compile/inc/src/main/scala/sbt/inc/Compile.scala @@ -16,12 +16,13 @@ object IncrementalCompile compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit, previous: Analysis, forEntry: File => Option[Analysis], - output: Output, log: Logger): (Boolean, Analysis) = + output: Output, log: Logger, + options: IncOptions): (Boolean, Analysis) = { val current = Stamps.initial(Stamp.exists, Stamp.hash, Stamp.lastModified) val internalMap = (f: File) => previous.relations.produced(f).headOption val externalAPI = getExternalAPI(entry, forEntry) - Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log) + Incremental.compile(sources, entry, previous, current, forEntry, doCompile(compile, internalMap, externalAPI, current, output), log, options) } def doCompile(compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback) => Unit, internalMap: File => Option[File], externalAPI: (File, String) => Option[Source], current: ReadStamps, output: Output) = (srcs: Set[File], changes: DependencyChanges) => { val callback = new AnalysisCallback(internalMap, externalAPI, current, output) diff --git a/compile/inc/src/main/scala/sbt/inc/IncOptions.scala b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala new file mode 100644 index 000000000..50f37b48e --- /dev/null +++ b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala @@ -0,0 +1,87 @@ +package sbt.inc + +/** + * Case class that represents all configuration options for incremental compiler. + * + * Those are options that configure incremental compiler itself and not underlying + * Java/Scala compiler. + */ +case class IncOptions( + /** After which step include whole transitive closure of invalidated source files. */ + val transitiveStep: Int, + /** + * What's the fraction of invalidated source files when we switch to recompiling + * all files and giving up incremental compilation altogether. That's useful in + * cases when probability that we end up recompiling most of source files but + * in multiple steps is high. Multi-step incremental recompilation is slower + * than recompiling everything in one step. + */ + val recompileAllFraction: Double, + /** Print very detail information about relations (like dependencies between source files). */ + val relationsDebug: Boolean, + /** + * Enable tools for debugging API changes. At the moment that option is unused but in the + * future it will enable for example: + * - disabling API hashing and API minimization (potentially very memory consuming) + * - dumping textual API representation into files + */ + val apiDebug: Boolean, + /** + * The directory where we dump textual representation of APIs. This method might be called + * only if apiDebug returns true. This is unused option at the moment as the needed functionality + * is not implemented yet. + */ + val apiDumpDirectory: Option[java.io.File]) + +object IncOptions { + val Default = IncOptions( + transitiveStep = 2, + recompileAllFraction = 0.5, + relationsDebug = false, + apiDebug = false, + apiDumpDirectory = None) + + val transitiveStepKey = "transitiveStep" + val recompileAllFractionKey = "recompileAllFraction" + val relationsDebugKey = "relationsDebug" + val apiDebugKey = "apiDebug" + val apiDumpDirectoryKey = "apiDumpDirectory" + + def fromStringMap(m: java.util.Map[String, String]): IncOptions = { + // all the code below doesn't look like idiomatic Scala for a good reason: we are working with Java API + def getTransitiveStep: Int = { + val k = transitiveStepKey + if (m.containsKey(k)) m.get(k).toInt else Default.transitiveStep + } + def getRecompileAllFraction: Double = { + val k = recompileAllFractionKey + if (m.containsKey(k)) m.get(k).toDouble else Default.recompileAllFraction + } + def getRelationsDebug: Boolean = { + val k = relationsDebugKey + if (m.containsKey(k)) m.get(k).toBoolean else Default.relationsDebug + } + def getApiDebug: Boolean = { + val k = apiDebugKey + if (m.containsKey(k)) m.get(k).toBoolean else Default.apiDebug + } + def getApiDumpDirectory: Option[java.io.File] = { + val k = apiDumpDirectoryKey + if (m.containsKey(k)) + Some(new java.io.File(m.get(k))) + else None + } + + IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDumpDirectory) + } + + def toStringMap(o: IncOptions): java.util.Map[String, String] = { + val m = new java.util.HashMap[String, String] + m.put(transitiveStepKey, o.transitiveStep.toString) + m.put(recompileAllFractionKey, o.recompileAllFraction.toString) + m.put(relationsDebugKey, o.relationsDebug.toString) + m.put(apiDebugKey, o.apiDebug.toString) + o.apiDumpDirectory.foreach(f => m.put(apiDumpDirectoryKey, f.toString)) + m + } +} diff --git a/compile/inc/src/main/scala/sbt/inc/Incremental.scala b/compile/inc/src/main/scala/sbt/inc/Incremental.scala index 811a0cc51..80fc7b6cc 100644 --- a/compile/inc/src/main/scala/sbt/inc/Incremental.scala +++ b/compile/inc/src/main/scala/sbt/inc/Incremental.scala @@ -22,9 +22,10 @@ object Incremental current: ReadStamps, forEntry: File => Option[Analysis], doCompile: (Set[File], DependencyChanges) => Analysis, - log: Logger)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) = + log: Logger, + options: IncOptions)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) = { - val initialChanges = changedInitial(entry, sources, previous, current, forEntry) + val initialChanges = changedInitial(entry, sources, previous, current, forEntry, options) val binaryChanges = new DependencyChanges { val modifiedBinaries = initialChanges.binaryDeps.toArray val modifiedClasses = initialChanges.external.modified.toArray @@ -32,20 +33,21 @@ object Incremental } val initialInv = invalidateInitial(previous.relations, initialChanges, log) log.debug("Initially invalidated: " + initialInv) - val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log) + val analysis = cycle(initialInv, sources, binaryChanges, previous, doCompile, 1, log, options) (!initialInv.isEmpty, analysis) } val incDebugProp = "xsbt.inc.debug" + private def incDebug(options: IncOptions): Boolean = options.relationsDebug || java.lang.Boolean.getBoolean(incDebugProp) // TODO: the Analysis for the last successful compilation should get returned + Boolean indicating success // TODO: full external name changes, scopeInvalidations def cycle(invalidatedRaw: Set[File], allSources: Set[File], binaryChanges: DependencyChanges, previous: Analysis, - doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger): Analysis = + doCompile: (Set[File], DependencyChanges) => Analysis, cycleNum: Int, log: Logger, options: IncOptions): Analysis = if(invalidatedRaw.isEmpty) previous else { - def debug(s: => String) = if(java.lang.Boolean.getBoolean(incDebugProp)) log.debug(s) else () + def debug(s: => String) = if (incDebug(options)) log.debug(s) else () val withPackageObjects = invalidatedRaw ++ invalidatedPackageObjects(invalidatedRaw, previous.relations) val invalidated = expand(withPackageObjects, allSources, log) val pruned = prune(invalidated, previous) @@ -54,10 +56,10 @@ object Incremental debug("********* Fresh: \n" + fresh.relations + "\n*********") val merged = pruned ++ fresh//.copy(relations = pruned.relations ++ fresh.relations, apis = pruned.apis ++ fresh.apis) debug("********* Merged: \n" + merged.relations + "\n*********") - val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _) + val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, options) debug("Changes:\n" + incChanges) val incInv = invalidateIncremental(merged.relations, incChanges, invalidated, cycleNum >= TransitiveStep, log) - cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log) + cycle(incInv, allSources, emptyChanges, merged, doCompile, cycleNum+1, log, options) } private[this] def emptyChanges: DependencyChanges = new DependencyChanges { val modifiedBinaries = new Array[File](0) @@ -81,7 +83,7 @@ object Incremental * providing the API before and after the last step. The functions should return * an empty API if the file did not/does not exist. */ - def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source): APIChanges[T] = + def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source, options: IncOptions): APIChanges[T] = { val oldApis = lastSources.toSeq map oldAPI val newApis = lastSources.toSeq map newAPI @@ -104,7 +106,8 @@ object Incremental case (co1, co2) => co1.sourceDirectory == co2.sourceDirectory && co1.outputDirectory == co2.outputDirectory } - def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps, forEntry: File => Option[Analysis])(implicit equivS: Equiv[Stamp]): InitialChanges = + def changedInitial(entry: String => Option[File], sources: Set[File], previousAnalysis: Analysis, current: ReadStamps, + forEntry: File => Option[Analysis], options: IncOptions)(implicit equivS: Equiv[Stamp]): InitialChanges = { val previous = previousAnalysis.stamps val previousAPIs = previousAnalysis.apis @@ -112,7 +115,7 @@ object Incremental val srcChanges = changes(previous.allInternalSources.toSet, sources, f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) ) val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet val binaryDepChanges = previous.allBinaries.filter( externalBinaryModified(entry, forEntry, previous, current)).toSet - val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry)) + val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry), options) InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges ) } diff --git a/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala b/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala index dc48a5d3b..5c02685e1 100644 --- a/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala +++ b/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala @@ -11,6 +11,7 @@ import inc._ import classpath.ClasspathUtilities import classfile.Analyze import inc.Locate.DefinesClass + import inc.IncOptions import CompileSetup._ import sbinary.DefaultProtocol.{ immutableMapFormat, immutableSetFormat, StringFormat } @@ -21,7 +22,7 @@ import inc._ final class CompileConfiguration(val sources: Seq[File], val classpath: Seq[File], val previousAnalysis: Analysis, val previousSetup: Option[CompileSetup], val currentSetup: CompileSetup, val progress: Option[CompileProgress], val getAnalysis: File => Option[Analysis], val definesClass: DefinesClass, - val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache) + val reporter: Reporter, val compiler: AnalyzingCompiler, val javac: xsbti.compile.JavaCompiler, val cache: GlobalsCache, val incOptions: IncOptions) class AggressiveCompile(cacheFile: File) { @@ -37,11 +38,12 @@ class AggressiveCompile(cacheFile: File) definesClass: DefinesClass = Locate.definesClass _, reporter: Reporter, compileOrder: CompileOrder = Mixed, - skip: Boolean = false)(implicit log: Logger): Analysis = + skip: Boolean = false, + incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis = { val setup = new CompileSetup(output, new CompileOptions(options, javacOptions), compiler.scalaInstance.actualVersion, compileOrder) compile1(sources, classpath, setup, progress, store, analysisMap, definesClass, - compiler, javac, reporter, skip, cache) + compiler, javac, reporter, skip, cache, incrementalCompilerOptions) } def withBootclasspath(args: CompilerArguments, classpath: Seq[File]): Seq[File] = @@ -56,14 +58,15 @@ class AggressiveCompile(cacheFile: File) compiler: AnalyzingCompiler, javac: xsbti.compile.JavaCompiler, reporter: Reporter, skip: Boolean, - cache: GlobalsCache)(implicit log: Logger): Analysis = + cache: GlobalsCache, + incrementalCompilerOptions: IncOptions)(implicit log: Logger): Analysis = { val (previousAnalysis, previousSetup) = extract(store.get()) if(skip) previousAnalysis else { val config = new CompileConfiguration(sources, classpath, previousAnalysis, previousSetup, setup, - progress, analysis, definesClass, reporter, compiler, javac, cache) + progress, analysis, definesClass, reporter, compiler, javac, cache, incrementalCompilerOptions) val (modified, result) = compile2(config) if(modified) store.set(result, setup) @@ -140,7 +143,7 @@ class AggressiveCompile(cacheFile: File) case Some(previous) if equiv.equiv(previous, currentSetup) => previousAnalysis case _ => Incremental.prune(sourcesSet, previousAnalysis) } - IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log) + IncrementalCompile(sourcesSet, entry, compile0, analysis, getAnalysis, output, log, incOptions) } private[this] def outputDirectories(output: Output): Seq[File] = output match { case single: SingleOutput => List(single.outputDirectory) diff --git a/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala b/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala index 626103563..daad493f1 100644 --- a/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala +++ b/compile/integration/src/main/scala/sbt/compiler/IncrementalCompiler.scala @@ -1,7 +1,7 @@ package sbt.compiler import sbt.CompileSetup - import sbt.inc.Analysis + import sbt.inc.{Analysis, IncOptions} import xsbti.{Logger, Maybe} import xsbti.compile._ @@ -17,7 +17,9 @@ object IC extends IncrementalCompiler[Analysis, AnalyzingCompiler] val agg = new AggressiveCompile(setup.cacheFile) val aMap = (f: File) => m2o(analysisMap(f)) val defClass = (f: File) => { val dc = definesClass(f); (name: String) => dc.apply(name) } - agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap, defClass, reporter, order, skip)(log) + val incOptions = IncOptions.fromStringMap(incrementalCompilerOptions) + agg(scalac, javac, sources, classpath, output, cache, m2o(progress), scalacOptions, javacOptions, aMap, + defClass, reporter, order, skip, incOptions)(log) } private[this] def m2o[S](opt: Maybe[S]): Option[S] = if(opt.isEmpty) None else Some(opt.get) diff --git a/interface/src/main/java/xsbti/compile/Setup.java b/interface/src/main/java/xsbti/compile/Setup.java index 9a2a6bf4c..edf250b8b 100644 --- a/interface/src/main/java/xsbti/compile/Setup.java +++ b/interface/src/main/java/xsbti/compile/Setup.java @@ -1,6 +1,8 @@ package xsbti.compile; import java.io.File; +import java.util.Map; + import xsbti.Maybe; import xsbti.Reporter; @@ -30,4 +32,16 @@ public interface Setup /** The reporter that should be used to report scala compilation to. */ Reporter reporter(); + + /** + * Returns incremental compiler options. + * + * @see sbt.inc.IncOptions for details + * + * You can get default options by calling sbt.inc.IncOptions.toStringMap(sbt.inc.IncOptions.Default). + * + * In the future, we'll extend API in xsbti to provide factory methods that would allow to obtain + * defaults values so one can depend on xsbti package only. + **/ + Map incrementalCompilerOptions(); } diff --git a/main/actions/src/main/scala/sbt/Compiler.scala b/main/actions/src/main/scala/sbt/Compiler.scala index 779474043..d74de78b1 100644 --- a/main/actions/src/main/scala/sbt/Compiler.scala +++ b/main/actions/src/main/scala/sbt/Compiler.scala @@ -17,7 +17,7 @@ object Compiler final case class Inputs(compilers: Compilers, config: Options, incSetup: IncSetup) final case class Options(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMapper: Position => Position, order: CompileOrder) - final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache) + final case class IncSetup(analysisMap: File => Option[Analysis], definesClass: DefinesClass, skip: Boolean, cacheFile: File, cache: GlobalsCache, incOptions: IncOptions) final case class Compilers(scalac: AnalyzingCompiler, javac: JavaTool) @deprecated("Use the other inputs variant.", "0.12.0") @@ -27,7 +27,7 @@ object Compiler val classesDirectory = outputDirectory / "classes" val cacheFile = outputDirectory / "cache_old_style" val augClasspath = classesDirectory.asFile +: classpath - val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh) + val incSetup = IncSetup(Map.empty, definesClass, false, cacheFile, CompilerCache.fresh, IncOptions.Default) inputs(augClasspath, sources, classesDirectory, options, javacOptions, maxErrors, Nil, order)(compilers, incSetup, log) } def inputs(classpath: Seq[File], sources: Seq[File], classesDirectory: File, options: Seq[String], javacOptions: Seq[String], maxErrors: Int, sourcePositionMappers: Seq[Position => Option[Position]], order: CompileOrder)(implicit compilers: Compilers, incSetup: IncSetup, log: Logger): Inputs = @@ -77,7 +77,7 @@ object Compiler val agg = new AggressiveCompile(cacheFile) agg(scalac, javac, sources, classpath, CompileOutput(classesDirectory), cache, None, options, javacOptions, - analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip)(log) + analysisMap, definesClass, new LoggerReporter(maxErrors, log, sourcePositionMapper), order, skip, incOptions)(log) } private[sbt] def foldMappers[A](mappers: Seq[A => Option[A]]) = diff --git a/main/src/main/scala/sbt/Defaults.scala b/main/src/main/scala/sbt/Defaults.scala index 4f65ba371..36c90dca9 100755 --- a/main/src/main/scala/sbt/Defaults.scala +++ b/main/src/main/scala/sbt/Defaults.scala @@ -204,6 +204,7 @@ object Defaults extends BuildCommon compilersSetting, javacOptions in GlobalScope :== Nil, scalacOptions in GlobalScope :== Nil, + incOptions in GlobalScope :== sbt.inc.IncOptions.Default, scalaInstance <<= scalaInstanceTask, scalaVersion in GlobalScope := appConfiguration.value.provider.scalaProvider.version, scalaBinaryVersion in GlobalScope := binaryScalaVersion(scalaVersion.value), @@ -650,8 +651,8 @@ object Defaults extends BuildCommon def compileTask = (compileInputs in compile, streams) map { (i,s) => Compiler(i,s.log) } def compileIncSetupTask = - (dependencyClasspath, skip in compile, definesClass, compilerCache, streams) map { (cp, skip, definesC, cache, s) => - Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache) + (dependencyClasspath, skip in compile, definesClass, compilerCache, streams, incOptions) map { (cp, skip, definesC, cache, s, incOptions) => + Compiler.IncSetup(analysisMap(cp), definesC, skip, s.cacheDirectory / "inc_compile", cache, incOptions) } def compileInputsSettings: Seq[Setting[_]] = Seq(compileInputs := { @@ -1024,7 +1025,7 @@ object Classpaths val si = Defaults.unmanagedScalaInstanceOnly.value.map(si => (si, scalaOrganization.value)) val show = Reference.display(thisProjectRef.value) cachedUpdate(s.cacheDirectory, show, ivyModule.value, updateConfiguration.value, si, skip = (skip in update).value, force = isRoot, depsUpdated = depsUpdated, log = s.log) - } + } def cachedUpdate(cacheFile: File, label: String, module: IvySbt#Module, config: UpdateConfiguration, scalaInstance: Option[(ScalaInstance, String)], skip: Boolean, force: Boolean, depsUpdated: Boolean, log: Logger): UpdateReport = { diff --git a/main/src/main/scala/sbt/Keys.scala b/main/src/main/scala/sbt/Keys.scala index 2fa696410..939f840c5 100644 --- a/main/src/main/scala/sbt/Keys.scala +++ b/main/src/main/scala/sbt/Keys.scala @@ -126,6 +126,7 @@ object Keys val maxErrors = SettingKey[Int]("max-errors", "The maximum number of errors, such as compile errors, to list.", ASetting) val scalacOptions = TaskKey[Seq[String]]("scalac-options", "Options for the Scala compiler.", BPlusTask) val javacOptions = TaskKey[Seq[String]]("javac-options", "Options for the Java compiler.", BPlusTask) + val incOptions = TaskKey[sbt.inc.IncOptions]("inc-options", "Options for the incremental compiler.", BTask) val compileOrder = SettingKey[CompileOrder]("compile-order", "Configures the order in which Java and sources within a single compilation are compiled. Valid values are: JavaThenScala, ScalaThenJava, or Mixed.", BPlusSetting) val initialCommands = SettingKey[String]("initial-commands", "Initial commands to execute when starting up the Scala interpreter.", AMinusSetting) val cleanupCommands = SettingKey[String]("cleanup-commands", "Commands to execute before the Scala interpreter exits.", BMinusSetting)