From 0cbbe8c2ed2d08cd7c70ffdde91f9821b6ab9030 Mon Sep 17 00:00:00 2001 From: Grzegorz Kossakowski Date: Thu, 8 May 2014 19:32:26 +0200 Subject: [PATCH 1/3] Add pending scripted test for Ant-style compilation. Add pending test for Ant-style incremental compilation. In that mode incremental compiler will recompile only the source files that were changed by the user and won't try to invalidate any dependencies. Once Ant-style incremental compilation is implemented this test should be passing. --- .../compiler-project/inc-ant-style/build.sbt | 28 +++++++++++++++++++ .../inc-ant-style/changes/A1.scala | 3 ++ .../compiler-project/inc-ant-style/pending | 9 ++++++ .../inc-ant-style/src/main/scala/A.scala | 5 ++++ .../inc-ant-style/src/main/scala/B.scala | 3 ++ .../inc-ant-style/src/main/scala/C.scala | 5 ++++ 6 files changed, 53 insertions(+) create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/changes/A1.scala create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/pending create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/A.scala create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/B.scala create mode 100644 sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/C.scala diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt b/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt new file mode 100644 index 000000000..b38650b6f --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt @@ -0,0 +1,28 @@ +logLevel := Level.Debug + +incOptions := incOptions.value.withNameHashing(true) + +/* Performs checks related to compilations: + * a) checks in which compilation given set of files was recompiled + * b) checks overall number of compilations performed + */ +TaskKey[Unit]("check-compilations") <<= (compile in Compile, scalaSource in Compile) map { (a: sbt.inc.Analysis, src: java.io.File) => + def relative(f: java.io.File): java.io.File = f.relativeTo(src) getOrElse f + val allCompilations = a.compilations.allCompilations + val recompiledFiles: Seq[Set[java.io.File]] = allCompilations map { c => + val recompiledFiles = a.apis.internal.collect { + case (file, api) if api.compilation.startTime == c.startTime => relative(file) + } + recompiledFiles.toSet + } + def recompiledFilesInIteration(iteration: Int, fileNames: Set[String]) = { + val files = fileNames.map(new java.io.File(_)) + assert(recompiledFiles(iteration) == files, "%s != %s".format(recompiledFiles(iteration), files)) + } + assert(allCompilations.size == 2) + // B.scala and C.scala are compiled at the beginning, in the Ant-style incremental compilation + // they are not rebuild when A.scala. + recompiledFilesInIteration(0, Set("B.scala", "C.scala")) + // A.scala is changed and recompiled + recompiledFilesInIteration(1, Set("A.scala")) +} diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/changes/A1.scala b/sbt/src/sbt-test/compiler-project/inc-ant-style/changes/A1.scala new file mode 100644 index 000000000..e86bab42e --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/changes/A1.scala @@ -0,0 +1,3 @@ +package test3 + +trait A \ No newline at end of file diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/pending b/sbt/src/sbt-test/compiler-project/inc-ant-style/pending new file mode 100644 index 000000000..72f5dba3f --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/pending @@ -0,0 +1,9 @@ +# introduces first compile iteration +> compile +# this change is local to method and does not change api so introduces +# only one additional compile iteration +$ copy-file changes/A1.scala src/main/scala/A.scala +# second iteration +> compile +# check if there are only two compile iterations performed +> check-compilations diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/A.scala b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/A.scala new file mode 100644 index 000000000..4ab580616 --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/A.scala @@ -0,0 +1,5 @@ +package test3 + +trait A { + def foo = 1 +} diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/B.scala b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/B.scala new file mode 100644 index 000000000..572e3c764 --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/B.scala @@ -0,0 +1,3 @@ +package test3 + +trait B extends A diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/C.scala b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/C.scala new file mode 100644 index 000000000..c8b935590 --- /dev/null +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/src/main/scala/C.scala @@ -0,0 +1,5 @@ +package test3 + +trait C { + def abc(a: A): Int = a.foo +} From ba88236b31880eb8bb45b35c2846af0e8d559978 Mon Sep 17 00:00:00 2001 From: Grzegorz Kossakowski Date: Thu, 8 May 2014 20:13:23 +0200 Subject: [PATCH 2/3] Add `antStyle` to IncOptions. Add an option that enables (to be implemented) Ant-style mode of incremental compilation. This option is unsupported and may go away at any point in the future. NOTE: Either `antStyle` or `nameHashing` mode can be enabled. This is being enforced with runtime assertion. --- .../src/main/scala/sbt/inc/IncOptions.scala | 65 ++++++++++++++----- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/compile/inc/src/main/scala/sbt/inc/IncOptions.scala b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala index e2a2cffb2..66c372784 100644 --- a/compile/inc/src/main/scala/sbt/inc/IncOptions.scala +++ b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala @@ -65,7 +65,22 @@ final class IncOptions( * 3. Hashing of public names is enabled. See `sbt.inc.AnalysisCallback` for details. * */ - val nameHashing: Boolean) extends Product with Serializable { + val nameHashing: Boolean, + /** + * THE `antStyle` OPTION IS UNSUPPORTED, MAY GO AWAY AT ANY POINT. + * + * Enables "ant-style" mode of incremental compilation. This mode emulates what Ant's scalac command does. + * The idea is to recompile just changed source files and not perform any invalidation of dependencies. This + * is a very naive mode of incremental compilation that very often leads to broken binaries. + * + * The Ant-style mode has been introduced because Scala team needs it for migration of Scala compiler to sbt. + * The name hashing algorithm doesn't work well with Scala compiler sources due to deep inheritance chains. + * There's a plan to refactor compiler's code to use more composition instead of inheritance. + * + * Once Scala compiler sources are refactored to work well with name hashing algorithm this option will be + * deleted immediately. + */ + val antStyle: Boolean) extends Product with Serializable { /** * Secondary constructor introduced to make IncOptions to be binary compatible with version that didn't have @@ -74,52 +89,60 @@ final class IncOptions( def this(transitiveStep: Int, recompileAllFraction: Double, relationsDebug: Boolean, apiDebug: Boolean, apiDiffContextSize: Int, apiDumpDirectory: Option[java.io.File], newClassfileManager: () => ClassfileManager) = { this(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, IncOptions.recompileOnMacroDefDefault, IncOptions.nameHashingDefault) + apiDumpDirectory, newClassfileManager, IncOptions.recompileOnMacroDefDefault, IncOptions.nameHashingDefault, + IncOptions.antStyleDefault) } + assert(!(antStyle && nameHashing), "Name hashing and Ant-style cannot be enabled at the same time.") + def withTransitiveStep(transitiveStep: Int): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withRecompileAllFraction(recompileAllFraction: Double): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withRelationsDebug(relationsDebug: Boolean): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withApiDebug(apiDebug: Boolean): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withApiDiffContextSize(apiDiffContextSize: Int): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withApiDumpDirectory(apiDumpDirectory: Option[File]): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withNewClassfileManager(newClassfileManager: () => ClassfileManager): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withRecompileOnMacroDef(recompileOnMacroDef: Boolean): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } def withNameHashing(nameHashing: Boolean): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) + } + + def withAntStyle(antStyle: Boolean): IncOptions = { + new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } //- EXPANDED CASE CLASS METHOD BEGIN -// @@ -130,14 +153,14 @@ final class IncOptions( apiDumpDirectory: Option[java.io.File] = this.apiDumpDirectory, newClassfileManager: () => ClassfileManager = this.newClassfileManager): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyle) } @deprecated("Methods generated for case class will be removed in the future.", "0.13.2") override def productPrefix: String = "IncOptions" @deprecated("Methods generated for case class will be removed in the future.", "0.13.2") - def productArity: Int = 9 + def productArity: Int = 10 @deprecated("Methods generated for case class will be removed in the future.", "0.13.2") def productElement(x$1: Int): Any = x$1 match { @@ -150,6 +173,7 @@ final class IncOptions( case 6 => IncOptions.this.newClassfileManager case 7 => IncOptions.this.recompileOnMacroDef case 8 => IncOptions.this.nameHashing + case 9 => IncOptions.this.antStyle case _ => throw new IndexOutOfBoundsException(x$1.toString()) } @@ -171,6 +195,7 @@ final class IncOptions( acc = Statics.mix(acc, Statics.anyHash(newClassfileManager)) acc = Statics.mix(acc, if (recompileOnMacroDef) 1231 else 1237) acc = Statics.mix(acc, if (nameHashing) 1231 else 1237) + acc = Statics.mix(acc, if (antStyle) 1231 else 1237) Statics.finalizeHash(acc, 9) } @@ -183,7 +208,8 @@ final class IncOptions( relationsDebug == IncOptions$1.relationsDebug && apiDebug == IncOptions$1.apiDebug && apiDiffContextSize == IncOptions$1.apiDiffContextSize && apiDumpDirectory == IncOptions$1.apiDumpDirectory && newClassfileManager == IncOptions$1.newClassfileManager && - recompileOnMacroDef == IncOptions$1.recompileOnMacroDef && nameHashing == IncOptions$1.nameHashing + recompileOnMacroDef == IncOptions$1.recompileOnMacroDef && nameHashing == IncOptions$1.nameHashing && + antStyle == IncOptions$1.antStyle })) } //- EXPANDED CASE CLASS METHOD END -// @@ -192,6 +218,7 @@ final class IncOptions( object IncOptions extends Serializable { private val recompileOnMacroDefDefault: Boolean = true private val nameHashingDefault: Boolean = false + private val antStyleDefault: Boolean = false val Default = IncOptions( // 1. recompile changed sources // 2(3). recompile direct dependencies and transitive public inheritance dependencies of sources with API changes in 1(2). @@ -220,7 +247,7 @@ object IncOptions extends Serializable { newClassfileManager: () => ClassfileManager, recompileOnMacroDef: Boolean, nameHashing: Boolean): IncOptions = { new IncOptions(transitiveStep, recompileAllFraction, relationsDebug, apiDebug, apiDiffContextSize, - apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing) + apiDumpDirectory, newClassfileManager, recompileOnMacroDef, nameHashing, antStyleDefault) } @deprecated("Methods generated for case class will be removed in the future.", "0.13.2") def unapply(x$0: IncOptions): Option[(Int, Double, Boolean, Boolean, Int, Option[java.io.File], () => AnyRef)] = { @@ -248,6 +275,7 @@ object IncOptions extends Serializable { private val apiDiffContextSizeKey = "apiDiffContextSize" private val recompileOnMacroDefKey = "recompileOnMacroDef" private val nameHashingKey = "nameHashing" + private val antStyleKey = "antStyle" def fromStringMap(m: java.util.Map[String, String]): IncOptions = { // all the code below doesn't look like idiomatic Scala for a good reason: we are working with Java API @@ -286,8 +314,13 @@ object IncOptions extends Serializable { if (m.containsKey(k)) m.get(k).toBoolean else Default.nameHashing } + def getAntStyle: Boolean = { + val k = antStyleKey + if (m.containsKey(k)) m.get(k).toBoolean else Default.antStyle + } + new IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDiffContextSize, - getApiDumpDirectory, ClassfileManager.deleteImmediately, getRecompileOnMacroDef, getNameHashing) + getApiDumpDirectory, ClassfileManager.deleteImmediately, getRecompileOnMacroDef, getNameHashing, getAntStyle) } def toStringMap(o: IncOptions): java.util.Map[String, String] = { From 680713f6663f3c6339394bbc5e905d26b99ce54c Mon Sep 17 00:00:00 2001 From: Grzegorz Kossakowski Date: Thu, 8 May 2014 20:26:22 +0200 Subject: [PATCH 3/3] Add Ant-style incremental compilation mode. This commit implements an Ant-style incremental compilation mode. This mode emulates what Ant's scalac command does. It recompiles just changed source files and does not perform any invalidation of dependencies. This is a very naive mode of incremental compilation that very often leads to broken binaries. The Ant-style mode is being introduced because Scala team needs it for migration of Scala compiler to sbt. The name hashing algorithm doesn't work well with Scala compiler sources due to deep inheritance chains. There's a plan to refactor compiler's code to use more composition instead of inheritance. Once Scala compiler sources are refactored to work well with name hashing algorithm, Ant-style mode will be deleted immediately. --- .../src/main/scala/sbt/inc/Incremental.scala | 27 ++++++++++++++++--- .../compiler-project/inc-ant-style/build.sbt | 2 +- .../inc-ant-style/{pending => test} | 0 3 files changed, 25 insertions(+), 4 deletions(-) rename sbt/src/sbt-test/compiler-project/inc-ant-style/{pending => test} (100%) diff --git a/compile/inc/src/main/scala/sbt/inc/Incremental.scala b/compile/inc/src/main/scala/sbt/inc/Incremental.scala index ccce0065e..01f3a8749 100644 --- a/compile/inc/src/main/scala/sbt/inc/Incremental.scala +++ b/compile/inc/src/main/scala/sbt/inc/Incremental.scala @@ -21,10 +21,12 @@ object Incremental { options: IncOptions)(implicit equivS: Equiv[Stamp]): (Boolean, Analysis) = { val incremental: IncrementalCommon = - if (!options.nameHashing) - new IncrementalDefaultImpl(log, options) - else + if (options.nameHashing) new IncrementalNameHashing(log, options) + else if (options.antStyle) + new IncrementalAntStyle(log, options) + else + new IncrementalDefaultImpl(log, options) val initialChanges = incremental.changedInitial(entry, sources, previous, current, forEntry) val binaryChanges = new DependencyChanges { val modifiedBinaries = initialChanges.binaryDeps.toArray @@ -573,3 +575,22 @@ private final class IncrementalNameHashing(log: Logger, options: IncOptions) ext f => relations.memberRef.internal.reverse(f) } + +private final class IncrementalAntStyle(log: Logger, options: IncOptions) extends IncrementalCommon(log, options) { + + /** Ant-style mode doesn't do anything special with package objects */ + override protected def invalidatedPackageObjects(invalidated: Set[File], relations: Relations): Set[File] = Set.empty + + /** In Ant-style mode we don't need to compare APIs because we don't perform any invalidation */ + override protected def sameAPI[T](src: T, a: Source, b: Source): Option[APIChange[T]] = None + + /** In Ant-style mode we don't perform any invalidation */ + override protected def invalidateByExternal(relations: Relations, externalAPIChange: APIChange[String]): Set[File] = Set.empty + + /** In Ant-style mode we don't perform any invalidation */ + override protected def invalidateSource(relations: Relations, change: APIChange[File]): Set[File] = Set.empty + + /** In Ant-style mode we don't need to perform any dependency analysis hence we can always return an empty set. */ + override protected def allDeps(relations: Relations): File => Set[File] = _ => Set.empty + +} diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt b/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt index b38650b6f..b25805b13 100644 --- a/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt +++ b/sbt/src/sbt-test/compiler-project/inc-ant-style/build.sbt @@ -1,6 +1,6 @@ logLevel := Level.Debug -incOptions := incOptions.value.withNameHashing(true) +incOptions := incOptions.value.withAntStyle(true) /* Performs checks related to compilations: * a) checks in which compilation given set of files was recompiled diff --git a/sbt/src/sbt-test/compiler-project/inc-ant-style/pending b/sbt/src/sbt-test/compiler-project/inc-ant-style/test similarity index 100% rename from sbt/src/sbt-test/compiler-project/inc-ant-style/pending rename to sbt/src/sbt-test/compiler-project/inc-ant-style/test