From c355bef20088623ba8a8568d3e6aba56aba96e94 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Fri, 26 Apr 2013 22:35:27 -0400 Subject: [PATCH 1/7] Track public inherited dependencies. There is a public inherited dependency on each (normalized) base class of a public template (class, module, trait, structural type). --- compile/interface/src/main/scala/xsbt/API.scala | 17 ++++++++++++++++- .../src/main/scala/xsbt/Analyzer.scala | 8 +++++--- .../src/main/scala/xsbt/CompilerInterface.scala | 5 +++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/compile/interface/src/main/scala/xsbt/API.scala b/compile/interface/src/main/scala/xsbt/API.scala index b20219823..451cc8aa8 100644 --- a/compile/interface/src/main/scala/xsbt/API.scala +++ b/compile/interface/src/main/scala/xsbt/API.scala @@ -39,6 +39,7 @@ final class API(val global: CallbackGlobal) extends Compat def processScalaUnit(unit: CompilationUnit) { val sourceFile = unit.source.file.file + currentSourceFile = sourceFile debug("Traversing " + sourceFile) val traverser = new TopLevelHandler(sourceFile) traverser.apply(unit.body) @@ -50,6 +51,10 @@ final class API(val global: CallbackGlobal) extends Compat } } + // Tracks the source file associated with the CompilationUnit currently being processed by the API phase. + // This is used when recording inheritance dependencies. + private[this] var currentSourceFile: File = _ + // this cache reduces duplicate work both here and when persisting // caches on other structures had minimal effect on time and cache size // (tried: Definition, Modifier, Path, Id, String) @@ -237,8 +242,18 @@ final class API(val global: CallbackGlobal) extends Compat mkStructure(s, baseTypes, ds, is) } - private def mkStructure(s: Symbol, bases: List[Type], declared: List[Symbol], inherited: List[Symbol]): xsbti.api.Structure = + // If true, this template is publicly visible and should be processed as a public inheritance dependency. + // Local classes and local refinements will never be traversed by the api phase, so we don't need to check for that. + private[this] def isPublicStructure(s: Symbol): Boolean = + s.isStructuralRefinement || + // do not consider templates that are private[this] or private + !(s.isPrivate && (s.privateWithin == NoSymbol || s.isLocal)) + + private def mkStructure(s: Symbol, bases: List[Type], declared: List[Symbol], inherited: List[Symbol]): xsbti.api.Structure = { + if(isPublicStructure(s)) + addInheritedDependencies(currentSourceFile, bases.map(_.dealias.typeSymbol)) new xsbti.api.Structure(lzy(types(s, bases)), lzy(processDefinitions(s, declared)), lzy(processDefinitions(s, inherited))) + } private def processDefinitions(in: Symbol, defs: List[Symbol]): Array[xsbti.api.Definition] = sort(defs.toArray).flatMap( (d: Symbol) => definition(in, d)) private[this] def sort(defs: Array[Symbol]): Array[Symbol] = { diff --git a/compile/interface/src/main/scala/xsbt/Analyzer.scala b/compile/interface/src/main/scala/xsbt/Analyzer.scala index 728224b96..0989f7a67 100644 --- a/compile/interface/src/main/scala/xsbt/Analyzer.scala +++ b/compile/interface/src/main/scala/xsbt/Analyzer.scala @@ -33,9 +33,11 @@ final class Analyzer(val global: CallbackGlobal) extends Compat // build dependencies structure val sourceFile = unit.source.file.file callback.beginSource(sourceFile) - for(on <- unit.depends) + for(on <- unit.depends) processDependency(on, inherited=false) + for(on <- inheritedDependencies.getOrElse(sourceFile, Nil: Iterable[Symbol])) processDependency(on, inherited=true) + def processDependency(on: Symbol, inherited: Boolean) { - def binaryDependency(file: File, className: String) = callback.binaryDependency(file, className, sourceFile) + def binaryDependency(file: File, className: String) = callback.binaryDependency(file, className, sourceFile /*, inherited*/) val onSource = on.sourceFile if(onSource == null) { @@ -53,7 +55,7 @@ final class Analyzer(val global: CallbackGlobal) extends Compat } } else - callback.sourceDependency(onSource.file, sourceFile) + callback.sourceDependency(onSource.file, sourceFile /*, inherited*/) } // build list of generated classes diff --git a/compile/interface/src/main/scala/xsbt/CompilerInterface.scala b/compile/interface/src/main/scala/xsbt/CompilerInterface.scala index abb1407cd..3a02bd138 100644 --- a/compile/interface/src/main/scala/xsbt/CompilerInterface.scala +++ b/compile/interface/src/main/scala/xsbt/CompilerInterface.scala @@ -42,6 +42,11 @@ sealed abstract class CallbackGlobal(settings: Settings, reporter: reporters.Rep case multi: MultipleOutput => multi.outputGroups.toStream map (_.outputDirectory) } } + // Map source files to public inherited dependencies. These dependencies are tracked as the symbol for the dealiased base class. + val inheritedDependencies = new mutable.HashMap[File, mutable.Set[Symbol]] + def addInheritedDependencies(file: File, deps: Iterable[Symbol]) { + inheritedDependencies.getOrElseUpdate(file, new mutable.HashSet) ++= deps + } } class InterfaceCompileFailed(val arguments: Array[String], val problems: Array[Problem], override val toString: String) extends xsbti.CompileFailed From 4dc75343aebba8d7080e73a4442dbce101afdb8d Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Fri, 26 Apr 2013 22:35:27 -0400 Subject: [PATCH 2/7] Record and persist public inheritance dependencies. Includes placeholders for adding public inherited dependencies for Java classes. --- .../inc/src/main/scala/sbt/inc/Analysis.scala | 16 +- .../inc/src/main/scala/sbt/inc/Compile.scala | 41 +++-- .../src/main/scala/sbt/inc/Relations.scala | 155 ++++++++++++++---- .../src/main/scala/xsbt/Analyzer.scala | 4 +- .../main/scala/sbt/inc/AnalysisFormats.scala | 8 +- .../src/main/java/xsbti/AnalysisCallback.java | 12 +- .../main/scala/sbt/classfile/Analyze.scala | 6 +- 7 files changed, 175 insertions(+), 67 deletions(-) diff --git a/compile/inc/src/main/scala/sbt/inc/Analysis.scala b/compile/inc/src/main/scala/sbt/inc/Analysis.scala index 3443aeeda..192d931a1 100644 --- a/compile/inc/src/main/scala/sbt/inc/Analysis.scala +++ b/compile/inc/src/main/scala/sbt/inc/Analysis.scala @@ -11,6 +11,7 @@ trait Analysis { val stamps: Stamps val apis: APIs + /** Mappings between sources, classes, and binaries. */ val relations: Relations val infos: SourceInfos /** Information about compiler runs accumulated since `clean` command has been run. */ @@ -21,9 +22,9 @@ trait Analysis def copy(stamps: Stamps = stamps, apis: APIs = apis, relations: Relations = relations, infos: SourceInfos = infos, compilations: Compilations = compilations): Analysis - def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File], info: SourceInfo): Analysis + def addSource(src: File, api: Source, stamp: Stamp, directInternal: Iterable[File], inheritedInternal: Iterable[File], info: SourceInfo): Analysis def addBinaryDep(src: File, dep: File, className: String, stamp: Stamp): Analysis - def addExternalDep(src: File, dep: String, api: Source): Analysis + def addExternalDep(src: File, dep: String, api: Source, inherited: Boolean): Analysis def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis def groupBy[K](f: (File => K)): Map[K, Analysis] @@ -60,8 +61,7 @@ object Analysis } } -private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relations, val infos: SourceInfos, - val compilations: Compilations) extends Analysis +private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relations, val infos: SourceInfos, val compilations: Compilations) extends Analysis { def ++ (o: Analysis): Analysis = new MAnalysis(stamps ++ o.stamps, apis ++ o.apis, relations ++ o.relations, infos ++ o.infos, compilations ++ o.compilations) @@ -78,14 +78,14 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat def copy(stamps: Stamps, apis: APIs, relations: Relations, infos: SourceInfos, compilations: Compilations = compilations): Analysis = new MAnalysis(stamps, apis, relations, infos, compilations) - def addSource(src: File, api: Source, stamp: Stamp, internalDeps: Iterable[File], info: SourceInfo): Analysis = - copy( stamps.markInternalSource(src, stamp), apis.markInternalSource(src, api), relations.addInternalSrcDeps(src, internalDeps), infos.add(src, info) ) + def addSource(src: File, api: Source, stamp: Stamp, directInternal: Iterable[File], inheritedInternal: Iterable[File], info: SourceInfo): Analysis = + copy( stamps.markInternalSource(src, stamp), apis.markInternalSource(src, api), relations.addInternalSrcDeps(src, directInternal, inheritedInternal), infos.add(src, info) ) def addBinaryDep(src: File, dep: File, className: String, stamp: Stamp): Analysis = copy( stamps.markBinary(dep, className, stamp), apis, relations.addBinaryDep(src, dep), infos ) - def addExternalDep(src: File, dep: String, depAPI: Source): Analysis = - copy( stamps, apis.markExternalAPI(dep, depAPI), relations.addExternalDep(src, dep), infos ) + def addExternalDep(src: File, dep: String, depAPI: Source, inherited: Boolean): Analysis = + copy( stamps, apis.markExternalAPI(dep, depAPI), relations.addExternalDep(src, dep, inherited), infos ) def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis = copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos ) diff --git a/compile/inc/src/main/scala/sbt/inc/Compile.scala b/compile/inc/src/main/scala/sbt/inc/Compile.scala index a4640d776..61a19d3dd 100644 --- a/compile/inc/src/main/scala/sbt/inc/Compile.scala +++ b/compile/inc/src/main/scala/sbt/inc/Compile.scala @@ -62,12 +62,17 @@ private final class AnalysisCallback(internalMap: File => Option[File], external private[this] val unreporteds = new HashMap[File, ListBuffer[Problem]] private[this] val reporteds = new HashMap[File, ListBuffer[Problem]] private[this] val binaryDeps = new HashMap[File, Set[File]] - // source file to set of generated (class file, class name) + // source file to set of generated (class file, class name) private[this] val classes = new HashMap[File, Set[(File, String)]] - // generated class file to its source file + // generated class file to its source file private[this] val classToSource = new HashMap[File, File] + // all internal source depenencies, including direct and inherited private[this] val sourceDeps = new HashMap[File, Set[File]] - private[this] val extSrcDeps = new ListBuffer[(File, String, Source)] + // inherited internal source dependencies + private[this] val inheritedSourceDeps = new HashMap[File, Set[File]] + // external source dependencies: + // (internal source, external source depended on, API of external dependency, true if an inheritance dependency) + private[this] val extSrcDeps = new ListBuffer[(File, String, Source, Boolean)] private[this] val binaryClassName = new HashMap[File, String] // source files containing a macro def. private[this] val macroSources = Set[File]() @@ -83,41 +88,45 @@ private final class AnalysisCallback(internalMap: File => Option[File], external } } - def sourceDependency(dependsOn: File, source: File) = if(source != dependsOn) add(sourceDeps, source, dependsOn) - def externalBinaryDependency(binary: File, className: String, source: File) + def sourceDependency(dependsOn: File, source: File, inherited: Boolean) = + if(source != dependsOn) { + add(sourceDeps, source, dependsOn) + if(inherited) add(inheritedSourceDeps, source, dependsOn) + } + def externalBinaryDependency(binary: File, className: String, source: File, inherited: Boolean) { binaryClassName.put(binary, className) add(binaryDeps, source, binary) } - def externalSourceDependency(triple: (File, String, Source)) = extSrcDeps += triple + def externalSourceDependency(t4: (File, String, Source, Boolean)) = extSrcDeps += t4 - def binaryDependency(classFile: File, name: String, source: File) = + def binaryDependency(classFile: File, name: String, source: File, inherited: Boolean) = internalMap(classFile) match { case Some(dependsOn) => // dependency is a product of a source not included in this compilation - sourceDependency(dependsOn, source) + sourceDependency(dependsOn, source, inherited) case None => classToSource.get(classFile) match { case Some(dependsOn) => // dependency is a product of a source in this compilation step, // but not in the same compiler run (as in javac v. scalac) - sourceDependency(dependsOn, source) + sourceDependency(dependsOn, source, inherited) case None => - externalDependency(classFile, name, source) + externalDependency(classFile, name, source, inherited) } } - private[this] def externalDependency(classFile: File, name: String, source: File): Unit = + private[this] def externalDependency(classFile: File, name: String, source: File, inherited: Boolean): Unit = externalAPI(classFile, name) match { case Some(api) => // dependency is a product of a source in another project - externalSourceDependency( (source, name, api) ) + externalSourceDependency( (source, name, api, inherited) ) case None => // dependency is some other binary on the classpath - externalBinaryDependency(classFile, name, source) + externalBinaryDependency(classFile, name, source, inherited) } def generatedClass(source: File, module: File, name: String) = @@ -148,10 +157,12 @@ private final class AnalysisCallback(internalMap: File => Option[File], external val hasMacro: Boolean = macroSources.contains(src) val s = new xsbti.api.Source(compilation, hash, api._2, api._1, hasMacro) val info = SourceInfos.makeInfo(getOrNil(reporteds, src), getOrNil(unreporteds, src)) - a.addSource(src, s, stamp, sourceDeps.getOrElse(src, Nil: Iterable[File]), info) + val direct = sourceDeps.getOrElse(src, Nil: Iterable[File]) + val publicInherited = inheritedSourceDeps.getOrElse(src, Nil: Iterable[File]) + a.addSource(src, s, stamp, direct, publicInherited, info) } def getOrNil[A,B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten - def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api)) => a.addExternalDep(source, name, api) } + def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api, inherited)) => a.addExternalDep(source, name, api, inherited) } def addCompilation(base: Analysis): Analysis = base.copy(compilations = base.compilations.add(compilation)) def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis = diff --git a/compile/inc/src/main/scala/sbt/inc/Relations.scala b/compile/inc/src/main/scala/sbt/inc/Relations.scala index a5f0fcf15..803a9d957 100644 --- a/compile/inc/src/main/scala/sbt/inc/Relations.scala +++ b/compile/inc/src/main/scala/sbt/inc/Relations.scala @@ -6,55 +6,133 @@ package inc import xsbti.api.Source import java.io.File +import Relations.{Source => RSource} + +/** Provides mappings between source files, generated classes (products), and binaries. +* Dependencies that are tracked include internal: a dependency on a source in the same compilation group (project), +* external: a dependency on a source in another compilation group (tracked as the name of the class), +* binary: a dependency on a class or jar file not generated by a source file in any tracked compilation group, +* inherited: a dependency that resulted from a public template inheriting, +* direct: any type of dependency, including inheritance. */ trait Relations { /** All sources _with at least one product_ . */ def allSources: collection.Set[File] + /** All products associates with sources. */ def allProducts: collection.Set[File] + + /** All files that are recorded as a binary dependency of a source file.*/ def allBinaryDeps: collection.Set[File] + + /** All files in this compilation group (project) that are recorded as a source dependency of a source file in this group.*/ def allInternalSrcDeps: collection.Set[File] + + /** All files in another compilation group (project) that are recorded as a source dependency of a source file in this group.*/ def allExternalDeps: collection.Set[String] + /** Fully qualified names of classes defined in source file `src`. */ def classNames(src: File): Set[String] + + /** Names of classes defined in source file `src`. */ def definesClass(name: String): Set[File] + /** The classes that were generated for source file `src`. */ def products(src: File): Set[File] + /** The source files that generated class file `prod`. This is typically a set containing a single file. */ def produced(prod: File): Set[File] + /** The binary dependencies for the source file `src`. */ def binaryDeps(src: File): Set[File] + /** The source files that depend on binary file `dep`. */ def usesBinary(dep: File): Set[File] - + + /** Internal source dependencies for `src`. This includes both direct and inherited dependencies. */ def internalSrcDeps(src: File): Set[File] + /** Internal source files that depend on internal source `dep`. This includes both direct and inherited dependencies. */ def usesInternalSrc(dep: File): Set[File] + /** External source dependencies that internal source file `src` depends on. This includes both direct and inherited dependencies. */ def externalDeps(src: File): Set[String] + /** Internal source dependencies that depend on external source file `dep`. This includes both direct and inherited dependencies. */ def usesExternal(dep: String): Set[File] + /** Records internal source file `src` as generating class file `prod` with top-level class `name`. */ def addProduct(src: File, prod: File, name: String): Relations - def addExternalDep(src: File, dependsOn: String): Relations - def addInternalSrcDeps(src: File, dependsOn: Iterable[File]): Relations + + /** Records internal source file `src` as depending on class `dependsOn` in an external source file. + * If `inherited` is true, this dependency is recorded as coming from a public template in `src` extending something in `dependsOn` (an inheritance dependency). + * Whatever the value of `inherited`, the dependency is also recorded as a direct dependency. */ + def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations + + /** Records internal source file `src` depending on a dependency binary dependency `dependsOn`.*/ def addBinaryDep(src: File, dependsOn: File): Relations + + /** Records internal source file `src` as having direct dependencies on internal source files `directDependsOn` + * and inheritance dependencies on `inheritedDependsOn`. Everything in `inheritedDependsOn` must be included in `directDependsOn`; + * this method does not automatically record direct dependencies like `addExternalDep` does.*/ + def addInternalSrcDeps(src: File, directDependsOn: Iterable[File], inheritedDependsOn: Iterable[File]): Relations def ++ (o: Relations): Relations + + /** Drops all dependency mappings from `sources`. This will not remove mappings to them (that is, mappings where they are dependencies).*/ def -- (sources: Iterable[File]): Relations + def groupBy[K](f: (File => K)): Map[K, Relations] - + + /** The relation between internal sources and generated class files. */ def srcProd: Relation[File, File] + + /** The dependency relation between internal sources and binaries. */ def binaryDep: Relation[File, File] + + /** The dependency relation between internal sources. This includes both direct and inherited dependencies.*/ def internalSrcDep: Relation[File, File] + + /** The dependency relation between internal and external sources. This includes both direct and inherited dependencies.*/ def externalDep: Relation[File, String] + + /** The dependency relations between sources. These include both direct and inherited dependencies.*/ + def direct: RSource + /** The inheritance dependency relations between sources.*/ + def publicInherited: RSource + + /** The relation between a source file and names of classes generated from it.*/ def classes: Relation[File, String] } + object Relations { + /** Tracks internal and external source dependencies for a specific dependency type, such as direct or inherited.*/ + final class Source private[sbt](val internal: Relation[File,File], val external: Relation[File,String]) { + def addInternal(source: File, dependsOn: Iterable[File]): Source = new Source(internal + (source, dependsOn), external) + def addExternal(source: File, dependsOn: String): Source = new Source(internal, external + (source, dependsOn)) + /** Drops all dependency mappings from `sources`. This will not remove mappings to them (that is, where they are dependencies).*/ + def --(sources: Iterable[File]): Source = new Source(internal -- sources, external -- sources) + def ++(o: Source): Source = new Source(internal ++ o.internal, external ++ o.external) + def groupBySource[K](f: File => K): Map[K, Source] = { + val i = internal.groupBy { case (a,b) => f(a) } + val e = external.groupBy { case (a,b) => f(a) } + val pairs = for( k <- i.keySet ++ e.keySet ) yield + (k, new Source( getOrEmpty(i, k), getOrEmpty(e, k) )) + pairs.toMap + } + } + + private[sbt] def getOrEmpty[A,B,K](m: Map[K, Relation[A,B]], k: K): Relation[A,B] = m.getOrElse(k, Relation.empty) + private[this] lazy val e = Relation.empty[File, File] - private[this] lazy val es = Relation.empty[File, String] - def empty: Relations = new MRelations(e, e, e, es, es) - def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], internalSrcDep: Relation[File, File], externalDep: Relation[File, String], classes: Relation[File, String]): Relations = - new MRelations(srcProd, binaryDep, internalSrcDep, externalDep, classes) + private[this] lazy val estr = Relation.empty[File, String] + private[this] lazy val es = new Source(e, estr) + + def emptySource: Source = es + def empty: Relations = new MRelations(e, e, es, es, estr) + + def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], direct: Source, publicInherited: Source, classes: Relation[File, String]): Relations = + new MRelations(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes) + def makeSource(internal: Relation[File,File], external: Relation[File,String]): Source = new Source(internal, external) } /** * `srcProd` is a relation between a source file and a product: (source, product). @@ -64,23 +142,27 @@ object Relations * This only includes dependencies on classes and jars that do not have a corresponding source/API to track instead. * A class or jar with a corresponding source should only be tracked in one of the source dependency relations. * -* `internalSrcDeps` is a relation between a source file and a source dependency in the same compilation group. -* Dependencies on sources in other projects belong in external source dependencies. +* `direct` defines relations for dependencies between internal and external source dependencies. It includes all types of +* dependencies, including inheritance. * -* `externalSrcDeps` is a relation between a source file and a source dependency in another compilation group. -* Dependencies on sources in the same group belong in internal source dependencies. +* `publicInherited` defines relations for internal and external source dependencies, only including dependencies +* introduced by inheritance. * * `classes` is a relation between a source file and its generated class names. */ private class MRelations(val srcProd: Relation[File, File], val binaryDep: Relation[File, File], - val internalSrcDep: Relation[File, File], val externalDep: Relation[File, String], val classes: Relation[File, String]) extends Relations + // direct should include everything in inherited + val direct: RSource, val publicInherited: RSource, val classes: Relation[File, String]) extends Relations { + def internalSrcDep: Relation[File, File] = direct.internal + def externalDep: Relation[File, String] = direct.external + def allSources: collection.Set[File] = srcProd._1s def allProducts: collection.Set[File] = srcProd._2s def allBinaryDeps: collection.Set[File] = binaryDep._2s - def allInternalSrcDeps: collection.Set[File] = internalSrcDep._2s - def allExternalDeps: collection.Set[String] = externalDep._2s + def allInternalSrcDeps: collection.Set[File] = direct.internal._2s + def allExternalDeps: collection.Set[String] = direct.external._2s def classNames(src: File): Set[String] = classes.forward(src) def definesClass(name: String): Set[File] = classes.reverse(name) @@ -91,45 +173,52 @@ private class MRelations(val srcProd: Relation[File, File], val binaryDep: Relat def binaryDeps(src: File): Set[File] = binaryDep.forward(src) def usesBinary(dep: File): Set[File] = binaryDep.reverse(dep) - def internalSrcDeps(src: File): Set[File] = internalSrcDep.forward(src) - def usesInternalSrc(dep: File): Set[File] = internalSrcDep.reverse(dep) + def internalSrcDeps(src: File): Set[File] = direct.internal.forward(src) + def usesInternalSrc(dep: File): Set[File] = direct.internal.reverse(dep) - def externalDeps(src: File): Set[String] = externalDep.forward(src) - def usesExternal(dep: String): Set[File] = externalDep.reverse(dep) + def externalDeps(src: File): Set[String] = direct.external.forward(src) + def usesExternal(dep: String): Set[File] = direct.external.reverse(dep) def addProduct(src: File, prod: File, name: String): Relations = - new MRelations( srcProd + (src, prod), binaryDep, internalSrcDep, externalDep, classes + (src, name) ) + new MRelations( srcProd + (src, prod), binaryDep, direct = direct, publicInherited = publicInherited, classes + (src, name) ) - def addExternalDep(src: File, dependsOn: String): Relations = - new MRelations( srcProd, binaryDep, internalSrcDep, externalDep + (src, dependsOn), classes ) + def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = { + val newI = if(inherited) publicInherited.addExternal(src, dependsOn) else publicInherited + val newD = direct.addExternal(src, dependsOn) + new MRelations( srcProd, binaryDep, direct = newD, publicInherited = newI, classes ) + } - def addInternalSrcDeps(src: File, dependsOn: Iterable[File]): Relations = - new MRelations( srcProd, binaryDep, internalSrcDep + (src, dependsOn ), externalDep, classes ) + def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations = { + val newI = publicInherited.addInternal(src, inherited) + val newD = direct.addInternal(src, dependsOn) + new MRelations( srcProd, binaryDep, direct = newD, publicInherited = newI, classes ) + } def addBinaryDep(src: File, dependsOn: File): Relations = - new MRelations( srcProd, binaryDep + (src, dependsOn), internalSrcDep, externalDep, classes ) + new MRelations( srcProd, binaryDep + (src, dependsOn), direct = direct, publicInherited = publicInherited, classes ) def ++ (o: Relations): Relations = - new MRelations(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, internalSrcDep ++ o.internalSrcDep, externalDep ++ o.externalDep, classes ++ o.classes) + new MRelations(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct = direct ++ o.direct, publicInherited = publicInherited ++ o.publicInherited, classes ++ o.classes) def -- (sources: Iterable[File]) = - new MRelations(srcProd -- sources, binaryDep -- sources, internalSrcDep -- sources, externalDep -- sources, classes -- sources) + new MRelations(srcProd -- sources, binaryDep -- sources, direct = direct -- sources, publicInherited = publicInherited -- sources, classes -- sources) def groupBy[K](f: File => K): Map[K, Relations] = { type MapRel[T] = Map[K, Relation[File, T]] - def outerJoin(srcProdMap: MapRel[File], binaryDepMap: MapRel[File], internalSrcDepMap: MapRel[File], - externalDepMap: MapRel[String], classesMap: MapRel[String]): Map[K, Relations] = + def outerJoin(srcProdMap: MapRel[File], binaryDepMap: MapRel[File], direct: Map[K, RSource], inherited: Map[K, RSource], + classesMap: MapRel[String]): Map[K, Relations] = { def kRelations(k: K): Relations = { - def get[T](m: Map[K, Relation[File, T]]) = m.getOrElse(k, Relation.empty) - new MRelations( get(srcProdMap), get(binaryDepMap), get(internalSrcDepMap), get(externalDepMap), get(classesMap) ) + def get[T](m: Map[K, Relation[File, T]]) = Relations.getOrEmpty(m, k) + def getSrc(m: Map[K, RSource]): RSource = m.getOrElse(k, Relations.emptySource) + new MRelations( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited), get(classesMap) ) } - val keys = (srcProdMap.keySet ++ binaryDepMap.keySet ++ internalSrcDepMap.keySet ++ externalDepMap.keySet ++ classesMap.keySet).toList + val keys = (srcProdMap.keySet ++ binaryDepMap.keySet ++ direct.keySet ++ inherited.keySet ++ classesMap.keySet).toList Map( keys.map( (k: K) => (k, kRelations(k)) ) : _*) } def f1[B](item: (File, B)): K = f(item._1) - outerJoin(srcProd.groupBy(f1), binaryDep.groupBy(f1), internalSrcDep.groupBy(f1), externalDep.groupBy(f1), classes.groupBy(f1)) + outerJoin(srcProd.groupBy(f1), binaryDep.groupBy(f1), direct.groupBySource(f), publicInherited.groupBySource(f), classes.groupBy(f1)) } diff --git a/compile/interface/src/main/scala/xsbt/Analyzer.scala b/compile/interface/src/main/scala/xsbt/Analyzer.scala index 0989f7a67..ff5fb577c 100644 --- a/compile/interface/src/main/scala/xsbt/Analyzer.scala +++ b/compile/interface/src/main/scala/xsbt/Analyzer.scala @@ -37,7 +37,7 @@ final class Analyzer(val global: CallbackGlobal) extends Compat for(on <- inheritedDependencies.getOrElse(sourceFile, Nil: Iterable[Symbol])) processDependency(on, inherited=true) def processDependency(on: Symbol, inherited: Boolean) { - def binaryDependency(file: File, className: String) = callback.binaryDependency(file, className, sourceFile /*, inherited*/) + def binaryDependency(file: File, className: String) = callback.binaryDependency(file, className, sourceFile, inherited) val onSource = on.sourceFile if(onSource == null) { @@ -55,7 +55,7 @@ final class Analyzer(val global: CallbackGlobal) extends Compat } } else - callback.sourceDependency(onSource.file, sourceFile /*, inherited*/) + callback.sourceDependency(onSource.file, sourceFile, inherited) } // build list of generated classes diff --git a/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala b/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala index 7f37f0b6b..cf6e05bd9 100644 --- a/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala +++ b/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala @@ -13,6 +13,7 @@ package inc import DefaultProtocol._ import DefaultProtocol.tuple2Format import Logger.{m2o, position, problem} + import Relations.{Source => RSource} object AnalysisFormats { @@ -97,8 +98,11 @@ object AnalysisFormats implicit def apisFormat(implicit internalF: Format[Map[File, Source]], externalF: Format[Map[String, Source]]): Format[APIs] = asProduct2( APIs.apply _)( as => (as.internal, as.external) )(internalF, externalF) - implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], intF: Format[RFF], extF: Format[RFS], csF: Format[RFS]): Format[Relations] = - asProduct5[Relations, RFF, RFF, RFF, RFS, RFS]( (a,b,c,d,e) => Relations.make(a,b,c,d,e) )( rs => (rs.srcProd, rs.binaryDep, rs.internalSrcDep, rs.externalDep, rs.classes) )(prodF, binF, intF, extF, csF) + implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], csF: Format[RFS]): Format[Relations] = + asProduct5[Relations, RFF, RFF, RSource, RSource, RFS]( (a,b,c,d,e) => Relations.make(a,b,c,d,e) )( rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.classes) )(prodF, binF, directF, inheritedF, csF) + + implicit def relationsSourceFormat(implicit internalFormat: Format[Relation[File, File]], externalFormat: Format[Relation[File,String]]): Format[RSource] = + asProduct2[RSource, RFF, RFS]( (a, b) => Relations.makeSource(a,b))( rs => (rs.internal, rs.external)) implicit def relationFormat[A,B](implicit af: Format[Map[A, Set[B]]], bf: Format[Map[B, Set[A]]]): Format[Relation[A,B]] = asProduct2[Relation[A,B], Map[A, Set[B]], Map[B, Set[A]]]( Relation.make _ )( r => (r.forwardMap, r.reverseMap) )(af, bf) diff --git a/interface/src/main/java/xsbti/AnalysisCallback.java b/interface/src/main/java/xsbti/AnalysisCallback.java index c23b43ecd..d00f5b7ed 100644 --- a/interface/src/main/java/xsbti/AnalysisCallback.java +++ b/interface/src/main/java/xsbti/AnalysisCallback.java @@ -12,11 +12,15 @@ public interface AnalysisCallback /** Called to indicate that the source file source depends on the source file * dependsOn. Note that only source files included in the current compilation will * passed to this method. Dependencies on classes generated by sources not in the current compilation will - * be passed as class dependencies to the classDependency method.*/ - public void sourceDependency(File dependsOn, File source); + * be passed as class dependencies to the classDependency method. + * If publicInherited is true, this dependency is a result of inheritance by a + * template accessible outside of the source file. */ + public void sourceDependency(File dependsOn, File source, boolean publicInherited); /** Called to indicate that the source file source depends on the top-level - * class named name from class or jar file binary. */ - public void binaryDependency(File binary, String name, File source); + * class named name from class or jar file binary. + * If publicInherited is true, this dependency is a result of inheritance by a + * template accessible outside of the source file. */ + public void binaryDependency(File binary, String name, File source, boolean publicInherited); /** Called to indicate that the source file source produces a class file at * module contain class name.*/ public void generatedClass(File source, File module, String name); diff --git a/util/classfile/src/main/scala/sbt/classfile/Analyze.scala b/util/classfile/src/main/scala/sbt/classfile/Analyze.scala index a85fc886d..843f129c5 100644 --- a/util/classfile/src/main/scala/sbt/classfile/Analyze.scala +++ b/util/classfile/src/main/scala/sbt/classfile/Analyze.scala @@ -49,14 +49,14 @@ private[sbt] object Analyze for (url <- Option(loader.getResource(tpe.replace('.', '/') + ClassExt)); file <- urlAsFile(url, log)) { if(url.getProtocol == "jar") - analysis.binaryDependency(file, tpe, source) + analysis.binaryDependency(file, tpe, source, /*inherited = */ false) // TODO: properly handle inherited else { assume(url.getProtocol == "file") productToSource.get(file) match { - case Some(dependsOn) => analysis.sourceDependency(dependsOn, source) - case None => analysis.binaryDependency(file, tpe, source) + case Some(dependsOn) => analysis.sourceDependency(dependsOn, source, /*inherited = */ false) + case None => analysis.binaryDependency(file, tpe, source, /*inherited = */ false) } } } From 658c3d06c4ae50cb18d0cbb26a4c32a858c5a248 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Fri, 26 Apr 2013 22:35:27 -0400 Subject: [PATCH 3/7] Use public inherited dependencies in incremental compilation invalidation. 1. All parents of public/exported classes/modules/packages are tracked as 'publicInherited' dependencies. These are dealiased and normalized so that the dependency is on the actual underlying template and not the source enclosing the alias. 2. All CompilationUnit.depends dependencies are direct dependencies. These include inherited dependencies. 3. When invalidating changed internal sources, a. Invalidate all inherited dependencies, transitively and include the originally modified sources, b. Invalidate all direct dependencies of these sources, c. Exclude any sources that were compiled in the previous step unless they depend on a newly invalidated source. 4. Invalidate changed external sources in the same way as #3 but remove the external sources from the final set. Only public inheritance dependencies need to be considered because a template that is not accessible outside its source file and that inherits from another file can be handled as a normal, direct dependency. Because the template isn't public, changes to its API will not propagate outside of the source file. Several existing tests cover the correctness, especially: 1. transitive-a covers direct, transitive dependencies with inferred return types 2. transitive-b covers inherited, transitive dependencies with inferred return types There are two new tests, one that tests that public inherited dependencies are tracked and one that verifies the basic invalidation progression. More tests are needed to verify the improvements that this algorithm brings: 1. Inheritance-related dependencies are processed in one step to avoid the otherwise unavoidable several steps. 2. Only immediate direct dependencies are ever processed, which should in many typical cases avoid large invalidation sets. --- .../src/main/scala/sbt/inc/Incremental.scala | 106 +++++++++--------- .../scala/sbt/inc/StronglyConnected.scala | 51 --------- .../inherited-dependencies/A.scala | 4 + .../inherited-dependencies/B.scala | 19 ++++ .../inherited-dependencies/C.scala | 1 + .../inherited-dependencies/D.scala | 1 + .../inherited-dependencies/E.scala | 1 + .../inherited-dependencies/F.scala | 3 + .../inherited-dependencies/G.scala | 1 + .../inherited-dependencies/J.scala | 1 + .../inherited-dependencies/build.sbt | 27 +++++ .../inherited-dependencies/test | 1 + .../less-inter-inv/A.scala | 3 + .../less-inter-inv/B.scala | 1 + .../less-inter-inv/C.scala | 1 + .../less-inter-inv/D.scala | 4 + .../less-inter-inv/E.scala | 3 + .../less-inter-inv/build.sbt | 10 ++ .../less-inter-inv/changes/A2.scala | 3 + .../source-dependencies/less-inter-inv/test | 9 ++ 20 files changed, 149 insertions(+), 101 deletions(-) delete mode 100644 compile/inc/src/main/scala/sbt/inc/StronglyConnected.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/A.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/B.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/C.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/D.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/E.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/F.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/G.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/J.scala create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/build.sbt create mode 100644 sbt/src/sbt-test/source-dependencies/inherited-dependencies/test create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/A.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/B.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/C.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/D.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/E.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/build.sbt create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/changes/A2.scala create mode 100644 sbt/src/sbt-test/source-dependencies/less-inter-inv/test diff --git a/compile/inc/src/main/scala/sbt/inc/Incremental.scala b/compile/inc/src/main/scala/sbt/inc/Incremental.scala index 561ed15b7..2f7f568ba 100644 --- a/compile/inc/src/main/scala/sbt/inc/Incremental.scala +++ b/compile/inc/src/main/scala/sbt/inc/Incremental.scala @@ -77,7 +77,7 @@ object Incremental debug("********* Merged: \n" + merged.relations + "\n*********") val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, options) - debug("Changes:\n" + incChanges) + debug("\nChanges:\n" + incChanges) val transitiveStep = options.transitiveStep val incInv = invalidateIncremental(merged.relations, incChanges, invalidated, cycleNum >= transitiveStep, log) cycle(incInv, allSources, emptyChanges, merged, doCompile, classfileManager, cycleNum+1, log, options) @@ -159,7 +159,7 @@ object Incremental if(transitive) transitiveDependencies(dependsOnSrc, changes.modified, log) else - invalidateStage2(dependsOnSrc, changes.modified, log) + invalidateIntermediate(previous, changes.modified, log) val dups = invalidateDuplicates(previous) if(dups.nonEmpty) @@ -176,53 +176,15 @@ object Incremental if(sources.size > 1) sources else Nil } toSet; - /** Only invalidates direct source dependencies. It excludes any sources that were recompiled during the previous run. - * Callers may want to augment the returned set with 'modified' or all sources recompiled up to this point. */ - def invalidateDirect(dependsOnSrc: File => Set[File], modified: Set[File]): Set[File] = - (modified flatMap dependsOnSrc) -- modified - - /** Invalidates transitive source dependencies including `modified`.*/ - @tailrec def invalidateTransitive(dependsOnSrc: File => Set[File], modified: Set[File], log: Logger): Set[File] = - { - val newInv = invalidateDirect(dependsOnSrc, modified) - log.debug("\tInvalidated direct: " + newInv) - if(newInv.isEmpty) modified else invalidateTransitive(dependsOnSrc, modified ++ newInv, log) - } - - /** Returns the transitive source dependencies of `initial`, excluding the files in `initial` in most cases. - * In three-stage incremental compilation, the `initial` files are the sources from step 2 that had API changes. - * Because strongly connected components (cycles) are included in step 2, the files with API changes shouldn't - * need to be compiled in step 3 if their dependencies haven't changed. If there are new cycles introduced after - * step 2, these can require step 2 sources to be included in step 3 recompilation. - */ + /** Returns the transitive source dependencies of `initial`. + * Because the intermediate steps do not pull in cycles, this result includes the initial files + * if they are part of a cycle containing newly invalidated files . */ def transitiveDependencies(dependsOnSrc: File => Set[File], initial: Set[File], log: Logger): Set[File] = { - // include any file that depends on included files - def recheck(included: Set[File], process: Set[File], excluded: Set[File]): Set[File] = - { - val newIncludes = (process flatMap dependsOnSrc) intersect excluded - if(newIncludes.isEmpty) - included - else - recheck(included ++ newIncludes, newIncludes, excluded -- newIncludes) - } - val transitiveOnly = transitiveDepsOnly(initial)(dependsOnSrc) - log.debug("Step 3 transitive dependencies:\n\t" + transitiveOnly) - val stage3 = recheck(transitiveOnly, transitiveOnly, initial) - log.debug("Step 3 sources from new step 2 source dependencies:\n\t" + (stage3 -- transitiveOnly)) - stage3 - } - - - def invalidateStage2(dependsOnSrc: File => Set[File], initial: Set[File], log: Logger): Set[File] = - { - val initAndImmediate = initial ++ initial.flatMap(dependsOnSrc) - log.debug("Step 2 changed sources and immdediate dependencies:\n\t" + initAndImmediate) - val components = sbt.inc.StronglyConnected(initAndImmediate)(dependsOnSrc) - log.debug("Non-trivial strongly connected components: " + components.filter(_.size > 1).mkString("\n\t", "\n\t", "")) - val inv = components.filter(initAndImmediate.exists).flatten - log.debug("Step 2 invalidated sources:\n\t" + inv) - inv + val transitiveWithInitial = transitiveDeps(initial)(dependsOnSrc) + val transitivePartial = includeInitialCond(initial, transitiveWithInitial, dependsOnSrc, log) + log.debug("Final step, transitive dependencies:\n\t" + transitivePartial) + transitivePartial } /** Invalidates sources based on initially detected 'changes' to the sources, products, and dependencies.*/ @@ -232,7 +194,7 @@ object Incremental val srcDirect = srcChanges.removed ++ srcChanges.removed.flatMap(previous.usesInternalSrc) ++ srcChanges.added ++ srcChanges.changed val byProduct = changes.removedProducts.flatMap(previous.produced) val byBinaryDep = changes.binaryDeps.flatMap(previous.usesBinary) - val byExtSrcDep = changes.external.modified.flatMap(previous.usesExternal) // ++ scopeInvalidations + val byExtSrcDep = invalidateByExternal(previous, changes.external.modified, log) //changes.external.modified.flatMap(previous.usesExternal) // ++ scopeInvalidations log.debug( "\nInitial source changes: \n\tremoved:" + srcChanges.removed + "\n\tadded: " + srcChanges.added + "\n\tmodified: " + srcChanges.changed + "\nRemoved products: " + changes.removedProducts + @@ -248,6 +210,51 @@ object Incremental srcDirect ++ byProduct ++ byBinaryDep ++ byExtSrcDep } + /** Sources invalidated by `external` sources in other projects according to the previous `relations`. */ + def invalidateByExternal(relations: Relations, external: Set[String], log: Logger): Set[File] = + { + // Propagate public inheritance dependencies transitively. + // This differs from normal because we need the initial crossing from externals to sources in this project. + val externalInheritedR = relations.publicInherited.external + val byExternalInherited = external flatMap externalInheritedR.reverse + val internalInheritedR = relations.publicInherited.internal + val transitiveInherited = transitiveDeps(byExternalInherited)(internalInheritedR.reverse _) + + // Get the direct dependencies of all sources transitively invalidated by inheritance + val directA = transitiveInherited flatMap relations.direct.internal.reverse + // Get the sources that directly depend on externals. This includes non-inheritance dependencies and is not transitive. + val directB = external flatMap relations.direct.external.reverse + transitiveInherited ++ directA ++ directB + } + /** Intermediate invalidation step: steps after the initial invalidation, but before the final transitive invalidation. */ + def invalidateIntermediate(relations: Relations, modified: Set[File], log: Logger): Set[File] = + { + def reverse(r: Relations.Source) = r.internal.reverse _ + invalidateSources(reverse(relations.direct), reverse(relations.publicInherited), modified, log) + } + /** Invalidates inheritance dependencies, transitively. Then, invalidates direct dependencies. Finally, excludes initial dependencies not + * included in a cycle with newly invalidated sources. */ + private[this] def invalidateSources(directDeps: File => Set[File], publicInherited: File => Set[File], initial: Set[File], log: Logger): Set[File] = + { + val transitiveInherited = transitiveDeps(initial)(publicInherited) + log.debug("Invalidated by transitive public inheritance: " + transitiveInherited) + val direct = transitiveInherited flatMap directDeps + log.debug("Invalidated by direct dependency: " + direct) + val all = transitiveInherited ++ direct + includeInitialCond(initial, all, f => directDeps(f) ++ publicInherited(f), log) + } + /** Conditionally include initial sources that are dependencies of newly invalidated sources. + ** Initial sources included in this step can be because of a cycle, but not always. */ + private[this] def includeInitialCond(initial: Set[File], currentInvalidations: Set[File], allDeps: File => Set[File], log: Logger): Set[File] = + { + val newInv = currentInvalidations -- initial + log.debug("New invalidations:\n\t" + newInv) + val transitiveOfNew = transitiveDeps(newInv)(allDeps) + val initialDependsOnNew = transitiveOfNew & initial + log.debug("Previously invalidated, but (transitively) depend on new invalidations:\n\t" + initialDependsOnNew) + newInv ++ initialDependsOnNew + } + def prune(invalidatedSrcs: Set[File], previous: Analysis): Analysis = prune(invalidatedSrcs, previous, ClassfileManager.deleteImmediately()) @@ -309,7 +316,7 @@ object Incremental def orEmpty(o: Option[Source]): Source = o getOrElse APIs.emptySource def orTrue(o: Option[Boolean]): Boolean = o getOrElse true - private[this] def transitiveDepsOnly[T](nodes: Iterable[T])(dependencies: T => Iterable[T]): Set[T] = + private[this] def transitiveDeps[T](nodes: Iterable[T])(dependencies: T => Iterable[T]): Set[T] = { val xs = new collection.mutable.HashSet[T] def all(ns: Iterable[T]): Unit = ns.foreach(visit) @@ -319,7 +326,6 @@ object Incremental all(dependencies(n)) } all(nodes) - xs --= nodes xs.toSet } diff --git a/compile/inc/src/main/scala/sbt/inc/StronglyConnected.scala b/compile/inc/src/main/scala/sbt/inc/StronglyConnected.scala deleted file mode 100644 index 89d8ac4a0..000000000 --- a/compile/inc/src/main/scala/sbt/inc/StronglyConnected.scala +++ /dev/null @@ -1,51 +0,0 @@ -package sbt.inc - -// stolen from Josh -object StronglyConnected -{ - def apply[N](nodes: Iterable[N])(dependencies: N => Iterable[N]): Set[Set[N]] = - { - val stack = new collection.mutable.Stack[N] - val onStack = new collection.mutable.HashSet[N] - val scc = new collection.mutable.ArrayBuffer[Set[N]] - val index = new collection.mutable.ArrayBuffer[N] - val lowLink = new collection.mutable.HashMap[N, Int] - - def tarjanImpl(v: N) - { - index += v - lowLink(v) = index.size-1 - stack.push(v) - onStack += v - for(n <- dependencies(v)) - { - if( !index.contains(n) ) - { - tarjanImpl(n) - lowLink(v) = math.min(lowLink(v), lowLink(n)) - } - else if(onStack(n)) - lowLink(v) = math.min(lowLink(v), index.indexOf(n)) - } - - if(lowLink(v) == index.indexOf(v)) - { - val components = new collection.mutable.ArrayBuffer[N] - def popLoop() - { - val popped = stack.pop() - onStack -= popped - components.append(popped) - if(popped != v) popLoop() - } - popLoop() - scc.append(components.toSet) - } - } - - for(node <- nodes) - if( !index.contains(node) ) - tarjanImpl(node) - scc.toSet - } -} diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/A.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/A.scala new file mode 100644 index 000000000..30853bb78 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/A.scala @@ -0,0 +1,4 @@ +// T is a type constructor [x]C +// C extends D +// E verifies the core type gets pulled out +trait A extends B.T[Int] with (E[Int] @unchecked) diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/B.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/B.scala new file mode 100644 index 000000000..9c6fbe046 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/B.scala @@ -0,0 +1,19 @@ +object B { + type T[x] = C +} + +class B { + // not public, so this shouldn't be tracked as an inherited dependency + private[this] class X extends D with E[Int] + + def x(i: Int) { + // not public, not an inherited dependency + trait Y extends D + } + + def y(j: Int) { + // not public + val w: D { def length: Int } = ??? + () + } +} \ No newline at end of file diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/C.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/C.scala new file mode 100644 index 000000000..360899d9f --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/C.scala @@ -0,0 +1 @@ +trait C extends D diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/D.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/D.scala new file mode 100644 index 000000000..804e77004 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/D.scala @@ -0,0 +1 @@ +trait D extends G.P diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/E.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/E.scala new file mode 100644 index 000000000..fa7c94867 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/E.scala @@ -0,0 +1 @@ +trait E[T] diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/F.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/F.scala new file mode 100644 index 000000000..8c26474b5 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/F.scala @@ -0,0 +1,3 @@ +class F { + def q: C { def length: Int } = ??? +} \ No newline at end of file diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/G.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/G.scala new file mode 100644 index 000000000..1fd92c068 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/G.scala @@ -0,0 +1 @@ +object G { trait P extends J } \ No newline at end of file diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/J.scala b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/J.scala new file mode 100644 index 000000000..62eeb6c96 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/J.scala @@ -0,0 +1 @@ +class J \ No newline at end of file diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/build.sbt b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/build.sbt new file mode 100644 index 000000000..15a6a4dba --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/build.sbt @@ -0,0 +1,27 @@ +lazy val verifyDeps = taskKey[Unit]("verify inherited dependencies are properly extracted") + +verifyDeps := { + val a = compile.in(Compile).value + same(a.relations.publicInherited.internal.forwardMap, expectedDeps.forwardMap) +} + +lazy val expected = Seq( + "A" -> Seq("C", "D", "E", "G", "J"), + "B" -> Seq(), + "C" -> Seq("D", "G", "J"), + "D" -> Seq("G", "J"), + "E" -> Seq(), + "F" -> Seq("C", "D", "G", "J"), + "G" -> Seq("J"), + "J" -> Seq() +) +lazy val pairs = + expected.map { case (from,tos) => + (toFile(from), tos.map(toFile)) + } +lazy val expectedDeps = (Relation.empty[File,File] /: pairs) { case (r, (x,ys)) => r + (x,ys) } +def toFile(s: String) = file(s + ".scala").getAbsoluteFile + +def same[T](x: T, y: T) { + assert(x == y, s"\nActual: $x, \nExpected: $y") +} diff --git a/sbt/src/sbt-test/source-dependencies/inherited-dependencies/test b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/test new file mode 100644 index 000000000..e5d477601 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/inherited-dependencies/test @@ -0,0 +1 @@ +> verifyDeps diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/A.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/A.scala new file mode 100644 index 000000000..a4f92f4fa --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/A.scala @@ -0,0 +1,3 @@ +class A { + def x = 3 +} diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/B.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/B.scala new file mode 100644 index 000000000..a18aec3db --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/B.scala @@ -0,0 +1 @@ +class B extends A diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/C.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/C.scala new file mode 100644 index 000000000..f6f5bb28f --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/C.scala @@ -0,0 +1 @@ +class C extends B diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/D.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/D.scala new file mode 100644 index 000000000..55959c2a9 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/D.scala @@ -0,0 +1,4 @@ +object D { + val c = new C + def x: String = c.x.toString +} diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/E.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/E.scala new file mode 100644 index 000000000..f393ca20d --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/E.scala @@ -0,0 +1,3 @@ +object E extends App { + assert(D.x == "3") +} diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/build.sbt b/sbt/src/sbt-test/source-dependencies/less-inter-inv/build.sbt new file mode 100644 index 000000000..d23dff705 --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/build.sbt @@ -0,0 +1,10 @@ +import complete.DefaultParsers._ + +val checkIterations = inputKey[Unit]("Verifies the accumlated number of iterations of incremental compilation.") + +checkIterations := { + val expected: Int = (Space ~> NatBasic).parsed + val actual: Int = (compile in Compile).value.compilations.allCompilations.size + assert(expected == actual, s"Expected $expected compilations, got $actual") +} + diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/changes/A2.scala b/sbt/src/sbt-test/source-dependencies/less-inter-inv/changes/A2.scala new file mode 100644 index 000000000..acab4a1ae --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/changes/A2.scala @@ -0,0 +1,3 @@ +class A { + def x = "3" +} diff --git a/sbt/src/sbt-test/source-dependencies/less-inter-inv/test b/sbt/src/sbt-test/source-dependencies/less-inter-inv/test new file mode 100644 index 000000000..c6df5698e --- /dev/null +++ b/sbt/src/sbt-test/source-dependencies/less-inter-inv/test @@ -0,0 +1,9 @@ +# 1 iteration from initial full compile +> run +$ copy-file changes/A2.scala A.scala + +# 1 iteration for the initial changes +# 1 iteration to recompile all descendents and direct dependencies +# no further iteration, because APIs of directs don't change +> run +> checkIterations 3 From bedc8dbb10ceb3da9de23e09d9a7fdc38a683fd0 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Fri, 26 Apr 2013 22:35:27 -0400 Subject: [PATCH 4/7] Push full transitive invalidation out a step since step 3 is now relatively cheap. --- compile/inc/src/main/scala/sbt/inc/IncOptions.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compile/inc/src/main/scala/sbt/inc/IncOptions.scala b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala index 5be46d85f..5d0604ee8 100644 --- a/compile/inc/src/main/scala/sbt/inc/IncOptions.scala +++ b/compile/inc/src/main/scala/sbt/inc/IncOptions.scala @@ -38,7 +38,10 @@ final case class IncOptions( object IncOptions { val Default = IncOptions( - transitiveStep = 2, + // 1. recompile changed sources + // 2(3). recompile direct dependencies and transitive public inheritance dependencies of sources with API changes in 1(2). + // 4. further changes invalidate all dependencies transitively to avoid too many steps + transitiveStep = 3, recompileAllFraction = 0.5, relationsDebug = false, apiDebug = false, From 429131bdd879ee8196ada25dcaf44ddc049890e1 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Sat, 27 Apr 2013 16:27:29 -0400 Subject: [PATCH 5/7] fix compilation error in TestCallback --- interface/src/test/scala/TestCallback.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface/src/test/scala/TestCallback.scala b/interface/src/test/scala/TestCallback.scala index 096d73a83..061457723 100644 --- a/interface/src/test/scala/TestCallback.scala +++ b/interface/src/test/scala/TestCallback.scala @@ -7,15 +7,15 @@ class TestCallback extends AnalysisCallback { val beganSources = new ArrayBuffer[File] val endedSources = new ArrayBuffer[File] - val sourceDependencies = new ArrayBuffer[(File, File)] - val binaryDependencies = new ArrayBuffer[(File, String, File)] + val sourceDependencies = new ArrayBuffer[(File, File, Boolean)] + val binaryDependencies = new ArrayBuffer[(File, String, File, Boolean)] val products = new ArrayBuffer[(File, File, String)] val apis = new ArrayBuffer[(File, xsbti.api.SourceAPI)] def beginSource(source: File) { beganSources += source } - def sourceDependency(dependsOn: File, source: File) { sourceDependencies += ((dependsOn, source)) } - def binaryDependency(binary: File, name: String, source: File) { binaryDependencies += ((binary, name, source)) } + def sourceDependency(dependsOn: File, source: File, inherited: Boolean) { sourceDependencies += ((dependsOn, source, inherited)) } + def binaryDependency(binary: File, name: String, source: File, inherited: Boolean) { binaryDependencies += ((binary, name, source, inherited)) } def generatedClass(source: File, module: File, name: String) { products += ((source, module, name)) } def endSource(source: File) { endedSources += source } From 435bd1d587bfc7a3ed1720d315032f80a4111e0a Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Wed, 1 May 2013 09:35:51 -0400 Subject: [PATCH 6/7] Only invalidate package objects that inherit from invalidated files. Originally described in cf355f18224332c3311b552cd9a777f5624426a4. --- compile/inc/src/main/scala/sbt/inc/Incremental.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/inc/src/main/scala/sbt/inc/Incremental.scala b/compile/inc/src/main/scala/sbt/inc/Incremental.scala index 2f7f568ba..88289a426 100644 --- a/compile/inc/src/main/scala/sbt/inc/Incremental.scala +++ b/compile/inc/src/main/scala/sbt/inc/Incremental.scala @@ -96,10 +96,10 @@ object Incremental else invalidated } - // Package objects are fragile: if they depend on an invalidated source, get "class file needed by package is missing" error + // Package objects are fragile: if they inherit from an invalidated source, get "class file needed by package is missing" error // This might be too conservative: we probably only need package objects for packages of invalidated sources. private[this] def invalidatedPackageObjects(invalidated: Set[File], relations: Relations): Set[File] = - invalidated flatMap relations.usesInternalSrc filter { _.getName == "package.scala" } + invalidated flatMap relations.publicInherited.internal.reverse filter { _.getName == "package.scala" } /** * Accepts the sources that were recompiled during the last step and functions From a867d8e87c158582b965776beacad6111967a5b7 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Wed, 1 May 2013 17:54:10 -0400 Subject: [PATCH 7/7] extract public inherited dependencies from Java class files --- .../api/src/main/scala/sbt/ClassToAPI.scala | 48 ++++++++++++++----- .../sbt/compiler/AggressiveCompile.scala | 6 ++- .../main/scala/sbt/classfile/Analyze.scala | 18 ++++--- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/compile/api/src/main/scala/sbt/ClassToAPI.scala b/compile/api/src/main/scala/sbt/ClassToAPI.scala index 5af7b3d50..bdab5e1e5 100644 --- a/compile/api/src/main/scala/sbt/ClassToAPI.scala +++ b/compile/api/src/main/scala/sbt/ClassToAPI.scala @@ -10,11 +10,18 @@ import collection.mutable object ClassToAPI { - def apply(c: Seq[Class[_]]): api.SourceAPI = + def apply(c: Seq[Class[_]]): api.SourceAPI = process(c)._1 + + // (api, public inherited classes) + def process(c: Seq[Class[_]]): (api.SourceAPI, Set[Class[_]]) = { val pkgs = packages(c).map(p => new api.Package(p)) - val defs = c.filter(isTopLevel).flatMap(toDefinitions(new mutable.HashMap)) - new api.SourceAPI(pkgs.toArray, defs.toArray) + val cmap = emptyClassMap + val defs = c.filter(isTopLevel).flatMap(toDefinitions(cmap)) + val source = new api.SourceAPI(pkgs.toArray, defs.toArray) + cmap.lz.foreach(_.get()) // force thunks to ensure all inherited dependencies are recorded + cmap.clear() + (source, cmap.inherited.toSet) } // Avoiding implicit allocation. @@ -35,9 +42,13 @@ object ClassToAPI def isTopLevel(c: Class[_]): Boolean = c.getEnclosingClass eq null - type ClassMap = mutable.Map[String, Seq[api.ClassLike]] + final class ClassMap private[sbt](private[sbt] val memo: mutable.Map[String, Seq[api.ClassLike]], private[sbt] val inherited: mutable.Set[Class[_]], private[sbt] val lz: mutable.Buffer[xsbti.api.Lazy[_]]) { + def clear() { memo.clear(); inherited.clear(); lz.clear() } + } + def emptyClassMap: ClassMap = new ClassMap(new mutable.HashMap, new mutable.HashSet, new mutable.ListBuffer) + def toDefinitions(cmap: ClassMap)(c: Class[_]): Seq[api.ClassLike] = - cmap.getOrElseUpdate(c.getName, toDefinitions0(c, cmap)) + cmap.memo.getOrElseUpdate(c.getName, toDefinitions0(c, cmap)) def toDefinitions0(c: Class[_], cmap: ClassMap): Seq[api.ClassLike] = { import api.DefinitionType.{ClassDef, Module, Trait} @@ -48,13 +59,14 @@ object ClassToAPI val name = c.getName val tpe = if(Modifier.isInterface(c.getModifiers)) Trait else ClassDef lazy val (static, instance) = structure(c, enclPkg, cmap) - val cls = new api.ClassLike(tpe, strict(Empty), lzy(instance), emptyStringArray, typeParameters(c.getTypeParameters), name, acc, mods, annots) - val stat = new api.ClassLike(Module, strict(Empty), lzy(static), emptyStringArray, emptyTypeParameterArray, name, acc, mods, annots) + val cls = new api.ClassLike(tpe, strict(Empty), lzy(instance, cmap), emptyStringArray, typeParameters(c.getTypeParameters), name, acc, mods, annots) + val stat = new api.ClassLike(Module, strict(Empty), lzy(static, cmap), emptyStringArray, emptyTypeParameterArray, name, acc, mods, annots) val defs = cls :: stat :: Nil - cmap(c.getName) = defs + cmap.memo(c.getName) = defs defs } + /** Returns the (static structure, instance structure, inherited classes) for `c`. */ def structure(c: Class[_], enclPkg: Option[String], cmap: ClassMap): (api.Structure, api.Structure) = { val methods = mergeMap(c, c.getDeclaredMethods, c.getMethods, methodToDef(enclPkg)) @@ -62,20 +74,29 @@ object ClassToAPI val constructors = mergeMap(c, c.getDeclaredConstructors, c.getConstructors, constructorToDef(enclPkg)) val classes = merge[Class[_]](c, c.getDeclaredClasses, c.getClasses, toDefinitions(cmap), (_: Seq[Class[_]]).partition(isStatic), _.getEnclosingClass != c) val all = (methods ++ fields ++ constructors ++ classes) - val parentTypes = parents(c) - val instanceStructure = new api.Structure(lzy(parentTypes.toArray), lzy(all.declared.toArray), lzy(all.inherited.toArray)) - val staticStructure = new api.Structure(lzyEmptyTpeArray, lzy(all.staticDeclared.toArray), lzy(all.staticInherited.toArray)) + val parentJavaTypes = allSuperTypes(c) + if(!Modifier.isPrivate(c.getModifiers)) + cmap.inherited ++= parentJavaTypes.collect { case c: Class[_] => c } + val parentTypes = types(parentJavaTypes) + val instanceStructure = new api.Structure(lzyS(parentTypes.toArray), lzyS(all.declared.toArray), lzyS(all.inherited.toArray)) + val staticStructure = new api.Structure(lzyEmptyTpeArray, lzyS(all.staticDeclared.toArray), lzyS(all.staticInherited.toArray)) (staticStructure, instanceStructure) } + private[this] def lzyS[T <: AnyRef](t: T): xsbti.api.Lazy[T] = lzy(t) def lzy[T <: AnyRef](t: => T): xsbti.api.Lazy[T] = xsbti.SafeLazy(t) + private[this] def lzy[T <: AnyRef](t: => T, cmap: ClassMap): xsbti.api.Lazy[T] = { + val s = lzy(t) + cmap.lz += s + s + } private val emptyStringArray = new Array[String](0) private val emptyTypeArray = new Array[xsbti.api.Type](0) private val emptyAnnotationArray = new Array[xsbti.api.Annotation](0) private val emptyTypeParameterArray = new Array[xsbti.api.TypeParameter](0) private val emptySimpleTypeArray = new Array[xsbti.api.SimpleType](0) - private val lzyEmptyTpeArray = lzy(emptyTypeArray) - private val lzyEmptyDefArray = lzy(new Array[xsbti.api.Definition](0)) + private val lzyEmptyTpeArray = lzyS(emptyTypeArray) + private val lzyEmptyDefArray = lzyS(new Array[xsbti.api.Definition](0)) private def allSuperTypes(t: Type): Seq[Type] = { @@ -101,6 +122,7 @@ object ClassToAPI accumulate(t).filterNot(_ == null).distinct } + @deprecated("No longer used", "0.13.0") def parents(c: Class[_]): Seq[api.Type] = types(allSuperTypes(c)) def types(ts: Seq[Type]): Array[api.Type] = ts filter (_ ne null) map reference toArray; def upperBounds(ts: Array[Type]): api.Type = diff --git a/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala b/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala index b33db6d84..2ff410e4d 100644 --- a/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala +++ b/compile/integration/src/main/scala/sbt/compiler/AggressiveCompile.scala @@ -126,7 +126,11 @@ class AggressiveCompile(cacheFile: File) javac.compile(javaSrcs.toArray, absClasspath.toArray, output, options.javacOptions.toArray, log) } - def readAPI(source: File, classes: Seq[Class[_]]) { callback.api(source, ClassToAPI(classes)) } + def readAPI(source: File, classes: Seq[Class[_]]): Set[String] = { + val (api, inherits) = ClassToAPI.process(classes) + callback.api(source, api) + inherits.map(_.getName) + } timed("Java analysis", log) { for ((classesFinder, oldClasses, srcs) <- memo) { diff --git a/util/classfile/src/main/scala/sbt/classfile/Analyze.scala b/util/classfile/src/main/scala/sbt/classfile/Analyze.scala index 843f129c5..117b718ae 100644 --- a/util/classfile/src/main/scala/sbt/classfile/Analyze.scala +++ b/util/classfile/src/main/scala/sbt/classfile/Analyze.scala @@ -15,7 +15,7 @@ import java.net.URL private[sbt] object Analyze { - def apply[T](newClasses: Seq[File], sources: Seq[File], log: Logger)(analysis: xsbti.AnalysisCallback, loader: ClassLoader, readAPI: (File,Seq[Class[_]]) => Unit) + def apply[T](newClasses: Seq[File], sources: Seq[File], log: Logger)(analysis: xsbti.AnalysisCallback, loader: ClassLoader, readAPI: (File,Seq[Class[_]]) => Set[String]) { val sourceMap = sources.toSet[File].groupBy(_.getName) @@ -42,29 +42,33 @@ private[sbt] object Analyze // get class to class dependencies and map back to source to class dependencies for( (source, classFiles) <- sourceToClassFiles ) { - def processDependency(tpe: String) + val publicInherited = readAPI(source, classFiles.toSeq.flatMap(c => load(c.className, Some("Error reading API from class file") ))) + + def processDependency(tpe: String, inherited: Boolean) { trapAndLog(log) { for (url <- Option(loader.getResource(tpe.replace('.', '/') + ClassExt)); file <- urlAsFile(url, log)) { if(url.getProtocol == "jar") - analysis.binaryDependency(file, tpe, source, /*inherited = */ false) // TODO: properly handle inherited + analysis.binaryDependency(file, tpe, source, inherited) else { assume(url.getProtocol == "file") productToSource.get(file) match { - case Some(dependsOn) => analysis.sourceDependency(dependsOn, source, /*inherited = */ false) - case None => analysis.binaryDependency(file, tpe, source, /*inherited = */ false) + case Some(dependsOn) => analysis.sourceDependency(dependsOn, source, inherited) + case None => analysis.binaryDependency(file, tpe, source, inherited) } } } } } + def processDependencies(tpes: Iterable[String], inherited: Boolean): Unit = tpes.foreach(tpe => processDependency(tpe, inherited)) - classFiles.flatMap(_.types).toSet.foreach(processDependency) - readAPI(source, classFiles.toSeq.flatMap(c => load(c.className, Some("Error reading API from class file") ))) + val notInherited = classFiles.flatMap(_.types).toSet -- publicInherited + processDependencies(notInherited, false) + processDependencies(publicInherited, true) analysis.endSource(source) }