diff --git a/compile/inc/src/main/scala/sbt/inc/Analysis.scala b/compile/inc/src/main/scala/sbt/inc/Analysis.scala index 956c146cb..212b6fc6d 100644 --- a/compile/inc/src/main/scala/sbt/inc/Analysis.scala +++ b/compile/inc/src/main/scala/sbt/inc/Analysis.scala @@ -59,7 +59,7 @@ object Analysis /** Merge multiple analysis objects into one. Deps will be internalized as needed. */ def merge(analyses: Traversable[Analysis]): Analysis = { - if (analyses.exists(_.relations.memberRefAndInheritanceDeps)) + if (analyses.exists(_.relations.nameHashing)) throw new IllegalArgumentException("Merging of Analyses that have" + "`relations.memberRefAndInheritanceDeps` set to `true` is not supported.") @@ -160,7 +160,7 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos ) def groupBy[K](discriminator: File => K): Map[K, Analysis] = { - if (relations.memberRefAndInheritanceDeps) + if (relations.nameHashing) throw new UnsupportedOperationException("Grouping of Analyses that have" + "`relations.memberRefAndInheritanceDeps` set to `true` is not supported.") diff --git a/compile/inc/src/main/scala/sbt/inc/Compile.scala b/compile/inc/src/main/scala/sbt/inc/Compile.scala index d9ad4cf51..2d325a4d9 100644 --- a/compile/inc/src/main/scala/sbt/inc/Compile.scala +++ b/compile/inc/src/main/scala/sbt/inc/Compile.scala @@ -67,6 +67,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external import collection.mutable.{HashMap, HashSet, ListBuffer, Map, Set} private[this] val apis = new HashMap[File, (Int, SourceAPI)] + private[this] val usedNames = new HashMap[File, Set[String]] private[this] val unreporteds = new HashMap[File, ListBuffer[Problem]] private[this] val reporteds = new HashMap[File, ListBuffer[Problem]] private[this] val binaryDeps = new HashMap[File, Set[File]] @@ -151,9 +152,11 @@ private final class AnalysisCallback(internalMap: File => Option[File], external apis(sourceFile) = (HashAPI(source), savedSource) } - def memberRefAndInheritanceDeps: Boolean = false // TODO: define the flag in IncOptions which controls this + def usedName(sourceFile: File, name: String) = add(usedNames, sourceFile, name) - def get: Analysis = addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) ) + def nameHashing: Boolean = false // TODO: define the flag in IncOptions which controls this + + def get: Analysis = addUsedNames( addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) ) ) def addProducts(base: Analysis): Analysis = addAll(base, classes) { case (a, src, (prod, name)) => a.addProduct(src, prod, current product prod, name ) } def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, binaryClassName(bin), current binary bin) ) def addSources(base: Analysis): Analysis = @@ -171,6 +174,9 @@ private final class AnalysisCallback(internalMap: File => Option[File], external def getOrNil[A,B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api, inherited)) => a.addExternalDep(source, name, api, inherited) } def addCompilation(base: Analysis): Analysis = base.copy(compilations = base.compilations.add(compilation)) + def addUsedNames(base: Analysis): Analysis = (base /: usedNames) { case (a, (src, names)) => + (a /: names) { case (a, name) => a.copy(relations = a.relations.addUsedName(src, name)) } + } def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis = (base /: m) { case (outer, (a, bs)) => diff --git a/compile/inc/src/main/scala/sbt/inc/Relations.scala b/compile/inc/src/main/scala/sbt/inc/Relations.scala index 3f1b34725..b03b519e0 100644 --- a/compile/inc/src/main/scala/sbt/inc/Relations.scala +++ b/compile/inc/src/main/scala/sbt/inc/Relations.scala @@ -58,6 +58,8 @@ trait Relations /** Internal source dependencies that depend on external source file `dep`. This includes both direct and inherited dependencies. */ def usesExternal(dep: String): Set[File] + private[inc] def usedNames(src: File): Set[String] + /** Records internal source file `src` as generating class file `prod` with top-level class `name`. */ def addProduct(src: File, prod: File, name: String): Relations @@ -74,6 +76,8 @@ trait Relations * this method does not automatically record direct dependencies like `addExternalDep` does.*/ def addInternalSrcDeps(src: File, directDependsOn: Iterable[File], inheritedDependsOn: Iterable[File]): Relations + private[inc] def addUsedName(src: File, name: String): Relations + /** Concatenates the two relations. Acts naively, i.e., doesn't internalize external deps on added files. */ def ++ (o: Relations): Relations @@ -105,7 +109,7 @@ trait Relations * This relation properly accounts for that so the invariant that `memberRef` is a superset * of `inheritance` is preserved. */ - def memberRef: SourceDependencies + private[inc] def memberRef: SourceDependencies /** * The source dependency relation between source files introduced by inheritance. @@ -135,7 +139,7 @@ trait Relations * resolved transitively. You should not rely on this behavior, though. * */ - def inheritance: SourceDependencies + private[inc] def inheritance: SourceDependencies /** The dependency relations between sources. These include both direct and inherited dependencies.*/ def direct: Source @@ -147,17 +151,26 @@ trait Relations def classes: Relation[File, String] /** - * Flag which indicates whether the new style (based on `memberRef` and `inheritance` source dependencies) - * of dependency tracking is enabled. When this flag is enabled access to `direct` and `publicInherited` - * relations is illegal and will cause runtime exception being thrown. + * Flag which indicates whether given Relations object supports operations needed by name hashing algorithm. * - * Conversely, when `memberRefAndInheritanceDeps` flag is disabled access to `memberRef` and `inheritance` - * relations is illegal and will cause runtime exception being thrown. + * At the moment the list includes the following operations: * - * The name of this flag is ugly but it's private to incremental compiler and it's temporary measure during - * our migration to the new dependency tracking. + * - memberRef: SourceDependencies + * - inheritance: SourceDependencies + * + * The `memberRef` and `inheritance` implement a new style source dependency tracking. When this flag is + * enabled access to `direct` and `publicInherited` relations is illegal and will cause runtime exception + * being thrown. That is done as an optimization that prevents from storing two overlapping sets of + * dependencies. + * + * Conversely, when `nameHashing` flag is disabled access to `memberRef` and `inheritance` + * relations is illegal and will cause runtime exception being thrown. */ - private[inc] def memberRefAndInheritanceDeps: Boolean + private[inc] def nameHashing: Boolean + /** + * Relation between source files and _unqualified_ term and type names used in given source file. + */ + private[inc] def names: Relation[File, String] } @@ -213,19 +226,21 @@ object Relations def emptySource: Source = es private[inc] lazy val emptySourceDependencies: SourceDependencies = new SourceDependencies(e, estr) - def empty: Relations = empty(memberRefAndInheritanceDeps = false) - def empty(memberRefAndInheritanceDeps: Boolean): Relations = - if (memberRefAndInheritanceDeps) - new MRelationsMemberRefAndInheritance(e, e, emptySourceDependencies, emptySourceDependencies, estr) + def empty: Relations = empty(nameHashing = false) + private[inc] def empty(nameHashing: Boolean): Relations = + if (nameHashing) + new MRelationsNameHashing(e, e, emptySourceDependencies, emptySourceDependencies, estr, estr) else - new MRelationsDirectAndPublicInherited(e, e, es, es, estr) + new MRelationsDefaultImpl(e, e, es, es, estr) def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], direct: Source, publicInherited: Source, classes: Relation[File, String]): Relations = - new MRelationsDirectAndPublicInherited(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes) + new MRelationsDefaultImpl(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes) private[inc] def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], - memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String]): Relations = - new MRelationsMemberRefAndInheritance(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance, classes) + memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String], + names: Relation[File, String]): Relations = + new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance, + classes, names) def makeSource(internal: Relation[File,File], external: Relation[File,String]): Source = new Source(internal, external) private[inc] def makeSourceDependencies(internal: Relation[File,File], external: Relation[File,String]): SourceDependencies = new SourceDependencies(internal, external) } @@ -276,6 +291,8 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b def externalDeps(src: File): Set[String] = externalDep.forward(src) def usesExternal(dep: String): Set[File] = externalDep.reverse(dep) + def usedNames(src: File): Set[String] = names.forward(src) + /** Making large Relations a little readable. */ private val userDir = sys.props("user.dir").stripSuffix("/") + "/" private def nocwd(s: String) = s stripPrefix userDir @@ -309,7 +326,7 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b * introduced by inheritance. * */ -private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], binaryDep: Relation[File, File], +private class MRelationsDefaultImpl(srcProd: Relation[File, File], binaryDep: Relation[File, File], // direct should include everything in inherited val direct: Source, val publicInherited: Source, classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes) @@ -317,45 +334,53 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], def internalSrcDep: Relation[File, File] = direct.internal def externalDep: Relation[File, String] = direct.external - def memberRefAndInheritanceDeps: Boolean = false + def nameHashing: Boolean = false def memberRef: SourceDependencies = throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " + - "when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?") + "when `nameHashing` flag is disabled.") def inheritance: SourceDependencies = throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " + - "when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?") + "when `nameHashing` flag is disabled.") def addProduct(src: File, prod: File, name: String): Relations = - new MRelationsDirectAndPublicInherited(srcProd + (src, prod), binaryDep, direct = direct, + new MRelationsDefaultImpl(srcProd + (src, prod), binaryDep, direct = direct, publicInherited = publicInherited, classes + (src, name)) def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = { val newI = if(inherited) publicInherited.addExternal(src, dependsOn) else publicInherited val newD = direct.addExternal(src, dependsOn) - new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes) + new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes) } def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations = { val newI = publicInherited.addInternal(src, inherited) val newD = direct.addInternal(src, dependsOn) - new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes) + new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes) } + def names: Relation[File, String] = + throw new UnsupportedOperationException("Tracking of used names is not supported " + + "when `nameHashing` is disabled.") + + def addUsedName(src: File, name: String): Relations = + throw new UnsupportedOperationException("Tracking of used names is not supported " + + "when `nameHashing` is disabled.") + def addBinaryDep(src: File, dependsOn: File): Relations = - new MRelationsDirectAndPublicInherited( srcProd, binaryDep + (src, dependsOn), direct = direct, + new MRelationsDefaultImpl( srcProd, binaryDep + (src, dependsOn), direct = direct, publicInherited = publicInherited, classes) def ++ (o: Relations): Relations = { - if (memberRefAndInheritanceDeps != o.memberRefAndInheritanceDeps) + if (nameHashing != o.nameHashing) throw new UnsupportedOperationException("The `++` operation is not supported for relations " + - "with different values of `memberRefAndInheritanceDeps` flag.") - new MRelationsDirectAndPublicInherited(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct, + "with different values of `nameHashing` flag.") + new MRelationsDefaultImpl(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct, publicInherited ++ o.publicInherited, classes ++ o.classes) } def -- (sources: Iterable[File]) = - new MRelationsDirectAndPublicInherited(srcProd -- sources, binaryDep -- sources, direct = direct -- sources, + new MRelationsDefaultImpl(srcProd -- sources, binaryDep -- sources, direct = direct -- sources, publicInherited = publicInherited -- sources, classes -- sources) @deprecated("Broken implementation. OK to remove in 0.14", "0.13.1") @@ -363,14 +388,15 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], { type MapRel[T] = Map[K, Relation[File, T]] def outerJoin(srcProdMap: MapRel[File], binaryDepMap: MapRel[File], direct: Map[K, Source], - inherited: Map[K, Source], classesMap: MapRel[String]): Map[K, Relations] = + inherited: Map[K, Source], classesMap: MapRel[String], + namesMap: MapRel[String]): Map[K, Relations] = { def kRelations(k: K): Relations = { def get[T](m: Map[K, Relation[File, T]]) = Relations.getOrEmpty(m, k) def getSrc(m: Map[K, Source]): Source = m.getOrElse(k, Relations.emptySource) def getSrcDeps(m: Map[K, SourceDependencies]): SourceDependencies = m.getOrElse(k, Relations.emptySourceDependencies) - new MRelationsDirectAndPublicInherited( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited), + new MRelationsDefaultImpl( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited), get(classesMap)) } val keys = (srcProdMap.keySet ++ binaryDepMap.keySet ++ direct.keySet ++ inherited.keySet ++ classesMap.keySet).toList @@ -380,11 +406,11 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], def f1[B](item: (File, B)): K = f(item._1) outerJoin(srcProd.groupBy(f1), binaryDep.groupBy(f1), direct.groupBySource(f), - publicInherited.groupBySource(f), classes.groupBy(f1)) + publicInherited.groupBySource(f), classes.groupBy(f1), names.groupBy(f1)) } override def equals(other: Any) = other match { - case o: MRelationsDirectAndPublicInherited => + case o: MRelationsDefaultImpl => srcProd == o.srcProd && binaryDep == o.binaryDep && direct == o.direct && publicInherited == o.publicInherited && classes == o.classes case _ => false @@ -408,7 +434,8 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], | src deps: %s | ext deps: %s | class names: %s - """.trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes) map relation_s : _*) + | used names: %s + """.trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes, names) map relation_s : _*) ) } @@ -417,62 +444,70 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], * dependencies. Therefore this class implements the new (compared to sbt 0.13.0) dependency tracking logic * needed by the name hashing invalidation algorithm. */ -private class MRelationsMemberRefAndInheritance(srcProd: Relation[File, File], binaryDep: Relation[File, File], +private class MRelationsNameHashing(srcProd: Relation[File, File], binaryDep: Relation[File, File], // memberRef should include everything in inherited val memberRef: SourceDependencies, val inheritance: SourceDependencies, - classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes) + classes: Relation[File, String], + val names: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes) { def direct: Source = throw new UnsupportedOperationException("The `direct` source dependencies relation is not supported " + - "when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?") + "when `nameHashing` flag is disabled.") def publicInherited: Source = throw new UnsupportedOperationException("The `publicInherited` source dependencies relation is not supported " + - "when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?") + "when `nameHashing` flag is disabled.") - val memberRefAndInheritanceDeps: Boolean = true + val nameHashing: Boolean = true def internalSrcDep: Relation[File, File] = memberRef.internal def externalDep: Relation[File, String] = memberRef.external def addProduct(src: File, prod: File, name: String): Relations = - new MRelationsMemberRefAndInheritance(srcProd + (src, prod), binaryDep, memberRef = memberRef, - inheritance = inheritance, classes + (src, name)) + new MRelationsNameHashing(srcProd + (src, prod), binaryDep, memberRef = memberRef, + inheritance = inheritance, classes + (src, name), names = names) def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = { val newIH = if(inherited) inheritance.addExternal(src, dependsOn) else inheritance val newMR = memberRef.addExternal(src, dependsOn) - new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes) + new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes, + names = names) } def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations = { val newIH = inheritance.addInternal(src, inherited) val newMR = memberRef.addInternal(src, dependsOn) - new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes) + new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes, + names = names) } + def addUsedName(src: File, name: String): Relations = + new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef, + inheritance = inheritance, classes, names = names + (src, name)) + def addBinaryDep(src: File, dependsOn: File): Relations = - new MRelationsMemberRefAndInheritance(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef, - inheritance = inheritance, classes) + new MRelationsNameHashing(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef, + inheritance = inheritance, classes, names = names) def ++ (o: Relations): Relations = { - if (!o.memberRefAndInheritanceDeps) + if (!o.nameHashing) throw new UnsupportedOperationException("The `++` operation is not supported for relations " + - "with different values of `memberRefAndInheritanceDeps` flag.") - new MRelationsMemberRefAndInheritance(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, + "with different values of `nameHashing` flag.") + new MRelationsNameHashing(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, memberRef = memberRef ++ o.memberRef, inheritance = inheritance ++ o.inheritance, - classes ++ o.classes) + classes ++ o.classes, names = names ++ o.names) } def -- (sources: Iterable[File]) = - new MRelationsMemberRefAndInheritance(srcProd -- sources, binaryDep -- sources, - memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources) + new MRelationsNameHashing(srcProd -- sources, binaryDep -- sources, + memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources, + names = names -- sources) def groupBy[K](f: File => K): Map[K, Relations] = { throw new UnsupportedOperationException("Merging of Analyses that have" + - "`relations.memberRefAndInheritanceDeps` set to `true` is not supported.") + "`relations.nameHashing` set to `true` is not supported.") } override def equals(other: Any) = other match { - case o: MRelationsMemberRefAndInheritance => + case o: MRelationsNameHashing => srcProd == o.srcProd && binaryDep == o.binaryDep && memberRef == o.memberRef && inheritance == o.inheritance && classes == o.classes case _ => false diff --git a/compile/interface/src/main/scala/xsbt/API.scala b/compile/interface/src/main/scala/xsbt/API.scala index 9c005cfe0..c65bef3c0 100644 --- a/compile/interface/src/main/scala/xsbt/API.scala +++ b/compile/interface/src/main/scala/xsbt/API.scala @@ -43,6 +43,12 @@ final class API(val global: CallbackGlobal) extends Compat val extractApi = new ExtractAPI[global.type](global, sourceFile) val traverser = new TopLevelHandler(extractApi) traverser.apply(unit.body) + if (global.callback.nameHashing) { + val extractUsedNames = new ExtractUsedNames[global.type](global) + val names = extractUsedNames.extract(unit) + debug("The " + sourceFile + " contains the following used names " + names) + names foreach { (name: String) => callback.usedName(sourceFile, name) } + } val packages = traverser.packages.toArray[String].map(p => new xsbti.api.Package(p)) val source = new xsbti.api.SourceAPI(packages, traverser.definitions.toArray[xsbti.api.Definition]) extractApi.forceStructures() diff --git a/compile/interface/src/main/scala/xsbt/Dependency.scala b/compile/interface/src/main/scala/xsbt/Dependency.scala index edb33197f..e9b482ef9 100644 --- a/compile/interface/src/main/scala/xsbt/Dependency.scala +++ b/compile/interface/src/main/scala/xsbt/Dependency.scala @@ -43,7 +43,7 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile { // build dependencies structure val sourceFile = unit.source.file.file - if (global.callback.memberRefAndInheritanceDeps) { + if (global.callback.nameHashing) { val dependenciesByMemberRef = extractDependenciesByMemberRef(unit) for(on <- dependenciesByMemberRef) processDependency(on, inherited=false) diff --git a/compile/interface/src/main/scala/xsbt/ExtractUsedNames.scala b/compile/interface/src/main/scala/xsbt/ExtractUsedNames.scala new file mode 100644 index 000000000..9f89a3459 --- /dev/null +++ b/compile/interface/src/main/scala/xsbt/ExtractUsedNames.scala @@ -0,0 +1,103 @@ +package xsbt + +import scala.tools.nsc._ + +/** + * Extracts simple names used in given compilation unit. + * + * Extracts simple (unqualified) names mentioned in given in non-definition position by collecting + * all symbols associated with non-definition trees and extracting names from all collected symbols. + * + * If given symbol is mentioned both in definition and in non-definition position (e.g. in member + * selection) then that symbol is collected. It means that names of symbols defined and used in the + * same compilation unit are extracted. We've considered not extracting names of those symbols + * as an optimization strategy. It turned out that this is not correct. Check + * https://github.com/gkossakowski/sbt/issues/3 for an example of scenario where it matters. + * + * All extracted names are returned in _decoded_ form. This way we stay consistent with the rest + * of incremental compiler which works with names in decoded form. + * + * Names mentioned in Import nodes are handled properly but require some special logic for two + * reasons: + * + * 1. import node itself has a term symbol associated with it with a name `. + * I (gkossakowski) tried to track down what role this symbol serves but I couldn't. + * It doesn't look like there are many places in Scala compiler that refer to + * that kind of symbols explicitly. + * 2. ImportSelector is not subtype of Tree therefore is not processed by `Tree.foreach` + * + * Another type of tree nodes that requires special handling is TypeTree. TypeTree nodes + * has a little bit odd representation: + * + * 1. TypeTree.hasSymbol always returns false even when TypeTree.symbol + * returns a symbol + * 2. The original tree from which given TypeTree was derived is stored + * in TypeTree.original but Tree.forech doesn't walk into original + * tree so we missed it + * + * The tree walking algorithm walks into TypeTree.original explicitly. + * + */ +class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) { + import global._ + + def extract(unit: CompilationUnit): Set[String] = { + val tree = unit.body + val extractedByTreeWalk = extractByTreeWalk(tree) + extractedByTreeWalk + } + + private def extractByTreeWalk(tree: Tree): Set[String] = { + val namesBuffer = collection.mutable.ListBuffer.empty[String] + def addSymbol(symbol: Symbol): Unit = { + val symbolNameAsString = symbol.name.decode.trim + namesBuffer += symbolNameAsString + } + def handleTreeNode(node: Tree): Unit = node match { + case _: DefTree | _: Template => () + // turns out that Import node has a TermSymbol associated with it + // I (Grzegorz) tried to understand why it's there and what does it represent but + // that logic was introduced in 2005 without any justification I'll just ignore the + // import node altogether and just process the selectors in the import node + case Import(_, selectors: List[ImportSelector]) => + def usedNameInImportSelector(name: Name): Unit = + if ((name != null) && (name != nme.WILDCARD)) namesBuffer += name.toString + selectors foreach { selector => + usedNameInImportSelector(selector.name) + usedNameInImportSelector(selector.rename) + } + // TODO: figure out whether we should process the original tree or walk the type + // the argument for processing the original tree: we process what user wrote + // the argument for processing the type: we catch all transformations that typer applies + // to types but that might be a bad thing because it might expand aliases eagerly which + // not what we need + case t: TypeTree if t.original != null => + t.original.foreach(handleTreeNode) + case t if t.hasSymbol && eligibleAsUsedName(t.symbol) => + addSymbol(t.symbol) + case _ => () + } + tree.foreach(handleTreeNode) + namesBuffer.toSet + } + + + /** + * Needed for compatibility with Scala 2.8 which doesn't define `tpnme` + */ + private object tpnme { + val EMPTY = nme.EMPTY.toTypeName + val EMPTY_PACKAGE_NAME = nme.EMPTY_PACKAGE_NAME.toTypeName + } + + private def eligibleAsUsedName(symbol: Symbol): Boolean = { + def emptyName(name: Name): Boolean = name match { + case nme.EMPTY | nme.EMPTY_PACKAGE_NAME | tpnme.EMPTY | tpnme.EMPTY_PACKAGE_NAME => true + case _ => false + } + + (symbol != NoSymbol) && + !symbol.isSynthetic && + !emptyName(symbol.name) + } +} diff --git a/compile/interface/src/test/scala/xsbt/DependencySpecification.scala b/compile/interface/src/test/scala/xsbt/DependencySpecification.scala index 89f465143..040ad1d6e 100644 --- a/compile/interface/src/test/scala/xsbt/DependencySpecification.scala +++ b/compile/interface/src/test/scala/xsbt/DependencySpecification.scala @@ -80,7 +80,7 @@ class DependencySpecification extends Specification { // E verifies the core type gets pulled out val srcH = "trait H extends G.T[Int] with (E[Int] @unchecked)" - val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true) + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) val sourceDependencies = compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD, 'E -> srcE, 'F -> srcF, 'G -> srcG, 'H -> srcH) sourceDependencies @@ -92,7 +92,7 @@ class DependencySpecification extends Specification { val srcC = "class C { private class Inner1 extends A }" val srcD = "class D { def foo: Unit = { class Inner2 extends B } }" - val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true) + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) val sourceDependencies = compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD) sourceDependencies @@ -104,7 +104,7 @@ class DependencySpecification extends Specification { val srcC = "trait C extends B" val srcD = "class D extends C" - val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true) + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) val sourceDependencies = compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD) sourceDependencies diff --git a/compile/interface/src/test/scala/xsbt/ExtractUsedNamesSpecification.scala b/compile/interface/src/test/scala/xsbt/ExtractUsedNamesSpecification.scala new file mode 100644 index 000000000..861edea62 --- /dev/null +++ b/compile/interface/src/test/scala/xsbt/ExtractUsedNamesSpecification.scala @@ -0,0 +1,108 @@ +package xsbt + +import org.junit.runner.RunWith +import xsbti.api.ClassLike +import xsbti.api.Def +import xsbti.api.Package +import xsbt.api.SameAPI +import org.junit.runners.JUnit4 + +import org.specs2.mutable.Specification + +@RunWith(classOf[JUnit4]) +class ExtractUsedNamesSpecification extends Specification { + + /** + * Standard names that appear in every compilation unit that has any class + * definition. + */ + private val standardNames = Set( + // AnyRef is added as default parent of a class + "scala", "AnyRef", + // class receives a default constructor which is internally called "" + "") + + "imported name" in { + val src = """ + |package a { class A } + |package b { + | import a.{A => A2} + |}""".stripMargin + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(src) + val expectedNames = standardNames ++ Set("a", "A", "A2", "b") + usedNames === expectedNames + } + + // test covers https://github.com/gkossakowski/sbt/issues/6 + "names in type tree" in { + val srcA = """| + |package a { + | class A { + | class C { class D } + | } + | class B[T] + | class BB + |}""".stripMargin + val srcB = """| + |package b { + | abstract class X { + | def foo: a.A#C#D + | def bar: a.B[a.BB] + | } + |}""".stripMargin + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB) + val expectedNames = standardNames ++ Set("a", "A", "B", "C", "D", "b", "X", "BB") + usedNames === expectedNames + } + + // test for https://github.com/gkossakowski/sbt/issues/5 + "symbolic names" in { + val srcA = """| + |class A { + | def `=`: Int = 3 + |}""".stripMargin + val srcB = """| + |class B { + | def foo(a: A) = a.`=` + |}""".stripMargin + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB) + val expectedNames = standardNames ++ Set("A", "a", "B", "=") + usedNames === expectedNames + } + + // test for https://github.com/gkossakowski/sbt/issues/3 + "used names from the same compilation unit" in { + val src = "class A { def foo: Int = 0; def bar: Int = foo }" + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(src) + val expectedNames = standardNames ++ Set("A", "foo", "Int") + usedNames === expectedNames + } + + // pending test for https://issues.scala-lang.org/browse/SI-7173 + "names of constants" in { + val src = "class A { final val foo = 12; def bar: Int = foo }" + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(src) + val expectedNames = standardNames ++ Set("A", "foo", "Int") + usedNames === expectedNames + }.pendingUntilFixed("Scala's type checker inlines constants so we can't see the original name.") + + // pending test for https://github.com/gkossakowski/sbt/issues/4 + // TODO: we should fix it by having special treatment of `selectDynamic` and `applyDynamic` calls + "names from method calls on Dynamic" in { + val srcA = """|import scala.language.dynamics + |class A extends Dynamic { + | def selectDynamic(name: String): Int = name.length + |}""".stripMargin + val srcB = "class B { def foo(a: A): Int = a.bla }" + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB) + val expectedNames = standardNames ++ Set("B", "A", "a", "Int", "selectDynamic", "bla") + usedNames === expectedNames + }.pendingUntilFixed("Call to Dynamic is desugared in type checker so Select nodes is turned into string literal.") + +} diff --git a/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala b/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala index 61fb08078..5362b1ca6 100644 --- a/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala +++ b/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala @@ -19,7 +19,7 @@ import ScalaCompilerForUnitTesting.ExtractedSourceDependencies * Provides common functionality needed for unit tests that require compiling * source code using Scala compiler. */ -class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false) { +class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { /** * Compiles given source code using Scala compiler and returns API representation @@ -30,6 +30,24 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false) analysisCallback.apis(tempSrcFile) } + def extractUsedNamesFromSrc(src: String): Set[String] = { + val (Seq(tempSrcFile), analysisCallback) = compileSrcs(src) + analysisCallback.usedNames(tempSrcFile).toSet + } + + /** + * Extract used names from src provided as the second argument. + * + * The purpose of the first argument is to define names that the second + * source is going to refer to. Both files are compiled in the same compiler + * Run but only names used in the second src file are returned. + */ + def extractUsedNamesFromSrc(definitionSrc: String, actualSrc: String): Set[String] = { + // we drop temp src file corresponding to the definition src file + val (Seq(_, tempSrcFile), analysisCallback) = compileSrcs(definitionSrc, actualSrc) + analysisCallback.usedNames(tempSrcFile).toSet + } + /** * Compiles given source code snippets (passed as Strings) using Scala compiler and returns extracted * dependencies between snippets. Source code snippets are identified by symbols. Each symbol should @@ -76,7 +94,7 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false) */ private def compileSrcs(srcs: String*): (Seq[File], TestCallback) = { withTemporaryDirectory { temp => - val analysisCallback = new TestCallback(memberRefAndInheritanceDeps) + val analysisCallback = new TestCallback(nameHashing) val classesDir = new File(temp, "classes") classesDir.mkdir() val compiler = prepareCompiler(classesDir, analysisCallback) diff --git a/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala b/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala index 5da25c868..5f2c7b9c6 100644 --- a/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala +++ b/compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala @@ -99,26 +99,26 @@ object AnalysisFormats implicit def apisFormat(implicit internalF: Format[Map[File, Source]], externalF: Format[Map[String, Source]]): Format[APIs] = asProduct2( APIs.apply _)( as => (as.internal, as.external) )(internalF, externalF) - implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS]): Format[Relations] = + implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS], namesF: Format[RFS]): Format[Relations] = { def makeRelation(srcProd: RFF, binaryDep: RFF, direct: RSource, publicInherited: RSource, memberRef: SourceDependencies, inheritance: SourceDependencies, classes: RFS, - memberRefAndInheritanceDeps: Boolean): Relations = if (memberRefAndInheritanceDeps) { + nameHashing: Boolean, names: RFS): Relations = if (nameHashing) { def isEmpty(sourceDependencies: RSource): Boolean = sourceDependencies.internal.all.isEmpty && sourceDependencies.external.all.isEmpty // we check direct dependencies only because publicInherited dependencies are subset of direct - assert(isEmpty(direct), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.") - Relations.make(srcProd, binaryDep, memberRef, inheritance, classes) + assert(isEmpty(direct), "Direct dependencies are not empty but `nameHashing` flag is enabled.") + Relations.make(srcProd, binaryDep, memberRef, inheritance, classes, names) } else { def isEmpty(sourceDependencies: SourceDependencies): Boolean = sourceDependencies.internal.all.isEmpty && sourceDependencies.external.all.isEmpty // we check memberRef dependencies only because inheritance dependencies are subset of memberRef - assert(isEmpty(memberRef), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.") + assert(isEmpty(memberRef), "Direct dependencies are not empty but `nameHashing` flag is enabled.") Relations.make(srcProd, binaryDep, direct, publicInherited, classes) } - asProduct8[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean]( (a,b,c,d,e,f,g,h) =>makeRelation(a,b,c,d,e,f,g,h) )( - rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.memberRefAndInheritanceDeps) )( - prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]]) + asProduct9[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean, RFS]( (a,b,c,d,e,f,g,h,i) =>makeRelation(a,b,c,d,e,f,g,h,i) )( + rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.nameHashing, rs.names) )( + prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]], namesF) } implicit def relationsSourceFormat(implicit internalFormat: Format[Relation[File, File]], externalFormat: Format[Relation[File,String]]): Format[RSource] = diff --git a/compile/persist/src/main/scala/sbt/inc/TextAnalysisFormat.scala b/compile/persist/src/main/scala/sbt/inc/TextAnalysisFormat.scala index 8c19fc94a..e4dea8a17 100644 --- a/compile/persist/src/main/scala/sbt/inc/TextAnalysisFormat.scala +++ b/compile/persist/src/main/scala/sbt/inc/TextAnalysisFormat.scala @@ -117,6 +117,8 @@ object TextAnalysisFormat { val memberRefExternalDep = "member reference external dependencies" val inheritanceInternalDep = "inheritance internal dependencies" val inheritanceExternalDep = "inheritance external dependencies" + + val usedNames = "used names" } def write(out: Writer, relations: Relations) { @@ -134,17 +136,17 @@ object TextAnalysisFormat { } } - val memberRefAndInheritanceDeps = relations.memberRefAndInheritanceDeps + val nameHashing = relations.nameHashing writeRelation(Headers.srcProd, relations.srcProd) writeRelation(Headers.binaryDep, relations.binaryDep) - val direct = if (memberRefAndInheritanceDeps) Relations.emptySource else relations.direct - val publicInherited = if (memberRefAndInheritanceDeps) + val direct = if (nameHashing) Relations.emptySource else relations.direct + val publicInherited = if (nameHashing) Relations.emptySource else relations.publicInherited - val memberRef = if (memberRefAndInheritanceDeps) + val memberRef = if (nameHashing) relations.memberRef else Relations.emptySourceDependencies - val inheritance = if (memberRefAndInheritanceDeps) + val inheritance = if (nameHashing) relations.inheritance else Relations.emptySourceDependencies writeRelation(Headers.directSrcDep, direct.internal) @@ -158,6 +160,7 @@ object TextAnalysisFormat { writeRelation(Headers.inheritanceExternalDep, inheritance.external) writeRelation(Headers.classes, relations.classes) + writeRelation(Headers.usedNames, relations.names) } def read(in: BufferedReader): Relations = { @@ -213,13 +216,17 @@ object TextAnalysisFormat { // we assume that invariant that says they are subsets of direct/memberRef holds assert((directSrcDeps == emptySource) || (memberRefSrcDeps == emptySourceDependencies), "One mechanism is supported for tracking source dependencies at the time") - val memberRefAndInheritanceDeps = memberRefSrcDeps != emptySourceDependencies + val nameHashing = memberRefSrcDeps != emptySourceDependencies val classes = readStringRelation(Headers.classes) + val names = readStringRelation(Headers.usedNames) - if (memberRefAndInheritanceDeps) - Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes) - else + if (nameHashing) + Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes, names) + else { + assert(names.all.isEmpty, s"When `nameHashing` is disabled `names` relation " + + "should be empty: $names") Relations.make(srcProd, binaryDep, directSrcDeps, publicInheritedSrcDeps, classes) + } } } diff --git a/interface/src/main/java/xsbti/AnalysisCallback.java b/interface/src/main/java/xsbti/AnalysisCallback.java index ff239ae74..0e083d4eb 100644 --- a/interface/src/main/java/xsbti/AnalysisCallback.java +++ b/interface/src/main/java/xsbti/AnalysisCallback.java @@ -24,12 +24,16 @@ public interface AnalysisCallback public void generatedClass(File source, File module, String name); /** Called when the public API of a source file is extracted. */ public void api(File sourceFile, xsbti.api.SourceAPI source); + public void usedName(File sourceFile, String names); /** Provides problems discovered during compilation. These may be reported (logged) or unreported. * Unreported problems are usually unreported because reporting was not enabled via a command line switch. */ public void problem(String what, Position pos, String msg, Severity severity, boolean reported); /** - * Determines whether member reference and inheritance dependencies should be extracted in given compiler - * run. + * Determines whether method calls through this interface should be interpreted as serving + * name hashing algorithm needs in given compiler run. + * + * In particular, it indicates whether member reference and inheritance dependencies should be + * extracted. * * As the signature suggests, this method's implementation is meant to be side-effect free. It's added * to AnalysisCallback because it indicates how other callback calls should be interpreted by both @@ -38,5 +42,5 @@ public interface AnalysisCallback * NOTE: This method is an implementation detail and can be removed at any point without deprecation. * Do not depend on it, please. */ - public boolean memberRefAndInheritanceDeps(); + public boolean nameHashing(); } \ No newline at end of file diff --git a/interface/src/test/scala/xsbti/TestCallback.scala b/interface/src/test/scala/xsbti/TestCallback.scala index 28bee5466..3ea7e32e1 100644 --- a/interface/src/test/scala/xsbti/TestCallback.scala +++ b/interface/src/test/scala/xsbti/TestCallback.scala @@ -4,17 +4,19 @@ import java.io.File import scala.collection.mutable.ArrayBuffer import xsbti.api.SourceAPI -class TestCallback(override val memberRefAndInheritanceDeps: Boolean = false) extends AnalysisCallback +class TestCallback(override val nameHashing: Boolean = false) extends AnalysisCallback { val sourceDependencies = new ArrayBuffer[(File, File, Boolean)] val binaryDependencies = new ArrayBuffer[(File, String, File, Boolean)] val products = new ArrayBuffer[(File, File, String)] + val usedNames = scala.collection.mutable.Map.empty[File, Set[String]].withDefaultValue(Set.empty) val apis: scala.collection.mutable.Map[File, SourceAPI] = scala.collection.mutable.Map.empty def sourceDependency(dependsOn: File, source: File, inherited: Boolean) { sourceDependencies += ((dependsOn, source, inherited)) } def binaryDependency(binary: File, name: String, source: File, inherited: Boolean) { binaryDependencies += ((binary, name, source, inherited)) } def generatedClass(source: File, module: File, name: String) { products += ((source, module, name)) } + def usedName(source: File, name: String) { usedNames(source) += name } def api(source: File, sourceAPI: SourceAPI): Unit = { assert(!apis.contains(source), s"The `api` method should be called once per source file: $source") apis(source) = sourceAPI diff --git a/project/Sbt.scala b/project/Sbt.scala index 39565833f..288eb25ba 100644 --- a/project/Sbt.scala +++ b/project/Sbt.scala @@ -270,6 +270,9 @@ object Sbt extends Build // we are expecting all of our dependencies to be on classpath so Scala compiler // can use them while constructing its own classpath for compilation fork in Test := true, + // needed because we fork tests and tests are ran in parallel so we have multiple Scala + // compiler instances that are memory hungry + javaOptions in Test += "-Xmx1G", artifact in (Compile, packageSrc) := Artifact(srcID).copy(configurations = Compile :: Nil).extra("e:component" -> srcID) ) def compilerSettings = Seq(