Merge pull request #1013 from gkossakowski/used-names-extraction

Used names extraction logic

compile/inc/src/main/scala/sbt/inc/Analysis.scala

@@ -59,7 +59,7 @@ object Analysis
 
 	/** Merge multiple analysis objects into one. Deps will be internalized as needed. */
 	def merge(analyses: Traversable[Analysis]): Analysis = {
-		if (analyses.exists(_.relations.memberRefAndInheritanceDeps))
+		if (analyses.exists(_.relations.nameHashing))
 			throw new IllegalArgumentException("Merging of Analyses that have" +
 				"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")
 
@@ -160,7 +160,7 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat
 		copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos )
 
 	def groupBy[K](discriminator: File => K): Map[K, Analysis] = {
-		if (relations.memberRefAndInheritanceDeps)
+		if (relations.nameHashing)
 			throw new UnsupportedOperationException("Grouping of Analyses that have" +
 				"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")
 

compile/inc/src/main/scala/sbt/inc/Compile.scala

@@ -67,6 +67,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
 	import collection.mutable.{HashMap, HashSet, ListBuffer, Map, Set}
 
 	private[this] val apis = new HashMap[File, (Int, SourceAPI)]
+	private[this] val usedNames = new HashMap[File, Set[String]]
 	private[this] val unreporteds = new HashMap[File, ListBuffer[Problem]]
 	private[this] val reporteds = new HashMap[File, ListBuffer[Problem]]
 	private[this] val binaryDeps = new HashMap[File, Set[File]]
@@ -151,9 +152,11 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
 		apis(sourceFile) = (HashAPI(source), savedSource)
 	}
 
-	def memberRefAndInheritanceDeps: Boolean = false // TODO: define the flag in IncOptions which controls this
+	def usedName(sourceFile: File, name: String) = add(usedNames, sourceFile, name)
 
-	def get: Analysis = addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) )
+	def nameHashing: Boolean = false // TODO: define the flag in IncOptions which controls this
+
+	def get: Analysis = addUsedNames( addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) ) )
 	def addProducts(base: Analysis): Analysis = addAll(base, classes) { case (a, src, (prod, name)) => a.addProduct(src, prod, current product prod, name ) }
 	def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, binaryClassName(bin), current binary bin) )
 	def addSources(base: Analysis): Analysis =
@@ -171,6 +174,9 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
 	def getOrNil[A,B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten
 	def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api, inherited)) => a.addExternalDep(source, name, api, inherited) }
 	def addCompilation(base: Analysis): Analysis = base.copy(compilations = base.compilations.add(compilation))
+	def addUsedNames(base: Analysis): Analysis = (base /: usedNames) { case (a, (src, names)) =>
+	  (a /: names) { case (a, name) => a.copy(relations = a.relations.addUsedName(src, name)) }
+	}
 
 	def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis =
 		(base /: m) { case (outer, (a, bs)) =>

compile/inc/src/main/scala/sbt/inc/Relations.scala

@@ -58,6 +58,8 @@ trait Relations
 	/** Internal source dependencies that depend on external source file `dep`.  This includes both direct and inherited dependencies.  */
 	def usesExternal(dep: String): Set[File]
 
+	private[inc] def usedNames(src: File): Set[String]
+
 	/** Records internal source file `src` as generating class file `prod` with top-level class `name`. */
 	def addProduct(src: File, prod: File, name: String): Relations
 
@@ -74,6 +76,8 @@ trait Relations
 	* this method does not automatically record direct dependencies like `addExternalDep` does.*/
 	def addInternalSrcDeps(src: File, directDependsOn: Iterable[File], inheritedDependsOn: Iterable[File]): Relations
 
+	private[inc] def addUsedName(src: File, name: String): Relations
+
 	/** Concatenates the two relations. Acts naively, i.e., doesn't internalize external deps on added files. */
 	def ++ (o: Relations): Relations
 
@@ -105,7 +109,7 @@ trait Relations
 	 * This relation properly accounts for that so the invariant that `memberRef` is a superset
 	 * of `inheritance` is preserved.
 	 */
-	def memberRef: SourceDependencies
+	private[inc] def memberRef: SourceDependencies
 
 	/**
 	 * The source dependency relation between source files introduced by inheritance.
@@ -135,7 +139,7 @@ trait Relations
 	 * resolved transitively. You should not rely on this behavior, though.
 	 *
 	 */
-	def inheritance: SourceDependencies
+	private[inc] def inheritance: SourceDependencies
 
 	/** The dependency relations between sources.  These include both direct and inherited dependencies.*/
 	def direct: Source
@@ -147,17 +151,26 @@ trait Relations
 	def classes: Relation[File, String]
 
 	/**
-	 * Flag which indicates whether the new style (based on `memberRef` and `inheritance` source dependencies)
+	 * Flag which indicates whether given Relations object supports operations needed by name hashing algorithm.
-	 * of dependency tracking is enabled. When this flag is enabled access to `direct` and `publicInherited`
-	 * relations is illegal and will cause runtime exception being thrown.
 	 *
-	 * Conversely, when `memberRefAndInheritanceDeps` flag is disabled access to `memberRef` and `inheritance`
+	 * At the moment the list includes the following operations:
-	 * relations is illegal and will cause runtime exception being thrown.
 	 *
-	 * The name of this flag is ugly but it's private to incremental compiler and it's temporary measure during
+	 *   - memberRef: SourceDependencies
-	 * our migration to the new dependency tracking.
+	 *   - inheritance: SourceDependencies
+	 *
+	 * The `memberRef` and `inheritance` implement a new style source dependency tracking. When this flag is
+	 * enabled access to `direct` and `publicInherited` relations is illegal and will cause runtime exception
+	 * being thrown. That is done as an optimization that prevents from storing two overlapping sets of
+	 * dependencies.
+	 *
+	 * Conversely, when `nameHashing` flag is disabled access to `memberRef` and `inheritance`
+	 * relations is illegal and will cause runtime exception being thrown.
 	 */
-	private[inc] def memberRefAndInheritanceDeps: Boolean
+	private[inc] def nameHashing: Boolean
+	/**
+	 * Relation between source files and _unqualified_ term and type names used in given source file.
+	 */
+	private[inc] def names: Relation[File, String]
 }
 
 
@@ -213,19 +226,21 @@ object Relations
 
 	def emptySource: Source = es
 	private[inc] lazy val emptySourceDependencies: SourceDependencies = new SourceDependencies(e, estr)
-	def empty: Relations = empty(memberRefAndInheritanceDeps = false)
+	def empty: Relations = empty(nameHashing = false)
-	def empty(memberRefAndInheritanceDeps: Boolean): Relations =
+	private[inc] def empty(nameHashing: Boolean): Relations =
-		if (memberRefAndInheritanceDeps)
+		if (nameHashing)
-			new MRelationsMemberRefAndInheritance(e, e, emptySourceDependencies, emptySourceDependencies, estr)
+			new MRelationsNameHashing(e, e, emptySourceDependencies, emptySourceDependencies, estr, estr)
 		else
-			new MRelationsDirectAndPublicInherited(e, e, es, es, estr)
+			new MRelationsDefaultImpl(e, e, es, es, estr)
 
 	def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], direct: Source, publicInherited: Source, classes: Relation[File, String]): Relations =
-		new MRelationsDirectAndPublicInherited(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes)
+		new MRelationsDefaultImpl(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes)
 
 	private[inc] def make(srcProd: Relation[File, File], binaryDep: Relation[File, File],
-			memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String]): Relations =
+			memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String],
-		new MRelationsMemberRefAndInheritance(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance, classes)
+			names: Relation[File, String]): Relations =
+		new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance,
+			classes, names)
 	def makeSource(internal: Relation[File,File], external: Relation[File,String]): Source = new Source(internal, external)
 	private[inc] def makeSourceDependencies(internal: Relation[File,File], external: Relation[File,String]): SourceDependencies = new SourceDependencies(internal, external)
 }
@@ -276,6 +291,8 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b
 	def externalDeps(src: File): Set[String] = externalDep.forward(src)
 	def usesExternal(dep: String): Set[File] = externalDep.reverse(dep)
 
+	def usedNames(src: File): Set[String] = names.forward(src)
+
   /** Making large Relations a little readable. */
   private val userDir = sys.props("user.dir").stripSuffix("/") + "/"
   private def nocwd(s: String)              = s stripPrefix userDir
@@ -309,7 +326,7 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b
  *    introduced by inheritance.
  *
  */
-private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], binaryDep: Relation[File, File],
+private class MRelationsDefaultImpl(srcProd: Relation[File, File], binaryDep: Relation[File, File],
 	// direct should include everything in inherited
 	val direct: Source, val publicInherited: Source,
 	classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
@@ -317,45 +334,53 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
 	def internalSrcDep: Relation[File, File] = direct.internal
 	def externalDep: Relation[File, String] = direct.external
 
-	def memberRefAndInheritanceDeps: Boolean = false
+	def nameHashing: Boolean = false
 
 	def memberRef: SourceDependencies =
 		throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " +
-			"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
+			"when `nameHashing` flag is disabled.")
 	def inheritance: SourceDependencies =
 		throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " +
-			"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
+			"when `nameHashing` flag is disabled.")
 
 	def addProduct(src: File, prod: File, name: String): Relations =
-		new MRelationsDirectAndPublicInherited(srcProd + (src, prod), binaryDep, direct = direct,
+		new MRelationsDefaultImpl(srcProd + (src, prod), binaryDep, direct = direct,
 			publicInherited = publicInherited, classes + (src, name))
 
 	def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = {
 		val newI = if(inherited) publicInherited.addExternal(src, dependsOn) else publicInherited
 		val newD = direct.addExternal(src, dependsOn)
-		new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
+		new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
 	}
 
 	def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations =
 		{
 			val newI = publicInherited.addInternal(src, inherited)
 			val newD = direct.addInternal(src, dependsOn)
-			new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
+			new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
 		}
 
+	def names: Relation[File, String] =
+		throw new UnsupportedOperationException("Tracking of used names is not supported " +
+			"when `nameHashing` is disabled.")
+
+	def addUsedName(src: File, name: String): Relations =
+		throw new UnsupportedOperationException("Tracking of used names is not supported " +
+			"when `nameHashing` is disabled.")
+
 	def addBinaryDep(src: File, dependsOn: File): Relations =
-		new MRelationsDirectAndPublicInherited( srcProd, binaryDep + (src, dependsOn), direct = direct,
+		new MRelationsDefaultImpl( srcProd, binaryDep + (src, dependsOn), direct = direct,
 			publicInherited = publicInherited, classes)
 
 	def ++ (o: Relations): Relations = {
-		if (memberRefAndInheritanceDeps != o.memberRefAndInheritanceDeps)
+		if (nameHashing != o.nameHashing)
 			throw new UnsupportedOperationException("The `++` operation is not supported for relations " +
-				"with different values of `memberRefAndInheritanceDeps` flag.")
+				"with different values of `nameHashing` flag.")
-		new MRelationsDirectAndPublicInherited(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct,
+		new MRelationsDefaultImpl(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct,
 				publicInherited ++ o.publicInherited, classes ++ o.classes)
 	}
 	def -- (sources: Iterable[File]) =
-		new MRelationsDirectAndPublicInherited(srcProd -- sources, binaryDep -- sources, direct = direct -- sources,
+		new MRelationsDefaultImpl(srcProd -- sources, binaryDep -- sources, direct = direct -- sources,
 				publicInherited = publicInherited -- sources, classes -- sources)
 
 	@deprecated("Broken implementation. OK to remove in 0.14", "0.13.1")
@@ -363,14 +388,15 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
 	{
 		type MapRel[T] = Map[K, Relation[File, T]]
 		def outerJoin(srcProdMap: MapRel[File], binaryDepMap: MapRel[File], direct: Map[K, Source],
-					  inherited: Map[K, Source], classesMap: MapRel[String]): Map[K, Relations] =
+					  inherited: Map[K, Source], classesMap: MapRel[String],
+					  namesMap: MapRel[String]): Map[K, Relations] =
 		{
 			def kRelations(k: K): Relations = {
 				def get[T](m: Map[K, Relation[File, T]]) = Relations.getOrEmpty(m, k)
 				def getSrc(m: Map[K, Source]): Source = m.getOrElse(k, Relations.emptySource)
 				def getSrcDeps(m: Map[K, SourceDependencies]): SourceDependencies =
 					m.getOrElse(k, Relations.emptySourceDependencies)
-				new MRelationsDirectAndPublicInherited( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited),
+				new MRelationsDefaultImpl( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited),
 					get(classesMap))
 			}
 			val keys = (srcProdMap.keySet ++ binaryDepMap.keySet ++ direct.keySet ++ inherited.keySet ++ classesMap.keySet).toList
@@ -380,11 +406,11 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
 		def f1[B](item: (File, B)): K = f(item._1)
 
 		outerJoin(srcProd.groupBy(f1), binaryDep.groupBy(f1), direct.groupBySource(f),
-				  publicInherited.groupBySource(f), classes.groupBy(f1))
+				  publicInherited.groupBySource(f), classes.groupBy(f1), names.groupBy(f1))
     }
 
 	override def equals(other: Any) = other match {
-		case o: MRelationsDirectAndPublicInherited =>
+		case o: MRelationsDefaultImpl =>
 			srcProd == o.srcProd && binaryDep == o.binaryDep && direct == o.direct &&
 			publicInherited == o.publicInherited && classes == o.classes
 		case _ => false
@@ -408,7 +434,8 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
 	  |  src deps: %s
 	  |  ext deps: %s
 	  |  class names: %s
-	  """.trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes) map relation_s : _*)
+	  |  used names: %s
+	  """.trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes, names) map relation_s : _*)
 	)
 }
 
@@ -417,62 +444,70 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
  * dependencies. Therefore this class implements the new (compared to sbt 0.13.0) dependency tracking logic
  * needed by the name hashing invalidation algorithm.
  */
-private class MRelationsMemberRefAndInheritance(srcProd: Relation[File, File], binaryDep: Relation[File, File],
+private class MRelationsNameHashing(srcProd: Relation[File, File], binaryDep: Relation[File, File],
 	// memberRef should include everything in inherited
 	val memberRef: SourceDependencies, val inheritance: SourceDependencies,
-	classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
+	classes: Relation[File, String],
+	val names: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
 {
 	def direct: Source =
 		throw new UnsupportedOperationException("The `direct` source dependencies relation is not supported " +
-			"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
+			"when `nameHashing` flag is disabled.")
 	def publicInherited: Source =
 		throw new UnsupportedOperationException("The `publicInherited` source dependencies relation is not supported " +
-			"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
+			"when `nameHashing` flag is disabled.")
 
-	val memberRefAndInheritanceDeps: Boolean = true
+	val nameHashing: Boolean = true
 
 	def internalSrcDep: Relation[File, File] = memberRef.internal
 	def externalDep: Relation[File, String] = memberRef.external
 
 	def addProduct(src: File, prod: File, name: String): Relations =
-		new MRelationsMemberRefAndInheritance(srcProd + (src, prod), binaryDep, memberRef = memberRef,
+		new MRelationsNameHashing(srcProd + (src, prod), binaryDep, memberRef = memberRef,
-			inheritance = inheritance, classes + (src, name))
+			inheritance = inheritance, classes + (src, name), names = names)
 
 	def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = {
 		val newIH = if(inherited) inheritance.addExternal(src, dependsOn) else inheritance
 		val newMR = memberRef.addExternal(src, dependsOn)
-		new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes)
+		new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes,
+			names = names)
 	}
 
 	def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations = {
 		val newIH = inheritance.addInternal(src, inherited)
 		val newMR = memberRef.addInternal(src, dependsOn)
-		new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes)
+		new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes,
+			names = names)
 	}
 
+	def addUsedName(src: File, name: String): Relations =
+		new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef,
+			inheritance = inheritance, classes, names = names + (src, name))
+
 	def addBinaryDep(src: File, dependsOn: File): Relations =
-		new MRelationsMemberRefAndInheritance(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef,
+		new MRelationsNameHashing(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef,
-			inheritance = inheritance, classes)
+			inheritance = inheritance, classes, names = names)
 
 	def ++ (o: Relations): Relations = {
-		if (!o.memberRefAndInheritanceDeps)
+		if (!o.nameHashing)
 			throw new UnsupportedOperationException("The `++` operation is not supported for relations " +
-				"with different values of `memberRefAndInheritanceDeps` flag.")
+				"with different values of `nameHashing` flag.")
-		new MRelationsMemberRefAndInheritance(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep,
+		new MRelationsNameHashing(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep,
 			memberRef = memberRef ++ o.memberRef, inheritance = inheritance ++ o.inheritance,
-			classes ++ o.classes)
+			classes ++ o.classes, names = names ++ o.names)
 	}
 	def -- (sources: Iterable[File]) =
-		new MRelationsMemberRefAndInheritance(srcProd -- sources, binaryDep -- sources,
+		new MRelationsNameHashing(srcProd -- sources, binaryDep -- sources,
-			memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources)
+			memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources,
+			names = names -- sources)
 
 	def groupBy[K](f: File => K): Map[K, Relations] = {
 		throw new UnsupportedOperationException("Merging of Analyses that have" +
-			"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")
+			"`relations.nameHashing` set to `true` is not supported.")
     }
 
 	override def equals(other: Any) = other match {
-		case o: MRelationsMemberRefAndInheritance =>
+		case o: MRelationsNameHashing =>
 			srcProd == o.srcProd && binaryDep == o.binaryDep && memberRef == o.memberRef &&
 			inheritance == o.inheritance && classes == o.classes
 		case _ => false

compile/interface/src/main/scala/xsbt/API.scala

@@ -43,6 +43,12 @@ final class API(val global: CallbackGlobal) extends Compat
 			val extractApi = new ExtractAPI[global.type](global, sourceFile)
 			val traverser = new TopLevelHandler(extractApi)
 			traverser.apply(unit.body)
+			if (global.callback.nameHashing) {
+				val extractUsedNames = new ExtractUsedNames[global.type](global)
+				val names = extractUsedNames.extract(unit)
+				debug("The " + sourceFile + " contains the following used names " + names)
+				names foreach { (name: String) => callback.usedName(sourceFile, name) }
+			}
 			val packages = traverser.packages.toArray[String].map(p => new xsbti.api.Package(p))
 			val source = new xsbti.api.SourceAPI(packages, traverser.definitions.toArray[xsbti.api.Definition])
 			extractApi.forceStructures()

compile/interface/src/main/scala/xsbt/Dependency.scala

@@ -43,7 +43,7 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile
 			{
 				// build dependencies structure
 				val sourceFile = unit.source.file.file
-				if (global.callback.memberRefAndInheritanceDeps) {
+				if (global.callback.nameHashing) {
 					val dependenciesByMemberRef = extractDependenciesByMemberRef(unit)
 					for(on <- dependenciesByMemberRef)
 						processDependency(on, inherited=false)

compile/interface/src/main/scala/xsbt/ExtractUsedNames.scala

@@ -0,0 +1,103 @@
+package xsbt
+
+import scala.tools.nsc._
+
+/**
+ * Extracts simple names used in given compilation unit.
+ *
+ * Extracts simple (unqualified) names mentioned in given in non-definition position by collecting
+ * all symbols associated with non-definition trees and extracting names from all collected symbols.
+ *
+ * If given symbol is mentioned both in definition and in non-definition position (e.g. in member
+ * selection) then that symbol is collected. It means that names of symbols defined and used in the
+ * same compilation unit are extracted. We've considered not extracting names of those symbols
+ * as an optimization strategy. It turned out that this is not correct.  Check
+ * https://github.com/gkossakowski/sbt/issues/3 for an example of scenario where it matters.
+ *
+ * All extracted names are returned in _decoded_ form. This way we stay consistent with the rest
+ * of incremental compiler which works with names in decoded form.
+ *
+ * Names mentioned in Import nodes are handled properly but require some special logic for two
+ * reasons:
+ *
+ *   1. import node itself has a term symbol associated with it with a name `<import`>.
+ *      I (gkossakowski) tried to track down what role this symbol serves but I couldn't.
+ *      It doesn't look like there are many places in Scala compiler that refer to
+ *      that kind of symbols explicitly.
+ *   2. ImportSelector is not subtype of Tree therefore is not processed by `Tree.foreach`
+ *
+ * Another type of tree nodes that requires special handling is TypeTree. TypeTree nodes
+ * has a little bit odd representation:
+ *
+ *   1. TypeTree.hasSymbol always returns false even when TypeTree.symbol
+ *      returns a symbol
+ *   2. The original tree from which given TypeTree was derived is stored
+ *      in TypeTree.original but Tree.forech doesn't walk into original
+ *      tree so we missed it
+ *
+ * The tree walking algorithm walks into TypeTree.original explicitly.
+ *
+ */
+class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) {
+	import global._
+
+	def extract(unit: CompilationUnit): Set[String] = {
+		val tree = unit.body
+		val extractedByTreeWalk = extractByTreeWalk(tree)
+		extractedByTreeWalk
+	}
+
+	private def extractByTreeWalk(tree: Tree): Set[String] = {
+		val namesBuffer = collection.mutable.ListBuffer.empty[String]
+		def addSymbol(symbol: Symbol): Unit = {
+			val symbolNameAsString = symbol.name.decode.trim
+			namesBuffer += symbolNameAsString
+		}
+		def handleTreeNode(node: Tree): Unit = node match {
+			case _: DefTree | _: Template => ()
+			// turns out that Import node has a TermSymbol associated with it
+			// I (Grzegorz) tried to understand why it's there and what does it represent but
+			// that logic was introduced in 2005 without any justification I'll just ignore the
+			// import node altogether and just process the selectors in the import node
+			case Import(_, selectors: List[ImportSelector]) =>
+				def usedNameInImportSelector(name: Name): Unit =
+					if ((name != null) && (name != nme.WILDCARD)) namesBuffer += name.toString
+				selectors foreach { selector =>
+					usedNameInImportSelector(selector.name)
+					usedNameInImportSelector(selector.rename)
+				}
+			// TODO: figure out whether we should process the original tree or walk the type
+			// the argument for processing the original tree: we process what user wrote
+			// the argument for processing the type: we catch all transformations that typer applies
+			// to types but that might be a bad thing because it might expand aliases eagerly which
+			// not what we need
+			case t: TypeTree if t.original != null =>
+				t.original.foreach(handleTreeNode)
+			case t if t.hasSymbol && eligibleAsUsedName(t.symbol) =>
+				addSymbol(t.symbol)
+			case _ => ()
+		}
+		tree.foreach(handleTreeNode)
+		namesBuffer.toSet
+	}
+
+
+	/**
+	 * Needed for compatibility with Scala 2.8 which doesn't define `tpnme`
+	 */
+	private object tpnme {
+		val EMPTY = nme.EMPTY.toTypeName
+		val EMPTY_PACKAGE_NAME = nme.EMPTY_PACKAGE_NAME.toTypeName
+	}
+
+	private def eligibleAsUsedName(symbol: Symbol): Boolean = {
+		def emptyName(name: Name): Boolean = name match {
+			case nme.EMPTY | nme.EMPTY_PACKAGE_NAME | tpnme.EMPTY | tpnme.EMPTY_PACKAGE_NAME => true
+			case _ => false
+		}
+
+		(symbol != NoSymbol) &&
+		!symbol.isSynthetic &&
+		!emptyName(symbol.name)
+	}
+}

compile/interface/src/test/scala/xsbt/DependencySpecification.scala

@@ -80,7 +80,7 @@ class DependencySpecification extends Specification {
 		// E verifies the core type gets pulled out
 		val srcH = "trait H extends G.T[Int] with (E[Int] @unchecked)"
 
-		val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
 		val sourceDependencies = compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC,
 			'D -> srcD, 'E -> srcE, 'F -> srcF, 'G -> srcG, 'H -> srcH)
 		sourceDependencies
@@ -92,7 +92,7 @@ class DependencySpecification extends Specification {
 		val srcC = "class C { private class Inner1 extends A }"
 		val srcD = "class D { def foo: Unit = { class Inner2 extends B } }"
 
-		val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
 		val sourceDependencies =
 			compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD)
 		sourceDependencies
@@ -104,7 +104,7 @@ class DependencySpecification extends Specification {
 		val srcC = "trait C extends B"
 		val srcD = "class D extends C"
 
-		val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
 		val sourceDependencies =
 			compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD)
 		sourceDependencies

compile/interface/src/test/scala/xsbt/ExtractUsedNamesSpecification.scala

@@ -0,0 +1,108 @@
+package xsbt
+
+import org.junit.runner.RunWith
+import xsbti.api.ClassLike
+import xsbti.api.Def
+import xsbti.api.Package
+import xsbt.api.SameAPI
+import org.junit.runners.JUnit4
+
+import org.specs2.mutable.Specification
+
+@RunWith(classOf[JUnit4])
+class ExtractUsedNamesSpecification extends Specification {
+
+	/**
+	 * Standard names that appear in every compilation unit that has any class
+	 * definition.
+	 */
+	private val standardNames = Set(
+		// AnyRef is added as default parent of a class
+		"scala", "AnyRef",
+		// class receives a default constructor which is internally called "<init>"
+		"<init>")
+
+	"imported name" in {
+		val src = """
+			|package a { class A }
+			|package b {
+			|	import a.{A => A2}
+			|}""".stripMargin
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
+		val expectedNames = standardNames ++ Set("a", "A", "A2", "b")
+		usedNames === expectedNames
+	}
+
+	// test covers https://github.com/gkossakowski/sbt/issues/6
+	"names in type tree" in {
+		val srcA = """|
+			|package a {
+			|  class A {
+			|    class C { class D }
+			|  }
+			|  class B[T]
+			|  class BB
+			|}""".stripMargin
+		val srcB = """|
+			|package b {
+			|	abstract class X {
+			|     def foo: a.A#C#D
+			|     def bar: a.B[a.BB]
+			|   }
+			|}""".stripMargin
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
+		val expectedNames = standardNames ++ Set("a", "A", "B", "C", "D", "b", "X", "BB")
+		usedNames === expectedNames
+	}
+
+	// test for https://github.com/gkossakowski/sbt/issues/5
+	"symbolic names" in {
+		val srcA = """|
+			|class A {
+			|  def `=`: Int = 3
+			|}""".stripMargin
+		val srcB = """|
+			|class B {
+			|  def foo(a: A) = a.`=`
+			|}""".stripMargin
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
+		val expectedNames = standardNames ++ Set("A", "a", "B", "=")
+		usedNames === expectedNames
+	}
+
+	// test for https://github.com/gkossakowski/sbt/issues/3
+	"used names from the same compilation unit" in {
+		val src = "class A { def foo: Int = 0; def bar: Int = foo }"
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
+		val expectedNames = standardNames ++ Set("A", "foo", "Int")
+		usedNames === expectedNames
+	}
+
+	// pending test for https://issues.scala-lang.org/browse/SI-7173
+	"names of constants" in {
+		val src = "class A { final val foo = 12; def bar: Int = foo }"
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
+		val expectedNames = standardNames ++ Set("A", "foo", "Int")
+		usedNames === expectedNames
+	}.pendingUntilFixed("Scala's type checker inlines constants so we can't see the original name.")
+
+	// pending test for https://github.com/gkossakowski/sbt/issues/4
+	// TODO: we should fix it by having special treatment of `selectDynamic` and `applyDynamic` calls
+	"names from method calls on Dynamic" in {
+		val srcA = """|import scala.language.dynamics
+			|class A extends Dynamic {
+			|	def selectDynamic(name: String): Int = name.length
+			|}""".stripMargin
+		val srcB = "class B { def foo(a: A): Int = a.bla }"
+		val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
+		val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
+		val expectedNames = standardNames ++ Set("B", "A", "a", "Int", "selectDynamic", "bla")
+		usedNames === expectedNames
+	}.pendingUntilFixed("Call to Dynamic is desugared in type checker so Select nodes is turned into string literal.")
+
+}

compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala

@@ -19,7 +19,7 @@ import ScalaCompilerForUnitTesting.ExtractedSourceDependencies
  * Provides common functionality needed for unit tests that require compiling
  * source code using Scala compiler.
  */
-class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false) {
+class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) {
 
 	/**
 	 * Compiles given source code using Scala compiler and returns API representation
@@ -30,6 +30,24 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false)
 		analysisCallback.apis(tempSrcFile)
 	}
 
+	def extractUsedNamesFromSrc(src: String): Set[String] = {
+		val (Seq(tempSrcFile), analysisCallback) = compileSrcs(src)
+		analysisCallback.usedNames(tempSrcFile).toSet
+	}
+
+	/**
+	 * Extract used names from src provided as the second argument.
+	 *
+	 * The purpose of the first argument is to define names that the second
+	 * source is going to refer to. Both files are compiled in the same compiler
+	 * Run but only names used in the second src file are returned.
+	 */
+	def extractUsedNamesFromSrc(definitionSrc: String, actualSrc: String): Set[String] = {
+		// we drop temp src file corresponding to the definition src file
+        val (Seq(_, tempSrcFile), analysisCallback) = compileSrcs(definitionSrc, actualSrc)
+        analysisCallback.usedNames(tempSrcFile).toSet
+	}
+
 	/**
 	 * Compiles given source code snippets (passed as Strings) using Scala compiler and returns extracted
 	 * dependencies between snippets. Source code snippets are identified by symbols. Each symbol should
@@ -76,7 +94,7 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false)
 	 */
 	private def compileSrcs(srcs: String*): (Seq[File], TestCallback) = {
 		withTemporaryDirectory { temp =>
-			val analysisCallback = new TestCallback(memberRefAndInheritanceDeps)
+			val analysisCallback = new TestCallback(nameHashing)
 			val classesDir = new File(temp, "classes")
 			classesDir.mkdir()
 			val compiler = prepareCompiler(classesDir, analysisCallback)

compile/persist/src/main/scala/sbt/inc/AnalysisFormats.scala

@@ -99,26 +99,26 @@ object AnalysisFormats
 	implicit def apisFormat(implicit internalF: Format[Map[File, Source]], externalF: Format[Map[String, Source]]): Format[APIs] =
 		asProduct2( APIs.apply _)( as => (as.internal, as.external) )(internalF, externalF)
 
-	implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS]): Format[Relations] =
+	implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS], namesF: Format[RFS]): Format[Relations] =
 		{
 			def makeRelation(srcProd: RFF, binaryDep: RFF, direct: RSource, publicInherited: RSource,
 					memberRef: SourceDependencies, inheritance: SourceDependencies, classes: RFS,
-					memberRefAndInheritanceDeps: Boolean): Relations = if (memberRefAndInheritanceDeps) {
+					nameHashing: Boolean, names: RFS): Relations = if (nameHashing) {
 				def isEmpty(sourceDependencies: RSource): Boolean =
 					sourceDependencies.internal.all.isEmpty &&  sourceDependencies.external.all.isEmpty
 				// we check direct dependencies only because publicInherited dependencies are subset of direct
-				assert(isEmpty(direct), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.")
+				assert(isEmpty(direct), "Direct dependencies are not empty but `nameHashing` flag is enabled.")
-				Relations.make(srcProd, binaryDep, memberRef, inheritance, classes)
+				Relations.make(srcProd, binaryDep, memberRef, inheritance, classes, names)
 			} else {
 				def isEmpty(sourceDependencies: SourceDependencies): Boolean =
 					sourceDependencies.internal.all.isEmpty &&  sourceDependencies.external.all.isEmpty
 				// we check memberRef dependencies only because inheritance dependencies are subset of memberRef
-				assert(isEmpty(memberRef), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.")
+				assert(isEmpty(memberRef), "Direct dependencies are not empty but `nameHashing` flag is enabled.")
 				Relations.make(srcProd, binaryDep, direct, publicInherited, classes)
 			}
-			asProduct8[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean]( (a,b,c,d,e,f,g,h) =>makeRelation(a,b,c,d,e,f,g,h) )(
+			asProduct9[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean, RFS]( (a,b,c,d,e,f,g,h,i) =>makeRelation(a,b,c,d,e,f,g,h,i) )(
-					rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.memberRefAndInheritanceDeps) )(
+					rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.nameHashing, rs.names) )(
-							prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]])
+							prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]], namesF)
 		}
 
 	implicit def relationsSourceFormat(implicit internalFormat: Format[Relation[File, File]], externalFormat: Format[Relation[File,String]]): Format[RSource] =

compile/persist/src/main/scala/sbt/inc/TextAnalysisFormat.scala

@@ -117,6 +117,8 @@ object TextAnalysisFormat {
 			val memberRefExternalDep = "member reference external dependencies"
 			val inheritanceInternalDep = "inheritance internal dependencies"
 			val inheritanceExternalDep = "inheritance external dependencies"
+
+			val usedNames = "used names"
 		}
 
 		def write(out: Writer, relations: Relations) {
@@ -134,17 +136,17 @@ object TextAnalysisFormat {
 				}
 			}
 
-			val memberRefAndInheritanceDeps = relations.memberRefAndInheritanceDeps
+			val nameHashing = relations.nameHashing
 			writeRelation(Headers.srcProd,          relations.srcProd)
 			writeRelation(Headers.binaryDep,        relations.binaryDep)
 
-			val direct = if (memberRefAndInheritanceDeps) Relations.emptySource else relations.direct
+			val direct = if (nameHashing) Relations.emptySource else relations.direct
-			val publicInherited = if (memberRefAndInheritanceDeps)
+			val publicInherited = if (nameHashing)
 				Relations.emptySource else relations.publicInherited
 
-			val memberRef = if (memberRefAndInheritanceDeps)
+			val memberRef = if (nameHashing)
 				relations.memberRef else Relations.emptySourceDependencies
-			val inheritance = if (memberRefAndInheritanceDeps)
+			val inheritance = if (nameHashing)
 				relations.inheritance else Relations.emptySourceDependencies
 
 			writeRelation(Headers.directSrcDep, direct.internal)
@@ -158,6 +160,7 @@ object TextAnalysisFormat {
 			writeRelation(Headers.inheritanceExternalDep, inheritance.external)
 
 			writeRelation(Headers.classes,        relations.classes)
+			writeRelation(Headers.usedNames, relations.names)
 		}
 
 		def read(in: BufferedReader): Relations = {
@@ -213,13 +216,17 @@ object TextAnalysisFormat {
 			// we assume that invariant that says they are subsets of direct/memberRef holds
 			assert((directSrcDeps == emptySource) || (memberRefSrcDeps == emptySourceDependencies),
 					"One mechanism is supported for tracking source dependencies at the time")
-			val memberRefAndInheritanceDeps = memberRefSrcDeps != emptySourceDependencies
+			val nameHashing = memberRefSrcDeps != emptySourceDependencies
 			val classes =          readStringRelation(Headers.classes)
+			val names = readStringRelation(Headers.usedNames)
 
-			if (memberRefAndInheritanceDeps)
+			if (nameHashing)
-				Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes)
+				Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes, names)
-			else
+			else {
+				assert(names.all.isEmpty, s"When `nameHashing` is disabled `names` relation " +
+					"should be empty: $names")
 				Relations.make(srcProd, binaryDep, directSrcDeps, publicInheritedSrcDeps, classes)
+			}
 		}
 	}
 

interface/src/main/java/xsbti/AnalysisCallback.java

@@ -24,12 +24,16 @@ public interface AnalysisCallback
 	public void generatedClass(File source, File module, String name);
 	/** Called when the public API of a source file is extracted. */
 	public void api(File sourceFile, xsbti.api.SourceAPI source);
+	public void usedName(File sourceFile, String names);
 	/** Provides problems discovered during compilation.  These may be reported (logged) or unreported.
 	* Unreported problems are usually unreported because reporting was not enabled via a command line switch. */
 	public void problem(String what, Position pos, String msg, Severity severity, boolean reported);
 	/**
-	 * Determines whether member reference and inheritance dependencies should be extracted in given compiler
+	 * Determines whether method calls through this interface should be interpreted as serving
-	 * run.
+	 * name hashing algorithm needs in given compiler run.
+	 *
+	 * In particular, it indicates whether member reference and inheritance dependencies should be
+	 * extracted.
 	 *
 	 * As the signature suggests, this method's implementation is meant to be side-effect free. It's added
 	 * to AnalysisCallback because it indicates how other callback calls should be interpreted by both
@@ -38,5 +42,5 @@ public interface AnalysisCallback
 	 * NOTE: This method is an implementation detail and can be removed at any point without deprecation.
 	 *       Do not depend on it, please.
 	 */
-	public boolean memberRefAndInheritanceDeps();
+	public boolean nameHashing();
 }

interface/src/test/scala/xsbti/TestCallback.scala

@@ -4,17 +4,19 @@ import java.io.File
 import scala.collection.mutable.ArrayBuffer
 import xsbti.api.SourceAPI
 
-class TestCallback(override val memberRefAndInheritanceDeps: Boolean = false) extends AnalysisCallback
+class TestCallback(override val nameHashing: Boolean = false) extends AnalysisCallback
 {
 	val sourceDependencies = new ArrayBuffer[(File, File, Boolean)]
 	val binaryDependencies = new ArrayBuffer[(File, String, File, Boolean)]
 	val products = new ArrayBuffer[(File, File, String)]
+	val usedNames = scala.collection.mutable.Map.empty[File, Set[String]].withDefaultValue(Set.empty)
 	val apis: scala.collection.mutable.Map[File, SourceAPI] = scala.collection.mutable.Map.empty
 
 	def sourceDependency(dependsOn: File, source: File, inherited: Boolean) { sourceDependencies += ((dependsOn, source, inherited)) }
 	def binaryDependency(binary: File, name: String, source: File, inherited: Boolean) { binaryDependencies += ((binary, name, source, inherited)) }
 	def generatedClass(source: File, module: File, name: String) { products += ((source, module, name)) }
 
+	def usedName(source: File, name: String) { usedNames(source) += name }
 	def api(source: File, sourceAPI: SourceAPI): Unit = {
 		assert(!apis.contains(source), s"The `api` method should be called once per source file: $source")
 		apis(source) = sourceAPI

project/Sbt.scala

@@ -270,6 +270,9 @@ object Sbt extends Build
 		// we are expecting all of our dependencies to be on classpath so Scala compiler
 		// can use them while constructing its own classpath for compilation
 		fork in Test := true,
+		// needed because we fork tests and tests are ran in parallel so we have multiple Scala
+		// compiler instances that are memory hungry
+		javaOptions in Test += "-Xmx1G",
 		artifact in (Compile, packageSrc) := Artifact(srcID).copy(configurations = Compile :: Nil).extra("e:component" -> srcID)
 	)
 	def compilerSettings = Seq(