Merge pull request #1013 from gkossakowski/used-names-extraction

Used names extraction logic
This commit is contained in:
Grzegorz Kossakowski 2013-12-03 03:30:21 -08:00
commit ec40eab92d
14 changed files with 377 additions and 85 deletions

View File

@ -59,7 +59,7 @@ object Analysis
/** Merge multiple analysis objects into one. Deps will be internalized as needed. */
def merge(analyses: Traversable[Analysis]): Analysis = {
if (analyses.exists(_.relations.memberRefAndInheritanceDeps))
if (analyses.exists(_.relations.nameHashing))
throw new IllegalArgumentException("Merging of Analyses that have" +
"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")
@ -160,7 +160,7 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat
copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos )
def groupBy[K](discriminator: File => K): Map[K, Analysis] = {
if (relations.memberRefAndInheritanceDeps)
if (relations.nameHashing)
throw new UnsupportedOperationException("Grouping of Analyses that have" +
"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")

View File

@ -67,6 +67,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
import collection.mutable.{HashMap, HashSet, ListBuffer, Map, Set}
private[this] val apis = new HashMap[File, (Int, SourceAPI)]
private[this] val usedNames = new HashMap[File, Set[String]]
private[this] val unreporteds = new HashMap[File, ListBuffer[Problem]]
private[this] val reporteds = new HashMap[File, ListBuffer[Problem]]
private[this] val binaryDeps = new HashMap[File, Set[File]]
@ -151,9 +152,11 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
apis(sourceFile) = (HashAPI(source), savedSource)
}
def memberRefAndInheritanceDeps: Boolean = false // TODO: define the flag in IncOptions which controls this
def usedName(sourceFile: File, name: String) = add(usedNames, sourceFile, name)
def get: Analysis = addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) )
def nameHashing: Boolean = false // TODO: define the flag in IncOptions which controls this
def get: Analysis = addUsedNames( addCompilation( addExternals( addBinaries( addProducts( addSources(Analysis.Empty) ) ) ) ) )
def addProducts(base: Analysis): Analysis = addAll(base, classes) { case (a, src, (prod, name)) => a.addProduct(src, prod, current product prod, name ) }
def addBinaries(base: Analysis): Analysis = addAll(base, binaryDeps)( (a, src, bin) => a.addBinaryDep(src, bin, binaryClassName(bin), current binary bin) )
def addSources(base: Analysis): Analysis =
@ -171,6 +174,9 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
def getOrNil[A,B](m: collection.Map[A, Seq[B]], a: A): Seq[B] = m.get(a).toList.flatten
def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api, inherited)) => a.addExternalDep(source, name, api, inherited) }
def addCompilation(base: Analysis): Analysis = base.copy(compilations = base.compilations.add(compilation))
def addUsedNames(base: Analysis): Analysis = (base /: usedNames) { case (a, (src, names)) =>
(a /: names) { case (a, name) => a.copy(relations = a.relations.addUsedName(src, name)) }
}
def addAll[A,B](base: Analysis, m: Map[A, Set[B]])( f: (Analysis, A, B) => Analysis): Analysis =
(base /: m) { case (outer, (a, bs)) =>

View File

@ -58,6 +58,8 @@ trait Relations
/** Internal source dependencies that depend on external source file `dep`. This includes both direct and inherited dependencies. */
def usesExternal(dep: String): Set[File]
private[inc] def usedNames(src: File): Set[String]
/** Records internal source file `src` as generating class file `prod` with top-level class `name`. */
def addProduct(src: File, prod: File, name: String): Relations
@ -74,6 +76,8 @@ trait Relations
* this method does not automatically record direct dependencies like `addExternalDep` does.*/
def addInternalSrcDeps(src: File, directDependsOn: Iterable[File], inheritedDependsOn: Iterable[File]): Relations
private[inc] def addUsedName(src: File, name: String): Relations
/** Concatenates the two relations. Acts naively, i.e., doesn't internalize external deps on added files. */
def ++ (o: Relations): Relations
@ -105,7 +109,7 @@ trait Relations
* This relation properly accounts for that so the invariant that `memberRef` is a superset
* of `inheritance` is preserved.
*/
def memberRef: SourceDependencies
private[inc] def memberRef: SourceDependencies
/**
* The source dependency relation between source files introduced by inheritance.
@ -135,7 +139,7 @@ trait Relations
* resolved transitively. You should not rely on this behavior, though.
*
*/
def inheritance: SourceDependencies
private[inc] def inheritance: SourceDependencies
/** The dependency relations between sources. These include both direct and inherited dependencies.*/
def direct: Source
@ -147,17 +151,26 @@ trait Relations
def classes: Relation[File, String]
/**
* Flag which indicates whether the new style (based on `memberRef` and `inheritance` source dependencies)
* of dependency tracking is enabled. When this flag is enabled access to `direct` and `publicInherited`
* relations is illegal and will cause runtime exception being thrown.
* Flag which indicates whether given Relations object supports operations needed by name hashing algorithm.
*
* Conversely, when `memberRefAndInheritanceDeps` flag is disabled access to `memberRef` and `inheritance`
* relations is illegal and will cause runtime exception being thrown.
* At the moment the list includes the following operations:
*
* The name of this flag is ugly but it's private to incremental compiler and it's temporary measure during
* our migration to the new dependency tracking.
* - memberRef: SourceDependencies
* - inheritance: SourceDependencies
*
* The `memberRef` and `inheritance` implement a new style source dependency tracking. When this flag is
* enabled access to `direct` and `publicInherited` relations is illegal and will cause runtime exception
* being thrown. That is done as an optimization that prevents from storing two overlapping sets of
* dependencies.
*
* Conversely, when `nameHashing` flag is disabled access to `memberRef` and `inheritance`
* relations is illegal and will cause runtime exception being thrown.
*/
private[inc] def memberRefAndInheritanceDeps: Boolean
private[inc] def nameHashing: Boolean
/**
* Relation between source files and _unqualified_ term and type names used in given source file.
*/
private[inc] def names: Relation[File, String]
}
@ -213,19 +226,21 @@ object Relations
def emptySource: Source = es
private[inc] lazy val emptySourceDependencies: SourceDependencies = new SourceDependencies(e, estr)
def empty: Relations = empty(memberRefAndInheritanceDeps = false)
def empty(memberRefAndInheritanceDeps: Boolean): Relations =
if (memberRefAndInheritanceDeps)
new MRelationsMemberRefAndInheritance(e, e, emptySourceDependencies, emptySourceDependencies, estr)
def empty: Relations = empty(nameHashing = false)
private[inc] def empty(nameHashing: Boolean): Relations =
if (nameHashing)
new MRelationsNameHashing(e, e, emptySourceDependencies, emptySourceDependencies, estr, estr)
else
new MRelationsDirectAndPublicInherited(e, e, es, es, estr)
new MRelationsDefaultImpl(e, e, es, es, estr)
def make(srcProd: Relation[File, File], binaryDep: Relation[File, File], direct: Source, publicInherited: Source, classes: Relation[File, String]): Relations =
new MRelationsDirectAndPublicInherited(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes)
new MRelationsDefaultImpl(srcProd, binaryDep, direct = direct, publicInherited = publicInherited, classes)
private[inc] def make(srcProd: Relation[File, File], binaryDep: Relation[File, File],
memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String]): Relations =
new MRelationsMemberRefAndInheritance(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance, classes)
memberRef: SourceDependencies, inheritance: SourceDependencies, classes: Relation[File, String],
names: Relation[File, String]): Relations =
new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef, inheritance = inheritance,
classes, names)
def makeSource(internal: Relation[File,File], external: Relation[File,String]): Source = new Source(internal, external)
private[inc] def makeSourceDependencies(internal: Relation[File,File], external: Relation[File,String]): SourceDependencies = new SourceDependencies(internal, external)
}
@ -276,6 +291,8 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b
def externalDeps(src: File): Set[String] = externalDep.forward(src)
def usesExternal(dep: String): Set[File] = externalDep.reverse(dep)
def usedNames(src: File): Set[String] = names.forward(src)
/** Making large Relations a little readable. */
private val userDir = sys.props("user.dir").stripSuffix("/") + "/"
private def nocwd(s: String) = s stripPrefix userDir
@ -309,7 +326,7 @@ private abstract class MRelationsCommon(val srcProd: Relation[File, File], val b
* introduced by inheritance.
*
*/
private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File], binaryDep: Relation[File, File],
private class MRelationsDefaultImpl(srcProd: Relation[File, File], binaryDep: Relation[File, File],
// direct should include everything in inherited
val direct: Source, val publicInherited: Source,
classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
@ -317,45 +334,53 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
def internalSrcDep: Relation[File, File] = direct.internal
def externalDep: Relation[File, String] = direct.external
def memberRefAndInheritanceDeps: Boolean = false
def nameHashing: Boolean = false
def memberRef: SourceDependencies =
throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " +
"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
"when `nameHashing` flag is disabled.")
def inheritance: SourceDependencies =
throw new UnsupportedOperationException("The `memberRef` source dependencies relation is not supported " +
"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
"when `nameHashing` flag is disabled.")
def addProduct(src: File, prod: File, name: String): Relations =
new MRelationsDirectAndPublicInherited(srcProd + (src, prod), binaryDep, direct = direct,
new MRelationsDefaultImpl(srcProd + (src, prod), binaryDep, direct = direct,
publicInherited = publicInherited, classes + (src, name))
def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = {
val newI = if(inherited) publicInherited.addExternal(src, dependsOn) else publicInherited
val newD = direct.addExternal(src, dependsOn)
new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
}
def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations =
{
val newI = publicInherited.addInternal(src, inherited)
val newD = direct.addInternal(src, dependsOn)
new MRelationsDirectAndPublicInherited( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
new MRelationsDefaultImpl( srcProd, binaryDep, direct = newD, publicInherited = newI, classes)
}
def names: Relation[File, String] =
throw new UnsupportedOperationException("Tracking of used names is not supported " +
"when `nameHashing` is disabled.")
def addUsedName(src: File, name: String): Relations =
throw new UnsupportedOperationException("Tracking of used names is not supported " +
"when `nameHashing` is disabled.")
def addBinaryDep(src: File, dependsOn: File): Relations =
new MRelationsDirectAndPublicInherited( srcProd, binaryDep + (src, dependsOn), direct = direct,
new MRelationsDefaultImpl( srcProd, binaryDep + (src, dependsOn), direct = direct,
publicInherited = publicInherited, classes)
def ++ (o: Relations): Relations = {
if (memberRefAndInheritanceDeps != o.memberRefAndInheritanceDeps)
if (nameHashing != o.nameHashing)
throw new UnsupportedOperationException("The `++` operation is not supported for relations " +
"with different values of `memberRefAndInheritanceDeps` flag.")
new MRelationsDirectAndPublicInherited(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct,
"with different values of `nameHashing` flag.")
new MRelationsDefaultImpl(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep, direct ++ o.direct,
publicInherited ++ o.publicInherited, classes ++ o.classes)
}
def -- (sources: Iterable[File]) =
new MRelationsDirectAndPublicInherited(srcProd -- sources, binaryDep -- sources, direct = direct -- sources,
new MRelationsDefaultImpl(srcProd -- sources, binaryDep -- sources, direct = direct -- sources,
publicInherited = publicInherited -- sources, classes -- sources)
@deprecated("Broken implementation. OK to remove in 0.14", "0.13.1")
@ -363,14 +388,15 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
{
type MapRel[T] = Map[K, Relation[File, T]]
def outerJoin(srcProdMap: MapRel[File], binaryDepMap: MapRel[File], direct: Map[K, Source],
inherited: Map[K, Source], classesMap: MapRel[String]): Map[K, Relations] =
inherited: Map[K, Source], classesMap: MapRel[String],
namesMap: MapRel[String]): Map[K, Relations] =
{
def kRelations(k: K): Relations = {
def get[T](m: Map[K, Relation[File, T]]) = Relations.getOrEmpty(m, k)
def getSrc(m: Map[K, Source]): Source = m.getOrElse(k, Relations.emptySource)
def getSrcDeps(m: Map[K, SourceDependencies]): SourceDependencies =
m.getOrElse(k, Relations.emptySourceDependencies)
new MRelationsDirectAndPublicInherited( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited),
new MRelationsDefaultImpl( get(srcProdMap), get(binaryDepMap), getSrc(direct), getSrc(inherited),
get(classesMap))
}
val keys = (srcProdMap.keySet ++ binaryDepMap.keySet ++ direct.keySet ++ inherited.keySet ++ classesMap.keySet).toList
@ -380,11 +406,11 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
def f1[B](item: (File, B)): K = f(item._1)
outerJoin(srcProd.groupBy(f1), binaryDep.groupBy(f1), direct.groupBySource(f),
publicInherited.groupBySource(f), classes.groupBy(f1))
publicInherited.groupBySource(f), classes.groupBy(f1), names.groupBy(f1))
}
override def equals(other: Any) = other match {
case o: MRelationsDirectAndPublicInherited =>
case o: MRelationsDefaultImpl =>
srcProd == o.srcProd && binaryDep == o.binaryDep && direct == o.direct &&
publicInherited == o.publicInherited && classes == o.classes
case _ => false
@ -408,7 +434,8 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
| src deps: %s
| ext deps: %s
| class names: %s
""".trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes) map relation_s : _*)
| used names: %s
""".trim.stripMargin.format(List(srcProd, binaryDep, internalSrcDep, externalDep, classes, names) map relation_s : _*)
)
}
@ -417,62 +444,70 @@ private class MRelationsDirectAndPublicInherited(srcProd: Relation[File, File],
* dependencies. Therefore this class implements the new (compared to sbt 0.13.0) dependency tracking logic
* needed by the name hashing invalidation algorithm.
*/
private class MRelationsMemberRefAndInheritance(srcProd: Relation[File, File], binaryDep: Relation[File, File],
private class MRelationsNameHashing(srcProd: Relation[File, File], binaryDep: Relation[File, File],
// memberRef should include everything in inherited
val memberRef: SourceDependencies, val inheritance: SourceDependencies,
classes: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
classes: Relation[File, String],
val names: Relation[File, String]) extends MRelationsCommon(srcProd, binaryDep, classes)
{
def direct: Source =
throw new UnsupportedOperationException("The `direct` source dependencies relation is not supported " +
"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
"when `nameHashing` flag is disabled.")
def publicInherited: Source =
throw new UnsupportedOperationException("The `publicInherited` source dependencies relation is not supported " +
"when `memberRefAndInheritanceDeps` is disabled. Do you have name hashing algorithm disabled?")
"when `nameHashing` flag is disabled.")
val memberRefAndInheritanceDeps: Boolean = true
val nameHashing: Boolean = true
def internalSrcDep: Relation[File, File] = memberRef.internal
def externalDep: Relation[File, String] = memberRef.external
def addProduct(src: File, prod: File, name: String): Relations =
new MRelationsMemberRefAndInheritance(srcProd + (src, prod), binaryDep, memberRef = memberRef,
inheritance = inheritance, classes + (src, name))
new MRelationsNameHashing(srcProd + (src, prod), binaryDep, memberRef = memberRef,
inheritance = inheritance, classes + (src, name), names = names)
def addExternalDep(src: File, dependsOn: String, inherited: Boolean): Relations = {
val newIH = if(inherited) inheritance.addExternal(src, dependsOn) else inheritance
val newMR = memberRef.addExternal(src, dependsOn)
new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes)
new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes,
names = names)
}
def addInternalSrcDeps(src: File, dependsOn: Iterable[File], inherited: Iterable[File]): Relations = {
val newIH = inheritance.addInternal(src, inherited)
val newMR = memberRef.addInternal(src, dependsOn)
new MRelationsMemberRefAndInheritance( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes)
new MRelationsNameHashing( srcProd, binaryDep, memberRef = newMR, inheritance = newIH, classes,
names = names)
}
def addUsedName(src: File, name: String): Relations =
new MRelationsNameHashing(srcProd, binaryDep, memberRef = memberRef,
inheritance = inheritance, classes, names = names + (src, name))
def addBinaryDep(src: File, dependsOn: File): Relations =
new MRelationsMemberRefAndInheritance(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef,
inheritance = inheritance, classes)
new MRelationsNameHashing(srcProd, binaryDep + (src, dependsOn), memberRef = memberRef,
inheritance = inheritance, classes, names = names)
def ++ (o: Relations): Relations = {
if (!o.memberRefAndInheritanceDeps)
if (!o.nameHashing)
throw new UnsupportedOperationException("The `++` operation is not supported for relations " +
"with different values of `memberRefAndInheritanceDeps` flag.")
new MRelationsMemberRefAndInheritance(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep,
"with different values of `nameHashing` flag.")
new MRelationsNameHashing(srcProd ++ o.srcProd, binaryDep ++ o.binaryDep,
memberRef = memberRef ++ o.memberRef, inheritance = inheritance ++ o.inheritance,
classes ++ o.classes)
classes ++ o.classes, names = names ++ o.names)
}
def -- (sources: Iterable[File]) =
new MRelationsMemberRefAndInheritance(srcProd -- sources, binaryDep -- sources,
memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources)
new MRelationsNameHashing(srcProd -- sources, binaryDep -- sources,
memberRef = memberRef -- sources, inheritance = inheritance -- sources, classes -- sources,
names = names -- sources)
def groupBy[K](f: File => K): Map[K, Relations] = {
throw new UnsupportedOperationException("Merging of Analyses that have" +
"`relations.memberRefAndInheritanceDeps` set to `true` is not supported.")
"`relations.nameHashing` set to `true` is not supported.")
}
override def equals(other: Any) = other match {
case o: MRelationsMemberRefAndInheritance =>
case o: MRelationsNameHashing =>
srcProd == o.srcProd && binaryDep == o.binaryDep && memberRef == o.memberRef &&
inheritance == o.inheritance && classes == o.classes
case _ => false

View File

@ -43,6 +43,12 @@ final class API(val global: CallbackGlobal) extends Compat
val extractApi = new ExtractAPI[global.type](global, sourceFile)
val traverser = new TopLevelHandler(extractApi)
traverser.apply(unit.body)
if (global.callback.nameHashing) {
val extractUsedNames = new ExtractUsedNames[global.type](global)
val names = extractUsedNames.extract(unit)
debug("The " + sourceFile + " contains the following used names " + names)
names foreach { (name: String) => callback.usedName(sourceFile, name) }
}
val packages = traverser.packages.toArray[String].map(p => new xsbti.api.Package(p))
val source = new xsbti.api.SourceAPI(packages, traverser.definitions.toArray[xsbti.api.Definition])
extractApi.forceStructures()

View File

@ -43,7 +43,7 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile
{
// build dependencies structure
val sourceFile = unit.source.file.file
if (global.callback.memberRefAndInheritanceDeps) {
if (global.callback.nameHashing) {
val dependenciesByMemberRef = extractDependenciesByMemberRef(unit)
for(on <- dependenciesByMemberRef)
processDependency(on, inherited=false)

View File

@ -0,0 +1,103 @@
package xsbt
import scala.tools.nsc._
/**
* Extracts simple names used in given compilation unit.
*
* Extracts simple (unqualified) names mentioned in given in non-definition position by collecting
* all symbols associated with non-definition trees and extracting names from all collected symbols.
*
* If given symbol is mentioned both in definition and in non-definition position (e.g. in member
* selection) then that symbol is collected. It means that names of symbols defined and used in the
* same compilation unit are extracted. We've considered not extracting names of those symbols
* as an optimization strategy. It turned out that this is not correct. Check
* https://github.com/gkossakowski/sbt/issues/3 for an example of scenario where it matters.
*
* All extracted names are returned in _decoded_ form. This way we stay consistent with the rest
* of incremental compiler which works with names in decoded form.
*
* Names mentioned in Import nodes are handled properly but require some special logic for two
* reasons:
*
* 1. import node itself has a term symbol associated with it with a name `<import`>.
* I (gkossakowski) tried to track down what role this symbol serves but I couldn't.
* It doesn't look like there are many places in Scala compiler that refer to
* that kind of symbols explicitly.
* 2. ImportSelector is not subtype of Tree therefore is not processed by `Tree.foreach`
*
* Another type of tree nodes that requires special handling is TypeTree. TypeTree nodes
* has a little bit odd representation:
*
* 1. TypeTree.hasSymbol always returns false even when TypeTree.symbol
* returns a symbol
* 2. The original tree from which given TypeTree was derived is stored
* in TypeTree.original but Tree.forech doesn't walk into original
* tree so we missed it
*
* The tree walking algorithm walks into TypeTree.original explicitly.
*
*/
class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) {
import global._
def extract(unit: CompilationUnit): Set[String] = {
val tree = unit.body
val extractedByTreeWalk = extractByTreeWalk(tree)
extractedByTreeWalk
}
private def extractByTreeWalk(tree: Tree): Set[String] = {
val namesBuffer = collection.mutable.ListBuffer.empty[String]
def addSymbol(symbol: Symbol): Unit = {
val symbolNameAsString = symbol.name.decode.trim
namesBuffer += symbolNameAsString
}
def handleTreeNode(node: Tree): Unit = node match {
case _: DefTree | _: Template => ()
// turns out that Import node has a TermSymbol associated with it
// I (Grzegorz) tried to understand why it's there and what does it represent but
// that logic was introduced in 2005 without any justification I'll just ignore the
// import node altogether and just process the selectors in the import node
case Import(_, selectors: List[ImportSelector]) =>
def usedNameInImportSelector(name: Name): Unit =
if ((name != null) && (name != nme.WILDCARD)) namesBuffer += name.toString
selectors foreach { selector =>
usedNameInImportSelector(selector.name)
usedNameInImportSelector(selector.rename)
}
// TODO: figure out whether we should process the original tree or walk the type
// the argument for processing the original tree: we process what user wrote
// the argument for processing the type: we catch all transformations that typer applies
// to types but that might be a bad thing because it might expand aliases eagerly which
// not what we need
case t: TypeTree if t.original != null =>
t.original.foreach(handleTreeNode)
case t if t.hasSymbol && eligibleAsUsedName(t.symbol) =>
addSymbol(t.symbol)
case _ => ()
}
tree.foreach(handleTreeNode)
namesBuffer.toSet
}
/**
* Needed for compatibility with Scala 2.8 which doesn't define `tpnme`
*/
private object tpnme {
val EMPTY = nme.EMPTY.toTypeName
val EMPTY_PACKAGE_NAME = nme.EMPTY_PACKAGE_NAME.toTypeName
}
private def eligibleAsUsedName(symbol: Symbol): Boolean = {
def emptyName(name: Name): Boolean = name match {
case nme.EMPTY | nme.EMPTY_PACKAGE_NAME | tpnme.EMPTY | tpnme.EMPTY_PACKAGE_NAME => true
case _ => false
}
(symbol != NoSymbol) &&
!symbol.isSynthetic &&
!emptyName(symbol.name)
}
}

View File

@ -80,7 +80,7 @@ class DependencySpecification extends Specification {
// E verifies the core type gets pulled out
val srcH = "trait H extends G.T[Int] with (E[Int] @unchecked)"
val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val sourceDependencies = compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC,
'D -> srcD, 'E -> srcE, 'F -> srcF, 'G -> srcG, 'H -> srcH)
sourceDependencies
@ -92,7 +92,7 @@ class DependencySpecification extends Specification {
val srcC = "class C { private class Inner1 extends A }"
val srcD = "class D { def foo: Unit = { class Inner2 extends B } }"
val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val sourceDependencies =
compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD)
sourceDependencies
@ -104,7 +104,7 @@ class DependencySpecification extends Specification {
val srcC = "trait C extends B"
val srcD = "class D extends C"
val compilerForTesting = new ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps = true)
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val sourceDependencies =
compilerForTesting.extractDependenciesFromSrcs('A -> srcA, 'B -> srcB, 'C -> srcC, 'D -> srcD)
sourceDependencies

View File

@ -0,0 +1,108 @@
package xsbt
import org.junit.runner.RunWith
import xsbti.api.ClassLike
import xsbti.api.Def
import xsbti.api.Package
import xsbt.api.SameAPI
import org.junit.runners.JUnit4
import org.specs2.mutable.Specification
@RunWith(classOf[JUnit4])
class ExtractUsedNamesSpecification extends Specification {
/**
* Standard names that appear in every compilation unit that has any class
* definition.
*/
private val standardNames = Set(
// AnyRef is added as default parent of a class
"scala", "AnyRef",
// class receives a default constructor which is internally called "<init>"
"<init>")
"imported name" in {
val src = """
|package a { class A }
|package b {
| import a.{A => A2}
|}""".stripMargin
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
val expectedNames = standardNames ++ Set("a", "A", "A2", "b")
usedNames === expectedNames
}
// test covers https://github.com/gkossakowski/sbt/issues/6
"names in type tree" in {
val srcA = """|
|package a {
| class A {
| class C { class D }
| }
| class B[T]
| class BB
|}""".stripMargin
val srcB = """|
|package b {
| abstract class X {
| def foo: a.A#C#D
| def bar: a.B[a.BB]
| }
|}""".stripMargin
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
val expectedNames = standardNames ++ Set("a", "A", "B", "C", "D", "b", "X", "BB")
usedNames === expectedNames
}
// test for https://github.com/gkossakowski/sbt/issues/5
"symbolic names" in {
val srcA = """|
|class A {
| def `=`: Int = 3
|}""".stripMargin
val srcB = """|
|class B {
| def foo(a: A) = a.`=`
|}""".stripMargin
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
val expectedNames = standardNames ++ Set("A", "a", "B", "=")
usedNames === expectedNames
}
// test for https://github.com/gkossakowski/sbt/issues/3
"used names from the same compilation unit" in {
val src = "class A { def foo: Int = 0; def bar: Int = foo }"
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
val expectedNames = standardNames ++ Set("A", "foo", "Int")
usedNames === expectedNames
}
// pending test for https://issues.scala-lang.org/browse/SI-7173
"names of constants" in {
val src = "class A { final val foo = 12; def bar: Int = foo }"
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(src)
val expectedNames = standardNames ++ Set("A", "foo", "Int")
usedNames === expectedNames
}.pendingUntilFixed("Scala's type checker inlines constants so we can't see the original name.")
// pending test for https://github.com/gkossakowski/sbt/issues/4
// TODO: we should fix it by having special treatment of `selectDynamic` and `applyDynamic` calls
"names from method calls on Dynamic" in {
val srcA = """|import scala.language.dynamics
|class A extends Dynamic {
| def selectDynamic(name: String): Int = name.length
|}""".stripMargin
val srcB = "class B { def foo(a: A): Int = a.bla }"
val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true)
val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB)
val expectedNames = standardNames ++ Set("B", "A", "a", "Int", "selectDynamic", "bla")
usedNames === expectedNames
}.pendingUntilFixed("Call to Dynamic is desugared in type checker so Select nodes is turned into string literal.")
}

View File

@ -19,7 +19,7 @@ import ScalaCompilerForUnitTesting.ExtractedSourceDependencies
* Provides common functionality needed for unit tests that require compiling
* source code using Scala compiler.
*/
class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false) {
class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) {
/**
* Compiles given source code using Scala compiler and returns API representation
@ -30,6 +30,24 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false)
analysisCallback.apis(tempSrcFile)
}
def extractUsedNamesFromSrc(src: String): Set[String] = {
val (Seq(tempSrcFile), analysisCallback) = compileSrcs(src)
analysisCallback.usedNames(tempSrcFile).toSet
}
/**
* Extract used names from src provided as the second argument.
*
* The purpose of the first argument is to define names that the second
* source is going to refer to. Both files are compiled in the same compiler
* Run but only names used in the second src file are returned.
*/
def extractUsedNamesFromSrc(definitionSrc: String, actualSrc: String): Set[String] = {
// we drop temp src file corresponding to the definition src file
val (Seq(_, tempSrcFile), analysisCallback) = compileSrcs(definitionSrc, actualSrc)
analysisCallback.usedNames(tempSrcFile).toSet
}
/**
* Compiles given source code snippets (passed as Strings) using Scala compiler and returns extracted
* dependencies between snippets. Source code snippets are identified by symbols. Each symbol should
@ -76,7 +94,7 @@ class ScalaCompilerForUnitTesting(memberRefAndInheritanceDeps: Boolean = false)
*/
private def compileSrcs(srcs: String*): (Seq[File], TestCallback) = {
withTemporaryDirectory { temp =>
val analysisCallback = new TestCallback(memberRefAndInheritanceDeps)
val analysisCallback = new TestCallback(nameHashing)
val classesDir = new File(temp, "classes")
classesDir.mkdir()
val compiler = prepareCompiler(classesDir, analysisCallback)

View File

@ -99,26 +99,26 @@ object AnalysisFormats
implicit def apisFormat(implicit internalF: Format[Map[File, Source]], externalF: Format[Map[String, Source]]): Format[APIs] =
asProduct2( APIs.apply _)( as => (as.internal, as.external) )(internalF, externalF)
implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS]): Format[Relations] =
implicit def relationsFormat(implicit prodF: Format[RFF], binF: Format[RFF], directF: Format[RSource], inheritedF: Format[RSource], memberRefF: Format[SourceDependencies], inheritanceF: Format[SourceDependencies], csF: Format[RFS], namesF: Format[RFS]): Format[Relations] =
{
def makeRelation(srcProd: RFF, binaryDep: RFF, direct: RSource, publicInherited: RSource,
memberRef: SourceDependencies, inheritance: SourceDependencies, classes: RFS,
memberRefAndInheritanceDeps: Boolean): Relations = if (memberRefAndInheritanceDeps) {
nameHashing: Boolean, names: RFS): Relations = if (nameHashing) {
def isEmpty(sourceDependencies: RSource): Boolean =
sourceDependencies.internal.all.isEmpty && sourceDependencies.external.all.isEmpty
// we check direct dependencies only because publicInherited dependencies are subset of direct
assert(isEmpty(direct), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.")
Relations.make(srcProd, binaryDep, memberRef, inheritance, classes)
assert(isEmpty(direct), "Direct dependencies are not empty but `nameHashing` flag is enabled.")
Relations.make(srcProd, binaryDep, memberRef, inheritance, classes, names)
} else {
def isEmpty(sourceDependencies: SourceDependencies): Boolean =
sourceDependencies.internal.all.isEmpty && sourceDependencies.external.all.isEmpty
// we check memberRef dependencies only because inheritance dependencies are subset of memberRef
assert(isEmpty(memberRef), "Direct dependencies are not empty but `memberRefAndInheritanceDeps` flag is enabled.")
assert(isEmpty(memberRef), "Direct dependencies are not empty but `nameHashing` flag is enabled.")
Relations.make(srcProd, binaryDep, direct, publicInherited, classes)
}
asProduct8[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean]( (a,b,c,d,e,f,g,h) =>makeRelation(a,b,c,d,e,f,g,h) )(
rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.memberRefAndInheritanceDeps) )(
prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]])
asProduct9[Relations, RFF, RFF, RSource, RSource, SourceDependencies, SourceDependencies, RFS, Boolean, RFS]( (a,b,c,d,e,f,g,h,i) =>makeRelation(a,b,c,d,e,f,g,h,i) )(
rs => (rs.srcProd, rs.binaryDep, rs.direct, rs.publicInherited, rs.memberRef, rs.inheritance, rs.classes, rs.nameHashing, rs.names) )(
prodF, binF, directF, inheritedF, memberRefF, inheritanceF, csF, implicitly[Format[Boolean]], namesF)
}
implicit def relationsSourceFormat(implicit internalFormat: Format[Relation[File, File]], externalFormat: Format[Relation[File,String]]): Format[RSource] =

View File

@ -117,6 +117,8 @@ object TextAnalysisFormat {
val memberRefExternalDep = "member reference external dependencies"
val inheritanceInternalDep = "inheritance internal dependencies"
val inheritanceExternalDep = "inheritance external dependencies"
val usedNames = "used names"
}
def write(out: Writer, relations: Relations) {
@ -134,17 +136,17 @@ object TextAnalysisFormat {
}
}
val memberRefAndInheritanceDeps = relations.memberRefAndInheritanceDeps
val nameHashing = relations.nameHashing
writeRelation(Headers.srcProd, relations.srcProd)
writeRelation(Headers.binaryDep, relations.binaryDep)
val direct = if (memberRefAndInheritanceDeps) Relations.emptySource else relations.direct
val publicInherited = if (memberRefAndInheritanceDeps)
val direct = if (nameHashing) Relations.emptySource else relations.direct
val publicInherited = if (nameHashing)
Relations.emptySource else relations.publicInherited
val memberRef = if (memberRefAndInheritanceDeps)
val memberRef = if (nameHashing)
relations.memberRef else Relations.emptySourceDependencies
val inheritance = if (memberRefAndInheritanceDeps)
val inheritance = if (nameHashing)
relations.inheritance else Relations.emptySourceDependencies
writeRelation(Headers.directSrcDep, direct.internal)
@ -158,6 +160,7 @@ object TextAnalysisFormat {
writeRelation(Headers.inheritanceExternalDep, inheritance.external)
writeRelation(Headers.classes, relations.classes)
writeRelation(Headers.usedNames, relations.names)
}
def read(in: BufferedReader): Relations = {
@ -213,13 +216,17 @@ object TextAnalysisFormat {
// we assume that invariant that says they are subsets of direct/memberRef holds
assert((directSrcDeps == emptySource) || (memberRefSrcDeps == emptySourceDependencies),
"One mechanism is supported for tracking source dependencies at the time")
val memberRefAndInheritanceDeps = memberRefSrcDeps != emptySourceDependencies
val nameHashing = memberRefSrcDeps != emptySourceDependencies
val classes = readStringRelation(Headers.classes)
val names = readStringRelation(Headers.usedNames)
if (memberRefAndInheritanceDeps)
Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes)
else
if (nameHashing)
Relations.make(srcProd, binaryDep, memberRefSrcDeps, inheritanceSrcDeps, classes, names)
else {
assert(names.all.isEmpty, s"When `nameHashing` is disabled `names` relation " +
"should be empty: $names")
Relations.make(srcProd, binaryDep, directSrcDeps, publicInheritedSrcDeps, classes)
}
}
}

View File

@ -24,12 +24,16 @@ public interface AnalysisCallback
public void generatedClass(File source, File module, String name);
/** Called when the public API of a source file is extracted. */
public void api(File sourceFile, xsbti.api.SourceAPI source);
public void usedName(File sourceFile, String names);
/** Provides problems discovered during compilation. These may be reported (logged) or unreported.
* Unreported problems are usually unreported because reporting was not enabled via a command line switch. */
public void problem(String what, Position pos, String msg, Severity severity, boolean reported);
/**
* Determines whether member reference and inheritance dependencies should be extracted in given compiler
* run.
* Determines whether method calls through this interface should be interpreted as serving
* name hashing algorithm needs in given compiler run.
*
* In particular, it indicates whether member reference and inheritance dependencies should be
* extracted.
*
* As the signature suggests, this method's implementation is meant to be side-effect free. It's added
* to AnalysisCallback because it indicates how other callback calls should be interpreted by both
@ -38,5 +42,5 @@ public interface AnalysisCallback
* NOTE: This method is an implementation detail and can be removed at any point without deprecation.
* Do not depend on it, please.
*/
public boolean memberRefAndInheritanceDeps();
public boolean nameHashing();
}

View File

@ -4,17 +4,19 @@ import java.io.File
import scala.collection.mutable.ArrayBuffer
import xsbti.api.SourceAPI
class TestCallback(override val memberRefAndInheritanceDeps: Boolean = false) extends AnalysisCallback
class TestCallback(override val nameHashing: Boolean = false) extends AnalysisCallback
{
val sourceDependencies = new ArrayBuffer[(File, File, Boolean)]
val binaryDependencies = new ArrayBuffer[(File, String, File, Boolean)]
val products = new ArrayBuffer[(File, File, String)]
val usedNames = scala.collection.mutable.Map.empty[File, Set[String]].withDefaultValue(Set.empty)
val apis: scala.collection.mutable.Map[File, SourceAPI] = scala.collection.mutable.Map.empty
def sourceDependency(dependsOn: File, source: File, inherited: Boolean) { sourceDependencies += ((dependsOn, source, inherited)) }
def binaryDependency(binary: File, name: String, source: File, inherited: Boolean) { binaryDependencies += ((binary, name, source, inherited)) }
def generatedClass(source: File, module: File, name: String) { products += ((source, module, name)) }
def usedName(source: File, name: String) { usedNames(source) += name }
def api(source: File, sourceAPI: SourceAPI): Unit = {
assert(!apis.contains(source), s"The `api` method should be called once per source file: $source")
apis(source) = sourceAPI

View File

@ -270,6 +270,9 @@ object Sbt extends Build
// we are expecting all of our dependencies to be on classpath so Scala compiler
// can use them while constructing its own classpath for compilation
fork in Test := true,
// needed because we fork tests and tests are ran in parallel so we have multiple Scala
// compiler instances that are memory hungry
javaOptions in Test += "-Xmx1G",
artifact in (Compile, packageSrc) := Artifact(srcID).copy(configurations = Compile :: Nil).extra("e:component" -> srcID)
)
def compilerSettings = Seq(