store hashes of API instead of full API. fixes #21

This commit is contained in:
Mark Harrah 2011-10-05 18:09:27 -04:00
parent b6fc7ba0a7
commit e4848efcc8
17 changed files with 406 additions and 43 deletions

View File

@ -1,5 +1,6 @@
package xsbt.api
import xsbti.SafeLazy
import xsbti.api._
import scala.collection.mutable.HashSet
@ -42,4 +43,27 @@ object APIUtil
super.visitParameterRef(ref)
}
}
def minimize(api: SourceAPI): SourceAPI =
new SourceAPI(api.packages, minimizeDefinitions(api.definitions))
def minimizeDefinitions(ds: Array[Definition]): Array[Definition] =
ds flatMap minimizeDefinition
def minimizeDefinition(d: Definition): Array[Definition] =
d match
{
case c: ClassLike => Array(minimizeClass(c))
case _ => Array()
}
def minimizeClass(c: ClassLike): ClassLike =
{
val struct = minimizeStructure(c.structure, c.definitionType == DefinitionType.Module)
new ClassLike(c.definitionType, lzy(emptyType), lzy(struct), c.typeParameters, c.name, c.access, c.modifiers, c.annotations)
}
def minimizeStructure(s: Structure, isModule: Boolean): Structure =
new Structure(lzy(s.parents), filterDefinitions(s.declared, isModule), filterDefinitions(s.inherited, isModule))
def filterDefinitions(ds: Array[Definition], isModule: Boolean): Lazy[Array[Definition]] =
lzy(if(isModule) ds filter Discovery.isMainMethod else Array())
private[this] def lzy[T <: AnyRef](t: T): Lazy[T] = SafeLazy.strict(t)
private[this] val emptyType = new EmptyType
}

View File

@ -1,8 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package compiler
package xsbt.api
final case class Discovered(baseClasses: Set[String], annotations: Set[String], hasMain: Boolean, isModule: Boolean)
{

View File

@ -1,8 +1,7 @@
/* sbt -- Simple Build Tool
* Copyright 2010 Mark Harrah
*/
package sbt
package compiler
package xsbt.api
import xsbti.api.{Path => APath, _}

340
compile/api/HashAPI.scala Normal file
View File

@ -0,0 +1,340 @@
/* sbt -- Simple Build Tool
* Copyright 2009, 2010, 2011 Mark Harrah
*/
package xsbt.api
import xsbti.api._
import util.MurmurHash
import TagTypeVariables.TypeVars
import HashAPI.Hash
object HashAPI
{
type Hash = Int
def apply(a: SourceAPI): Hash =
{
/** de Bruijn levels for type parameters in source a and b*/
val tags = TagTypeVariables(a)
(new HashAPI(tags, false, true)).hashAPI(a)
}
}
final class HashAPI(tags: TypeVars, includePrivate: Boolean, includeParamNames: Boolean)
{
import scala.collection.mutable
import MurmurHash._
private[this] val visitedStructures = visitedMap[Structure]
private[this] val visitedClassLike = visitedMap[ClassLike]
private[this] def visitedMap[T] = new mutable.HashMap[T, List[Hash]]
private[this] def visit[T](map: mutable.Map[T, List[Hash]], t: T)(hashF: T => Unit)
{
map.put(t, hash :: map.getOrElse(t,Nil)) match {
case Some(x :: _) => extend(x)
case _ =>
hashF(t)
for(hs <- map(t))
extend(hs)
map.put(t, hash :: Nil)
}
}
private[this] final val ValHash = 1
private[this] final val VarHash = 2
private[this] final val DefHash = 3
private[this] final val ClassHash = 4
private[this] final val TypeDeclHash = 5
private[this] final val TypeAliasHash = 6
private[this] final val PublicHash = 30
private[this] final val ProtectedHash = 31
private[this] final val PrivateHash = 32
private[this] final val UnqualifiedHash = 33
private[this] final val ThisQualifierHash = 34
private[this] final val IdQualifierHash = 35
private[this] final val IdPathHash = 20
private[this] final val SuperHash = 21
private[this] final val ThisPathHash = 22
private[this] final val ValueParamsHash = 40
private[this] final val ClassPendingHash = 41
private[this] final val StructurePendingHash = 42
private[this] final val EmptyTypeHash = 51
private[this] final val ParameterRefHash = 52
private[this] final val SingletonHash = 53
private[this] final val ProjectionHash = 54
private[this] final val ParameterizedHash = 55
private[this] final val AnnotatedHash = 56
private[this] final val PolymorphicHash = 57
private[this] final val ConstantHash = 58
private[this] final val ExistentialHash = 59
private[this] final val StructureHash = 60
private[this] final val TrueHash = 97
private[this] final val FalseHash = 98
private[this] var hash: Hash = startHash(0)
private[this] var magicA: Hash = startMagicA
private[this] var magicB: Hash = startMagicB
@inline final def hashString(s: String): Unit = extend(stringHash(s))
@inline final def hashBoolean(b: Boolean): Unit = extend(if(b) TrueHash else FalseHash)
@inline final def hashSeq[T](s: Seq[T], hashF: T => Unit)
{
extend(s.length)
s foreach hashF
}
final def hashSymmetric[T](ts: TraversableOnce[T], hashF: T => Unit): Unit =
{
val current = hash
val hashes = ts.toList map { t =>
hash = startHash(1)
hashF(t)
hash
}
hash = symmetricHash(hashes, current)
}
@inline final def extend(a: Hash)
{
hash = extendHash(hash, a, magicA, magicB)
magicA = nextMagicA(magicA)
magicB = nextMagicB(magicB)
}
def hashModifiers(m: Modifiers) = extend(m.raw)
def hashAPI(s: SourceAPI): Hash =
{
hash = startHash(0)
hashSymmetric(s.packages, hashPackage)
hashDefinitions(s.definitions, true)
hash
}
def hashPackage(p: Package) = hashString(p.name)
def hashDefinitions(ds: Seq[Definition], topLevel: Boolean): Unit =
{
val defs = SameAPI.filterDefinitions(ds, topLevel, includePrivate)
hashSymmetric(defs, hashDefinition)
}
def hashDefinition(d: Definition) =
{
hashString(d.name)
hashAnnotations(d.annotations)
hashModifiers(d.modifiers)
hashAccess(d.access)
d match
{
case c: ClassLike => hashClass(c)
case f: FieldLike => hashField(f)
case d: Def => hashDef(d)
case t: TypeDeclaration => hashTypeDeclaration(t)
case t: TypeAlias => hashTypeAlias(t)
}
}
final def hashClass(c: ClassLike): Unit = visit(visitedClassLike, c)(hashClass0)
def hashClass0(c: ClassLike)
{
extend(ClassHash)
hashParameterizedDefinition(c)
hashType(c.selfType)
hashStructure(c.structure)
}
def hashField(f: FieldLike)
{
f match
{
case v: Var => extend(VarHash)
case v: Val => extend(ValHash)
}
hashType(f.tpe)
}
def hashDef(d: Def)
{
extend(DefHash)
hashParameterizedDefinition(d)
hashValueParameters(d.valueParameters)
hashType(d.returnType)
}
def hashAccess(a: Access): Unit =
a match
{
case pub: Public => extend(PublicHash)
case qual: Qualified => hashQualified(qual)
}
def hashQualified(qual: Qualified): Unit =
{
qual match
{
case p: Protected => extend(ProtectedHash)
case p: Private => extend(PrivateHash)
}
hashQualifier(qual.qualifier)
}
def hashQualifier(qual: Qualifier): Unit =
qual match
{
case _: Unqualified => extend(UnqualifiedHash)
case _: ThisQualifier => extend(ThisQualifierHash)
case id: IdQualifier =>
extend(IdQualifierHash)
hashString(id.value)
}
def hashValueParameters(valueParameters: Seq[ParameterList]) = hashSeq(valueParameters, hashValueParameterList)
def hashValueParameterList(list: ParameterList) =
{
extend(ValueParamsHash)
hashBoolean(list.isImplicit)
hashSeq(list.parameters, hashValueParameter)
}
def hashValueParameter(parameter: MethodParameter) =
{
hashString(parameter.name)
hashType(parameter.tpe)
extend(parameter.modifier.ordinal)
hashBoolean(parameter.hasDefault)
}
def hashParameterizedDefinition[T <: ParameterizedDefinition](d: T)
{
hashTypeParameters(d.typeParameters)
}
def hashTypeDeclaration(d: TypeDeclaration)
{
extend(TypeDeclHash)
hashParameterizedDefinition(d)
hashType(d.lowerBound)
hashType(d.upperBound)
}
def hashTypeAlias(d: TypeAlias)
{
extend(TypeAliasHash)
hashParameterizedDefinition(d)
hashType(d.tpe)
}
def hashTypeParameters(parameters: Seq[TypeParameter]) = hashSeq(parameters, hashTypeParameter)
def hashTypeParameter(parameter: TypeParameter)
{
extend(parameter.variance.ordinal)
hashTypeParameters(parameter.typeParameters)
hashType(parameter.lowerBound)
hashType(parameter.upperBound)
hashAnnotations(parameter.annotations)
}
def hashAnnotations(annotations: Seq[Annotation]) = hashSeq(annotations, hashAnnotation)
def hashAnnotation(annotation: Annotation) =
{
hashType(annotation.base)
hashAnnotationArguments(annotation.arguments)
}
def hashAnnotationArguments(args: Seq[AnnotationArgument]) = hashSeq(args, hashAnnotationArgument)
def hashAnnotationArgument(arg: AnnotationArgument)
{
hashString(arg.name)
hashString(arg.value)
}
def hashTypes(ts: Seq[Type]) = hashSeq(ts, hashType)
def hashType(t: Type)
{
t match
{
case s: Structure => hashStructure(s)
case e: Existential => hashExistential(e)
case c: Constant => hashConstant(c)
case p: Polymorphic => hashPolymorphic(p)
case a: Annotated => hashAnnotated(a)
case p: Parameterized => hashParameterized(p)
case p: Projection => hashProjection(p)
case _: EmptyType => extend(EmptyTypeHash)
case s: Singleton => hashSingleton(s)
case pr: ParameterRef => hashParameterRef(pr)
}
}
def hashParameterRef(p: ParameterRef)
{
extend(ParameterRefHash)
tags.get(p.id) match {
case Some((a,b)) => extend(a); extend(b)
case None => extend(-1)
}
}
def hashSingleton(s: Singleton)
{
extend(SingletonHash)
hashPath(s.path)
}
def hashPath(path: Path) = hashSeq(path.components, hashPathComponent)
def hashPathComponent(pc: PathComponent) = pc match
{
case _: This => extend(ThisPathHash)
case s: Super => hashSuperPath(s)
case id: Id => hashIdPath(id)
}
def hashSuperPath(s: Super)
{
extend(SuperHash)
hashPath(s.qualifier)
}
def hashIdPath(id: Id)
{
extend(IdPathHash)
hashString(id.id)
}
def hashConstant(c: Constant) =
{
extend(ConstantHash)
hashString(c.value)
hashType(c.baseType)
}
def hashExistential(e: Existential) =
{
extend(ExistentialHash)
hashParameters(e.clause, e.baseType)
}
def hashPolymorphic(p: Polymorphic) =
{
extend(PolymorphicHash)
hashParameters(p.parameters, p.baseType)
}
def hashProjection(p: Projection) =
{
extend(ProjectionHash)
hashString(p.id)
hashType(p.prefix)
}
def hashParameterized(p: Parameterized)
{
extend(ParameterizedHash)
hashType(p.baseType)
hashTypes(p.typeArguments)
}
def hashAnnotated(a: Annotated)
{
extend(AnnotatedHash)
hashType(a.baseType)
hashAnnotations(a.annotations)
}
final def hashStructure(structure: Structure) = visit(visitedStructures, structure)(hashStructure0)
def hashStructure0(structure: Structure)
{
extend(StructureHash)
hashTypes(structure.parents)
hashDefinitions(structure.declared, false)
hashDefinitions(structure.inherited, false)
}
def hashParameters(parameters: Seq[TypeParameter], base: Type): Unit =
{
hashTypeParameters(parameters)
hashType(base)
}
}

View File

@ -43,7 +43,8 @@ object TopLevel
/** Checks the API of two source files for equality.*/
object SameAPI
{
def apply(a: SourceAPI, b: SourceAPI) =
def apply(a: Source, b: Source): Boolean = a.apiHash == b.apiHash && (a.hash.length > 0 && b.hash.length > 0) && apply(a.api, b.api)
def apply(a: SourceAPI, b: SourceAPI): Boolean =
{
val start = System.currentTimeMillis
@ -82,6 +83,19 @@ object SameAPI
map = map.updated(name, d :: map.getOrElse(name, Nil) )
map
}
/** Removes definitions that should not be considered for API equality.
* All top-level definitions are always considered: 'private' only means package-private.
* Other definitions are considered if they are not qualified with 'private[this]' or 'private'.*/
def filterDefinitions(d: Seq[Definition], topLevel: Boolean, includePrivate: Boolean) = if(topLevel || includePrivate) d else d.filter(isNonPrivate)
def isNonPrivate(d: Definition): Boolean = isNonPrivate(d.access)
/** Returns false if the `access` is `Private` and qualified, true otherwise.*/
def isNonPrivate(access: Access): Boolean =
access match
{
case p: Private if !p.qualifier.isInstanceOf[IdQualifier] => false
case _ => true
}
}
/** Used to implement API equality. All comparisons must be done between constructs in source files `a` and `b`. For example, when doing:
* `sameDefinitions(as, bs)`, `as` must be definitions from source file `a` and `bs` must be definitions from source file `b`. This is in order
@ -117,8 +131,8 @@ class SameAPI(tagsA: TypeVars, tagsB: TypeVars, includePrivate: Boolean, include
sameDefinitions(a.definitions, b.definitions, true)
def sameDefinitions(a: Seq[Definition], b: Seq[Definition], topLevel: Boolean): Boolean =
{
val (avalues, atypes) = separateDefinitions(filterDefinitions(a, topLevel))
val (bvalues, btypes) = separateDefinitions(filterDefinitions(b, topLevel))
val (avalues, atypes) = separateDefinitions(filterDefinitions(a, topLevel, includePrivate))
val (bvalues, btypes) = separateDefinitions(filterDefinitions(b, topLevel, includePrivate))
debug(sameDefinitions(byName(avalues), byName(bvalues)), "Value definitions differed") &&
debug(sameDefinitions(byName(atypes), byName(btypes)), "Type definitions differed")
}
@ -126,19 +140,6 @@ class SameAPI(tagsA: TypeVars, tagsB: TypeVars, includePrivate: Boolean, include
debug(sameStrings(a.keySet, b.keySet), "\tDefinition strings differed (a: " + (a.keySet -- b.keySet) + ", b: " + (b.keySet -- a.keySet) + ")") &&
zippedEntries(a,b).forall(tupled(sameNamedDefinitions))
/** Removes definitions that should not be considered for API equality.
* All top-level definitions are always considered: 'private' only means package-private.
* Other definitions are considered if they are not qualified with 'private[this]' or 'private'.*/
def filterDefinitions(d: Seq[Definition], topLevel: Boolean) = if(topLevel || includePrivate) d else d.filter(isNonPrivate)
def isNonPrivate(d: Definition): Boolean = isNonPrivate(d.access)
/** Returns false if the `access` is `Private` and qualified, true otherwise.*/
def isNonPrivate(access: Access): Boolean =
access match
{
case p: Private if !p.qualifier.isInstanceOf[IdQualifier] => false
case _ => true
}
/** Checks that the definitions in `a` are the same as those in `b`, ignoring order.
* Each list is assumed to have already been checked to have the same names (by `sameDefinitions`, for example).*/
def sameNamedDefinitions(a: List[Definition], b: List[Definition]): Boolean =

View File

@ -1,3 +0,0 @@
Simple Build Tool: Discovery Component
Copyright 2010 Mark Harrah
Licensed under BSD-style license (see LICENSE)

View File

@ -38,7 +38,7 @@ object APIs
val emptyAPI = new xsbti.api.SourceAPI(Array(), Array())
val emptyCompilation = new xsbti.api.Compilation(-1, "")
val emptySource = new xsbti.api.Source(emptyCompilation, Array(), emptyAPI)
val emptySource = new xsbti.api.Source(emptyCompilation, Array(), emptyAPI, 0)
def getAPI[T](map: Map[T, Source], src: T): Source = map.getOrElse(src, emptySource)
}

View File

@ -43,7 +43,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
import collection.mutable.{HashMap, HashSet, ListBuffer, Map, Set}
private val apis = new HashMap[File, SourceAPI]
private val apis = new HashMap[File, (Int, SourceAPI)]
private val binaryDeps = new HashMap[File, Set[File]]
private val classes = new HashMap[File, Set[(File, String)]]
private val sourceDeps = new HashMap[File, Set[File]]
@ -81,7 +81,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
def generatedClass(source: File, module: File, name: String) = add(classes, source, (module, name))
def api(sourceFile: File, source: SourceAPI) { apis(sourceFile) = source }
def api(sourceFile: File, source: SourceAPI) { apis(sourceFile) = (xsbt.api.HashAPI(source), xsbt.api.APIUtil.minimize(source)) }
def endSource(sourcePath: File): Unit =
assert(apis.contains(sourcePath))
@ -92,7 +92,7 @@ private final class AnalysisCallback(internalMap: File => Option[File], external
(base /: apis) { case (a, (src, api) ) =>
val stamp = current.internalSource(src)
val hash = stamp match { case h: Hash => h.value; case _ => new Array[Byte](0) }
val s = new xsbti.api.Source(compilation, hash, api)
val s = new xsbti.api.Source(compilation, hash, api._2, api._1)
a.addSource(src, s, stamp, sourceDeps.getOrElse(src, Nil: Iterable[File]))
}
def addExternals(base: Analysis): Analysis = (base /: extSrcDeps) { case (a, (source, name, api)) => a.addExternalDep(source, name, api) }

View File

@ -59,7 +59,7 @@ object Incremental
new APIChanges(modifiedAPIs, changedNames)
}
def sameSource(a: Source, b: Source): Boolean = shortcutSameSource(a, b) || SameAPI(a.api, b.api)
def sameSource(a: Source, b: Source): Boolean = shortcutSameSource(a, b) || SameAPI(a,b)
def shortcutSameSource(a: Source, b: Source): Boolean = !a.hash.isEmpty && !b.hash.isEmpty && sameCompilation(a.compilation, b.compilation) && (a.hash deepEquals b.hash)
def sameCompilation(a: Compilation, b: Compilation): Boolean = a.startTime == b.startTime && a.target == b.target

View File

@ -1,9 +1,10 @@
package sbt
package compiler
package xsbt.api
import java.io.File
import java.net.URLClassLoader
import org.specs.Specification
import sbt.WithFiles
import sbt.compiler.{CallbackTest,TestCompile}
/** Verifies that the analyzer plugin properly detects main methods. The main method must be
* public with the right signature and be defined on a public, top-level module.*/

View File

@ -1,6 +1,7 @@
package sbt
package compiler
package xsbt.api
import sbt.WithFiles
import sbt.compiler.{CallbackTest,TestCompile}
import java.io.File
import org.specs.Specification

View File

@ -1,6 +1,7 @@
package sbt
package compiler
package xsbt.api
import sbt.WithFiles
import sbt.compiler.{CallbackTest,TestCompile}
import java.io.File
import org.specs.Specification

View File

@ -2,6 +2,7 @@ Source
compilation: Compilation
hash: Byte*
api: SourceAPI
apiHash: Int
SourceAPI
packages : Package*

View File

@ -5,7 +5,7 @@ package sbt
import Build.data
import Scope.{fillTaskAxis, GlobalScope, ThisScope}
import compiler.Discovery
import xsbt.api.Discovery
import Project.{inConfig, Initialize, inScope, inTask, ScopedKey, Setting, SettingsDefinition}
import Artifact.{DocClassifier, SourceClassifier}
import Configurations.{Compile, CompilerPlugin, IntegrationTest, names, Provided, Runtime, Test}

View File

@ -5,7 +5,8 @@ package sbt
import java.io.File
import java.net.URI
import compiler.{Discovered,Discovery,Eval,EvalImports}
import compiler.{Eval,EvalImports}
import xsbt.api.{Discovered,Discovery}
import classpath.ClasspathUtilities
import scala.annotation.tailrec
import collection.mutable

View File

@ -4,7 +4,7 @@
package sbt
import std._
import compiler.{Discovered,Discovery}
import xsbt.api.{Discovered,Discovery}
import inc.Analysis
import TaskExtra._
import Types._

View File

@ -44,7 +44,8 @@ object Sbt extends Build
lazy val interfaceSub = project(file("interface"), "Interface") settings(interfaceSettings : _*)
// defines operations on the API of a source, including determining whether it has changed and converting it to a string
lazy val apiSub = baseProject(compilePath / "api", "API") dependsOn(interfaceSub)
// and discovery of subclasses and annotations
lazy val apiSub = testedBaseProject(compilePath / "api", "API") dependsOn(interfaceSub)
/***** Utilities *****/
@ -97,10 +98,7 @@ object Sbt extends Build
lazy val compilePersistSub = baseProject(compilePath / "persist", "Persist") dependsOn(compileIncrementalSub, apiSub) settings(sbinary)
// sbt-side interface to compiler. Calls compiler-side interface reflectively
lazy val compilerSub = testedBaseProject(compilePath, "Compile") dependsOn(launchInterfaceSub, interfaceSub % "compile;test->test", ivySub, ioSub, classpathSub,
logSub % "test->test", launchSub % "test->test", apiSub % "test->test") settings( compilerSettings : _*)
// Searches the source API data structures, currently looks for subclasses and annotations
lazy val discoverySub = testedBaseProject(compilePath / "discover", "Discovery") dependsOn(compileIncrementalSub, apiSub, compilerSub % "test->test")
logSub % "test->test", launchSub % "test->test", apiSub % "test") settings( compilerSettings : _*)
lazy val scriptedBaseSub = baseProject(scriptedPath / "base", "Scripted Framework") dependsOn(ioSub, processSub)
lazy val scriptedSbtSub = baseProject(scriptedPath / "sbt", "Scripted sbt") dependsOn(ioSub, logSub, processSub, scriptedBaseSub, launchInterfaceSub % "provided")
@ -109,7 +107,7 @@ object Sbt extends Build
// Implementation and support code for defining actions.
lazy val actionsSub = baseProject(mainPath / "actions", "Actions") dependsOn(
classfileSub, classpathSub, compileIncrementalSub, compilePersistSub, compilerSub, completeSub, discoverySub,
classfileSub, classpathSub, compileIncrementalSub, compilePersistSub, compilerSub, completeSub, apiSub,
interfaceSub, ioSub, ivySub, logSub, processSub, runSub, stdTaskSub, taskSub, trackingSub, testingSub)
// The main integration project for sbt. It brings all of the subsystems together, configures them, and provides for overriding conventions.