From 30cdb1a1bb4e62c74ac64374e31cb5228768a6c8 Mon Sep 17 00:00:00 2001 From: Benjy Date: Sun, 13 Oct 2013 21:48:02 -0700 Subject: [PATCH] Add correct implementations of merge/groupBy to Analysis. Copious comments to explain the non-trivial logic. --- .../inc/src/main/scala/sbt/inc/Analysis.scala | 185 +++++++++++++++--- 1 file changed, 157 insertions(+), 28 deletions(-) diff --git a/compile/inc/src/main/scala/sbt/inc/Analysis.scala b/compile/inc/src/main/scala/sbt/inc/Analysis.scala index 192d931a1..f5e2ec735 100644 --- a/compile/inc/src/main/scala/sbt/inc/Analysis.scala +++ b/compile/inc/src/main/scala/sbt/inc/Analysis.scala @@ -7,6 +7,22 @@ package inc import xsbti.api.Source import java.io.File + +/** + * The merge/groupBy functionality requires understanding of the concepts of internalizing/externalizing dependencies: + * + * Say we have source files X, Y. And say we have some analysis A_X containing X as a source, and likewise for A_Y and Y. + * If X depends on Y then A_X contains an external dependency X -> Y. + * + * However if we merge A_X and A_Y into a combined analysis A_XY, then A_XY contains X and Y as sources, and therefore + * X -> Y must be converted to an internal dependency in A_XY. We refer to this as "internalizing" the dependency. + * + * The reverse transformation must occur if we group an analysis A_XY into A_X and A_Y, so that the dependency X->Y + * crosses the boundary. We refer to this as "externalizing" the dependency. + * + * These transformations are complicated by the fact that internal dependencies are expressed as source file -> source file, + * but external dependencies are expressed as source file -> fully-qualified class name. + */ trait Analysis { val stamps: Stamps @@ -16,9 +32,13 @@ trait Analysis val infos: SourceInfos /** Information about compiler runs accumulated since `clean` command has been run. */ val compilations: Compilations - - def ++(other: Analysis): Analysis + + /** Concatenates Analysis objects naively, i.e., doesn't internalize external deps on added files. See `Analysis.merge`. */ + def ++ (other: Analysis): Analysis + + /** Drops all analysis information for `sources` naively, i.e., doesn't externalize internal deps on removed files. */ def -- (sources: Iterable[File]): Analysis + def copy(stamps: Stamps = stamps, apis: APIs = apis, relations: Relations = relations, infos: SourceInfos = infos, compilations: Compilations = compilations): Analysis @@ -27,7 +47,8 @@ trait Analysis def addExternalDep(src: File, dep: String, api: Source, inherited: Boolean): Analysis def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis - def groupBy[K](f: (File => K)): Map[K, Analysis] + /** Partitions this Analysis using the discriminator function. Externalizes internal deps that cross partitions. */ + def groupBy[K](discriminator: (File => K)): Map[K, Analysis] override lazy val toString = Analysis.summary(this) } @@ -35,6 +56,48 @@ trait Analysis object Analysis { lazy val Empty: Analysis = new MAnalysis(Stamps.empty, APIs.empty, Relations.empty, SourceInfos.empty, Compilations.empty) + + /** Merge multiple analysis objects into one. Deps will be internalized as needed. */ + def merge(analyses: Traversable[Analysis]): Analysis = { + // Merge the Relations, internalizing deps as needed. + val mergedSrcProd = Relation.merge(analyses map { _.relations.srcProd }) + val mergedBinaryDep = Relation.merge(analyses map { _.relations.binaryDep }) + val mergedClasses = Relation.merge(analyses map { _.relations.classes }) + + val stillInternal = Relation.merge(analyses map { _.relations.direct.internal }) + val (internalized, stillExternal) = Relation.merge(analyses map { _.relations.direct.external }) partition { case (a, b) => mergedClasses._2s.contains(b) } + val internalizedFiles = Relation.reconstruct(internalized.forwardMap mapValues { _ flatMap mergedClasses.reverse }) + val mergedInternal = stillInternal ++ internalizedFiles + + val stillInternalPI = Relation.merge(analyses map { _.relations.publicInherited.internal }) + val (internalizedPI, stillExternalPI) = Relation.merge(analyses map { _.relations.publicInherited.external }) partition { case (a, b) => mergedClasses._2s.contains(b) } + val internalizedFilesPI = Relation.reconstruct(internalizedPI.forwardMap mapValues { _ flatMap mergedClasses.reverse }) + val mergedInternalPI = stillInternalPI ++ internalizedFilesPI + + val mergedRelations = Relations.make( + mergedSrcProd, + mergedBinaryDep, + Relations.makeSource(mergedInternal, stillExternal), + Relations.makeSource(mergedInternalPI, stillExternalPI), + mergedClasses + ) + + // Merge the APIs, internalizing APIs for targets of dependencies we internalized above. + val concatenatedAPIs = (APIs.empty /: (analyses map {_.apis}))(_ ++ _) + val stillInternalAPIs = concatenatedAPIs.internal + val (internalizedAPIs, stillExternalAPIs) = concatenatedAPIs.external partition { x: (String, Source) => internalized._2s.contains(x._1) } + val internalizedFilesAPIs = internalizedAPIs flatMap { + case (cls: String, source: Source) => mergedRelations.definesClass(cls) map { file: File => (file, concatenatedAPIs.internalAPI(file)) } + } + val mergedAPIs = APIs(stillInternalAPIs ++ internalizedFilesAPIs, stillExternalAPIs) + + val mergedStamps = Stamps.merge(analyses map { _.stamps }) + val mergedInfos = SourceInfos.merge(analyses map { _.infos }) + val mergedCompilations = Compilations.merge(analyses map { _.compilations }) + + new MAnalysis(mergedStamps, mergedAPIs, mergedRelations, mergedInfos, mergedCompilations) + } + def summary(a: Analysis): String = { val (j, s) = a.apis.allInternalSources.partition(_.getName.endsWith(".java")) @@ -64,17 +127,19 @@ object Analysis private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relations, val infos: SourceInfos, val compilations: Compilations) extends Analysis { def ++ (o: Analysis): Analysis = new MAnalysis(stamps ++ o.stamps, apis ++ o.apis, relations ++ o.relations, - infos ++ o.infos, compilations ++ o.compilations) + infos ++ o.infos, compilations ++ o.compilations) + def -- (sources: Iterable[File]): Analysis = { val newRelations = relations -- sources - def keep[T](f: (Relations, T) => Set[_]): T => Boolean = keepFor(newRelations)(f) - + def keep[T](f: (Relations, T) => Set[_]): T => Boolean = !f(newRelations, _).isEmpty + val newAPIs = apis.removeInternal(sources).filterExt( keep(_ usesExternal _) ) val newStamps = stamps.filter( keep(_ produced _), sources, keep(_ usesBinary _)) val newInfos = infos -- sources new MAnalysis(newStamps, newAPIs, newRelations, newInfos, compilations) } + def copy(stamps: Stamps, apis: APIs, relations: Relations, infos: SourceInfos, compilations: Compilations = compilations): Analysis = new MAnalysis(stamps, apis, relations, infos, compilations) @@ -90,31 +155,95 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis = copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos ) - def groupBy[K](f: File => K): Map[K, Analysis] = - { - def outerJoin(stampsMap: Map[K, Stamps], apisMap: Map[K, APIs], relationsMap: Map[K, Relations], infosMap: Map[K, SourceInfos]): Map[K, Analysis] = - { - def kAnalysis(k: K): Analysis = - new MAnalysis( - stampsMap.getOrElse(k, Stamps.empty), - apisMap.getOrElse(k, APIs.empty), - relationsMap.getOrElse(k, Relations.empty), - infosMap.getOrElse(k, SourceInfos.empty), - compilations - ) - val keys = (stampsMap.keySet ++ apisMap.keySet ++ relationsMap.keySet ++ infosMap.keySet).toList - Map( keys map( (k: K) => (k, kAnalysis(k)) ) :_*) + def groupBy[K](discriminator: File => K): Map[K, Analysis] = { + def discriminator1(x: (File, _)) = discriminator(x._1) // Apply the discriminator to the first coordinate. + + val kSrcProd = relations.srcProd.groupBy(discriminator1) + val kBinaryDep = relations.binaryDep.groupBy(discriminator1) + val kClasses = relations.classes.groupBy(discriminator1) + val kSourceInfos = infos.allInfos.groupBy(discriminator1) + + val (kStillInternal, kExternalized) = relations.direct.internal partition { case (a, b) => discriminator(a) == discriminator(b) } match { + case (i, e) => (i.groupBy(discriminator1), e.groupBy(discriminator1)) } + val kStillExternal = relations.direct.external.groupBy(discriminator1) - val relationsMap: Map[K, Relations] = relations.groupBy(f) - def keepMap[T](f: (Relations, T) => Set[_]): Map[K, T => Boolean] = - relationsMap map { item => (item._1, keepFor(item._2)(f) ) } - val keepExternal: Map[K, String => Boolean] = keepMap(_ usesExternal _) - val keepProduced: Map[K, File => Boolean] = keepMap(_ produced _) - val keepBinary: Map[K, File => Boolean] = keepMap(_ usesBinary _) - outerJoin(stamps.groupBy(keepProduced, f, keepBinary), apis.groupBy(f, keepExternal), relationsMap, infos.groupBy(f)) + // Find all possible groups. + val allMaps = kSrcProd :: kBinaryDep :: kStillInternal :: kExternalized :: kStillExternal :: kClasses :: kSourceInfos :: Nil + val allKeys: Set[K] = (Set.empty[K] /: (allMaps map { _.keySet }))(_ ++ _) + + // Map from file to a single representative class defined in that file. + // This is correct (for now): currently all classes in an external dep share the same Source object, + // and a change to any of them will act like a change to all of them. + // We don't use all the top-level classes in source.api.definitions, even though that's more intuitively + // correct, because this can cause huge bloat of the analysis file. + def getRepresentativeClass(file: File): Option[String] = apis.internalAPI(file).api.definitions.headOption map { _.name } + + // Create an Analysis for each group. + (for (k <- allKeys) yield { + def getFrom[A, B](m: Map[K, Relation[A, B]]): Relation[A, B] = m.getOrElse(k, Relation.empty) + + // Products and binary deps. + val srcProd = getFrom(kSrcProd) + val binaryDep = getFrom(kBinaryDep) + + // Direct Sources. + val stillInternal = getFrom(kStillInternal) + val stillExternal = getFrom(kStillExternal) + val externalized = getFrom(kExternalized) + val externalizedClasses = Relation.reconstruct(externalized.forwardMap mapValues { _ flatMap getRepresentativeClass }) + val newExternal = stillExternal ++ externalizedClasses + + // Public inherited sources. + val stillInternalPI = stillInternal filter relations.publicInherited.internal.contains + val stillExternalPI = stillExternal filter relations.publicInherited.external.contains + val externalizedPI = externalized filter relations.publicInherited.internal.contains + val externalizedClassesPI = Relation.reconstruct(externalizedPI.forwardMap mapValues { _ flatMap getRepresentativeClass }) + val newExternalPI = stillExternalPI ++ externalizedClassesPI + + // Class names. + val classes = getFrom(kClasses) + + // Create new relations for this group. + val newRelations = Relations.make( + srcProd, + binaryDep, + Relations.makeSource(stillInternal, newExternal), + Relations.makeSource(stillInternalPI, newExternalPI), + classes + ) + + // Compute new API mappings. + def apisFor[T](m: Map[T, Source], x: Traversable[T]): Map[T, Source] = + (x map { e: T => (e, m.get(e)) } collect { case (t, Some(source)) => (t, source)}).toMap + val stillInternalAPIs = apisFor(apis.internal, srcProd._1s) + val stillExternalAPIs = apisFor(apis.external, stillExternal._2s) + val externalizedAPIs = apisFor(apis.internal, externalized._2s) + val externalizedClassesAPIs = externalizedAPIs flatMap { + case (file: File, source: Source) => getRepresentativeClass(file) map { cls: String => (cls, source) } + } + val newAPIs = APIs(stillInternalAPIs, stillExternalAPIs ++ externalizedClassesAPIs) + + // New stamps. + val newStamps = Stamps( + stamps.products.filterKeys(srcProd._2s.contains), + stamps.sources.filterKeys({ discriminator(_) == k }), + stamps.binaries.filterKeys(binaryDep._2s.contains), + stamps.classNames.filterKeys(binaryDep._2s.contains)) + + // New infos. + val newSourceInfos = SourceInfos.make(kSourceInfos.getOrElse(k, Map.empty)) + + (k, new MAnalysis(newStamps, newAPIs, newRelations, newSourceInfos, compilations)) + }).toMap } - private def keepFor[T](newRelations: Relations)(f: (Relations, T) => Set[_]): T => Boolean = file => !f(newRelations, file).isEmpty + override def equals(other: Any) = other match { + // Note: Equality doesn't consider source infos or compilations. + case o: MAnalysis => stamps == o.stamps && apis == o.apis && relations == o.relations + case _ => false + } + + override lazy val hashCode = (stamps :: apis :: relations :: Nil).hashCode }