Add correct implementations of merge/groupBy to Analysis.

Copious comments to explain the non-trivial logic.

compile/inc/src/main/scala/sbt/inc/Analysis.scala

@@ -7,6 +7,22 @@ package inc
 import xsbti.api.Source
 import java.io.File
 
+
+/**
+ * The merge/groupBy functionality requires understanding of the concepts of internalizing/externalizing dependencies:
+ *
+ * Say we have source files X, Y. And say we have some analysis A_X containing X as a source, and likewise for A_Y and Y.
+ * If X depends on Y then A_X contains an external dependency X -> Y.
+ *
+ * However if we merge A_X and A_Y into a combined analysis A_XY, then A_XY contains X and Y as sources, and therefore
+ * X -> Y must be converted to an internal dependency in A_XY. We refer to this as "internalizing" the dependency.
+ *
+ * The reverse transformation must occur if we group an analysis A_XY into A_X and A_Y, so that the dependency X->Y
+ * crosses the boundary. We refer to this as "externalizing" the dependency.
+ *
+ * These transformations are complicated by the fact that internal dependencies are expressed as source file -> source file,
+ * but external dependencies are expressed as source file -> fully-qualified class name.
+ */
 trait Analysis
 {
 	val stamps: Stamps
@@ -16,9 +32,13 @@ trait Analysis
 	val infos: SourceInfos
 	/** Information about compiler runs accumulated since `clean` command has been run. */
 	val compilations: Compilations
-	
+
-	def ++(other: Analysis): Analysis
+	/** Concatenates Analysis objects naively, i.e., doesn't internalize external deps on added files. See `Analysis.merge`. */
+	def ++ (other: Analysis): Analysis
+
+	/** Drops all analysis information for `sources` naively, i.e., doesn't externalize internal deps on removed files. */
 	def -- (sources: Iterable[File]): Analysis
+
 	def copy(stamps: Stamps = stamps, apis: APIs = apis, relations: Relations = relations, infos: SourceInfos = infos,
 	    compilations: Compilations = compilations): Analysis
 	
@@ -27,7 +47,8 @@ trait Analysis
 	def addExternalDep(src: File, dep: String, api: Source, inherited: Boolean): Analysis
 	def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis
 
-	def groupBy[K](f: (File => K)): Map[K, Analysis]
+	/** Partitions this Analysis using the discriminator function. Externalizes internal deps that cross partitions. */
+	def groupBy[K](discriminator: (File => K)): Map[K, Analysis]
 
 	override lazy val toString = Analysis.summary(this)
 }
@@ -35,6 +56,48 @@ trait Analysis
 object Analysis
 {
 	lazy val Empty: Analysis = new MAnalysis(Stamps.empty, APIs.empty, Relations.empty, SourceInfos.empty, Compilations.empty)
+
+	/** Merge multiple analysis objects into one. Deps will be internalized as needed. */
+	def merge(analyses: Traversable[Analysis]): Analysis = {
+		// Merge the Relations, internalizing deps as needed.
+		val mergedSrcProd = Relation.merge(analyses map { _.relations.srcProd })
+		val mergedBinaryDep = Relation.merge(analyses map { _.relations.binaryDep })
+		val mergedClasses = Relation.merge(analyses map { _.relations.classes })
+
+		val stillInternal = Relation.merge(analyses map { _.relations.direct.internal })
+		val (internalized, stillExternal) = Relation.merge(analyses map { _.relations.direct.external }) partition { case (a, b) => mergedClasses._2s.contains(b) }
+		val internalizedFiles = Relation.reconstruct(internalized.forwardMap mapValues { _ flatMap mergedClasses.reverse })
+		val mergedInternal = stillInternal ++ internalizedFiles
+
+		val stillInternalPI = Relation.merge(analyses map { _.relations.publicInherited.internal })
+		val (internalizedPI, stillExternalPI) = Relation.merge(analyses map { _.relations.publicInherited.external }) partition { case (a, b) => mergedClasses._2s.contains(b) }
+		val internalizedFilesPI = Relation.reconstruct(internalizedPI.forwardMap mapValues { _ flatMap mergedClasses.reverse })
+		val mergedInternalPI = stillInternalPI ++ internalizedFilesPI
+
+		val mergedRelations = Relations.make(
+			mergedSrcProd,
+			mergedBinaryDep,
+			Relations.makeSource(mergedInternal, stillExternal),
+			Relations.makeSource(mergedInternalPI, stillExternalPI),
+			mergedClasses
+		)
+
+		// Merge the APIs, internalizing APIs for targets of dependencies we internalized above.
+		val concatenatedAPIs = (APIs.empty /: (analyses map {_.apis}))(_ ++ _)
+		val stillInternalAPIs = concatenatedAPIs.internal
+		val (internalizedAPIs, stillExternalAPIs) = concatenatedAPIs.external partition { x: (String, Source) => internalized._2s.contains(x._1) }
+		val internalizedFilesAPIs = internalizedAPIs flatMap {
+			case (cls: String, source: Source) => mergedRelations.definesClass(cls) map { file: File => (file, concatenatedAPIs.internalAPI(file)) }
+		}
+		val mergedAPIs = APIs(stillInternalAPIs ++ internalizedFilesAPIs, stillExternalAPIs)
+
+		val mergedStamps = Stamps.merge(analyses map { _.stamps })
+		val mergedInfos = SourceInfos.merge(analyses map { _.infos })
+		val mergedCompilations = Compilations.merge(analyses map { _.compilations })
+
+		new MAnalysis(mergedStamps, mergedAPIs, mergedRelations, mergedInfos, mergedCompilations)
+	}
+
 	def summary(a: Analysis): String =
 	{
 		val (j, s) = a.apis.allInternalSources.partition(_.getName.endsWith(".java"))
@@ -64,17 +127,19 @@ object Analysis
 private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relations, val infos: SourceInfos, val compilations: Compilations) extends Analysis
 {
 	def ++ (o: Analysis): Analysis = new MAnalysis(stamps ++ o.stamps, apis ++ o.apis, relations ++ o.relations,
-	    infos ++ o.infos, compilations ++ o.compilations)
+		infos ++ o.infos, compilations ++ o.compilations)
+
 	def -- (sources: Iterable[File]): Analysis =
 	{
 		val newRelations = relations -- sources
-		def keep[T](f: (Relations, T) => Set[_]): T => Boolean = keepFor(newRelations)(f)
+		def keep[T](f: (Relations, T) => Set[_]): T => Boolean = !f(newRelations, _).isEmpty
-		
+
 		val newAPIs = apis.removeInternal(sources).filterExt( keep(_ usesExternal _) )
 		val newStamps = stamps.filter( keep(_ produced _), sources, keep(_ usesBinary _))
 		val newInfos = infos -- sources
 		new MAnalysis(newStamps, newAPIs, newRelations, newInfos, compilations)
 	}
+
 	def copy(stamps: Stamps, apis: APIs, relations: Relations, infos: SourceInfos, compilations: Compilations = compilations): Analysis =
 	  new MAnalysis(stamps, apis, relations, infos, compilations)
 
@@ -90,31 +155,95 @@ private class MAnalysis(val stamps: Stamps, val apis: APIs, val relations: Relat
 	def addProduct(src: File, product: File, stamp: Stamp, name: String): Analysis =
 		copy( stamps.markProduct(product, stamp), apis, relations.addProduct(src, product, name), infos )
 
-	def groupBy[K](f: File => K): Map[K, Analysis] =
+	def groupBy[K](discriminator: File => K): Map[K, Analysis] = {
-	{
+		def discriminator1(x: (File, _)) = discriminator(x._1)  // Apply the discriminator to the first coordinate.
-		def outerJoin(stampsMap: Map[K, Stamps], apisMap: Map[K, APIs], relationsMap: Map[K, Relations], infosMap: Map[K, SourceInfos]): Map[K, Analysis] =
+
-		{
+		val kSrcProd = relations.srcProd.groupBy(discriminator1)
-			def kAnalysis(k: K): Analysis =
+		val kBinaryDep = relations.binaryDep.groupBy(discriminator1)
-				new MAnalysis(
+		val kClasses = relations.classes.groupBy(discriminator1)
-					stampsMap.getOrElse(k, Stamps.empty),
+		val kSourceInfos = infos.allInfos.groupBy(discriminator1)
-					apisMap.getOrElse(k, APIs.empty),
+
-					relationsMap.getOrElse(k, Relations.empty),
+		val (kStillInternal, kExternalized) = relations.direct.internal partition { case (a, b) => discriminator(a) == discriminator(b) } match {
-					infosMap.getOrElse(k, SourceInfos.empty),
+			case (i, e) => (i.groupBy(discriminator1), e.groupBy(discriminator1))
-					compilations
-				)
-			val keys = (stampsMap.keySet ++ apisMap.keySet ++ relationsMap.keySet ++ infosMap.keySet).toList
-			Map( keys map( (k: K) => (k, kAnalysis(k)) ) :_*)
 		}
+		val kStillExternal = relations.direct.external.groupBy(discriminator1)
 
-		val relationsMap: Map[K, Relations] = relations.groupBy(f)
-		def keepMap[T](f: (Relations, T) => Set[_]): Map[K, T => Boolean] =
-			relationsMap map { item => (item._1, keepFor(item._2)(f) ) }
 
-		val keepExternal: Map[K, String => Boolean] = keepMap(_ usesExternal _)
+		// Find all possible groups.
-		val keepProduced: Map[K, File => Boolean] = keepMap(_ produced _)
+		val allMaps = kSrcProd :: kBinaryDep :: kStillInternal :: kExternalized :: kStillExternal :: kClasses :: kSourceInfos :: Nil
-		val keepBinary: Map[K, File => Boolean] = keepMap(_ usesBinary _)
+		val allKeys: Set[K] = (Set.empty[K] /: (allMaps map { _.keySet }))(_ ++ _)
-		outerJoin(stamps.groupBy(keepProduced, f, keepBinary), apis.groupBy(f, keepExternal), relationsMap, infos.groupBy(f))
+
+		// Map from file to a single representative class defined in that file.
+		// This is correct (for now): currently all classes in an external dep share the same Source object,
+		// and a change to any of them will act like a change to all of them.
+		// We don't use all the top-level classes in source.api.definitions, even though that's more intuitively
+		// correct, because this can cause huge bloat of the analysis file.
+		def getRepresentativeClass(file: File): Option[String] = apis.internalAPI(file).api.definitions.headOption map { _.name }
+
+		// Create an Analysis for each group.
+		(for (k <- allKeys) yield {
+			def getFrom[A, B](m: Map[K, Relation[A, B]]): Relation[A, B] = m.getOrElse(k, Relation.empty)
+
+			// Products and binary deps.
+			val srcProd = getFrom(kSrcProd)
+			val binaryDep = getFrom(kBinaryDep)
+
+			// Direct Sources.
+			val stillInternal = getFrom(kStillInternal)
+			val stillExternal = getFrom(kStillExternal)
+			val externalized = getFrom(kExternalized)
+			val externalizedClasses = Relation.reconstruct(externalized.forwardMap mapValues { _ flatMap getRepresentativeClass })
+			val newExternal = stillExternal ++ externalizedClasses
+
+			// Public inherited sources.
+			val stillInternalPI = stillInternal filter relations.publicInherited.internal.contains
+			val stillExternalPI = stillExternal filter relations.publicInherited.external.contains
+			val externalizedPI = externalized filter relations.publicInherited.internal.contains
+			val externalizedClassesPI = Relation.reconstruct(externalizedPI.forwardMap mapValues { _ flatMap getRepresentativeClass })
+			val newExternalPI = stillExternalPI ++ externalizedClassesPI
+
+			// Class names.
+			val classes = getFrom(kClasses)
+
+			// Create new relations for this group.
+			val newRelations = Relations.make(
+				srcProd,
+				binaryDep,
+				Relations.makeSource(stillInternal, newExternal),
+			  Relations.makeSource(stillInternalPI, newExternalPI),
+				classes
+			)
+
+			// Compute new API mappings.
+			def apisFor[T](m: Map[T, Source], x: Traversable[T]): Map[T, Source] =
+				(x map { e: T => (e, m.get(e)) } collect { case (t, Some(source)) => (t, source)}).toMap
+			val stillInternalAPIs = apisFor(apis.internal, srcProd._1s)
+			val stillExternalAPIs = apisFor(apis.external, stillExternal._2s)
+			val externalizedAPIs = apisFor(apis.internal, externalized._2s)
+			val externalizedClassesAPIs = externalizedAPIs flatMap {
+				case (file: File, source: Source) => getRepresentativeClass(file) map { cls: String => (cls, source) }
+			}
+			val newAPIs = APIs(stillInternalAPIs, stillExternalAPIs ++ externalizedClassesAPIs)
+
+			// New stamps.
+			val newStamps = Stamps(
+				stamps.products.filterKeys(srcProd._2s.contains),
+				stamps.sources.filterKeys({ discriminator(_) == k }),
+				stamps.binaries.filterKeys(binaryDep._2s.contains),
+				stamps.classNames.filterKeys(binaryDep._2s.contains))
+
+			// New infos.
+			val newSourceInfos = SourceInfos.make(kSourceInfos.getOrElse(k, Map.empty))
+
+			(k, new MAnalysis(newStamps, newAPIs, newRelations, newSourceInfos, compilations))
+		}).toMap
 	}
 
-	private def keepFor[T](newRelations: Relations)(f: (Relations, T) => Set[_]): T => Boolean = file => !f(newRelations, file).isEmpty
+	override def equals(other: Any) = other match {
+		// Note: Equality doesn't consider source infos or compilations.
+		case o: MAnalysis => stamps == o.stamps && apis == o.apis && relations == o.relations
+		case _ => false
+	}
+
+	override lazy val hashCode = (stamps :: apis :: relations :: Nil).hashCode
 }