Merge api-diff into 0.13

compile/inc/src/main/scala/sbt/inc/APIDiff.scala

@@ -0,0 +1,67 @@
+package sbt.inc
+
+import xsbti.api.SourceAPI
+import xsbt.api.ShowAPI
+import xsbt.api.DefaultShowAPI._
+import java.lang.reflect.Method
+import java.util.{List => JList}
+
+/**
+ * A class which computes diffs (unified diffs) between two textual representations of an API.
+ *
+ * Internally, it uses java-diff-utils library but it calls it through reflection so there's
+ * no hard dependency on java-diff-utils.
+ *
+ * The reflective lookup of java-diff-utils library is performed in the constructor. Exceptions
+ * thrown by reflection are passed as-is to the caller of the constructor.
+ *
+ * @throws ClassNotFoundException if difflib.DiffUtils class cannot be located
+ * @throws LinkageError
+ * @throws ExceptionInInitializerError
+ */
+class APIDiff {
+
+	import APIDiff._
+
+	private val diffUtilsClass = Class.forName(diffUtilsClassName)
+	// method signature: diff(List<?>, List<?>)
+	private val diffMethod: Method =
+		diffUtilsClass.getMethod(diffMethodName, classOf[JList[_]], classOf[JList[_]])
+
+	private val generateUnifiedDiffMethod: Method = {
+		val patchClass = Class.forName(patchClassName)
+		// method signature: generateUnifiedDiff(String, String, List<String>, Patch, int)
+		diffUtilsClass.getMethod(generateUnifiedDiffMethodName, classOf[String],
+				classOf[String], classOf[JList[String]], patchClass, classOf[Int])
+	}
+
+	/**
+	 * Generates an unified diff between textual representations of `api1` and `api2`.
+	 */
+	def generateApiDiff(fileName: String, api1: SourceAPI, api2: SourceAPI, contextSize: Int): String = {
+		val api1Str = ShowAPI.show(api1)
+		val api2Str = ShowAPI.show(api2)
+		generateApiDiff(fileName, api1Str, api2Str, contextSize)
+	}
+
+	private def generateApiDiff(fileName: String, f1: String, f2: String, contextSize: Int): String = {
+		assert((diffMethod != null) && (generateUnifiedDiffMethod != null), "APIDiff isn't properly initialized.")
+		import scala.collection.JavaConverters._
+		def asJavaList[T](it: Iterator[T]): java.util.List[T] = it.toSeq.asJava
+		val f1Lines = asJavaList(f1.lines)
+		val f2Lines = asJavaList(f2.lines)
+		//val diff = DiffUtils.diff(f1Lines, f2Lines)
+		val diff /*: Patch*/ = diffMethod.invoke(null, f1Lines, f2Lines)
+		val unifiedPatch: JList[String] = generateUnifiedDiffMethod.invoke(null, fileName, fileName, f1Lines, diff,
+				(contextSize: java.lang.Integer)).asInstanceOf[JList[String]]
+		unifiedPatch.asScala.mkString("\n")
+	}
+
+}
+
+object APIDiff {
+	private val diffUtilsClassName = "difflib.DiffUtils"
+	private val patchClassName = "difflib.Patch"
+	private val diffMethodName = "diff"
+	private val generateUnifiedDiffMethodName = "generateUnifiedDiff"
+}

compile/inc/src/main/scala/sbt/inc/IncOptions.scala

@@ -23,9 +23,17 @@ final case class IncOptions(
 	* Enable tools for debugging API changes. At the moment this option is unused but in the
 	* future it will enable for example:
 	*   - disabling API hashing and API minimization (potentially very memory consuming)
-	*   - dumping textual API representation into files
+	*   - diffing textual API representation which helps understanding what kind of changes
+	*     to APIs are visible to the incremental compiler
 	*/
 	apiDebug: Boolean,
+	/**
+	 * Controls context size (in lines) displayed when diffs are produced for textual API
+	 * representation.
+	 *
+	 * This option is used only when `apiDebug == true`.
+	 */
+	apiDiffContextSize: Int,
 	/**
 	* The directory where we dump textual representation of APIs. This method might be called
 	* only if apiDebug returns true. This is unused option at the moment as the needed functionality
@@ -45,6 +53,7 @@ object IncOptions {
 		recompileAllFraction = 0.5,
 		relationsDebug = false,
 		apiDebug = false,
+		apiDiffContextSize = 5,
 		apiDumpDirectory = None,
 		newClassfileManager = ClassfileManager.deleteImmediately
 	)
@@ -57,6 +66,7 @@ object IncOptions {
 	val relationsDebugKey       = "relationsDebug"
 	val apiDebugKey             = "apiDebug"
 	val apiDumpDirectoryKey     = "apiDumpDirectory"
+	val apiDiffContextSize      = "apiDiffContextSize"
 
 	def fromStringMap(m: java.util.Map[String, String]): IncOptions = {
 		// all the code below doesn't look like idiomatic Scala for a good reason: we are working with Java API
@@ -76,6 +86,10 @@ object IncOptions {
 			val k = apiDebugKey
 			if (m.containsKey(k)) m.get(k).toBoolean else Default.apiDebug
 		}
+		def getApiDiffContextSize: Int = {
+			val k = apiDiffContextSize
+			if (m.containsKey(k)) m.get(k).toInt else Default.apiDiffContextSize
+		}
 		def getApiDumpDirectory: Option[java.io.File] = {
 			val k = apiDumpDirectoryKey
 			if (m.containsKey(k))
@@ -83,7 +97,8 @@ object IncOptions {
 			else None
 		}
 
-		IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDumpDirectory, ClassfileManager.deleteImmediately)
+		IncOptions(getTransitiveStep, getRecompileAllFraction, getRelationsDebug, getApiDebug, getApiDiffContextSize,
+			getApiDumpDirectory, ClassfileManager.deleteImmediately)
 	}
 
 	def toStringMap(o: IncOptions): java.util.Map[String, String] = {
@@ -93,6 +108,7 @@ object IncOptions {
 		m.put(relationsDebugKey, o.relationsDebug.toString)
 		m.put(apiDebugKey, o.apiDebug.toString)
 		o.apiDumpDirectory.foreach(f => m.put(apiDumpDirectoryKey, f.toString))
+		m.put(apiDiffContextSize, o.apiDiffContextSize.toString)
 		m
 	}
 }

compile/inc/src/main/scala/sbt/inc/Incremental.scala

@@ -76,7 +76,7 @@ object Incremental
 			val merged = pruned ++ fresh//.copy(relations = pruned.relations ++ fresh.relations, apis = pruned.apis ++ fresh.apis)
 			debug("********* Merged: \n" + merged.relations + "\n*********")
 
-			val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, options)
+			val incChanges = changedIncremental(invalidated, previous.apis.internalAPI _, merged.apis.internalAPI _, log, options)
 			debug("\nChanges:\n" + incChanges)
 			val transitiveStep = options.transitiveStep
 			val incInv = invalidateIncremental(merged.relations, incChanges, invalidated, cycleNum >= transitiveStep, log)
@@ -101,17 +101,48 @@ object Incremental
 	private[this] def invalidatedPackageObjects(invalidated: Set[File], relations: Relations): Set[File] =
 		invalidated flatMap relations.publicInherited.internal.reverse filter { _.getName == "package.scala" }
 
+	/**
+	 * Logs API changes using debug-level logging. The API are obtained using the APIDiff class.
+	 *
+	 * NOTE: This method creates a new APIDiff instance on every invocation.
+	 */
+	private def logApiChanges[T](changes: (collection.Set[T], Seq[Source], Seq[Source]), log: Logger,
+			options: IncOptions): Unit = {
+		val contextSize = options.apiDiffContextSize
+		try {
+			val apiDiff = new APIDiff
+			changes.zipped foreach {
+				case (src, oldApi, newApi) =>
+					val apiUnifiedPatch = apiDiff.generateApiDiff(src.toString, oldApi.api, newApi.api, contextSize)
+					log.debug("Detected a change in a public API:\n" + apiUnifiedPatch)
+			}
+		} catch {
+			case e: ClassNotFoundException =>
+				log.error("You have api debugging enabled but DiffUtils library cannot be found on sbt's classpath")
+			case e: LinkageError =>
+				log.error("Encoutared linkage error while trying to load DiffUtils library.")
+				log.trace(e)
+			case e: Exception =>
+				log.error("An exception has been thrown while trying to dump an api diff.")
+				log.trace(e)
+		}
+	}
+
 	/**
 	* Accepts the sources that were recompiled during the last step and functions
 	* providing the API before and after the last step.  The functions should return
 	* an empty API if the file did not/does not exist.
 	*/
-	def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source, options: IncOptions): APIChanges[T] =
+	def changedIncremental[T](lastSources: collection.Set[T], oldAPI: T => Source, newAPI: T => Source, log: Logger, options: IncOptions): APIChanges[T] =
 	{
 		val oldApis = lastSources.toSeq map oldAPI
 		val newApis = lastSources.toSeq map newAPI
 		val changes = (lastSources, oldApis, newApis).zipped.filter { (src, oldApi, newApi) => !sameSource(oldApi, newApi) }
 
+		if (apiDebug(options) && changes.zipped.nonEmpty) {
+			logApiChanges(changes, log, options)
+		}
+
 		val changedNames = TopLevel.nameChanges(changes._3, changes._2 )
 
 		val modifiedAPIs = changes._1.toSet
@@ -138,7 +169,7 @@ object Incremental
 		val srcChanges = changes(previous.allInternalSources.toSet, sources,  f => !equivS.equiv( previous.internalSource(f), current.internalSource(f) ) )
 		val removedProducts = previous.allProducts.filter( p => !equivS.equiv( previous.product(p), current.product(p) ) ).toSet
 		val binaryDepChanges = previous.allBinaries.filter( externalBinaryModified(entry, forEntry, previous, current, log)).toSet
-		val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry), options)
+		val extChanges = changedIncremental(previousAPIs.allExternals, previousAPIs.externalAPI _, currentExternalAPI(entry, forEntry), log, options)
 
 		InitialChanges(srcChanges, removedProducts, binaryDepChanges, extChanges )
 	}

launch/src/main/input_resources/sbt/sbt.boot.properties

@@ -8,6 +8,7 @@
   class: ${sbt.main.class-sbt.xMain}
   components: xsbti,extra
   cross-versioned: ${sbt.cross.versioned-false}
+  resources: ${sbt.extraClasspath-}
 
 [repositories]
   local

launch/src/main/scala/xsbt/boot/ConfigurationParser.scala

@@ -18,7 +18,7 @@ object ConfigurationParser
 	def trim(s: Array[String]) = s.map(_.trim).toList
 	def ids(value: String) = trim(substituteVariables(value).split(",")).filter(isNonEmpty)
 
-	private[this] lazy val VarPattern = Pattern.compile("""\$\{([\w.]+)(\-(.+))?\}""")
+	private[this] lazy val VarPattern = Pattern.compile("""\$\{([\w.]+)(\-(.*))?\}""")
 	def substituteVariables(s: String): String = if(s.indexOf('$') >= 0) substituteVariables0(s) else s
 	// scala.util.Regex brought in 30kB, so we code it explicitly
 	def substituteVariables0(s: String): String =