Use a machine-global lock on Ivy because caches are not safe for concurrent access

ivy/ComponentManager.scala

@@ -39,7 +39,9 @@ class ComponentManager(globalLock: xsbti.GlobalLock, provider: xsbti.ComponentPr
 
 		lockLocalCache { getOrElse(fromGlobal) }
 	}
+	/** This is used to lock the local cache in project/boot/.  By checking the local cache first, we can avoid grabbing a global lock. */
 	private def lockLocalCache[T](action: => T): T = lock(provider.lockFile)( action )
+	/** This is used to ensure atomic access to components in the global Ivy cache.*/
 	private def lockGlobalCache[T](action: => T): T = lock(IvyCache.lockFile)( action )
 	private def lock[T](file: File)(action: => T): T = globalLock(file, new Callable[T] { def call = action })
 	/** Get the file for component 'id', throwing an exception if no files or multiple files exist for the component. */
@@ -53,12 +55,12 @@ class ComponentManager(globalLock: xsbti.GlobalLock, provider: xsbti.ComponentPr
 
 	def define(id: String, files: Iterable[File]) = lockLocalCache { provider.defineComponent(id, files.toSeq.toArray) }
 	/** Retrieve the file for component 'id' from the local repository. */
-	private def update(id: String): Unit = IvyCache.withCachedJar(sbtModuleID(id), log)(jar => define(id, Seq(jar)) )
+	private def update(id: String): Unit = IvyCache.withCachedJar(sbtModuleID(id), Some(globalLock), log)(jar => define(id, Seq(jar)) )
 
 	private def sbtModuleID(id: String) = ModuleID("org.scala-tools.sbt", id, ComponentManager.stampedVersion)
 	/** Install the files for component 'id' to the local repository.  This is usually used after writing files to the directory returned by 'location'. */
-	def cache(id: String): Unit = IvyCache.cacheJar(sbtModuleID(id), file(id)(IfMissing.Fail), log)
+	def cache(id: String): Unit = IvyCache.cacheJar(sbtModuleID(id), file(id)(IfMissing.Fail), Some(globalLock), log)
-	def clearCache(id: String): Unit = lockGlobalCache { IvyCache.clearCachedJar(sbtModuleID(id), log) }
+	def clearCache(id: String): Unit = lockGlobalCache { IvyCache.clearCachedJar(sbtModuleID(id), Some(globalLock), log) }
 }
 class InvalidComponent(msg: String, cause: Throwable) extends RuntimeException(msg, cause)
 {

ivy/Ivy.scala

@@ -6,6 +6,7 @@ package sbt
 import Artifact.{defaultExtension, defaultType}
 
 import java.io.File
+import java.util.concurrent.Callable
 
 import org.apache.ivy.{core, plugins, util, Ivy}
 import core.IvyPatternHelper
@@ -29,13 +30,25 @@ final class IvySbt(configuration: IvyConfiguration)
 	*/
 	private lazy val logger = new IvyLoggerInterface(log)
 	private def withDefaultLogger[T](f: => T): T =
-		IvySbt.synchronized // Ivy is not thread-safe.  In particular, it uses a static DocumentBuilder, which is not thread-safe
+	{
+		def action() =
+			IvySbt.synchronized 
+			{
+				val originalLogger = Message.getDefaultLogger
+				Message.setDefaultLogger(logger)
+				try { f }
+				finally { Message.setDefaultLogger(originalLogger) }
+			}
+		// Ivy is not thread-safe nor can the cache be used concurrently.
+		// If provided a GlobalLock, we can use that to ensure safe access to the cache.
+		// Otherwise, we can at least synchronize within the JVM.
+		//   For thread-safety In particular, Ivy uses a static DocumentBuilder, which is not thread-safe.
+		configuration.lock match
 		{
-			val originalLogger = Message.getDefaultLogger
+			case Some(lock) => lock(ivyLockFile, new Callable[T] { def call = action() })
-			Message.setDefaultLogger(logger)
+			case None => action()
-			try { f }
-			finally { Message.setDefaultLogger(originalLogger) }
 		}
+	}
 	private lazy val settings =
 	{
 		val is = new IvySettings
@@ -56,6 +69,8 @@ final class IvySbt(configuration: IvyConfiguration)
 		i.getLoggerEngine.pushLogger(logger)
 		i
 	}
+	// Must be the same file as is used in Update in the launcher
+	private lazy val ivyLockFile = new File(settings.getDefaultIvyUserDir, ".sbt.ivy.lock")
 	/** ========== End Configuration/Setup ============*/
 
 	/** Uses the configured Ivy instance within a safe context.*/
@@ -178,7 +193,15 @@ private object IvySbt
 	private def configureCache(settings: IvySettings, dir: Option[File])
 	{
 		val cacheDir = dir.getOrElse(settings.getDefaultRepositoryCacheBasedir())
-		val manager = new DefaultRepositoryCacheManager("default-cache", settings, cacheDir)
+		val manager = new DefaultRepositoryCacheManager("default-cache", settings, cacheDir) {
+			override def clean() { delete(getBasedir); true }
+			private final def deleteAll(fs: Seq[File]) = if(fs ne null) fs foreach delete
+			private final def delete(f: File)
+			{
+				if(f.isDirectory) deleteAll(f.listFiles)
+				try { f.delete } catch { case _: java.io.IOException =>  }
+			}
+		}
 		manager.setUseOrigin(true)
 		manager.setChangingMatcher(PatternMatcher.REGEXP);
 		manager.setChangingPattern(".*-SNAPSHOT");

ivy/IvyCache.scala

@@ -29,36 +29,36 @@ object IvyCache
 {
 	def lockFile = new File(System.getProperty("java.io.tmpdir"), "sbt.cache.lock")
 	/** Caches the given 'file' with the given ID.  It may be retrieved or cleared using this ID.*/
-	def cacheJar(moduleID: ModuleID, file: File, log: IvyLogger)
+	def cacheJar(moduleID: ModuleID, file: File, lock: Option[xsbti.GlobalLock], log: IvyLogger)
 	{
 		val artifact = defaultArtifact(moduleID)
 		val resolved = new ResolvedResource(new FileResource(new IvyFileRepository, file), moduleID.revision)
-		withDefaultCache(log) { cache =>
+		withDefaultCache(lock, log) { cache =>
 			val resolver = new ArtifactResourceResolver { def resolve(artifact: IvyArtifact) = resolved }
 			cache.download(artifact, resolver, new FileDownloader, new CacheDownloadOptions)
 		}
 	}
 	/** Clears the cache of the jar for the given ID.*/
-	def clearCachedJar(id: ModuleID, log: IvyLogger)
+	def clearCachedJar(id: ModuleID, lock: Option[xsbti.GlobalLock], log: IvyLogger)
 	{
-		try { withCachedJar(id, log)(_.delete) }
+		try { withCachedJar(id, lock, log)(_.delete) }
 		catch { case e: Exception => log.debug("Error cleaning cached jar: " + e.toString) }
 	}
 	/** Copies the cached jar for the given ID to the directory 'toDirectory'.  If the jar is not in the cache, NotInCache is thrown.*/
-	def retrieveCachedJar(id: ModuleID, toDirectory: File, log: IvyLogger) =
+	def retrieveCachedJar(id: ModuleID, toDirectory: File, lock: Option[xsbti.GlobalLock], log: IvyLogger) =
-		withCachedJar(id, log) { cachedFile =>
+		withCachedJar(id, lock, log) { cachedFile =>
 			val copyTo = new File(toDirectory, cachedFile.getName)
 			FileUtil.copy(cachedFile, copyTo, null)
 			copyTo
 		}
 
 	/** Get the location of the cached jar for the given ID in the Ivy cache.  If the jar is not in the cache, NotInCache is thrown .*/
-	def withCachedJar[T](id: ModuleID, log: IvyLogger)(f: File => T): T =
+	def withCachedJar[T](id: ModuleID, lock: Option[xsbti.GlobalLock], log: IvyLogger)(f: File => T): T =
 	{
 		val cachedFile =
 			try
 			{
-				withDefaultCache(log) { cache =>
+				withDefaultCache(lock, log) { cache =>
 					val artifact = defaultArtifact(id)
 					cache.getArchiveFileInCache(artifact, unknownOrigin(artifact))
 				}
@@ -68,9 +68,9 @@ object IvyCache
 		if(cachedFile.exists) f(cachedFile) else throw new NotInCache(id)
 	}
 	/** Calls the given function with the default Ivy cache.*/
-	def withDefaultCache[T](log: IvyLogger)(f: DefaultRepositoryCacheManager => T): T =
+	def withDefaultCache[T](lock: Option[xsbti.GlobalLock], log: IvyLogger)(f: DefaultRepositoryCacheManager => T): T =
 	{
-		val (ivy, local) = basicLocalIvy(log)
+		val (ivy, local) = basicLocalIvy(lock, log)
 		ivy.withIvy { ivy =>
 			val cache = ivy.getSettings.getDefaultRepositoryCacheManager.asInstanceOf[DefaultRepositoryCacheManager]
 			cache.setUseOrigin(false)
@@ -79,11 +79,11 @@ object IvyCache
 	}
 	private def unknownOrigin(artifact: IvyArtifact) = ArtifactOrigin.unkwnown(artifact)
 	/** A minimal Ivy setup with only a local resolver and the current directory as the base directory.*/
-	private def basicLocalIvy(log: IvyLogger) =
+	private def basicLocalIvy(lock: Option[xsbti.GlobalLock], log: IvyLogger) =
 	{
 		val local = Resolver.defaultLocal
 		val paths = new IvyPaths(new File("."), None)
-		val conf = new InlineIvyConfiguration(paths, Seq(local), Nil, log)
+		val conf = new InlineIvyConfiguration(paths, Seq(local), Nil, lock, log)
 		(new IvySbt(conf), local)
 	}
 	/** Creates a default jar artifact based on the given ID.*/

ivy/IvyConfigurations.scala

@@ -10,28 +10,29 @@ final class IvyPaths(val baseDirectory: File, val cacheDirectory: Option[File])
 
 sealed trait IvyConfiguration extends NotNull
 {
+	def lock: Option[xsbti.GlobalLock]
 	def baseDirectory: File
 	def log: IvyLogger
 }
 final class InlineIvyConfiguration(val paths: IvyPaths, val resolvers: Seq[Resolver], 
-	val moduleConfigurations: Seq[ModuleConfiguration], val log: IvyLogger) extends IvyConfiguration
+	val moduleConfigurations: Seq[ModuleConfiguration], val lock: Option[xsbti.GlobalLock], val log: IvyLogger) extends IvyConfiguration
 {
 	def baseDirectory = paths.baseDirectory
 }
-final class ExternalIvyConfiguration(val baseDirectory: File, val file: File, val log: IvyLogger) extends IvyConfiguration
+final class ExternalIvyConfiguration(val baseDirectory: File, val file: File, val lock: Option[xsbti.GlobalLock], val log: IvyLogger) extends IvyConfiguration
 
 object IvyConfiguration
 {
 	/** Called to configure Ivy when inline resolvers are not specified.
 	* This will configure Ivy with an 'ivy-settings.xml' file if there is one or else use default resolvers.*/
-	def apply(paths: IvyPaths, log: IvyLogger): IvyConfiguration =
+	def apply(paths: IvyPaths, lock: Option[xsbti.GlobalLock], log: IvyLogger): IvyConfiguration =
 	{
 		log.debug("Autodetecting configuration.")
 		val defaultIvyConfigFile = IvySbt.defaultIvyConfiguration(paths.baseDirectory)
 		if(defaultIvyConfigFile.canRead)
-			new ExternalIvyConfiguration(paths.baseDirectory, defaultIvyConfigFile, log)
+			new ExternalIvyConfiguration(paths.baseDirectory, defaultIvyConfigFile, lock, log)
 		else
-			new InlineIvyConfiguration(paths, Resolver.withDefaultResolvers(Nil), Nil, log)
+			new InlineIvyConfiguration(paths, Resolver.withDefaultResolvers(Nil), Nil, lock, log)
 	}
 }