luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use a machine-global lock on Ivy because caches are not safe for concurrent access

This commit is contained in:
Mark Harrah 2010-01-28 19:31:04 -05:00
parent 3e3519b3a7
commit a958fa6484
6 changed files with 61 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ivy/ComponentManager.scala
View File

@ -39,7 +39,9 @@ class ComponentManager(globalLock: xsbti.GlobalLock, provider: xsbti.ComponentPr
lockLocalCache { getOrElse(fromGlobal) }
}
/** This is used to lock the local cache in project/boot/. By checking the local cache first, we can avoid grabbing a global lock. */
private def lockLocalCache[T](action: => T): T = lock(provider.lockFile)( action )
/** This is used to ensure atomic access to components in the global Ivy cache.*/
private def lockGlobalCache[T](action: => T): T = lock(IvyCache.lockFile)( action )
private def lock[T](file: File)(action: => T): T = globalLock(file, new Callable[T] { def call = action })
/** Get the file for component 'id', throwing an exception if no files or multiple files exist for the component. */
@ -53,12 +55,12 @@ class ComponentManager(globalLock: xsbti.GlobalLock, provider: xsbti.ComponentPr
def define(id: String, files: Iterable[File]) = lockLocalCache { provider.defineComponent(id, files.toSeq.toArray) }
/** Retrieve the file for component 'id' from the local repository. */
private def update(id: String): Unit = IvyCache.withCachedJar(sbtModuleID(id), log)(jar => define(id, Seq(jar)) )
private def update(id: String): Unit = IvyCache.withCachedJar(sbtModuleID(id), Some(globalLock), log)(jar => define(id, Seq(jar)) )
private def sbtModuleID(id: String) = ModuleID("org.scala-tools.sbt", id, ComponentManager.stampedVersion)
/** Install the files for component 'id' to the local repository. This is usually used after writing files to the directory returned by 'location'. */
def cache(id: String): Unit = IvyCache.cacheJar(sbtModuleID(id), file(id)(IfMissing.Fail), log)
def clearCache(id: String): Unit = lockGlobalCache { IvyCache.clearCachedJar(sbtModuleID(id), log) }
def cache(id: String): Unit = IvyCache.cacheJar(sbtModuleID(id), file(id)(IfMissing.Fail), Some(globalLock), log)
def clearCache(id: String): Unit = lockGlobalCache { IvyCache.clearCachedJar(sbtModuleID(id), Some(globalLock), log) }
}
class InvalidComponent(msg: String, cause: Throwable) extends RuntimeException(msg, cause)
{

35
ivy/Ivy.scala
View File

@ -6,6 +6,7 @@ package sbt
import Artifact.{defaultExtension, defaultType}
import java.io.File
import java.util.concurrent.Callable
import org.apache.ivy.{core, plugins, util, Ivy}
import core.IvyPatternHelper
@ -29,13 +30,25 @@ final class IvySbt(configuration: IvyConfiguration)
*/
private lazy val logger = new IvyLoggerInterface(log)
private def withDefaultLogger[T](f: => T): T =
IvySbt.synchronized // Ivy is not thread-safe. In particular, it uses a static DocumentBuilder, which is not thread-safe
{
def action() =
IvySbt.synchronized
{
val originalLogger = Message.getDefaultLogger
Message.setDefaultLogger(logger)
try { f }
finally { Message.setDefaultLogger(originalLogger) }
}
// Ivy is not thread-safe nor can the cache be used concurrently.
// If provided a GlobalLock, we can use that to ensure safe access to the cache.
// Otherwise, we can at least synchronize within the JVM.
// For thread-safety In particular, Ivy uses a static DocumentBuilder, which is not thread-safe.
configuration.lock match
{
val originalLogger = Message.getDefaultLogger
Message.setDefaultLogger(logger)
try { f }
finally { Message.setDefaultLogger(originalLogger) }
case Some(lock) => lock(ivyLockFile, new Callable[T] { def call = action() })
case None => action()
}
}
private lazy val settings =
{
val is = new IvySettings
@ -56,6 +69,8 @@ final class IvySbt(configuration: IvyConfiguration)
i.getLoggerEngine.pushLogger(logger)
i
}
// Must be the same file as is used in Update in the launcher
private lazy val ivyLockFile = new File(settings.getDefaultIvyUserDir, ".sbt.ivy.lock")
/** ========== End Configuration/Setup ============*/
/** Uses the configured Ivy instance within a safe context.*/
@ -178,7 +193,15 @@ private object IvySbt
private def configureCache(settings: IvySettings, dir: Option[File])
{
val cacheDir = dir.getOrElse(settings.getDefaultRepositoryCacheBasedir())
val manager = new DefaultRepositoryCacheManager("default-cache", settings, cacheDir)
val manager = new DefaultRepositoryCacheManager("default-cache", settings, cacheDir) {
override def clean() { delete(getBasedir); true }
private final def deleteAll(fs: Seq[File]) = if(fs ne null) fs foreach delete
private final def delete(f: File)
{
if(f.isDirectory) deleteAll(f.listFiles)
try { f.delete } catch { case _: java.io.IOException => }
}
}
manager.setUseOrigin(true)
manager.setChangingMatcher(PatternMatcher.REGEXP);
manager.setChangingPattern(".*-SNAPSHOT");

24
ivy/IvyCache.scala
View File

@ -29,36 +29,36 @@ object IvyCache
{
def lockFile = new File(System.getProperty("java.io.tmpdir"), "sbt.cache.lock")
/** Caches the given 'file' with the given ID. It may be retrieved or cleared using this ID.*/
def cacheJar(moduleID: ModuleID, file: File, log: IvyLogger)
def cacheJar(moduleID: ModuleID, file: File, lock: Option[xsbti.GlobalLock], log: IvyLogger)
{
val artifact = defaultArtifact(moduleID)
val resolved = new ResolvedResource(new FileResource(new IvyFileRepository, file), moduleID.revision)
withDefaultCache(log) { cache =>
withDefaultCache(lock, log) { cache =>
val resolver = new ArtifactResourceResolver { def resolve(artifact: IvyArtifact) = resolved }
cache.download(artifact, resolver, new FileDownloader, new CacheDownloadOptions)
}
}
/** Clears the cache of the jar for the given ID.*/
def clearCachedJar(id: ModuleID, log: IvyLogger)
def clearCachedJar(id: ModuleID, lock: Option[xsbti.GlobalLock], log: IvyLogger)
{
try { withCachedJar(id, log)(_.delete) }
try { withCachedJar(id, lock, log)(_.delete) }
catch { case e: Exception => log.debug("Error cleaning cached jar: " + e.toString) }
}
/** Copies the cached jar for the given ID to the directory 'toDirectory'. If the jar is not in the cache, NotInCache is thrown.*/
def retrieveCachedJar(id: ModuleID, toDirectory: File, log: IvyLogger) =
withCachedJar(id, log) { cachedFile =>
def retrieveCachedJar(id: ModuleID, toDirectory: File, lock: Option[xsbti.GlobalLock], log: IvyLogger) =
withCachedJar(id, lock, log) { cachedFile =>
val copyTo = new File(toDirectory, cachedFile.getName)
FileUtil.copy(cachedFile, copyTo, null)
copyTo
}
/** Get the location of the cached jar for the given ID in the Ivy cache. If the jar is not in the cache, NotInCache is thrown .*/
def withCachedJar[T](id: ModuleID, log: IvyLogger)(f: File => T): T =
def withCachedJar[T](id: ModuleID, lock: Option[xsbti.GlobalLock], log: IvyLogger)(f: File => T): T =
{
val cachedFile =
try
{
withDefaultCache(log) { cache =>
withDefaultCache(lock, log) { cache =>
val artifact = defaultArtifact(id)
cache.getArchiveFileInCache(artifact, unknownOrigin(artifact))
}
@ -68,9 +68,9 @@ object IvyCache
if(cachedFile.exists) f(cachedFile) else throw new NotInCache(id)
}
/** Calls the given function with the default Ivy cache.*/
def withDefaultCache[T](log: IvyLogger)(f: DefaultRepositoryCacheManager => T): T =
def withDefaultCache[T](lock: Option[xsbti.GlobalLock], log: IvyLogger)(f: DefaultRepositoryCacheManager => T): T =
{
val (ivy, local) = basicLocalIvy(log)
val (ivy, local) = basicLocalIvy(lock, log)
ivy.withIvy { ivy =>
val cache = ivy.getSettings.getDefaultRepositoryCacheManager.asInstanceOf[DefaultRepositoryCacheManager]
cache.setUseOrigin(false)
@ -79,11 +79,11 @@ object IvyCache
}
private def unknownOrigin(artifact: IvyArtifact) = ArtifactOrigin.unkwnown(artifact)
/** A minimal Ivy setup with only a local resolver and the current directory as the base directory.*/
private def basicLocalIvy(log: IvyLogger) =
private def basicLocalIvy(lock: Option[xsbti.GlobalLock], log: IvyLogger) =
{
val local = Resolver.defaultLocal
val paths = new IvyPaths(new File("."), None)
val conf = new InlineIvyConfiguration(paths, Seq(local), Nil, log)
val conf = new InlineIvyConfiguration(paths, Seq(local), Nil, lock, log)
(new IvySbt(conf), local)
}
/** Creates a default jar artifact based on the given ID.*/

11
ivy/IvyConfigurations.scala
View File

@ -10,28 +10,29 @@ final class IvyPaths(val baseDirectory: File, val cacheDirectory: Option[File])
sealed trait IvyConfiguration extends NotNull
{
def lock: Option[xsbti.GlobalLock]
def baseDirectory: File
def log: IvyLogger
}
final class InlineIvyConfiguration(val paths: IvyPaths, val resolvers: Seq[Resolver],
val moduleConfigurations: Seq[ModuleConfiguration], val log: IvyLogger) extends IvyConfiguration
val moduleConfigurations: Seq[ModuleConfiguration], val lock: Option[xsbti.GlobalLock], val log: IvyLogger) extends IvyConfiguration
{
def baseDirectory = paths.baseDirectory
}
final class ExternalIvyConfiguration(val baseDirectory: File, val file: File, val log: IvyLogger) extends IvyConfiguration
final class ExternalIvyConfiguration(val baseDirectory: File, val file: File, val lock: Option[xsbti.GlobalLock], val log: IvyLogger) extends IvyConfiguration
object IvyConfiguration
{
/** Called to configure Ivy when inline resolvers are not specified.
* This will configure Ivy with an 'ivy-settings.xml' file if there is one or else use default resolvers.*/
def apply(paths: IvyPaths, log: IvyLogger): IvyConfiguration =
def apply(paths: IvyPaths, lock: Option[xsbti.GlobalLock], log: IvyLogger): IvyConfiguration =
{
log.debug("Autodetecting configuration.")
val defaultIvyConfigFile = IvySbt.defaultIvyConfiguration(paths.baseDirectory)
if(defaultIvyConfigFile.canRead)
new ExternalIvyConfiguration(paths.baseDirectory, defaultIvyConfigFile, log)
new ExternalIvyConfiguration(paths.baseDirectory, defaultIvyConfigFile, lock, log)
else
new InlineIvyConfiguration(paths, Resolver.withDefaultResolvers(Nil), Nil, log)
new InlineIvyConfiguration(paths, Resolver.withDefaultResolvers(Nil), Nil, lock, log)
}
}

1
launch/Cache.scala
View File

@ -2,7 +2,6 @@ package xsbt.boot
import java.lang.ref.{Reference, SoftReference}
import java.util.HashMap
import java.lang.ref.{Reference, SoftReference}
final class Cache[K,V](create: K => V) extends NotNull
{

10
launch/Update.scala
View File

@ -5,6 +5,7 @@ package xsbt.boot
import Pre._
import java.io.{File, FileWriter, PrintWriter, Writer}
import java.util.concurrent.Callable
import java.util.regex.Pattern
import org.apache.ivy.{core, plugins, util, Ivy}
@ -56,11 +57,18 @@ final class Update(config: UpdateConfiguration)
ivy.bind()
ivy
}
// should be the same file as is used in the Ivy module
private lazy val ivyLockFile = new File(settings.getDefaultIvyUserDir, ".sbt.ivy.lock")
/** The main entry point of this class for use by the Update module. It runs Ivy */
def apply(target: UpdateTarget): Boolean =
{
Message.setDefaultLogger(new SbtIvyLogger(logWriter))
val action = new Callable[Boolean] { def call = lockedApply(target) }
Locks(ivyLockFile, action)
}
private def lockedApply(target: UpdateTarget) =
{
ivy.pushContext()
try { update(target); true }
catch
@ -309,6 +317,6 @@ private object SbtIvyLogger
{
val IgnorePrefix = "impossible to define"
val UnknownResolver = "unknown resolver"
def acceptError(msg: String) = (msg ne null) && !msg.startsWith(UnknownResolver)
def acceptError(msg: String) = acceptMessage(msg) && !msg.startsWith(UnknownResolver)
def acceptMessage(msg: String) = (msg ne null) && !msg.startsWith(IgnorePrefix)
}