From 2acaa0ad25fa8f1af73b2fb8245d96e56d0d0c05 Mon Sep 17 00:00:00 2001 From: Mark Harrah Date: Thu, 20 Aug 2009 00:02:06 -0400 Subject: [PATCH] Tests and fixes for component manager and cache interface. --- ivy/ComponentManager.scala | 30 ++++- ivy/Ivy.scala | 16 +-- ivy/IvyActions.scala | 38 +++---- ivy/IvyCache.scala | 103 ++++++++++++++++++ ivy/IvyInterface.scala | 26 ++--- ivy/src/test/scala/ComponentManagerTest.scala | 95 ++++++++++++++++ ivy/src/test/scala/TestIvyLogger.scala | 9 ++ 7 files changed, 267 insertions(+), 50 deletions(-) create mode 100644 ivy/IvyCache.scala create mode 100644 ivy/src/test/scala/ComponentManagerTest.scala create mode 100644 ivy/src/test/scala/TestIvyLogger.scala diff --git a/ivy/ComponentManager.scala b/ivy/ComponentManager.scala index f1d441916..fc1209758 100644 --- a/ivy/ComponentManager.scala +++ b/ivy/ComponentManager.scala @@ -9,12 +9,16 @@ import xsbti.Versions * version of Scala. * * The component manager provides services to install and retrieve components to the local repository. -* This is used for source jars so that the compilation need not be repeated for other projects on the same +* This is used for compiled source jars so that the compilation need not be repeated for other projects on the same * machine. */ class ComponentManager(baseDirectory: File, log: IvyLogger) extends NotNull { + /** Get the location where files for component 'id' are stored. This method does not ensure that the component is retrieved from the + * local repository. By default, the location returned is is baseDirectory / id.*/ def location(id: String): File = new File(baseDirectory, id) + /** Get the location where files for component 'id' are stored. If the component has not yet been retrieved from the local repository, + * it is retrieved first. */ def directory(id: String): File = { val dir = location(id) @@ -22,24 +26,38 @@ class ComponentManager(baseDirectory: File, log: IvyLogger) extends NotNull update(id) dir } + // get the contents of the given directory, wrapping a null result in an empty list. private def contents(dir: File): Seq[File] = { val fs = dir.listFiles if(fs == null) Nil else fs } + /** Get all of the files for component 'id', throwing an exception if no files exist for the component. */ def files(id: String): Iterable[File] = { val fs = contents(directory(id)) - if(!fs.isEmpty) fs else error("Could not find required component '" + id + "'") + if(!fs.isEmpty) fs else invalid("Could not find required component '" + id + "'") } + /** Get the file for component 'id', throwing an exception if no files or multiple files exist for the component. */ def file(id: String): File = files(id).toList match { case x :: Nil => x - case xs => error("Expected single file for component '" + id + "', found: " + xs.mkString(", ")) + case xs => invalid("Expected single file for component '" + id + "', found: " + xs.mkString(", ")) } - + private def invalid(msg: String) = throw new InvalidComponent(msg) + private def invalid(e: NotInCache) = throw new InvalidComponent(e.getMessage, e) + + /** Retrieve the file for component 'id' from the local repository. */ def update(id: String): Unit = - IvyActions.basicRetrieveLocal(sbtModuleID("manager"), Seq(sbtModuleID(id)), location(id), log) + try { IvyCache.retrieveCachedJar(sbtModuleID(id), location(id), log) } + catch { case e: NotInCache => invalid(e) } + def sbtModuleID(id: String) = ModuleID("org.scala-tools.sbt", id, Versions.Sbt) - def cache(id: String): Unit = IvyActions.basicPublishLocal(sbtModuleID(id), Nil, files(id), log) + /** Install the files for component 'id' to the local repository. This is usually used after writing files to the directory returned by 'location'. */ + def cache(id: String): Unit = IvyCache.cacheJar(sbtModuleID(id), file(id), log) + def clearCache(id: String): Unit = IvyCache.clearCachedJar(sbtModuleID(id), log) +} +class InvalidComponent(msg: String, cause: Throwable) extends RuntimeException(msg, cause) +{ + def this(msg: String) = this(msg, null) } \ No newline at end of file diff --git a/ivy/Ivy.scala b/ivy/Ivy.scala index d85a46733..ad7e4866c 100644 --- a/ivy/Ivy.scala +++ b/ivy/Ivy.scala @@ -73,13 +73,13 @@ final class IvySbt(configuration: IvyConfiguration) try { f(ivy) } finally { ivy.popContext() } } - + final class Module(val moduleConfiguration: ModuleConfiguration) extends NotNull { def logger = configuration.log def withModule[T](f: (Ivy,DefaultModuleDescriptor,String) => T): T = withIvy[T] { ivy => f(ivy, moduleDescriptor, defaultConfig) } - + import moduleConfiguration._ private lazy val (moduleDescriptor: DefaultModuleDescriptor, defaultConfig: String) = { @@ -97,7 +97,7 @@ final class IvySbt(configuration: IvyConfiguration) val moduleID = newConfiguredModuleID val defaultConf = defaultConfiguration getOrElse Configurations.config(ModuleDescriptor.DEFAULT_CONFIGURATION) log.debug("Using inline dependencies specified in Scala" + (if(ivyXML.isEmpty) "." else " and XML.")) - + val parser = IvySbt.parseIvyXML(ivy.getSettings, IvySbt.wrapped(module, ivyXML), moduleID, defaultConf.name, validate) IvySbt.addArtifacts(moduleID, artifacts) @@ -112,7 +112,7 @@ final class IvySbt(configuration: IvyConfiguration) configurations.foreach(config => mod.addConfiguration(IvySbt.toIvyConfiguration(config))) mod } - + /** Parses the given Maven pom 'pomFile'.*/ private def readPom(pomFile: File) = { @@ -164,11 +164,11 @@ private object IvySbt val DefaultIvyConfigFilename = "ivysettings.xml" val DefaultIvyFilename = "ivy.xml" val DefaultMavenFilename = "pom.xml" - + private def defaultIvyFile(project: File) = new File(project, DefaultIvyFilename) private def defaultIvyConfiguration(project: File) = new File(project, DefaultIvyConfigFilename) private def defaultPOM(project: File) = new File(project, DefaultMavenFilename) - + /** Sets the resolvers for 'settings' to 'resolvers'. This is done by creating a new chain and making it the default. */ private def setResolvers(settings: IvySettings, resolvers: Seq[Resolver], log: IvyLogger) { @@ -207,7 +207,7 @@ private object IvySbt moduleID.check() } /** Converts the given sbt module id into an Ivy ModuleRevisionId.*/ - private def toID(m: ModuleID) = + private[xsbt] def toID(m: ModuleID) = { import m._ ModuleRevisionId.newInstance(organization, name, revision) @@ -253,7 +253,7 @@ private object IvySbt parser.parse() parser } - + /** This method is used to add inline dependencies to the provided module. */ def addDependencies(moduleID: DefaultModuleDescriptor, dependencies: Iterable[ModuleID], parser: CustomXmlParser.CustomParser) { diff --git a/ivy/IvyActions.scala b/ivy/IvyActions.scala index cb7f74b62..3b95062a9 100644 --- a/ivy/IvyActions.scala +++ b/ivy/IvyActions.scala @@ -6,6 +6,7 @@ import org.apache.ivy.{core, plugins, util, Ivy} import core.cache.DefaultRepositoryCacheManager import core.LogOptions import core.deliver.DeliverOptions +import core.install.InstallOptions import core.module.descriptor.{DefaultArtifact, DefaultDependencyArtifactDescriptor, MDArtifact} import core.module.descriptor.{DefaultDependencyDescriptor, DefaultModuleDescriptor, DependencyDescriptor, ModuleDescriptor} import core.module.id.{ArtifactId,ModuleId, ModuleRevisionId} @@ -18,33 +19,24 @@ final class UpdateConfiguration(val retrieveDirectory: File, val outputPattern: object IvyActions { - def basicPublishLocal(moduleID: ModuleID, dependencies: Iterable[ModuleID], artifactFiles: Iterable[File], log: IvyLogger) + /** Installs the dependencies of the given 'module' from the resolver named 'from' to the resolver named 'to'.*/ + def install(module: IvySbt#Module, from: String, to: String) { - val artifacts = artifactFiles.map(Artifact.defaultArtifact) - val (ivy, local) = basicLocalIvy(log) - val module = new ivy.Module(ModuleConfiguration(moduleID, dependencies, artifacts)) - val srcArtifactPatterns = artifactFiles.map(_.getAbsolutePath) - publish(module, local.name, srcArtifactPatterns, None, None) - } - def basicRetrieveLocal(moduleID: ModuleID, dependencies: Iterable[ModuleID], to: File, log: IvyLogger) - { - val (ivy, local) = basicLocalIvy(log) - val module = new ivy.Module(ModuleConfiguration(moduleID, dependencies, Nil)) - val up = new UpdateConfiguration(to, defaultOutputPattern, false, true) - update(module, up) - } - def defaultOutputPattern = "[artifact]-[revision](-[classifier]).[ext]" - private def basicLocalIvy(log: IvyLogger) = - { - val local = Resolver.defaultLocal - val paths = new IvyPaths(new File("."), None) - val conf = new IvyConfiguration(paths, Seq(local), log) - (new IvySbt(conf), local) + module.withModule { (ivy, md, default) => + for(dependency <- md.getDependencies) + { + module.logger.info("Installing " + dependency) + val options = new InstallOptions + options.setValidate(module.moduleConfiguration.validate) + options.setTransitive(dependency.isTransitive) + ivy.install(dependency.getDependencyRevisionId, from, to, options) + } + } } /** Clears the Ivy cache, as configured by 'config'. */ def cleanCache(ivy: IvySbt) = ivy.withIvy { _.getSettings.getRepositoryCacheManagers.foreach(_.clean()) } - + /** Creates a Maven pom from the given Ivy configuration*/ def makePom(module: IvySbt#Module, extraDependencies: Iterable[ModuleID], configurations: Option[Iterable[Configuration]], output: File) { @@ -90,7 +82,7 @@ object IvyActions newModule.addDependency(translated) newModule } - + def deliver(module: IvySbt#Module, status: String, deliverIvyPattern: String, extraDependencies: Iterable[ModuleID], configurations: Option[Iterable[Configuration]], quiet: Boolean) { module.withModule { case (ivy, md, default) => diff --git a/ivy/IvyCache.scala b/ivy/IvyCache.scala new file mode 100644 index 000000000..cd6fdb2b2 --- /dev/null +++ b/ivy/IvyCache.scala @@ -0,0 +1,103 @@ +package xsbt + +import java.io.File +import java.net.URL + +import org.apache.ivy.{core, plugins, util} +import core.cache.{ArtifactOrigin, CacheDownloadOptions, DefaultRepositoryCacheManager} +import core.module.descriptor.{Artifact => IvyArtifact, DefaultArtifact} +import plugins.repository.file.{FileRepository=>IvyFileRepository, FileResource} +import plugins.repository.{ArtifactResourceResolver, Resource, ResourceDownloader} +import plugins.resolver.util.ResolvedResource +import util.FileUtil + +class NotInCache(val id: ModuleID, cause: Throwable) + extends RuntimeException(NotInCache(id, cause), cause) +{ + def this(id: ModuleID) = this(id, null) +} +private object NotInCache +{ + def apply(id: ModuleID, cause: Throwable) = + { + val postfix = if(cause == null) "" else (": " +cause.toString) + "File for " + id + " not in cache" + postfix + } +} +/** Provides methods for working at the level of a single jar file with the default Ivy cache.*/ +object IvyCache +{ + /** Caches the given 'file' with the given ID. It may be retrieved or cleared using this ID.*/ + def cacheJar(moduleID: ModuleID, file: File, log: IvyLogger) + { + val artifact = defaultArtifact(moduleID) + val resolved = new ResolvedResource(new FileResource(new IvyFileRepository, file), moduleID.revision) + withDefaultCache(log) { cache => + val resolver = new ArtifactResourceResolver { def resolve(artifact: IvyArtifact) = resolved } + cache.download(artifact, resolver, new FileDownloader, new CacheDownloadOptions) + } + } + /** Clears the cache of the jar for the given ID.*/ + def clearCachedJar(id: ModuleID, log: IvyLogger) + { + try { getCachedFile(id, log).delete } + catch { case e: Exception => log.debug("Error cleaning cached jar: " + e.toString) } + } + /** Copies the cached jar for the given ID to the directory 'toDirectory'. If the jar is not in the cache, NotInCache is thrown.*/ + def retrieveCachedJar(id: ModuleID, toDirectory: File, log: IvyLogger) = + { + val cachedFile = getCachedFile(id, log) + val copyTo = new File(toDirectory, cachedFile.getName) + FileUtil.copy(cachedFile, copyTo, null) + copyTo + } + /** Get the location of the cached jar for the given ID in the Ivy cache. If the jar is not in the cache, NotInCache is thrown.*/ + def getCachedFile(id: ModuleID, log: IvyLogger): File = + { + val cachedFile = + try + { + withDefaultCache(log) { cache => + val artifact = defaultArtifact(id) + cache.getArchiveFileInCache(artifact, unknownOrigin(artifact)) + } + } + catch { case e: Exception => throw new NotInCache(id, e) } + + if(cachedFile.exists) cachedFile else throw new NotInCache(id) + } + /** Calls the given function with the default Ivy cache.*/ + def withDefaultCache[T](log: IvyLogger)(f: DefaultRepositoryCacheManager => T): T = + { + val (ivy, local) = basicLocalIvy(log) + ivy.withIvy { ivy => + val cache = ivy.getSettings.getDefaultRepositoryCacheManager.asInstanceOf[DefaultRepositoryCacheManager] + cache.setUseOrigin(false) + f(cache) + } + } + private def unknownOrigin(artifact: IvyArtifact) = ArtifactOrigin.unkwnown(artifact) + /** A minimal Ivy setup with only a local resolver and the current directory as the base directory.*/ + private def basicLocalIvy(log: IvyLogger) = + { + val local = Resolver.defaultLocal + val paths = new IvyPaths(new File("."), None) + val conf = new IvyConfiguration(paths, Seq(local), log) + (new IvySbt(conf), local) + } + /** Creates a default jar artifact based on the given ID.*/ + private def defaultArtifact(moduleID: ModuleID): IvyArtifact = + new DefaultArtifact(IvySbt.toID(moduleID), null, moduleID.name, "jar", "jar") +} +/** Required by Ivy for copying to the cache.*/ +private class FileDownloader extends ResourceDownloader with NotNull +{ + def download(artifact: IvyArtifact, resource: Resource, dest: File) + { + if(dest.exists()) dest.delete() + val part = new File(dest.getAbsolutePath + ".part") + FileUtil.copy(resource.openStream, part, null) + if(!part.renameTo(dest)) + error("Could not move temporary file " + part + " to final location " + dest) + } +} \ No newline at end of file diff --git a/ivy/IvyInterface.scala b/ivy/IvyInterface.scala index d31ac00c3..5b8312cc9 100644 --- a/ivy/IvyInterface.scala +++ b/ivy/IvyInterface.scala @@ -72,7 +72,7 @@ sealed abstract class PatternsBasedRepository extends Resolver /** The object representing the configured patterns for this repository. */ def patterns: Patterns - + /** Enables maven 2 compatibility for this repository. */ def mavenStyle() = copy(patterns.mavenStyle()) /** Adds the given patterns for resolving/publishing Ivy files.*/ @@ -100,10 +100,10 @@ sealed abstract class SshBasedRepository extends PatternsBasedRepository type RepositoryType <: SshBasedRepository protected def copy(connection: SshConnection): RepositoryType private def copy(authentication: SshAuthentication): RepositoryType = copy(connection.copy(Some(authentication))) - + /** The object representing the configured ssh connection for this repository. */ def connection: SshConnection - + /** Configures this to use the specified user name and password when connecting to the remote repository. */ def as(user: String, password: String): RepositoryType = copy(new PasswordAuthentication(user, password)) /** Configures this to use the specified keyfile and password for the keyfile when connecting to the remote repository. */ @@ -218,7 +218,7 @@ object Resolver } private def baseRepository[T](baseURI: java.net.URI)(construct: Patterns => T)(implicit basePatterns: Patterns): T = construct(resolvePatterns(baseURI.normalize, basePatterns)) - + /** If `base` is None, `patterns` is returned unchanged. * Otherwise, the ivy file and artifact patterns in `patterns` are resolved against the given base. */ private def resolvePatterns(base: Option[String], patterns: Patterns): Patterns = @@ -236,7 +236,7 @@ object Resolver } /** Constructs a `URI` with the path component set to `path` and the other components set to null.*/ private def pathURI(path: String) = new URI(null, null, path, null) - + def defaultFileConfiguration = FileConfiguration(true, None) def mavenStylePatterns = Patterns(Nil, mavenStyleBasePattern :: Nil, true) def ivyStylePatterns = Patterns(Nil, Nil, false) @@ -244,11 +244,11 @@ object Resolver def defaultPatterns = mavenStylePatterns def mavenStyleBasePattern = "[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]" def localBasePattern = "[organisation]/[module]/[revision]/[type]s/[artifact].[ext]" - + def userRoot = System.getProperty("user.home") def userMavenRoot = userRoot + "/.m2/repository/" def userIvyRoot = userRoot + "/.ivy2/" - + def defaultLocal = defaultUserFileRepository("local") def defaultShared = defaultUserFileRepository("shared") def defaultUserFileRepository(id: String) = file(id, new File(userIvyRoot, id))(defaultIvyPatterns) @@ -263,7 +263,7 @@ object Configurations { def config(name: String) = new Configuration(name) def defaultMavenConfigurations = Compile :: Runtime :: Test :: Provided :: System :: Optional :: Sources :: Javadoc :: Nil - + lazy val Default = config("default") lazy val Compile = config("compile") lazy val IntegrationTest = config("it") hide @@ -276,7 +276,7 @@ object Configurations lazy val Optional = config("optional") lazy val CompilerPlugin = config("plugin") hide - + private[xsbt] val DefaultMavenConfiguration = defaultConfiguration(true) private[xsbt] val DefaultIvyConfiguration = defaultConfiguration(false) private[xsbt] def DefaultConfiguration(mavenStyle: Boolean) = if(mavenStyle) DefaultMavenConfiguration else DefaultIvyConfiguration @@ -285,7 +285,7 @@ object Configurations val base = if(mavenStyle) Configurations.Compile else Configurations.Default config(base.name + "->default(compile)") } - + private[xsbt] def removeDuplicates(configs: Iterable[Configuration]) = Set(scala.collection.mutable.Map(configs.map(config => (config.name, config)).toSeq: _*).values.toList: _*) } /** Represents an Ivy configuration. */ @@ -311,8 +311,8 @@ object Artifact def apply(name: String, url: URL): Artifact =Artifact(name, extract(url, defaultType), extract(url, defaultExtension), None, Nil, Some(url)) val defaultExtension = "jar" val defaultType = "jar" - private[this] def extract(url: URL, default: String): String = extract(url.toString, default) - private[this] def extract(name: String, default: String): String = + def extract(url: URL, default: String): String = extract(url.toString, default) + def extract(name: String, default: String): String = { val i = name.lastIndexOf('.') if(i >= 0) @@ -346,7 +346,7 @@ object Credentials { val properties = new scala.collection.mutable.HashMap[String, String] def get(keys: List[String]) = keys.flatMap(properties.get).firstOption.toRight(keys.head + " not specified in credentials file: " + path) - + impl.MapUtilities.read(properties, path, log) orElse { List.separate( List(RealmKeys, HostKeys, UserKeys, PasswordKeys).map(get) ) match diff --git a/ivy/src/test/scala/ComponentManagerTest.scala b/ivy/src/test/scala/ComponentManagerTest.scala new file mode 100644 index 000000000..5d88297f4 --- /dev/null +++ b/ivy/src/test/scala/ComponentManagerTest.scala @@ -0,0 +1,95 @@ +package xsbt + +import java.io.File +import org.specs._ +import FileUtilities.{createDirectory, delete, touch, withTemporaryDirectory} +import org.apache.ivy.util.ChecksumHelper + +object ComponentManagerTest extends Specification +{ + val TestID = "manager-test" + "Component manager" should { + "throw an exception if 'file' is called for a non-existing component" in { + withManager { _.file(TestID) must throwA[InvalidComponent] } + } + "throw an exception if 'file' is called for an empty component" in { + withManager { manager => + createDirectory(manager.location(TestID)) + ( manager.file(TestID) ) must throwA[InvalidComponent] + } + } + "return the file for a single-file component" in { + withManager { manager => + createFiles(manager, TestID, "a") match { case Seq(x) => + manager.file(TestID).getAbsoluteFile must beEqualTo(x.getAbsoluteFile) + } + } + } + + "throw an exception if 'file' is called for multi-file component" in { + withManager { manager => + createFiles(manager, TestID, "a", "b") + ( manager.file(TestID) ) must throwA[InvalidComponent] + } + } + "return the files for a multi-file component" in { + withManager { manager => + val files = createFiles(manager, TestID, "a", "b") + manager.files(TestID) must haveTheSameElementsAs(files) + } + } + "return the files for a single-file component" in { + withManager { manager => + val files = createFiles(manager, TestID, "a") + manager.files(TestID) must haveTheSameElementsAs(files) + } + } + "throw an exception if 'files' is called for a non-existing component" in { + withManager { _.files(TestID) must throwA[InvalidComponent] } + } + + "properly cache a file and then retrieve it to an unresolved component" in { + withManager { manager => + val file = createFile(manager, TestID, "a") + val hash = checksum(file) + try + { + manager.cache(TestID) + delete(manager.location(TestID)) + FileUtilities.listFiles(manager.location(TestID)).toList must haveSize(0) + checksum(manager.file(TestID)) must beEqualTo(hash) + } + finally { manager.clearCache(TestID) } + } + } + + "not retrieve to a component already resolved" in { + withManager { manager => + val file = createFile(manager, TestID, "a") + try + { + manager.cache(TestID) + val idDirectory = manager.location(TestID) + delete(idDirectory) + createDirectory(idDirectory) + manager.file(TestID) must throwA[InvalidComponent] + } + finally { manager.clearCache(TestID) } + } + } + } + private def checksum(file: File) = ChecksumHelper.computeAsString(file, "sha1") + private def createFile(manager: ComponentManager, id: String, name: String): File = createFiles(manager, id, name).toList.head + private def createFiles(manager: ComponentManager, id: String, names: String*): Seq[File] = + { + val dir = manager.location(id) + createDirectory(dir) + names.map { name => + val testFile = new File(dir, name) + touch(testFile) + testFile + } + } + private def withManager[T](f: ComponentManager => T): T = + TestIvyLogger( logger => withTemporaryDirectory { temp => f(new ComponentManager(temp, logger)) } ) +} \ No newline at end of file diff --git a/ivy/src/test/scala/TestIvyLogger.scala b/ivy/src/test/scala/TestIvyLogger.scala new file mode 100644 index 000000000..5ed7716d2 --- /dev/null +++ b/ivy/src/test/scala/TestIvyLogger.scala @@ -0,0 +1,9 @@ +package xsbt + +import xsbti.TestLogger + +class TestIvyLogger extends TestLogger with IvyLogger +object TestIvyLogger +{ + def apply[T](f: TestIvyLogger => T): T = TestLogger(new TestIvyLogger)(f) +} \ No newline at end of file