From b95759a138e626b7a8277c31b4c5690559777029 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Sat, 29 Mar 2014 13:06:27 -0400 Subject: [PATCH 1/4] Bump to our own nightly of ivy 2.4.0 (RC1+). * Testing the infrastructure to use our own instance of ivy * Allow us to push bug-fixes/performance directly into ivy on our own pace, as we submit patches back to ivy master --- ivy/src/main/scala/sbt/IvyLogger.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivy/src/main/scala/sbt/IvyLogger.scala b/ivy/src/main/scala/sbt/IvyLogger.scala index acd07781b..6e1b136fe 100644 --- a/ivy/src/main/scala/sbt/IvyLogger.scala +++ b/ivy/src/main/scala/sbt/IvyLogger.scala @@ -31,7 +31,7 @@ private final class IvyLoggerInterface(logger: Logger) extends MessageLogger def warn(msg: String) = logger.warn(msg) def error(msg: String) = if(SbtIvyLogger.acceptError(msg)) logger.error(msg) - private def emptyList = java.util.Collections.emptyList[T forSome { type T}] + private def emptyList = java.util.Collections.emptyList[String] def getProblems = emptyList def getWarns = emptyList def getErrors = emptyList From 80c09619bffc1f2bfbf4021eea28f0b337376e51 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Fri, 4 Apr 2014 16:38:44 -0400 Subject: [PATCH 2/4] Ensure that if artifact is published, we overwrite default checksums. Fixes # 1233 * Add a new "shim" for RepositoryResolvers that modifies the "put" method so that it will ignore the overwrite flag for checksums. --- ivy/src/main/scala/sbt/ConvertResolver.scala | 77 +++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/ivy/src/main/scala/sbt/ConvertResolver.scala b/ivy/src/main/scala/sbt/ConvertResolver.scala index a93a57011..ab1cf95ce 100644 --- a/ivy/src/main/scala/sbt/ConvertResolver.scala +++ b/ivy/src/main/scala/sbt/ConvertResolver.scala @@ -10,13 +10,84 @@ import core.module.id.ModuleRevisionId import core.module.descriptor.DependencyDescriptor import core.resolve.ResolveData import core.settings.IvySettings -import plugins.resolver.{BasicResolver, DependencyResolver, IBiblioResolver} +import plugins.resolver.{BasicResolver, DependencyResolver, IBiblioResolver, RepositoryResolver} import plugins.resolver.{AbstractPatternsBasedResolver, AbstractSshBasedResolver, FileSystemResolver, SFTPResolver, SshResolver, URLResolver} import plugins.repository.url.{URLRepository => URLRepo} import plugins.repository.file.{FileRepository => FileRepo, FileResource} +import java.io.File +import org.apache.ivy.util.ChecksumHelper +import org.apache.ivy.core.module.descriptor.{Artifact=>IArtifact} + private object ConvertResolver { + /** This class contains all the reflective lookups used in the + * checksum-friendly URL publishing shim. + */ + private object ChecksumFriendlyURLResolver { + import java.lang.reflect.AccessibleObject + private def reflectiveLookup[A <: AccessibleObject](f: Class[_] => A): Option[A] = + try { + val cls = classOf[RepositoryResolver] + val thing = f(cls) + thing.setAccessible(true) + Some(thing) + } catch { + case e: java.lang.ReflectiveOperationException => None + } + private val signerNameField: Option[java.lang.reflect.Field] = + reflectiveLookup(_.getDeclaredField("signerName")) + private val putChecksumMethod: Option[java.lang.reflect.Method] = + reflectiveLookup(_.getDeclaredMethod("putChecksum", + classOf[IArtifact], classOf[File], classOf[String], + classOf[Boolean], classOf[String])) + private val putSignatureMethod: Option[java.lang.reflect.Method] = + reflectiveLookup(_.getDeclaredMethod("putSignature", + classOf[IArtifact], classOf[File], classOf[String], + classOf[Boolean])) + } + /** + * The default behavior of ivy's overwrite flags ignores the fact that a lot of repositories + * will autogenerate checksums *for* an artifact if it doesn't already exist. Therefore + * if we succeed in publishing an artifact, we need to just blast the checksums in place. + * This acts as a "shim" on RepositoryResolvers so that we can hook our methods into + * both the IBiblioResolver + URLResolver without having to duplicate the code in two + * places. However, this does mean our use of reflection is awesome. + * + * TODO - See about contributing back to ivy. + */ + private trait ChecksumFriendlyURLResolver extends RepositoryResolver { + import ChecksumFriendlyURLResolver._ + private def signerName: String = signerNameField match { + case Some(field) => field.get(this).asInstanceOf[String] + case None => null + } + override protected def put(artifact: IArtifact, src: File, dest: String, overwrite: Boolean): Unit = { + // verify the checksum algorithms before uploading artifacts! + val checksums = getChecksumAlgorithms() + val repository = getRepository() + for { + checksum <- checksums + if !ChecksumHelper.isKnownAlgorithm(checksum) + } throw new IllegalArgumentException("Unknown checksum algorithm: " + checksum) + repository.put(artifact, src, dest, overwrite); + // Fix for sbt#1156 - Artifactory will auto-generate MD5/sha1 files, so + // we need to overwrite what it has. + for (checksum <- checksums) { + putChecksumMethod match { + case Some(method) => method.invoke(this, artifact, src, dest, true: java.lang.Boolean, checksum) + case None => // TODO - issue warning? + } + } + if (signerName != null) { + putSignatureMethod match { + case None => () + case Some(method) => method.invoke(artifact, src, dest, true: java.lang.Boolean) + } + } + } + } + /** Converts the given sbt resolver into an Ivy resolver..*/ def apply(r: Resolver, settings: IvySettings, log: Logger) = { @@ -25,7 +96,7 @@ private object ConvertResolver case repo: MavenRepository => { val pattern = Collections.singletonList(Resolver.resolvePattern(repo.root, Resolver.mavenStyleBasePattern)) - final class PluginCapableResolver extends IBiblioResolver with DescriptorRequired { + final class PluginCapableResolver extends IBiblioResolver with ChecksumFriendlyURLResolver with DescriptorRequired { def setPatterns() { // done this way for access to protected methods. setArtifactPatterns(pattern) setIvyPatterns(pattern) @@ -77,7 +148,7 @@ private object ConvertResolver } case repo: URLRepository => { - val resolver = new URLResolver with DescriptorRequired + val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired resolver.setName(repo.name) initializePatterns(resolver, repo.patterns, settings) resolver From 8bd930cbe59932c0d39394b828d5848f0c0bbe5c Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Fri, 4 Apr 2014 20:25:37 -0400 Subject: [PATCH 3/4] Remove JDK7 features from Resolver shim. --- ivy/src/main/scala/sbt/ConvertResolver.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ivy/src/main/scala/sbt/ConvertResolver.scala b/ivy/src/main/scala/sbt/ConvertResolver.scala index ab1cf95ce..74c5c119c 100644 --- a/ivy/src/main/scala/sbt/ConvertResolver.scala +++ b/ivy/src/main/scala/sbt/ConvertResolver.scala @@ -25,15 +25,22 @@ private object ConvertResolver * checksum-friendly URL publishing shim. */ private object ChecksumFriendlyURLResolver { - import java.lang.reflect.AccessibleObject + // TODO - When we dump JDK6 support we can remove this hackery + // import java.lang.reflect.AccessibleObject + type AccessibleObject = { + def setAccessible(value: Boolean): Unit + } private def reflectiveLookup[A <: AccessibleObject](f: Class[_] => A): Option[A] = try { val cls = classOf[RepositoryResolver] val thing = f(cls) + import scala.language.reflectiveCalls thing.setAccessible(true) Some(thing) } catch { - case e: java.lang.ReflectiveOperationException => None + case (_: java.lang.NoSuchFieldException) | + (_: java.lang.SecurityException) | + (_: java.lang.NoSuchMethodException) => None } private val signerNameField: Option[java.lang.reflect.Field] = reflectiveLookup(_.getDeclaredField("signerName")) From 1742e837c4f5ab0ed4ba89c7f131cf779a393ecb Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sat, 5 Apr 2014 01:51:08 -0700 Subject: [PATCH 4/4] Roll back sbt/sbt@482f99 (aggressive deletion of cache). Fixes #1091. * sbt 0.12.3 introduced sbt/sbt@482f99 to fix #637 and #641. * The underlying issue of #641 was an incorrect classifier, and it was fixed in sbt/sbt@718fa9 for #683 and shipped as sbt 0.13.0. --- ivy/src/main/scala/sbt/Ivy.scala | 36 -------------------------------- 1 file changed, 36 deletions(-) diff --git a/ivy/src/main/scala/sbt/Ivy.scala b/ivy/src/main/scala/sbt/Ivy.scala index 2408992e6..a6519c6bc 100644 --- a/ivy/src/main/scala/sbt/Ivy.scala +++ b/ivy/src/main/scala/sbt/Ivy.scala @@ -7,7 +7,6 @@ import Resolver.PluginPattern import java.io.File import java.net.URI -import java.text.ParseException import java.util.concurrent.Callable import java.util.{Collection, Collections => CS} import CS.singleton @@ -24,9 +23,7 @@ import core.settings.IvySettings import plugins.latest.LatestRevisionStrategy import plugins.matcher.PatternMatcher import plugins.parser.m2.PomModuleDescriptorParser -import plugins.repository.ResourceDownloader import plugins.resolver.{ChainResolver, DependencyResolver} -import plugins.resolver.util.ResolvedResource import util.{Message, MessageLogger} import util.extendable.ExtendableItem @@ -358,41 +355,8 @@ private object IvySbt case pr: ProjectResolver => true case _ => false } - /** This is overridden to delete outofdate artifacts of changing modules that are not listed in the metadata. - * This occurs for artifacts with classifiers, for example. */ - @throws(classOf[ParseException]) - override def cacheModuleDescriptor(resolver: DependencyResolver, mdRef: ResolvedResource, dd: DependencyDescriptor, moduleArtifact: IArtifact, downloader: ResourceDownloader, options: CacheMetadataOptions): ResolvedModuleRevision = - { - val rmrRaw = super.cacheModuleDescriptor(null, mdRef, dd, moduleArtifact, downloader, options) - val rmr = resetArtifactResolver(rmrRaw) - val mrid = moduleArtifact.getModuleRevisionId - def shouldClear(): Boolean = rmr != null && - ( (rmr.getReport != null && rmr.getReport.isSearched && isChanging(dd, mrid)) || - isProjectResolver(rmr.getResolver) ) - // only handle changing modules whose metadata actually changed. - // Typically, the publication date in the metadata has to change to get here. - if(shouldClear()) { - // this is the locally cached metadata as originally retrieved (e.g. the pom) - val original = rmr.getReport.getOriginalLocalFile - if(original != null) { - // delete all files in subdirectories that are older than the original metadata file's publication date - // The publication date is used because the metadata will be redownloaded for changing files, - // so the last modified time changes, but the publication date doesn't - val pubDate = rmrRaw.getPublicationDate - val lm = if(pubDate eq null) original.lastModified else pubDate.getTime - val indirectFiles = PathFinder(original.getParentFile).*(DirectoryFilter).**(-DirectoryFilter).get.toList - val older = indirectFiles.filter(f => f.lastModified < lm).toList - Message.verbose("Deleting additional old artifacts from cache for changed module " + mrid + older.mkString(":\n\t", "\n\t", "")) - IO.delete(older) - } - } - rmr - } // ignore the original resolver wherever possible to avoid issues like #704 override def saveResolvers(descriptor: ModuleDescriptor, metadataResolverName: String, artifactResolverName: String) {} - - def isChanging(dd: DependencyDescriptor, requestedRevisionId: ModuleRevisionId): Boolean = - !localOnly && (dd.isChanging || requestedRevisionId.getRevision.contains("-SNAPSHOT")) } manager.setArtifactPattern(PluginPattern + manager.getArtifactPattern) manager.setDataFilePattern(PluginPattern + manager.getDataFilePattern)