From 80ee35b01cbb180922cc7b4b05c61ccd53637ea5 Mon Sep 17 00:00:00 2001 From: jvican Date: Thu, 25 May 2017 22:09:12 +0200 Subject: [PATCH 1/8] Add custom checksum checks to ivy --- .../librarymanagement/ConvertResolver.scala | 74 +++++++++++++++++-- .../ivyint/SbtChainResolver.scala | 1 + .../src/test/scala/BaseIvySpecification.scala | 2 +- .../src/test/scala/OfflineModeSpec.scala | 2 +- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala index 82760272c..2a75124c5 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala @@ -5,10 +5,15 @@ package sbt.internal.librarymanagement import java.net.URL import java.util.Collections + import org.apache.ivy.core.module.descriptor.DependencyDescriptor import org.apache.ivy.core.resolve.ResolveData import org.apache.ivy.core.settings.IvySettings -import org.apache.ivy.plugins.repository.{ RepositoryCopyProgressListener, TransferEvent } +import org.apache.ivy.plugins.repository.{ + RepositoryCopyProgressListener, + Resource, + TransferEvent +} import org.apache.ivy.plugins.resolver.{ BasicResolver, DependencyResolver, @@ -24,9 +29,10 @@ import org.apache.ivy.plugins.resolver.{ URLResolver } import org.apache.ivy.plugins.repository.url.{ URLRepository => URLRepo } -import org.apache.ivy.plugins.repository.file.{ FileRepository => FileRepo, FileResource } -import java.io.{ IOException, File } -import org.apache.ivy.util.{ FileUtil, ChecksumHelper } +import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => FileRepo } +import java.io.{ File, IOException } + +import org.apache.ivy.util.{ ChecksumHelper, FileUtil, Message } import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact } import sbt.io.IO import sbt.util.Logger @@ -156,6 +162,7 @@ private[sbt] object ConvertResolver { extends IBiblioResolver with ChecksumFriendlyURLResolver with DescriptorRequired { + override def getResource(resource: Resource, dest: File): Long = get(resource, dest) def setPatterns(): Unit = { // done this way for access to protected methods. setArtifactPatterns(pattern) @@ -170,7 +177,9 @@ private[sbt] object ConvertResolver { resolver } case repo: SshRepository => { - val resolver = new SshResolver with DescriptorRequired + val resolver = new SshResolver with DescriptorRequired { + override def getResource(resource: Resource, dest: File): Long = get(resource, dest) + } initializeSSHResolver(resolver, repo, settings) repo.publishPermissions.foreach(perm => resolver.setPublishPermissions(perm)) resolver @@ -187,6 +196,7 @@ private[sbt] object ConvertResolver { // in local files for non-changing revisions. // This will be fully enforced in sbt 1.0. setRepository(new WarnOnOverwriteFileRepo()) + override def getResource(resource: Resource, dest: File): Long = get(resource, dest) } resolver.setName(repo.name) initializePatterns(resolver, repo.patterns, settings) @@ -196,7 +206,9 @@ private[sbt] object ConvertResolver { resolver } case repo: URLRepository => { - val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired + val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired { + override def getResource(resource: Resource, dest: File): Long = get(resource, dest) + } resolver.setName(repo.name) initializePatterns(resolver, repo.patterns, settings) resolver @@ -208,11 +220,59 @@ private[sbt] object ConvertResolver { } private sealed trait DescriptorRequired extends BasicResolver { + // Works around implementation restriction to access protected method `get` + def getResource(resource: Resource, dest: File): Long + + override def getAndCheck(resource: Resource, dest: File): Long = { + // Follows the same semantics that private method `check` as defined in ivy `BasicResolver` + def check(resource: Resource, destination: File, algorithm: String) = { + if (!ChecksumHelper.isKnownAlgorithm(algorithm)) { + throw new IllegalArgumentException(s"Unknown checksum algorithm: $algorithm") + } + val checksumResource = resource.clone(s"${resource.getName}.$algorithm") + if (checksumResource.exists) { + Message.debug(s"$algorithm file found for $resource: checking...") + val checksumFile = File.createTempFile("ivytmp", algorithm) + try { + getResource(checksumResource, checksumFile) + try { + ChecksumHelper.check(dest, checksumFile, algorithm) + Message.verbose(s"$algorithm OK for $resource") + true + } catch { + case e: IOException => + dest.delete() + throw e + } + } finally { + checksumFile.delete() + } + } else false + } + + val size = getResource(resource, dest) + val checksums = getChecksumAlgorithms + checksums.foldLeft(false) { (failed, checksum) => + // Continue checking until we hit a failure + if (failed) failed + else check(resource, dest, checksum) + } + size + } + var i = 0 override def getDependency(dd: DependencyDescriptor, data: ResolveData) = { + val moduleID = IvyRetrieve.toModuleID(dd.getDependencyRevisionId) + print(" " * i) + println(s"Downloading and checking for $moduleID") + i += 2 val prev = descriptorString(isAllownomd) setDescriptor(descriptorString(hasExplicitURL(dd))) - try super.getDependency(dd, data) + val t = try super.getDependency(dd, data) finally setDescriptor(prev) + i -= 2 + print(" " * i) + println(s"End $moduleID") + t } def descriptorString(optional: Boolean) = if (optional) BasicResolver.DESCRIPTOR_OPTIONAL else BasicResolver.DESCRIPTOR_REQUIRED diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala index a364b9d77..ad6245055 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala @@ -31,6 +31,7 @@ private[sbt] case class SbtChainResolver( updateOptions: UpdateOptions, log: Logger ) extends ChainResolver { + override def setCheckmodified(check: Boolean): Unit = super.setCheckmodified(check) override def equals(o: Any): Boolean = o match { case o: SbtChainResolver => diff --git a/librarymanagement/src/test/scala/BaseIvySpecification.scala b/librarymanagement/src/test/scala/BaseIvySpecification.scala index d87baa862..6068496d6 100644 --- a/librarymanagement/src/test/scala/BaseIvySpecification.scala +++ b/librarymanagement/src/test/scala/BaseIvySpecification.scala @@ -53,8 +53,8 @@ trait BaseIvySpecification extends UnitSpec { def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = { val paths = IvyPaths(currentBase, Some(currentTarget)) val other = Vector.empty - val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) val check = Vector.empty + val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) val resCacheDir = currentTarget / "resolution-cache" new InlineIvyConfiguration(paths, resolvers, diff --git a/librarymanagement/src/test/scala/OfflineModeSpec.scala b/librarymanagement/src/test/scala/OfflineModeSpec.scala index 0b941f405..04d8cf836 100644 --- a/librarymanagement/src/test/scala/OfflineModeSpec.scala +++ b/librarymanagement/src/test/scala/OfflineModeSpec.scala @@ -44,7 +44,7 @@ class OfflineModeSpec extends BaseIvySpecification with DependencyBuilders { val offlineResolution = IvyActions.updateEither(toResolve, offlineConf, warningConf, noClock, targetDir, log) - assert(offlineResolution.isRight) + assert(offlineResolution.isRight, s"Offline resolution has failed with $offlineResolution.") val resolveTime = offlineResolution.right.get.stats.resolveTime // Only check the estimate for the non cached resolution, otherwise resolution is cached From 7500291dfc71d02e8c7e9cea1a6b82faa7012ce3 Mon Sep 17 00:00:00 2001 From: jvican Date: Thu, 25 May 2017 22:15:46 +0200 Subject: [PATCH 2/8] Fix bug in updateOptionsFormat --- .../librarymanagement/formats/UpdateOptionsFormat.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala index 5074e8a0e..832af13ad 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala @@ -35,7 +35,7 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re xs._3, xs._4, xs._5, - ConvertResolver.defaultConvert, + PartialFunction.empty, xs._6 ) ) From 892f6e3775456b6d74021e18b60a2f4cae84648b Mon Sep 17 00:00:00 2001 From: jvican Date: Thu, 25 May 2017 23:30:00 +0200 Subject: [PATCH 3/8] Add checksum to ModuleID A checksum in a `ModuleID` ensures that the checksum of the resolved module id matches. This checksum check is done at the same time ivy checks it. --- .../main/contraband/librarymanagement.json | 31 +++++++++++++------ .../sbt/librarymanagement/ModuleIDExtra.scala | 7 +++-- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/librarymanagement/src/main/contraband/librarymanagement.json b/librarymanagement/src/main/contraband/librarymanagement.json index d28da3b69..d098603d8 100644 --- a/librarymanagement/src/main/contraband/librarymanagement.json +++ b/librarymanagement/src/main/contraband/librarymanagement.json @@ -267,22 +267,33 @@ { "name": "organization", "type": "String" }, { "name": "name", "type": "String" }, { "name": "revision", "type": "String" }, - { "name": "configurations", "type": "Option[String]", "default": "None", "since": "0.0.1" }, - { "name": "isChanging", "type": "boolean", "default": "false", "since": "0.0.1" }, - { "name": "isTransitive", "type": "boolean", "default": "true", "since": "0.0.1" }, - { "name": "isForce", "type": "boolean", "default": "false", "since": "0.0.1" }, - { "name": "explicitArtifacts", "type": "sbt.librarymanagement.Artifact*", "default": "Vector.empty", "since": "0.0.1" }, - { "name": "inclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" }, - { "name": "exclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" }, - { "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" }, - { "name": "crossVersion", "type": "sbt.librarymanagement.CrossVersion", "default": "sbt.librarymanagement.Disabled()", "since": "0.0.1" }, - { "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" } + { "name": "configurations", "type": "Option[String]", "default": "None", "since": "0.0.1" }, + { "name": "isChanging", "type": "boolean", "default": "false", "since": "0.0.1" }, + { "name": "isTransitive", "type": "boolean", "default": "true", "since": "0.0.1" }, + { "name": "isForce", "type": "boolean", "default": "false", "since": "0.0.1" }, + { "name": "explicitArtifacts", "type": "sbt.librarymanagement.Artifact*", "default": "Vector.empty", "since": "0.0.1" }, + { "name": "inclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" }, + { "name": "exclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" }, + { "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" }, + { "name": "crossVersion", "type": "sbt.librarymanagement.CrossVersion", "default": "sbt.librarymanagement.Disabled()", "since": "0.0.1" }, + { "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" }, + { "name": "checksum", "type": "Option[sbt.librarymanagement.Checksum]", "default": "None", "since": "0.0.1" } ], "toString": [ "this.toStringImpl" ], "parentsCompanion": "sbt.librarymanagement.ModuleIDFunctions" }, + { + "name": "Checksum", + "namespace": "sbt.librarymanagement", + "target": "Scala", + "type": "record", + "fields": [ + { "name": "digest", "type": "String" }, + { "name": "type", "type": "String", "default": "sha1" } + ] + }, { "name": "ModuleInfo", "namespace": "sbt.librarymanagement", diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala index 5b89b5419..e52df5a4d 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala @@ -22,6 +22,7 @@ abstract class ModuleIDExtra { def extraAttributes: Map[String, String] def crossVersion: CrossVersion def branchName: Option[String] + def checksum: Option[Checksum] protected[this] def copy( organization: String = organization, @@ -36,12 +37,14 @@ abstract class ModuleIDExtra { exclusions: Vector[ExclusionRule] = exclusions, extraAttributes: Map[String, String] = extraAttributes, crossVersion: CrossVersion = crossVersion, - branchName: Option[String] = branchName + branchName: Option[String] = branchName, + checksum: Option[Checksum] = checksum ): ModuleID protected def toStringImpl: String = s"""$organization:$name:$revision""" + - (configurations match { case Some(s) => ":" + s; case None => "" }) + { + (checksum match { case Some(s) => s": ${s.`type`} ${s.digest}"; case None => "" }) + + (configurations match { case Some(s) => ":" + s; case None => "" }) + { val attr = attributeString if (attr == "") "" else " " + attr From 4c8036c7cb780d1209f997daf1016462b1c290b7 Mon Sep 17 00:00:00 2001 From: jvican Date: Fri, 26 May 2017 10:44:24 +0200 Subject: [PATCH 4/8] Add `managedChecksums` to `UpdateOptions` --- .../formats/UpdateOptionsFormat.scala | 6 ++++-- .../scala/sbt/librarymanagement/UpdateOptions.scala | 10 ++++++++++ .../src/test/scala/BaseIvySpecification.scala | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala index 832af13ad..217beb77f 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala @@ -26,17 +26,19 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re uo.latestSnapshots, uo.consolidatedResolution, uo.cachedResolution, + uo.managedChecksums, uo.moduleResolvers ), - (xs: (String, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => + (xs: (String, Boolean, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => new UpdateOptions( levels(xs._1), xs._2, xs._3, xs._4, xs._5, + xs._6, PartialFunction.empty, - xs._6 + xs._7 ) ) diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala index 4c3810055..7253f0edc 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala @@ -22,6 +22,8 @@ final class UpdateOptions private[sbt] ( val consolidatedResolution: Boolean, // If set to true, use cached resolution. val cachedResolution: Boolean, + // If set to true, use cached resolution. + val managedChecksums: Boolean, // Extension point for an alternative resolver converter. val resolverConverter: UpdateOptions.ResolverConverter, // Map the unique resolver to be checked for the module ID @@ -54,12 +56,16 @@ final class UpdateOptions private[sbt] ( def withModuleResolvers(moduleResolvers: Map[ModuleID, Resolver]): UpdateOptions = copy(moduleResolvers = moduleResolvers) + def withManagedChecksums(managedChecksums: Boolean): UpdateOptions = + copy(managedChecksums = managedChecksums) + private[sbt] def copy( circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel, interProjectFirst: Boolean = this.interProjectFirst, latestSnapshots: Boolean = this.latestSnapshots, consolidatedResolution: Boolean = this.consolidatedResolution, cachedResolution: Boolean = this.cachedResolution, + managedChecksums: Boolean = this.managedChecksums, resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter, moduleResolvers: Map[ModuleID, Resolver] = this.moduleResolvers ): UpdateOptions = @@ -69,6 +75,7 @@ final class UpdateOptions private[sbt] ( latestSnapshots, consolidatedResolution, cachedResolution, + managedChecksums, resolverConverter, moduleResolvers ) @@ -79,6 +86,7 @@ final class UpdateOptions private[sbt] ( this.interProjectFirst == o.interProjectFirst && this.latestSnapshots == o.latestSnapshots && this.cachedResolution == o.cachedResolution && + this.managedChecksums == o.managedChecksums && this.resolverConverter == o.resolverConverter && this.moduleResolvers == o.moduleResolvers case _ => false @@ -90,6 +98,7 @@ final class UpdateOptions private[sbt] ( hash = hash * 31 + this.interProjectFirst.## hash = hash * 31 + this.latestSnapshots.## hash = hash * 31 + this.cachedResolution.## + hash = hash * 31 + this.managedChecksums.## hash = hash * 31 + this.resolverConverter.## hash = hash * 31 + this.moduleResolvers.## hash @@ -106,6 +115,7 @@ object UpdateOptions { latestSnapshots = true, consolidatedResolution = false, cachedResolution = false, + managedChecksums = false, resolverConverter = PartialFunction.empty, moduleResolvers = Map.empty ) diff --git a/librarymanagement/src/test/scala/BaseIvySpecification.scala b/librarymanagement/src/test/scala/BaseIvySpecification.scala index 6068496d6..7ab89de56 100644 --- a/librarymanagement/src/test/scala/BaseIvySpecification.scala +++ b/librarymanagement/src/test/scala/BaseIvySpecification.scala @@ -53,7 +53,7 @@ trait BaseIvySpecification extends UnitSpec { def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = { val paths = IvyPaths(currentBase, Some(currentTarget)) val other = Vector.empty - val check = Vector.empty + val check = IvySbt.DefaultChecksums.headOption.toVector val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) val resCacheDir = currentTarget / "resolution-cache" new InlineIvyConfiguration(paths, From 7bf60557d9d756bfeb6bb59e69e04ec1f1404c19 Mon Sep 17 00:00:00 2001 From: jvican Date: Fri, 26 May 2017 12:10:29 +0200 Subject: [PATCH 5/8] Implement `managedChecksums` in ivy Managed checksums tells ivy to forget about checking checksums for jar files and just persist them in the cache. The user that enables that option will take care of verifying they are correct. This is a big deal because: 1. Ivy takes *a lot of time* checking timestamps for big jars, and does it sequentially. The user (sbt) can do better by running these checks in parallel, speeding up the whole resolution process! 2. The fact that the sha files are not present in the cache means that build tools cannot check if a checksum is correct without preprocessing the jar. Note that the user is responsible for keeping the cache consistent. If the checksum is not correct, the user should report it and overwrite the checksum file with the valid digest. --- .../librarymanagement/ConvertResolver.scala | 100 ++++++++++-------- .../librarymanagement/IvyActions.scala | 22 +++- .../sbt/librarymanagement/UpdateOptions.scala | 5 +- .../src/test/scala/BaseIvySpecification.scala | 2 +- .../src/test/scala/OfflineModeSpec.scala | 2 +- .../ManagedChecksumsSpec.scala | 75 +++++++++++++ 6 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala index 2a75124c5..839083dd7 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala @@ -141,18 +141,24 @@ private[sbt] object ConvertResolver { def apply(r: Resolver, settings: IvySettings, log: Logger): DependencyResolver = apply(r, settings, UpdateOptions(), log) + private[librarymanagement] val ManagedChecksums = "managedChecksums" + /** Converts the given sbt resolver into an Ivy resolver. */ def apply( r: Resolver, settings: IvySettings, updateOptions: UpdateOptions, log: Logger - ): DependencyResolver = + ): DependencyResolver = { + // Pass in to the resolver converter the update options via ivy settings + settings.setVariable(ManagedChecksums, updateOptions.managedChecksums.toString) (updateOptions.resolverConverter orElse defaultConvert)((r, settings, log)) + } /** The default implementation of converter. */ lazy val defaultConvert: ResolverConverter = { case (r, settings, log) => + val managedChecksums = settings.getVariable(ManagedChecksums).toBoolean r match { case repo: MavenRepository => { val pattern = Collections.singletonList( @@ -162,6 +168,7 @@ private[sbt] object ConvertResolver { extends IBiblioResolver with ChecksumFriendlyURLResolver with DescriptorRequired { + override val managedChecksumsEnabled: Boolean = managedChecksums override def getResource(resource: Resource, dest: File): Long = get(resource, dest) def setPatterns(): Unit = { // done this way for access to protected methods. @@ -178,6 +185,7 @@ private[sbt] object ConvertResolver { } case repo: SshRepository => { val resolver = new SshResolver with DescriptorRequired { + override val managedChecksumsEnabled: Boolean = managedChecksums override def getResource(resource: Resource, dest: File): Long = get(resource, dest) } initializeSSHResolver(resolver, repo, settings) @@ -196,6 +204,7 @@ private[sbt] object ConvertResolver { // in local files for non-changing revisions. // This will be fully enforced in sbt 1.0. setRepository(new WarnOnOverwriteFileRepo()) + override val managedChecksumsEnabled: Boolean = managedChecksums override def getResource(resource: Resource, dest: File): Long = get(resource, dest) } resolver.setName(repo.name) @@ -207,6 +216,7 @@ private[sbt] object ConvertResolver { } case repo: URLRepository => { val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired { + override val managedChecksumsEnabled: Boolean = managedChecksums override def getResource(resource: Resource, dest: File): Long = get(resource, dest) } resolver.setName(repo.name) @@ -223,55 +233,61 @@ private[sbt] object ConvertResolver { // Works around implementation restriction to access protected method `get` def getResource(resource: Resource, dest: File): Long - override def getAndCheck(resource: Resource, dest: File): Long = { - // Follows the same semantics that private method `check` as defined in ivy `BasicResolver` - def check(resource: Resource, destination: File, algorithm: String) = { - if (!ChecksumHelper.isKnownAlgorithm(algorithm)) { - throw new IllegalArgumentException(s"Unknown checksum algorithm: $algorithm") - } - val checksumResource = resource.clone(s"${resource.getName}.$algorithm") - if (checksumResource.exists) { - Message.debug(s"$algorithm file found for $resource: checking...") - val checksumFile = File.createTempFile("ivytmp", algorithm) - try { - getResource(checksumResource, checksumFile) - try { - ChecksumHelper.check(dest, checksumFile, algorithm) - Message.verbose(s"$algorithm OK for $resource") - true - } catch { - case e: IOException => - dest.delete() - throw e - } - } finally { - checksumFile.delete() - } - } else false - } + /** + * Defines an option to tell ivy to disable checksums when downloading and + * let the user handle verifying these checksums. + * + * This means that the checksums are stored in the ivy cache directory. This + * is good for reproducibility from outside ivy. Sbt can check that jars are + * not corrupted, ever, independently of trusting whatever it's there in the + * local directory. + */ + def managedChecksumsEnabled: Boolean - val size = getResource(resource, dest) - val checksums = getChecksumAlgorithms - checksums.foldLeft(false) { (failed, checksum) => - // Continue checking until we hit a failure - if (failed) failed - else check(resource, dest, checksum) + import sbt.io.syntax._ + private def downloadChecksum(resource: Resource, + target: File, + targetChecksumFile: File, + algorithm: String): Boolean = { + if (!ChecksumHelper.isKnownAlgorithm(algorithm)) + throw new IllegalArgumentException(s"Unknown checksum algorithm: $algorithm") + + val checksumResource = resource.clone(s"${resource.getName}.$algorithm") + if (!checksumResource.exists) false + else { + Message.debug(s"$algorithm file found for $resource: downloading...") + // Resource must be cleaned up outside of this function if it's invalid + getResource(checksumResource, targetChecksumFile) + true } - size } - var i = 0 + + private final val PartEnd = ".part" + private final val JarEnd = ".jar" + private final val TemporaryJar = JarEnd + PartEnd + override def getAndCheck(resource: Resource, target: File): Long = { + val targetPath = target.getAbsolutePath + if (!managedChecksumsEnabled || !targetPath.endsWith(TemporaryJar)) { + super.getAndCheck(resource, target) + } else { + // This is where we differ from ivy behaviour + val size = getResource(resource, target) + val checksumAlgorithms = getChecksumAlgorithms + checksumAlgorithms.foldLeft(false) { (checked, algorithm) => + // Continue checking until we hit a failure + val checksumFile = new File(targetPath.stripSuffix(PartEnd) + s".$algorithm") + if (checked) checked + else downloadChecksum(resource, target, checksumFile, algorithm) + } + size + } + } + override def getDependency(dd: DependencyDescriptor, data: ResolveData) = { - val moduleID = IvyRetrieve.toModuleID(dd.getDependencyRevisionId) - print(" " * i) - println(s"Downloading and checking for $moduleID") - i += 2 val prev = descriptorString(isAllownomd) setDescriptor(descriptorString(hasExplicitURL(dd))) val t = try super.getDependency(dd, data) finally setDescriptor(prev) - i -= 2 - print(" " * i) - println(s"End $moduleID") t } def descriptorString(optional: Boolean) = diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyActions.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyActions.scala index 638aa4770..d989d2cd4 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyActions.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyActions.scala @@ -540,6 +540,7 @@ object IvyActions { report: UpdateReport, config: RetrieveConfiguration ): UpdateReport = { + val copyChecksums = ivy.getVariable(ConvertResolver.ManagedChecksums).toBoolean val toRetrieve = config.configurationsToRetrieve val base = config.retrieveDirectory val pattern = config.outputPattern @@ -551,9 +552,9 @@ object IvyActions { val toCopy = new collection.mutable.HashSet[(File, File)] val retReport = report retrieve { (conf, mid, art, cached) => configurationNames match { - case None => performRetrieve(conf, mid, art, base, pattern, cached, toCopy) + case None => performRetrieve(conf, mid, art, base, pattern, cached, copyChecksums, toCopy) case Some(names) if names(conf) => - performRetrieve(conf, mid, art, base, pattern, cached, toCopy) + performRetrieve(conf, mid, art, base, pattern, cached, copyChecksums, toCopy) case _ => cached } } @@ -577,10 +578,27 @@ object IvyActions { base: File, pattern: String, cached: File, + copyChecksums: Boolean, toCopy: collection.mutable.HashSet[(File, File)] ): File = { val to = retrieveTarget(conf, mid, art, base, pattern) toCopy += ((cached, to)) + + if (copyChecksums) { + // Copy over to the lib managed directory any checksum for a jar if it exists + // TODO(jvican): Support user-provided checksums + val cachePath = cached.getAbsolutePath + IvySbt.DefaultChecksums.foreach { checksum => + if (cachePath.endsWith(".jar")) { + val cacheChecksum = new File(s"$cachePath.$checksum") + if (cacheChecksum.exists()) { + val toChecksum = new File(s"${to.getAbsolutePath}.$checksum") + toCopy += ((cacheChecksum, toChecksum)) + } + } + } + } + to } diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala index 7253f0edc..bc210670f 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala @@ -9,6 +9,9 @@ import sbt.util.Logger * While UpdateConfiguration is passed into update at runtime, * UpdateOption is intended to be used while setting up the Ivy object. * + * @param managedChecksums Managed checksums tells ivy whether it should only download the + * checksum files and let the caller handle the verification. + * * See also UpdateConfiguration in IvyActions.scala. */ final class UpdateOptions private[sbt] ( @@ -22,7 +25,7 @@ final class UpdateOptions private[sbt] ( val consolidatedResolution: Boolean, // If set to true, use cached resolution. val cachedResolution: Boolean, - // If set to true, use cached resolution. + // If set to true, use managed checksums. val managedChecksums: Boolean, // Extension point for an alternative resolver converter. val resolverConverter: UpdateOptions.ResolverConverter, diff --git a/librarymanagement/src/test/scala/BaseIvySpecification.scala b/librarymanagement/src/test/scala/BaseIvySpecification.scala index 7ab89de56..6068496d6 100644 --- a/librarymanagement/src/test/scala/BaseIvySpecification.scala +++ b/librarymanagement/src/test/scala/BaseIvySpecification.scala @@ -53,7 +53,7 @@ trait BaseIvySpecification extends UnitSpec { def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = { val paths = IvyPaths(currentBase, Some(currentTarget)) val other = Vector.empty - val check = IvySbt.DefaultChecksums.headOption.toVector + val check = Vector.empty val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) val resCacheDir = currentTarget / "resolution-cache" new InlineIvyConfiguration(paths, diff --git a/librarymanagement/src/test/scala/OfflineModeSpec.scala b/librarymanagement/src/test/scala/OfflineModeSpec.scala index 04d8cf836..ea6b5f1d7 100644 --- a/librarymanagement/src/test/scala/OfflineModeSpec.scala +++ b/librarymanagement/src/test/scala/OfflineModeSpec.scala @@ -3,7 +3,7 @@ package sbt.librarymanagement import org.scalatest.Assertion import sbt.internal.librarymanagement._ import sbt.internal.librarymanagement.impl.DependencyBuilders -import sbt.io.IO +import sbt.io.{ FileFilter, IO, Path } class OfflineModeSpec extends BaseIvySpecification with DependencyBuilders { private final def targetDir = Some(currentDependency) diff --git a/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala b/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala new file mode 100644 index 000000000..9dc0649de --- /dev/null +++ b/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala @@ -0,0 +1,75 @@ +package sbt.librarymanagement + +import java.io.File + +import org.apache.ivy.util.Message +import org.scalatest.Assertion +import sbt.internal.librarymanagement.{ + BaseIvySpecification, + InlineIvyConfiguration, + IvyActions, + IvyConfiguration, + IvyPaths, + IvySbt, + LogicalClock, + UnresolvedWarningConfiguration +} +import sbt.internal.librarymanagement.impl.DependencyBuilders +import sbt.io.IO + +class ManagedChecksumsSpec extends BaseIvySpecification with DependencyBuilders { + private final def targetDir = Some(currentDependency) + private final def onlineConf = makeUpdateConfiguration(false) + private final def warningConf = UnresolvedWarningConfiguration() + private final def noClock = LogicalClock.unknown + private final val Checksum = "sha1" + + def avro177 = ModuleID("org.apache.avro", "avro", "1.7.7") + def dataAvro1940 = ModuleID("com.linkedin.pegasus", "data-avro", "1.9.40") + def netty320 = ModuleID("org.jboss.netty", "netty", "3.2.0.Final") + final def dependencies: Vector[ModuleID] = + Vector(avro177, dataAvro1940, netty320).map(_.withConfigurations(Some("compile"))) + + import sbt.io.syntax._ + override def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = { + val paths = IvyPaths(currentBase, Some(currentTarget)) + val other = Vector.empty + val check = Vector(Checksum) + val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) + val resCacheDir = currentTarget / "resolution-cache" + new InlineIvyConfiguration(paths, + resolvers, + other, + moduleConfs, + None, + check, + Some(resCacheDir), + uo, + log) + } + + def cleanAll(): Unit = { + cleanIvyCache() + IO.delete(currentTarget) + IO.delete(currentManaged) + IO.delete(currentDependency) + } + + def assertChecksumExists(file: File) = { + val shaFile = new File(file.getAbsolutePath + s".$Checksum") + Message.info(s"Checking $shaFile exists...") + assert(shaFile.exists(), s"The checksum $Checksum for $file does not exist") + } + + "Managed checksums" should "should download the checksum files" in { + cleanAll() + val updateOptions = UpdateOptions().withManagedChecksums(true) + val toResolve = module(defaultModuleId, dependencies, None, updateOptions) + val res = IvyActions.updateEither(toResolve, onlineConf, warningConf, noClock, targetDir, log) + assert(res.isRight, s"Resolution with managed checksums failed! $res") + val updateReport = res.right.get + val allModuleReports = updateReport.configurations.flatMap(_.modules) + val allArtifacts: Seq[File] = allModuleReports.flatMap(_.artifacts.map(_._2)) + allArtifacts.foreach(assertChecksumExists) + } +} From e3a52c3e1f27ca4a63be22ddf9c41d705c75dfd9 Mon Sep 17 00:00:00 2001 From: jvican Date: Fri, 26 May 2017 12:26:03 +0200 Subject: [PATCH 6/8] Make explicit where we deviate from ivy behaviour --- .../sbt/internal/librarymanagement/ConvertResolver.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala index 839083dd7..c7df8a641 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala @@ -265,12 +265,13 @@ private[sbt] object ConvertResolver { private final val PartEnd = ".part" private final val JarEnd = ".jar" private final val TemporaryJar = JarEnd + PartEnd + override def getAndCheck(resource: Resource, target: File): Long = { val targetPath = target.getAbsolutePath if (!managedChecksumsEnabled || !targetPath.endsWith(TemporaryJar)) { super.getAndCheck(resource, target) } else { - // This is where we differ from ivy behaviour + // +ivy deviation val size = getResource(resource, target) val checksumAlgorithms = getChecksumAlgorithms checksumAlgorithms.foldLeft(false) { (checked, algorithm) => @@ -279,6 +280,7 @@ private[sbt] object ConvertResolver { if (checked) checked else downloadChecksum(resource, target, checksumFile, algorithm) } + // -ivy deviation size } } From 27bf1308532b03b44b3b13471b38eb9feb5b4d43 Mon Sep 17 00:00:00 2001 From: jvican Date: Fri, 26 May 2017 20:10:53 +0200 Subject: [PATCH 7/8] Move `managedChecksums` to InlineIvyConfiguration `UpdateOptions` is only meant for user-provided options. Since this option should not be exposed to users, it's moved to inline ivy configuration, whose parameters require information about the build and is usually defined project-wide instead of `Global`-wide. --- .../src/main/contraband/librarymanagement.json | 4 +++- .../librarymanagement/ConvertResolver.scala | 7 ++----- .../scala/sbt/internal/librarymanagement/Ivy.scala | 1 + .../sbt/internal/librarymanagement/IvyCache.scala | 1 + .../formats/UpdateOptionsFormat.scala | 6 ++---- .../scala/sbt/librarymanagement/UpdateOptions.scala | 13 ------------- .../src/test/scala/BaseIvySpecification.scala | 2 ++ .../src/test/scala/CustomPomParserTest.scala | 1 + .../librarymanagement/ManagedChecksumsSpec.scala | 3 ++- 9 files changed, 14 insertions(+), 24 deletions(-) diff --git a/librarymanagement/src/main/contraband/librarymanagement.json b/librarymanagement/src/main/contraband/librarymanagement.json index d098603d8..06dea8671 100644 --- a/librarymanagement/src/main/contraband/librarymanagement.json +++ b/librarymanagement/src/main/contraband/librarymanagement.json @@ -742,6 +742,7 @@ { "name": "otherResolvers", "type": "sbt.librarymanagement.Resolver*" }, { "name": "moduleConfigurations", "type": "sbt.librarymanagement.ModuleConfiguration*" }, { "name": "checksums", "type": "String*" }, + { "name": "managedChecksums", "type": "Boolean" }, { "name": "resolutionCacheDir", "type": "java.io.File?" } ], "extra": [ @@ -752,12 +753,13 @@ " moduleConfigurations: Vector[sbt.librarymanagement.ModuleConfiguration],", " lock: Option[xsbti.GlobalLock],", " checksums: Vector[String],", + " managedChecksums: Boolean,", " resolutionCacheDir: Option[java.io.File],", " updateOptions: sbt.librarymanagement.UpdateOptions,", " log: xsbti.Logger", ") =", " this(lock, paths.baseDirectory, log, updateOptions, paths, resolvers, otherResolvers,", - " moduleConfigurations, checksums, resolutionCacheDir)" + " moduleConfigurations, checksums, managedChecksums, resolutionCacheDir)" ] }, { diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala index c7df8a641..bba247e80 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala @@ -141,7 +141,7 @@ private[sbt] object ConvertResolver { def apply(r: Resolver, settings: IvySettings, log: Logger): DependencyResolver = apply(r, settings, UpdateOptions(), log) - private[librarymanagement] val ManagedChecksums = "managedChecksums" + private[librarymanagement] val ManagedChecksums = "sbt.managedChecksums" /** Converts the given sbt resolver into an Ivy resolver. */ def apply( @@ -149,11 +149,8 @@ private[sbt] object ConvertResolver { settings: IvySettings, updateOptions: UpdateOptions, log: Logger - ): DependencyResolver = { - // Pass in to the resolver converter the update options via ivy settings - settings.setVariable(ManagedChecksums, updateOptions.managedChecksums.toString) + ): DependencyResolver = (updateOptions.resolverConverter orElse defaultConvert)((r, settings, log)) - } /** The default implementation of converter. */ lazy val defaultConvert: ResolverConverter = { diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala index e27eb1907..38bba8076 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala @@ -87,6 +87,7 @@ final class IvySbt(val configuration: IvyConfiguration) { self => IvySbt.loadURI(is, e.uri) case i: InlineIvyConfiguration => is.setVariable("ivy.checksums", i.checksums mkString ",") + is.setVariable(ConvertResolver.ManagedChecksums, i.managedChecksums.toString) i.paths.ivyHome foreach is.setDefaultIvyUserDir val log = configuration.log IvySbt.configureCache(is, i.resolutionCacheDir) diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyCache.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyCache.scala index bef341acc..5a17a2d82 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyCache.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyCache.scala @@ -110,6 +110,7 @@ class IvyCache(val ivyHome: Option[File]) { Vector.empty, lock, IvySbt.DefaultChecksums, + false, None, UpdateOptions(), log diff --git a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala index 217beb77f..832af13ad 100644 --- a/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala +++ b/librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala @@ -26,19 +26,17 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re uo.latestSnapshots, uo.consolidatedResolution, uo.cachedResolution, - uo.managedChecksums, uo.moduleResolvers ), - (xs: (String, Boolean, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => + (xs: (String, Boolean, Boolean, Boolean, Boolean, Map[ModuleID, Resolver])) => new UpdateOptions( levels(xs._1), xs._2, xs._3, xs._4, xs._5, - xs._6, PartialFunction.empty, - xs._7 + xs._6 ) ) diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala index bc210670f..4c3810055 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/UpdateOptions.scala @@ -9,9 +9,6 @@ import sbt.util.Logger * While UpdateConfiguration is passed into update at runtime, * UpdateOption is intended to be used while setting up the Ivy object. * - * @param managedChecksums Managed checksums tells ivy whether it should only download the - * checksum files and let the caller handle the verification. - * * See also UpdateConfiguration in IvyActions.scala. */ final class UpdateOptions private[sbt] ( @@ -25,8 +22,6 @@ final class UpdateOptions private[sbt] ( val consolidatedResolution: Boolean, // If set to true, use cached resolution. val cachedResolution: Boolean, - // If set to true, use managed checksums. - val managedChecksums: Boolean, // Extension point for an alternative resolver converter. val resolverConverter: UpdateOptions.ResolverConverter, // Map the unique resolver to be checked for the module ID @@ -59,16 +54,12 @@ final class UpdateOptions private[sbt] ( def withModuleResolvers(moduleResolvers: Map[ModuleID, Resolver]): UpdateOptions = copy(moduleResolvers = moduleResolvers) - def withManagedChecksums(managedChecksums: Boolean): UpdateOptions = - copy(managedChecksums = managedChecksums) - private[sbt] def copy( circularDependencyLevel: CircularDependencyLevel = this.circularDependencyLevel, interProjectFirst: Boolean = this.interProjectFirst, latestSnapshots: Boolean = this.latestSnapshots, consolidatedResolution: Boolean = this.consolidatedResolution, cachedResolution: Boolean = this.cachedResolution, - managedChecksums: Boolean = this.managedChecksums, resolverConverter: UpdateOptions.ResolverConverter = this.resolverConverter, moduleResolvers: Map[ModuleID, Resolver] = this.moduleResolvers ): UpdateOptions = @@ -78,7 +69,6 @@ final class UpdateOptions private[sbt] ( latestSnapshots, consolidatedResolution, cachedResolution, - managedChecksums, resolverConverter, moduleResolvers ) @@ -89,7 +79,6 @@ final class UpdateOptions private[sbt] ( this.interProjectFirst == o.interProjectFirst && this.latestSnapshots == o.latestSnapshots && this.cachedResolution == o.cachedResolution && - this.managedChecksums == o.managedChecksums && this.resolverConverter == o.resolverConverter && this.moduleResolvers == o.moduleResolvers case _ => false @@ -101,7 +90,6 @@ final class UpdateOptions private[sbt] ( hash = hash * 31 + this.interProjectFirst.## hash = hash * 31 + this.latestSnapshots.## hash = hash * 31 + this.cachedResolution.## - hash = hash * 31 + this.managedChecksums.## hash = hash * 31 + this.resolverConverter.## hash = hash * 31 + this.moduleResolvers.## hash @@ -118,7 +106,6 @@ object UpdateOptions { latestSnapshots = true, consolidatedResolution = false, cachedResolution = false, - managedChecksums = false, resolverConverter = PartialFunction.empty, moduleResolvers = Map.empty ) diff --git a/librarymanagement/src/test/scala/BaseIvySpecification.scala b/librarymanagement/src/test/scala/BaseIvySpecification.scala index 6068496d6..5fa82c305 100644 --- a/librarymanagement/src/test/scala/BaseIvySpecification.scala +++ b/librarymanagement/src/test/scala/BaseIvySpecification.scala @@ -54,6 +54,7 @@ trait BaseIvySpecification extends UnitSpec { val paths = IvyPaths(currentBase, Some(currentTarget)) val other = Vector.empty val check = Vector.empty + val managedChecksums = false val moduleConfs = Vector(ModuleConfiguration("*", chainResolver)) val resCacheDir = currentTarget / "resolution-cache" new InlineIvyConfiguration(paths, @@ -62,6 +63,7 @@ trait BaseIvySpecification extends UnitSpec { moduleConfs, None, check, + managedChecksums, Some(resCacheDir), uo, log) diff --git a/librarymanagement/src/test/scala/CustomPomParserTest.scala b/librarymanagement/src/test/scala/CustomPomParserTest.scala index 950562638..488f683a8 100644 --- a/librarymanagement/src/test/scala/CustomPomParserTest.scala +++ b/librarymanagement/src/test/scala/CustomPomParserTest.scala @@ -21,6 +21,7 @@ class CustomPomParserTest extends UnitSpec { Vector.empty, None, Vector("sha1", "md5"), + false, None, UpdateOptions(), log) diff --git a/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala b/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala index 9dc0649de..493d600e5 100644 --- a/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala +++ b/librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala @@ -43,6 +43,7 @@ class ManagedChecksumsSpec extends BaseIvySpecification with DependencyBuilders moduleConfs, None, check, + managedChecksums = true, Some(resCacheDir), uo, log) @@ -63,7 +64,7 @@ class ManagedChecksumsSpec extends BaseIvySpecification with DependencyBuilders "Managed checksums" should "should download the checksum files" in { cleanAll() - val updateOptions = UpdateOptions().withManagedChecksums(true) + val updateOptions = UpdateOptions() val toResolve = module(defaultModuleId, dependencies, None, updateOptions) val res = IvyActions.updateEither(toResolve, onlineConf, warningConf, noClock, targetDir, log) assert(res.isRight, s"Resolution with managed checksums failed! $res") From 9d0dfce8696d1ba73d4b2e42905da9ba64927892 Mon Sep 17 00:00:00 2001 From: jvican Date: Sat, 27 May 2017 23:48:21 +0200 Subject: [PATCH 8/8] Move checksum to artifact This is a more elegant solution than before since module IDs can bring different artifacts (in updateClassifiers, for instance). --- .../src/main/contraband/librarymanagement.json | 6 +++--- .../scala/sbt/librarymanagement/ArtifactExtra.scala | 10 ++++++---- .../scala/sbt/librarymanagement/ModuleIDExtra.scala | 7 ++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/librarymanagement/src/main/contraband/librarymanagement.json b/librarymanagement/src/main/contraband/librarymanagement.json index 06dea8671..1ea41015e 100644 --- a/librarymanagement/src/main/contraband/librarymanagement.json +++ b/librarymanagement/src/main/contraband/librarymanagement.json @@ -15,7 +15,8 @@ { "name": "classifier", "type": "Option[String]", "default": "None", "since": "0.0.1" }, { "name": "configurations", "type": "sbt.librarymanagement.Configuration*", "default": "Vector.empty", "since": "0.0.1" }, { "name": "url", "type": "Option[java.net.URL]", "default": "None", "since": "0.0.1" }, - { "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" } + { "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" }, + { "name": "checksum", "type": "Option[sbt.librarymanagement.Checksum]", "default": "None", "since": "0.0.1" } ], "parentsCompanion": "sbt.librarymanagement.ArtifactFunctions" }, @@ -276,8 +277,7 @@ { "name": "exclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" }, { "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" }, { "name": "crossVersion", "type": "sbt.librarymanagement.CrossVersion", "default": "sbt.librarymanagement.Disabled()", "since": "0.0.1" }, - { "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" }, - { "name": "checksum", "type": "Option[sbt.librarymanagement.Checksum]", "default": "None", "since": "0.0.1" } + { "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" } ], "toString": [ "this.toStringImpl" diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala index 989ef8ce1..d3f7a6e7e 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala @@ -14,6 +14,7 @@ abstract class ArtifactExtra { def configurations: Vector[Configuration] def url: Option[URL] def extraAttributes: Map[String, String] + def checksum: Option[Checksum] protected[this] def copy( name: String = name, @@ -22,7 +23,8 @@ abstract class ArtifactExtra { classifier: Option[String] = classifier, configurations: Vector[Configuration] = configurations, url: Option[URL] = url, - extraAttributes: Map[String, String] = extraAttributes + extraAttributes: Map[String, String] = extraAttributes, + checksum: Option[Checksum] = checksum ): Artifact def extra(attributes: (String, String)*) = @@ -33,7 +35,7 @@ import Configurations.{ Optional, Pom, Test } abstract class ArtifactFunctions { def apply(name: String, extra: Map[String, String]): Artifact = - Artifact(name, DefaultType, DefaultExtension, None, Vector.empty, None, extra) + Artifact(name, DefaultType, DefaultExtension, None, Vector.empty, None, extra, None) def apply(name: String, classifier: String): Artifact = Artifact(name, DefaultType, DefaultExtension, Some(classifier), Vector.empty, None) def apply(name: String, `type`: String, extension: String): Artifact = @@ -50,6 +52,7 @@ abstract class ArtifactFunctions { Some(url) ) + private final val empty = Map.empty[String, String] def apply( name: String, `type`: String, @@ -57,8 +60,7 @@ abstract class ArtifactFunctions { classifier: Option[String], configurations: Vector[Configuration], url: Option[URL] - ): Artifact = - Artifact(name, `type`, extension, classifier, configurations, url, Map.empty[String, String]) + ): Artifact = Artifact(name, `type`, extension, classifier, configurations, url, empty, None) val DefaultExtension = "jar" val DefaultType = "jar" diff --git a/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala b/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala index e52df5a4d..5b89b5419 100644 --- a/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala +++ b/librarymanagement/src/main/scala/sbt/librarymanagement/ModuleIDExtra.scala @@ -22,7 +22,6 @@ abstract class ModuleIDExtra { def extraAttributes: Map[String, String] def crossVersion: CrossVersion def branchName: Option[String] - def checksum: Option[Checksum] protected[this] def copy( organization: String = organization, @@ -37,14 +36,12 @@ abstract class ModuleIDExtra { exclusions: Vector[ExclusionRule] = exclusions, extraAttributes: Map[String, String] = extraAttributes, crossVersion: CrossVersion = crossVersion, - branchName: Option[String] = branchName, - checksum: Option[Checksum] = checksum + branchName: Option[String] = branchName ): ModuleID protected def toStringImpl: String = s"""$organization:$name:$revision""" + - (checksum match { case Some(s) => s": ${s.`type`} ${s.digest}"; case None => "" }) + - (configurations match { case Some(s) => ":" + s; case None => "" }) + { + (configurations match { case Some(s) => ":" + s; case None => "" }) + { val attr = attributeString if (attr == "") "" else " " + attr