luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #105 from scalacenter/managed-checksums 

Implement `managedChecksums` in ivy
This commit is contained in:
eugene yokota 2017-05-27 18:01:57 -04:00 committed by GitHub
parent be972124c6 9d0dfce869
commit fae2b0de51
12 changed files with 219 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
librarymanagement/src/main/contraband/librarymanagement.json
View File

@ -15,7 +15,8 @@
{ "name": "classifier", "type": "Option[String]", "default": "None", "since": "0.0.1" },
{ "name": "configurations", "type": "sbt.librarymanagement.Configuration*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "url", "type": "Option[java.net.URL]", "default": "None", "since": "0.0.1" },
{ "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" }
{ "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" },
{ "name": "checksum", "type": "Option[sbt.librarymanagement.Checksum]", "default": "None", "since": "0.0.1" }
],
"parentsCompanion": "sbt.librarymanagement.ArtifactFunctions"
},
@ -267,22 +268,32 @@
{ "name": "organization", "type": "String" },
{ "name": "name", "type": "String" },
{ "name": "revision", "type": "String" },
{ "name": "configurations", "type": "Option[String]", "default": "None", "since": "0.0.1" },
{ "name": "isChanging", "type": "boolean", "default": "false", "since": "0.0.1" },
{ "name": "isTransitive", "type": "boolean", "default": "true", "since": "0.0.1" },
{ "name": "isForce", "type": "boolean", "default": "false", "since": "0.0.1" },
{ "name": "explicitArtifacts", "type": "sbt.librarymanagement.Artifact*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "inclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "exclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" },
{ "name": "crossVersion", "type": "sbt.librarymanagement.CrossVersion", "default": "sbt.librarymanagement.Disabled()", "since": "0.0.1" },
{ "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" }
{ "name": "configurations", "type": "Option[String]", "default": "None", "since": "0.0.1" },
{ "name": "isChanging", "type": "boolean", "default": "false", "since": "0.0.1" },
{ "name": "isTransitive", "type": "boolean", "default": "true", "since": "0.0.1" },
{ "name": "isForce", "type": "boolean", "default": "false", "since": "0.0.1" },
{ "name": "explicitArtifacts", "type": "sbt.librarymanagement.Artifact*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "inclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "exclusions", "type": "sbt.librarymanagement.InclExclRule*", "default": "Vector.empty", "since": "0.0.1" },
{ "name": "extraAttributes", "type": "Map[String, String]", "default": "Map.empty", "since": "0.0.1" },
{ "name": "crossVersion", "type": "sbt.librarymanagement.CrossVersion", "default": "sbt.librarymanagement.Disabled()", "since": "0.0.1" },
{ "name": "branchName", "type": "Option[String]", "default": "None", "since": "0.0.1" }
],
"toString": [
"this.toStringImpl"
],
"parentsCompanion": "sbt.librarymanagement.ModuleIDFunctions"
},
{
"name": "Checksum",
"namespace": "sbt.librarymanagement",
"target": "Scala",
"type": "record",
"fields": [
{ "name": "digest", "type": "String" },
{ "name": "type", "type": "String", "default": "sha1" }
]
},
{
"name": "ModuleInfo",
"namespace": "sbt.librarymanagement",
@ -731,6 +742,7 @@
{ "name": "otherResolvers", "type": "sbt.librarymanagement.Resolver*" },
{ "name": "moduleConfigurations", "type": "sbt.librarymanagement.ModuleConfiguration*" },
{ "name": "checksums", "type": "String*" },
{ "name": "managedChecksums", "type": "Boolean" },
{ "name": "resolutionCacheDir", "type": "java.io.File?" }
],
"extra": [
@ -741,12 +753,13 @@
" moduleConfigurations: Vector[sbt.librarymanagement.ModuleConfiguration],",
" lock: Option[xsbti.GlobalLock],",
" checksums: Vector[String],",
" managedChecksums: Boolean,",
" resolutionCacheDir: Option[java.io.File],",
" updateOptions: sbt.librarymanagement.UpdateOptions,",
" log: xsbti.Logger",
") =",
" this(lock, paths.baseDirectory, log, updateOptions, paths, resolvers, otherResolvers,",
" moduleConfigurations, checksums, resolutionCacheDir)"
" moduleConfigurations, checksums, managedChecksums, resolutionCacheDir)"
]
},
{

89
librarymanagement/src/main/scala/sbt/internal/librarymanagement/ConvertResolver.scala
View File

@ -5,10 +5,15 @@ package sbt.internal.librarymanagement
import java.net.URL
import java.util.Collections
import org.apache.ivy.core.module.descriptor.DependencyDescriptor
import org.apache.ivy.core.resolve.ResolveData
import org.apache.ivy.core.settings.IvySettings
import org.apache.ivy.plugins.repository.{ RepositoryCopyProgressListener, TransferEvent }
import org.apache.ivy.plugins.repository.{
RepositoryCopyProgressListener,
Resource,
TransferEvent
}
import org.apache.ivy.plugins.resolver.{
BasicResolver,
DependencyResolver,
@ -24,9 +29,10 @@ import org.apache.ivy.plugins.resolver.{
URLResolver
}
import org.apache.ivy.plugins.repository.url.{ URLRepository => URLRepo }
import org.apache.ivy.plugins.repository.file.{ FileRepository => FileRepo, FileResource }
import java.io.{ IOException, File }
import org.apache.ivy.util.{ FileUtil, ChecksumHelper }
import org.apache.ivy.plugins.repository.file.{ FileResource, FileRepository => FileRepo }
import java.io.{ File, IOException }
import org.apache.ivy.util.{ ChecksumHelper, FileUtil, Message }
import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact }
import sbt.io.IO
import sbt.util.Logger
@ -135,6 +141,8 @@ private[sbt] object ConvertResolver {
def apply(r: Resolver, settings: IvySettings, log: Logger): DependencyResolver =
apply(r, settings, UpdateOptions(), log)
private[librarymanagement] val ManagedChecksums = "sbt.managedChecksums"
/** Converts the given sbt resolver into an Ivy resolver. */
def apply(
r: Resolver,
@ -147,6 +155,7 @@ private[sbt] object ConvertResolver {
/** The default implementation of converter. */
lazy val defaultConvert: ResolverConverter = {
case (r, settings, log) =>
val managedChecksums = settings.getVariable(ManagedChecksums).toBoolean
r match {
case repo: MavenRepository => {
val pattern = Collections.singletonList(
@ -156,6 +165,8 @@ private[sbt] object ConvertResolver {
extends IBiblioResolver
with ChecksumFriendlyURLResolver
with DescriptorRequired {
override val managedChecksumsEnabled: Boolean = managedChecksums
override def getResource(resource: Resource, dest: File): Long = get(resource, dest)
def setPatterns(): Unit = {
// done this way for access to protected methods.
setArtifactPatterns(pattern)
@ -170,7 +181,10 @@ private[sbt] object ConvertResolver {
resolver
}
case repo: SshRepository => {
val resolver = new SshResolver with DescriptorRequired
val resolver = new SshResolver with DescriptorRequired {
override val managedChecksumsEnabled: Boolean = managedChecksums
override def getResource(resource: Resource, dest: File): Long = get(resource, dest)
}
initializeSSHResolver(resolver, repo, settings)
repo.publishPermissions.foreach(perm => resolver.setPublishPermissions(perm))
resolver
@ -187,6 +201,8 @@ private[sbt] object ConvertResolver {
// in local files for non-changing revisions.
// This will be fully enforced in sbt 1.0.
setRepository(new WarnOnOverwriteFileRepo())
override val managedChecksumsEnabled: Boolean = managedChecksums
override def getResource(resource: Resource, dest: File): Long = get(resource, dest)
}
resolver.setName(repo.name)
initializePatterns(resolver, repo.patterns, settings)
@ -196,7 +212,10 @@ private[sbt] object ConvertResolver {
resolver
}
case repo: URLRepository => {
val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired
val resolver = new URLResolver with ChecksumFriendlyURLResolver with DescriptorRequired {
override val managedChecksumsEnabled: Boolean = managedChecksums
override def getResource(resource: Resource, dest: File): Long = get(resource, dest)
}
resolver.setName(repo.name)
initializePatterns(resolver, repo.patterns, settings)
resolver
@ -208,11 +227,67 @@ private[sbt] object ConvertResolver {
}
private sealed trait DescriptorRequired extends BasicResolver {
// Works around implementation restriction to access protected method `get`
def getResource(resource: Resource, dest: File): Long
/**
* Defines an option to tell ivy to disable checksums when downloading and
* let the user handle verifying these checksums.
*
* This means that the checksums are stored in the ivy cache directory. This
* is good for reproducibility from outside ivy. Sbt can check that jars are
* not corrupted, ever, independently of trusting whatever it's there in the
* local directory.
*/
def managedChecksumsEnabled: Boolean
import sbt.io.syntax._
private def downloadChecksum(resource: Resource,
target: File,
targetChecksumFile: File,
algorithm: String): Boolean = {
if (!ChecksumHelper.isKnownAlgorithm(algorithm))
throw new IllegalArgumentException(s"Unknown checksum algorithm: $algorithm")
val checksumResource = resource.clone(s"${resource.getName}.$algorithm")
if (!checksumResource.exists) false
else {
Message.debug(s"$algorithm file found for $resource: downloading...")
// Resource must be cleaned up outside of this function if it's invalid
getResource(checksumResource, targetChecksumFile)
true
}
}
private final val PartEnd = ".part"
private final val JarEnd = ".jar"
private final val TemporaryJar = JarEnd + PartEnd
override def getAndCheck(resource: Resource, target: File): Long = {
val targetPath = target.getAbsolutePath
if (!managedChecksumsEnabled || !targetPath.endsWith(TemporaryJar)) {
super.getAndCheck(resource, target)
} else {
// +ivy deviation
val size = getResource(resource, target)
val checksumAlgorithms = getChecksumAlgorithms
checksumAlgorithms.foldLeft(false) { (checked, algorithm) =>
// Continue checking until we hit a failure
val checksumFile = new File(targetPath.stripSuffix(PartEnd) + s".$algorithm")
if (checked) checked
else downloadChecksum(resource, target, checksumFile, algorithm)
}
// -ivy deviation
size
}
}
override def getDependency(dd: DependencyDescriptor, data: ResolveData) = {
val prev = descriptorString(isAllownomd)
setDescriptor(descriptorString(hasExplicitURL(dd)))
try super.getDependency(dd, data)
val t = try super.getDependency(dd, data)
finally setDescriptor(prev)
t
}
def descriptorString(optional: Boolean) =
if (optional) BasicResolver.DESCRIPTOR_OPTIONAL else BasicResolver.DESCRIPTOR_REQUIRED

1
librarymanagement/src/main/scala/sbt/internal/librarymanagement/Ivy.scala
View File

@ -98,6 +98,7 @@ final class IvySbt(val configuration: IvyConfiguration) { self =>
IvySbt.loadURI(is, e.uri)
case i: InlineIvyConfiguration =>
is.setVariable("ivy.checksums", i.checksums mkString ",")
is.setVariable(ConvertResolver.ManagedChecksums, i.managedChecksums.toString)
i.paths.ivyHome foreach is.setDefaultIvyUserDir
val log = configuration.log
IvySbt.configureCache(is, i.resolutionCacheDir)

22
librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyActions.scala
View File

@ -540,6 +540,7 @@ object IvyActions {
report: UpdateReport,
config: RetrieveConfiguration
): UpdateReport = {
val copyChecksums = ivy.getVariable(ConvertResolver.ManagedChecksums).toBoolean
val toRetrieve = config.configurationsToRetrieve
val base = config.retrieveDirectory
val pattern = config.outputPattern
@ -551,9 +552,9 @@ object IvyActions {
val toCopy = new collection.mutable.HashSet[(File, File)]
val retReport = report retrieve { (conf, mid, art, cached) =>
configurationNames match {
case None => performRetrieve(conf, mid, art, base, pattern, cached, toCopy)
case None => performRetrieve(conf, mid, art, base, pattern, cached, copyChecksums, toCopy)
case Some(names) if names(conf) =>
performRetrieve(conf, mid, art, base, pattern, cached, toCopy)
performRetrieve(conf, mid, art, base, pattern, cached, copyChecksums, toCopy)
case _ => cached
}
}
@ -577,10 +578,27 @@ object IvyActions {
base: File,
pattern: String,
cached: File,
copyChecksums: Boolean,
toCopy: collection.mutable.HashSet[(File, File)]
): File = {
val to = retrieveTarget(conf, mid, art, base, pattern)
toCopy += ((cached, to))
if (copyChecksums) {
// Copy over to the lib managed directory any checksum for a jar if it exists
// TODO(jvican): Support user-provided checksums
val cachePath = cached.getAbsolutePath
IvySbt.DefaultChecksums.foreach { checksum =>
if (cachePath.endsWith(".jar")) {
val cacheChecksum = new File(s"$cachePath.$checksum")
if (cacheChecksum.exists()) {
val toChecksum = new File(s"${to.getAbsolutePath}.$checksum")
toCopy += ((cacheChecksum, toChecksum))
}
}
}
}
to
}

1
librarymanagement/src/main/scala/sbt/internal/librarymanagement/IvyCache.scala
View File

@ -110,6 +110,7 @@ class IvyCache(val ivyHome: Option[File]) {
Vector.empty,
lock,
IvySbt.DefaultChecksums,
false,
None,
UpdateOptions(),
log

2
librarymanagement/src/main/scala/sbt/internal/librarymanagement/formats/UpdateOptionsFormat.scala
View File

@ -35,7 +35,7 @@ trait UpdateOptionsFormat { self: BasicJsonProtocol with ModuleIDFormats with Re
xs._3,
xs._4,
xs._5,
ConvertResolver.defaultConvert,
PartialFunction.empty,
xs._6
)
)

1
librarymanagement/src/main/scala/sbt/internal/librarymanagement/ivyint/SbtChainResolver.scala
View File

@ -31,6 +31,7 @@ private[sbt] case class SbtChainResolver(
updateOptions: UpdateOptions,
log: Logger
) extends ChainResolver {
override def setCheckmodified(check: Boolean): Unit = super.setCheckmodified(check)
override def equals(o: Any): Boolean = o match {
case o: SbtChainResolver =>

10
librarymanagement/src/main/scala/sbt/librarymanagement/ArtifactExtra.scala
View File

@ -14,6 +14,7 @@ abstract class ArtifactExtra {
def configurations: Vector[Configuration]
def url: Option[URL]
def extraAttributes: Map[String, String]
def checksum: Option[Checksum]
protected[this] def copy(
name: String = name,
@ -22,7 +23,8 @@ abstract class ArtifactExtra {
classifier: Option[String] = classifier,
configurations: Vector[Configuration] = configurations,
url: Option[URL] = url,
extraAttributes: Map[String, String] = extraAttributes
extraAttributes: Map[String, String] = extraAttributes,
checksum: Option[Checksum] = checksum
): Artifact
def extra(attributes: (String, String)*) =
@ -33,7 +35,7 @@ import Configurations.{ Optional, Pom, Test }
abstract class ArtifactFunctions {
def apply(name: String, extra: Map[String, String]): Artifact =
Artifact(name, DefaultType, DefaultExtension, None, Vector.empty, None, extra)
Artifact(name, DefaultType, DefaultExtension, None, Vector.empty, None, extra, None)
def apply(name: String, classifier: String): Artifact =
Artifact(name, DefaultType, DefaultExtension, Some(classifier), Vector.empty, None)
def apply(name: String, `type`: String, extension: String): Artifact =
@ -50,6 +52,7 @@ abstract class ArtifactFunctions {
Some(url)
)
private final val empty = Map.empty[String, String]
def apply(
name: String,
`type`: String,
@ -57,8 +60,7 @@ abstract class ArtifactFunctions {
classifier: Option[String],
configurations: Vector[Configuration],
url: Option[URL]
): Artifact =
Artifact(name, `type`, extension, classifier, configurations, url, Map.empty[String, String])
): Artifact = Artifact(name, `type`, extension, classifier, configurations, url, empty, None)
val DefaultExtension = "jar"
val DefaultType = "jar"

4
librarymanagement/src/test/scala/BaseIvySpecification.scala
View File

@ -53,8 +53,9 @@ trait BaseIvySpecification extends UnitSpec {
def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = {
val paths = IvyPaths(currentBase, Some(currentTarget))
val other = Vector.empty
val moduleConfs = Vector(ModuleConfiguration("*", chainResolver))
val check = Vector.empty
val managedChecksums = false
val moduleConfs = Vector(ModuleConfiguration("*", chainResolver))
val resCacheDir = currentTarget / "resolution-cache"
new InlineIvyConfiguration(paths,
resolvers,
@ -62,6 +63,7 @@ trait BaseIvySpecification extends UnitSpec {
moduleConfs,
None,
check,
managedChecksums,
Some(resCacheDir),
uo,
log)

1
librarymanagement/src/test/scala/CustomPomParserTest.scala
View File

@ -21,6 +21,7 @@ class CustomPomParserTest extends UnitSpec {
Vector.empty,
None,
Vector("sha1", "md5"),
false,
None,
UpdateOptions(),
log)

4
librarymanagement/src/test/scala/OfflineModeSpec.scala
View File

@ -3,7 +3,7 @@ package sbt.librarymanagement
import org.scalatest.Assertion
import sbt.internal.librarymanagement._
import sbt.internal.librarymanagement.impl.DependencyBuilders
import sbt.io.IO
import sbt.io.{ FileFilter, IO, Path }
class OfflineModeSpec extends BaseIvySpecification with DependencyBuilders {
private final def targetDir = Some(currentDependency)
@ -44,7 +44,7 @@ class OfflineModeSpec extends BaseIvySpecification with DependencyBuilders {
val offlineResolution =
IvyActions.updateEither(toResolve, offlineConf, warningConf, noClock, targetDir, log)
assert(offlineResolution.isRight)
assert(offlineResolution.isRight, s"Offline resolution has failed with $offlineResolution.")
val resolveTime = offlineResolution.right.get.stats.resolveTime
// Only check the estimate for the non cached resolution, otherwise resolution is cached

76
librarymanagement/src/test/scala/sbt/internal/librarymanagement/ManagedChecksumsSpec.scala Normal file
View File

@ -0,0 +1,76 @@
package sbt.librarymanagement
import java.io.File
import org.apache.ivy.util.Message
import org.scalatest.Assertion
import sbt.internal.librarymanagement.{
BaseIvySpecification,
InlineIvyConfiguration,
IvyActions,
IvyConfiguration,
IvyPaths,
IvySbt,
LogicalClock,
UnresolvedWarningConfiguration
}
import sbt.internal.librarymanagement.impl.DependencyBuilders
import sbt.io.IO
class ManagedChecksumsSpec extends BaseIvySpecification with DependencyBuilders {
private final def targetDir = Some(currentDependency)
private final def onlineConf = makeUpdateConfiguration(false)
private final def warningConf = UnresolvedWarningConfiguration()
private final def noClock = LogicalClock.unknown
private final val Checksum = "sha1"
def avro177 = ModuleID("org.apache.avro", "avro", "1.7.7")
def dataAvro1940 = ModuleID("com.linkedin.pegasus", "data-avro", "1.9.40")
def netty320 = ModuleID("org.jboss.netty", "netty", "3.2.0.Final")
final def dependencies: Vector[ModuleID] =
Vector(avro177, dataAvro1940, netty320).map(_.withConfigurations(Some("compile")))
import sbt.io.syntax._
override def mkIvyConfiguration(uo: UpdateOptions): IvyConfiguration = {
val paths = IvyPaths(currentBase, Some(currentTarget))
val other = Vector.empty
val check = Vector(Checksum)
val moduleConfs = Vector(ModuleConfiguration("*", chainResolver))
val resCacheDir = currentTarget / "resolution-cache"
new InlineIvyConfiguration(paths,
resolvers,
other,
moduleConfs,
None,
check,
managedChecksums = true,
Some(resCacheDir),
uo,
log)
}
def cleanAll(): Unit = {
cleanIvyCache()
IO.delete(currentTarget)
IO.delete(currentManaged)
IO.delete(currentDependency)
}
def assertChecksumExists(file: File) = {
val shaFile = new File(file.getAbsolutePath + s".$Checksum")
Message.info(s"Checking $shaFile exists...")
assert(shaFile.exists(), s"The checksum $Checksum for $file does not exist")
}
"Managed checksums" should "should download the checksum files" in {
cleanAll()
val updateOptions = UpdateOptions()
val toResolve = module(defaultModuleId, dependencies, None, updateOptions)
val res = IvyActions.updateEither(toResolve, onlineConf, warningConf, noClock, targetDir, log)
assert(res.isRight, s"Resolution with managed checksums failed! $res")
val updateReport = res.right.get
val allModuleReports = updateReport.configurations.flatMap(_.modules)
val allArtifacts: Seq[File] = allModuleReports.flatMap(_.artifacts.map(_._2))
allArtifacts.foreach(assertChecksumExists)
}
}