luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Copy local artifact to cache (#831)

This commit is contained in:
Dorothy Ordogh 2018-04-26 02:16:09 -07:00 committed by Alexandre Archambault
parent 07989f0040
commit c469899040
8 changed files with 86 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bootstrap/src/main/java/coursier/Bootstrap.java
View File

@ -186,7 +186,8 @@ public class Bootstrap {
if (protocol.equals("file") || protocol.equals(bootstrapProtocol)) {
localURLs.add(url);
} else {
File dest = CachePath.localFile(url.toString(), cacheDir, null);
// fourth argument is false because we don't want to store local files when bootstrapping
File dest = CachePath.localFile(url.toString(), cacheDir, null, false);
if (dest.exists()) {
localURLs.add(dest.toURI().toURL());
@ -200,7 +201,8 @@ public class Bootstrap {
completionService.submit(new Callable<URL>() {
@Override
public URL call() throws Exception {
final File dest = CachePath.localFile(url.toString(), cacheDir, null);
// fourth argument is false because we don't want to store local files when bootstrapping
final File dest = CachePath.localFile(url.toString(), cacheDir, null, false);
if (!dest.exists()) {
FileOutputStream out = null;

32
cache/jvm/src/main/scala/coursier/Cache.scala vendored
View File

@ -37,8 +37,8 @@ object Cache {
// Check SHA-1 if available, else be fine with no checksum
val defaultChecksums = Seq(Some("SHA-1"), None)
def localFile(url: String, cache: File, user: Option[String]): File =
CachePath.localFile(url, cache, user.orNull)
def localFile(url: String, cache: File, user: Option[String], localArtifactsShouldBeCached: Boolean): File =
CachePath.localFile(url, cache, user.orNull, localArtifactsShouldBeCached)
private def readFullyTo(
in: InputStream,
@ -363,7 +363,8 @@ object Cache {
cachePolicy: CachePolicy,
pool: ExecutorService,
logger: Option[Logger],
ttl: Option[Duration]
ttl: Option[Duration],
localArtifactsShouldBeCached: Boolean
)(implicit S: Schedulable[F]): F[Seq[((File, String), Either[FileError, Unit])]] = {
// Reference file - if it exists, and we get not found errors on some URLs, we assume
@ -371,7 +372,7 @@ object Cache {
val referenceFileOpt = artifact
.extra
.get("metadata")
.map(a => localFile(a.url, cache, a.authentication.map(_.user)))
.map(a => localFile(a.url, cache, a.authentication.map(_.user), localArtifactsShouldBeCached))
def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
@ -779,7 +780,7 @@ object Cache {
case Some(required) =>
cachePolicy0 match {
case CachePolicy.LocalOnly | CachePolicy.LocalUpdateChanging | CachePolicy.LocalUpdate =>
val file = localFile(required.url, cache, artifact.authentication.map(_.user))
val file = localFile(required.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
localInfo(file, required.url).flatMap {
case true =>
EitherT(S.point[Either[FileError, Unit]](Right(())))
@ -793,10 +794,10 @@ object Cache {
val tasks =
for (url <- urls) yield {
val file = localFile(url, cache, artifact.authentication.map(_.user))
val file = localFile(url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
def res =
if (url.startsWith("file:/")) {
if (url.startsWith("file:/") && !localArtifactsShouldBeCached) {
// for debug purposes, flaky with URL-encoded chars anyway
// def filtered(s: String) =
// s.stripPrefix("file:/").stripPrefix("//").stripSuffix("/")
@ -879,15 +880,16 @@ object Cache {
artifact: Artifact,
sumType: String,
cache: File,
pool: ExecutorService
pool: ExecutorService,
localArtifactsShouldBeCached: Boolean = false
)(implicit S: Schedulable[F]): EitherT[F, FileError, Unit] = {
val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user))
val localFile0 = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
EitherT {
artifact.checksumUrls.get(sumType) match {
case Some(sumUrl) =>
val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user))
val sumFile = localFile(sumUrl, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
S.schedule(pool) {
val sumOpt = parseRawChecksum(Files.readAllBytes(sumFile.toPath))
@ -941,7 +943,8 @@ object Cache {
logger: Option[Logger] = None,
pool: ExecutorService = defaultPool,
ttl: Option[Duration] = defaultTtl,
retry: Int = 1
retry: Int = 1,
localArtifactsShouldBeCached: Boolean = false
)(implicit S: Schedulable[F]): EitherT[F, FileError, File] = {
val checksums0 = if (checksums.isEmpty) Seq(None) else checksums
@ -954,7 +957,8 @@ object Cache {
cachePolicy,
pool,
logger = logger,
ttl = ttl
ttl = ttl,
localArtifactsShouldBeCached
)) { results =>
val checksum = checksums0.find {
case None => true
@ -982,7 +986,7 @@ object Cache {
res.flatMap {
case (f, None) => EitherT(S.point[Either[FileError, File]](Right(f)))
case (f, Some(c)) =>
validateChecksum(artifact, c, cache, pool).map(_ => f)
validateChecksum(artifact, c, cache, pool, localArtifactsShouldBeCached).map(_ => f)
}.leftFlatMap {
case err: FileError.WrongChecksum =>
if (retry <= 0) {
@ -991,7 +995,7 @@ object Cache {
else {
EitherT {
S.schedule[Either[FileError, Unit]](pool) {
val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user))
val badFile = localFile(artifact.url, cache, artifact.authentication.map(_.user), localArtifactsShouldBeCached)
badFile.delete()
logger.foreach(_.removedCorruptFile(artifact.url, badFile, Some(err)))
Right(())

5
cli/src/main/scala-2.12/coursier/cli/Helper.scala
View File

@ -295,7 +295,7 @@ class Helper(
}
}.toMap
val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls)
val depsWithUrlRepo: FallbackDependenciesRepository = FallbackDependenciesRepository(depsWithUrls, cacheFileArtifacts)
// Prepend FallbackDependenciesRepository to the repository list
// so that dependencies with URIs are resolved against this repo
@ -646,7 +646,8 @@ class Helper(
logger = logger,
pool = pool,
ttl = ttl0,
retry = common.retryCount
retry = common.retryCount,
cacheFileArtifacts
)
(file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_))

6
cli/src/main/scala-2.12/coursier/cli/options/CommonOptions.scala
View File

@ -98,7 +98,11 @@ final case class CommonOptions(
cacheOptions: CacheOptions = CacheOptions(),
@Help("Retry limit for Checksum error when fetching a file")
retryCount: Int = 1
retryCount: Int = 1,
@Help("Flag that specifies if a local artifact should be cached.")
@Short("cfa")
cacheFileArtifacts: Boolean = false
) {
val verbosityLevel = Tag.unwrap(verbose) - (if (quiet) 1 else 0)

5
cli/src/main/scala-2.12/coursier/cli/spark/Assembly.scala
View File

@ -208,7 +208,8 @@ object Assembly {
extraDependencies: Seq[String],
options: CommonOptions,
artifactTypes: Set[String],
checksumSeed: Array[Byte] = "v1".getBytes(UTF_8)
checksumSeed: Array[Byte] = "v1".getBytes(UTF_8),
localArtifactsShouldBeCached: Boolean = false
): Either[String, (File, Seq[File])] = {
val helper = sparkJarsHelper(scalaVersion, sparkVersion, yarnVersion, default, extraDependencies, options)
@ -219,7 +220,7 @@ object Assembly {
val checksums = artifacts.map { a =>
val f = a.checksumUrls.get("SHA-1") match {
case Some(url) =>
Cache.localFile(url, helper.cache, a.authentication.map(_.user))
Cache.localFile(url, helper.cache, a.authentication.map(_.user), localArtifactsShouldBeCached)
case None =>
throw new Exception(s"SHA-1 file not found for ${a.url}")
}

52
cli/src/test/scala-2.12/coursier/cli/CliFetchIntegrationTest.scala
View File

@ -448,9 +448,9 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
/**
* Result:
* |└─ org.apache.commons:commons-compress:1.5
* |└─ a:b:c
*/
"local dep url" should "have coursier-fetch-test.jar" in withFile() {
"local file dep url" should "have coursier-fetch-test.jar and cached for second run" in withFile() {
(jsonFile, _) => {
withFile("tada", "coursier-fetch-test", ".jar") {
(testFile, _) => {
@ -458,7 +458,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
val encodedUrl = encode("file://" + path, "UTF-8")
val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath)
val commonOpt = CommonOptions(jsonOutputFile = jsonFile.getPath, cacheFileArtifacts = true)
val fetchOpt = FetchOptions(common = commonOpt)
// fetch with encoded url set to temp jar
@ -466,25 +466,47 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
fetchOpt,
RemainingArgs(
Seq(
"org.apache.commons:commons-compress:1.5,url=" + encodedUrl
"a:b:c,url=" + encodedUrl
),
Seq()
)
)
val node: ReportNode = getReportFromJson(jsonFile)
val node1: ReportNode = getReportFromJson(jsonFile)
val depNodes: Seq[DepNode] = node.dependencies
.filter(_.coord == "org.apache.commons:commons-compress:1.5")
val depNodes1: Seq[DepNode] = node1.dependencies
.filter(_.coord == "a:b:c")
.sortBy(fileNameLength)
assert(depNodes.length == 1)
assert(depNodes1.length == 1)
val urlInJsonFile = depNodes.head.file.get
assert(urlInJsonFile.contains(path))
val urlInJsonFile1 = depNodes1.head.file.get
assert(urlInJsonFile1.contains(path))
// open jar and inspect contents
val fileContents = Source.fromFile(urlInJsonFile).getLines.mkString
assert(fileContents == "tada")
val fileContents1 = Source.fromFile(urlInJsonFile1).getLines.mkString
assert(fileContents1 == "tada")
testFile.delete()
Fetch.run(
fetchOpt,
RemainingArgs(
Seq(
"a:b:c,url=" + encodedUrl
),
Seq()
)
)
val node2: ReportNode = getReportFromJson(jsonFile)
val depNodes2: Seq[DepNode] = node2.dependencies
.filter(_.coord == "a:b:c")
.sortBy(fileNameLength)
assert(depNodes2.length == 1)
val urlInJsonFile2 = depNodes2.head.file.get
assert(urlInJsonFile2.contains("coursier/cache") && urlInJsonFile2.contains(testFile.toString))
}
}
}
@ -525,7 +547,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
/**
* Result:
* |└─ a:b:c
* |└─ h:i:j
*/
"external dep url with arbitrary coords" should "fetch junit-4.12.jar" in withFile() {
(jsonFile, _) => {
@ -540,7 +562,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
fetchOpt,
RemainingArgs(
Seq(
"a:b:c,url=" + externalUrl
"h:i:j,url=" + externalUrl
),
Seq()
)
@ -549,7 +571,7 @@ class CliFetchIntegrationTest extends FlatSpec with CliTestLib with Matchers {
val node: ReportNode = getReportFromJson(jsonFile)
val depNodes: Seq[DepNode] = node.dependencies
.filter(_.coord == "a:b:c")
.filter(_.coord == "h:i:j")
.sortBy(fileNameLength)
assert(depNodes.length == 1)
depNodes.head.file.map( f => assert(f.contains("junit/junit/4.12/junit-4.12.jar"))).orElse(fail("Not Defined"))

19
extra/src/main/scala/coursier/FallbackDependenciesRepository.scala
View File

@ -7,7 +7,7 @@ import coursier.util.{EitherT, Monad}
object FallbackDependenciesRepository {
def exists(url: URL): Boolean = {
def exists(url: URL, localArtifactsShouldBeCached: Boolean): Boolean = {
// Sometimes HEAD attempts fail even though standard GETs are fine.
// E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar
@ -15,8 +15,14 @@ object FallbackDependenciesRepository {
val protocolSpecificAttemptOpt = {
def ifFile: Boolean =
new File(url.getPath).exists() // FIXME Escaping / de-escaping needed here?
def ifFile: Option[Boolean] = {
if (localArtifactsShouldBeCached && !new File(url.getPath).exists()) {
val cachePath = coursier.Cache.default + "/file" // Use '/file' here because the protocol becomes part of the cache path
Some(new File(cachePath, url.getPath).exists())
} else {
Some(new File(url.getPath).exists()) // FIXME Escaping / de-escaping needed here?
}
}
def ifHttp: Option[Boolean] = {
// HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275
@ -41,7 +47,7 @@ object FallbackDependenciesRepository {
}
url.getProtocol match {
case "file" => Some(ifFile)
case "file" => ifFile
case "http" | "https" => ifHttp
case _ => None
}
@ -71,7 +77,8 @@ object FallbackDependenciesRepository {
}
final case class FallbackDependenciesRepository(
fallbacks: Map[(Module, String), (URL, Boolean)]
fallbacks: Map[(Module, String), (URL, Boolean)],
localArtifactsShouldBeCached: Boolean = false
) extends Repository {
private val source: Artifact.Source =
@ -113,7 +120,7 @@ final case class FallbackDependenciesRepository(
else {
val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
if (FallbackDependenciesRepository.exists(url)) {
if (FallbackDependenciesRepository.exists(url, localArtifactsShouldBeCached)) {
val proj = Project(
module,
version,

6
paths/src/main/java/coursier/CachePath.java
View File

@ -36,14 +36,14 @@ public class CachePath {
return ch > 128 || " %$&+,:;=?@<>#%".indexOf(ch) >= 0;
}
public static File localFile(String url, File cache, String user) throws MalformedURLException {
public static File localFile(String url, File cache, String user, boolean localArtifactsShouldBeCached) throws MalformedURLException {
// use the File constructor accepting a URI in case of problem with the two cases below?
if (url.startsWith("file:///"))
if (url.startsWith("file:///") && !localArtifactsShouldBeCached)
return new File(url.substring("file://".length()));
if (url.startsWith("file:/"))
if (url.startsWith("file:/") && !localArtifactsShouldBeCached)
return new File(url.substring("file:".length()));
String[] split = url.split(":", 2);