diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index 162fc545f..d70447079 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter case class CommonOptions( @HelpMessage("Keep optional dependencies (Maven)") keepOptional: Boolean, - @HelpMessage("Off-line mode: only use cache and local repositories") - @ExtraName("c") - offline: Boolean, - @HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly") - @ExtraName("f") - force: Boolean, + @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)") + @ValueDescription("offline|update-changing|update|missing|force") + @ExtraName("m") + mode: String, @HelpMessage("Quiet output") @ExtraName("q") quiet: Boolean, diff --git a/cli/src/main/scala/coursier/cli/Helper.scala b/cli/src/main/scala/coursier/cli/Helper.scala index 459d32f84..20371b125 100644 --- a/cli/src/main/scala/coursier/cli/Helper.scala +++ b/cli/src/main/scala/coursier/cli/Helper.scala @@ -2,7 +2,6 @@ package coursier package cli import java.io.{ OutputStreamWriter, File } -import java.util.UUID import coursier.ivy.IvyRepository @@ -10,22 +9,6 @@ import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task object Helper { - def validate(common: CommonOptions) = { - import common._ - - if (force && offline) { - Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.") - sys.exit(255) - } - - if (parallel <= 0) { - Console.err.println(s"Error: invalid --parallel (-n) value: $parallel") - sys.exit(255) - } - - ??? - } - def fileRepr(f: File) = f.toString def errPrintln(s: String) = Console.err.println(s) @@ -56,13 +39,23 @@ class Helper( import common._ import Helper.errPrintln - implicit val cachePolicy = - if (offline) - CachePolicy.LocalOnly - else if (force) - CachePolicy.ForceDownload - else - CachePolicy.Default + val cachePolicies = mode match { + case "offline" => + Seq(CachePolicy.LocalOnly) + case "update-changing" => + Seq(CachePolicy.UpdateChanging) + case "update" => + Seq(CachePolicy.Update) + case "missing" => + Seq(CachePolicy.FetchMissing) + case "force" => + Seq(CachePolicy.ForceDownload) + case "default" => + Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing) + case other => + errPrintln(s"Unrecognized mode: $other") + sys.exit(255) + } val files = Files( @@ -200,10 +193,14 @@ class Helper( else None logger.foreach(_.init()) + + val fetchs = cachePolicies.map(p => + files.fetch(logger = logger)(cachePolicy = p) + ) val fetchQuiet = coursier.Fetch( repositories, - files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority - files.fetch(logger = logger) + fetchs.head, + fetchs.tail: _* ) val fetch0 = if (verbose0 <= 0) fetchQuiet @@ -296,8 +293,16 @@ class Helper( } def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = { - if (verbose0 >= 0) - errPrintln("Fetching artifacts") + if (verbose0 >= 0) { + val msg = cachePolicies match { + case Seq(CachePolicy.LocalOnly) => + "Checking artifacts" + case _ => + "Fetching artifacts" + } + + errPrintln(msg) + } val artifacts0 = res.artifacts val main0 = main || (!sources && !javadoc) val artifacts = artifacts0.flatMap{ artifact => @@ -318,7 +323,11 @@ class Helper( else None logger.foreach(_.init()) - val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->)) + val tasks = artifacts.map(artifact => + (files.file(artifact, logger = logger)(cachePolicy = cachePolicies.head) /: cachePolicies.tail)( + _ orElse files.file(artifact, logger = logger)(_) + ).run.map(artifact.->) + ) def printTask = Task { if (verbose0 >= 1 && artifacts.nonEmpty) println(s"Found ${artifacts.length} artifacts") diff --git a/core/shared/src/main/scala/coursier/core/Definitions.scala b/core/shared/src/main/scala/coursier/core/Definitions.scala index e9b52a889..82db9ae31 100644 --- a/core/shared/src/main/scala/coursier/core/Definitions.scala +++ b/core/shared/src/main/scala/coursier/core/Definitions.scala @@ -136,7 +136,8 @@ case class Artifact( url: String, checksumUrls: Map[String, String], extra: Map[String, Artifact], - attributes: Attributes + attributes: Attributes, + changing: Boolean ) object Artifact { diff --git a/core/shared/src/main/scala/coursier/core/Repository.scala b/core/shared/src/main/scala/coursier/core/Repository.scala index 8c853874c..2d0f96dcf 100644 --- a/core/shared/src/main/scala/coursier/core/Repository.scala +++ b/core/shared/src/main/scala/coursier/core/Repository.scala @@ -29,16 +29,34 @@ object Repository { def withDefaultSignature: Artifact = underlying.copy(extra = underlying.extra ++ Seq( "sig" -> - Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", "")) + Artifact( + underlying.url + ".asc", + Map.empty, + Map.empty, + Attributes("asc", ""), + changing = underlying.changing + ) .withDefaultChecksums )) def withJavadocSources: Artifact = { val base = underlying.url.stripSuffix(".jar") underlying.copy(extra = underlying.extra ++ Seq( - "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + "sources" -> Artifact( + base + "-sources.jar", + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = underlying.changing + ) .withDefaultChecksums .withDefaultSignature, - "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + "javadoc" -> Artifact( + base + "-javadoc.jar", + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = underlying.changing + ) .withDefaultChecksums .withDefaultSignature )) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index ab0c814c9..0b496a5c0 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -69,7 +69,7 @@ object IvyRepository { } -case class IvyRepository(pattern: String) extends Repository { +case class IvyRepository(pattern: String, changing: Option[Boolean] = None) extends Repository { import Repository._ import IvyRepository._ @@ -170,7 +170,8 @@ case class IvyRepository(pattern: String) extends Repository { url, Map.empty, Map.empty, - Attributes(p.`type`, p.ext) + Attributes(p.`type`, p.ext), + changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable ) .withDefaultChecksums .withDefaultSignature @@ -194,7 +195,8 @@ case class IvyRepository(pattern: String) extends Repository { url, Map.empty, Map.empty, - Attributes("ivy", "") + Attributes("ivy", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) .withDefaultChecksums .withDefaultSignature diff --git a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala index 6fc0b02f3..d0522f13e 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala @@ -48,7 +48,8 @@ object MavenRepository { case class MavenRepository( root: String, - ivyLike: Boolean = false + ivyLike: Boolean = false, + changing: Option[Boolean] = None ) extends Repository { import Repository._ @@ -85,7 +86,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = changing.getOrElse(version.contains("-SNAPSHOT")) ) .withDefaultChecksums .withDefaultSignature @@ -106,7 +108,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums .withDefaultSignature @@ -133,7 +136,8 @@ case class MavenRepository( root0 + path.mkString("/"), Map.empty, Map.empty, - Attributes("pom", "") + Attributes("pom", ""), + changing = true ) .withDefaultChecksums .withDefaultSignature diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index 02c780d3f..26973836d 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -2,7 +2,12 @@ package coursier.maven import coursier.core._ -case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { +case class MavenSource( + root: String, + ivyLike: Boolean, + changing: Option[Boolean] = None +) extends Artifact.Source { + import Repository._ import MavenRepository._ @@ -39,12 +44,14 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { ) } + val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT")) var artifact = Artifact( root + path.mkString("/"), Map.empty, Map.empty, - dependency.attributes + dependency.attributes, + changing = changing0 ) .withDefaultChecksums @@ -62,10 +69,22 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source { artifact .copy( extra = artifact.extra ++ Map( - "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes? + "sources" -> Artifact( + srcPath, + Map.empty, + Map.empty, + Attributes("jar", "src"), // Are these the right attributes? + changing = changing0 + ) .withDefaultChecksums .withDefaultSignature, - "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above + "javadoc" -> Artifact( + javadocPath, + Map.empty, + Map.empty, + Attributes("jar", "javadoc"), // Same comment as above + changing = changing0 + ) .withDefaultChecksums .withDefaultSignature )) diff --git a/files/src/main/scala/coursier/CachePolicy.scala b/files/src/main/scala/coursier/CachePolicy.scala index 10c76392f..0bf51e6e3 100644 --- a/files/src/main/scala/coursier/CachePolicy.scala +++ b/files/src/main/scala/coursier/CachePolicy.scala @@ -1,49 +1,11 @@ package coursier -import scalaz.\/ -import scalaz.concurrent.Task - -sealed trait CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] -} +sealed trait CachePolicy extends Product with Serializable object CachePolicy { - def saving[E,T]( - remote: => Task[E \/ T] )( - save: T => Task[Unit] - ): Task[E \/ T] = { - for { - res <- remote - _ <- res.fold(_ => Task.now(()), t => save(t)) - } yield res - } - - case object Default extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res)) - } - case object LocalOnly extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - local - } - case object ForceDownload extends CachePolicy { - def apply[T]( - tryRemote: T => Boolean )( - local: => Task[T] )( - remote: Option[T] => Task[T] - ): Task[T] = - remote(None) - } + case object LocalOnly extends CachePolicy + case object UpdateChanging extends CachePolicy + case object Update extends CachePolicy + case object FetchMissing extends CachePolicy + case object ForceDownload extends CachePolicy } diff --git a/files/src/main/scala/coursier/Files.scala b/files/src/main/scala/coursier/Files.scala index e798338d8..d6e5c9eb8 100644 --- a/files/src/main/scala/coursier/Files.scala +++ b/files/src/main/scala/coursier/Files.scala @@ -1,6 +1,6 @@ package coursier -import java.net.URL +import java.net.{HttpURLConnection, URL} import java.nio.channels.{ OverlappingFileLockException, FileLock } import java.security.MessageDigest import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService} @@ -66,13 +66,24 @@ case class Files( .extra .getOrElse("local", artifact) - val checksumPairs = checksums - .intersect(artifact0.checksumUrls.keySet) - .intersect(artifact.checksumUrls.keySet) - .toSeq - .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + val pairs = + Seq(artifact0.url -> artifact.url) ++ { + checksums + .intersect(artifact0.checksumUrls.keySet) + .intersect(artifact.checksumUrls.keySet) + .toSeq + .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType)) + } - val pairs = (artifact0.url -> artifact.url) +: checksumPairs + def urlConn(url: String) = { + val conn = new URL(url).openConnection() // FIXME Should this be closed? + // Dummy user-agent instead of the default "Java/...", + // so that we are not returned incomplete/erroneous metadata + // (Maven 2 compatibility? - happens for snapshot versioning metadata, + // this is SO FUCKING CRAZY) + conn.setRequestProperty("User-Agent", "") + conn + } def locally(file: File, url: String): EitherT[Task, FileError, File] = @@ -86,32 +97,57 @@ case class Files( } } - def downloadIfDifferent(file: File, url: String): EitherT[Task, FileError, Boolean] = { - ??? - } - - def test = { - val t: Task[List[((File, String), FileError \/ Boolean)]] = Nondeterminism[Task].gather(checksumPairs.map { case (file, url) => - val f = new File(file) - downloadIfDifferent(f, url).run.map((f, url) -> _) - }) - - t.map { l => - val noChange = l.nonEmpty && l.forall { case (_, e) => e.exists(x => x) } - - val anyChange = l.exists { case (_, e) => e.exists(x => !x) } - val anyRecoverableError = l.exists { - case (_, -\/(err: FileError.Recoverable)) => true - case _ => false + def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + \/- { + val lastModified = file.lastModified() + if (lastModified > 0L) + Some(lastModified) + else + None + } : FileError \/ Option[Long] } } - } + def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] = + EitherT { + Task { + urlConn(url) match { + case c: HttpURLConnection => + c.setRequestMethod("HEAD") + val remoteLastModified = c.getLastModified - // FIXME Things can go wrong here and are possibly not properly handled, + \/- { + if (remoteLastModified > 0L) + Some(remoteLastModified) + else + None + } + + case other => + -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)")) + } + } + } + + def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = + for { + fileLastModOpt <- fileLastModified(file) + urlLastModOpt <- urlLastModified(url) + } yield { + val fromDatesOpt = for { + fileLastMod <- fileLastModOpt + urlLastMod <- urlLastModOpt + } yield fileLastMod < urlLastMod + + fromDatesOpt.getOrElse(true) + } + + // FIXME Things can go wrong here and are not properly handled, // e.g. what if the connection gets closed during the transfer? // (partial file on disk?) - def remote(file: File, url: String): EitherT[Task, FileError, File] = + def remote(file: File, url: String): EitherT[Task, FileError, Unit] = EitherT { Task { try { @@ -121,91 +157,113 @@ case class Files( logger.foreach(_.downloadingArtifact(url, file)) val r = try { - val conn = new URL(url).openConnection() // FIXME Should this be closed? - // Dummy user-agent instead of the default "Java/...", - // so that we are not returned incomplete/erroneous metadata - // (Maven 2 compatibility? - happens for snapshot versioning metadata, - // this is SO FUCKING CRAZY) - conn.setRequestProperty("User-Agent", "") + val conn = urlConn(url) for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L)) logger.foreach(_.downloadLength(url, len)) val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize) - val result = try { - file.getParentFile.mkdirs() - val out = new FileOutputStream(file) + val result = try { - var lock: FileLock = null + file.getParentFile.mkdirs() + val out = new FileOutputStream(file) try { - lock = out.getChannel.tryLock() - if (lock == null) - -\/(FileError.Locked(file)) - else { - val b = Array.fill[Byte](Files.bufferSize)(0) + var lock: FileLock = null + try { + lock = out.getChannel.tryLock() + if (lock == null) + -\/(FileError.Locked(file)) + else { + val b = Array.fill[Byte](Files.bufferSize)(0) - @tailrec - def helper(count: Long): Unit = { - val read = in.read(b) - if (read >= 0) { - out.write(b, 0, read) - out.flush() - logger.foreach(_.downloadProgress(url, count + read)) - helper(count + read) + @tailrec + def helper(count: Long): Unit = { + val read = in.read(b) + if (read >= 0) { + out.write(b, 0, read) + out.flush() + logger.foreach(_.downloadProgress(url, count + read)) + helper(count + read) + } } - } - helper(0L) - \/-(file) + helper(0L) + \/-(()) + } } - } catch { case e: OverlappingFileLockException => - -\/(FileError.Locked(file)) - } finally if (lock != null) lock.release() - } finally out.close() - } finally in.close() + catch { + case e: OverlappingFileLockException => + -\/(FileError.Locked(file)) + } + finally if (lock != null) lock.release() + } finally out.close() + } finally in.close() for (lastModified <- Option(conn.getLastModified).filter(_ > 0L)) file.setLastModified(lastModified) result - } catch { case e: Exception => + } + catch { case e: Exception => logger.foreach(_.downloadedArtifact(url, success = false)) throw e - } finally { + } + finally { urlLocks.remove(url) } logger.foreach(_.downloadedArtifact(url, success = true)) r } else -\/(FileError.ConcurrentDownload(url)) - } catch { case e: Exception => + } + catch { case e: Exception => -\/(FileError.DownloadError(e.getMessage)) } } } + def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] = + EitherT { + Task { + if (file.exists()) { + logger.foreach(_.foundLocally(url, file)) + \/-(()) + } else + -\/(FileError.NotFound(file.toString)) + } + } val tasks = for ((f, url) <- pairs) yield { val file = new File(f) - if (url != ("file:" + f) && url != ("file://" + f)) { - assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url") - cachePolicy[FileError \/ File]( - _.isLeft )( - locally(file, url).run )( - _ => remote(file, url).run - ).map(e => (file, url) -> e.map(_ => ())) - } else - Task { - (file, url) -> { - if (file.exists()) - \/-(()) - else - -\/(FileError.NotFound(file.toString)) + val isRemote = url != ("file:" + f) && url != ("file://" + f) + val cachePolicy0 = + if (!isRemote) + CachePolicy.LocalOnly + else if (cachePolicy == CachePolicy.UpdateChanging && !artifact.changing) + CachePolicy.FetchMissing + else + cachePolicy + + val res = cachePolicy match { + case CachePolicy.LocalOnly => + checkFileExists(file, url) + case CachePolicy.UpdateChanging | CachePolicy.Update => + shouldDownload(file, url).flatMap { + case true => + remote(file, url) + case false => + EitherT(Task.now(\/-(()) : FileError \/ Unit)) } - } + case CachePolicy.FetchMissing => + checkFileExists(file, url) orElse remote(file, url) + case CachePolicy.ForceDownload => + remote(file, url) + } + + res.run.map((file, url) -> _) } Nondeterminism[Task].gather(tasks)