Changes in cache policy

cli/src/main/scala/coursier/cli/Coursier.scala

@@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter
 case class CommonOptions(
   @HelpMessage("Keep optional dependencies (Maven)")
     keepOptional: Boolean,
-  @HelpMessage("Off-line mode: only use cache and local repositories")
+  @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)")
-  @ExtraName("c")
+  @ValueDescription("offline|update-changing|update|missing|force")
-    offline: Boolean,
+  @ExtraName("m")
-  @HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly")
+    mode: String,
-  @ExtraName("f")
-    force: Boolean,
   @HelpMessage("Quiet output")
   @ExtraName("q")
     quiet: Boolean,

cli/src/main/scala/coursier/cli/Helper.scala

@@ -2,7 +2,6 @@ package coursier
 package cli
 
 import java.io.{ OutputStreamWriter, File }
-import java.util.UUID
 
 import coursier.ivy.IvyRepository
 
@@ -10,22 +9,6 @@ import scalaz.{ \/-, -\/ }
 import scalaz.concurrent.Task
 
 object Helper {
-  def validate(common: CommonOptions) = {
-    import common._
-
-    if (force && offline) {
-      Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.")
-      sys.exit(255)
-    }
-
-    if (parallel <= 0) {
-      Console.err.println(s"Error: invalid --parallel (-n) value: $parallel")
-      sys.exit(255)
-    }
-
-    ???
-  }
-
   def fileRepr(f: File) = f.toString
 
   def errPrintln(s: String) = Console.err.println(s)
@@ -56,13 +39,23 @@ class Helper(
   import common._
   import Helper.errPrintln
 
-  implicit val cachePolicy =
+  val cachePolicies = mode match {
-    if (offline)
+    case "offline" =>
-      CachePolicy.LocalOnly
+      Seq(CachePolicy.LocalOnly)
-    else if (force)
+    case "update-changing" =>
-      CachePolicy.ForceDownload
+      Seq(CachePolicy.UpdateChanging)
-    else
+    case "update" =>
-      CachePolicy.Default
+      Seq(CachePolicy.Update)
+    case "missing" =>
+      Seq(CachePolicy.FetchMissing)
+    case "force" =>
+      Seq(CachePolicy.ForceDownload)
+    case "default" =>
+      Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing)
+    case other =>
+      errPrintln(s"Unrecognized mode: $other")
+      sys.exit(255)
+  }
 
   val files =
     Files(
@@ -200,10 +193,14 @@ class Helper(
     else
       None
   logger.foreach(_.init())
+
+  val fetchs = cachePolicies.map(p =>
+    files.fetch(logger = logger)(cachePolicy = p)
+  )
   val fetchQuiet = coursier.Fetch(
     repositories,
-    files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority
+    fetchs.head,
-    files.fetch(logger = logger)
+    fetchs.tail: _*
   )
   val fetch0 =
     if (verbose0 <= 0) fetchQuiet
@@ -296,8 +293,16 @@ class Helper(
   }
 
   def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = {
-    if (verbose0 >= 0)
+    if (verbose0 >= 0) {
-      errPrintln("Fetching artifacts")
+      val msg = cachePolicies match {
+        case Seq(CachePolicy.LocalOnly) =>
+          "Checking artifacts"
+        case _ =>
+          "Fetching artifacts"
+      }
+
+      errPrintln(msg)
+    }
     val artifacts0 = res.artifacts
     val main0 = main || (!sources && !javadoc)
     val artifacts = artifacts0.flatMap{ artifact =>
@@ -318,7 +323,11 @@ class Helper(
       else
         None
     logger.foreach(_.init())
-    val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->))
+    val tasks = artifacts.map(artifact =>
+      (files.file(artifact, logger = logger)(cachePolicy = cachePolicies.head) /: cachePolicies.tail)(
+        _ orElse files.file(artifact, logger = logger)(_)
+      ).run.map(artifact.->)
+    )
     def printTask = Task {
       if (verbose0 >= 1 && artifacts.nonEmpty)
         println(s"Found ${artifacts.length} artifacts")

core/shared/src/main/scala/coursier/core/Definitions.scala

@@ -136,7 +136,8 @@ case class Artifact(
   url: String,
   checksumUrls: Map[String, String],
   extra: Map[String, Artifact],
-  attributes: Attributes
+  attributes: Attributes,
+  changing: Boolean
 )
 
 object Artifact {

core/shared/src/main/scala/coursier/core/Repository.scala

@@ -29,16 +29,34 @@ object Repository {
     def withDefaultSignature: Artifact =
       underlying.copy(extra = underlying.extra ++ Seq(
         "sig" ->
-          Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", ""))
+          Artifact(
+            underlying.url + ".asc",
+            Map.empty,
+            Map.empty,
+            Attributes("asc", ""),
+            changing = underlying.changing
+          )
             .withDefaultChecksums
       ))
     def withJavadocSources: Artifact = {
       val base = underlying.url.stripSuffix(".jar")
       underlying.copy(extra = underlying.extra ++ Seq(
-        "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
+        "sources" -> Artifact(
+          base + "-sources.jar",
+          Map.empty,
+          Map.empty,
+          Attributes("jar", "src"),  // Are these the right attributes?
+          changing = underlying.changing
+        )
           .withDefaultChecksums
           .withDefaultSignature,
-        "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
+        "javadoc" -> Artifact(
+          base + "-javadoc.jar",
+          Map.empty,
+          Map.empty,
+          Attributes("jar", "javadoc"), // Same comment as above
+          changing = underlying.changing
+        )
           .withDefaultChecksums
           .withDefaultSignature
       ))

core/shared/src/main/scala/coursier/ivy/IvyRepository.scala

@@ -69,7 +69,7 @@ object IvyRepository {
 
 }
 
-case class IvyRepository(pattern: String) extends Repository {
+case class IvyRepository(pattern: String, changing: Option[Boolean] = None) extends Repository {
 
   import Repository._
   import IvyRepository._
@@ -170,7 +170,8 @@ case class IvyRepository(pattern: String) extends Repository {
             url,
             Map.empty,
             Map.empty,
-            Attributes(p.`type`, p.ext)
+            Attributes(p.`type`, p.ext),
+            changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable
           )
             .withDefaultChecksums
             .withDefaultSignature
@@ -194,7 +195,8 @@ case class IvyRepository(pattern: String) extends Repository {
           url,
           Map.empty,
           Map.empty,
-          Attributes("ivy", "")
+          Attributes("ivy", ""),
+          changing = changing.getOrElse(version.contains("-SNAPSHOT"))
         )
           .withDefaultChecksums
           .withDefaultSignature

core/shared/src/main/scala/coursier/maven/MavenRepository.scala

@@ -48,7 +48,8 @@ object MavenRepository {
 
 case class MavenRepository(
   root: String,
-  ivyLike: Boolean = false
+  ivyLike: Boolean = false,
+  changing: Option[Boolean] = None
 ) extends Repository {
 
   import Repository._
@@ -85,7 +86,8 @@ case class MavenRepository(
       root0 + path.mkString("/"),
       Map.empty,
       Map.empty,
-      Attributes("pom", "")
+      Attributes("pom", ""),
+      changing = changing.getOrElse(version.contains("-SNAPSHOT"))
     )
     .withDefaultChecksums
     .withDefaultSignature
@@ -106,7 +108,8 @@ case class MavenRepository(
           root0 + path.mkString("/"),
           Map.empty,
           Map.empty,
-          Attributes("pom", "")
+          Attributes("pom", ""),
+          changing = true
         )
         .withDefaultChecksums
         .withDefaultSignature
@@ -133,7 +136,8 @@ case class MavenRepository(
           root0 + path.mkString("/"),
           Map.empty,
           Map.empty,
-          Attributes("pom", "")
+          Attributes("pom", ""),
+          changing = true
         )
         .withDefaultChecksums
         .withDefaultSignature

core/shared/src/main/scala/coursier/maven/MavenSource.scala

@@ -2,7 +2,12 @@ package coursier.maven
 
 import coursier.core._
 
-case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
+case class MavenSource(
+  root: String,
+  ivyLike: Boolean,
+  changing: Option[Boolean] = None
+) extends Artifact.Source {
+
   import Repository._
   import MavenRepository._
 
@@ -39,12 +44,14 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
         )
       }
 
+    val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT"))
     var artifact =
       Artifact(
         root + path.mkString("/"),
         Map.empty,
         Map.empty,
-        dependency.attributes
+        dependency.attributes,
+        changing = changing0
       )
       .withDefaultChecksums
 
@@ -62,10 +69,22 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
           artifact
             .copy(
               extra = artifact.extra ++ Map(
-                "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
+                "sources" -> Artifact(
+                  srcPath,
+                  Map.empty,
+                  Map.empty,
+                  Attributes("jar", "src"), // Are these the right attributes?
+                  changing = changing0
+                )
                   .withDefaultChecksums
                   .withDefaultSignature,
-                "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
+                "javadoc" -> Artifact(
+                  javadocPath,
+                  Map.empty,
+                  Map.empty,
+                  Attributes("jar", "javadoc"), // Same comment as above
+                  changing = changing0
+                )
                   .withDefaultChecksums
                   .withDefaultSignature
             ))

files/src/main/scala/coursier/CachePolicy.scala

@@ -1,49 +1,11 @@
 package coursier
 
-import scalaz.\/
+sealed trait CachePolicy extends Product with Serializable
-import scalaz.concurrent.Task
-
-sealed trait CachePolicy {
-  def apply[T](
-    tryRemote: T => Boolean )(
-    local: => Task[T] )(
-    remote: Option[T] => Task[T]
-  ): Task[T]
-}
 
 object CachePolicy {
-  def saving[E,T](
+  case object LocalOnly extends CachePolicy
-    remote: => Task[E \/ T] )(
+  case object UpdateChanging extends CachePolicy
-    save: T => Task[Unit]
+  case object Update extends CachePolicy
-  ): Task[E \/ T] = {
+  case object FetchMissing extends CachePolicy
-    for {
+  case object ForceDownload extends CachePolicy
-      res <- remote
-      _ <- res.fold(_ => Task.now(()), t => save(t))
-    } yield res
-  }
-
-  case object Default extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res))
-  }
-  case object LocalOnly extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local
-  }
-  case object ForceDownload extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      remote(None)
-  }
 }

files/src/main/scala/coursier/Files.scala

@@ -1,6 +1,6 @@
 package coursier
 
-import java.net.URL
+import java.net.{HttpURLConnection, URL}
 import java.nio.channels.{ OverlappingFileLockException, FileLock }
 import java.security.MessageDigest
 import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService}
@@ -66,13 +66,24 @@ case class Files(
       .extra
       .getOrElse("local", artifact)
 
-    val checksumPairs = checksums
+    val pairs =
-      .intersect(artifact0.checksumUrls.keySet)
+      Seq(artifact0.url -> artifact.url) ++ {
-      .intersect(artifact.checksumUrls.keySet)
+        checksums
-      .toSeq
+          .intersect(artifact0.checksumUrls.keySet)
-      .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
+          .intersect(artifact.checksumUrls.keySet)
+          .toSeq
+          .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
+      }
 
-    val pairs = (artifact0.url -> artifact.url) +: checksumPairs
+    def urlConn(url: String) = {
+      val conn = new URL(url).openConnection() // FIXME Should this be closed?
+      // Dummy user-agent instead of the default "Java/...",
+      // so that we are not returned incomplete/erroneous metadata
+      // (Maven 2 compatibility? - happens for snapshot versioning metadata,
+      // this is SO FUCKING CRAZY)
+      conn.setRequestProperty("User-Agent", "")
+      conn
+    }
 
 
     def locally(file: File, url: String): EitherT[Task, FileError, File] =
@@ -86,32 +97,57 @@ case class Files(
         }
       }
 
-    def downloadIfDifferent(file: File, url: String): EitherT[Task, FileError, Boolean] = {
+    def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] =
-      ???
+      EitherT {
-    }
+        Task {
-
+          \/- {
-    def test = {
+            val lastModified = file.lastModified()
-      val t: Task[List[((File, String), FileError \/ Boolean)]] = Nondeterminism[Task].gather(checksumPairs.map { case (file, url) =>
+            if (lastModified > 0L)
-        val f = new File(file)
+              Some(lastModified)
-        downloadIfDifferent(f, url).run.map((f, url) -> _)
+            else
-      })
+              None
-
+          } : FileError \/ Option[Long]
-      t.map { l =>
-        val noChange = l.nonEmpty && l.forall { case (_, e) => e.exists(x => x) }
-
-        val anyChange = l.exists { case (_, e) => e.exists(x => !x) }
-        val anyRecoverableError = l.exists {
-          case (_, -\/(err: FileError.Recoverable)) => true
-          case _ => false
         }
       }
 
-    }
+    def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] =
+      EitherT {
+        Task {
+          urlConn(url) match {
+            case c: HttpURLConnection =>
+              c.setRequestMethod("HEAD")
+              val remoteLastModified = c.getLastModified
 
-    // FIXME Things can go wrong here and are possibly not properly handled,
+              \/- {
+                if (remoteLastModified > 0L)
+                  Some(remoteLastModified)
+                else
+                  None
+              }
+
+            case other =>
+              -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
+          }
+        }
+      }
+
+    def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] =
+      for {
+        fileLastModOpt <- fileLastModified(file)
+        urlLastModOpt <- urlLastModified(url)
+      } yield {
+        val fromDatesOpt = for {
+          fileLastMod <- fileLastModOpt
+          urlLastMod <- urlLastModOpt
+        } yield fileLastMod < urlLastMod
+
+        fromDatesOpt.getOrElse(true)
+      }
+
+    // FIXME Things can go wrong here and are not properly handled,
     // e.g. what if the connection gets closed during the transfer?
     // (partial file on disk?)
-    def remote(file: File, url: String): EitherT[Task, FileError, File] =
+    def remote(file: File, url: String): EitherT[Task, FileError, Unit] =
       EitherT {
         Task {
           try {
@@ -121,91 +157,113 @@ case class Files(
               logger.foreach(_.downloadingArtifact(url, file))
 
               val r = try {
-                val conn = new URL(url).openConnection() // FIXME Should this be closed?
+                val conn = urlConn(url)
-                // Dummy user-agent instead of the default "Java/...",
-                // so that we are not returned incomplete/erroneous metadata
-                // (Maven 2 compatibility? - happens for snapshot versioning metadata,
-                // this is SO FUCKING CRAZY)
-                conn.setRequestProperty("User-Agent", "")
 
                 for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L))
                   logger.foreach(_.downloadLength(url, len))
 
                 val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize)
 
-                val result = try {
+                val result =
-                  file.getParentFile.mkdirs()
-                  val out = new FileOutputStream(file)
                   try {
-                    var lock: FileLock = null
+                    file.getParentFile.mkdirs()
+                    val out = new FileOutputStream(file)
                     try {
-                      lock = out.getChannel.tryLock()
+                      var lock: FileLock = null
-                      if (lock == null)
+                      try {
-                        -\/(FileError.Locked(file))
+                        lock = out.getChannel.tryLock()
-                      else {
+                        if (lock == null)
-                        val b = Array.fill[Byte](Files.bufferSize)(0)
+                          -\/(FileError.Locked(file))
+                        else {
+                          val b = Array.fill[Byte](Files.bufferSize)(0)
 
-                        @tailrec
+                          @tailrec
-                        def helper(count: Long): Unit = {
+                          def helper(count: Long): Unit = {
-                          val read = in.read(b)
+                            val read = in.read(b)
-                          if (read >= 0) {
+                            if (read >= 0) {
-                            out.write(b, 0, read)
+                              out.write(b, 0, read)
-                            out.flush()
+                              out.flush()
-                            logger.foreach(_.downloadProgress(url, count + read))
+                              logger.foreach(_.downloadProgress(url, count + read))
-                            helper(count + read)
+                              helper(count + read)
+                            }
                           }
-                        }
 
-                        helper(0L)
+                          helper(0L)
-                        \/-(file)
+                          \/-(())
+                        }
                       }
-                    } catch { case e: OverlappingFileLockException =>
+                      catch {
-                      -\/(FileError.Locked(file))
+                        case e: OverlappingFileLockException =>
-                    } finally if (lock != null) lock.release()
+                          -\/(FileError.Locked(file))
-                  } finally out.close()
+                      }
-                } finally in.close()
+                      finally if (lock != null) lock.release()
+                    } finally out.close()
+                  } finally in.close()
 
                 for (lastModified <- Option(conn.getLastModified).filter(_ > 0L))
                   file.setLastModified(lastModified)
 
                 result
-              } catch { case e: Exception =>
+              }
+              catch { case e: Exception =>
                 logger.foreach(_.downloadedArtifact(url, success = false))
                 throw e
-              } finally {
+              }
+              finally {
                 urlLocks.remove(url)
               }
               logger.foreach(_.downloadedArtifact(url, success = true))
               r
             } else
               -\/(FileError.ConcurrentDownload(url))
-          } catch { case e: Exception =>
+          }
+          catch { case e: Exception =>
             -\/(FileError.DownloadError(e.getMessage))
           }
         }
       }
 
+    def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] =
+      EitherT {
+        Task {
+          if (file.exists()) {
+            logger.foreach(_.foundLocally(url, file))
+            \/-(())
+          } else
+            -\/(FileError.NotFound(file.toString))
+        }
+      }
 
     val tasks =
       for ((f, url) <- pairs) yield {
         val file = new File(f)
 
-        if (url != ("file:" + f) && url != ("file://" + f)) {
+        val isRemote = url != ("file:" + f) && url != ("file://" + f)
-          assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url")
+        val cachePolicy0 =
-          cachePolicy[FileError \/ File](
+          if (!isRemote)
-            _.isLeft )(
+            CachePolicy.LocalOnly
-            locally(file, url).run )(
+          else if (cachePolicy == CachePolicy.UpdateChanging && !artifact.changing)
-            _ => remote(file, url).run
+            CachePolicy.FetchMissing
-          ).map(e => (file, url) -> e.map(_ => ()))
+          else
-        } else
+            cachePolicy
-          Task {
+
-            (file, url) -> {
+        val res = cachePolicy match {
-              if (file.exists())
+          case CachePolicy.LocalOnly =>
-                \/-(())
+            checkFileExists(file, url)
-              else
+          case CachePolicy.UpdateChanging | CachePolicy.Update =>
-                -\/(FileError.NotFound(file.toString))
+            shouldDownload(file, url).flatMap {
+              case true =>
+                remote(file, url)
+              case false =>
+                EitherT(Task.now(\/-(()) : FileError \/ Unit))
             }
-          }
+          case CachePolicy.FetchMissing =>
+            checkFileExists(file, url) orElse remote(file, url)
+          case CachePolicy.ForceDownload =>
+            remote(file, url)
+        }
+
+        res.run.map((file, url) -> _)
       }
 
     Nondeterminism[Task].gather(tasks)