Changes in cache policy

2015-12-30 01:34:33 +01:00 · 2015-12-30 01:34:33 +01:00 · 46732be5c9
parent 91d4ad0d18
commit 46732be5c9
9 changed files with 241 additions and 170 deletions
--- a/cli/src/main/scala/coursier/cli/Coursier.scala
+++ b/cli/src/main/scala/coursier/cli/Coursier.scala
@ -12,12 +12,10 @@ import coursier.util.ClasspathFilter
 case class CommonOptions(
  @HelpMessage("Keep optional dependencies (Maven)")
    keepOptional: Boolean,
-  @HelpMessage("Off-line mode: only use cache and local repositories")
-  @ExtraName("c")
-    offline: Boolean,
-  @HelpMessage("Force download: for remote repositories only: re-download items, that is, don't use cache directly")
-  @ExtraName("f")
-    force: Boolean,
+  @HelpMessage("Download mode (default: missing, that is fetch things missing from cache)")
+  @ValueDescription("offline|update-changing|update|missing|force")
+  @ExtraName("m")
+    mode: String,
  @HelpMessage("Quiet output")
  @ExtraName("q")
    quiet: Boolean,
--- a/cli/src/main/scala/coursier/cli/Helper.scala
+++ b/cli/src/main/scala/coursier/cli/Helper.scala
@ -2,7 +2,6 @@ package coursier
 package cli

 import java.io.{ OutputStreamWriter, File }
-import java.util.UUID

 import coursier.ivy.IvyRepository

@ -10,22 +9,6 @@ import scalaz.{ \/-, -\/ }
 import scalaz.concurrent.Task

 object Helper {
-  def validate(common: CommonOptions) = {
-    import common._
-
-    if (force && offline) {
-      Console.err.println("Error: --offline (-c) and --force (-f) options can't be specified at the same time.")
-      sys.exit(255)
-    }
-
-    if (parallel <= 0) {
-      Console.err.println(s"Error: invalid --parallel (-n) value: $parallel")
-      sys.exit(255)
-    }
-
-    ???
-  }
-
  def fileRepr(f: File) = f.toString

  def errPrintln(s: String) = Console.err.println(s)
@ -56,13 +39,23 @@ class Helper(
  import common._
  import Helper.errPrintln

-  implicit val cachePolicy =
-    if (offline)
-      CachePolicy.LocalOnly
-    else if (force)
-      CachePolicy.ForceDownload
-    else
-      CachePolicy.Default
+  val cachePolicies = mode match {
+    case "offline" =>
+      Seq(CachePolicy.LocalOnly)
+    case "update-changing" =>
+      Seq(CachePolicy.UpdateChanging)
+    case "update" =>
+      Seq(CachePolicy.Update)
+    case "missing" =>
+      Seq(CachePolicy.FetchMissing)
+    case "force" =>
+      Seq(CachePolicy.ForceDownload)
+    case "default" =>
+      Seq(CachePolicy.LocalOnly, CachePolicy.FetchMissing)
+    case other =>
+      errPrintln(s"Unrecognized mode: $other")
+      sys.exit(255)
+  }

  val files =
    Files(
@ -200,10 +193,14 @@ class Helper(
    else
      None
  logger.foreach(_.init())
+
+  val fetchs = cachePolicies.map(p =>
+    files.fetch(logger = logger)(cachePolicy = p)
+  )
  val fetchQuiet = coursier.Fetch(
    repositories,
-    files.fetch(logger = logger)(cachePolicy = CachePolicy.LocalOnly), // local files get the priority
-    files.fetch(logger = logger)
+    fetchs.head,
+    fetchs.tail: _*
  )
  val fetch0 =
    if (verbose0 <= 0) fetchQuiet
@ -296,8 +293,16 @@ class Helper(
  }

  def fetch(main: Boolean, sources: Boolean, javadoc: Boolean): Seq[File] = {
-    if (verbose0 >= 0)
-      errPrintln("Fetching artifacts")
+    if (verbose0 >= 0) {
+      val msg = cachePolicies match {
+        case Seq(CachePolicy.LocalOnly) =>
+          "Checking artifacts"
+        case _ =>
+          "Fetching artifacts"
+      }
+
+      errPrintln(msg)
+    }
    val artifacts0 = res.artifacts
    val main0 = main || (!sources && !javadoc)
    val artifacts = artifacts0.flatMap{ artifact =>
@ -318,7 +323,11 @@ class Helper(
      else
        None
    logger.foreach(_.init())
-    val tasks = artifacts.map(artifact => files.file(artifact, logger = logger).run.map(artifact.->))
+    val tasks = artifacts.map(artifact =>
+      (files.file(artifact, logger = logger)(cachePolicy = cachePolicies.head) /: cachePolicies.tail)(
+        _ orElse files.file(artifact, logger = logger)(_)
+      ).run.map(artifact.->)
+    )
    def printTask = Task {
      if (verbose0 >= 1 && artifacts.nonEmpty)
        println(s"Found ${artifacts.length} artifacts")
--- a/core/shared/src/main/scala/coursier/core/Definitions.scala
+++ b/core/shared/src/main/scala/coursier/core/Definitions.scala
@ -136,7 +136,8 @@ case class Artifact(
  url: String,
  checksumUrls: Map[String, String],
  extra: Map[String, Artifact],
-  attributes: Attributes
+  attributes: Attributes,
+  changing: Boolean
 )

 object Artifact {
--- a/core/shared/src/main/scala/coursier/core/Repository.scala
+++ b/core/shared/src/main/scala/coursier/core/Repository.scala
@ -29,16 +29,34 @@ object Repository {
    def withDefaultSignature: Artifact =
      underlying.copy(extra = underlying.extra ++ Seq(
        "sig" ->
-          Artifact(underlying.url + ".asc", Map.empty, Map.empty, Attributes("asc", ""))
+          Artifact(
+            underlying.url + ".asc",
+            Map.empty,
+            Map.empty,
+            Attributes("asc", ""),
+            changing = underlying.changing
+          )
            .withDefaultChecksums
      ))
    def withJavadocSources: Artifact = {
      val base = underlying.url.stripSuffix(".jar")
      underlying.copy(extra = underlying.extra ++ Seq(
-        "sources" -> Artifact(base + "-sources.jar", Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
+        "sources" -> Artifact(
+          base + "-sources.jar",
+          Map.empty,
+          Map.empty,
+          Attributes("jar", "src"),  // Are these the right attributes?
+          changing = underlying.changing
+        )
          .withDefaultChecksums
          .withDefaultSignature,
-        "javadoc" -> Artifact(base + "-javadoc.jar", Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
+        "javadoc" -> Artifact(
+          base + "-javadoc.jar",
+          Map.empty,
+          Map.empty,
+          Attributes("jar", "javadoc"), // Same comment as above
+          changing = underlying.changing
+        )
          .withDefaultChecksums
          .withDefaultSignature
      ))
--- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
+++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala
@ -69,7 +69,7 @@ object IvyRepository {

 }

-case class IvyRepository(pattern: String) extends Repository {
+case class IvyRepository(pattern: String, changing: Option[Boolean] = None) extends Repository {

  import Repository._
  import IvyRepository._
@ -170,7 +170,8 @@ case class IvyRepository(pattern: String) extends Repository {
            url,
            Map.empty,
            Map.empty,
-            Attributes(p.`type`, p.ext)
+            Attributes(p.`type`, p.ext),
+            changing = changing.getOrElse(project.version.contains("-SNAPSHOT")) // could be more reliable
          )
            .withDefaultChecksums
            .withDefaultSignature
@ -194,7 +195,8 @@ case class IvyRepository(pattern: String) extends Repository {
          url,
          Map.empty,
          Map.empty,
-          Attributes("ivy", "")
+          Attributes("ivy", ""),
+          changing = changing.getOrElse(version.contains("-SNAPSHOT"))
        )
          .withDefaultChecksums
          .withDefaultSignature
--- a/core/shared/src/main/scala/coursier/maven/MavenRepository.scala
+++ b/core/shared/src/main/scala/coursier/maven/MavenRepository.scala
@ -48,7 +48,8 @@ object MavenRepository {

 case class MavenRepository(
  root: String,
-  ivyLike: Boolean = false
+  ivyLike: Boolean = false,
+  changing: Option[Boolean] = None
 ) extends Repository {

  import Repository._
@ -85,7 +86,8 @@ case class MavenRepository(
      root0 + path.mkString("/"),
      Map.empty,
      Map.empty,
-      Attributes("pom", "")
+      Attributes("pom", ""),
+      changing = changing.getOrElse(version.contains("-SNAPSHOT"))
    )
    .withDefaultChecksums
    .withDefaultSignature
@ -106,7 +108,8 @@ case class MavenRepository(
          root0 + path.mkString("/"),
          Map.empty,
          Map.empty,
-          Attributes("pom", "")
+          Attributes("pom", ""),
+          changing = true
        )
        .withDefaultChecksums
        .withDefaultSignature
@ -133,7 +136,8 @@ case class MavenRepository(
          root0 + path.mkString("/"),
          Map.empty,
          Map.empty,
-          Attributes("pom", "")
+          Attributes("pom", ""),
+          changing = true
        )
        .withDefaultChecksums
        .withDefaultSignature
--- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala
+++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala
@ -2,7 +2,12 @@ package coursier.maven

 import coursier.core._

-case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
+case class MavenSource(
+  root: String,
+  ivyLike: Boolean,
+  changing: Option[Boolean] = None
+) extends Artifact.Source {
+
  import Repository._
  import MavenRepository._

@ -39,12 +44,14 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
        )
      }

+    val changing0 = changing.getOrElse(project.version.contains("-SNAPSHOT"))
    var artifact =
      Artifact(
        root + path.mkString("/"),
        Map.empty,
        Map.empty,
-        dependency.attributes
+        dependency.attributes,
+        changing = changing0
      )
      .withDefaultChecksums

@ -62,10 +69,22 @@ case class MavenSource(root: String, ivyLike: Boolean) extends Artifact.Source {
          artifact
            .copy(
              extra = artifact.extra ++ Map(
-                "sources" -> Artifact(srcPath, Map.empty, Map.empty, Attributes("jar", "src")) // Are these the right attributes?
+                "sources" -> Artifact(
+                  srcPath,
+                  Map.empty,
+                  Map.empty,
+                  Attributes("jar", "src"), // Are these the right attributes?
+                  changing = changing0
+                )
                  .withDefaultChecksums
                  .withDefaultSignature,
-                "javadoc" -> Artifact(javadocPath, Map.empty, Map.empty, Attributes("jar", "javadoc")) // Same comment as above
+                "javadoc" -> Artifact(
+                  javadocPath,
+                  Map.empty,
+                  Map.empty,
+                  Attributes("jar", "javadoc"), // Same comment as above
+                  changing = changing0
+                )
                  .withDefaultChecksums
                  .withDefaultSignature
            ))
--- a/files/src/main/scala/coursier/CachePolicy.scala
+++ b/files/src/main/scala/coursier/CachePolicy.scala
@ -1,49 +1,11 @@
 package coursier

-import scalaz.\/
-import scalaz.concurrent.Task
-
-sealed trait CachePolicy {
-  def apply[T](
-    tryRemote: T => Boolean )(
-    local: => Task[T] )(
-    remote: Option[T] => Task[T]
-  ): Task[T]
-}
+sealed trait CachePolicy extends Product with Serializable

 object CachePolicy {
-  def saving[E,T](
-    remote: => Task[E \/ T] )(
-    save: T => Task[Unit]
-  ): Task[E \/ T] = {
-    for {
-      res <- remote
-      _ <- res.fold(_ => Task.now(()), t => save(t))
-    } yield res
-  }
-
-  case object Default extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local.flatMap(res => if (tryRemote(res)) remote(Some(res)) else Task.now(res))
-  }
-  case object LocalOnly extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      local
-  }
-  case object ForceDownload extends CachePolicy {
-    def apply[T](
-      tryRemote: T => Boolean )(
-      local: => Task[T] )(
-      remote: Option[T] => Task[T]
-    ): Task[T] =
-      remote(None)
-  }
+  case object LocalOnly extends CachePolicy
+  case object UpdateChanging extends CachePolicy
+  case object Update extends CachePolicy
+  case object FetchMissing extends CachePolicy
+  case object ForceDownload extends CachePolicy
 }
--- a/files/src/main/scala/coursier/Files.scala
+++ b/files/src/main/scala/coursier/Files.scala
@ -1,6 +1,6 @@
 package coursier

-import java.net.URL
+import java.net.{HttpURLConnection, URL}
 import java.nio.channels.{ OverlappingFileLockException, FileLock }
 import java.security.MessageDigest
 import java.util.concurrent.{ConcurrentHashMap, Executors, ExecutorService}
@ -66,13 +66,24 @@ case class Files(
      .extra
      .getOrElse("local", artifact)

-    val checksumPairs = checksums
-      .intersect(artifact0.checksumUrls.keySet)
-      .intersect(artifact.checksumUrls.keySet)
-      .toSeq
-      .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
+    val pairs =
+      Seq(artifact0.url -> artifact.url) ++ {
+        checksums
+          .intersect(artifact0.checksumUrls.keySet)
+          .intersect(artifact.checksumUrls.keySet)
+          .toSeq
+          .map(sumType => artifact0.checksumUrls(sumType) -> artifact.checksumUrls(sumType))
+      }

-    val pairs = (artifact0.url -> artifact.url) +: checksumPairs
+    def urlConn(url: String) = {
+      val conn = new URL(url).openConnection() // FIXME Should this be closed?
+      // Dummy user-agent instead of the default "Java/...",
+      // so that we are not returned incomplete/erroneous metadata
+      // (Maven 2 compatibility? - happens for snapshot versioning metadata,
+      // this is SO FUCKING CRAZY)
+      conn.setRequestProperty("User-Agent", "")
+      conn
+    }


    def locally(file: File, url: String): EitherT[Task, FileError, File] =
@ -86,32 +97,57 @@ case class Files(
        }
      }

-    def downloadIfDifferent(file: File, url: String): EitherT[Task, FileError, Boolean] = {
-      ???
-    }
-
-    def test = {
-      val t: Task[List[((File, String), FileError \/ Boolean)]] = Nondeterminism[Task].gather(checksumPairs.map { case (file, url) =>
-        val f = new File(file)
-        downloadIfDifferent(f, url).run.map((f, url) -> _)
-      })
-
-      t.map { l =>
-        val noChange = l.nonEmpty && l.forall { case (_, e) => e.exists(x => x) }
-
-        val anyChange = l.exists { case (_, e) => e.exists(x => !x) }
-        val anyRecoverableError = l.exists {
-          case (_, -\/(err: FileError.Recoverable)) => true
-          case _ => false
+    def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] =
+      EitherT {
+        Task {
+          \/- {
+            val lastModified = file.lastModified()
+            if (lastModified > 0L)
+              Some(lastModified)
+            else
+              None
+          } : FileError \/ Option[Long]
        }
      }

-    }
+    def urlLastModified(url: String): EitherT[Task, FileError, Option[Long]] =
+      EitherT {
+        Task {
+          urlConn(url) match {
+            case c: HttpURLConnection =>
+              c.setRequestMethod("HEAD")
+              val remoteLastModified = c.getLastModified

-    // FIXME Things can go wrong here and are possibly not properly handled,
+              \/- {
+                if (remoteLastModified > 0L)
+                  Some(remoteLastModified)
+                else
+                  None
+              }
+
+            case other =>
+              -\/(FileError.DownloadError(s"Cannot do HEAD request with connection $other ($url)"))
+          }
+        }
+      }
+
+    def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] =
+      for {
+        fileLastModOpt <- fileLastModified(file)
+        urlLastModOpt <- urlLastModified(url)
+      } yield {
+        val fromDatesOpt = for {
+          fileLastMod <- fileLastModOpt
+          urlLastMod <- urlLastModOpt
+        } yield fileLastMod < urlLastMod
+
+        fromDatesOpt.getOrElse(true)
+      }
+
+    // FIXME Things can go wrong here and are not properly handled,
    // e.g. what if the connection gets closed during the transfer?
    // (partial file on disk?)
-    def remote(file: File, url: String): EitherT[Task, FileError, File] =
+    def remote(file: File, url: String): EitherT[Task, FileError, Unit] =
      EitherT {
        Task {
          try {
@ -121,91 +157,113 @@ case class Files(
              logger.foreach(_.downloadingArtifact(url, file))

              val r = try {
-                val conn = new URL(url).openConnection() // FIXME Should this be closed?
-                // Dummy user-agent instead of the default "Java/...",
-                // so that we are not returned incomplete/erroneous metadata
-                // (Maven 2 compatibility? - happens for snapshot versioning metadata,
-                // this is SO FUCKING CRAZY)
-                conn.setRequestProperty("User-Agent", "")
+                val conn = urlConn(url)

                for (len <- Option(conn.getContentLengthLong).filter(_ >= 0L))
                  logger.foreach(_.downloadLength(url, len))

                val in = new BufferedInputStream(conn.getInputStream, Files.bufferSize)

-                val result = try {
-                  file.getParentFile.mkdirs()
-                  val out = new FileOutputStream(file)
+                val result =
                  try {
-                    var lock: FileLock = null
+                    file.getParentFile.mkdirs()
+                    val out = new FileOutputStream(file)
                    try {
-                      lock = out.getChannel.tryLock()
-                      if (lock == null)
-                        -\/(FileError.Locked(file))
-                      else {
-                        val b = Array.fill[Byte](Files.bufferSize)(0)
+                      var lock: FileLock = null
+                      try {
+                        lock = out.getChannel.tryLock()
+                        if (lock == null)
+                          -\/(FileError.Locked(file))
+                        else {
+                          val b = Array.fill[Byte](Files.bufferSize)(0)

-                        @tailrec
-                        def helper(count: Long): Unit = {
-                          val read = in.read(b)
-                          if (read >= 0) {
-                            out.write(b, 0, read)
-                            out.flush()
-                            logger.foreach(_.downloadProgress(url, count + read))
-                            helper(count + read)
+                          @tailrec
+                          def helper(count: Long): Unit = {
+                            val read = in.read(b)
+                            if (read >= 0) {
+                              out.write(b, 0, read)
+                              out.flush()
+                              logger.foreach(_.downloadProgress(url, count + read))
+                              helper(count + read)
+                            }
                          }
-                        }

-                        helper(0L)
-                        \/-(file)
+                          helper(0L)
+                          \/-(())
+                        }
                      }
-                    } catch { case e: OverlappingFileLockException =>
-                      -\/(FileError.Locked(file))
-                    } finally if (lock != null) lock.release()
-                  } finally out.close()
-                } finally in.close()
+                      catch {
+                        case e: OverlappingFileLockException =>
+                          -\/(FileError.Locked(file))
+                      }
+                      finally if (lock != null) lock.release()
+                    } finally out.close()
+                  } finally in.close()

                for (lastModified <- Option(conn.getLastModified).filter(_ > 0L))
                  file.setLastModified(lastModified)

                result
-              } catch { case e: Exception =>
+              }
+              catch { case e: Exception =>
                logger.foreach(_.downloadedArtifact(url, success = false))
                throw e
-              } finally {
+              }
+              finally {
                urlLocks.remove(url)
              }
              logger.foreach(_.downloadedArtifact(url, success = true))
              r
            } else
              -\/(FileError.ConcurrentDownload(url))
-          } catch { case e: Exception =>
+          }
+          catch { case e: Exception =>
            -\/(FileError.DownloadError(e.getMessage))
          }
        }
      }

+    def checkFileExists(file: File, url: String): EitherT[Task, FileError, Unit] =
+      EitherT {
+        Task {
+          if (file.exists()) {
+            logger.foreach(_.foundLocally(url, file))
+            \/-(())
+          } else
+            -\/(FileError.NotFound(file.toString))
+        }
+      }

    val tasks =
      for ((f, url) <- pairs) yield {
        val file = new File(f)

-        if (url != ("file:" + f) && url != ("file://" + f)) {
-          assert(!f.startsWith("file:/"), s"Wrong file detection: $f, $url")
-          cachePolicy[FileError \/ File](
-            _.isLeft )(
-            locally(file, url).run )(
-            _ => remote(file, url).run
-          ).map(e => (file, url) -> e.map(_ => ()))
-        } else
-          Task {
-            (file, url) -> {
-              if (file.exists())
-                \/-(())
-              else
-                -\/(FileError.NotFound(file.toString))
+        val isRemote = url != ("file:" + f) && url != ("file://" + f)
+        val cachePolicy0 =
+          if (!isRemote)
+            CachePolicy.LocalOnly
+          else if (cachePolicy == CachePolicy.UpdateChanging && !artifact.changing)
+            CachePolicy.FetchMissing
+          else
+            cachePolicy
+
+        val res = cachePolicy match {
+          case CachePolicy.LocalOnly =>
+            checkFileExists(file, url)
+          case CachePolicy.UpdateChanging | CachePolicy.Update =>
+            shouldDownload(file, url).flatMap {
+              case true =>
+                remote(file, url)
+              case false =>
+                EitherT(Task.now(\/-(()) : FileError \/ Unit))
            }
-          }
+          case CachePolicy.FetchMissing =>
+            checkFileExists(file, url) orElse remote(file, url)
+          case CachePolicy.ForceDownload =>
+            remote(file, url)
+        }
+
+        res.run.map((file, url) -> _)
      }

    Nondeterminism[Task].gather(tasks)