Add support for TTL

2016-05-31 15:18:29 +02:00 · 2016-05-31 15:18:29 +02:00 · b76fbf363a
parent 6aad1c6310
commit b76fbf363a
9 changed files with 192 additions and 69 deletions
--- a/build.sbt
+++ b/build.sbt
@ -248,6 +248,9 @@ lazy val cache = project
      import com.typesafe.tools.mima.core.ProblemFilters._
      
      Seq(
+        // Since 1.0.0-M13
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.file"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.fetch"),
        // Since 1.0.0-M12
        // Remove deprecated / unused helper method
        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.readFully"),
--- a/cache/src/main/scala/coursier/Cache.scala
+++ b/cache/src/main/scala/coursier/Cache.scala
@ -20,6 +20,9 @@ import scalaz.concurrent.{ Task, Strategy }

 import java.io.{ Serializable => _, _ }

+import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration }
+import scala.util.Try
+
 trait AuthenticatedURLConnection extends URLConnection {
  def authenticate(authentication: Authentication): Unit
 }
@ -306,7 +309,8 @@ object Cache {
    checksums: Set[String],
    cachePolicy: CachePolicy,
    pool: ExecutorService,
-    logger: Option[Logger] = None
+    logger: Option[Logger] = None,
+    ttl: Option[FiniteDuration] = defaultTtl
  ): Task[Seq[((File, String), FileError \/ Unit)]] = {

    implicit val pool0 = pool
@ -402,7 +406,44 @@ object Cache {
        file.exists()
      }

+    def ttlFile(file: File): File =
+      new File(file.getParent, s".${file.getName}.checked")
+
+    def lastCheck(file: File): Task[Option[Long]] = {
+
+      val ttlFile0 = ttlFile(file)
+
+      Task {
+        if (ttlFile0.exists())
+          Some(ttlFile0.lastModified()).filter(_ > 0L)
+        else
+          None
+      }
+    }
+
+    /** Not wrapped in a `Task` !!! */
+    def doTouchCheckFile(file: File): Unit = {
+      val ts = System.currentTimeMillis()
+      val f = ttlFile(file)
+      if (f.exists())
+        f.setLastModified(ts)
+      else {
+        val fos = new FileOutputStream(f)
+        fos.write(Array.empty[Byte])
+        fos.close()
+      }
+    }
+
    def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = {
+
+      def checkNeeded = ttl.map(_.toMillis).filter(_ > 0L).fold(Task.now(true)) { ttlMs =>
+        lastCheck(file).flatMap {
+          case None => Task.now(true)
+          case Some(ts) =>
+            Task(System.currentTimeMillis()).map(_ > ts + ttlMs)
+        }
+      }
+
      def check = for {
        fileLastModOpt <- fileLastModified(file)
        urlLastModOpt <- urlLastModified(url, fileLastModOpt, logger)
@ -420,7 +461,20 @@ object Cache {
          case false =>
            Task.now(true.right)
          case true =>
-            check.run
+            checkNeeded.flatMap {
+              case false =>
+                Task.now(false.right)
+              case true =>
+                check.run.flatMap {
+                  case \/-(false) =>
+                    Task {
+                      doTouchCheckFile(file)
+                      \/-(false)
+                    }
+                  case other =>
+                    Task.now(other)
+                }
+            }
        }
      }
    }
@ -493,7 +547,7 @@ object Cache {
                      tmp.getParentFile.mkdirs()
                      new FileOutputStream(tmp, partialDownload)
                    }
-                    try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L))
+                    try readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)
                    finally out.close()
                  } finally in.close()

@ -505,7 +559,9 @@ object Cache {
                for (lastModified <- Option(conn.getLastModified) if lastModified > 0L)
                  file.setLastModified(lastModified)

-                result
+                doTouchCheckFile(file)
+
+                result.right
              }
            }
          }
@ -725,7 +781,8 @@ object Cache {
    cachePolicy: CachePolicy = CachePolicy.FetchMissing,
    checksums: Seq[Option[String]] = defaultChecksums,
    logger: Option[Logger] = None,
-    pool: ExecutorService = defaultPool
+    pool: ExecutorService = defaultPool,
+    ttl: Option[FiniteDuration] = defaultTtl
  ): EitherT[Task, FileError, File] = {

    implicit val pool0 = pool
@ -739,7 +796,8 @@ object Cache {
        checksums = checksums0.collect { case Some(c) => c }.toSet,
        cachePolicy,
        pool,
-        logger = logger
+        logger = logger,
+        ttl = ttl
      ).map { results =>
        val checksum = checksums0.find {
          case None => true
@ -776,7 +834,8 @@ object Cache {
    cachePolicy: CachePolicy = CachePolicy.FetchMissing,
    checksums: Seq[Option[String]] = defaultChecksums,
    logger: Option[Logger] = None,
-    pool: ExecutorService = defaultPool
+    pool: ExecutorService = defaultPool,
+    ttl: Option[FiniteDuration] = defaultTtl
  ): Fetch.Content[Task] = {
    artifact =>
      file(
@ -785,7 +844,8 @@ object Cache {
        cachePolicy,
        checksums = checksums,
        logger = logger,
-        pool = pool
+        pool = pool,
+        ttl = ttl
      ).leftMap(_.describe).map { f =>
        // FIXME Catch error here?
        new String(NioFiles.readAllBytes(f.toPath), "UTF-8")
@ -830,6 +890,20 @@ object Cache {
  lazy val defaultPool =
    Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory)

+  lazy val defaultTtl: Option[FiniteDuration] = {
+    def fromString(s: String) =
+      Try(Duration(s)).toOption.collect {
+        case d: FiniteDuration => d
+      }
+
+    val fromEnv = sys.env.get("COURSIER_TTL").flatMap(fromString)
+    def fromProps = sys.props.get("coursier.ttl").flatMap(fromString)
+    def default = 24.days
+
+    fromEnv
+      .orElse(fromProps)
+      .orElse(Some(default))
+  }

  private val urlLocks = new ConcurrentHashMap[String, Object]

--- a/cache/src/main/scala/coursier/CachePolicy.scala
+++ b/cache/src/main/scala/coursier/CachePolicy.scala
@ -55,4 +55,47 @@ object CachePolicy {
    * Erases files already in cache.
    */
  case object ForceDownload extends CachePolicy
+
+
+  private val baseDefault = Seq(
+    CachePolicy.LocalOnly,
+    CachePolicy.FetchMissing
+  )
+
+  def default: Seq[CachePolicy] = {
+
+    def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] =
+      value.filter(_.nonEmpty).flatMap {
+        str =>
+          CacheParse.cachePolicies(str) match {
+            case scalaz.Success(Seq()) =>
+              Console.err.println(
+                s"Warning: no mode found in $description, ignoring it."
+              )
+              None
+            case scalaz.Success(policies) =>
+              Some(policies)
+            case scalaz.Failure(errors) =>
+              Console.err.println(
+                s"Warning: unrecognized mode in $description, ignoring it."
+              )
+              None
+          }
+      }
+
+    val fromEnv = fromOption(
+      sys.env.get("COURSIER_MODE"),
+      "COURSIER_MODE environment variable"
+    )
+
+    def fromProps = fromOption(
+      sys.props.get("coursier.mode"),
+      "Java property coursier.mode"
+    )
+
+    fromEnv
+      .orElse(fromProps)
+      .getOrElse(baseDefault)
+  }
+
 }
--- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala
+++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala
@ -10,6 +10,8 @@ import coursier.ivy.IvyRepository
 import coursier.util.{Print, Parse}

 import scala.annotation.tailrec
+import scala.concurrent.duration.{ Duration, FiniteDuration }
+
 import scalaz.{Failure, Success, \/-, -\/}
 import scalaz.concurrent.{ Task, Strategy }

@ -80,15 +82,35 @@ class Helper(

  import Util._

-  val cachePoliciesValidation = CacheParse.cachePolicies(common.mode)
+  val ttl0 =
+    if (ttl.isEmpty)
+      Cache.defaultTtl
+    else {
+      val d = try {
+        Duration(ttl)
+      } catch {
+        case e: Exception =>
+          prematureExit(s"Unrecognized TTL duration: $ttl")
+      }

-  val cachePolicies = cachePoliciesValidation match {
-    case Success(cp) => cp
-    case Failure(errors) =>
-      prematureExit(
-        s"Error parsing modes:\n${errors.list.map("  "+_).mkString("\n")}"
-      )
-  }
+      d match {
+        case f: FiniteDuration => Some(f)
+        case _ =>
+          prematureExit(s"Non finite TTL duration: $ttl")
+      }
+    }
+
+  val cachePolicies =
+    if (common.mode.isEmpty)
+      CachePolicy.default
+    else
+      CacheParse.cachePolicies(common.mode) match {
+        case Success(cp) => cp
+        case Failure(errors) =>
+          prematureExit(
+            s"Error parsing modes:\n${errors.list.map("  "+_).mkString("\n")}"
+          )
+      }

  val cache = new File(cacheOptions.cache)

@ -279,7 +301,7 @@ class Helper(
      None

  val fetchs = cachePolicies.map(p =>
-    Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool)
+    Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool, ttl = ttl0)
  )
  val fetchQuiet = coursier.Fetch.from(
    repositories,
@ -524,11 +546,21 @@ class Helper(
    if (verbosityLevel >= 1 && artifacts0.nonEmpty)
      println(s"  Found ${artifacts0.length} artifacts")

-    val tasks = artifacts0.map(artifact =>
-      (Cache.file(artifact, cache, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
-        _ orElse Cache.file(artifact, cache, _, checksums = checksums, logger = logger, pool = pool)
-      ).run.map(artifact.->)
-    )
+    val tasks = artifacts0.map { artifact =>
+      def file(policy: CachePolicy) = Cache.file(
+        artifact,
+        cache,
+        policy,
+        checksums = checksums,
+        logger = logger,
+        pool = pool,
+        ttl = ttl0
+      )
+
+      (file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_))
+        .run
+        .map(artifact.->)
+    }

    logger.foreach(_.init())

--- a/cli/src/main/scala-2.11/coursier/cli/Options.scala
+++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala
@ -11,7 +11,11 @@ case class CommonOptions(
  @Help("Download mode (default: missing, that is fetch things missing from cache)")
  @Value("offline|update-changing|update|missing|force")
  @Short("m")
-    mode: String = "default",
+    mode: String = "",
+  @Help("TTL duration (e.g. \"24 hours\")")
+  @Value("duration")
+  @Short("l")
+    ttl: String,
  @Help("Quiet output")
  @Short("q")
    quiet: Boolean,
--- a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala
+++ b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala
@ -15,6 +15,7 @@ object CoursierPlugin extends AutoPlugin {
    val coursierChecksums = Keys.coursierChecksums
    val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums
    val coursierCachePolicies = Keys.coursierCachePolicies
+    val coursierTtl = Keys.coursierTtl
    val coursierVerbosity = Keys.coursierVerbosity
    val coursierSourceRepositories = Keys.coursierSourceRepositories
    val coursierResolvers = Keys.coursierResolvers
@ -58,7 +59,8 @@ object CoursierPlugin extends AutoPlugin {
    coursierMaxIterations := 50,
    coursierChecksums := Seq(Some("SHA-1"), None),
    coursierArtifactsChecksums := Seq(None),
-    coursierCachePolicies := Settings.defaultCachePolicies,
+    coursierCachePolicies := CachePolicy.default,
+    coursierTtl := Cache.defaultTtl,
    coursierVerbosity := Settings.defaultVerbosityLevel,
    coursierSourceRepositories := Nil,
    coursierResolvers <<= Tasks.coursierResolversTask,
--- a/plugin/src/main/scala-2.10/coursier/Keys.scala
+++ b/plugin/src/main/scala-2.10/coursier/Keys.scala
@ -4,14 +4,18 @@ import java.io.File
 import java.net.URL

 import coursier.core.Publication
+
 import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey }

+import scala.concurrent.duration.FiniteDuration
+
 object Keys {
  val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads")
  val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations")
  val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums")
  val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums")
  val coursierCachePolicies = SettingKey[Seq[CachePolicy]]("coursier-cache-policies")
+  val coursierTtl = SettingKey[Option[FiniteDuration]]("coursier-ttl")

  val coursierVerbosity = SettingKey[Int]("coursier-verbosity")

--- a/plugin/src/main/scala-2.10/coursier/Settings.scala
+++ b/plugin/src/main/scala-2.10/coursier/Settings.scala
@ -40,46 +40,4 @@ object Settings {
      .getOrElse(baseDefaultVerbosityLevel)
  }

-
-  private val baseDefaultCachePolicies = Seq(
-    CachePolicy.LocalOnly,
-    CachePolicy.FetchMissing
-  )
-
-  def defaultCachePolicies: Seq[CachePolicy] = {
-
-    def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] =
-      value.filter(_.nonEmpty).flatMap {
-        str =>
-          CacheParse.cachePolicies(str) match {
-            case scalaz.Success(Seq()) =>
-              Console.err.println(
-                s"Warning: no mode found in $description, ignoring it."
-              )
-              None
-            case scalaz.Success(policies) =>
-              Some(policies)
-            case scalaz.Failure(errors) =>
-              Console.err.println(
-                s"Warning: unrecognized mode in $description, ignoring it."
-              )
-              None
-          }
-      }
-
-    val fromEnv = fromOption(
-      sys.env.get("COURSIER_MODE"),
-      "COURSIER_MODE environment variable"
-    )
-
-    def fromProps = fromOption(
-      sys.props.get("coursier.mode"),
-      "Java property coursier.mode"
-    )
-
-    fromEnv
-      .orElse(fromProps)
-      .getOrElse(baseDefaultCachePolicies)
-  }
-
 }
--- a/plugin/src/main/scala-2.10/coursier/Tasks.scala
+++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala
@ -269,6 +269,7 @@ object Tasks {
      val checksums = coursierChecksums.value
      val maxIterations = coursierMaxIterations.value
      val cachePolicies = coursierCachePolicies.value
+      val ttl = coursierTtl.value
      val cache = coursierCache.value

      val log = streams.value.log
@ -461,9 +462,9 @@ object Tasks {

        val fetch = Fetch.from(
          repositories,
-          Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool),
+          Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl),
          cachePolicies.tail.map(p =>
-            Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool)
+            Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl)
          ): _*
        )

@ -610,6 +611,7 @@ object Tasks {
      val parallelDownloads = coursierParallelDownloads.value
      val artifactsChecksums = coursierArtifactsChecksums.value
      val cachePolicies = coursierCachePolicies.value
+      val ttl = coursierTtl.value
      val cache = coursierCache.value

      val log = streams.value.log
@ -687,7 +689,8 @@ object Tasks {
              p,
              checksums = artifactsChecksums,
              logger = Some(artifactsLogger),
-              pool = pool
+              pool = pool,
+              ttl = ttl
            )

          cachePolicies.tail