Add support for TTL

build.sbt

@@ -248,6 +248,9 @@ lazy val cache = project
       import com.typesafe.tools.mima.core.ProblemFilters._
       
       Seq(
+        // Since 1.0.0-M13
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.file"),
+        ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.fetch"),
         // Since 1.0.0-M12
         // Remove deprecated / unused helper method
         ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.readFully"),

cache/src/main/scala/coursier/Cache.scala

@@ -20,6 +20,9 @@ import scalaz.concurrent.{ Task, Strategy }
 
 import java.io.{ Serializable => _, _ }
 
+import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration }
+import scala.util.Try
+
 trait AuthenticatedURLConnection extends URLConnection {
   def authenticate(authentication: Authentication): Unit
 }
@@ -306,7 +309,8 @@ object Cache {
     checksums: Set[String],
     cachePolicy: CachePolicy,
     pool: ExecutorService,
-    logger: Option[Logger] = None
+    logger: Option[Logger] = None,
+    ttl: Option[FiniteDuration] = defaultTtl
   ): Task[Seq[((File, String), FileError \/ Unit)]] = {
 
     implicit val pool0 = pool
@@ -402,7 +406,44 @@ object Cache {
         file.exists()
       }
 
+    def ttlFile(file: File): File =
+      new File(file.getParent, s".${file.getName}.checked")
+
+    def lastCheck(file: File): Task[Option[Long]] = {
+
+      val ttlFile0 = ttlFile(file)
+
+      Task {
+        if (ttlFile0.exists())
+          Some(ttlFile0.lastModified()).filter(_ > 0L)
+        else
+          None
+      }
+    }
+
+    /** Not wrapped in a `Task` !!! */
+    def doTouchCheckFile(file: File): Unit = {
+      val ts = System.currentTimeMillis()
+      val f = ttlFile(file)
+      if (f.exists())
+        f.setLastModified(ts)
+      else {
+        val fos = new FileOutputStream(f)
+        fos.write(Array.empty[Byte])
+        fos.close()
+      }
+    }
+
     def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = {
+
+      def checkNeeded = ttl.map(_.toMillis).filter(_ > 0L).fold(Task.now(true)) { ttlMs =>
+        lastCheck(file).flatMap {
+          case None => Task.now(true)
+          case Some(ts) =>
+            Task(System.currentTimeMillis()).map(_ > ts + ttlMs)
+        }
+      }
+
       def check = for {
         fileLastModOpt <- fileLastModified(file)
         urlLastModOpt <- urlLastModified(url, fileLastModOpt, logger)
@@ -420,7 +461,20 @@ object Cache {
           case false =>
             Task.now(true.right)
           case true =>
-            check.run
+            checkNeeded.flatMap {
+              case false =>
+                Task.now(false.right)
+              case true =>
+                check.run.flatMap {
+                  case \/-(false) =>
+                    Task {
+                      doTouchCheckFile(file)
+                      \/-(false)
+                    }
+                  case other =>
+                    Task.now(other)
+                }
+            }
         }
       }
     }
@@ -493,7 +547,7 @@ object Cache {
                       tmp.getParentFile.mkdirs()
                       new FileOutputStream(tmp, partialDownload)
                     }
-                    try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L))
+                    try readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)
                     finally out.close()
                   } finally in.close()
 
@@ -505,7 +559,9 @@ object Cache {
                 for (lastModified <- Option(conn.getLastModified) if lastModified > 0L)
                   file.setLastModified(lastModified)
 
-                result
+                doTouchCheckFile(file)
+
+                result.right
               }
             }
           }
@@ -725,7 +781,8 @@ object Cache {
     cachePolicy: CachePolicy = CachePolicy.FetchMissing,
     checksums: Seq[Option[String]] = defaultChecksums,
     logger: Option[Logger] = None,
-    pool: ExecutorService = defaultPool
+    pool: ExecutorService = defaultPool,
+    ttl: Option[FiniteDuration] = defaultTtl
   ): EitherT[Task, FileError, File] = {
 
     implicit val pool0 = pool
@@ -739,7 +796,8 @@ object Cache {
         checksums = checksums0.collect { case Some(c) => c }.toSet,
         cachePolicy,
         pool,
-        logger = logger
+        logger = logger,
+        ttl = ttl
       ).map { results =>
         val checksum = checksums0.find {
           case None => true
@@ -776,7 +834,8 @@ object Cache {
     cachePolicy: CachePolicy = CachePolicy.FetchMissing,
     checksums: Seq[Option[String]] = defaultChecksums,
     logger: Option[Logger] = None,
-    pool: ExecutorService = defaultPool
+    pool: ExecutorService = defaultPool,
+    ttl: Option[FiniteDuration] = defaultTtl
   ): Fetch.Content[Task] = {
     artifact =>
       file(
@@ -785,7 +844,8 @@ object Cache {
         cachePolicy,
         checksums = checksums,
         logger = logger,
-        pool = pool
+        pool = pool,
+        ttl = ttl
       ).leftMap(_.describe).map { f =>
         // FIXME Catch error here?
         new String(NioFiles.readAllBytes(f.toPath), "UTF-8")
@@ -830,6 +890,20 @@ object Cache {
   lazy val defaultPool =
     Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory)
 
+  lazy val defaultTtl: Option[FiniteDuration] = {
+    def fromString(s: String) =
+      Try(Duration(s)).toOption.collect {
+        case d: FiniteDuration => d
+      }
+
+    val fromEnv = sys.env.get("COURSIER_TTL").flatMap(fromString)
+    def fromProps = sys.props.get("coursier.ttl").flatMap(fromString)
+    def default = 24.days
+
+    fromEnv
+      .orElse(fromProps)
+      .orElse(Some(default))
+  }
 
   private val urlLocks = new ConcurrentHashMap[String, Object]
 

cache/src/main/scala/coursier/CachePolicy.scala

@@ -55,4 +55,47 @@ object CachePolicy {
     * Erases files already in cache.
     */
   case object ForceDownload extends CachePolicy
+
+
+  private val baseDefault = Seq(
+    CachePolicy.LocalOnly,
+    CachePolicy.FetchMissing
+  )
+
+  def default: Seq[CachePolicy] = {
+
+    def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] =
+      value.filter(_.nonEmpty).flatMap {
+        str =>
+          CacheParse.cachePolicies(str) match {
+            case scalaz.Success(Seq()) =>
+              Console.err.println(
+                s"Warning: no mode found in $description, ignoring it."
+              )
+              None
+            case scalaz.Success(policies) =>
+              Some(policies)
+            case scalaz.Failure(errors) =>
+              Console.err.println(
+                s"Warning: unrecognized mode in $description, ignoring it."
+              )
+              None
+          }
+      }
+
+    val fromEnv = fromOption(
+      sys.env.get("COURSIER_MODE"),
+      "COURSIER_MODE environment variable"
+    )
+
+    def fromProps = fromOption(
+      sys.props.get("coursier.mode"),
+      "Java property coursier.mode"
+    )
+
+    fromEnv
+      .orElse(fromProps)
+      .getOrElse(baseDefault)
+  }
+
 }

cli/src/main/scala-2.11/coursier/cli/Helper.scala

@@ -10,6 +10,8 @@ import coursier.ivy.IvyRepository
 import coursier.util.{Print, Parse}
 
 import scala.annotation.tailrec
+import scala.concurrent.duration.{ Duration, FiniteDuration }
+
 import scalaz.{Failure, Success, \/-, -\/}
 import scalaz.concurrent.{ Task, Strategy }
 
@@ -80,15 +82,35 @@ class Helper(
 
   import Util._
 
-  val cachePoliciesValidation = CacheParse.cachePolicies(common.mode)
+  val ttl0 =
+    if (ttl.isEmpty)
+      Cache.defaultTtl
+    else {
+      val d = try {
+        Duration(ttl)
+      } catch {
+        case e: Exception =>
+          prematureExit(s"Unrecognized TTL duration: $ttl")
+      }
 
-  val cachePolicies = cachePoliciesValidation match {
+      d match {
-    case Success(cp) => cp
+        case f: FiniteDuration => Some(f)
-    case Failure(errors) =>
+        case _ =>
-      prematureExit(
+          prematureExit(s"Non finite TTL duration: $ttl")
-        s"Error parsing modes:\n${errors.list.map("  "+_).mkString("\n")}"
+      }
-      )
+    }
-  }
+
+  val cachePolicies =
+    if (common.mode.isEmpty)
+      CachePolicy.default
+    else
+      CacheParse.cachePolicies(common.mode) match {
+        case Success(cp) => cp
+        case Failure(errors) =>
+          prematureExit(
+            s"Error parsing modes:\n${errors.list.map("  "+_).mkString("\n")}"
+          )
+      }
 
   val cache = new File(cacheOptions.cache)
 
@@ -279,7 +301,7 @@ class Helper(
       None
 
   val fetchs = cachePolicies.map(p =>
-    Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool)
+    Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool, ttl = ttl0)
   )
   val fetchQuiet = coursier.Fetch.from(
     repositories,
@@ -524,11 +546,21 @@ class Helper(
     if (verbosityLevel >= 1 && artifacts0.nonEmpty)
       println(s"  Found ${artifacts0.length} artifacts")
 
-    val tasks = artifacts0.map(artifact =>
+    val tasks = artifacts0.map { artifact =>
-      (Cache.file(artifact, cache, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)(
+      def file(policy: CachePolicy) = Cache.file(
-        _ orElse Cache.file(artifact, cache, _, checksums = checksums, logger = logger, pool = pool)
+        artifact,
-      ).run.map(artifact.->)
+        cache,
-    )
+        policy,
+        checksums = checksums,
+        logger = logger,
+        pool = pool,
+        ttl = ttl0
+      )
+
+      (file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_))
+        .run
+        .map(artifact.->)
+    }
 
     logger.foreach(_.init())
 

cli/src/main/scala-2.11/coursier/cli/Options.scala

@@ -11,7 +11,11 @@ case class CommonOptions(
   @Help("Download mode (default: missing, that is fetch things missing from cache)")
   @Value("offline|update-changing|update|missing|force")
   @Short("m")
-    mode: String = "default",
+    mode: String = "",
+  @Help("TTL duration (e.g. \"24 hours\")")
+  @Value("duration")
+  @Short("l")
+    ttl: String,
   @Help("Quiet output")
   @Short("q")
     quiet: Boolean,

plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala

@@ -15,6 +15,7 @@ object CoursierPlugin extends AutoPlugin {
     val coursierChecksums = Keys.coursierChecksums
     val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums
     val coursierCachePolicies = Keys.coursierCachePolicies
+    val coursierTtl = Keys.coursierTtl
     val coursierVerbosity = Keys.coursierVerbosity
     val coursierSourceRepositories = Keys.coursierSourceRepositories
     val coursierResolvers = Keys.coursierResolvers
@@ -58,7 +59,8 @@ object CoursierPlugin extends AutoPlugin {
     coursierMaxIterations := 50,
     coursierChecksums := Seq(Some("SHA-1"), None),
     coursierArtifactsChecksums := Seq(None),
-    coursierCachePolicies := Settings.defaultCachePolicies,
+    coursierCachePolicies := CachePolicy.default,
+    coursierTtl := Cache.defaultTtl,
     coursierVerbosity := Settings.defaultVerbosityLevel,
     coursierSourceRepositories := Nil,
     coursierResolvers <<= Tasks.coursierResolversTask,

plugin/src/main/scala-2.10/coursier/Keys.scala

@@ -4,14 +4,18 @@ import java.io.File
 import java.net.URL
 
 import coursier.core.Publication
+
 import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey }
 
+import scala.concurrent.duration.FiniteDuration
+
 object Keys {
   val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads")
   val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations")
   val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums")
   val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums")
   val coursierCachePolicies = SettingKey[Seq[CachePolicy]]("coursier-cache-policies")
+  val coursierTtl = SettingKey[Option[FiniteDuration]]("coursier-ttl")
 
   val coursierVerbosity = SettingKey[Int]("coursier-verbosity")
 

plugin/src/main/scala-2.10/coursier/Settings.scala

@@ -40,46 +40,4 @@ object Settings {
       .getOrElse(baseDefaultVerbosityLevel)
   }
 
-
-  private val baseDefaultCachePolicies = Seq(
-    CachePolicy.LocalOnly,
-    CachePolicy.FetchMissing
-  )
-
-  def defaultCachePolicies: Seq[CachePolicy] = {
-
-    def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] =
-      value.filter(_.nonEmpty).flatMap {
-        str =>
-          CacheParse.cachePolicies(str) match {
-            case scalaz.Success(Seq()) =>
-              Console.err.println(
-                s"Warning: no mode found in $description, ignoring it."
-              )
-              None
-            case scalaz.Success(policies) =>
-              Some(policies)
-            case scalaz.Failure(errors) =>
-              Console.err.println(
-                s"Warning: unrecognized mode in $description, ignoring it."
-              )
-              None
-          }
-      }
-
-    val fromEnv = fromOption(
-      sys.env.get("COURSIER_MODE"),
-      "COURSIER_MODE environment variable"
-    )
-
-    def fromProps = fromOption(
-      sys.props.get("coursier.mode"),
-      "Java property coursier.mode"
-    )
-
-    fromEnv
-      .orElse(fromProps)
-      .getOrElse(baseDefaultCachePolicies)
-  }
-
 }

plugin/src/main/scala-2.10/coursier/Tasks.scala

@@ -269,6 +269,7 @@ object Tasks {
       val checksums = coursierChecksums.value
       val maxIterations = coursierMaxIterations.value
       val cachePolicies = coursierCachePolicies.value
+      val ttl = coursierTtl.value
       val cache = coursierCache.value
 
       val log = streams.value.log
@@ -461,9 +462,9 @@ object Tasks {
 
         val fetch = Fetch.from(
           repositories,
-          Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool),
+          Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl),
           cachePolicies.tail.map(p =>
-            Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool)
+            Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl)
           ): _*
         )
 
@@ -610,6 +611,7 @@ object Tasks {
       val parallelDownloads = coursierParallelDownloads.value
       val artifactsChecksums = coursierArtifactsChecksums.value
       val cachePolicies = coursierCachePolicies.value
+      val ttl = coursierTtl.value
       val cache = coursierCache.value
 
       val log = streams.value.log
@@ -687,7 +689,8 @@ object Tasks {
               p,
               checksums = artifactsChecksums,
               logger = Some(artifactsLogger),
-              pool = pool
+              pool = pool,
+              ttl = ttl
             )
 
           cachePolicies.tail