From b76fbf363a3add3ba2e9d9476d92004d1406232b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 31 May 2016 15:18:29 +0200 Subject: [PATCH] Add support for TTL --- build.sbt | 3 + cache/src/main/scala/coursier/Cache.scala | 90 +++++++++++++++++-- .../src/main/scala/coursier/CachePolicy.scala | 43 +++++++++ .../main/scala-2.11/coursier/cli/Helper.scala | 60 ++++++++++--- .../scala-2.11/coursier/cli/Options.scala | 6 +- .../scala-2.10/coursier/CoursierPlugin.scala | 4 +- .../src/main/scala-2.10/coursier/Keys.scala | 4 + .../main/scala-2.10/coursier/Settings.scala | 42 --------- .../src/main/scala-2.10/coursier/Tasks.scala | 9 +- 9 files changed, 192 insertions(+), 69 deletions(-) diff --git a/build.sbt b/build.sbt index 5a3e39a4e..9b5a470a8 100644 --- a/build.sbt +++ b/build.sbt @@ -248,6 +248,9 @@ lazy val cache = project import com.typesafe.tools.mima.core.ProblemFilters._ Seq( + // Since 1.0.0-M13 + ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.file"), + ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.fetch"), // Since 1.0.0-M12 // Remove deprecated / unused helper method ProblemFilters.exclude[MissingMethodProblem]("coursier.Cache.readFully"), diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index f0f651749..f00f793ec 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -20,6 +20,9 @@ import scalaz.concurrent.{ Task, Strategy } import java.io.{ Serializable => _, _ } +import scala.concurrent.duration.{ Duration, DurationInt, FiniteDuration } +import scala.util.Try + trait AuthenticatedURLConnection extends URLConnection { def authenticate(authentication: Authentication): Unit } @@ -306,7 +309,8 @@ object Cache { checksums: Set[String], cachePolicy: CachePolicy, pool: ExecutorService, - logger: Option[Logger] = None + logger: Option[Logger] = None, + ttl: Option[FiniteDuration] = defaultTtl ): Task[Seq[((File, String), FileError \/ Unit)]] = { implicit val pool0 = pool @@ -402,7 +406,44 @@ object Cache { file.exists() } + def ttlFile(file: File): File = + new File(file.getParent, s".${file.getName}.checked") + + def lastCheck(file: File): Task[Option[Long]] = { + + val ttlFile0 = ttlFile(file) + + Task { + if (ttlFile0.exists()) + Some(ttlFile0.lastModified()).filter(_ > 0L) + else + None + } + } + + /** Not wrapped in a `Task` !!! */ + def doTouchCheckFile(file: File): Unit = { + val ts = System.currentTimeMillis() + val f = ttlFile(file) + if (f.exists()) + f.setLastModified(ts) + else { + val fos = new FileOutputStream(f) + fos.write(Array.empty[Byte]) + fos.close() + } + } + def shouldDownload(file: File, url: String): EitherT[Task, FileError, Boolean] = { + + def checkNeeded = ttl.map(_.toMillis).filter(_ > 0L).fold(Task.now(true)) { ttlMs => + lastCheck(file).flatMap { + case None => Task.now(true) + case Some(ts) => + Task(System.currentTimeMillis()).map(_ > ts + ttlMs) + } + } + def check = for { fileLastModOpt <- fileLastModified(file) urlLastModOpt <- urlLastModified(url, fileLastModOpt, logger) @@ -420,7 +461,20 @@ object Cache { case false => Task.now(true.right) case true => - check.run + checkNeeded.flatMap { + case false => + Task.now(false.right) + case true => + check.run.flatMap { + case \/-(false) => + Task { + doTouchCheckFile(file) + \/-(false) + } + case other => + Task.now(other) + } + } } } } @@ -493,7 +547,7 @@ object Cache { tmp.getParentFile.mkdirs() new FileOutputStream(tmp, partialDownload) } - try \/-(readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L)) + try readFullyTo(in, out, logger, url, if (partialDownload) alreadyDownloaded else 0L) finally out.close() } finally in.close() @@ -505,7 +559,9 @@ object Cache { for (lastModified <- Option(conn.getLastModified) if lastModified > 0L) file.setLastModified(lastModified) - result + doTouchCheckFile(file) + + result.right } } } @@ -725,7 +781,8 @@ object Cache { cachePolicy: CachePolicy = CachePolicy.FetchMissing, checksums: Seq[Option[String]] = defaultChecksums, logger: Option[Logger] = None, - pool: ExecutorService = defaultPool + pool: ExecutorService = defaultPool, + ttl: Option[FiniteDuration] = defaultTtl ): EitherT[Task, FileError, File] = { implicit val pool0 = pool @@ -739,7 +796,8 @@ object Cache { checksums = checksums0.collect { case Some(c) => c }.toSet, cachePolicy, pool, - logger = logger + logger = logger, + ttl = ttl ).map { results => val checksum = checksums0.find { case None => true @@ -776,7 +834,8 @@ object Cache { cachePolicy: CachePolicy = CachePolicy.FetchMissing, checksums: Seq[Option[String]] = defaultChecksums, logger: Option[Logger] = None, - pool: ExecutorService = defaultPool + pool: ExecutorService = defaultPool, + ttl: Option[FiniteDuration] = defaultTtl ): Fetch.Content[Task] = { artifact => file( @@ -785,7 +844,8 @@ object Cache { cachePolicy, checksums = checksums, logger = logger, - pool = pool + pool = pool, + ttl = ttl ).leftMap(_.describe).map { f => // FIXME Catch error here? new String(NioFiles.readAllBytes(f.toPath), "UTF-8") @@ -830,6 +890,20 @@ object Cache { lazy val defaultPool = Executors.newFixedThreadPool(defaultConcurrentDownloadCount, Strategy.DefaultDaemonThreadFactory) + lazy val defaultTtl: Option[FiniteDuration] = { + def fromString(s: String) = + Try(Duration(s)).toOption.collect { + case d: FiniteDuration => d + } + + val fromEnv = sys.env.get("COURSIER_TTL").flatMap(fromString) + def fromProps = sys.props.get("coursier.ttl").flatMap(fromString) + def default = 24.days + + fromEnv + .orElse(fromProps) + .orElse(Some(default)) + } private val urlLocks = new ConcurrentHashMap[String, Object] diff --git a/cache/src/main/scala/coursier/CachePolicy.scala b/cache/src/main/scala/coursier/CachePolicy.scala index 16a16d87d..da83b4e6a 100644 --- a/cache/src/main/scala/coursier/CachePolicy.scala +++ b/cache/src/main/scala/coursier/CachePolicy.scala @@ -55,4 +55,47 @@ object CachePolicy { * Erases files already in cache. */ case object ForceDownload extends CachePolicy + + + private val baseDefault = Seq( + CachePolicy.LocalOnly, + CachePolicy.FetchMissing + ) + + def default: Seq[CachePolicy] = { + + def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] = + value.filter(_.nonEmpty).flatMap { + str => + CacheParse.cachePolicies(str) match { + case scalaz.Success(Seq()) => + Console.err.println( + s"Warning: no mode found in $description, ignoring it." + ) + None + case scalaz.Success(policies) => + Some(policies) + case scalaz.Failure(errors) => + Console.err.println( + s"Warning: unrecognized mode in $description, ignoring it." + ) + None + } + } + + val fromEnv = fromOption( + sys.env.get("COURSIER_MODE"), + "COURSIER_MODE environment variable" + ) + + def fromProps = fromOption( + sys.props.get("coursier.mode"), + "Java property coursier.mode" + ) + + fromEnv + .orElse(fromProps) + .getOrElse(baseDefault) + } + } diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index a1de3bde1..219b97e6e 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -10,6 +10,8 @@ import coursier.ivy.IvyRepository import coursier.util.{Print, Parse} import scala.annotation.tailrec +import scala.concurrent.duration.{ Duration, FiniteDuration } + import scalaz.{Failure, Success, \/-, -\/} import scalaz.concurrent.{ Task, Strategy } @@ -80,15 +82,35 @@ class Helper( import Util._ - val cachePoliciesValidation = CacheParse.cachePolicies(common.mode) + val ttl0 = + if (ttl.isEmpty) + Cache.defaultTtl + else { + val d = try { + Duration(ttl) + } catch { + case e: Exception => + prematureExit(s"Unrecognized TTL duration: $ttl") + } - val cachePolicies = cachePoliciesValidation match { - case Success(cp) => cp - case Failure(errors) => - prematureExit( - s"Error parsing modes:\n${errors.list.map(" "+_).mkString("\n")}" - ) - } + d match { + case f: FiniteDuration => Some(f) + case _ => + prematureExit(s"Non finite TTL duration: $ttl") + } + } + + val cachePolicies = + if (common.mode.isEmpty) + CachePolicy.default + else + CacheParse.cachePolicies(common.mode) match { + case Success(cp) => cp + case Failure(errors) => + prematureExit( + s"Error parsing modes:\n${errors.list.map(" "+_).mkString("\n")}" + ) + } val cache = new File(cacheOptions.cache) @@ -279,7 +301,7 @@ class Helper( None val fetchs = cachePolicies.map(p => - Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool) + Cache.fetch(cache, p, checksums = checksums, logger = logger, pool = pool, ttl = ttl0) ) val fetchQuiet = coursier.Fetch.from( repositories, @@ -524,11 +546,21 @@ class Helper( if (verbosityLevel >= 1 && artifacts0.nonEmpty) println(s" Found ${artifacts0.length} artifacts") - val tasks = artifacts0.map(artifact => - (Cache.file(artifact, cache, cachePolicies.head, checksums = checksums, logger = logger, pool = pool) /: cachePolicies.tail)( - _ orElse Cache.file(artifact, cache, _, checksums = checksums, logger = logger, pool = pool) - ).run.map(artifact.->) - ) + val tasks = artifacts0.map { artifact => + def file(policy: CachePolicy) = Cache.file( + artifact, + cache, + policy, + checksums = checksums, + logger = logger, + pool = pool, + ttl = ttl0 + ) + + (file(cachePolicies.head) /: cachePolicies.tail)(_ orElse file(_)) + .run + .map(artifact.->) + } logger.foreach(_.init()) diff --git a/cli/src/main/scala-2.11/coursier/cli/Options.scala b/cli/src/main/scala-2.11/coursier/cli/Options.scala index e51e077cf..5439722a2 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Options.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Options.scala @@ -11,7 +11,11 @@ case class CommonOptions( @Help("Download mode (default: missing, that is fetch things missing from cache)") @Value("offline|update-changing|update|missing|force") @Short("m") - mode: String = "default", + mode: String = "", + @Help("TTL duration (e.g. \"24 hours\")") + @Value("duration") + @Short("l") + ttl: String, @Help("Quiet output") @Short("q") quiet: Boolean, diff --git a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala index e9ef4f01f..c4f6e4ace 100644 --- a/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala +++ b/plugin/src/main/scala-2.10/coursier/CoursierPlugin.scala @@ -15,6 +15,7 @@ object CoursierPlugin extends AutoPlugin { val coursierChecksums = Keys.coursierChecksums val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums val coursierCachePolicies = Keys.coursierCachePolicies + val coursierTtl = Keys.coursierTtl val coursierVerbosity = Keys.coursierVerbosity val coursierSourceRepositories = Keys.coursierSourceRepositories val coursierResolvers = Keys.coursierResolvers @@ -58,7 +59,8 @@ object CoursierPlugin extends AutoPlugin { coursierMaxIterations := 50, coursierChecksums := Seq(Some("SHA-1"), None), coursierArtifactsChecksums := Seq(None), - coursierCachePolicies := Settings.defaultCachePolicies, + coursierCachePolicies := CachePolicy.default, + coursierTtl := Cache.defaultTtl, coursierVerbosity := Settings.defaultVerbosityLevel, coursierSourceRepositories := Nil, coursierResolvers <<= Tasks.coursierResolversTask, diff --git a/plugin/src/main/scala-2.10/coursier/Keys.scala b/plugin/src/main/scala-2.10/coursier/Keys.scala index 4c5e20a52..86388bb98 100644 --- a/plugin/src/main/scala-2.10/coursier/Keys.scala +++ b/plugin/src/main/scala-2.10/coursier/Keys.scala @@ -4,14 +4,18 @@ import java.io.File import java.net.URL import coursier.core.Publication + import sbt.{ GetClassifiersModule, Resolver, SettingKey, TaskKey } +import scala.concurrent.duration.FiniteDuration + object Keys { val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads") val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations") val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums") val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums") val coursierCachePolicies = SettingKey[Seq[CachePolicy]]("coursier-cache-policies") + val coursierTtl = SettingKey[Option[FiniteDuration]]("coursier-ttl") val coursierVerbosity = SettingKey[Int]("coursier-verbosity") diff --git a/plugin/src/main/scala-2.10/coursier/Settings.scala b/plugin/src/main/scala-2.10/coursier/Settings.scala index a62f8899e..0629d1726 100644 --- a/plugin/src/main/scala-2.10/coursier/Settings.scala +++ b/plugin/src/main/scala-2.10/coursier/Settings.scala @@ -40,46 +40,4 @@ object Settings { .getOrElse(baseDefaultVerbosityLevel) } - - private val baseDefaultCachePolicies = Seq( - CachePolicy.LocalOnly, - CachePolicy.FetchMissing - ) - - def defaultCachePolicies: Seq[CachePolicy] = { - - def fromOption(value: Option[String], description: String): Option[Seq[CachePolicy]] = - value.filter(_.nonEmpty).flatMap { - str => - CacheParse.cachePolicies(str) match { - case scalaz.Success(Seq()) => - Console.err.println( - s"Warning: no mode found in $description, ignoring it." - ) - None - case scalaz.Success(policies) => - Some(policies) - case scalaz.Failure(errors) => - Console.err.println( - s"Warning: unrecognized mode in $description, ignoring it." - ) - None - } - } - - val fromEnv = fromOption( - sys.env.get("COURSIER_MODE"), - "COURSIER_MODE environment variable" - ) - - def fromProps = fromOption( - sys.props.get("coursier.mode"), - "Java property coursier.mode" - ) - - fromEnv - .orElse(fromProps) - .getOrElse(baseDefaultCachePolicies) - } - } diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index bfd3c2ca0..1798afce1 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -269,6 +269,7 @@ object Tasks { val checksums = coursierChecksums.value val maxIterations = coursierMaxIterations.value val cachePolicies = coursierCachePolicies.value + val ttl = coursierTtl.value val cache = coursierCache.value val log = streams.value.log @@ -461,9 +462,9 @@ object Tasks { val fetch = Fetch.from( repositories, - Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool), + Cache.fetch(cache, cachePolicies.head, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl), cachePolicies.tail.map(p => - Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool) + Cache.fetch(cache, p, checksums = checksums, logger = Some(resLogger), pool = pool, ttl = ttl) ): _* ) @@ -610,6 +611,7 @@ object Tasks { val parallelDownloads = coursierParallelDownloads.value val artifactsChecksums = coursierArtifactsChecksums.value val cachePolicies = coursierCachePolicies.value + val ttl = coursierTtl.value val cache = coursierCache.value val log = streams.value.log @@ -687,7 +689,8 @@ object Tasks { p, checksums = artifactsChecksums, logger = Some(artifactsLogger), - pool = pool + pool = pool, + ttl = ttl ) cachePolicies.tail