From be3fa1b94f39ef62a1777c94319428a1a874021d Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Sat, 4 Jul 2015 15:19:40 +0100 Subject: [PATCH] Better repository management --- .../main/scala/coursier/cli/Coursier.scala | 107 ++++++++++---- .../scala/coursier/cli/Repositories.scala | 87 +++++++++++ files/src/main/scala/coursier/Cache.scala | 137 ++++++++++++++++++ 3 files changed, 303 insertions(+), 28 deletions(-) create mode 100644 cli/src/main/scala/coursier/cli/Repositories.scala create mode 100644 files/src/main/scala/coursier/Cache.scala diff --git a/cli/src/main/scala/coursier/cli/Coursier.scala b/cli/src/main/scala/coursier/cli/Coursier.scala index e0d391a20..1fc1357cc 100644 --- a/cli/src/main/scala/coursier/cli/Coursier.scala +++ b/cli/src/main/scala/coursier/cli/Coursier.scala @@ -4,9 +4,9 @@ package cli import java.io.File import caseapp._ -import coursier.core.{ MavenRepository, Parse, Repository, CachePolicy } +import coursier.core.{ MavenRepository, Parse, CachePolicy } -import scalaz.{\/-, -\/} +import scalaz.{ \/-, -\/ } import scalaz.concurrent.Task case class Coursier( @@ -16,7 +16,8 @@ case class Coursier( @ExtraName("P") @ExtraName("cp") classpath: Boolean, @ExtraName("c") offline: Boolean, @ExtraName("v") verbose: List[Unit], - @ExtraName("N") maxIterations: Int = 100 + @ExtraName("N") maxIterations: Int = 100, + @ExtraName("r") repository: List[String] ) extends App { val verbose0 = verbose.length @@ -26,15 +27,30 @@ case class Coursier( else scope.map(Parse.scope) val scopes = scopes0.toSet - val centralCacheDir = new File(sys.props("user.home") + "/.coursier/cache/metadata/central") - val centralFilesCacheDir = new File(sys.props("user.home") + "/.coursier/cache/files/central") - def fileRepr(f: File) = f.toString def println(s: String) = Console.err.println(s) - val logger: MavenRepository.Logger with Files.Logger = + def defaultLogger: MavenRepository.Logger with Files.Logger = + new MavenRepository.Logger with Files.Logger { + def downloading(url: String) = + println(s"Downloading $url") + def downloaded(url: String, success: Boolean) = + if (!success) + println(s"Failed to download $url") + def readingFromCache(f: File) = {} + def puttingInCache(f: File) = {} + + def foundLocally(f: File) = {} + def downloadingArtifact(url: String) = + println(s"Downloading $url") + def downloadedArtifact(url: String, success: Boolean) = + if (!success) + println(s"Failed to download $url") + } + + def verboseLogger: MavenRepository.Logger with Files.Logger = new MavenRepository.Logger with Files.Logger { def downloading(url: String) = println(s"Downloading $url") @@ -60,22 +76,54 @@ case class Coursier( ) } + val logger = if (verbose0 <= 0) defaultLogger else verboseLogger + implicit val cachePolicy = if (offline) CachePolicy.LocalOnly else CachePolicy.Default - val cachedMavenCentral = Repository.mavenCentral.copy( - cache = Some(centralCacheDir), - logger = if (verbose0 <= 1) None else Some(logger) - ) - val repositories = Seq[Repository]( - cachedMavenCentral, - Repository.ivy2Local.copy( - logger = if (verbose0 <= 1) None else Some(logger) + val cache = Cache.default + + val repositoryIds = { + val repository0 = repository + .flatMap(_.split(',')) + .map(_.trim) + .filter(_.nonEmpty) + + if (repository0.isEmpty) + cache.default() + else + repository0 + } + + val existingRepo = cache + .list() + .map(_._1) + .toSet + if (repositoryIds.exists(!existingRepo(_))) { + val notFound = repositoryIds + .filter(!existingRepo(_)) + + Console.err.println( + (if (notFound.lengthCompare(1) == 1) "Repository" else "Repositories") + + " not found: " + + notFound.mkString(", ") ) - ) + + sys.exit(1) + } + + + val (repositories0, fileCaches) = cache + .list() + .map{case (_, repo, cacheEntry) => (repo, cacheEntry)} + .unzip + + val repositories = repositories0 + .map(_.copy(logger = Some(logger))) + val (splitDependencies, malformed) = remainingArgs.toList .map(_.split(":", 3).toSeq) @@ -159,13 +207,9 @@ case class Coursier( val artifacts = res.artifacts - val files = new Files( - Seq( - cachedMavenCentral.root -> centralFilesCacheDir - ), - () => ???, - if (verbose0 <= 0) None else Some(logger) - ) + val files = cache + .files() + .copy(logger = Some(logger)) val tasks = artifacts.map(artifact => files.file(artifact, cachePolicy).run.map(artifact.->)) val task = Task.gatherUnordered(tasks) @@ -181,11 +225,18 @@ case class Coursier( } } - Console.out.println( - files0 - .map(_.toString) - .mkString(File.pathSeparator) - ) + if (classpath) + Console.out.println( + files0 + .map(_.toString) + .mkString(File.pathSeparator) + ) + else + println( + files0 + .map(_.toString) + .mkString("\n") + ) } } diff --git a/cli/src/main/scala/coursier/cli/Repositories.scala b/cli/src/main/scala/coursier/cli/Repositories.scala new file mode 100644 index 000000000..37903016f --- /dev/null +++ b/cli/src/main/scala/coursier/cli/Repositories.scala @@ -0,0 +1,87 @@ +package coursier.cli + +import coursier.Cache +import caseapp._ + +// TODO: allow removing a repository (with confirmations, etc.) +case class Repositories( + @ValueDescription("id:baseUrl") @ExtraName("a") add: List[String], + @ExtraName("L") list: Boolean, + @ExtraName("l") defaultList: Boolean, + ivyLike: Boolean +) extends App { + + if (add.exists(!_.contains(":"))) { + CaseApp.printUsage[Repositories](err = true) + sys.exit(255) + } + + val add0 = add + .map{ s => + val Seq(id, baseUrl) = s.split(":", 2).toSeq + id -> baseUrl + } + + if ( + add0.exists(_._1.contains("/")) || + add0.exists(_._1.startsWith(".")) || + add0.exists(_._1.isEmpty) + ) { + CaseApp.printUsage[Repositories](err = true) + sys.exit(255) + } + + + val cache = Cache.default + + if (cache.cache.exists() && !cache.cache.isDirectory) { + Console.err.println(s"Error: ${cache.cache} not a directory") + sys.exit(1) + } + + if (!cache.cache.exists()) + cache.init() + + val current = cache.list().map(_._1).toSet + + val alreadyAdded = add0 + .map(_._1) + .filter(current) + + if (alreadyAdded.nonEmpty) { + Console.err.println(s"Error: already added: ${alreadyAdded.mkString(", ")}") + sys.exit(1) + } + + for ((id, baseUrl0) <- add0) { + val baseUrl = + if (baseUrl0.endsWith("/")) + baseUrl0 + else + baseUrl0 + "/" + + cache.add(id, baseUrl, ivyLike = ivyLike) + } + + if (defaultList) { + val map = cache.repositoryMap() + + for (id <- cache.default(withNotFound = true)) + map.get(id) match { + case Some(repo) => + println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) + case None => + println(s"$id (not found)") + } + } + + if (list) + for ((id, repo, _) <- cache.list().sortBy(_._1)) { + println(s"$id: ${repo.root}" + (if (repo.ivyLike) " (Ivy-like)" else "")) + } + +} + +object Repositories extends AppOf[Repositories] { + val parser = default +} diff --git a/files/src/main/scala/coursier/Cache.scala b/files/src/main/scala/coursier/Cache.scala new file mode 100644 index 000000000..ebd80eff9 --- /dev/null +++ b/files/src/main/scala/coursier/Cache.scala @@ -0,0 +1,137 @@ +package coursier + +import java.io.{PrintWriter, File} + +import coursier.core.MavenRepository + +import scala.io.Source + +object Cache { + + def mavenRepository(lines: Seq[String]): Option[MavenRepository] = { + def isMaven = + lines + .find(_.startsWith("maven:")) + .map(_.stripPrefix("maven:").trim) + .contains("true") + + def ivyLike = + lines + .find(_.startsWith("ivy-like:")) + .map(_.stripPrefix("ivy-like:").trim) + .contains("true") + + def base = + lines + .find(_.startsWith("base:")) + .map(_.stripPrefix("base:").trim) + .filter(_.nonEmpty) + + if (isMaven) + base.map(MavenRepository(_, ivyLike = ivyLike)) + else + None + } + + lazy val default = Cache(new File(sys.props("user.home") + "/.coursier/cache")) + +} + +case class Cache(cache: File) { + + import Cache._ + + lazy val repoDir = new File(cache, "repositories") + lazy val metadataBase = new File(cache, "metadata") + lazy val fileBase = new File(cache, "files") + + lazy val defaultFile = new File(repoDir, ".default") + + def add(id: String, base: String, ivyLike: Boolean): Unit = { + repoDir.mkdirs() + val f = new File(repoDir, id) + val w = new PrintWriter(f) + try w.println((Seq("maven: true", s"base: $base") ++ (if (ivyLike) Seq("ivy-like: true") else Nil)).mkString("\n")) + finally w.close() + } + + def addCentral(): Unit = + add("central", "https://repo1.maven.org/maven2/", ivyLike = false) + + def addIvy2Local(): Unit = + add("ivy2local", "file://" + sys.props("user.home") + "/.ivy2/local/", ivyLike = true) + + def init(ifEmpty: Boolean = true): Unit = + if (!ifEmpty || !cache.exists()) { + repoDir.mkdirs() + metadataBase.mkdirs() + fileBase.mkdirs() + addCentral() + addIvy2Local() + setDefault("ivy2local", "central") + } + + def setDefault(ids: String*): Unit = { + defaultFile.getParentFile.mkdirs() + val w = new PrintWriter(defaultFile) + try w.println(ids.mkString("\n")) + finally w.close() + } + + def list(): Seq[(String, MavenRepository, (String, File))] = + Option(repoDir.listFiles()) + .map(_.toSeq) + .getOrElse(Nil) + .filter(f => f.isFile && !f.getName.startsWith(".")) + .flatMap { f => + val name = f.getName + val lines = Source.fromFile(f).getLines().toList + mavenRepository(lines) + .map(repo => + (name, repo.copy(cache = Some(new File(metadataBase, name))), (repo.root, new File(fileBase, name))) + ) + } + + def map(): Map[String, (MavenRepository, (String, File))] = + list() + .map{case (id, repo, fileCache) => id -> (repo, fileCache) } + .toMap + + + def repositories(): Seq[MavenRepository] = + list().map(_._2) + + def repositoryMap(): Map[String, MavenRepository] = + list() + .map{case (id, repo, _) => id -> repo} + .toMap + + def fileCaches(): Seq[(String, File)] = + list().map(_._3) + + def default(withNotFound: Boolean = false): Seq[String] = + if (defaultFile.exists()) { + val default0 = + Source.fromFile(defaultFile) + .getLines() + .map(_.trim) + .filter(_.nonEmpty) + .toList + + val found = list() + .map(_._1) + .toSet + + default0 + .filter(found) + } else + Nil + + def files(): Files = { + val map0 = map() + val default0 = default() + + new Files(default0.map(map0(_)._2), () => ???) + } + +}