From c476b08f3298be1ba8c104d0400b3529412efa2b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 14 Jan 2016 00:42:10 +0100 Subject: [PATCH 1/3] Add support for Ivy repositories with different artifact / metadata patterns --- .../scala/coursier/ivy/IvyRepository.scala | 139 ++--------------- .../src/main/scala/coursier/ivy/Pattern.scala | 142 ++++++++++++++++++ .../main/scala-2.10/coursier/FromSbt.scala | 10 +- 3 files changed, 157 insertions(+), 134 deletions(-) create mode 100644 core/shared/src/main/scala/coursier/ivy/Pattern.scala diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index fb14f9738..5fc7e077d 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -2,83 +2,12 @@ package coursier.ivy import coursier.Fetch import coursier.core._ -import scala.annotation.tailrec -import scala.util.matching.Regex + import scalaz._ -import java.util.regex.Pattern.quote - -object IvyRepository { - - val optionalPartRegex = (quote("(") + "[^" + quote("{()}") + "]*" + quote(")")).r - val variableRegex = (quote("[") + "[^" + quote("{[()]}") + "]*" + quote("]")).r - val propertyRegex = (quote("${") + "[^" + quote("{[()]}") + "]*" + quote("}")).r - - sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable { - require(effectiveStart <= effectiveEnd) - def start = effectiveStart - def end = effectiveEnd - - // FIXME Some kind of validation should be used here, to report all the missing variables, - // not only the first one missing. - def apply(content: String): Map[String, String] => String \/ String - } - object PatternPart { - case class Literal(override val effectiveStart: Int, override val effectiveEnd: Int) extends PatternPart(effectiveStart, effectiveEnd) { - def apply(content: String): Map[String, String] => String \/ String = { - assert(content.length == effectiveEnd - effectiveStart) - val matches = variableRegex.findAllMatchIn(content).toList - - variables => - @tailrec - def helper(idx: Int, matches: List[Regex.Match], b: StringBuilder): String \/ String = - if (idx >= content.length) - \/-(b.result()) - else { - assert(matches.headOption.forall(_.start >= idx)) - matches.headOption.filter(_.start == idx) match { - case Some(m) => - val variableName = content.substring(m.start + 1, m.end - 1) - variables.get(variableName) match { - case None => -\/(s"Variable not found: $variableName") - case Some(value) => - b ++= value - helper(m.end, matches.tail, b) - } - case None => - val nextIdx = matches.headOption.fold(content.length)(_.start) - b ++= content.substring(idx, nextIdx) - helper(nextIdx, matches, b) - } - } - - helper(0, matches, new StringBuilder) - } - } - case class Optional(start0: Int, end0: Int) extends PatternPart(start0 + 1, end0 - 1) { - override def start = start0 - override def end = end0 - - def apply(content: String): Map[String, String] => String \/ String = { - assert(content.length == effectiveEnd - effectiveStart) - val inner = Literal(effectiveStart, effectiveEnd).apply(content) - - variables => - \/-(inner(variables).fold(_ => "", x => x)) - } - } - } - - def substituteProperties(s: String, properties: Map[String, String]): String = - propertyRegex.findAllMatchIn(s).toVector.foldRight(s) { case (m, s0) => - val key = s0.substring(m.start + "${".length, m.end - "}".length) - val value = properties.getOrElse(key, "") - s0.take(m.start) + value + s0.drop(m.end) - } - -} case class IvyRepository( pattern: String, + metadataPatternOpt: Option[String] = None, changing: Option[Boolean] = None, properties: Map[String, String] = Map.empty, withChecksums: Boolean = true, @@ -86,62 +15,12 @@ case class IvyRepository( withArtifacts: Boolean = true ) extends Repository { + def metadataPattern: String = metadataPatternOpt.getOrElse(pattern) + import Repository._ - import IvyRepository._ - private val pattern0 = substituteProperties(pattern, properties) - - val parts = { - val optionalParts = optionalPartRegex.findAllMatchIn(pattern0).toList.map { m => - PatternPart.Optional(m.start, m.end) - } - - val len = pattern0.length - - @tailrec - def helper( - idx: Int, - opt: List[PatternPart.Optional], - acc: List[PatternPart] - ): Vector[PatternPart] = - if (idx >= len) - acc.toVector.reverse - else - opt match { - case Nil => - helper(len, Nil, PatternPart.Literal(idx, len) :: acc) - case (opt0 @ PatternPart.Optional(start0, end0)) :: rem => - if (idx < start0) - helper(start0, opt, PatternPart.Literal(idx, start0) :: acc) - else { - assert(idx == start0, s"idx: $idx, start0: $start0") - helper(end0, rem, opt0 :: acc) - } - } - - helper(0, optionalParts, Nil) - } - - assert(pattern0.isEmpty == parts.isEmpty) - if (pattern0.nonEmpty) { - for ((a, b) <- parts.zip(parts.tail)) - assert(a.end == b.start) - assert(parts.head.start == 0) - assert(parts.last.end == pattern0.length) - } - - private val substituteHelpers = parts.map { part => - part(pattern0.substring(part.effectiveStart, part.effectiveEnd)) - } - - def substitute(variables: Map[String, String]): String \/ String = - substituteHelpers.foldLeft[String \/ String](\/-("")) { - case (acc0, helper) => - for { - acc <- acc0 - s <- helper(variables) - } yield acc + s - } + private val pattern0 = Pattern(pattern, properties) + private val metadataPattern0 = Pattern(metadataPattern, properties) // See http://ant.apache.org/ivy/history/latest-milestone/concept.html for a // list of variables that should be supported. @@ -194,7 +73,7 @@ case class IvyRepository( } val retainedWithUrl = retained.flatMap { p => - substitute(variables( + pattern0.substitute(variables( dependency.module, dependency.version, p.`type`, @@ -236,7 +115,7 @@ case class IvyRepository( val eitherArtifact: String \/ Artifact = for { - url <- substitute( + url <- metadataPattern0.substitute( variables(module, version, "ivy", "ivy", "xml", None) ) } yield { @@ -269,4 +148,4 @@ case class IvyRepository( } yield (source, proj) } -} \ No newline at end of file +} diff --git a/core/shared/src/main/scala/coursier/ivy/Pattern.scala b/core/shared/src/main/scala/coursier/ivy/Pattern.scala new file mode 100644 index 000000000..5054bd6dc --- /dev/null +++ b/core/shared/src/main/scala/coursier/ivy/Pattern.scala @@ -0,0 +1,142 @@ +package coursier.ivy + +import scala.annotation.tailrec + +import scalaz._ + +import scala.util.matching.Regex +import java.util.regex.Pattern.quote + +object Pattern { + + val propertyRegex = (quote("${") + "[^" + quote("{[()]}") + "]*" + quote("}")).r + val optionalPartRegex = (quote("(") + "[^" + quote("{()}") + "]*" + quote(")")).r + val variableRegex = (quote("[") + "[^" + quote("{[()]}") + "]*" + quote("]")).r + + sealed abstract class PatternPart(val effectiveStart: Int, val effectiveEnd: Int) extends Product with Serializable { + require(effectiveStart <= effectiveEnd) + def start = effectiveStart + def end = effectiveEnd + + // FIXME Some kind of validation should be used here, to report all the missing variables, + // not only the first one missing. + def apply(content: String): Map[String, String] => String \/ String + } + object PatternPart { + case class Literal(override val effectiveStart: Int, override val effectiveEnd: Int) extends PatternPart(effectiveStart, effectiveEnd) { + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val matches = variableRegex.findAllMatchIn(content).toList + + variables => + @tailrec + def helper(idx: Int, matches: List[Regex.Match], b: StringBuilder): String \/ String = + if (idx >= content.length) + \/-(b.result()) + else { + assert(matches.headOption.forall(_.start >= idx)) + matches.headOption.filter(_.start == idx) match { + case Some(m) => + val variableName = content.substring(m.start + 1, m.end - 1) + variables.get(variableName) match { + case None => -\/(s"Variable not found: $variableName") + case Some(value) => + b ++= value + helper(m.end, matches.tail, b) + } + case None => + val nextIdx = matches.headOption.fold(content.length)(_.start) + b ++= content.substring(idx, nextIdx) + helper(nextIdx, matches, b) + } + } + + helper(0, matches, new StringBuilder) + } + } + case class Optional(start0: Int, end0: Int) extends PatternPart(start0 + 1, end0 - 1) { + override def start = start0 + override def end = end0 + + def apply(content: String): Map[String, String] => String \/ String = { + assert(content.length == effectiveEnd - effectiveStart) + val inner = Literal(effectiveStart, effectiveEnd).apply(content) + + variables => + \/-(inner(variables).fold(_ => "", x => x)) + } + } + } + + + def substituteProperties(s: String, properties: Map[String, String]): String = + propertyRegex.findAllMatchIn(s).toVector.foldRight(s) { case (m, s0) => + val key = s0.substring(m.start + "${".length, m.end - "}".length) + val value = properties.getOrElse(key, "") + s0.take(m.start) + value + s0.drop(m.end) + } + +} + +case class Pattern( + pattern: String, + properties: Map[String, String] +) { + + import Pattern._ + + private val pattern0 = substituteProperties(pattern, properties) + + val parts = { + val optionalParts = optionalPartRegex.findAllMatchIn(pattern0).toList.map { m => + PatternPart.Optional(m.start, m.end) + } + + val len = pattern0.length + + @tailrec + def helper( + idx: Int, + opt: List[PatternPart.Optional], + acc: List[PatternPart] + ): Vector[PatternPart] = + if (idx >= len) + acc.toVector.reverse + else + opt match { + case Nil => + helper(len, Nil, PatternPart.Literal(idx, len) :: acc) + case (opt0 @ PatternPart.Optional(start0, end0)) :: rem => + if (idx < start0) + helper(start0, opt, PatternPart.Literal(idx, start0) :: acc) + else { + assert(idx == start0, s"idx: $idx, start0: $start0") + helper(end0, rem, opt0 :: acc) + } + } + + helper(0, optionalParts, Nil) + } + + assert(pattern0.isEmpty == parts.isEmpty) + if (pattern0.nonEmpty) { + for ((a, b) <- parts.zip(parts.tail)) + assert(a.end == b.start) + assert(parts.head.start == 0) + assert(parts.last.end == pattern0.length) + } + + private val substituteHelpers = parts.map { part => + part(pattern0.substring(part.effectiveStart, part.effectiveEnd)) + } + + def substitute(variables: Map[String, String]): String \/ String = + substituteHelpers.foldLeft[String \/ String](\/-("")) { + case (acc0, helper) => + for { + acc <- acc0 + s <- helper(variables) + } yield acc + s + } + +} diff --git a/plugin/src/main/scala-2.10/coursier/FromSbt.scala b/plugin/src/main/scala-2.10/coursier/FromSbt.scala index 4e19c3a53..65a139b9c 100644 --- a/plugin/src/main/scala-2.10/coursier/FromSbt.scala +++ b/plugin/src/main/scala-2.10/coursier/FromSbt.scala @@ -115,20 +115,22 @@ object FromSbt { case sbt.FileRepository(_, _, patterns) if patterns.ivyPatterns.lengthCompare(1) == 0 && - patterns.ivyPatterns == patterns.artifactPatterns => + patterns.artifactPatterns.lengthCompare(1) == 0 => Some(IvyRepository( - "file://" + patterns.ivyPatterns.head, + "file://" + patterns.artifactPatterns.head, + metadataPatternOpt = Some("file://" + patterns.ivyPatterns.head), changing = Some(true), properties = ivyProperties )) case sbt.URLRepository(_, patterns) if patterns.ivyPatterns.lengthCompare(1) == 0 && - patterns.ivyPatterns == patterns.artifactPatterns => + patterns.artifactPatterns.lengthCompare(1) == 0 => Some(IvyRepository( - patterns.ivyPatterns.head, + patterns.artifactPatterns.head, + metadataPatternOpt = Some(patterns.ivyPatterns.head), changing = None, properties = ivyProperties )) From a9f3403520f9e8836d208ea3828f137ed5cdef8b Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 14 Jan 2016 00:47:53 +0100 Subject: [PATCH 2/3] Add support for ~/.ivy2/cache as a repository --- cache/src/main/scala/coursier/Cache.scala | 24 +++++++++++++++++-- .../src/main/scala/coursier/CacheParse.scala | 2 ++ .../scala/coursier/ivy/IvyRepository.scala | 22 ++++++++++++++--- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 76871be43..7448fb091 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -467,13 +467,33 @@ object Cache { } } + private lazy val ivy2HomeUri = { + // a bit touchy on Windows... - don't try to manually write down the URI with s"file://..." + val str = new File(sys.props("user.home") + "/.ivy2/").toURI.toString + if (str.endsWith("/")) + str + else + str + "/" + } + lazy val ivy2Local = IvyRepository( - // a bit touchy on Windows... - don't try to get the URI manually like s"file://..." - new File(sys.props("user.home") + "/.ivy2/local/").toURI.toString + + ivy2HomeUri + "local/" + "[organisation]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/" + "[artifact](-[classifier]).[ext]" ) + lazy val ivy2Cache = IvyRepository( + ivy2HomeUri + "cache/" + + "(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[organisation]/[module]/[type]s/[artifact]-[revision](-[classifier]).[ext]", + metadataPatternOpt = Some( + ivy2HomeUri + "cache/" + + "(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[organisation]/[module]/[type]-[revision](-[classifier]).[ext]" + ), + withChecksums = false, + withSignatures = false, + dropInfoAttributes = true + ) + lazy val defaultBase = new File( sys.env.getOrElse( "COURSIER_CACHE", diff --git a/cache/src/main/scala/coursier/CacheParse.scala b/cache/src/main/scala/coursier/CacheParse.scala index 69409ecad..8914e3ad4 100644 --- a/cache/src/main/scala/coursier/CacheParse.scala +++ b/cache/src/main/scala/coursier/CacheParse.scala @@ -10,6 +10,8 @@ object CacheParse { def repository(s: String): Validation[String, Repository] = if (s == "ivy2local" || s == "ivy2Local") Cache.ivy2Local.success + else if (s == "ivy2cache" || s == "ivy2Cache") + Cache.ivy2Cache.success else { val repo = Parse.repository(s) diff --git a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala index 5fc7e077d..599b5ceac 100644 --- a/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala +++ b/core/shared/src/main/scala/coursier/ivy/IvyRepository.scala @@ -12,7 +12,9 @@ case class IvyRepository( properties: Map[String, String] = Map.empty, withChecksums: Boolean = true, withSignatures: Boolean = true, - withArtifacts: Boolean = true + withArtifacts: Boolean = true, + // hack for SBT putting infos in properties + dropInfoAttributes: Boolean = false ) extends Repository { def metadataPattern: String = metadataPatternOpt.getOrElse(pattern) @@ -138,14 +140,28 @@ case class IvyRepository( for { artifact <- EitherT(F.point(eitherArtifact)) ivy <- fetch(artifact) - proj <- EitherT(F.point { + proj0 <- EitherT(F.point { for { xml <- \/.fromEither(compatibility.xmlParse(ivy)) _ <- if (xml.label == "ivy-module") \/-(()) else -\/("Module definition not found") proj <- IvyXml.project(xml) } yield proj }) - } yield (source, proj) + } yield { + val proj = + if (dropInfoAttributes) + proj0.copy( + module = proj0.module.copy( + attributes = proj0.module.attributes.filter { + case (k, _) => !k.startsWith("info.") + } + ) + ) + else + proj0 + + (source, proj) + } } } From a8d5a2c4c9d889e7de4d9fcb6c0a95753e9792f5 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 14 Jan 2016 01:05:36 +0100 Subject: [PATCH 3/3] Accept absent checksums by default SHA-1 still checked first if present --- cache/src/main/scala/coursier/Cache.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 7448fb091..121f4416d 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -16,6 +16,9 @@ import java.io.{ Serializable => _, _ } object Cache { + // Check SHA-1 if available, else be fine with no checksum + val defaultChecksums = Seq(Some("SHA-1"), None) + private def withLocal(artifact: Artifact, cache: Seq[(String, File)]): Artifact = { def local(url: String) = if (url.startsWith("file:///")) @@ -398,7 +401,7 @@ object Cache { artifact: Artifact, cache: Seq[(String, File)] = default, cachePolicy: CachePolicy = CachePolicy.FetchMissing, - checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + checksums: Seq[Option[String]] = defaultChecksums, logger: Option[Logger] = None, pool: ExecutorService = defaultPool ): EitherT[Task, FileError, File] = { @@ -449,7 +452,7 @@ object Cache { def fetch( cache: Seq[(String, File)] = default, cachePolicy: CachePolicy = CachePolicy.FetchMissing, - checksums: Seq[Option[String]] = Seq(Some("SHA-1")), + checksums: Seq[Option[String]] = defaultChecksums, logger: Option[Logger] = None, pool: ExecutorService = defaultPool ): Fetch.Content[Task] = {