diff --git a/appveyor.yml b/appveyor.yml index ab7b7a389..1e2c1663a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,8 +24,8 @@ install: build_script: - sbt ++2.11.11 clean compile coreJVM/publishLocal http-server/publishLocal - sbt ++2.10.6 clean compile - - sbt ++2.12.1 coreJVM/publishLocal cache/publishLocal # to make the scripted sbt 1.0 tests happy - - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted sbt 0.13 tests happy + - sbt ++2.12.1 coreJVM/publishLocal cache/publishLocal extra/publishLocal # to make the scripted sbt 1.0 tests happy + - sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal extra/publishLocal # to make the scripted sbt 0.13 tests happy test_script: - ps: Start-Job -filepath .\scripts\start-it-auth-server.ps1 -ArgumentList $pwd - ps: Start-Sleep -s 15 # wait for the first server to have downloaded its dependencies diff --git a/build.sbt b/build.sbt index d0ab3fd3a..a854bb232 100644 --- a/build.sbt +++ b/build.sbt @@ -88,8 +88,15 @@ lazy val bootstrap = project renameMainJar("bootstrap.jar") ) +lazy val extra = project + .dependsOn(coreJvm) + .settings( + shared, + coursierPrefix + ) + lazy val cli = project - .dependsOn(coreJvm, cache) + .dependsOn(coreJvm, cache, extra) .settings( shared, dontPublishIn("2.10", "2.12"), @@ -166,7 +173,7 @@ lazy val doc = project // Don't try to compile that if you're not in 2.10 lazy val `sbt-coursier` = project - .dependsOn(coreJvm, cache) + .dependsOn(coreJvm, cache, extra) .settings(plugin) // Don't try to compile that if you're not in 2.10 @@ -224,6 +231,7 @@ lazy val jvm = project testsJvm, cache, bootstrap, + extra, cli, `sbt-coursier`, `sbt-shading`, @@ -262,6 +270,7 @@ lazy val coursier = project testsJs, cache, bootstrap, + extra, cli, `sbt-coursier`, `sbt-shading`, diff --git a/cache/src/main/scala/coursier/Cache.scala b/cache/src/main/scala/coursier/Cache.scala index 7e04877e7..b52170b75 100644 --- a/cache/src/main/scala/coursier/Cache.scala +++ b/cache/src/main/scala/coursier/Cache.scala @@ -957,8 +957,6 @@ object Cache { } } - private val utf8Bom = "\ufeff" - def fetch( cache: File = default, cachePolicy: CachePolicy = CachePolicy.UpdateChanging, @@ -981,7 +979,7 @@ object Cache { def notFound(f: File) = Left(s"${f.getCanonicalPath} not found") def read(f: File) = - try Right(new String(FileUtil.readAllBytes(f), "UTF-8").stripPrefix(utf8Bom)) + try Right(new String(FileUtil.readAllBytes(f), "UTF-8")) catch { case NonFatal(e) => Left(s"Could not read (file:${f.getCanonicalPath}): ${e.getMessage}") diff --git a/cli/src/main/scala-2.11/coursier/cli/Fetch.scala b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala index 916c0e546..16209d579 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Fetch.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Fetch.scala @@ -23,6 +23,9 @@ final case class Fetch( ) ) + // Some progress lines seem to be scraped without this. + Console.out.flush() + val out = if (options.classpath) files0 diff --git a/cli/src/main/scala-2.11/coursier/cli/Helper.scala b/cli/src/main/scala-2.11/coursier/cli/Helper.scala index 45b1cc2bb..591726ba3 100644 --- a/cli/src/main/scala-2.11/coursier/cli/Helper.scala +++ b/cli/src/main/scala-2.11/coursier/cli/Helper.scala @@ -7,7 +7,7 @@ import java.util.jar.{ Manifest => JManifest } import java.util.concurrent.Executors import coursier.cli.scaladex.Scaladex -import coursier.cli.typelevel.Typelevel +import coursier.extra.Typelevel import coursier.ivy.IvyRepository import coursier.util.{Print, Parse} diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/Entities.scala b/core/jvm/src/main/scala/coursier/core/compatibility/Entities.scala new file mode 100644 index 000000000..d0ee727c5 --- /dev/null +++ b/core/jvm/src/main/scala/coursier/core/compatibility/Entities.scala @@ -0,0 +1,99 @@ +package coursier.core.compatibility + +object Entities { + + // Generated via https://gist.github.com/alexarchambault/79388ff31ec8cbddf6607b55ab2f6527 + + val entities = Vector( + (" ", " "), + ("¡", "¡"), + ("¢", "¢"), + ("£", "£"), + ("¤", "¤"), + ("¥", "¥"), + ("¦", "¦"), + ("§", "§"), + ("¨", "¨"), + ("©", "©"), + ("ª", "ª"), + ("«", "«"), + ("¬", "¬"), + ("­", "­"), + ("®", "®"), + ("¯", "¯"), + ("°", "°"), + ("±", "±"), + ("´", "´"), + ("µ", "µ"), + ("¶", "¶"), + ("·", "·"), + ("¸", "¸"), + ("º", "º"), + ("»", "»"), + ("¿", "¿"), + ("À", "À"), + ("Á", "Á"), + ("Â", "Â"), + ("Ã", "Ã"), + ("Ä", "Ä"), + ("Å", "Å"), + ("Æ", "Æ"), + ("Ç", "Ç"), + ("È", "È"), + ("É", "É"), + ("Ê", "Ê"), + ("Ë", "Ë"), + ("Ì", "Ì"), + ("Í", "Í"), + ("Î", "Î"), + ("Ï", "Ï"), + ("Ð", "Ð"), + ("Ñ", "Ñ"), + ("Ò", "Ò"), + ("Ó", "Ó"), + ("Ô", "Ô"), + ("Õ", "Õ"), + ("Ö", "Ö"), + ("×", "×"), + ("Ø", "Ø"), + ("Ù", "Ù"), + ("Ú", "Ú"), + ("Û", "Û"), + ("Ü", "Ü"), + ("Ý", "Ý"), + ("Þ", "Þ"), + ("ß", "ß"), + ("à", "à"), + ("á", "á"), + ("â", "â"), + ("ã", "ã"), + ("ä", "ä"), + ("å", "å"), + ("æ", "æ"), + ("ç", "ç"), + ("è", "è"), + ("é", "é"), + ("ê", "ê"), + ("ë", "ë"), + ("ì", "ì"), + ("í", "í"), + ("î", "î"), + ("ï", "ï"), + ("ð", "ð"), + ("ñ", "ñ"), + ("ò", "ò"), + ("ó", "ó"), + ("ô", "ô"), + ("õ", "õ"), + ("ö", "ö"), + ("÷", "÷"), + ("ø", "ø"), + ("ù", "ù"), + ("ú", "ú"), + ("û", "û"), + ("ü", "ü"), + ("ý", "ý"), + ("þ", "þ"), + ("ÿ", "ÿ") + ) +} diff --git a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala index 124266ae2..87146078f 100644 --- a/core/jvm/src/main/scala/coursier/core/compatibility/package.scala +++ b/core/jvm/src/main/scala/coursier/core/compatibility/package.scala @@ -2,6 +2,8 @@ package coursier.core import coursier.util.Xml +import java.util.regex.Pattern.quote + import scala.collection.JavaConverters._ import scala.xml.{ Attribute, MetaData, Null } @@ -14,9 +16,23 @@ package object compatibility { def letter = c.isLetter } + private val entityPattern = (quote("&") + "[a-zA-Z]+" + quote(";")).r + + private val utf8Bom = "\ufeff" + def xmlParse(s: String): Either[String, Xml.Node] = { + + val content = + if (entityPattern.findFirstIn(s).isEmpty) + s + else + Entities.entities.foldLeft(s) { + case (s0, (target, replacement)) => + s0.replace(target, replacement) + } + def parse = - try Right(scala.xml.XML.loadString(s)) + try Right(scala.xml.XML.loadString(content.stripPrefix(utf8Bom))) catch { case e: Exception => Left(e.toString + Option(e.getMessage).fold("")(" (" + _ + ")")) } def fromNode(node: scala.xml.Node): Xml.Node = diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 3b0d4129d..9fc848f68 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -31,11 +31,7 @@ object Resolution { def fromActivation = profile.activation.isActive(properties, osInfo, jdkVersion) - val res = fromUserOrDefault.getOrElse(fromActivation) - - // println(s"Profile\n$profile\n$res\n") - - res + fromUserOrDefault.getOrElse(fromActivation) } /** @@ -785,31 +781,42 @@ final case class Resolution( project: Project ): Set[ModuleVersion] = { - val approxProperties0 = - project.parent - .flatMap(projectCache.get) - .map(_._2.properties) - .fold(project.properties)(project.properties ++ _) + val needsParent = + project.parent.exists { par => + val parentFound = projectCache.contains(par) || errorCache.contains(par) + !parentFound + } - val approxProperties = propertiesMap(approxProperties0) ++ projectProperties(project) + if (needsParent) + project.parent.toSet + else { - val profileDependencies = - profiles( - project, - approxProperties, - osInfo, - jdkVersion, - userActivations - ).flatMap(p => p.dependencies ++ p.dependencyManagement) + val approxProperties0 = + project.parent + .flatMap(projectCache.get) + .map(_._2.properties) + .fold(project.properties)(project.properties ++ _) - val modules = withProperties( - project.dependencies ++ project.dependencyManagement ++ profileDependencies, - approxProperties - ).collect { - case ("import", dep) => dep.moduleVersion + val approxProperties = propertiesMap(approxProperties0) ++ projectProperties(project) + + val profileDependencies = + profiles( + project, + approxProperties, + osInfo, + jdkVersion, + userActivations + ).flatMap(p => p.dependencies ++ p.dependencyManagement) + + val modules = withProperties( + project.dependencies ++ project.dependencyManagement ++ profileDependencies, + approxProperties + ).collect { + case ("import", dep) => dep.moduleVersion + } + + modules.toSet } - - modules.toSet ++ project.parent } /** diff --git a/core/shared/src/main/scala/coursier/maven/MavenSource.scala b/core/shared/src/main/scala/coursier/maven/MavenSource.scala index aa790df10..05a1495fd 100644 --- a/core/shared/src/main/scala/coursier/maven/MavenSource.scala +++ b/core/shared/src/main/scala/coursier/maven/MavenSource.scala @@ -95,6 +95,8 @@ final case class MavenSource( } } + private val types = Map("sha1" -> "SHA-1", "md5" -> "MD5", "asc" -> "sig") + private def artifactsKnownPublications( dependency: Dependency, project: Project, @@ -134,40 +136,40 @@ final case class MavenSource( } } - def extensionIsExtra(publications: Seq[EnrichedPublication], ext: String, tpe: String): Seq[EnrichedPublication] = { - - val (withExt, other) = publications.partition(_.publication.ext.endsWith("." + ext)) - - var withExtMap = withExt.map(p => (p.publication.classifier, p.publication.ext.stripSuffix("." + ext)) -> p).toMap - - other.map { p => - val key = (p.publication.classifier, p.publication.ext) - withExtMap.get(key).fold(p) { sigPub => - withExtMap -= key - p.copy( - extra = p.extra + (tpe -> sigPub) - ) - } - } ++ withExtMap.values - } - def groupedEnrichedPublications(publications: Seq[Publication]): Seq[EnrichedPublication] = { - def helperSameName(publications: Seq[Publication]): Seq[EnrichedPublication] = { + def helper(publications: Seq[Publication]): Seq[EnrichedPublication] = { - val publications0 = publications.map { pub => - EnrichedPublication(pub, Map()) + var publications0 = publications + .map { pub => + pub.ext -> EnrichedPublication(pub, Map()) + } + .toMap + + val byLength = publications0.toVector.sortBy(-_._1.length) + + for { + (ext, _) <- byLength + idx = ext.lastIndexOf('.') + if idx >= 0 + subExt = ext.substring(idx + 1) + baseExt = ext.substring(0, idx) + tpe <- types.get(subExt) + mainPub <- publications0.get(baseExt) + } { + val pub = publications0(ext) + publications0 += baseExt -> mainPub.copy( + extra = mainPub.extra + (tpe -> pub) + ) + publications0 -= ext } - Seq("sha1" -> "SHA-1", "md5" -> "MD5", "asc" -> "sig").foldLeft(publications0) { - case (pub, (ext, tpe)) => - extensionIsExtra(pub, ext, tpe) - } + publications0.values.toVector } publications - .groupBy(_.name) - .mapValues(helperSameName) + .groupBy(p => (p.name, p.classifier)) + .mapValues(helper) .values .toVector .flatten diff --git a/cli/src/main/scala-2.11/coursier/cli/typelevel/Typelevel.scala b/extra/src/main/scala/coursier/extra/Typelevel.scala similarity index 95% rename from cli/src/main/scala-2.11/coursier/cli/typelevel/Typelevel.scala rename to extra/src/main/scala/coursier/extra/Typelevel.scala index 72c290077..b0d88adfb 100644 --- a/cli/src/main/scala-2.11/coursier/cli/typelevel/Typelevel.scala +++ b/extra/src/main/scala/coursier/extra/Typelevel.scala @@ -1,4 +1,4 @@ -package coursier.cli.typelevel +package coursier.extra import coursier.{Dependency, Module} diff --git a/sbt-coursier/src/main/scala/coursier/Tasks.scala b/sbt-coursier/src/main/scala/coursier/Tasks.scala index d6db4d8c2..2b80cb095 100644 --- a/sbt-coursier/src/main/scala/coursier/Tasks.scala +++ b/sbt-coursier/src/main/scala/coursier/Tasks.scala @@ -5,6 +5,7 @@ import java.net.URL import java.util.concurrent.{ExecutorService, Executors} import coursier.core.{Authentication, Publication} +import coursier.extra.Typelevel import coursier.ivy.{IvyRepository, PropertiesPattern} import coursier.Keys._ import coursier.Structure._ @@ -504,6 +505,8 @@ object Tasks { val userEnabledProfiles = mavenProfiles.value + val typelevel = scalaOrganization.value == Typelevel.typelevelOrg + val startRes = Resolution( currentProject.dependencies.map(_._2).toSet, filter = Some(dep => !dep.optional), @@ -517,7 +520,8 @@ object Tasks { userForceVersions ++ forcedScalaModules(so, sv) ++ interProjectDependencies.map(_.moduleVersion), - projectCache = parentProjectCache + projectCache = parentProjectCache, + mapDependencies = if (typelevel) Some(Typelevel.swap(_)) else None ) if (verbosityLevel >= 2) { diff --git a/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/build.sbt b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/build.sbt new file mode 100644 index 000000000..f61164147 --- /dev/null +++ b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/build.sbt @@ -0,0 +1,9 @@ +scalaVersion := "2.12.2-bin-typelevel-4" +scalaOrganization := "org.typelevel" +scalacOptions += "-Yinduction-heuristics" + +libraryDependencies ++= Seq( + "com.47deg" %% "freestyle" % "0.1.0", + // CrossVersion.patch not available in sbt 0.13.8 + compilerPlugin("org.scalamacros" % "paradise_2.12.2" % "2.1.0") +) diff --git a/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/project/plugins.sbt b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/project/plugins.sbt new file mode 100644 index 000000000..152225a9e --- /dev/null +++ b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/project/plugins.sbt @@ -0,0 +1,11 @@ +{ + val pluginVersion = sys.props.getOrElse( + "plugin.version", + throw new RuntimeException( + """|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin + ) + ) + + addSbtPlugin("io.get-coursier" % "sbt-coursier" % pluginVersion) +} diff --git a/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/src/main/scala/Foo.scala b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/src/main/scala/Foo.scala new file mode 100644 index 000000000..cf64974eb --- /dev/null +++ b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/src/main/scala/Foo.scala @@ -0,0 +1,5 @@ +import freestyle._ + +@free trait Foo { + def bzz: FS[java.lang.String] +} diff --git a/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/test b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/test new file mode 100644 index 000000000..5df2af1f3 --- /dev/null +++ b/sbt-coursier/src/sbt-test/sbt-coursier/typelevel-with-dependencies/test @@ -0,0 +1 @@ +> compile diff --git a/scripts/travis.sh b/scripts/travis.sh index 5345f8f86..b81098d0d 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -75,14 +75,14 @@ is212() { } runSbtCoursierTests() { - sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal "sbt-coursier/scripted sbt-coursier/*" + sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal extra/publishLocal "sbt-coursier/scripted sbt-coursier/*" if [ "$SCALA_VERSION" = "2.10" ]; then sbt ++$SCALA_VERSION "sbt-coursier/scripted sbt-coursier-0.13/*" fi } runSbtShadingTests() { - sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal sbt-coursier/publishLocal "sbt-shading/scripted sbt-shading/*" + sbt ++$SCALA_VERSION coreJVM/publishLocal cache/publishLocal extra/publishLocal sbt-coursier/publishLocal "sbt-shading/scripted sbt-shading/*" if [ "$SCALA_VERSION" = "2.10" ]; then sbt ++$SCALA_VERSION "sbt-shading/scripted sbt-shading-0.13/*" fi @@ -128,7 +128,7 @@ testLauncherJava6() { } testSbtCoursierJava6() { - sbt ++${SCALA_VERSION} coreJVM/publishLocal cache/publishLocal sbt-coursier/publishLocal + sbt ++${SCALA_VERSION} coreJVM/publishLocal cache/publishLocal extra/publishLocal sbt-coursier/publishLocal git clone https://github.com/alexarchambault/scalacheck-shapeless.git cd scalacheck-shapeless diff --git a/tests/shared/src/test/resources/resolutions/org.codehaus.plexus/plexus/1.0.4 b/tests/shared/src/test/resources/resolutions/org.codehaus.plexus/plexus/1.0.4 new file mode 100644 index 000000000..8d5ac7bf8 --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.codehaus.plexus/plexus/1.0.4 @@ -0,0 +1 @@ +org.codehaus.plexus:plexus:1.0.4:compile diff --git a/tests/shared/src/test/resources/resolutions/org.kie/kie-api/6.5.0.Final b/tests/shared/src/test/resources/resolutions/org.kie/kie-api/6.5.0.Final new file mode 100644 index 000000000..30b15922a --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.kie/kie-api/6.5.0.Final @@ -0,0 +1,2 @@ +org.kie:kie-api:6.5.0.Final:compile +org.slf4j:slf4j-api:1.7.2:compile diff --git a/tests/shared/src/test/resources/resolutions/org.yaml/snakeyaml/1.17 b/tests/shared/src/test/resources/resolutions/org.yaml/snakeyaml/1.17 new file mode 100644 index 000000000..11fee1420 --- /dev/null +++ b/tests/shared/src/test/resources/resolutions/org.yaml/snakeyaml/1.17 @@ -0,0 +1 @@ +org.yaml:snakeyaml:1.17:compile diff --git a/tests/shared/src/test/scala/coursier/test/CentralTests.scala b/tests/shared/src/test/scala/coursier/test/CentralTests.scala index e9ce23931..1a6d490c4 100644 --- a/tests/shared/src/test/scala/coursier/test/CentralTests.scala +++ b/tests/shared/src/test/scala/coursier/test/CentralTests.scala @@ -32,6 +32,14 @@ object CentralTests extends TestSuite { ) .process .run(fetch) + .map { res => + + assert(res.metadataErrors.isEmpty) + assert(res.conflicts.isEmpty) + assert(res.isDone) + + res + } .runF } @@ -155,9 +163,14 @@ object CentralTests extends TestSuite { assert(res.conflicts.isEmpty) assert(res.isDone) - val artifacts = classifierOpt.fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c))).map(_._2).filter { a => - a.`type` == artifactType - } + val artifacts = classifierOpt + .fold(res.dependencyArtifacts)(c => res.dependencyClassifiersArtifacts(Seq(c))) + .map(_._2) + .filter { + if (artifactType == "*") _ => true + else + _.`type` == artifactType + } f(artifacts) } @@ -607,6 +620,55 @@ object CentralTests extends TestSuite { } } } + + 'entities - { + 'odash - resolutionCheck( + Module("org.codehaus.plexus", "plexus"), + "1.0.4" + ) + } + + 'parentBeforeImports - { + * - resolutionCheck( + Module("org.kie", "kie-api"), + "6.5.0.Final", + extraRepo = Some(MavenRepository("https://repository.jboss.org/nexus/content/repositories/public")) + ) + } + + 'signaturesOfSignatures - { + val mod = Module("org.yaml", "snakeyaml") + val ver = "1.17" + + def hasSha1(a: Artifact) = a.extra.contains("SHA-1") + def hasMd5(a: Artifact) = a.extra.contains("MD5") + def hasSig(a: Artifact) = a.extra.contains("sig") + def sigHasSig(a: Artifact) = a.extra.get("sig").exists(hasSig) + + * - resolutionCheck(mod, ver) + + * - withArtifacts(mod, ver, "*") { artifacts => + + val jarOpt = artifacts.find(_.`type` == "bundle") + val pomOpt = artifacts.find(_.`type` == "pom") + + if (artifacts.length != 2 || jarOpt.isEmpty || pomOpt.isEmpty) + artifacts.foreach(println) + + assert(artifacts.length == 2) + assert(jarOpt.nonEmpty) + assert(pomOpt.nonEmpty) + + assert(jarOpt.forall(hasSha1)) + assert(pomOpt.forall(hasSha1)) + assert(jarOpt.forall(hasMd5)) + assert(pomOpt.forall(hasMd5)) + assert(jarOpt.forall(hasSig)) + assert(pomOpt.forall(hasSig)) + assert(jarOpt.forall(sigHasSig)) + assert(pomOpt.forall(sigHasSig)) + } + } } }