Merge pull request #430 from alexarchambault/topic/artifact-type

Tweak artifact type handling for Maven repos
This commit is contained in:
Alexandre Archambault 2017-02-14 10:16:59 +01:00 committed by GitHub
commit 114d2f95ad
143 changed files with 505 additions and 294 deletions

View File

@ -37,6 +37,7 @@ function isMasterOrDevelop() {
-- \
-d tests/jvm/src/test/resources/test-repo/http/abc.com \
-u user -P pass -r realm \
--list-pages \
-v &
# TODO Add coverage once https://github.com/scoverage/sbt-scoverage/issues/111 is fixed
@ -47,7 +48,7 @@ RUN_SHADING_TESTS=1
if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then
SBT_COMMANDS="$SBT_COMMANDS publishLocal" # to make the scripted tests happy
SBT_COMMANDS="$SBT_COMMANDS plugin/scripted"
SBT_COMMANDS="$SBT_COMMANDS sbt-coursier/scripted"
if [ "$RUN_SHADING_TESTS" = 1 ]; then
# for the shading scripted test
@ -68,7 +69,7 @@ if echo "$TRAVIS_SCALA_VERSION" | grep -q "^2\.10"; then
rm -rf jarjar
fi
SBT_COMMANDS="$SBT_COMMANDS plugin/publishLocal sbt-shading/scripted"
SBT_COMMANDS="$SBT_COMMANDS sbt-coursier/publishLocal sbt-shading/scripted"
fi
fi

View File

@ -27,10 +27,10 @@ build_script:
- sbt ++2.10.6 clean compile
- sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted tests happy
test_script:
- ps: Start-Job { & java -jar -noverify C:\projects\coursier\coursier launch -r http://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d /C:/projects/coursier/tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm -v }
- ps: Start-Job { & java -jar -noverify C:\projects\coursier\coursier launch -r http://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d /C:/projects/coursier/tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm --list-pages -v }
- sbt ++2.12.1 testsJVM/test testsJVM/it:test # Would node be around for testsJS/test?
- sbt ++2.11.8 testsJVM/test testsJVM/it:test
- sbt ++2.10.6 testsJVM/test testsJVM/it:test plugin/scripted plugin/publishLocal sbt-shading/scripted
- sbt ++2.10.6 testsJVM/test testsJVM/it:test sbt-coursier/scripted sbt-coursier/publishLocal sbt-shading/scripted
cache:
- C:\Users\appveyor\.ivy2\cache
- C:\Users\appveyor\.m2

View File

@ -20,6 +20,42 @@ lazy val core = crossProject
import com.typesafe.tools.mima.core._
Seq(
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPublications"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.defaultPackaging"),
ProblemFilters.exclude[MissingClassProblem]("coursier.maven.MavenSource$DocSourcesArtifactExtensions"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageDirectoryElements"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageSubDirectories"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.compatibility.package.listWebPageFiles"),
ProblemFilters.exclude[MissingTypesProblem]("coursier.core.Project$"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Project.apply"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.core.Project.copy$default$13"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.core.Project.copy$default$12"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Project.copy"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.core.Project.this"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.copy$default$5"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.packagingBlacklist"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.copy"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.this"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.apply$default$5"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.ignorePackaging"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.<init>$default$5"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.maven.MavenRepository.apply"),
ProblemFilters.exclude[FinalClassProblem]("coursier.core.Activation$Os"),
ProblemFilters.exclude[FinalClassProblem]("coursier.core.Version"),
ProblemFilters.exclude[FinalClassProblem]("coursier.core.Authentication"),
ProblemFilters.exclude[FinalClassProblem]("coursier.core.VersionInterval"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.Pattern"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.Pattern$Chunk$Opt"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.PropertiesPattern$ChunkOrProperty$Const"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.PropertiesPattern$ChunkOrProperty$Opt"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.PropertiesPattern"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.Pattern$Chunk$Var"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.IvyRepository"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.Pattern$Chunk$Const"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.PropertiesPattern$ChunkOrProperty$Prop"),
ProblemFilters.exclude[FinalClassProblem]("coursier.ivy.PropertiesPattern$ChunkOrProperty$Var"),
ProblemFilters.exclude[FinalClassProblem]("coursier.maven.MavenRepository"),
ProblemFilters.exclude[FinalClassProblem]("coursier.maven.MavenSource"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.package#Resolution.apply$default$9"),
ProblemFilters.exclude[DirectMissingMethodProblem]("coursier.package#Resolution.apply"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("coursier.core.Resolution.copy$default$9"),
@ -126,6 +162,9 @@ lazy val cache = project
import com.typesafe.tools.mima.core._
Seq(
ProblemFilters.exclude[FinalClassProblem]("coursier.TermDisplay$DownloadInfo"),
ProblemFilters.exclude[FinalClassProblem]("coursier.TermDisplay$CheckUpdateInfo"),
ProblemFilters.exclude[FinalClassProblem]("coursier.util.Base64$B64Scheme"),
ProblemFilters.exclude[MissingClassProblem]("coursier.TermDisplay$Message$Stop$"),
ProblemFilters.exclude[MissingClassProblem]("coursier.TermDisplay$Message"),
ProblemFilters.exclude[MissingClassProblem]("coursier.TermDisplay$Message$"),
@ -326,16 +365,13 @@ lazy val doc = project
)
// Don't try to compile that if you're not in 2.10
lazy val plugin = project
lazy val `sbt-coursier` = project
.dependsOn(coreJvm, cache)
.settings(pluginSettings)
.settings(
name := "sbt-coursier"
)
// Don't try to compile that if you're not in 2.10
lazy val `sbt-shading` = project
.dependsOn(plugin)
.dependsOn(`sbt-coursier`)
.settings(pluginSettings)
.settings(
// Warning: this version doesn't handle well class names with '$'s
@ -385,7 +421,7 @@ lazy val `coursier` = project.in(file("."))
cache,
bootstrap,
cli,
plugin,
`sbt-coursier`,
`sbt-shading`,
web,
doc,

View File

@ -1,7 +1,8 @@
package coursier
import java.io._
import java.net.URL
import scala.language.implicitConversions
import scalaz._
import scalaz.concurrent.Task

View File

@ -79,7 +79,7 @@ object TermDisplay {
def display(): String
}
private case class DownloadInfo(
private final case class DownloadInfo(
downloaded: Long,
previouslyDownloaded: Long,
length: Option[Long],
@ -127,7 +127,7 @@ object TermDisplay {
private def formatTimestamp(ts: Long): String =
format.format(new Timestamp(ts))
private case class CheckUpdateInfo(
private final case class CheckUpdateInfo(
currentTimeOpt: Option[Long],
remoteTimeOpt: Option[Long],
isDone: Boolean
@ -292,7 +292,7 @@ object TermDisplay {
.toVector
.filter {
case (url, _) =>
!url.endsWith(".sha1") && !url.endsWith(".md5")
!url.endsWith(".sha1") && !url.endsWith(".md5") && !url.endsWith("/")
}
.sortBy { case (url, _) => url }

View File

@ -16,7 +16,7 @@ import scala.collection.mutable.ArrayBuilder
object Base64 {
case class B64Scheme(encodeTable: Array[Char], strictPadding: Boolean = true,
final case class B64Scheme(encodeTable: Array[Char], strictPadding: Boolean = true,
postEncode: String => String = identity,
preDecode: String => String = identity) {
lazy val decodeTable = {

View File

@ -11,7 +11,7 @@ import caseapp._
import coursier.cli.util.Zip
import coursier.internal.FileUtil
case class Bootstrap(
final case class Bootstrap(
@Recurse
artifactOptions: ArtifactOptions,
@Recurse

View File

@ -7,7 +7,7 @@ import caseapp.core.{ ArgsApp, CommandsMessages }
import shapeless.union.Union
// Temporary, see comment in Coursier below
case class CoursierCommandHelper(
final case class CoursierCommandHelper(
command: CoursierCommandHelper.U
) extends ArgsApp {
def setRemainingArgs(remainingArgs: Seq[String], extraArgs: Seq[String]): Unit =

View File

@ -7,7 +7,7 @@ import caseapp._
import scala.language.reflectiveCalls
case class Fetch(
final case class Fetch(
@Recurse
options: FetchOptions
) extends App {

View File

@ -79,7 +79,7 @@ trait ExtraArgsApp extends caseapp.core.DefaultArgsApp {
extraArgs1
}
case class Launch(
final case class Launch(
@Recurse
options: LaunchOptions
) extends App with ExtraArgsApp {

View File

@ -5,7 +5,7 @@ import caseapp.{ HelpMessage => Help, ValueDescription => Value, ExtraName => Sh
import coursier.util.Parse
case class CommonOptions(
final case class CommonOptions(
@Help("Keep optional dependencies (Maven)")
keepOptional: Boolean = false,
@Help("Download mode (default: missing, that is fetch things missing from cache)")
@ -94,13 +94,13 @@ case class CommonOptions(
lazy val classifier0 = classifier.flatMap(_.split(',')).filter(_.nonEmpty)
}
case class CacheOptions(
final case class CacheOptions(
@Help("Cache directory (defaults to environment variable COURSIER_CACHE or ~/.coursier/cache/v1)")
@Short("C")
cache: String = Cache.default.toString
)
case class IsolatedLoaderOptions(
final case class IsolatedLoaderOptions(
@Value("target:dependency")
@Short("I")
isolated: List[String] = Nil,
@ -174,7 +174,7 @@ object ArtifactOptions {
def defaultArtifactTypes = Set("jar", "bundle")
}
case class ArtifactOptions(
final case class ArtifactOptions(
@Help("Artifact types that should be retained (e.g. jar, src, doc, etc.) - defaults to jar,bundle")
@Value("type1,type2,...")
@Short("A")
@ -197,7 +197,7 @@ case class ArtifactOptions(
}
}
case class FetchOptions(
final case class FetchOptions(
@Help("Fetch source artifacts")
@Short("S")
sources: Boolean = false,
@ -213,7 +213,7 @@ case class FetchOptions(
common: CommonOptions = CommonOptions()
)
case class LaunchOptions(
final case class LaunchOptions(
@Short("M")
@Short("main")
mainClass: String = "",
@ -226,7 +226,7 @@ case class LaunchOptions(
common: CommonOptions = CommonOptions()
)
case class BootstrapOptions(
final case class BootstrapOptions(
@Short("M")
@Short("main")
mainClass: String = "",
@ -253,7 +253,7 @@ case class BootstrapOptions(
common: CommonOptions = CommonOptions()
)
case class SparkSubmitOptions(
final case class SparkSubmitOptions(
@Short("M")
@Short("main")
@Help("Main class to be launched (optional if in manifest)")

View File

@ -3,7 +3,7 @@ package cli
import caseapp._
case class Resolve(
final case class Resolve(
@Recurse
common: CommonOptions
) extends App {

View File

@ -50,7 +50,7 @@ object SparkSubmit {
* @author Han Ju
*/
@CommandName("spark-submit")
case class SparkSubmit(
final case class SparkSubmit(
@Recurse
options: SparkSubmitOptions
) extends App with ExtraArgsApp {

View File

@ -24,8 +24,8 @@ object Assembly {
def path: String
}
case class Exclude(path: String) extends PathRule
case class ExcludePattern(path: Pattern) extends Rule
final case class Exclude(path: String) extends PathRule
final case class ExcludePattern(path: Pattern) extends Rule
object ExcludePattern {
def apply(s: String): ExcludePattern =
@ -34,8 +34,8 @@ object Assembly {
// TODO Accept a separator: Array[Byte] argument in these
// (to separate content with a line return in particular)
case class Append(path: String) extends PathRule
case class AppendPattern(path: Pattern) extends Rule
final case class Append(path: String) extends PathRule
final case class AppendPattern(path: Pattern) extends Rule
object AppendPattern {
def apply(s: String): AppendPattern =

View File

@ -6,6 +6,8 @@ import org.scalajs.dom.raw.NodeList
import coursier.util.Xml
import scala.collection.mutable.ListBuffer
package object compatibility {
def option[A](a: js.Dynamic): Option[A] =
if (js.isUndefined(a)) None
@ -93,14 +95,22 @@ package object compatibility {
def encodeURIComponent(s: String): String =
g.encodeURIComponent(s).asInstanceOf[String]
def listWebPageSubDirectories(url: String, page: String): Seq[String] = {
// TODO
???
}
// FIXME Won't work in the browser
lazy val cheerio = g.require("cheerio")
def listWebPageFiles(url: String, page: String): Seq[String] = {
// TODO
???
}
def listWebPageRawElements(page: String): Seq[String] = {
val jquery = cheerio.load(page)
val links = new ListBuffer[String]
jquery("a").each({ self: js.Dynamic =>
val href = jquery(self).attr("href")
if (!js.isUndefined(href))
links += href.asInstanceOf[String]
()
}: js.ThisFunction0[js.Dynamic, Unit])
links.result()
}
}

View File

@ -56,25 +56,11 @@ package object compatibility {
def encodeURIComponent(s: String): String =
new java.net.URI(null, null, null, -1, s, null, null) .toASCIIString
def listWebPageDirectoryElements(url: String, page: String, directories: Boolean): Seq[String] =
def listWebPageRawElements(page: String): Seq[String] =
Jsoup.parse(page)
.select("a")
.asScala
.toVector
.map(_.attr("href"))
.collect {
case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
elem
.stripSuffix("/")
.stripPrefix(url)
.stripPrefix(":") // bintray typically prepends these
}
.filter(n => !n.contains("/") && n != "." && n != "..")
def listWebPageSubDirectories(url: String, page: String): Seq[String] =
listWebPageDirectoryElements(url, page, directories = true)
def listWebPageFiles(url: String, page: String): Seq[String] =
listWebPageDirectoryElements(url, page, directories = false)
}

View File

@ -1,6 +1,6 @@
package coursier
import coursier.maven.MavenSource
import scala.language.higherKinds
import scalaz._

View File

@ -47,7 +47,7 @@ final case class Activation(
object Activation {
case class Os(
final case class Os(
arch: Option[String],
families: Set[String],
name: Option[String],

View File

@ -81,6 +81,8 @@ final case class Project(
profiles: Seq[Profile],
versions: Option[Versions],
snapshotVersioning: Option[SnapshotVersioning],
packagingOpt: Option[String],
/**
* Optional exact version used to get this project metadata.
* May not match `version` for projects having a wrong version in their metadata.
@ -219,7 +221,7 @@ object Artifact {
}
}
case class Authentication(
final case class Authentication(
user: String,
password: String
) {

View File

@ -63,7 +63,7 @@ object Resolution {
type Key = (String, String, String)
def key(dep: Dependency): Key =
(dep.module.organization, dep.module.name, dep.attributes.`type`)
(dep.module.organization, dep.module.name, if (dep.attributes.`type`.isEmpty) "jar" else dep.attributes.`type`)
def add(
dict: Map[Key, (String, Dependency)],
@ -351,6 +351,7 @@ object Resolution {
private val mavenScopes = {
val base = Map[String, Set[String]](
"compile" -> Set("compile"),
"optional" -> Set("compile", "optional"),
"provided" -> Set(),
"runtime" -> Set("compile", "runtime"),
"test" -> Set()
@ -463,10 +464,16 @@ object Resolution {
default
else
keepOpt.fold(default) { keep =>
if (keep(config))
// really keeping the from.configuration, with its fallback config part
Seq(dep.copy(configuration = from.configuration))
else
if (keep(config)) {
val depConfig =
if (actualConfig == "optional")
defaultConfiguration
else
// really keeping the from.configuration, with its fallback config part
from.configuration
Seq(dep.copy(configuration = depConfig))
} else
Nil
}
}

View File

@ -3,6 +3,7 @@ package core
import scalaz._
import scala.annotation.tailrec
import scala.language.higherKinds
sealed abstract class ResolutionProcess {

View File

@ -8,7 +8,7 @@ import coursier.core.compatibility._
*
* Same kind of ordering as aether-util/src/main/java/org/eclipse/aether/util/version/GenericVersion.java
*/
case class Version(repr: String) extends Ordered[Version] {
final case class Version(repr: String) extends Ordered[Version] {
lazy val items = Version.items(repr)
lazy val rawItems: Seq[Version.Item] = {
val (first, tokens) = Version.Tokenizer(repr)

View File

@ -3,10 +3,12 @@ package coursier.core
import scalaz.{ -\/, \/, \/- }
import scalaz.Scalaz.ToEitherOps
case class VersionInterval(from: Option[Version],
to: Option[Version],
fromIncluded: Boolean,
toIncluded: Boolean) {
final case class VersionInterval(
from: Option[Version],
to: Option[Version],
fromIncluded: Boolean,
toIncluded: Boolean
) {
def isValid: Boolean = {
val fromToOrder =

View File

@ -2,11 +2,14 @@ package coursier.ivy
import coursier.Fetch
import coursier.core._
import coursier.util.WebPage
import scala.language.higherKinds
import scalaz._
import scalaz.Scalaz._
case class IvyRepository(
final case class IvyRepository(
pattern: Pattern,
metadataPatternOpt: Option[Pattern],
changing: Option[Boolean],
@ -147,7 +150,7 @@ case class IvyRepository(
}
def fromWebPage(url: String, s: String) = {
val subDirs = coursier.core.compatibility.listWebPageSubDirectories(url, s)
val subDirs = WebPage.listDirectories(url, s)
val versions = subDirs.map(Parse.version).collect { case Some(v) => v }
val versionsInItv = versions.filter(itv.contains)

View File

@ -80,7 +80,7 @@ object IvyXml {
version,
toConf,
allConfsExcludes ++ excludes.getOrElse(fromConf, Set.empty),
Attributes("jar", ""), // should come from possible artifact nodes
Attributes("", ""), // should come from possible artifact nodes
optional = false,
transitive = transitive
)
@ -158,6 +158,7 @@ object IvyXml {
None,
None,
None,
None,
if (publicationsOpt.isEmpty)
// no publications node -> default JAR artifact
Seq("*" -> Publication(module.name, "jar", "jar", ""))

View File

@ -1,10 +1,12 @@
package coursier.ivy
import scala.language.implicitConversions
import scalaz._, Scalaz._
import fastparse.all._
case class PropertiesPattern(chunks: Seq[PropertiesPattern.ChunkOrProperty]) {
final case class PropertiesPattern(chunks: Seq[PropertiesPattern.ChunkOrProperty]) {
def string: String = chunks.map(_.string).mkString
@ -51,7 +53,7 @@ case class PropertiesPattern(chunks: Seq[PropertiesPattern.ChunkOrProperty]) {
}
}
case class Pattern(chunks: Seq[Pattern.Chunk]) {
final case class Pattern(chunks: Seq[Pattern.Chunk]) {
def +:(chunk: Pattern.Chunk): Pattern =
Pattern(chunk +: chunks)
@ -101,17 +103,17 @@ object PropertiesPattern {
}
object ChunkOrProperty {
case class Prop(name: String, alternative: Option[Seq[ChunkOrProperty]]) extends ChunkOrProperty {
final case class Prop(name: String, alternative: Option[Seq[ChunkOrProperty]]) extends ChunkOrProperty {
def string: String =
s"$${" + name + alternative.fold("")(alt => "-" + alt.map(_.string).mkString) + "}"
}
case class Var(name: String) extends ChunkOrProperty {
final case class Var(name: String) extends ChunkOrProperty {
def string: String = "[" + name + "]"
}
case class Opt(content: ChunkOrProperty*) extends ChunkOrProperty {
final case class Opt(content: ChunkOrProperty*) extends ChunkOrProperty {
def string: String = "(" + content.map(_.string).mkString + ")"
}
case class Const(value: String) extends ChunkOrProperty {
final case class Const(value: String) extends ChunkOrProperty {
def string: String = value
}
@ -159,13 +161,13 @@ object Pattern {
}
object Chunk {
case class Var(name: String) extends Chunk {
final case class Var(name: String) extends Chunk {
def string: String = "[" + name + "]"
}
case class Opt(content: Chunk*) extends Chunk {
final case class Opt(content: Chunk*) extends Chunk {
def string: String = "(" + content.map(_.string).mkString + ")"
}
case class Const(value: String) extends Chunk {
final case class Const(value: String) extends Chunk {
def string: String = value
}

View File

@ -3,6 +3,7 @@ package coursier.maven
import coursier.Fetch
import coursier.core._
import coursier.core.compatibility.encodeURIComponent
import coursier.util.WebPage
import scala.language.higherKinds
import scalaz._
@ -45,19 +46,6 @@ object MavenRepository {
"test" -> Seq("runtime")
)
val defaultPackaging = "jar"
def defaultPublications(moduleName: String, packaging: String) = Seq(
"compile" -> Publication(
moduleName,
packaging,
MavenSource.typeExtension(packaging),
MavenSource.typeDefaultClassifier(packaging)
),
"docs" -> Publication(moduleName, "doc", "jar", "javadoc"),
"sources" -> Publication(moduleName, "src", "jar", "sources")
)
def dirModuleName(module: Module, sbtAttrStub: Boolean): String =
if (sbtAttrStub) {
var name = module.name
@ -69,19 +57,14 @@ object MavenRepository {
} else
module.name
val ignorePackaging = Set(
Module("org.apache.zookeeper", "zookeeper", Map.empty)
)
}
case class MavenRepository(
final case class MavenRepository(
root: String,
changing: Option[Boolean] = None,
/** Hackish hack for sbt plugins mainly - what this does really sucks */
sbtAttrStub: Boolean = false,
authentication: Option[Authentication] = None,
packagingBlacklist: Set[Module] = MavenRepository.ignorePackaging
authentication: Option[Authentication] = None
) extends Repository {
import Repository._
@ -90,20 +73,29 @@ case class MavenRepository(
val root0 = if (root.endsWith("/")) root else root + "/"
val source = MavenSource(root0, changing, sbtAttrStub, authentication)
private def modulePath(
module: Module,
version: String
): Seq[String] =
module.organization.split('.').toSeq ++ Seq(
dirModuleName(module, sbtAttrStub),
version
)
private def urlFor(path: Seq[String]): String =
root0 + path.map(encodeURIComponent).mkString("/")
def projectArtifact(
module: Module,
version: String,
versioningValue: Option[String]
): Artifact = {
val path = module.organization.split('.').toSeq ++ Seq(
dirModuleName(module, sbtAttrStub),
version,
val path = modulePath(module, version) :+
s"${module.name}-${versioningValue getOrElse version}.pom"
)
Artifact(
root0 + path.map(encodeURIComponent).mkString("/"),
urlFor(path),
Map.empty,
Map.empty,
Attributes("pom", ""),
@ -123,7 +115,7 @@ case class MavenRepository(
val artifact =
Artifact(
root0 + path.map(encodeURIComponent).mkString("/"),
urlFor(path),
Map.empty,
Map.empty,
Attributes("pom", ""),
@ -141,15 +133,11 @@ case class MavenRepository(
version: String
): Option[Artifact] = {
val path = module.organization.split('.').toSeq ++ Seq(
dirModuleName(module, sbtAttrStub),
version,
"maven-metadata.xml"
)
val path = modulePath(module, version) :+ "maven-metadata.xml"
val artifact =
Artifact(
root0 + path.map(encodeURIComponent).mkString("/"),
urlFor(path),
Map.empty,
Map.empty,
Attributes("pom", ""),
@ -257,30 +245,94 @@ case class MavenRepository(
F: Monad[F]
): EitherT[F, String, Project] = {
fetch(projectArtifact(module, version, versioningValue)).flatMap { str =>
EitherT {
F.point[String \/ Project] {
for {
xml <- \/.fromEither(compatibility.xmlParse(str))
_ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
proj <- Pom.project(xml)
} yield {
val packagingOpt =
if (packagingBlacklist(module))
None
else
Pom.packagingOpt(xml)
def parseRawPom(str: String) =
for {
xml <- \/.fromEither(compatibility.xmlParse(str))
_ <- if (xml.label == "project") \/-(()) else -\/("Project definition not found")
proj <- Pom.project(xml)
} yield proj
proj.copy(
configurations = defaultConfigurations,
publications = defaultPublications(
module.name,
packagingOpt.getOrElse(defaultPackaging)
)
)
}
def artifactFor(url: String) =
Artifact(
url,
Map.empty,
Map.empty,
Attributes("", ""),
changing = true,
authentication
)
def isArtifact(fileName: String, prefix: String): Option[(String, String)] =
// TODO There should be a regex for that...
if (fileName.startsWith(prefix)) {
val end = fileName.stripPrefix(prefix)
val idx = end.lastIndexOf('.')
if (idx >= 0) {
val ext = end.drop(idx + 1)
val rem = end.take(idx)
if (rem.isEmpty)
Some(("", ext))
else if (rem.startsWith("-"))
Some((rem.drop(1), ext))
else
None
} else
None
} else
None
val listFilesUrl = urlFor(modulePath(module, version)) + "/"
for {
str <- fetch(projectArtifact(module, version, versioningValue))
rawListFilesPage <- fetch(artifactFor(listFilesUrl))
proj0 <- EitherT(F.point[String \/ Project](parseRawPom(str)))
} yield {
val files = WebPage.listFiles(listFilesUrl, rawListFilesPage)
val versioning = proj0
.snapshotVersioning
.flatMap(versioning =>
mavenVersioning(versioning, "", "")
)
val prefix = s"${module.name}-${versioning.getOrElse(version)}"
val packagingTpeMap = proj0.packagingOpt
.map { packaging =>
(MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
}
}
.toMap
val foundPublications = files
.flatMap(isArtifact(_, prefix))
.map {
case (classifier, ext) =>
val tpe = packagingTpeMap.getOrElse(
(classifier, ext),
MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
)
val config = MavenSource.typeDefaultConfig(tpe).getOrElse("compile")
config -> Publication(
module.name,
tpe,
ext,
classifier
)
}
val proj = Pom.addOptionalDependenciesInConfig(
proj0.copy(configurations = defaultConfigurations),
Set("", "compile"),
"optional"
)
proj.copy(
actualVersionOpt = Some(version),
publications = foundPublications
)
}
}

View File

@ -2,7 +2,7 @@ package coursier.maven
import coursier.core._
case class MavenSource(
final case class MavenSource(
root: String,
changing: Option[Boolean] = None,
/** See doc on MavenRepository */
@ -13,34 +13,6 @@ case class MavenSource(
import Repository._
import MavenRepository._
private implicit class DocSourcesArtifactExtensions(val underlying: Artifact) {
def withJavadocSources: Artifact = {
val base = underlying.url.stripSuffix(".jar")
underlying.copy(extra = underlying.extra ++ Seq(
"sources" -> Artifact(
base + "-sources.jar",
Map.empty,
Map.empty,
Attributes("jar", "src"), // Are these the right attributes?
changing = underlying.changing,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature,
"javadoc" -> Artifact(
base + "-javadoc.jar",
Map.empty,
Map.empty,
Attributes("jar", "javadoc"), // Same comment as above
changing = underlying.changing,
authentication = authentication
)
.withDefaultChecksums
.withDefaultSignature
))
}
}
def artifacts(
dependency: Dependency,
project: Project,
@ -73,9 +45,9 @@ case class MavenSource(
)
.withDefaultChecksums
if (publication.ext == "jar") {
if (publication.ext == "jar")
// TODO Get available signature / checksums from directory listing
artifact = artifact.withDefaultSignature
}
artifact
}
@ -92,68 +64,33 @@ case class MavenSource(
val publications0 = overrideClassifiers match {
case Some(classifiers) =>
val classifiersSet = classifiers.toSet
val publications = project.publications.collect {
project.publications.collect {
case (_, p) if classifiersSet(p.classifier) =>
p
}
// Unlike with Ivy metadata, Maven POMs don't list the available publications (~artifacts)
// so we give a chance to any classifier we're given by returning some publications
// no matter what, even if we're unsure they're available.
if (publications.isEmpty)
classifiers.map { classifier =>
Publication(
dependency.module.name,
"jar",
"jar",
classifier
)
}
else
publications
case None =>
val publications =
if (dependency.attributes.classifier.nonEmpty)
// FIXME We're ignoring dependency.attributes.`type` in this case
project.publications.collect {
case (_, p) if p.classifier == dependency.attributes.classifier =>
p
}
else if (dependency.attributes.`type`.nonEmpty)
project.publications.collect {
case (_, p) if p.`type` == dependency.attributes.`type` =>
p
}
else
project.publications.collect {
case (_, p) if p.classifier.isEmpty =>
p
}
// See comment above
if (publications.isEmpty) {
val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
val extension = MavenSource.typeExtension(type0)
val classifier =
if (dependency.attributes.classifier.isEmpty)
MavenSource.typeDefaultClassifier(type0)
else
dependency.attributes.classifier
Seq(
Publication(
dependency.module.name,
type0,
extension,
classifier
)
)
} else
publications
if (dependency.attributes.classifier.nonEmpty)
// FIXME We're ignoring dependency.attributes.`type` in this case
project.publications.collect {
case (_, p) if p.classifier == dependency.attributes.classifier =>
p
}
else if (dependency.attributes.`type`.nonEmpty)
project.publications.collect {
case (_, p)
if p.`type` == dependency.attributes.`type` ||
p.ext == dependency.attributes.`type` // wow
=>
p
}
else
project.publications.collect {
case (_, p) if p.classifier.isEmpty =>
p
}
}
publications0.map(artifactWithExtra)
@ -194,4 +131,22 @@ object MavenSource {
def typeDefaultClassifier(`type`: String): String =
typeDefaultClassifierOpt(`type`).getOrElse("")
}
val classifierExtensionDefaultTypes: Map[(String, String), String] = Map(
("tests", "jar") -> "test-jar",
("javadoc", "jar") -> "doc",
("sources", "jar") -> "src"
// don't know much about "client" classifier, not including it here
)
def classifierExtensionDefaultTypeOpt(classifier: String, ext: String): Option[String] =
classifierExtensionDefaultTypes.get((classifier, ext))
val typeDefaultConfigs: Map[String, String] = Map(
"doc" -> "docs",
"src" -> "sources"
)
def typeDefaultConfig(`type`: String): Option[String] =
typeDefaultConfigs.get(`type`)
}

View File

@ -28,9 +28,6 @@ object Pom {
private def readVersion(node: Node) =
text(node, "version", "Version").getOrElse("").trim
private val defaultType = "jar"
private val defaultClassifier = ""
def dependency(node: Node): String \/ (String, Dependency) = {
for {
mod <- module(node)
@ -52,7 +49,7 @@ object Pom {
version0,
"",
exclusions.map(mod => (mod.organization, mod.name)).toSet,
Attributes(typeOpt getOrElse defaultType, classifierOpt getOrElse defaultClassifier),
Attributes(typeOpt.getOrElse(""), classifierOpt.getOrElse("")),
optional,
transitive = true
)
@ -253,6 +250,7 @@ object Pom {
profiles,
None,
None,
packagingOpt(pom),
None,
Nil,
Info(
@ -453,4 +451,24 @@ object Pom {
} yield modVers :+ modVer
}
}
def addOptionalDependenciesInConfig(
proj: Project,
fromConfigs: Set[String],
optionalConfig: String
): Project = {
val optionalDeps = proj.dependencies.collect {
case (conf, dep) if dep.optional && fromConfigs(conf) =>
optionalConfig -> dep.copy(optional = false)
}
val configurations = proj.configurations +
(optionalConfig -> (proj.configurations.getOrElse(optionalConfig, Nil) ++ fromConfigs.filter(_.nonEmpty)).distinct)
proj.copy(
configurations = configurations,
dependencies = proj.dependencies ++ optionalDeps
)
}
}

View File

@ -12,7 +12,7 @@ package object coursier {
version: String,
// Substituted by Resolver with its own default configuration (compile)
configuration: String = "",
attributes: Attributes = Attributes("jar"),
attributes: Attributes = Attributes(),
exclusions: Set[(String, String)] = Set.empty,
optional: Boolean = false,
transitive: Boolean = true

View File

@ -71,7 +71,7 @@ object Print {
else
("", "", "")
case class Elem(dep: Dependency, excluded: Boolean) {
final case class Elem(dep: Dependency, excluded: Boolean) {
lazy val reconciledVersion = resolution.reconciledVersions
.getOrElse(dep.module, dep.version)
@ -144,7 +144,7 @@ object Print {
if (reverse) {
case class Parent(
final case class Parent(
module: Module,
version: String,
dependsOn: Module,

View File

@ -0,0 +1,22 @@
package coursier.util
object WebPage {
def listElements(url: String, page: String, directories: Boolean): Seq[String] =
coursier.core.compatibility.listWebPageRawElements(page)
.collect {
case elem if elem.nonEmpty && elem.endsWith("/") == directories =>
elem
.stripSuffix("/")
.stripPrefix(url)
.stripPrefix(":") // bintray typically prepends these
}
.filter(n => !n.contains("/") && n != "." && n != "..")
def listDirectories(url: String, page: String): Seq[String] =
listElements(url, page, directories = true)
def listFiles(url: String, page: String): Seq[String] =
listElements(url, page, directories = false)
}

View File

@ -4,6 +4,8 @@ import org.scalajs.dom.raw.{ Event, XMLHttpRequest }
import scala.concurrent.{ ExecutionContext, Promise, Future }
import scala.language.implicitConversions
import scala.scalajs.js
import js.Dynamic.{ global => g }

View File

@ -5,10 +5,10 @@ import java.net.NetworkInterface
import java.nio.channels.{ FileLock, OverlappingFileLockException }
import org.http4s.dsl._
import org.http4s.headers.Authorization
import org.http4s.headers.{ Authorization, `Content-Type` }
import org.http4s.server.HttpService
import org.http4s.server.blaze.BlazeBuilder
import org.http4s.{ BasicCredentials, Challenge, EmptyBody, Request, Response }
import org.http4s.{ BasicCredentials, Challenge, EmptyBody, MediaType, Request, Response }
import caseapp._
@ -16,7 +16,7 @@ import scala.collection.JavaConverters._
import scalaz.concurrent.Task
case class HttpServerApp(
final case class HttpServerApp(
@ExtraName("d")
@ValueDescription("served directory")
directory: String,
@ -45,7 +45,10 @@ case class HttpServerApp(
password: String,
@ExtraName("r")
@ValueDescription("realm")
realm: String
realm: String,
@ExtraName("l")
@HelpMessage("Generate content listing pages for directories")
listPages: Boolean
) extends App {
val baseDir = new File(if (directory.isEmpty) "." else directory)
@ -171,16 +174,65 @@ case class HttpServerApp(
Locked()
}
def isDirectory(f: File): Task[Option[Boolean]] =
Task {
if (f.isDirectory)
Some(true)
else if (f.isFile)
Some(false)
else
None
}
def directoryListingPage(dir: File, title: String): Task[String] =
Task {
val entries = dir
.listFiles()
.flatMap { f =>
def name = f.getName
if (f.isDirectory)
Seq(name + "/")
else if (f.isFile)
Seq(name)
else
Nil
}
// meh escaping
// TODO Use to scalatags to generate that
s"""<!DOCTYPE html>
|<html>
|<head>
|<title>$title</title>
|</head>
|<body>
|<ul>
|${entries.map(e => " <li><a href=\"" + e + "\">" + e + "</a></li>").mkString("\n")}
|</ul>
|</body>
|</html>
""".stripMargin
}
def getService = authenticated {
case (method @ (GET | HEAD)) -> path =>
if (verbosityLevel >= 1)
Console.err.println(s"${method.name} $path")
val f = new File(baseDir, path.toList.mkString("/"))
val resp = if (f.exists())
Ok(f)
else
NotFound()
val relPath = path.toList.mkString("/")
val f = new File(baseDir, relPath)
val resp =
for {
isDirOpt <- isDirectory(f)
resp <- isDirOpt match {
case Some(true) if listPages =>
directoryListingPage(f, relPath).flatMap(page =>
Ok(page).withContentType(Some(`Content-Type`(MediaType.`text/html`)))
)
case Some(false) => Ok(f)
case _ => NotFound()
}
} yield resp
method match {
case HEAD =>

View File

@ -2,7 +2,8 @@
"repository": "https://github.com/alexarchambault/coursier.git",
"license": "Apache-2.0",
"devDependencies": {
"xmldom": "latest",
"xhr2": "latest"
"cheerio": "0.22.0",
"xmldom": "0.1.27",
"xhr2": "0.1.4"
}
}

View File

@ -12,7 +12,6 @@ object CoursierPlugin extends AutoPlugin {
object autoImport {
val coursierParallelDownloads = Keys.coursierParallelDownloads
val coursierMaxIterations = Keys.coursierMaxIterations
val coursierDefaultArtifactType = Keys.coursierDefaultArtifactType
val coursierChecksums = Keys.coursierChecksums
val coursierArtifactsChecksums = Keys.coursierArtifactsChecksums
val coursierCachePolicies = Keys.coursierCachePolicies
@ -76,7 +75,6 @@ object CoursierPlugin extends AutoPlugin {
) = Seq(
coursierParallelDownloads := 6,
coursierMaxIterations := 50,
coursierDefaultArtifactType := "",
coursierChecksums := Seq(Some("SHA-1"), None),
coursierArtifactsChecksums := Seq(None),
coursierCachePolicies := CachePolicy.default,

View File

@ -15,11 +15,11 @@ sealed abstract class Credentials extends Product with Serializable {
object Credentials {
case class Direct(user: String, password: String) extends Credentials {
final case class Direct(user: String, password: String) extends Credentials {
override def toString = s"Direct($user, ******)"
}
case class FromFile(file: File) extends Credentials {
final case class FromFile(file: File) extends Credentials {
private lazy val props = {
val p = new Properties()

View File

@ -3,6 +3,8 @@ package coursier
import java.io.{ File, FileNotFoundException, IOException }
import java.net.{ HttpURLConnection, URL, URLConnection }
import scala.language.higherKinds
import scalaz.{ EitherT, Monad }
object FallbackDependenciesRepository {
@ -63,7 +65,7 @@ object FallbackDependenciesRepository {
}
case class FallbackDependenciesRepository(
final case class FallbackDependenciesRepository(
fallbacks: Map[(Module, String), (URL, Boolean)]
) extends Repository {
@ -76,8 +78,10 @@ case class FallbackDependenciesRepository(
fallbacks.get(dependency.moduleVersion) match {
case None => Nil
case Some((url, changing)) =>
val url0 = url.toString
val ext = url0.substring(url0.lastIndexOf('.') + 1)
Seq(
Artifact(url.toString, Map.empty, Map.empty, Attributes("jar", ""), changing, None)
Artifact(url0, Map.empty, Map.empty, Attributes(ext, ""), changing, None)
)
}
}
@ -119,6 +123,7 @@ case class FallbackDependenciesRepository(
None,
None,
None,
None,
Nil,
Info.empty
)

View File

@ -53,8 +53,7 @@ object FromSbt {
def dependencies(
module: ModuleID,
scalaVersion: String,
scalaBinaryVersion: String,
defaultArtifactType: String
scalaBinaryVersion: String
): Seq[(String, Dependency)] = {
// TODO Warn about unsupported properties in `module`
@ -76,10 +75,10 @@ object FromSbt {
val attributes =
if (module.explicitArtifacts.isEmpty)
Seq(Attributes(defaultArtifactType, ""))
Seq(Attributes("", ""))
else
module.explicitArtifacts.map { a =>
Attributes(`type` = a.extension, classifier = a.classifier.getOrElse(""))
Attributes(`type` = a.`type`, classifier = a.classifier.getOrElse(""))
}
for {
@ -107,15 +106,14 @@ object FromSbt {
allDependencies: Seq[ModuleID],
ivyConfigurations: Map[String, Seq[String]],
scalaVersion: String,
scalaBinaryVersion: String,
defaultArtifactType: String
scalaBinaryVersion: String
): Project = {
// FIXME Ignored for now - easy to support though
// val sbtDepOverrides = dependencyOverrides.value
// val sbtExclusions = excludeDependencies.value
val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion, defaultArtifactType))
val deps = allDependencies.flatMap(dependencies(_, scalaVersion, scalaBinaryVersion))
Project(
Module(
@ -133,6 +131,7 @@ object FromSbt {
None,
None,
None,
None,
Nil,
Info.empty
)

View File

@ -1,8 +1,10 @@
package coursier
import scala.language.higherKinds
import scalaz.{ -\/, \/-, Monad, EitherT }
case class InterProjectRepository(projects: Seq[Project]) extends Repository {
final case class InterProjectRepository(projects: Seq[Project]) extends Repository {
private val map = projects
.map { proj => proj.moduleVersion -> proj }

View File

@ -13,7 +13,6 @@ import scalaz.\/
object Keys {
val coursierParallelDownloads = SettingKey[Int]("coursier-parallel-downloads")
val coursierMaxIterations = SettingKey[Int]("coursier-max-iterations")
val coursierDefaultArtifactType = SettingKey[String]("coursier-default-artifact-type")
val coursierChecksums = SettingKey[Seq[Option[String]]]("coursier-checksums")
val coursierArtifactsChecksums = SettingKey[Seq[Option[String]]]("coursier-artifacts-checksums")
val coursierCachePolicies = SettingKey[Seq[CachePolicy]]("coursier-cache-policies")

View File

@ -39,11 +39,11 @@ sealed abstract class ResolutionError extends Product with Serializable {
object ResolutionError {
case object MaximumIterationsReached extends ResolutionError
case class UnknownException(ex: Throwable) extends ResolutionError
case class UnknownDownloadException(ex: Throwable) extends ResolutionError
case class Conflicts(description: String) extends ResolutionError
final case class UnknownException(ex: Throwable) extends ResolutionError
final case class UnknownDownloadException(ex: Throwable) extends ResolutionError
final case class Conflicts(description: String) extends ResolutionError
case class MetadataDownloadErrors(errors: Seq[(Dependency, Seq[String])]) extends ResolutionError {
final case class MetadataDownloadErrors(errors: Seq[(Dependency, Seq[String])]) extends ResolutionError {
def description(): String = {
def grouped(errs: Seq[String]) =
@ -86,7 +86,7 @@ object ResolutionError {
}
}
case class DownloadErrors(errors: Seq[FileError]) extends ResolutionError {
final case class DownloadErrors(errors: Seq[FileError]) extends ResolutionError {
def description(verbose: Boolean): String = {

View File

@ -2,6 +2,8 @@ package coursier
import sbt._
import scala.language.implicitConversions
// things from sbt-structure
object Structure {
import Def.Initialize._

View File

@ -1,6 +1,6 @@
package coursier
import java.io.{ OutputStreamWriter, File }
import java.io.{ File, InputStream, OutputStreamWriter }
import java.net.URL
import java.util.concurrent.{ ExecutorService, Executors }
@ -118,19 +118,21 @@ object Tasks {
lazy val projId = projectID.in(projectRef).get(state)
lazy val sv = scalaVersion.in(projectRef).get(state)
lazy val sbv = scalaBinaryVersion.in(projectRef).get(state)
lazy val defaultArtifactType = coursierDefaultArtifactType.in(projectRef).get(state)
for {
allDependencies <- allDependenciesTask
} yield {
val configMap = configurations
.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }
.toMap
FromSbt.project(
projId,
allDependencies,
configurations.map { cfg => cfg.name -> cfg.extendsConfigs.map(_.name) }.toMap,
configMap,
sv,
sbv,
defaultArtifactType
sbv
)
}
}
@ -250,7 +252,7 @@ object Tasks {
}
}
private case class ResolutionCacheKey(
private final case class ResolutionCacheKey(
project: Project,
repositories: Seq[Repository],
userEnabledProfiles: Set[String],
@ -258,7 +260,7 @@ object Tasks {
sbtClassifiers: Boolean
)
private case class ReportCacheKey(
private final case class ReportCacheKey(
project: Project,
resolution: Resolution,
withClassifiers: Boolean,
@ -333,15 +335,13 @@ object Tasks {
if (sbtClassifiers) {
val sv = scalaVersion.value
val sbv = scalaBinaryVersion.value
val defaultArtifactType = coursierDefaultArtifactType.value
val proj = FromSbt.project(
cm.id,
cm.modules,
cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap,
sv,
sbv,
defaultArtifactType
sbv
)
val fallbackDeps = FromSbt.fallbackDependencies(
@ -828,15 +828,13 @@ object Tasks {
if (sbtClassifiers) {
val sv = scalaVersion.value
val sbv = scalaBinaryVersion.value
val defaultArtifactType = coursierDefaultArtifactType.value
FromSbt.project(
cm.id,
cm.modules,
cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap,
sv,
sbv,
defaultArtifactType
sbv
)
} else {
val proj = coursierProject.value
@ -977,15 +975,13 @@ object Tasks {
val cm = coursierSbtClassifiersModule.value
val sv = scalaVersion.value
val sbv = scalaBinaryVersion.value
val defaultArtifactType = coursierDefaultArtifactType.value
FromSbt.project(
cm.id,
cm.modules,
cm.configurations.map(cfg => cfg.name -> cfg.extendsConfigs.map(_.name)).toMap,
sv,
sbv,
defaultArtifactType
sbv
)
} else {
val proj = coursierProject.value

Some files were not shown because too many files have changed in this diff Show More