luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add fallback if directory listings are not available

This commit is contained in:
Alexandre Archambault 2017-03-23 15:19:07 +01:00
parent 3afc3433ba
commit b54438c1ba
14 changed files with 221 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
appveyor.yml
View File

@ -26,7 +26,9 @@ build_script:
- sbt ++2.10.6 clean compile
- sbt ++2.10.6 coreJVM/publishLocal cache/publishLocal # to make the scripted tests happy
test_script:
- ps: Start-Job -filepath .\scripts\start-it-server.ps1 -ArgumentList $pwd
- ps: Start-Job -filepath .\scripts\start-it-auth-server.ps1 -ArgumentList $pwd
- ps: Start-Sleep -s 15 # wait for the first server to have downloaded its dependencies
- ps: Start-Job -filepath .\scripts\start-it-no-listing-server.ps1 -ArgumentList $pwd
- sbt ++2.12.1 testsJVM/test testsJVM/it:test # Would node be around for testsJS/test?
- sbt ++2.11.8 testsJVM/test testsJVM/it:test
- sbt ++2.10.6 testsJVM/test testsJVM/it:test sbt-coursier/scripted sbt-coursier/publishLocal sbt-shading/scripted

82
cache/src/main/scala/coursier/Cache.scala vendored
View File

@ -319,6 +319,43 @@ object Cache {
def url(s: String): URL =
new URL(null, s, handlerFor(s).orNull)
def urlConnection(url0: String, authentication: Option[Authentication]) = {
var conn: URLConnection = null
try {
conn = url(url0).openConnection() // FIXME Should this be closed?
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata)
conn.setRequestProperty("User-Agent", "")
for (auth <- authentication)
conn match {
case authenticated: AuthenticatedURLConnection =>
authenticated.authenticate(auth)
case conn0: HttpURLConnection =>
conn0.setRequestProperty(
"Authorization",
"Basic " + basicAuthenticationEncode(auth.user, auth.password)
)
case _ =>
// FIXME Authentication is ignored
}
conn
} catch {
case NonFatal(e) =>
if (conn != null)
conn match {
case conn0: HttpURLConnection =>
conn0.getInputStream.close()
conn0.disconnect()
case _ =>
}
throw e
}
}
private def download(
artifact: Artifact,
cache: File,
@ -340,45 +377,6 @@ object Cache {
def referenceFileExists: Boolean = referenceFileOpt.exists(_.exists())
def urlConn(url0: String) = {
var conn: URLConnection = null
try {
conn = url(url0).openConnection() // FIXME Should this be closed?
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
// this is SO FSCKING CRAZY)
conn.setRequestProperty("User-Agent", "")
for (auth <- artifact.authentication)
conn match {
case authenticated: AuthenticatedURLConnection =>
authenticated.authenticate(auth)
case conn0: HttpURLConnection =>
conn0.setRequestProperty(
"Authorization",
"Basic " + basicAuthenticationEncode(auth.user, auth.password)
)
case _ =>
// FIXME Authentication is ignored
}
conn
} catch {
case NonFatal(e) =>
if (conn != null)
conn match {
case conn0: HttpURLConnection =>
conn0.getInputStream.close()
conn0.disconnect()
case _ =>
}
throw e
}
}
def fileLastModified(file: File): EitherT[Task, FileError, Option[Long]] =
EitherT {
Task {
@ -402,7 +400,7 @@ object Cache {
var conn: URLConnection = null
try {
conn = urlConn(url)
conn = urlConnection(url, artifact.authentication)
conn match {
case c: HttpURLConnection =>
@ -558,7 +556,7 @@ object Cache {
var conn: URLConnection = null
try {
conn = urlConn(url)
conn = urlConnection(url, artifact.authentication)
val partialDownload = conn match {
case conn0: HttpURLConnection if alreadyDownloaded > 0L =>
@ -571,7 +569,7 @@ object Cache {
// unrecognized Content-Range header -> start a new connection with no resume
conn0.getInputStream.close()
conn0.disconnect()
conn = urlConn(url)
conn = urlConnection(url, artifact.authentication)
false
}
}

11
cache/src/main/scala/coursier/Platform.scala vendored
View File

@ -42,19 +42,12 @@ object Platform {
val artifact: Fetch.Content[Task] = { artifact =>
EitherT {
val url = Cache.url(artifact.url)
val conn = url.openConnection()
// Dummy user-agent instead of the default "Java/...",
// so that we are not returned incomplete/erroneous metadata
// (Maven 2 compatibility? - happens for snapshot versioning metadata,
// this is SO FUCKING CRAZY)
conn.setRequestProperty("User-Agent", "")
val conn = Cache.urlConnection(artifact.url, artifact.authentication)
readFully(conn.getInputStream())
}
}
implicit def fetch(
def fetch(
repositories: Seq[core.Repository]
): Fetch.Metadata[Task] =
Fetch.from(repositories, Platform.artifact)

57
core/shared/src/main/scala/coursier/maven/MavenRepository.scala
View File

@ -286,35 +286,46 @@ final case class MavenRepository(
for {
str <- fetch(projectArtifact(module, version, versioningValue))
rawListFilesPage <- fetch(artifactFor(listFilesUrl))
rawListFilesPageOpt <- EitherT(F.map(fetch(artifactFor(listFilesUrl)).run) {
e => \/-(e.toOption): String \/ Option[String]
})
proj0 <- EitherT(F.point[String \/ Project](parseRawPom(str)))
} yield {
val files = WebPage.listFiles(listFilesUrl, rawListFilesPage)
val foundPublications =
rawListFilesPageOpt match {
case Some(rawListFilesPage) =>
val prefix = s"${module.name}-${versioningValue.getOrElse(version)}"
val files = WebPage.listFiles(listFilesUrl, rawListFilesPage)
val packagingTpeMap = proj0.packagingOpt
.map { packaging =>
(MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
}
.toMap
val prefix = s"${module.name}-${versioningValue.getOrElse(version)}"
val foundPublications = files
.flatMap(isArtifact(_, prefix))
.map {
case (classifier, ext) =>
val tpe = packagingTpeMap.getOrElse(
(classifier, ext),
MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
)
val config = MavenSource.typeDefaultConfig(tpe).getOrElse("compile")
config -> Publication(
module.name,
tpe,
ext,
classifier
)
val packagingTpeMap = proj0.packagingOpt
.map { packaging =>
(MavenSource.typeDefaultClassifier(packaging), MavenSource.typeExtension(packaging)) -> packaging
}
.toMap
files
.flatMap(isArtifact(_, prefix))
.map {
case (classifier, ext) =>
val tpe = packagingTpeMap.getOrElse(
(classifier, ext),
MavenSource.classifierExtensionDefaultTypeOpt(classifier, ext).getOrElse(ext)
)
val config = MavenSource.typeDefaultConfig(tpe).getOrElse("compile")
config -> Publication(
module.name,
tpe,
ext,
classifier
)
}
case None =>
// Publications can't be listed - MavenSource then handles that
Nil
}
val proj = Pom.addOptionalDependenciesInConfig(

43
core/shared/src/main/scala/coursier/maven/MavenSource.scala
View File

@ -65,14 +65,47 @@ final case class MavenSource(
case Some(classifiers) =>
val classifiersSet = classifiers.toSet
project.publications.collect {
case (_, p) if classifiersSet(p.classifier) =>
p
}
if (project.publications.isEmpty)
// For repositories not providing directory listings, give a try to some publications anyway
classifiers.map { classifier =>
Publication(
dependency.module.name,
"jar",
"jar",
classifier
)
}
else
project.publications.collect {
case (_, p) if classifiersSet(p.classifier) =>
p
}
case None =>
if (dependency.attributes.classifier.nonEmpty)
if (project.publications.isEmpty) {
// For repositories not providing directory listings, give a try to some publications anyway
val type0 = if (dependency.attributes.`type`.isEmpty) "jar" else dependency.attributes.`type`
val extension = MavenSource.typeExtension(type0)
val classifier =
if (dependency.attributes.classifier.isEmpty)
MavenSource.typeDefaultClassifier(type0)
else
dependency.attributes.classifier
Seq(
Publication(
dependency.module.name,
type0,
extension,
classifier
)
)
} else if (dependency.attributes.classifier.nonEmpty)
// FIXME We're ignoring dependency.attributes.`type` in this case
project.publications.collect {
case (_, p) if p.classifier == dependency.attributes.classifier =>

2
fetch-js/src/main/scala/coursier/Platform.scala
View File

@ -88,7 +88,7 @@ object Platform {
)
}
implicit def fetch(
def fetch(
repositories: Seq[core.Repository]
): Fetch.Metadata[Task] =
Fetch.from(repositories, Platform.artifact)

15
scripts/launch-test-repo.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -e
VERSION="${VERSION:-1.0.0-SNAPSHOT}"
cd "$(dirname "$0")/.."
./coursier launch \
"io.get-coursier:http-server-java7_2.11:$VERSION" \
-r https://dl.bintray.com/scalaz/releases \
-- \
-d tests/jvm/src/test/resources/test-repo/http/abc.com \
-u user -P pass -r realm \
-v \
"$@" &

3
scripts/start-it-auth-server.ps1 Normal file
View File

@ -0,0 +1,3 @@
# see https://stackoverflow.com/questions/2224350/powershell-start-job-working-directory/2246542#2246542
Set-Location $args[0]
& java -jar -noverify coursier launch -r https://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm --port 8080 --list-pages -v

2
scripts/start-it-server.ps1 → scripts/start-it-no-listing-server.ps1
View File

@ -1,3 +1,3 @@
# see https://stackoverflow.com/questions/2224350/powershell-start-job-working-directory/2246542#2246542
Set-Location $args[0]
& java -jar -noverify coursier launch -r https://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm --list-pages -v
& java -jar -noverify coursier launch -r https://dl.bintray.com/scalaz/releases io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT -- -d tests/jvm/src/test/resources/test-repo/http/abc.com -u user -P pass -r realm --port 8081 -v

16
scripts/travis.sh
View File

@ -20,19 +20,19 @@ downloadInstallSbtExtras() {
chmod +x bin/sbt
}
launchTestRepo() {
./scripts/launch-test-repo.sh "$@"
}
integrationTestsRequirements() {
# Required for ~/.ivy2/local repo tests
sbt ++2.11.8 coreJVM/publishLocal http-server/publishLocal
# Required for HTTP authentication tests
coursier launch \
io.get-coursier:http-server-java7_2.11:1.0.0-SNAPSHOT \
-r https://dl.bintray.com/scalaz/releases \
-- \
-d tests/jvm/src/test/resources/test-repo/http/abc.com \
-u user -P pass -r realm \
--list-pages \
-v &
launchTestRepo --port 8080 --list-pages
# Required for missing directory listing tests (no --list-pages)
launchTestRepo --port 8081
}
setupCustomJarjar() {

71
tests/jvm/src/it/scala/coursier/test/DirectoryListingTests.scala Normal file
View File

@ -0,0 +1,71 @@
package coursier.test
import coursier._
import coursier.core.Authentication
import utest._
object DirectoryListingTests extends TestSuite {
val user = "user"
val password = "pass"
val withListingRepo = MavenRepository(
"http://localhost:8080",
authentication = Some(Authentication(user, password))
)
val withoutListingRepo = MavenRepository(
"http://localhost:8081",
authentication = Some(Authentication(user, password))
)
val module = Module("com.abc", "test")
val version = "0.1"
val tests = TestSuite {
'withListing - {
'jar - CentralTests.withArtifacts(
module,
version,
"jar",
extraRepo = Some(withListingRepo)
) {
artifacts =>
assert(artifacts.length == 1)
}
'jarFoo - CentralTests.withArtifacts(
module,
version,
"jar-foo",
extraRepo = Some(withListingRepo)
) {
artifacts =>
assert(artifacts.length == 1)
}
}
'withoutListing - {
'jar - CentralTests.withArtifacts(
module,
version,
"jar",
extraRepo = Some(withoutListingRepo)
) {
artifacts =>
assert(artifacts.length == 1)
}
'jarFoo - CentralTests.withArtifacts(
module,
version,
"jar-foo",
extraRepo = Some(withoutListingRepo)
) {
artifacts =>
assert(artifacts.length == 0)
}
}
}
}

1
tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.jar Normal file
View File

@ -0,0 +1 @@
nothing here

1
tests/jvm/src/test/resources/test-repo/http/abc.com/com/abc/test/0.1/test-0.1.jar-foo Normal file
View File

@ -0,0 +1 @@
nothing here

4
tests/shared/src/test/scala/coursier/test/CentralTests.scala
View File

@ -23,13 +23,15 @@ object CentralTests extends TestSuite {
) = {
val repositories0 = extraRepo.toSeq ++ repositories
val fetch = Platform.fetch(repositories0)
Resolution(
deps,
filter = filter,
userActivations = profiles.map(_.iterator.map(_ -> true).toMap)
)
.process
.run(repositories0)
.run(fetch)
.runF
}