luke
/
sbt
mirror of https://github.com/sbt/sbt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for custom protocols (#327) 

* Add configuration to fetch custom protcol handlers

* Tweak things

Co-authored-by: Alexandre Archambault <alexandre.archambault@gmail.com>
This commit is contained in:
Guillaume Massé 2021-11-23 15:14:52 -05:00 committed by GitHub
parent 2eaf7aef43
commit 92e40c2225
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 303 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
build.sbt
View File

@ -39,7 +39,19 @@ lazy val `lm-coursier` = project
else lm2_13Version
},
"org.scalatest" %% "scalatest" % "3.2.7" % Test
)
),
Test / test := {
(publishLocal in customProtocolForTest212).value
(publishLocal in customProtocolForTest213).value
(publishLocal in customProtocolJavaForTest).value
(Test / test).value
},
Test / testOnly := {
(publishLocal in customProtocolForTest212).value
(publishLocal in customProtocolForTest213).value
(publishLocal in customProtocolJavaForTest).value
(Test / testOnly).evaluated
}
)
lazy val `lm-coursier-shaded` = project
@ -153,6 +165,38 @@ lazy val `sbt-coursier` = project
}
)
lazy val customProtocolForTest212 = project
.in(file("modules/custom-protocol-for-test-2-12"))
.settings(
sourceDirectory := file("modules/custom-protocol-for-test/src").toPath.toAbsolutePath.toFile,
scalaVersion := scala212,
organization := "org.example",
moduleName := "customprotocol-handler",
version := "0.1.0",
dontPublish
)
lazy val customProtocolForTest213 = project
.in(file("modules/custom-protocol-for-test-2-13"))
.settings(
sourceDirectory := file("modules/custom-protocol-for-test/src").toPath.toAbsolutePath.toFile,
scalaVersion := scala213,
organization := "org.example",
moduleName := "customprotocol-handler",
version := "0.1.0",
dontPublish
)
lazy val customProtocolJavaForTest = project
.in(file("modules/custom-protocol-java-for-test"))
.settings(
crossPaths := false,
organization := "org.example",
moduleName := "customprotocoljava-handler",
version := "0.1.0",
dontPublish
)
lazy val `sbt-coursier-root` = project
.in(file("."))
.disablePlugins(MimaPlugin)

10
modules/custom-protocol-for-test/src/main/scala/coursier/cache/protocol/CustomprotocolHandler.scala vendored Normal file
View File

@ -0,0 +1,10 @@
package coursier.cache.protocol
import java.net.{URL, URLConnection, URLStreamHandler, URLStreamHandlerFactory}
class CustomprotocolHandler extends URLStreamHandlerFactory {
def createURLStreamHandler(protocol: String): URLStreamHandler = new URLStreamHandler {
protected def openConnection(url: URL): URLConnection =
new URL("https://repo1.maven.org/maven2" + url.getPath()).openConnection()
}
}

18
modules/custom-protocol-java-for-test/src/main/java/coursier/cache/protocol/CustomprotocoljavaHandler.java vendored Normal file
View File

@ -0,0 +1,18 @@
package coursier.cache.protocol;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLStreamHandler;
import java.net.URLStreamHandlerFactory;
import java.io.IOException;
public class CustomprotocoljavaHandler implements URLStreamHandlerFactory {
public URLStreamHandler createURLStreamHandler(String protocol) {
return new URLStreamHandler() {
protected URLConnection openConnection(URL url) throws IOException {
return new URL("https://repo1.maven.org/maven2" + url.getPath()).openConnection();
}
};
}
}

8
modules/lm-coursier/src/main/scala/lmcoursier/CoursierConfiguration.scala
View File

@ -6,10 +6,12 @@ import dataclass.data
import coursier.cache.CacheDefaults
import lmcoursier.credentials.Credentials
import lmcoursier.definitions.{Authentication, CacheLogger, CachePolicy, FromCoursier, Module, ModuleMatchers, Project, Reconciliation, Strict}
import sbt.librarymanagement.{Resolver, UpdateConfiguration}
import sbt.librarymanagement.{Resolver, UpdateConfiguration, ModuleID, CrossVersion, ModuleInfo, ModuleDescriptorConfiguration}
import xsbti.Logger
import scala.concurrent.duration.Duration
import java.net.URL
import java.net.URLClassLoader
@data class CoursierConfiguration(
log: Option[Logger] = None,
@ -54,7 +56,9 @@ import scala.concurrent.duration.Duration
@since
sbtClassifiers: Boolean = false,
@since
providedInCompile: Boolean = false // unused, kept for binary compatibility
providedInCompile: Boolean = false, // unused, kept for binary compatibility
@since
protocolHandlerDependencies: Seq[ModuleID] = Vector.empty,
) {
def withLog(log: Logger): CoursierConfiguration =

104
modules/lm-coursier/src/main/scala/lmcoursier/CoursierDependencyResolution.scala
View File

@ -1,6 +1,7 @@
package lmcoursier
import java.io.File
import java.net.{URL, URLClassLoader, URLConnection, MalformedURLException}
import coursier.{Organization, Resolution, organizationString}
import coursier.core.{Classifier, Configuration}
@ -15,10 +16,86 @@ import sbt.util.Logger
import coursier.core.Dependency
import coursier.core.Publication
class CoursierDependencyResolution(conf: CoursierConfiguration) extends DependencyResolutionInterface {
import scala.util.{Try, Failure}
class CoursierDependencyResolution(
conf: CoursierConfiguration,
protocolHandlerConfiguration: Option[CoursierConfiguration],
bootstrappingProtocolHandler: Boolean
) extends DependencyResolutionInterface {
def this(conf: CoursierConfiguration) =
this(
conf,
protocolHandlerConfiguration = None,
bootstrappingProtocolHandler = true
)
lmcoursier.CoursierConfiguration.checkLegacyCache()
private var protocolHandlerClassLoader: Option[ClassLoader] = None
private val protocolHandlerClassLoaderLock = new Object
private def fetchProtocolHandlerClassLoader(
configuration: UpdateConfiguration,
uwconfig: UnresolvedWarningConfiguration,
log: Logger
): ClassLoader = {
val conf0 = protocolHandlerConfiguration.getOrElse(conf)
def isUnknownProtocol(rawURL: String): Boolean = {
Try(new URL(rawURL)) match {
case Failure(ex) if ex.getMessage.startsWith("unknown protocol: ") => true
case _ => false
}
}
val confWithoutUnknownProtocol =
conf0.withResolvers(
conf0.resolvers.filter {
case maven: MavenRepository =>
!isUnknownProtocol(maven.root)
case _ =>
true
}
)
val resolution = new CoursierDependencyResolution(
conf = confWithoutUnknownProtocol,
protocolHandlerConfiguration = None,
bootstrappingProtocolHandler = false
)
val fakeModule =
ModuleDescriptorConfiguration(
ModuleID("lmcoursier", "lmcoursier", "0.1.0"),
ModuleInfo("protocol-handler")
)
.withDependencies(conf0.protocolHandlerDependencies.toVector)
val reportOrUnresolved = resolution.update(moduleDescriptor(fakeModule), configuration, uwconfig, log)
val report = reportOrUnresolved match {
case Right(report0) =>
report0
case Left(unresolvedWarning) =>
import sbt.util.ShowLines._
unresolvedWarning.lines.foreach(log.warn(_))
throw unresolvedWarning.resolveException
}
val jars =
for {
reportConfiguration <- report.configurations.filter(_.configuration.name == "runtime")
module <- reportConfiguration.modules
(_, jar) <- module.artifacts
} yield jar
new URLClassLoader(jars.map(_.toURI().toURL()).toArray)
}
/*
* Based on earlier implementations by @leonardehrenfried (https://github.com/sbt/librarymanagement/pull/190)
* and @andreaTP (https://github.com/sbt/librarymanagement/pull/270), then adapted to the code from the former
@ -35,6 +112,14 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
log: Logger
): Either[UnresolvedWarning, UpdateReport] = {
if (bootstrappingProtocolHandler && protocolHandlerClassLoader.isEmpty)
protocolHandlerClassLoaderLock.synchronized {
if (bootstrappingProtocolHandler && protocolHandlerClassLoader.isEmpty) {
val classLoader = fetchProtocolHandlerClassLoader(configuration, uwconfig, log)
protocolHandlerClassLoader = Some(classLoader)
}
}
val conf = this.conf.withUpdateConfiguration(configuration)
// TODO Take stuff in configuration into account? uwconfig too?
@ -103,7 +188,8 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
resolver,
ivyProperties,
log,
authenticationByRepositoryId.get(resolver.name).map(ToCoursier.authentication)
authenticationByRepositoryId.get(resolver.name).map(ToCoursier.authentication),
protocolHandlerClassLoader.toSeq,
)
}
@ -213,7 +299,8 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
includeSignatures = false,
sbtBootJarOverrides = sbtBootJarOverrides,
classpathOrder = conf.classpathOrder,
missingOk = conf.missingOk
missingOk = conf.missingOk,
classLoaders = protocolHandlerClassLoader.toSeq,
)
val e = for {
@ -264,13 +351,22 @@ class CoursierDependencyResolution(conf: CoursierConfiguration) extends Dependen
} else
throw ex
}
}
object CoursierDependencyResolution {
def apply(configuration: CoursierConfiguration): DependencyResolution =
DependencyResolution(new CoursierDependencyResolution(configuration))
def apply(configuration: CoursierConfiguration,
protocolHandlerConfiguration: Option[CoursierConfiguration]): DependencyResolution =
DependencyResolution(
new CoursierDependencyResolution(
configuration,
protocolHandlerConfiguration,
bootstrappingProtocolHandler = true
)
)
def defaultCacheLocation: File =
CacheDefaults.location
}

21
modules/lm-coursier/src/main/scala/lmcoursier/internal/Resolvers.scala
View File

@ -30,10 +30,11 @@ object Resolvers {
private def mavenRepositoryOpt(
root: String,
log: Logger,
authentication: Option[Authentication]
authentication: Option[Authentication],
classLoaders: Seq[ClassLoader]
): Option[MavenRepository] =
try {
CacheUrl.url(root) // ensure root is a URL whose protocol can be handled here
CacheUrl.url(root, classLoaders) // ensure root is a URL whose protocol can be handled here
val root0 = if (root.endsWith("/")) root else root + "/"
Some(
MavenRepository(
@ -69,11 +70,12 @@ object Resolvers {
resolver: Resolver,
ivyProperties: Map[String, String],
log: Logger,
authentication: Option[Authentication]
authentication: Option[Authentication],
classLoaders: Seq[ClassLoader]
): Option[Repository] =
resolver match {
case r: sbt.librarymanagement.MavenRepository =>
mavenRepositoryOpt(r.root, log, authentication)
mavenRepositoryOpt(r.root, log, authentication, classLoaders)
case r: FileRepository
if r.patterns.ivyPatterns.lengthCompare(1) == 0 &&
@ -103,18 +105,18 @@ object Resolvers {
Some(repo)
case Some(mavenCompatibleBase) =>
mavenRepositoryOpt(pathToUriString(mavenCompatibleBase), log, authentication)
mavenRepositoryOpt(pathToUriString(mavenCompatibleBase), log, authentication, classLoaders)
}
case r: URLRepository if patternMatchGuard(r.patterns) =>
parseMavenCompatResolver(log, ivyProperties, authentication, r.patterns)
parseMavenCompatResolver(log, ivyProperties, authentication, r.patterns, classLoaders)
case raw: RawRepository if raw.name == "inter-project" => // sbt.RawRepository.equals just compares names anyway
None
// Pattern Match resolver-type-specific RawRepositories
case IBiblioRepository(p) =>
parseMavenCompatResolver(log, ivyProperties, authentication, p)
parseMavenCompatResolver(log, ivyProperties, authentication, p, classLoaders)
case other =>
log.warn(s"Unrecognized repository ${other.name}, ignoring it")
@ -159,7 +161,8 @@ object Resolvers {
log: Logger,
ivyProperties: Map[String, String],
authentication: Option[Authentication],
patterns: Patterns
patterns: Patterns,
classLoaders: Seq[ClassLoader],
): Option[Repository] = {
val mavenCompatibleBaseOpt0 = mavenCompatibleBaseOpt(patterns)
@ -185,7 +188,7 @@ object Resolvers {
Some(repo)
case Some(mavenCompatibleBase) =>
mavenRepositoryOpt(mavenCompatibleBase, log, authentication)
mavenRepositoryOpt(mavenCompatibleBase, log, authentication, classLoaders)
}
}
}

28
modules/lm-coursier/src/main/scala/lmcoursier/internal/SbtUpdateReport.scala
View File

@ -59,8 +59,8 @@ private[internal] object SbtUpdateReport {
.withIsTransitive(dependency.transitive)
}
private val artifact = caching[(Module, Map[String, String], Publication, Artifact), sbt.librarymanagement.Artifact] {
case (module, extraProperties, pub, artifact) =>
private val artifact = caching[(Module, Map[String, String], Publication, Artifact, Seq[ClassLoader]), sbt.librarymanagement.Artifact] {
case (module, extraProperties, pub, artifact, classLoaders) =>
sbt.librarymanagement.Artifact(pub.name)
.withType(pub.`type`.value)
.withExtension(pub.ext.value)
@ -70,20 +70,20 @@ private[internal] object SbtUpdateReport {
.orElse(MavenAttributes.typeDefaultClassifierOpt(pub.`type`))
.map(_.value)
)
.withUrl(Some(CacheUrl.url(artifact.url)))
.withUrl(Some(CacheUrl.url(artifact.url, classLoaders)))
.withExtraAttributes(module.attributes ++ extraProperties)
}
private val moduleReport = caching[(Dependency, Seq[(Dependency, ProjectInfo)], Project, Seq[(Publication, Artifact, Option[File])]), ModuleReport] {
case (dependency, dependees, project, artifacts) =>
private val moduleReport = caching[(Dependency, Seq[(Dependency, ProjectInfo)], Project, Seq[(Publication, Artifact, Option[File])], Seq[ClassLoader]), ModuleReport] {
case (dependency, dependees, project, artifacts, classLoaders) =>
val sbtArtifacts = artifacts.collect {
case (pub, artifact0, Some(file)) =>
(artifact((dependency.module, infoProperties(project).toMap, pub, artifact0)), file)
(artifact((dependency.module, infoProperties(project).toMap, pub, artifact0, classLoaders)), file)
}
val sbtMissingArtifacts = artifacts.collect {
case (pub, artifact0, None) =>
artifact((dependency.module, infoProperties(project).toMap, pub, artifact0))
artifact((dependency.module, infoProperties(project).toMap, pub, artifact0, classLoaders))
}
val publicationDate = project.info.publication.map { dt =>
@ -140,7 +140,8 @@ private[internal] object SbtUpdateReport {
log: Logger,
includeSignatures: Boolean,
classpathOrder: Boolean,
missingOk: Boolean
missingOk: Boolean,
classLoaders: Seq[ClassLoader]
): Vector[ModuleReport] = {
val deps = classifiersOpt match {
@ -280,7 +281,8 @@ private[internal] object SbtUpdateReport {
dep,
dependees,
proj,
filesOpt
filesOpt,
classLoaders,
))
}
}
@ -297,7 +299,8 @@ private[internal] object SbtUpdateReport {
includeSignatures: Boolean,
classpathOrder: Boolean,
missingOk: Boolean,
forceVersions: Map[Module, String]
forceVersions: Map[Module, String],
classLoaders: Seq[ClassLoader],
): UpdateReport = {
val configReports = resolutions.map {
@ -313,7 +316,8 @@ private[internal] object SbtUpdateReport {
log,
includeSignatures = includeSignatures,
classpathOrder = classpathOrder,
missingOk = missingOk
missingOk = missingOk,
classLoaders = classLoaders,
)
val reports0 = subRes.rootDependencies match {
@ -355,7 +359,7 @@ private[internal] object SbtUpdateReport {
// should not happen
ProjectInfo(c.dependeeVersion, Vector.empty, Vector.empty)
}
val rep = moduleReport((dep, Seq((dependee, dependeeProj)), proj.withVersion(c.wantedVersion), Nil))
val rep = moduleReport((dep, Seq((dependee, dependeeProj)), proj.withVersion(c.wantedVersion), Nil, classLoaders))
.withEvicted(true)
.withEvictedData(Some("version selection")) // ??? put latest-revision like sbt/ivy here?
OrganizationArtifactReport(c.module.organization.value, c.module.name.value, Vector(rep))

3
modules/lm-coursier/src/main/scala/lmcoursier/internal/UpdateParams.scala
View File

@ -19,7 +19,8 @@ final case class UpdateParams(
includeSignatures: Boolean,
sbtBootJarOverrides: Map[(Module, String), File],
classpathOrder: Boolean,
missingOk: Boolean
missingOk: Boolean,
classLoaders: Seq[ClassLoader]
) {
def artifactFileOpt(

3
modules/lm-coursier/src/main/scala/lmcoursier/internal/UpdateRun.scala
View File

@ -83,7 +83,8 @@ object UpdateRun {
includeSignatures = params.includeSignatures,
classpathOrder = params.classpathOrder,
missingOk = params.missingOk,
params.forceVersions
params.forceVersions,
params.classLoaders,
)
}

82
modules/lm-coursier/src/test/scala/lmcoursier/ResolutionSpec.scala
View File

@ -118,6 +118,88 @@ final class ResolutionSpec extends AnyPropSpec with Matchers {
resolution should be('right)
}
property("resolve with resolvers using a custom protocols") {
val sbtModule = "org.scala-sbt" % "sbt" % "1.1.0"
val dependencies = Vector(sbtModule)
val protocolHandlerDependencies = Vector(
"org.example" %% "customprotocol-handler" % "0.1.0"
)
val resolvers = Vector(
"custom" at "customprotocol://host"
)
val configuration =
CoursierConfiguration()
.withResolvers(resolvers)
val protocolHandlerConfiguration =
Some(
CoursierConfiguration()
.withProtocolHandlerDependencies(protocolHandlerDependencies)
.withResolvers(Resolver.combineDefaultResolvers(Vector.empty))
)
val lmEngine =
CoursierDependencyResolution(
configuration = configuration,
protocolHandlerConfiguration = protocolHandlerConfiguration
)
val coursierModule = module(lmEngine, stubModule, dependencies, Some("2.12.13"))
val resolution =
lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log)
val report = resolution.right.get
val modules = report.configurations.flatMap(_.modules)
modules.map(_.module).map(module => (module.organization, module.name, module.revision)) should contain(
(sbtModule.organization, sbtModule.name, sbtModule.revision)
)
}
property("resolve with resolvers using a custom protocols written in java") {
val sbtModule = "org.scala-sbt" % "sbt" % "1.1.0"
val dependencies = Vector(sbtModule)
val protocolHandlerDependencies = Vector(
"org.example" % "customprotocoljava-handler" % "0.1.0"
)
val resolvers = Vector(
"custom" at "customprotocoljava://host"
)
val configuration =
CoursierConfiguration()
.withResolvers(resolvers)
val protocolHandlerConfiguration =
Some(
CoursierConfiguration()
.withProtocolHandlerDependencies(protocolHandlerDependencies)
.withResolvers(Resolver.combineDefaultResolvers(Vector.empty))
)
val lmEngine =
CoursierDependencyResolution(
configuration = configuration,
protocolHandlerConfiguration = protocolHandlerConfiguration
)
val coursierModule = module(lmEngine, stubModule, dependencies, Some("2.12.13"))
val resolution =
lmEngine.update(coursierModule, UpdateConfiguration(), UnresolvedWarningConfiguration(), log)
val report = resolution.right.get
val modules = report.configurations.flatMap(_.modules)
modules.map(_.module).map(module => (module.organization, module.name, module.revision)) should contain(
(sbtModule.organization, sbtModule.name, sbtModule.revision)
)
}
property("resolve plugin") {
val pluginAttributes = Map("scalaVersion" -> "2.12", "sbtVersion" -> "1.0")
val dependencies =

3
modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/ResolutionTasks.scala
View File

@ -134,7 +134,8 @@ object ResolutionTasks {
Authentication(a.user, a.password)
.withOptional(a.optional)
.withRealmOpt(a.realmOpt)
}
},
Seq(),
)
}

3
modules/sbt-coursier/src/main/scala/coursier/sbtcoursier/UpdateTasks.scala
View File

@ -133,7 +133,8 @@ object UpdateTasks {
includeSignatures,
sbtBootJarOverrides,
classpathOrder = true,
missingOk = sbtClassifiers
missingOk = sbtClassifiers,
classLoaders = Nil,
)
val rep = UpdateRun.update(params, verbosityLevel, log)

1
modules/sbt-coursier/src/sbt-test/sbt-coursier/s3/build.sbt
View File

@ -30,6 +30,7 @@ check := {
),
log = s.log,
authentication = None,
classLoaders = Seq()
)
}

2
modules/sbt-lm-coursier/src/main/scala/coursier/sbtlmcoursier/LmCoursierPlugin.scala
View File

@ -208,7 +208,7 @@ object LmCoursierPlugin extends AutoPlugin {
}
private def mkDependencyResolution: Def.Initialize[Task[DependencyResolution]] =
Def.task {
CoursierDependencyResolution(coursierConfiguration.value)
CoursierDependencyResolution(coursierConfiguration.value, None)
}
}

4
project/Settings.scala
View File

@ -89,4 +89,8 @@ object Settings {
Seq(f)
}
lazy val dontPublish = Seq(
publish := {}
)
}