[2.x] feat: Ivyless publish to Ivy repo (#8686)

Implements ivyless publish task as part of #7639 (drop Ivy dependency).
Fixes #8639.

- Add ivylessPublish for URLRepository: HTTP PUT with optional Basic auth,
  same layout as ivyless publishLocal (artifacts + ivy.xml + checksums).
- Add ivylessPublishToFile for FileRepository: write to local path for
  testing without HTTP server.
- Add ivylessPublishTask: when useIvy is false, use ivyless path for
  URLRepository or FileRepository; otherwise use Ivy.
- Wire publish in Defaults to LibraryManagement.ivylessPublishTask
  (tagged Publish, Network).
- Add scripted test dependency-management/ivyless-publish using
  Resolver.file to verify ivyless publish produces identical layout.

Credentials supported via allCredentials (Basic auth for PUT).
This commit is contained in:
bitloi 2026-02-04 14:20:38 -05:00 committed by GitHub
parent b0601b4c6c
commit b460bb871e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 519 additions and 14 deletions

View File

@ -2991,7 +2991,7 @@ object Classpaths {
.map(m => d.withRevision(m.module.revision))
}.distinct
}.value,
publish := publishOrSkip(publishConfiguration, publish / skip).value,
publish := LibraryManagement.ivylessPublishTask.tag(Tags.Publish, Tags.Network).value,
publishLocal := LibraryManagement.ivylessPublishLocalTask.value,
publishM2 := publishOrSkip(publishM2Configuration, publishM2 / skip).value,
credentials ++= Def.uncached {

View File

@ -9,9 +9,12 @@
package sbt
package internal
import java.io.File
import java.io.{ File, IOException }
import java.net.{ URI, URL }
import java.util.concurrent.Callable
import gigahorse.{ AuthScheme }
import gigahorse.support.apachehttp.Gigahorse
import sbt.Def.ScopedKey
import sbt.internal.librarymanagement.*
import sbt.librarymanagement.*
@ -21,7 +24,8 @@ import sbt.io.IO
import sbt.io.syntax.*
import sbt.ProjectExtra.*
import sjsonnew.JsonFormat
import scala.concurrent.duration.FiniteDuration
import scala.concurrent.*
import scala.concurrent.duration.*
import lmcoursier.definitions.Project as CsrProject
private[sbt] object LibraryManagement {
@ -588,6 +592,17 @@ private[sbt] object LibraryManagement {
IO.write(checksumFile, digest.hashHexString)
log.debug(s"Wrote checksum: $checksumFile")
// Write ivy.xml first (so ivys/ exists even if artifact copy fails)
val ivysDir = moduleDir / "ivys"
val ivyXmlFile = ivysDir / "ivy.xml"
IO.createDirectory(ivysDir)
val ivyXmlContent = lmcoursier.IvyXml(project, Nil, Nil)
if !ivyXmlFile.exists || overwrite then
IO.write(ivyXmlFile, ivyXmlContent)
log.info(s"Published $ivyXmlFile")
writeChecksums(ivyXmlFile)
else log.warn(s"$ivyXmlFile already exists, skipping (overwrite=$overwrite)")
// Publish each artifact
artifacts.foreach: (artifact, sourceFile) =>
val folder = typeToFolder(artifact.`type`)
@ -604,19 +619,269 @@ private[sbt] object LibraryManagement {
log.info(s"Published $targetFile")
writeChecksums(targetFile)
else log.warn(s"$targetFile already exists, skipping (overwrite=$overwrite)")
// Generate and write ivy.xml
val ivyXmlContent = lmcoursier.IvyXml(project, Nil, Nil)
val ivysDir = moduleDir / "ivys"
val ivyXmlFile = ivysDir / "ivy.xml"
if !ivyXmlFile.exists || overwrite then
IO.createDirectory(ivysDir)
IO.write(ivyXmlFile, ivyXmlContent)
log.info(s"Published $ivyXmlFile")
writeChecksums(ivyXmlFile)
else log.warn(s"$ivyXmlFile already exists, skipping (overwrite=$overwrite)")
end ivylessPublishLocal
/**
* Substitutes Ivy pattern placeholders for artifact URL.
* Matches ivylessPublishLocal layout: [organisation]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext]
*/
private def substituteIvyArtifactPattern(
pattern: String,
org: String,
moduleName: String,
version: String,
typeFolder: String,
artifactName: String,
classifier: String,
ext: String
): String = {
var s = pattern
s = s.replace("[organisation]", org)
s = s.replace("[module]", moduleName)
s = s.replace("[revision]", version)
s = s.replace("[type]s", typeFolder)
s = s.replace("[artifact]", artifactName)
s = s.replace("[ext]", ext)
if (classifier.nonEmpty) s = s.replace("(-[classifier])", s"-$classifier")
else s = s.replace("(-[classifier])", "")
// Remove optional Ivy pattern parts (scala/sbt version, branch) for ivyless layout
s = s.replaceAll("\\(scala_[^)]+/\\)", "").replaceAll("\\(sbt_[^)]+/\\)", "")
s = s.replaceAll("\\(\\[branch\\]/\\)", "")
s
}
/**
* HTTP PUT a file to a URL with optional Basic auth.
* Uses Gigahorse (Apache HttpClient) per sbt tech stack.
*/
private def httpPut(
url: URL,
sourceFile: File,
credentials: Option[Credentials.DirectCredentials],
log: Logger
): Unit = {
val baseReq = Gigahorse.url(url.toString).put(sourceFile)
val req = credentials.filter(_.host == url.getHost) match {
case Some(dc) => baseReq.withAuth(dc.userName, dc.passwd, AuthScheme.Basic)
case None => baseReq
}
val f = sbt.librarymanagement.Http.http.processFull(req)
val response = Await.result(f, 5.minutes)
if (response.status < 200 || response.status >= 300) {
val body = response.bodyAsString
throw new IOException(
s"PUT $url failed: ${response.status} ${response.statusText}$body"
)
}
log.info(s"Published $url")
}
/**
* Publishes artifacts to a remote Ivy repo (URLRepository) without using Apache Ivy.
* Uses HTTP PUT; supports credentials. Produces the same layout as ivylessPublishLocal.
*/
def ivylessPublish(
project: CsrProject,
artifacts: Vector[(Artifact, File)],
checksumAlgorithms: Vector[String],
urlRepo: sbt.librarymanagement.URLRepository,
credentials: Seq[Credentials],
overwrite: Boolean,
log: Logger
): Unit = {
val org = project.module.organization.value
val moduleName = project.module.name.value
val version = project.version
val artifactPattern = urlRepo.patterns.artifactPatterns.headOption.getOrElse(
sys.error("URLRepository has no artifact pattern")
)
val ivyPattern = urlRepo.patterns.ivyPatterns.headOption.getOrElse(
sys.error("URLRepository has no ivy pattern")
)
val directCreds = credentials.collect { case d: Credentials.DirectCredentials => d }
def typeToFolder(tpe: String): String = tpe match
case "jar" => "jars"
case "src" | "source" | "sources" => "srcs"
case "doc" | "docs" | "javadoc" | "javadocs" => "docs"
case "pom" => "poms"
case "ivy" => "ivys"
case other => other + "s"
def writeChecksums(file: File): Vector[(File, String)] =
checksumAlgorithms.map { algo =>
val digestAlgo = algo.toLowerCase match
case "md5" => sbt.util.Digest.Md5
case "sha1" => sbt.util.Digest.Sha1
case other =>
throw new IllegalArgumentException(s"Unsupported checksum algorithm: $other")
val digest = sbt.util.Digest(digestAlgo, file.toPath)
val content = digest.hashHexString
val suffix = "." + algo.toLowerCase
val tmpFile = File.createTempFile("checksum", suffix)
IO.write(tmpFile, content)
(tmpFile, suffix)
}.toVector
artifacts.foreach { case (artifact, sourceFile) =>
val folder = typeToFolder(artifact.`type`)
val classifier = artifact.classifier.map("-" + _).getOrElse("")
val artifactName = moduleName
val pathPattern = substituteIvyArtifactPattern(
artifactPattern,
org,
moduleName,
version,
folder,
artifactName,
classifier,
artifact.extension
)
val url = URI.create(pathPattern).toURL()
httpPut(url, sourceFile, directCreds.find(_.host == url.getHost), log)
val checksums = writeChecksums(sourceFile)
checksums.foreach { case (cf, suffix) =>
val checksumUrl = URI.create(pathPattern + suffix).toURL()
try httpPut(checksumUrl, cf, directCreds.find(_.host == url.getHost), log)
finally cf.delete()
}
}
val ivyXmlContent = lmcoursier.IvyXml(project, Nil, Nil)
val ivyPathPattern = substituteIvyArtifactPattern(
ivyPattern,
org,
moduleName,
version,
"ivys",
"ivy",
"",
"xml"
)
val ivyUrl = URI.create(ivyPathPattern).toURL()
val ivyTmp = File.createTempFile("ivy", ".xml")
try {
IO.write(ivyTmp, ivyXmlContent)
httpPut(ivyUrl, ivyTmp, directCreds.find(_.host == ivyUrl.getHost), log)
val checksums = writeChecksums(ivyTmp)
checksums.foreach { case (cf, suffix) =>
val checksumUrl = URI.create(ivyPathPattern + suffix).toURL()
try httpPut(checksumUrl, cf, directCreds.find(_.host == ivyUrl.getHost), log)
finally cf.delete()
}
} finally ivyTmp.delete()
}
/**
* Publishes artifacts to a local file repo (FileRepository) without using Apache Ivy.
* Same layout as ivylessPublishLocal; used for testing without an HTTP server.
*/
def ivylessPublishToFile(
project: CsrProject,
artifacts: Vector[(Artifact, File)],
checksumAlgorithms: Vector[String],
fileRepo: sbt.librarymanagement.FileRepository,
overwrite: Boolean,
log: Logger
): Unit = {
val pattern = fileRepo.patterns.artifactPatterns.headOption.getOrElse(
sys.error("FileRepository has no artifact pattern")
)
val baseStr =
if (pattern.contains("[organisation]"))
pattern.substring(0, pattern.indexOf("[organisation]"))
else pattern
val normalized = baseStr.replace('\\', '/').stripSuffix("/")
val localRepoBase =
if (normalized.startsWith("file:")) new File(new java.net.URI(normalized))
else new File(normalized)
val repoDir = localRepoBase.getAbsoluteFile
log.info(s"Ivyless publish to file repo: $repoDir")
ivylessPublishLocal(project, artifacts, checksumAlgorithms, repoDir, overwrite, log)
}
/**
* Task initializer for ivyless publish (remote Ivy repo or file repo).
* When useIvy is false and publishTo is URLRepository or FileRepository, uses ivyless publish; otherwise uses Ivy.
*/
def ivylessPublishTask: Def.Initialize[Task[Unit]] =
import Keys.*
Def.ifS(Def.task { (publish / skip).value })(
Def.task {
val log = streams.value.log
val ref = thisProjectRef.value
log.debug(s"Skipping publish for ${Reference.display(ref)}")
}
)(
Def.ifS(Def.task { useIvy.value })(
Def.task {
val log = streams.value.log
val conf = publishConfiguration.value
val module = ivyModule.value
val publisherInterface = publisher.value
publisherInterface.publish(module, conf, log)
}
)(
Def.task {
val log = streams.value.log
val resolver = sbt.Classpaths.getPublishTo(publishTo.value)
val project = csrProject.value.withPublications(csrPublications.value)
val config = publishConfiguration.value
val artifacts = config.artifacts.map { case (a, f) => (a, f) }
resolver match {
case urlRepo: sbt.librarymanagement.URLRepository =>
val creds = allCredentials.value
ivylessPublish(
project,
artifacts,
config.checksums,
urlRepo,
creds,
config.overwrite,
log
)
case fileRepo: sbt.librarymanagement.FileRepository =>
ivylessPublishToFile(
project,
artifacts,
config.checksums,
fileRepo,
config.overwrite,
log
)
case pbr: sbt.librarymanagement.PatternsBasedRepository
if pbr.patterns.artifactPatterns.headOption.exists { pat =>
pat.contains("[organisation]") && !pat.trim.startsWith("http")
} =>
// File repo detected by pattern (e.g. scripted classloader makes type match fail)
val pat = pbr.patterns.artifactPatterns.head
val baseStr =
pat.substring(0, pat.indexOf("[organisation]")).replace('\\', '/').stripSuffix("/")
val repoDir =
(if (baseStr.startsWith("file:")) new File(new java.net.URI(baseStr))
else new File(baseStr)).getAbsoluteFile
log.info(s"Ivyless publish to file repo: $repoDir")
ivylessPublishLocal(
project,
artifacts,
config.checksums,
repoDir,
config.overwrite,
log
)
case _ =>
log.warn(
"Ivyless publish only supports URLRepository (Resolver.url) or FileRepository (Resolver.file). Falling back to Ivy."
)
val conf = publishConfiguration.value
val module = ivyModule.value
val publisherInterface = publisher.value
publisherInterface.publish(module, conf, log)
}
}
)
)
/**
* Task initializer for ivyless publishLocal.
* Uses Def.ifS for proper selective functor behavior.

View File

@ -0,0 +1,95 @@
ThisBuild / csrCacheDirectory := (ThisBuild / baseDirectory).value / "coursier-cache"
name := "lib1"
organization := "com.example"
version := "0.1.0-SNAPSHOT"
scalaVersion := "3.8.1"
val publishRepoBase = settingKey[File]("Base directory for publish repo (HTTP server writes here)")
publishRepoBase := baseDirectory.value / "repo"
val publishPort = 3030
// Publish to HTTP server (localhost) - ivyless publish uses PUT
// Resolver.url expects java.net.URL; in build.sbt "url" is sbt.URI, so use java.net.URL explicitly
publishTo := Some(
Resolver.url("test-repo", new java.net.URL(s"http://localhost:$publishPort/"))(Resolver.ivyStylePatterns)
.withAllowInsecureProtocol(true)
)
useIvy := false
Compile / packageDoc / publishArtifact := false
Compile / packageSrc / publishArtifact := false
val startPublishServer = taskKey[Unit]("Start HTTP server that accepts PUT to repo directory")
startPublishServer := {
HttpPutServer.start(publishPort, publishRepoBase.value)
streams.value.log.info(s"HTTP PUT server started on port $publishPort, writing to ${publishRepoBase.value}")
}
val stopPublishServer = taskKey[Unit]("Stop HTTP server")
stopPublishServer := {
HttpPutServer.stop()
streams.value.log.info("HTTP PUT server stopped")
}
val publishToHttp = taskKey[Unit]("Publish to HTTP server (start server, publish, stop server)")
publishToHttp := {
startPublishServer.value
try publish.value
finally stopPublishServer.value
}
val checkIvylessPublish = taskKey[Unit]("Check that ivyless publish produced the expected files")
checkIvylessPublish := {
val log = streams.value.log
val base = publishRepoBase.value
val org = organization.value
val moduleName = normalizedName.value + "_3"
val ver = version.value
val moduleDir = base / org / moduleName / ver
log.info(s"Checking published files in $moduleDir")
def listDir(dir: File, indent: String = ""): Unit = {
if (dir.exists) {
dir.listFiles.foreach { f =>
log.info(s"$indent${f.getName}")
if (f.isDirectory) listDir(f, indent + " ")
}
} else {
log.info(s"${indent}Directory does not exist: $dir")
}
}
log.info("Contents of publish repo:")
listDir(base)
val expectedDirs = Seq("jars", "poms", "ivys")
expectedDirs.foreach { dir =>
val d = moduleDir / dir
assert(d.exists && d.isDirectory, s"Expected directory $d to exist")
}
val jarFile = moduleDir / "jars" / s"$moduleName.jar"
assert(jarFile.exists, s"Expected $jarFile to exist")
assert((moduleDir / "jars" / s"$moduleName.jar.md5").exists, s"Expected md5 checksum to exist")
assert((moduleDir / "jars" / s"$moduleName.jar.sha1").exists, s"Expected sha1 checksum to exist")
val ivyFile = moduleDir / "ivys" / "ivy.xml"
assert(ivyFile.exists, s"Expected $ivyFile to exist")
assert((moduleDir / "ivys" / "ivy.xml.md5").exists, s"Expected ivy.xml md5 checksum to exist")
assert((moduleDir / "ivys" / "ivy.xml.sha1").exists, s"Expected ivy.xml sha1 checksum to exist")
val ivyContent = IO.read(ivyFile)
assert(ivyContent.contains(s"""organisation="$org""""), s"ivy.xml should contain organisation")
assert(ivyContent.contains(s"""module="$moduleName""""), s"ivy.xml should contain module name")
assert(ivyContent.contains(s"""revision="$ver""""), s"ivy.xml should contain revision")
log.info("All ivyless publish (HTTP) checks passed!")
}
val cleanPublishRepo = taskKey[Unit]("Clean the publish repo")
cleanPublishRepo := {
IO.delete(publishRepoBase.value)
}

View File

@ -0,0 +1,49 @@
import java.io._
import java.net.InetSocketAddress
import java.nio.file.{ Files, Paths }
import com.sun.net.httpserver.{ HttpExchange, HttpHandler, HttpServer }
/** Minimal HTTP server that accepts PUT and writes to a base directory (for ivyless publish scripted test). */
object HttpPutServer {
private var server: HttpServer = null
def start(port: Int, baseDir: java.io.File): Unit = {
if (server != null) stop()
server = HttpServer.create(new InetSocketAddress(port), 0)
server.createContext("/", new HttpHandler {
override def handle(ex: HttpExchange): Unit = {
val method = ex.getRequestMethod
if ("PUT".equalsIgnoreCase(method)) {
val path = ex.getRequestURI.getRawPath
val relativePath = if (path.startsWith("/")) path.substring(1) else path
val targetFile = new File(baseDir, relativePath.replace("/", File.separator))
targetFile.getParentFile.mkdirs()
val in = ex.getRequestBody
val out = new FileOutputStream(targetFile)
try {
val buf = new Array[Byte](8192)
var n = 0
while ({ n = in.read(buf); n } != -1) out.write(buf, 0, n)
} finally {
out.close()
in.close()
}
ex.sendResponseHeaders(200, -1)
ex.close()
} else {
ex.sendResponseHeaders(405, -1)
ex.close()
}
}
})
server.setExecutor(null)
server.start()
}
def stop(): Unit = {
if (server != null) {
server.stop(0)
server = null
}
}
}

View File

@ -0,0 +1 @@
// empty plugins file

View File

@ -0,0 +1,5 @@
package com.example
object Lib {
def hello: String = "Hello from lib1!"
}

View File

@ -0,0 +1,7 @@
# Test ivyless publish to HTTP server (issue #8639)
# HTTP server accepts PUT to repo directory; publish uses ivyless publisher
# and produces the same layout as ivyless publishLocal
> cleanPublishRepo
> publishToHttp
> checkIvylessPublish

View File

@ -0,0 +1,70 @@
ThisBuild / csrCacheDirectory := (ThisBuild / baseDirectory).value / "coursier-cache"
name := "lib1"
organization := "com.example"
version := "0.1.0-SNAPSHOT"
scalaVersion := "3.8.1"
// Publish to a file repo (tests ivyless publish without HTTP server)
val publishRepoBase = settingKey[File]("Base directory for publish repo")
publishRepoBase := baseDirectory.value / "repo"
publishTo := Some(Resolver.file("test-repo", publishRepoBase.value)(Resolver.ivyStylePatterns))
useIvy := false
Compile / packageDoc / publishArtifact := false
Compile / packageSrc / publishArtifact := false
val checkIvylessPublish = taskKey[Unit]("Check that ivyless publish produced the expected files")
checkIvylessPublish := {
val log = streams.value.log
val base = publishRepoBase.value
val org = organization.value
val moduleName = normalizedName.value + "_3"
val ver = version.value
val moduleDir = base / org / moduleName / ver
log.info(s"Checking published files in $moduleDir")
def listDir(dir: File, indent: String = ""): Unit = {
if (dir.exists) {
dir.listFiles.foreach { f =>
log.info(s"$indent${f.getName}")
if (f.isDirectory) listDir(f, indent + " ")
}
} else {
log.info(s"${indent}Directory does not exist: $dir")
}
}
log.info("Contents of publish repo:")
listDir(base)
val expectedDirs = Seq("jars", "poms", "ivys")
expectedDirs.foreach { dir =>
val d = moduleDir / dir
assert(d.exists && d.isDirectory, s"Expected directory $d to exist")
}
val jarFile = moduleDir / "jars" / s"$moduleName.jar"
assert(jarFile.exists, s"Expected $jarFile to exist")
assert((moduleDir / "jars" / s"$moduleName.jar.md5").exists, s"Expected md5 checksum to exist")
assert((moduleDir / "jars" / s"$moduleName.jar.sha1").exists, s"Expected sha1 checksum to exist")
val ivyFile = moduleDir / "ivys" / "ivy.xml"
assert(ivyFile.exists, s"Expected $ivyFile to exist")
assert((moduleDir / "ivys" / "ivy.xml.md5").exists, s"Expected ivy.xml md5 checksum to exist")
assert((moduleDir / "ivys" / "ivy.xml.sha1").exists, s"Expected ivy.xml sha1 checksum to exist")
val ivyContent = IO.read(ivyFile)
assert(ivyContent.contains(s"""organisation="$org""""), s"ivy.xml should contain organisation")
assert(ivyContent.contains(s"""module="$moduleName""""), s"ivy.xml should contain module name")
assert(ivyContent.contains(s"""revision="$ver""""), s"ivy.xml should contain revision")
log.info("All ivyless publish checks passed!")
}
val cleanPublishRepo = taskKey[Unit]("Clean the publish repo")
cleanPublishRepo := {
IO.delete(publishRepoBase.value)
}

View File

@ -0,0 +1 @@
// empty plugins file

View File

@ -0,0 +1,5 @@
package com.example
object Lib {
def hello: String = "Hello from lib1!"
}

View File

@ -0,0 +1,7 @@
# Test ivyless publish (remote/file repo)
# When useIvy is false and publishTo is Resolver.file, publish uses ivyless publisher
# and produces the same layout as ivyless publishLocal
> cleanPublishRepo
> publish
> checkIvylessPublish