From 5e65beb1455b91c96b7b24ed1e0bfb3531667ec9 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 24 Sep 2014 14:37:26 -0400 Subject: [PATCH 1/4] Fix URL resolver so that we can push Maven artifacts to local maven repositories. * Hook "LocalIfFile" repository to also handle publishing * Add test to ensure mavne publishign works. --- ivy/src/main/scala/sbt/ConvertResolver.scala | 35 +++++++++++++++++-- .../publish-to-maven-local-file/build.sbt | 27 ++++++++++++++ .../publish-to-maven-local-file/test | 2 ++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt create mode 100644 sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test diff --git a/ivy/src/main/scala/sbt/ConvertResolver.scala b/ivy/src/main/scala/sbt/ConvertResolver.scala index 70a0530e1..35cfb669f 100644 --- a/ivy/src/main/scala/sbt/ConvertResolver.scala +++ b/ivy/src/main/scala/sbt/ConvertResolver.scala @@ -9,12 +9,13 @@ import org.apache.ivy.core.module.id.ModuleRevisionId import org.apache.ivy.core.module.descriptor.DependencyDescriptor import org.apache.ivy.core.resolve.ResolveData import org.apache.ivy.core.settings.IvySettings +import org.apache.ivy.plugins.repository.{ RepositoryCopyProgressListener, TransferEvent } import org.apache.ivy.plugins.resolver.{ BasicResolver, DependencyResolver, IBiblioResolver, RepositoryResolver } import org.apache.ivy.plugins.resolver.{ AbstractPatternsBasedResolver, AbstractSshBasedResolver, FileSystemResolver, SFTPResolver, SshResolver, URLResolver } import org.apache.ivy.plugins.repository.url.{ URLRepository => URLRepo } import org.apache.ivy.plugins.repository.file.{ FileRepository => FileRepo, FileResource } -import java.io.File -import org.apache.ivy.util.ChecksumHelper +import java.io.{ IOException, File } +import org.apache.ivy.util.{ FileUtil, ChecksumHelper } import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact } private[sbt] object ConvertResolver { @@ -216,6 +217,7 @@ private[sbt] object ConvertResolver { */ private[this] final class LocalIfFileRepo extends URLRepo { private[this] val repo = new WarnOnOverwriteFileRepo() + private[this] val progress = new RepositoryCopyProgressListener(this); override def getResource(source: String) = { val url = new URL(source) if (url.getProtocol == IO.FileScheme) @@ -223,6 +225,35 @@ private[sbt] object ConvertResolver { else super.getResource(source) } + + override def put(source: File, destination: String, overwrite: Boolean): Unit = { + val url = new URL(destination) + if (url.getProtocol != IO.FileScheme) super.put(source, destination, overwrite) + else { + // Here we duplicate the put method for files so we don't just bail on trying ot use Http handler + val resource = getResource(destination) + if (!overwrite && resource.exists()) { + throw new IOException("destination file exists and overwrite == false"); + } + fireTransferInitiated(resource, TransferEvent.REQUEST_PUT); + try { + var totalLength = source.length + if (totalLength > 0) { + progress.setTotalLength(totalLength); + } + FileUtil.copy(source, new java.io.File(url.toURI), progress) + } catch { + case ex: IOException => + fireTransferError(ex) + throw ex + case ex: RuntimeException => + fireTransferError(ex) + throw ex + } finally { + progress.setTotalLength(null); + } + } + } } private[this] final class WarnOnOverwriteFileRepo extends FileRepo() { diff --git a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt new file mode 100644 index 000000000..d2fda41f7 --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt @@ -0,0 +1,27 @@ + + +lazy val localRemote = + MavenRepository("remote-repo", "file:///tmp/remote-repo") + +lazy val common = + project + .settings( + name := "published-maven", + organization := "com.example", + version := "1.0.0-SNAPSHOT", + publishTo := Some(localRemote) + ) + +lazy val analyze = + project + .dependsOn(common) + .settings( + name := "bad-dependency", + organization := "com.example", + version := "1.0.0-SNAPSHOT", + resolvers += localRemote, + fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project") + ) + + + diff --git a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test new file mode 100644 index 000000000..152da2cfd --- /dev/null +++ b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test @@ -0,0 +1,2 @@ +> common/publishM2 +> analyze/update From d865251686057b8643831e0082e75d4f9c40b4bb Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 24 Sep 2014 14:41:32 -0400 Subject: [PATCH 2/4] Add workaround (and warning message) for when ChainResolver gets a null publication date while we investigate other issues. * Attempt to set publication date to last modified time, if the stars align * Issue warning about undefined resolution behavior otherwise * Add scripted test which exercises the NPE issue in resolving -SNAPSHOTs. * Commit scalariform style edit in Act.scala * After parsing and transforming the pom, check for pub date. * If we don't have a pub date, try to grab lastModified from the URL * If we can't do anything, issue a warning about the problem artifact. --- ivy/src/main/scala/sbt/CustomPomParser.scala | 3 ++- .../scala/sbt/ivyint/SbtChainResolver.scala | 24 +++++++++++++++++-- main/src/main/scala/sbt/Act.scala | 4 ++-- .../publish-to-maven-local-file/build.sbt | 11 +++++---- .../publish-to-maven-local-file/test | 3 ++- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/ivy/src/main/scala/sbt/CustomPomParser.scala b/ivy/src/main/scala/sbt/CustomPomParser.scala index 7bc878b2b..202216700 100644 --- a/ivy/src/main/scala/sbt/CustomPomParser.scala +++ b/ivy/src/main/scala/sbt/CustomPomParser.scala @@ -54,7 +54,8 @@ object CustomPomParser { lazy val registerDefault: Unit = ModuleDescriptorParserRegistry.getInstance.addParser(default) def defaultTransform(parser: ModuleDescriptorParser, md: ModuleDescriptor): ModuleDescriptor = - if (transformedByThisVersion(md)) md else defaultTransformImpl(parser, md) + if (transformedByThisVersion(md)) md + else defaultTransformImpl(parser, md) private[this] def transformedByThisVersion(md: ModuleDescriptor): Boolean = { diff --git a/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala b/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala index 685431124..9a2b483aa 100644 --- a/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala +++ b/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala @@ -7,7 +7,7 @@ import java.util.Date import org.apache.ivy.core.settings.IvySettings import org.apache.ivy.core.{ IvyContext, LogOptions } -import org.apache.ivy.core.module.descriptor.{ ModuleDescriptor, DependencyDescriptor, Artifact => IArtifact } +import org.apache.ivy.core.module.descriptor.{ Artifact => IArtifact, DefaultModuleDescriptor, DefaultDependencyDescriptor, ModuleDescriptor, DependencyDescriptor } import org.apache.ivy.core.resolve.{ ResolvedModuleRevision, ResolveData } import org.apache.ivy.plugins.latest.LatestStrategy import org.apache.ivy.plugins.repository.file.{ FileRepository => IFileRepository, FileResource } @@ -94,6 +94,23 @@ class SbtChainResolver(name: String, resolvers: Seq[DependencyResolver], setting } else temp map { x => (forcedRevision(x), resolver) } ) + retval match { + case Right(Some((rmr, _))) => + rmr.getDescriptor.getPublicationDate match { + case null => + val ivf = resolver.findIvyFileRef(dd, data) + val lmd = new java.util.Date(ivf.getLastModified) + rmr.getDescriptor match { + case dmd: DefaultModuleDescriptor => + Message.info(s"Getting null publication date from resolver: ${resolver} for ${rmr.getId}, setting to: ${lmd}") + dmd.setPublicationDate(lmd) + case _ => + Message.warn(s"Getting null publication date from resolver: ${resolver} for ${rmr.getId}, resolution order is undefined!") + } + case _ => // All other cases ok + } + case _ => + } retval } catch { case ex: Exception => @@ -111,7 +128,10 @@ class SbtChainResolver(name: String, resolvers: Seq[DependencyResolver], setting val sorted = if (useLatest) (foundRevisions.sortBy { case (rmr, _) => - rmr.getDescriptor.getPublicationDate.getTime + rmr.getDescriptor.getPublicationDate match { + case null => 0L + case d => d.getTime + } }).reverse.headOption map { case (rmr, resolver) => // Now that we know the real latest revision, let's force Ivy to use it diff --git a/main/src/main/scala/sbt/Act.scala b/main/src/main/scala/sbt/Act.scala index 272f7e074..f0aa68dd4 100644 --- a/main/src/main/scala/sbt/Act.scala +++ b/main/src/main/scala/sbt/Act.scala @@ -163,8 +163,8 @@ object Act { } def resolveTask(task: ParsedAxis[AttributeKey[_]]): Option[AttributeKey[_]] = task match { - case ParsedGlobal | Omitted => None - case t: ParsedValue[AttributeKey[_]]@unchecked => Some(t.value) + case ParsedGlobal | Omitted => None + case t: ParsedValue[AttributeKey[_]] @unchecked => Some(t.value) } def filterStrings(base: Parser[String], valid: Set[String], label: String): Parser[String] = diff --git a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt index d2fda41f7..1b7c85bf5 100644 --- a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt +++ b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt @@ -6,10 +6,12 @@ lazy val localRemote = lazy val common = project .settings( - name := "published-maven", - organization := "com.example", - version := "1.0.0-SNAPSHOT", - publishTo := Some(localRemote) + name := "config", + organization := "com.typesafe", + version := "0.4.9-SNAPSHOT", + publishTo := Some(localRemote), + autoScalaLibrary := false, + crossPaths := false ) lazy val analyze = @@ -20,6 +22,7 @@ lazy val analyze = organization := "com.example", version := "1.0.0-SNAPSHOT", resolvers += localRemote, + resolvers += Resolver.sonatypeRepo("snapshots"), fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project") ) diff --git a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test index 152da2cfd..46f62f6f7 100644 --- a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test +++ b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/test @@ -1,2 +1,3 @@ > common/publishM2 -> analyze/update +> common/publish +> analyze/update \ No newline at end of file From b09be9bf83801c59dbde9eebdba663287a14826b Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Wed, 24 Sep 2014 18:41:40 -0400 Subject: [PATCH 3/4] Adding notes for ChainResolver fixes. Fixes #1611 --- notes/0.13.7/maven-null-publish-date.md | 15 +++++++++++++++ .../publish-to-maven-local-file/build.sbt | 1 + 2 files changed, 16 insertions(+) create mode 100644 notes/0.13.7/maven-null-publish-date.md diff --git a/notes/0.13.7/maven-null-publish-date.md b/notes/0.13.7/maven-null-publish-date.md new file mode 100644 index 000000000..b06d213eb --- /dev/null +++ b/notes/0.13.7/maven-null-publish-date.md @@ -0,0 +1,15 @@ + [1618]: https://github.com/sbt/sbt/pull/1618 + [1611]: Https://github.com/sbt/sbt/issues/1611 + [@jsuereth]: https://github.com/jsuereth + + +### Improvements + +* You can now publish to maven repositories that are `file` URLs. + +### Fixes + +* When resolving from maven, and unable to read maven-metadata.xml file (common given the divergence in + Maven 3 and Ivy 2), we attempt to use LastModified timestamp in lieu of "published" timestamp. + [#1618][1618] by [@jsuereth][@jsuereth] +* NPE exception when using ChainResolver and maven repositories [#1611]/[1611] by [@jsuereth][@jsuereth] diff --git a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt index 1b7c85bf5..06c06a9dc 100644 --- a/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt +++ b/sbt/src/sbt-test/dependency-management/publish-to-maven-local-file/build.sbt @@ -22,6 +22,7 @@ lazy val analyze = organization := "com.example", version := "1.0.0-SNAPSHOT", resolvers += localRemote, + resolvers += Resolver.mavenLocal, resolvers += Resolver.sonatypeRepo("snapshots"), fullResolvers := fullResolvers.value.filterNot(_.name == "inter-project") ) From ec5dfee517ec1532e81acb4720fce9b9be7e8ad4 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 25 Sep 2014 08:08:26 -0400 Subject: [PATCH 4/4] Fix NPE in ChainrResolver when there is no ivy file for a dependency. --- ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala b/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala index 9a2b483aa..f7dad987d 100644 --- a/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala +++ b/ivy/src/main/scala/sbt/ivyint/SbtChainResolver.scala @@ -98,10 +98,13 @@ class SbtChainResolver(name: String, resolvers: Seq[DependencyResolver], setting case Right(Some((rmr, _))) => rmr.getDescriptor.getPublicationDate match { case null => - val ivf = resolver.findIvyFileRef(dd, data) - val lmd = new java.util.Date(ivf.getLastModified) - rmr.getDescriptor match { - case dmd: DefaultModuleDescriptor => + (resolver.findIvyFileRef(dd, data), rmr.getDescriptor) match { + case (null, _) => + // In this instance, the dependency is specified by a direct URL or some other sort of "non-ivy" file + if (dd.isChanging) + Message.warn(s"Resolving a changing dependency (${rmr.getId}) with no ivy/pom file!, resolution order is undefined!") + case (ivf, dmd: DefaultModuleDescriptor) => + val lmd = new java.util.Date(ivf.getLastModified) Message.info(s"Getting null publication date from resolver: ${resolver} for ${rmr.getId}, setting to: ${lmd}") dmd.setPublicationDate(lmd) case _ =>