From 641af2af91958bd0e571d4060349e24fdac7d174 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 5 Apr 2016 16:24:39 +0200 Subject: [PATCH 1/7] Add simple-http-server project --- build.sbt | 14 +- .../scala/coursier/SimpleHttpServer.scala | 178 ++++++++++++++++++ 2 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 simple-web-server/src/main/scala/coursier/SimpleHttpServer.scala diff --git a/build.sbt b/build.sbt index ddc9c2d9d..431c04701 100644 --- a/build.sbt +++ b/build.sbt @@ -488,8 +488,20 @@ lazy val plugin = project scriptedBufferLog := false ) +lazy val `simple-web-server` = project + .settings(commonSettings) + .settings(packAutoSettings) + .settings( + libraryDependencies ++= Seq( + "org.http4s" %% "http4s-blaze-server" % "0.13.2", + "org.http4s" %% "http4s-dsl" % "0.13.2", + "org.slf4j" % "slf4j-nop" % "1.7.19", + "com.github.alexarchambault" %% "case-app" % "1.0.0-RC2" + ) + ) + lazy val `coursier` = project.in(file(".")) - .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, plugin, web, doc) + .aggregate(coreJvm, coreJs, `fetch-js`, testsJvm, testsJs, cache, bootstrap, cli, plugin, web, doc, `simple-web-server`) .settings(commonSettings) .settings(noPublishSettings) .settings(releaseSettings) diff --git a/simple-web-server/src/main/scala/coursier/SimpleHttpServer.scala b/simple-web-server/src/main/scala/coursier/SimpleHttpServer.scala new file mode 100644 index 000000000..3bd6b8fdb --- /dev/null +++ b/simple-web-server/src/main/scala/coursier/SimpleHttpServer.scala @@ -0,0 +1,178 @@ +package coursier + +import java.io.{ File, FileOutputStream } +import java.nio.channels.{ FileLock, OverlappingFileLockException } + +import org.http4s.dsl._ +import org.http4s.headers.Authorization +import org.http4s.server.blaze.BlazeBuilder +import org.http4s.{ BasicCredentials, Challenge, HttpService, Request, Response } + +import caseapp._ + +import scalaz.concurrent.Task + +case class SimpleHttpServerApp( + @ExtraName("d") + @ValueDescription("served directory") + directory: String, + @ExtraName("h") + @ValueDescription("host") + host: String = "0.0.0.0", + @ExtraName("p") + @ValueDescription("port") + port: Int = 8080, + @ExtraName("P") + acceptPost: Boolean, + @ExtraName("t") + acceptPut: Boolean, + @ExtraName("w") + @HelpMessage("Accept write requests. Equivalent to -P -t") + acceptWrite: Boolean, + @ExtraName("v") + verbose: Int @@ Counter, + @ExtraName("u") + @ValueDescription("user") + user: String, + @ExtraName("P") + @ValueDescription("password") + password: String, + @ExtraName("r") + @ValueDescription("realm") + realm: String +) extends App { + + val baseDir = new File(if (directory.isEmpty) "." else directory) + + val verbosityLevel = Tag.unwrap(verbose) + + def write(path: Seq[String], req: Request): Boolean = { + + val f = new File(baseDir, path.toList.mkString("/")) + f.getParentFile.mkdirs() + + var os: FileOutputStream = null + var lock: FileLock = null + try { + os = new FileOutputStream(f) + lock = + try os.getChannel.tryLock() + catch { + case _: OverlappingFileLockException => + null + } + + if (lock == null) + false + else { + req.body.runLog.run.foreach { b => + b.copyToStream(os) + } + + true + } + } finally { + if (lock != null) + lock.release() + if (os != null) + os.close() + } + } + + if (user.nonEmpty && password.isEmpty) + Console.err.println( + "Warning: authentication enabled but no password specified. " + + "Specify one with the --password or -P option." + ) + + if (password.nonEmpty && user.isEmpty) + Console.err.println( + "Warning: authentication enabled but no user specified. " + + "Specify one with the --user or -u option." + ) + + if ((user.nonEmpty || password.nonEmpty) && realm.isEmpty) + Console.err.println( + "Warning: authentication enabled but no realm specified. " + + "Specify one with the --realm or -r option." + ) + + val unauthorized = Unauthorized(Challenge("Basic", realm)) + + def authenticated(pf: PartialFunction[Request, Task[Response]]): HttpService = + authenticated0(HttpService(pf)) + + def authenticated0(service: HttpService): HttpService = + if (user.isEmpty && password.isEmpty) + service + else + HttpService { + case req => + req.headers.get(Authorization) match { + case None => + unauthorized + case Some(auth) => + auth.credentials match { + case basic: BasicCredentials => + if (basic.username == user && basic.password == password) + service.run(req) + else + unauthorized + } + } + } + + def putService = authenticated { + case req @ PUT -> path => + if (verbosityLevel >= 1) + Console.err.println(s"PUT $path") + + if (write(path.toList, req)) + Ok() + else + Locked() + } + + def postService = authenticated { + case req @ POST -> path => + if (verbosityLevel >= 1) + Console.err.println(s"POST $path") + + if (write(path.toList, req)) + Ok() + else + Locked() + } + + def getService = authenticated { + case GET -> path => + if (verbosityLevel >= 1) + Console.err.println(s"GET $path") + + val f = new File(baseDir, path.toList.mkString("/")) + if (f.exists()) + Ok(f) + else + NotFound() + } + + val builder = { + var b = BlazeBuilder.bindHttp(port, host) + + if (acceptWrite || acceptPut) + b = b.mountService(putService) + if (acceptWrite || acceptPost) + b = b.mountService(postService) + + b = b.mountService(getService) + + b + } + + builder + .run + .awaitShutdown() + +} + +object SimpleHttpServer extends AppOf[SimpleHttpServerApp] From 05c248df8229069cedad6fc901a2c5bdc4db2a42 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 27 Apr 2016 01:12:14 +0200 Subject: [PATCH 2/7] Safety check --- .../coursier/core/ResolutionProcess.scala | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala index d88284b20..1df1dd1ac 100644 --- a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala @@ -65,35 +65,44 @@ final case class Missing( def next(results: Fetch.MD): ResolutionProcess = { - val errors = results - .collect{case (modVer, -\/(errs)) => modVer -> errs } - val successes = results - .collect{case (modVer, \/-(repoProj)) => modVer -> repoProj } - - val depMgmtMissing0 = successes - .map{case (_, (_, proj)) => current.dependencyManagementMissing(proj) } - .fold(Set.empty)(_ ++ _) - - val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) - - def cont0(res: Resolution) = { - val res0 = - successes.foldLeft(res){case (acc, (modVer, (source, proj))) => - acc.copyWithCache(projectCache = acc.projectCache + ( - modVer -> (source, acc.withDependencyManagement(proj)) - )) - } - - Continue(res0, cont) + val errors = results.collect { + case (modVer, -\/(errs)) => + modVer -> errs + } + val successes = results.collect { + case (modVer, \/-(repoProj)) => + modVer -> repoProj } - val current0 = current - .copyWithCache(errorCache = current.errorCache ++ errors) + def cont0(res: Resolution): ResolutionProcess = { - if (depMgmtMissing.isEmpty) - cont0(current0) - else - Missing(depMgmtMissing.toSeq, current0, cont0) + val depMgmtMissing0 = successes.map { + case (_, (_, proj)) => + res.dependencyManagementMissing(proj) + }.fold(Set.empty)(_ ++ _) + + val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) + + if (depMgmtMissing.isEmpty) { + val res0 = successes.foldLeft(res) { + case (acc, (modVer, (source, proj))) => + acc.copyWithCache( + projectCache = acc.projectCache + ( + modVer -> (source, acc.withDependencyManagement(proj)) + ) + ) + } + + Continue(res0, cont) + } else + Missing(depMgmtMissing.toSeq, res, cont0) + } + + val current0 = current.copyWithCache( + errorCache = current.errorCache ++ errors + ) + + cont0(current0) } } From 9072dacb0eec371a0c462321238139479961bce3 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 27 Apr 2016 01:12:33 +0200 Subject: [PATCH 3/7] Fix message --- plugin/src/main/scala-2.10/coursier/Tasks.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/src/main/scala-2.10/coursier/Tasks.scala b/plugin/src/main/scala-2.10/coursier/Tasks.scala index d1a464985..4b6809ed2 100644 --- a/plugin/src/main/scala-2.10/coursier/Tasks.scala +++ b/plugin/src/main/scala-2.10/coursier/Tasks.scala @@ -399,7 +399,7 @@ object Tasks { } if (verbosityLevel >= 0) - log.info(s"Resolving ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") + log.info(s"Updating ${currentProject.module.organization}:${currentProject.module.name}:${currentProject.version}") if (verbosityLevel >= 2) for (depRepr <- depsRepr(currentProject.dependencies)) log.info(s" $depRepr") From 5b1165b79e6cd0b4257764b41cdec82a0359f00f Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 28 Apr 2016 21:19:37 +0200 Subject: [PATCH 4/7] Add notes about maven model --- .../main/scala/coursier/core/Resolution.scala | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/Resolution.scala b/core/shared/src/main/scala/coursier/core/Resolution.scala index 171baee4f..eaca558d5 100644 --- a/core/shared/src/main/scala/coursier/core/Resolution.scala +++ b/core/shared/src/main/scala/coursier/core/Resolution.scala @@ -372,9 +372,7 @@ object Resolution { project: Project ): Seq[Dependency] = { - // Here, we're substituting properties also in dependencies that - // come from parents or dependency management. This may not be - // the right thing to do. + // section numbers in the comments refer to withDependencyManagement val properties = propertiesMap(projectProperties(project)) @@ -386,10 +384,9 @@ object Resolution { val keepOpt = mavenScopes.get(config) withExclusions( + // 2.1 & 2.2 depsWithDependencyManagement( - // Important: properties have to be applied to both, - // so that dep mgmt can be matched properly - // Tested with org.ow2.asm:asm-commons:5.0.2 in CentralTests + // 1.7 withProperties(project.dependencies, properties), withProperties(project.dependencyManagement, properties) ), @@ -779,6 +776,36 @@ final case class Resolution( */ def withDependencyManagement(project: Project): Project = { + /* + + Loosely following what [Maven says](http://maven.apache.org/components/ref/3.3.9/maven-model-builder/): + (thanks to @MasseGuillaume for pointing that doc out) + + phase 1 + 1.1 profile activation: see available activators. Notice that model interpolation hasn't happened yet, then interpolation for file-based activation is limited to ${basedir} (since Maven 3), System properties and request properties + 1.2 raw model validation: ModelValidator (javadoc), with its DefaultModelValidator implementation (source) + 1.3 model normalization - merge duplicates: ModelNormalizer (javadoc), with its DefaultModelNormalizer implementation (source) + 1.4 profile injection: ProfileInjector (javadoc), with its DefaultProfileInjector implementation (source) + 1.5 parent resolution until super-pom + 1.6 inheritance assembly: InheritanceAssembler (javadoc), with its DefaultInheritanceAssembler implementation (source). Notice that project.url, project.scm.connection, project.scm.developerConnection, project.scm.url and project.distributionManagement.site.url have a special treatment: if not overridden in child, the default value is parent's one with child artifact id appended + 1.7 model interpolation (see below) + N/A url normalization: UrlNormalizer (javadoc), with its DefaultUrlNormalizer implementation (source) + phase 2, with optional plugin processing + N/A model path translation: ModelPathTranslator (javadoc), with its DefaultModelPathTranslator implementation (source) + N/A plugin management injection: PluginManagementInjector (javadoc), with its DefaultPluginManagementInjector implementation (source) + N/A (optional) lifecycle bindings injection: LifecycleBindingsInjector (javadoc), with its DefaultLifecycleBindingsInjector implementation (source) + 2.1 dependency management import (for dependencies of type pom in the section) + 2.2 dependency management injection: DependencyManagementInjector (javadoc), with its DefaultDependencyManagementInjector implementation (source) + 2.3 model normalization - inject default values: ModelNormalizer (javadoc), with its DefaultModelNormalizer implementation (source) + N/A (optional) reports configuration: ReportConfigurationExpander (javadoc), with its DefaultReportConfigurationExpander implementation (source) + N/A (optional) reports conversion to decoupled site plugin: ReportingConverter (javadoc), with its DefaultReportingConverter implementation (source) + N/A (optional) plugins configuration: PluginConfigurationExpander (javadoc), with its DefaultPluginConfigurationExpander implementation (source) + 2.4 effective model validation: ModelValidator (javadoc), with its DefaultModelValidator implementation (source) + + N/A: does not apply here (related to plugins, path of project being built, ...) + + */ + // A bit fragile, but seems to work // TODO Add non regression test for the touchy org.glassfish.jersey.core:jersey-client:2.19 // (for the way it uses org.glassfish.hk2:hk2-bom,2.4.0-b25) @@ -789,6 +816,7 @@ final case class Resolution( .map(projectCache(_)._2.properties.toMap) .fold(project.properties)(project.properties ++ _) + // 1.1 (see above) val approxProperties = propertiesMap(approxProperties0) ++ projectProperties(project) val profiles0 = profiles( @@ -797,6 +825,9 @@ final case class Resolution( profileActivation getOrElse defaultProfileActivation ) + // 1.2 made from Pom.scala (TODO look at the very details?) + + // 1.3 & 1.4 (if only vaguely so) val dependencies0 = addDependencies( (project.dependencies +: profiles0.map(_.dependencies)).map(withProperties(_, approxProperties)) ) @@ -808,7 +839,7 @@ final case class Resolution( acc ++ p.properties } - val deps0 = ( + val deps0 = dependencies0 .collect { case ("import", dep) => dep.moduleVersion @@ -817,10 +848,9 @@ final case class Resolution( .collect { case ("import", dep) => dep.moduleVersion } ++ - project.parent - ) + project.parent // belongs to 1.5 & 1.6 - val deps = deps0.filter(projectCache.contains) + val deps = deps0.filter(projectCache.contains) val projs = deps .map(projectCache(_)._2) @@ -843,7 +873,7 @@ final case class Resolution( .filterNot{case (config, dep) => config == "import" && depsSet(dep.moduleVersion) } ++ - project.parent + project.parent // belongs to 1.5 & 1.6 .filter(projectCache.contains) .toSeq .flatMap(projectCache(_)._2.dependencies), @@ -851,7 +881,7 @@ final case class Resolution( .filterNot{case (config, dep) => config == "import" && depsSet(dep.moduleVersion) }, - properties = project.parent + properties = project.parent // belongs to 1.5 & 1.6 .filter(projectCache.contains) .map(projectCache(_)._2.properties) .fold(properties0)(properties0 ++ _) From bf919d4fc1fa7e96e2ba84e3b57d3142a7b78608 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 28 Apr 2016 21:19:41 +0200 Subject: [PATCH 5/7] Nasty bug Occurring randomly :-| - hard to unit test... --- .../coursier/core/ResolutionProcess.scala | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala index 1df1dd1ac..6e802b7ac 100644 --- a/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala +++ b/core/shared/src/main/scala/coursier/core/ResolutionProcess.scala @@ -77,14 +77,38 @@ final case class Missing( def cont0(res: Resolution): ResolutionProcess = { val depMgmtMissing0 = successes.map { - case (_, (_, proj)) => - res.dependencyManagementMissing(proj) - }.fold(Set.empty)(_ ++ _) + case elem @ (_, (_, proj)) => + elem -> res.dependencyManagementMissing(proj) + } - val depMgmtMissing = depMgmtMissing0 -- results.map(_._1) + val depMgmtMissing = depMgmtMissing0.map(_._2).fold(Set.empty)(_ ++ _) -- results.map(_._1) if (depMgmtMissing.isEmpty) { - val res0 = successes.foldLeft(res) { + + type Elem = ((Module, String), (Artifact.Source, Project)) + val modVer = depMgmtMissing0.map(_._1._1).toSet + + @tailrec + def order(map: Map[Elem, Set[(Module, String)]], acc: List[Elem]): List[Elem] = + if (map.isEmpty) + acc.reverse + else { + val min = map.map(_._2.size).min // should be 0 + val (toAdd, remaining) = map.partition { + case (k, v) => v.size == min + } + val acc0 = toAdd.keys.foldLeft(acc)(_.::(_)) + val remainingKeys = remaining.keySet.map(_._1) + val map0 = remaining.map { + case (k, v) => + k -> v.intersect(remainingKeys) + } + order(map0, acc0) + } + + val orderedSuccesses = order(depMgmtMissing0.map { case (k, v) => k -> v.intersect(modVer) }.toMap, Nil) + + val res0 = orderedSuccesses.foldLeft(res) { case (acc, (modVer, (source, proj))) => acc.copyWithCache( projectCache = acc.projectCache + ( From 869d57c2f4a581f02e997da597575dd50149ccad Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 28 Apr 2016 23:17:07 +0200 Subject: [PATCH 6/7] Tidying --- .../main/scala-2.10/coursier/MakeIvyXml.scala | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala index 8a75ae8d9..4c1bc7c74 100644 --- a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala +++ b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala @@ -6,27 +6,26 @@ object MakeIvyXml { def apply(project: Project): Node = { - val baseInfoAttrs = .attributes - - val infoAttrs = project.module.attributes.foldLeft(baseInfoAttrs) { + val infoAttrs = project.module.attributes.foldLeft[xml.MetaData](xml.Null) { case (acc, (k, v)) => new PrefixedAttribute("e", k, v, acc) } val licenseElems = project.info.licenses.map { case (name, urlOpt) => - var n = - for (url <- urlOpt) - n = n % .attributes - n + val n = + + urlOpt.fold(n) { url => + n % .attributes + } } val infoElem = { - + {licenseElems} {project.info.description} @@ -34,10 +33,11 @@ object MakeIvyXml { val confElems = project.configurations.toVector.map { case (name, extends0) => - var n = + val n = if (extends0.nonEmpty) - n = n % .attributes - n + n % .attributes + else + n } val publications = project @@ -47,10 +47,12 @@ object MakeIvyXml { val publicationElems = publications.map { case (pub, configs) => - var n = + val n = + if (pub.classifier.nonEmpty) - n = n % .attributes - n + n % .attributes + else + n } val dependencyElems = project.dependencies.toVector.map { From f24a71f2fc8a9820e2fb0f072459af680f900e37 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 28 Apr 2016 23:17:11 +0200 Subject: [PATCH 7/7] Add missing Ivy attributes in dependencies of published artifacts Fixes https://github.com/alexarchambault/coursier/issues/237 --- plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala index 4c1bc7c74..45ef60556 100644 --- a/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala +++ b/plugin/src/main/scala-2.10/coursier/MakeIvyXml.scala @@ -62,9 +62,16 @@ object MakeIvyXml { } - ${dep.configuration}"}> + val n = ${dep.configuration}"}> {excludes} + + val moduleAttrs = dep.module.attributes.foldLeft[xml.MetaData](xml.Null) { + case (acc, (k, v)) => + new PrefixedAttribute("e", k, v, acc) + } + + n % moduleAttrs }