From 21e5838cfd3e3fe987a1bd6e7dadfd9201e516ce Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Sun, 26 Jun 2022 13:15:50 -0400 Subject: [PATCH] Build Graal native image using sbt-native-image --- .appveyor.yml | 162 --------------------------------------- .github/workflows/ci.yml | 6 ++ build.sbt | 76 +++--------------- project/plugins.sbt | 1 + 4 files changed, 18 insertions(+), 227 deletions(-) delete mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index a58675259..000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,162 +0,0 @@ -image: - - MacOS - - Visual Studio 2015 - - Visual Studio 2019 - - Ubuntu - -build: off - -init: - - git config --global core.autocrlf input - -for: -- - matrix: - only: - - image: Ubuntu - - branches: - only: - - build-graal - artifacts: - - path: client/target/bin/sbtn - name: sbtn - - install: - - curl -sL https://github.com/sbt/sbt/releases/download/v1.3.10/sbt-1.3.10.tgz > ~/sbt-bin.tgz - - mkdir ~/sbt - - tar -xf ~/sbt-bin.tgz --directory ~/sbt - - curl -sL https://raw.githubusercontent.com/shyiko/jabba/0.11.0/install.sh | bash && . ~/.jabba/jabba.sh - - jabba install adopt@1.8.0-275 - - jabba use adopt@1.8.0-275 - - curl -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.1.0/graalvm-ce-java8-linux-amd64-20.1.0.tar.gz > graalvm.tar.gz - - tar -xf graalvm.tar.gz - - export PATH="~/sbt/sbt/bin:$PATH" - - export PATH="$PATH:~/.jabba/jdk/adopt@1.8.0-275/bin" - - export JAVA_HOME="~/.jabba/jdk/adopt@1.8.0-275" - - test_script: - - export PATH="$PATH:~/.jabba/jdk/adopt@1.8.0-275/bin" - - export PATH="$PATH:graalvm-ce-java8-20.1.0/bin" - - gu install native-image - - sbt "-Dsbt.io.virtual=false" "-Dsbt.native-image=$(pwd)/graalvm-ce-java8-20.1.0/bin/native-image" "sbtClientProj/buildNativeThinClient" - -- - matrix: - only: - - image: MacOS - - branches: - only: - - build-graal - artifacts: - - path: client/target/bin/sbtn - name: mac-native-sbt-client - - install: - - curl -sL https://github.com/sbt/sbt/releases/download/v1.3.10/sbt-1.3.10.tgz > ~/sbt-bin.tgz - - mkdir ~/sbt - - tar -xf ~/sbt-bin.tgz --directory ~/sbt - - curl -sL https://raw.githubusercontent.com/shyiko/jabba/0.11.0/install.sh | bash && . ~/.jabba/jabba.sh - - jabba install adopt@1.8.0-222 - - jabba use adopt@1.8.0-222 - - curl -sL https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.1.0/graalvm-ce-java8-darwin-amd64-20.1.0.tar.gz > graalvm.tar.gz - - tar -xf graalvm.tar.gz - - export PATH="~/sbt/sbt/bin:$PATH" - - export PATH="$PATH:~/.jabba/jdk/adopt@1.8.0-222/bin" - - export JAVA_HOME="~/.jabba/jdk/adopt@1.8.0-222" - - test_script: - - export PATH="$PATH:~/.jabba/jdk/adopt@1.8.0-222/Contents/Home/bin" - - export PATH="$PATH:graalvm-ce-java8-20.1.0/Contents/Home/bin" - - gu install native-image - - sbt "-Dsbt.io.virtual=false" "-Dsbt.native-image=$(pwd)/graalvm-ce-java8-20.1.0/Contents/Home/bin/native-image" "sbtClientProj/buildNativeThinClient" - -- - matrix: - only: - - image: Visual Studio 2015 - branches: - only: - - build-graal - - artifacts: - - path: client\target\bin\sbtn.exe - name: sbtn.exe - install: - - cinst adoptopenjdk8 -params 'installdir=C:\\jdk8' - - SET CI=true - #- choco install windows-sdk-7.1 kb2519277 - - call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" - - - ps: | - Add-Type -AssemblyName System.IO.Compression.FileSystem - if (!(Test-Path -Path "C:\sbt" )) { - (new-object System.Net.WebClient).DownloadFile( - 'https://github.com/sbt/sbt/releases/download/v1.3.10/sbt-1.3.10.zip', - 'C:\sbt-bin.zip' - ) - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\sbt-bin.zip", "C:\sbt") - } - if (!(Test-Path -Path "C:\graalvm-ce-java8-20.2.0-dev" )) { - (new-object System.Net.WebClient).DownloadFile( - 'https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-20.1.0/graalvm-ce-java8-windows-amd64-20.1.0.zip', - 'C:\graalvm-ce-java8-20.1.0.zip' - ) - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\graalvm-ce-java8-20.1.0.zip", "C:\") - } - if (!(Test-Path -Path "C:\zulu-jdk7" )) { - (new-object System.Net.WebClient).DownloadFile( - 'https://cdn.azul.com/zulu/bin/zulu7.38.0.11-ca-jdk7.0.262-win_x64.zip', - 'C:\zulu-jdk7.zip' - ) - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\zulu-jdk7.zip", "C:\") - } - - SET PATH=C:\graalvm-ce-java8-20.1.0\bin;%PATH% - - SET PATH=C:\sbt\sbt\bin;%PATH% - - SET JAVA_HOME=C:\jdk8 - - gu install native-image - - rm .sbtopts - - cache: - - '%USERPROFILE%\.ivy2\cache' - - '%LOCALAPPDATA%\Coursier\Cache\v1' - - '%USERPROFILE%\.sbt' - - test_script: - - sbt "-Dsbt.io.virtual=false" "-Dsbt.native-image=C:\graalvm-ce-java8-20.1.0\bin\native-image.cmd" "sbtClientProj/buildNativeThinClient" -- - matrix: - only: - - image: Visual Studio 2019 - branches: - except: - - build-graal - install: - - cinst adoptopenjdk8 -params 'installdir=C:\\jdk8' - - SET JAVA_HOME=C:\jdk8 - - SET PATH=C:\jdk8\bin;%PATH% - - SET CI=true - - - ps: | - Add-Type -AssemblyName System.IO.Compression.FileSystem - if (!(Test-Path -Path "C:\sbt" )) { - (new-object System.Net.WebClient).DownloadFile( - 'https://github.com/sbt/sbt/releases/download/v1.3.10/sbt-1.3.10.zip', - 'C:\sbt-bin.zip' - ) - [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\sbt-bin.zip", "C:\sbt") - } - - SET PATH=C:\sbt\sbt\bin;%PATH% - - SET SBT_OPTS=-Xmx4g -Dsbt.supershell=never -Dfile.encoding=UTF8 - - rm .sbtopts - - cache: - - '%USERPROFILE%\.ivy2\cache' - - '%LOCALAPPDATA%\Coursier\Cache\v1' - - '%USERPROFILE%\.sbt' - - test_script: - # The server tests often fail in CI when run together so just run a single test to ensure - # that the thin client works on windows - - sbt "-Dsbt.io.virtual=false" "scripted actions/* reporter/source-mapper classloader-cache/* nio/* watch/*" "serverTestProj/testOnly testpkg.ClientTest" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d09fa04f6..f3ab86d21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,8 @@ jobs: if: ${{ matrix.jobtype == 7 }} shell: bash run: | + # test building sbtn on Linux + sbt "-Dsbt.io.virtual=false" nativeImage # test launcher script echo build using JDK 8 test using JDK 8 and JDK 11 cd launcher-package @@ -165,6 +167,8 @@ jobs: if: ${{ matrix.jobtype == 8 }} shell: bash run: | + # test building sbtn on macOS + sbt "-Dsbt.io.virtual=false" nativeImage # test launcher script echo build using JDK 8, test using JDK 8, on macOS cd launcher-package @@ -175,6 +179,8 @@ jobs: if: ${{ matrix.jobtype == 9 }} shell: bash run: | + # test building sbtn on Windows + sbt "-Dsbt.io.virtual=false" nativeImage # test launcher script echo build using JDK 8, test using JDK 8, on Windows cd launcher-package diff --git a/build.sbt b/build.sbt index ea690b5b3..2fc3ae23e 100644 --- a/build.sbt +++ b/build.sbt @@ -211,7 +211,7 @@ lazy val sbtRoot: Project = (project in file(".")) mimaSettings, mimaPreviousArtifacts := Set.empty, buildThinClient := (sbtClientProj / buildThinClient).evaluated, - buildNativeThinClient := (sbtClientProj / buildNativeThinClient).value, + nativeImage := (sbtClientProj / nativeImage).value, installNativeThinClient := { // nativeInstallDirectory can be set globally or in a gitignored local file val dir = nativeInstallDirectory.?.value @@ -226,7 +226,7 @@ lazy val sbtRoot: Project = (project in file(".")) } } val base = baseDirectory.value.toPath - val exec = (sbtClientProj / buildNativeThinClient).value + val exec = (sbtClientProj / nativeImage).value.toPath streams.value.log.info(s"installing thin client ${base.relativize(exec)} to ${target}") Files.copy(exec, target, java.nio.file.StandardCopyOption.REPLACE_EXISTING) } @@ -1143,17 +1143,11 @@ lazy val serverTestProj = (project in file("server-test")) val isWin = scala.util.Properties.isWin val buildThinClient = inputKey[JPath]("generate a java implementation of the thin client") -val thinClientClasspath = - taskKey[Seq[JPath]]("Generate the classpath for thin client (compacted for windows)") -val thinClientNativeImageCommand = taskKey[String]("The native image command") -val thinClientNativeImageOptions = settingKey[Seq[String]]("The native image options") -val thinClientNativeImageClass = settingKey[String]("The class for the native image") -val buildNativeThinClient = taskKey[JPath]("Generate a native executable") // Use a TaskKey rather than SettingKey for nativeInstallDirectory so it can left unset by default val nativeInstallDirectory = taskKey[JPath]("The install directory for the native executable") val installNativeThinClient = inputKey[JPath]("Install the native executable") -val nativeThinClientPath = settingKey[JPath]("The location of the native executable") lazy val sbtClientProj = (project in file("client")) + .enablePlugins(NativeImagePlugin) .dependsOn(commandProj) .settings( commonBaseSettings, @@ -1164,65 +1158,17 @@ lazy val sbtClientProj = (project in file("client")) crossPaths := false, exportJars := true, libraryDependencies += jansi, - libraryDependencies += scalatest % "test", - /* - * On windows, the raw classpath is too large to be a command argument to an - * external process so we create symbolic links with short names to get the - * classpath length under the limit. - */ - thinClientClasspath := { - val original = (Compile / fullClasspathAsJars).value.map(_.data) - val outputDir = target.value / "thinclientcp" - IO.createDirectory(outputDir) - Files.walk(outputDir.toPath).forEach { - case f if f.getFileName.toString.endsWith(".jar") => Files.deleteIfExists(f) - case _ => - } - original.zipWithIndex.map { - case (f, i) => Files.createSymbolicLink(outputDir.toPath / s"$i.jar", f.toPath) - } + libraryDependencies += scalatest % Test, + Compile / mainClass := Some("sbt.client.Client"), + nativeImageReady := { () => + () }, - thinClientNativeImageCommand := System.getProperty("sbt.native-image", "native-image").toString, - buildNativeThinClient / name := s"sbtn${if (isWin) ".exe" else ""}", - nativeThinClientPath := target.value.toPath / "bin" / (buildNativeThinClient / name).value, - thinClientNativeImageClass := "sbt.client.Client", - buildNativeThinClient := { - val hasChanges = thinClientClasspath.outputFileChanges.hasChanges - val cpString = - thinClientClasspath.value.map(_.getFileName).mkString(java.io.File.pathSeparator) - val prefix = Seq(thinClientNativeImageCommand.value, "-cp", cpString) - val full = prefix ++ thinClientNativeImageOptions.value :+ thinClientNativeImageClass.value - val dir = target.value - if (hasChanges || !Files.exists(nativeThinClientPath.value)) { - val pb = new java.lang.ProcessBuilder(full: _*) - pb.directory(dir / "thinclientcp") - val proc = pb.start() - val thread = new Thread { - setDaemon(true) - val is = proc.getInputStream - val es = proc.getErrorStream - - override def run(): Unit = { - Thread.sleep(100) - while (proc.isAlive) { - if (is.available > 0 || es.available > 0) { - while (is.available > 0) System.out.print(is.read.toChar) - while (es.available > 0) System.err.print(es.read.toChar) - } - if (proc.isAlive) Thread.sleep(10) - } - } - } - thread.start() - proc.waitFor(5, java.util.concurrent.TimeUnit.MINUTES) - assert(proc.exitValue == 0, s"Exit value ${proc.exitValue} was nonzero") - } - nativeThinClientPath.value - }, - thinClientNativeImageOptions := Seq( + nativeImage / name := s"sbtn${if (isWin) ".exe" else ""}", + nativeImageOutput := target.value / "bin" / (nativeImage / name).value, + nativeImageOptions ++= Seq( "--no-fallback", s"--initialize-at-run-time=sbt.client", - "--verbose", + // "--verbose", "-H:IncludeResourceBundles=jline.console.completer.CandidateListCompletionHandler", "-H:+ReportExceptionStackTraces", "-H:-ParseRuntimeOptions", diff --git a/project/plugins.sbt b/project/plugins.sbt index 00c24984e..7f677b694 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -11,4 +11,5 @@ addSbtPlugin("com.lightbend" % "sbt-whitesource" % "0.1.14") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.8.1") addSbtPlugin("com.swoval" % "sbt-java-format" % "0.3.1") +addSbtPlugin("org.scalameta" % "sbt-native-image" % "0.3.1") addDependencyTreePlugin