diff --git a/docs/getting-started.md b/docs/getting-started.md index a5fb83501..6f1ca8b39 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -156,17 +156,18 @@ of Java versions. Please open an issue if your build tool is not listed in the table below. Feel free to subscribe to the tracking issues to receive updates on your build tool. -| Build tool | Support | Tracking issue | -| ---------- | ------- | -------------------------------------------------------------------------------- | -| Gradle | ✅ | | -| Maven | ✅ | | -| Bazel | ❌ | [sourcegraph/lsif-java#88](https://github.com/sourcegraph/lsif-java/issues/88) | -| Buck | ❌ | [sourcegraph/lsif-java#99](https://github.com/sourcegraph/lsif-java/issues/99) | -| sbt | ❌ | [sourcegraph/lsif-java#110](https://github.com/sourcegraph/lsif-java/issues/110) | +| Build tool | Single repo navigation | Cross-repo navigation | Tracking issue | +| -------------- | ---------------------- | --------------------- | -------------------------------------------------------------------------------- | +| Maven | ✅ | ✅ | | +| Gradle v4.0+ | ✅ | ✅ | | +| Gradle v2.2.1+ | ✅ | ❌ | [sourcegraph/lsif-java#167](https://github.com/sourcegraph/lsif-java/issues/167) | +| Bazel | ❌ | ❌ | [sourcegraph/lsif-java#88](https://github.com/sourcegraph/lsif-java/issues/88) | +| Buck | ❌ | ❌ | [sourcegraph/lsif-java#99](https://github.com/sourcegraph/lsif-java/issues/99) | +| sbt | ❌ | ❌ | [sourcegraph/lsif-java#110](https://github.com/sourcegraph/lsif-java/issues/110) | **✅**: automatic indexing is fully supported. Please report a bug if the `lsif-java index` command does not work on your codebase. -**❌**: automatic inference is not supported but (!) you may still be able to -use `lsif-java` by configuring it manually using the instructions +**❌**: automatic indexing is not supported but (!) you may still be able to use +`lsif-java` by configuring it manually using the instructions [here](manual-configuration.md). diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala index 3cf9a1a54..0d63539c0 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/Embedded.scala @@ -23,11 +23,20 @@ object Embedded { private def javacErrorpath(tmp: Path) = tmp.resolve("errorpath.txt") def customJavac(sourceroot: Path, targetroot: Path, tmp: Path): Path = { - val javac = tmp.resolve("javac") + val bin = tmp.resolve("bin") + val javac = bin.resolve("javac") + val java = bin.resolve("java") val pluginpath = Embedded.semanticdbJar(tmp) val errorpath = javacErrorpath(tmp) val javacopts = targetroot.resolve("javacopts.txt") Files.createDirectories(targetroot) + Files.createDirectories(bin) + Files.write( + java, + """#!/usr/bin/env bash + |java "$@" + |""".stripMargin.getBytes(StandardCharsets.UTF_8) + ) val newJavacopts = tmp.resolve("javac_newarguments") val injectSemanticdbArguments = List[String]( "java", @@ -60,6 +69,7 @@ object Embedded { |""".stripMargin Files.write(javac, script.getBytes(StandardCharsets.UTF_8)) javac.toFile.setExecutable(true) + java.toFile.setExecutable(true) javac } diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala index 869d9becc..d8a944a25 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleBuildTool.scala @@ -37,31 +37,49 @@ class GradleBuildTool(index: IndexCommand) extends BuildTool("Gradle", index) { else "gradle" - TemporaryFiles.withDirectory(index.cleanup) { tmp => + TemporaryFiles.withDirectory(index) { tmp => val toolchains = GradleJavaToolchains .fromWorkspace(this, index, gradleCommand, tmp) - val script = initScript(toolchains, tmp).toString - - val buildCommand = ListBuffer.empty[String] - buildCommand ++= - List( - gradleCommand, - "--init-script", - script, - "-Porg.gradle.java.installations.auto-detect=false", - "-Porg.gradle.java.installations.auto-download=false", - s"-Porg.gradle.java.installations.paths=${toolchains.paths()}", - s"--no-daemon" - ) - buildCommand ++= index.finalBuildCommand(List("clean", "compileTestJava")) - buildCommand += lsifJavaDependencies + toolchains.gradleVersion match { + case Some(gradleVersion) + if gradleVersion.startsWith("6.7") && + toolchains.toolchains.nonEmpty => + index + .app + .error( + "lsif-java does not support Gradle 6.7 when used together with Java toolchains. " + + "To fix this problem, upgrade to Gradle version 6.8 or newer and try again." + ) + CommandResult(1, Nil) + case _ => + runCompileCommand(toolchains) + } + } + } - val result = index.process(buildCommand.toList: _*) - printDebugLogs(tmp) - Embedded - .reportUnexpectedJavacErrors(index.app.reporter, tmp) - .getOrElse(result) + private def runCompileCommand( + toolchains: GradleJavaToolchains + ): CommandResult = { + val script = initScript(toolchains, toolchains.tmp).toString + val buildCommand = ListBuffer.empty[String] + buildCommand += toolchains.gradleCommand + buildCommand += s"--no-daemon" + buildCommand += "--init-script" + buildCommand += script + if (toolchains.toolchains.nonEmpty) { + buildCommand += "-Porg.gradle.java.installations.auto-detect=false" + buildCommand += "-Porg.gradle.java.installations.auto-download=false" + buildCommand += + s"-Porg.gradle.java.installations.paths=${toolchains.paths()}" } + buildCommand ++= index.finalBuildCommand(List("clean", "compileTestJava")) + buildCommand += lsifJavaDependencies + + val result = index.process(buildCommand.toList: _*) + printDebugLogs(toolchains.tmp) + Embedded + .reportUnexpectedJavacErrors(index.app.reporter, toolchains.tmp) + .getOrElse(result) } private def lsifJavaDependencies = "lsifJavaDependencies" diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala index a3e5141a1..5e9fccef5 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/GradleJavaToolchains.scala @@ -13,6 +13,8 @@ case class GradleJavaToolchains( toolchains: List[GradleJavaCompiler], tool: GradleBuildTool, index: IndexCommand, + gradleVersion: Option[String], + gradleCommand: String, tmp: Path ) { @@ -53,9 +55,20 @@ object GradleJavaToolchains { ): GradleJavaToolchains = { val scriptPath = tmp.resolve("java-toolchains.gradle") val toolchainsPath = tmp.resolve("java-toolchains.txt") + val gradleVersionPath = tmp.resolve("gradle-version.txt") val taskName = "lsifDetectJavaToolchains" val script = - s"""|allprojects { + s"""| + |try { + | java.nio.file.Files.write( + | java.nio.file.Paths.get('$gradleVersionPath'), + | [gradle.gradleVersion], + | java.nio.file.StandardOpenOption.TRUNCATE_EXISTING, + | java.nio.file.StandardOpenOption.CREATE) + |} catch (Exception e) { + | // Ignore errors. + |} + |allprojects { | task $taskName { | def out = java.nio.file.Paths.get('$toolchainsPath') | doLast { @@ -92,6 +105,20 @@ object GradleJavaToolchains { Nil } - GradleJavaToolchains(toolchains, tool, index, tmp) + val gradleVersion = + if (Files.isRegularFile(gradleVersionPath)) { + Some(new String(Files.readAllBytes(gradleVersionPath)).trim) + } else { + None + } + + GradleJavaToolchains( + toolchains, + tool, + index, + gradleVersion = gradleVersion, + gradleCommand = gradleCommand, + tmp = tmp + ) } } diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala index 56e6a0eb8..4b0bf204c 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/MavenBuildTool.scala @@ -17,7 +17,7 @@ class MavenBuildTool(index: IndexCommand) extends BuildTool("Maven", index) { Files.isRegularFile(index.workingDirectory.resolve("pom.xml")) override def generateSemanticdb(): CommandResult = { - TemporaryFiles.withDirectory(index.cleanup) { tmp => + TemporaryFiles.withDirectory(index) { tmp => val mvnw = index.workingDirectory.resolve("mvnw") val mavenScript = if (Files.isRegularFile(mvnw)) diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala index 0e994c34c..281e420bd 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/TemporaryFiles.scala @@ -4,15 +4,21 @@ import java.nio.file.Files import java.nio.file.Path import com.sourcegraph.io.DeleteVisitor +import com.sourcegraph.lsif_java.commands.IndexCommand object TemporaryFiles { - def withDirectory[T](cleanup: Boolean)(fn: Path => T): T = { - val tmp = Files.createTempDirectory("lsif-java") - try fn(tmp) - finally { - if (cleanup) { - Files.walkFileTree(tmp, new DeleteVisitor()) - } + def withDirectory[T](index: IndexCommand)(fn: Path => T): T = { + index.temporaryDirectory match { + case Some(tmp) => + fn(tmp) + case None => + val tmp = Files.createTempDirectory("lsif-java") + try fn(tmp) + finally { + if (index.cleanup) { + Files.walkFileTree(tmp, new DeleteVisitor()) + } + } } } } diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala index 6fb077de1..1a84d16d6 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/commands/IndexCommand.scala @@ -49,6 +49,8 @@ case class IndexCommand( @Description("URL to a PackageHub instance") @Hidden // Hidden because it's not supposed to be used yet by normal users. packagehub: Option[String] = None, + @Hidden // Hidden because it's only used for testing purposes + temporaryDirectory: Option[Path] = None, @Description( "Optional. The build command to use to compile all sources. " + "Defaults to a build-specific command. For example, the default command for Maven command is 'clean verify -DskipTests'." + diff --git a/semanticdb-agent/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbAgent.java b/semanticdb-agent/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbAgent.java index bc2e74c25..ccf94cc84 100644 --- a/semanticdb-agent/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbAgent.java +++ b/semanticdb-agent/src/main/java/com/sourcegraph/semanticdb_javac/SemanticdbAgent.java @@ -1,22 +1,21 @@ package com.sourcegraph.semanticdb_javac; -import java.nio.charset.StandardCharsets; -import net.bytebuddy.agent.builder.AgentBuilder; -import net.bytebuddy.asm.Advice; +import static net.bytebuddy.implementation.bytecode.assign.Assigner.Typing.DYNAMIC; +import static net.bytebuddy.matcher.ElementMatchers.named; import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.lang.instrument.Instrumentation; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import java.util.NoSuchElementException; - -import static net.bytebuddy.implementation.bytecode.assign.Assigner.Typing.DYNAMIC; -import static net.bytebuddy.matcher.ElementMatchers.named; +import net.bytebuddy.agent.builder.AgentBuilder; +import net.bytebuddy.asm.Advice; /** Java agent that injects SemanticDB into the Java compilation process. */ public class SemanticdbAgent { diff --git a/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala b/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala index 166fa2812..4867c82fc 100644 --- a/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala +++ b/tests/buildTools/src/test/scala/tests/BaseBuildToolSuite.scala @@ -1,6 +1,7 @@ package tests import java.nio.file.FileSystems +import java.nio.file.Files import java.nio.file.Path import scala.meta.internal.io.FileIO @@ -12,6 +13,7 @@ import moped.testkit.DeleteVisitor import moped.testkit.FileLayout import moped.testkit.MopedSuite import munit.TestOptions +import os.Shellable abstract class BaseBuildToolSuite extends MopedSuite(LsifJava.app) { override def environmentVariables: Map[String, String] = sys.env @@ -36,23 +38,40 @@ abstract class BaseBuildToolSuite extends MopedSuite(LsifJava.app) { expectedSemanticdbFiles: Int = 0, extraArguments: List[String] = Nil, expectedError: String = "", - expectedPackages: String = "" + expectedPackages: String = "", + initCommand: => List[String] = Nil ): Unit = { test(options) { + if (initCommand.nonEmpty) { + os.proc(Shellable(initCommand)).call(os.Path(workingDirectory)) + } FileLayout.fromString(original, root = workingDirectory) val targetroot = workingDirectory.resolve("targetroot") val arguments = - List("index", "--targetroot", targetroot.toString) ++ extraArguments + List[String]( + "index", + "--temporary-directory", + Files.createDirectories(cacheDirectory).toString, + "--targetroot", + targetroot.toString + ) ++ extraArguments val exit = app().run(arguments) + if (extraArguments.contains("--verbose")) { + println(app.capturedOutput) + } if (expectedError.nonEmpty) { assert(clue(exit) != 0, clues(app.capturedOutput)) assertNoDiff(app.capturedOutput, expectedError) } else { assertEquals(exit, 0, clues(app.capturedOutput)) } - val semanticdbFiles = FileIO - .listAllFilesRecursively(AbsolutePath(targetroot)) - .filter(p => semanticdbPattern.matches(p.toNIO)) + val semanticdbFiles = + if (!Files.isDirectory(targetroot)) + Nil + else + FileIO + .listAllFilesRecursively(AbsolutePath(targetroot)) + .filter(p => semanticdbPattern.matches(p.toNIO)) if (semanticdbFiles.length != expectedSemanticdbFiles) { fail( s"Expected $expectedSemanticdbFiles SemanticDB file(s) to be generated.", diff --git a/tests/buildTools/src/test/scala/tests/GradleBuildToolSuite.scala b/tests/buildTools/src/test/scala/tests/GradleBuildToolSuite.scala index 9955d80a7..d6d8c9fe0 100644 --- a/tests/buildTools/src/test/scala/tests/GradleBuildToolSuite.scala +++ b/tests/buildTools/src/test/scala/tests/GradleBuildToolSuite.scala @@ -1,32 +1,95 @@ package tests -import munit.IgnoreSuite +import java.nio.file.Files +import java.nio.file.StandardOpenOption -// FIXME: disabled because of https://github.com/sourcegraph/lsif-java/issues/164 -@IgnoreSuite class GradleBuildToolSuite extends BaseBuildToolSuite { + def gradleVersion(version: String = ""): List[String] = { + createEmptyBuildScript() + if (version.isEmpty || version == "latest") + List("gradle", "wrapper") + else + List("gradle", "wrapper", "--gradle-version", version) + } + + def createEmptyBuildScript(): Unit = { + val script = workingDirectory.resolve("build.gradle") + Files.createDirectories(script.getParent) + Files.write( + script, + Array.emptyByteArray, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ) + } + + List("latest" -> "implementation", "4.0" -> "compile").foreach { + case (version, config) => + checkBuild( + s"basic-$version", + s"""|/build.gradle + |apply plugin: 'java' + |repositories { + | mavenCentral() + |} + |dependencies { + | $config 'junit:junit:4.13.1' + |} + |/src/main/java/Example.java + |import org.junit.Assert; + |public class Example {} + |/src/test/java/ExampleSuite.java + |public class ExampleSuite {} + |""".stripMargin, + expectedSemanticdbFiles = 2, + expectedPackages = + """|maven:junit:junit:4.13.1 + |maven:org.hamcrest:hamcrest-core:1.3 + |""".stripMargin, + initCommand = { + gradleVersion(version) + } + ) + } + + List("3.3", "2.2.1").foreach { version => + checkBuild( + s"legacy-$version", + s"""|/build.gradle + |apply plugin: 'java' + |/src/main/java/Example.java + |public class Example {} + |/src/test/java/ExampleSuite.java + |public class ExampleSuite {} + |""".stripMargin, + expectedSemanticdbFiles = 2, + initCommand = gradleVersion(version) + // NOTE(olafur): no packages because we use more modern APIs. + ) + } + checkBuild( - "basic", + "toolchains-latest", """|/build.gradle |apply plugin: 'java' + |java { + | toolchain { + | languageVersion = JavaLanguageVersion.of(8) + | } + |} |repositories { mavenCentral() } |dependencies { implementation "junit:junit:4.13.1" } |/src/main/java/Example.java - |import org.junit.Assert; |public class Example {} |/src/test/java/ExampleSuite.java |public class ExampleSuite {} |""".stripMargin, - expectedSemanticdbFiles = 2, - expectedPackages = - """|maven:junit:junit:4.11 - |maven:org.hamcrest:hamcrest-core:1.3 - |""".stripMargin + 2 ) checkBuild( - "toolchains", + "toolchains-6.7", """|/build.gradle |apply plugin: 'java' |java { @@ -34,14 +97,30 @@ class GradleBuildToolSuite extends BaseBuildToolSuite { | languageVersion = JavaLanguageVersion.of(8) | } |} - |repositories { mavenCentral() } - |dependencies { implementation "junit:junit:4.13.1" } |/src/main/java/Example.java |public class Example {} - |/src/test/java/ExampleSuite.java - |public class ExampleSuite {} |""".stripMargin, - 2 + expectedError = + """|info: /workingDirectory/gradlew --init-script /cache/java-toolchains.gradle lsifDetectJavaToolchains + |error: lsif-java does not support Gradle 6.7 when used together with Java toolchains. To fix this problem, upgrade to Gradle version 6.8 or newer and try again. + |""".stripMargin, + initCommand = gradleVersion("6.7") + ) + + checkBuild( + "toolchains-6.8", + """|/build.gradle + |apply plugin: 'java' + |java { + | toolchain { + | languageVersion = JavaLanguageVersion.of(8) + | } + |} + |/src/main/java/Example.java + |public class Example {} + |""".stripMargin, + expectedSemanticdbFiles = 1, + initCommand = gradleVersion("6.8") ) checkBuild(