From f6264a0dc00803eff35271d1c49eb4ab6fe7a755 Mon Sep 17 00:00:00 2001 From: Alon Albert Date: Wed, 8 May 2024 16:53:44 -0700 Subject: [PATCH] Add a Test Framework For Parsers Also include a failing test for https://github.com/romainguy/kotlin-explorer/issues/45 Will fix it in the next cl. --- .gitignore | 3 +- build.gradle.kts | 1 + gradle/libs.versions.toml | 1 + local.properties | 1 + .../romainguy/kotlin/explorer/Disassembly.kt | 49 +------ .../explorer/build/ByteCodeDecompiler.kt | 39 ++++++ .../kotlin/explorer/build/KolinCompiler.kt | 41 ++++++ .../explorer/bytecode/ByteCodeParser.kt | 2 +- .../explorer/bytecode/ByteCodeParserTest.kt | 38 ++++++ .../kotlin/explorer/testing/Build.kt | 121 ++++++++++++++++++ src/jvmTest/kotlin/testData/Issue_45.kt | 98 ++++++++++++++ 11 files changed, 349 insertions(+), 45 deletions(-) create mode 100644 local.properties create mode 100644 src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/ByteCodeDecompiler.kt create mode 100644 src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/KolinCompiler.kt create mode 100644 src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParserTest.kt create mode 100644 src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/testing/Build.kt create mode 100644 src/jvmTest/kotlin/testData/Issue_45.kt diff --git a/.gitignore b/.gitignore index 3a1237ad..8ee74e4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ .gradle build/ !gradle/wrapper/gradle-wrapper.jar -!**/src/main/**/build/ -!**/src/test/**/build/ +!**/src/**/build/ ### IntelliJ IDEA ### .idea/modules.xml diff --git a/build.gradle.kts b/build.gradle.kts index db7680ae..94512242 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -47,6 +47,7 @@ kotlin { val jvmTest by getting { dependencies { implementation(libs.junit4) + implementation(libs.kotlin.test) implementation(libs.truth) } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dcf7d3aa..8a3f1d7a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,6 +19,7 @@ jewel = { group = "org.jetbrains.jewel", name = "jewel-int-ui-standalone-241", v jewel-decorated = { group = "org.jetbrains.jewel", name = "jewel-int-ui-decorated-window-241", version.ref = "jewel" } jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna" } junit4 = { group = "junit", name = "junit", version.ref = "junit4" } +kotlin-test = { group = "org.jetbrains.kotlin", name = "kotlin-test", version.ref = "kotlin" } lifecycle = { group = "org.jetbrains.androidx.lifecycle", name = "lifecycle-runtime", version.ref = "lifecycle" } lifecycle-compose = { group = "org.jetbrains.androidx.lifecycle", name = "lifecycle-runtime-compose", version.ref = "lifecycle" } lifecycle-viewmodel = { group = "org.jetbrains.androidx.lifecycle", name = "lifecycle-viewmodel", version.ref = "lifecycle" } diff --git a/local.properties b/local.properties new file mode 100644 index 00000000..b6842565 --- /dev/null +++ b/local.properties @@ -0,0 +1 @@ +kotlin.home=/opt/kotlinc diff --git a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/Disassembly.kt b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/Disassembly.kt index b64bc92b..3c2e46c7 100644 --- a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/Disassembly.kt +++ b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/Disassembly.kt @@ -16,6 +16,8 @@ package dev.romainguy.kotlin.explorer +import dev.romainguy.kotlin.explorer.build.ByteCodeDecompiler +import dev.romainguy.kotlin.explorer.build.KotlinCompiler import dev.romainguy.kotlin.explorer.bytecode.ByteCodeParser import dev.romainguy.kotlin.explorer.code.CodeContent import dev.romainguy.kotlin.explorer.dex.DexDumpParser @@ -33,6 +35,7 @@ private const val TotalDisassemblySteps = 6 private const val TotalRunSteps = 2 private const val Done = 1f +private val byteCodeDecompiler = ByteCodeDecompiler() private val byteCodeParser = ByteCodeParser() private val dexDumpParser = DexDumpParser() private val oatDumpParser = OatDumpParser() @@ -56,10 +59,7 @@ suspend fun buildAndRun( val path = directory.resolve("KotlinExplorer.kt") Files.writeString(path, source) - val kotlinc = process( - *buildKotlincCommand(toolPaths, path), - directory = directory - ) + val kotlinc = KotlinCompiler(toolPaths, directory).compile(path) if (kotlinc.exitCode != 0) { launch(ui) { @@ -114,10 +114,7 @@ suspend fun buildAndDisassemble( val path = directory.resolve("KotlinExplorer.kt") Files.writeString(path, source) - val kotlinc = process( - *buildKotlincCommand(toolPaths, path), - directory = directory - ) + val kotlinc = KotlinCompiler(toolPaths, directory).compile(path) if (kotlinc.exitCode != 0) { launch(ui) { @@ -129,11 +126,7 @@ suspend fun buildAndDisassemble( launch(ui) { onStatusUpdate("Disassembling ByteCode…", step++ / TotalDisassemblySteps) } - val javap = process( - *buildJavapCommand(directory), - directory = directory - ) - + val javap = byteCodeDecompiler.decompile(directory) launch { onByteCode(byteCodeParser.parse(javap.output)) } if (javap.exitCode != 0) { @@ -249,18 +242,6 @@ private fun buildJavaCommand(toolPaths: ToolPaths): Array { return command.toTypedArray() } -private fun buildJavapCommand(directory: Path): Array { - val command = mutableListOf("javap", "-p", "-l", "-c") - val classFiles = Files - .list(directory) - .filter { path -> path.extension == "class" } - .map { file -> file.fileName.toString() } - .sorted() - .collect(Collectors.toList()) - command.addAll(classFiles) - return command.toTypedArray() -} - private fun buildR8Command( toolPaths: ToolPaths, directory: Path, @@ -318,22 +299,6 @@ private fun buildR8Command( return command.toTypedArray() } -private fun buildKotlincCommand(toolPaths: ToolPaths, path: Path): Array { - val command = mutableListOf( - toolPaths.kotlinc.toString(), - path.toString(), - "-Xmulti-platform", - "-Xno-param-assertions", - "-Xno-call-assertions", - "-Xno-receiver-assertions", - "-classpath", - toolPaths.kotlinLibs.joinToString(":") { jar -> jar.toString() } - + ":${toolPaths.platform}" - ) - - return command.toTypedArray() -} - private fun writeR8Rules(directory: Path) { // Match $ANDROID_HOME/tools/proguard/proguard-android-optimize.txt Files.writeString( @@ -343,7 +308,7 @@ private fun writeR8Rules(directory: Path) { -allowaccessmodification -dontpreverify -dontobfuscate - -keep,allowoptimization class !kotlin.**,!kotlinx.** { + -keep,allow optimization class !kotlin.**,!kotlinx.** { ; }""".trimIndent() ) diff --git a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/ByteCodeDecompiler.kt b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/ByteCodeDecompiler.kt new file mode 100644 index 00000000..0ec677ce --- /dev/null +++ b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/ByteCodeDecompiler.kt @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2024 Romain Guy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.romainguy.kotlin.explorer.build + +import dev.romainguy.kotlin.explorer.process +import java.nio.file.Path +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.extension +import kotlin.io.path.pathString +import kotlin.io.path.walk + +class ByteCodeDecompiler { + suspend fun decompile(directory: Path) = process(*buildJavapCommand(directory), directory = directory) + + @OptIn(ExperimentalPathApi::class) + private fun buildJavapCommand(directory: Path): Array { + val command = mutableListOf("javap", "-p", "-l", "-c") + val classFiles = directory.walk() + .filter { path -> path.extension == "class" } + .map { path -> directory.relativize(path).pathString } + .sorted() + command.addAll(classFiles) + return command.toTypedArray() + } +} diff --git a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/KolinCompiler.kt b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/KolinCompiler.kt new file mode 100644 index 00000000..d51694d8 --- /dev/null +++ b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/build/KolinCompiler.kt @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 Romain Guy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.romainguy.kotlin.explorer.build + +import dev.romainguy.kotlin.explorer.ToolPaths +import dev.romainguy.kotlin.explorer.process +import java.nio.file.Path + +class KotlinCompiler(private val toolPaths: ToolPaths, private val outputDirectory: Path) { + + suspend fun compile(source: Path) = process(*buildCompileCommand(source), directory = outputDirectory) + + private fun buildCompileCommand(file: Path): Array { + val command = mutableListOf( + toolPaths.kotlinc.toString(), + file.toString(), + "-Xmulti-platform", + "-Xno-param-assertions", + "-Xno-call-assertions", + "-Xno-receiver-assertions", + "-classpath", + (toolPaths.kotlinLibs + toolPaths.platform).joinToString(":") { jar -> jar.toString() } + ) + + return command.toTypedArray() + } +} diff --git a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParser.kt b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParser.kt index 66bb875b..1c8f7ed5 100644 --- a/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParser.kt +++ b/src/jvmMain/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParser.kt @@ -29,7 +29,7 @@ import dev.romainguy.kotlin.explorer.getValue * public final class KotlinExplorerKt { * ``` */ -private val ClassRegex = Regex("^.* class [_a-zA-Z][_\\w]+ \\{$") +private val ClassRegex = Regex("^.* class [_a-zA-Z][_\\w.]+ \\{$") /** * Examples: diff --git a/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParserTest.kt b/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParserTest.kt new file mode 100644 index 00000000..4e152866 --- /dev/null +++ b/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/bytecode/ByteCodeParserTest.kt @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Romain Guy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.romainguy.kotlin.explorer.bytecode + +import dev.romainguy.kotlin.explorer.testing.Build +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class ByteCodeParserTest { + @get:Rule + val temporaryFolder = TemporaryFolder() + + private val build by lazy { Build.getInstance(temporaryFolder.root.toPath()) } + private val byteCodeParser = ByteCodeParser() + + @Test + fun issue_45() { + val byteCode = build?.generateByteCode("Issue_45.kt") ?: return + val content = byteCodeParser.parse(byteCode) + println(content) + + } +} diff --git a/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/testing/Build.kt b/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/testing/Build.kt new file mode 100644 index 00000000..30359501 --- /dev/null +++ b/src/jvmTest/kotlin/dev/romainguy/kotlin/explorer/testing/Build.kt @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2024 Romain Guy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.romainguy.kotlin.explorer.testing + +import dev.romainguy.kotlin.explorer.ToolPaths +import dev.romainguy.kotlin.explorer.build.ByteCodeDecompiler +import dev.romainguy.kotlin.explorer.build.KotlinCompiler +import kotlinx.coroutines.runBlocking +import java.nio.file.Path +import java.util.* +import kotlin.io.path.createDirectory +import kotlin.io.path.notExists +import kotlin.io.path.reader +import kotlin.io.path.writeText +import kotlin.test.fail + +class Build private constructor(private val outputDirectory: Path) { + private val cwd = Path.of(System.getProperty("user.dir")) + private val testData = cwd.resolve("src/jvmTest/kotlin/testData") + private val toolPaths = createToolsPath() + private val kotlinCompiler = KotlinCompiler(toolPaths, outputDirectory) + private val byteCodeDecompiler = ByteCodeDecompiler() + + fun generateByteCode(testFile: String): String { + val path = testData.resolve(testFile) + if (path.notExists()) { + fail("$path does not exists") + } + return runBlocking { + kotlinCompile(path) + val result = byteCodeDecompiler.decompile(outputDirectory) + if (result.exitCode != 0) { + System.err.println(result.output) + fail("javap error") + } + + // Save the fine under so we can examine it when needed + val saveFile = cwd.resolve("build/testData/${testFile.replace(".kt", ".javap")}") + if (saveFile.parent.notExists()) { + saveFile.parent.createDirectory() + } + saveFile.writeText(result.output) + + result.output + } + } + + private suspend fun kotlinCompile(path: Path) { + val result = kotlinCompiler.compile(path) + if (result.exitCode != 0) { + System.err.println(result.output) + fail("kotlinc error") + } + } + + private fun createToolsPath(): ToolPaths { + val properties = Properties() + properties.load(cwd.resolve("local.properties").reader()) + + val kotlinHome = getKotlinHome(properties) + val androidHome = getAndroidHome(properties) + val toolPaths = ToolPaths(Path.of(""), androidHome, kotlinHome) + if (toolPaths.isKotlinHomeValid && toolPaths.isAndroidHomeValid) { + return toolPaths + } + fail("Invalid ToolsPath: KOTLIN_HOME=${kotlinHome} ANDROID_HOME=$androidHome") + } + + companion object { + fun getInstance(outputDirectory: Path): Build? { + return try { + Build(outputDirectory) + } catch (e: Throwable) { + System.err.println("Failed to setup the builder.") + e.printStackTrace(System.err) + null + } + } + } +} + +private fun getKotlinHome(properties: Properties): Path { + val pathString = System.getenv("KOTLIN_HOME") ?: properties.getProperty("kotlin.home") + + if (pathString == null) { + fail("Could not find Android SDK") + } + val path = Path.of(pathString) + if (path.notExists()) { + fail("Could not find Android SDK") + } + return path +} + +private fun getAndroidHome(properties: Properties): Path { + val path = + when (val androidHome: String? = System.getenv("ANDROID_HOME") ?: properties.getProperty("android.home")) { + null -> Path.of(System.getProperty("user.home")).resolve("Android/Sdk") + else -> Path.of(androidHome) + } + + if (path.notExists()) { + throw IllegalStateException("Could not find Android SDK") + } + return path +} + diff --git a/src/jvmTest/kotlin/testData/Issue_45.kt b/src/jvmTest/kotlin/testData/Issue_45.kt new file mode 100644 index 00000000..47767345 --- /dev/null +++ b/src/jvmTest/kotlin/testData/Issue_45.kt @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2024 Romain Guy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testData + +inline fun Int.uncheckedCoerceIn(minimumValue: Int, maximumValue: Int) = + this.coerceAtLeast(minimumValue).coerceAtMost(maximumValue) + +inline fun Rect(l: Int, t: Int, r: Int, b: Int) = + Rect(( + ((l.uncheckedCoerceIn(0, 8) and 0xf) shl 12) or + ((t.uncheckedCoerceIn(0, 8) and 0xf) shl 8) or + ((r.uncheckedCoerceIn(0, 8) and 0xf) shl 4) or + ((b.uncheckedCoerceIn(0, 8) and 0xf) ) + ).toShort()) + +@JvmInline +value class Rect @PublishedApi internal constructor(@PublishedApi internal val points: Short) { + inline val l: Int get() = points.toInt() ushr 12 + inline val t: Int get() = (points.toInt() shr 8) and 0xf + inline val r: Int get() = (points.toInt() shr 4) and 0xf + inline val b: Int get() = points.toInt() and 0xf + + override fun toString() = "Rect($l, $t, $r, $b)" +} + +inline fun Grid() = Grid(0L) +inline fun Grid(r: Rect) = Grid(rasterize(r.l, r.t, r.r, r.b)) +inline fun Grid(l: Int, t: Int, r: Int, b: Int) = Grid(rasterize(l, t, r, b)) + +@JvmInline +value class Grid @PublishedApi internal constructor(@PublishedApi internal val cells: Long) { + inline fun forEach(block: (Int, Int) -> Unit) { + var v = cells + while (v.hasNext()) { + val index = v.get() + block(index and 0x7, index ushr 3) + v = v.next() + } + } + + inline operator fun get(x: Int, y: Int) = + ((cells ushr ((7 - y) shl 3)) and (0x1L shl (7 - x))) != 0L + + inline operator fun plus(r: Rect) = + Grid(cells or rasterize(r.l, r.t, r.r, r.b)) + + inline operator fun minus(r: Rect) = + Grid(cells and rasterize(r.l, r.t, r.r, r.b).inv()) + + inline infix fun and(r: Rect) = + Grid(cells and rasterize(r.l, r.t, r.r, r.b)) + + inline fun intersects(r: Rect) = + (cells and rasterize(r.l, r.t, r.r, r.b)) != 0L + + override fun toString() = buildString { + for (y in 0..7) { + val line = (cells ushr (56 - y shl 3) and 0xffL).toString(2).padStart(8, '0') + appendLine(line) + } + } +} + +@PublishedApi +internal inline fun Long.get() = 63 - countTrailingZeroBits() + +@PublishedApi +internal inline fun Long.hasNext() = this != 0L +@PublishedApi +internal inline fun Long.next() = this and (this - 1L) + +@PublishedApi +internal fun rasterize(l: Int, t: Int, r: Int, b: Int): Long { + val w = r - l + val h = b - t + val scanline = 0xffL ushr (8 - w) shl (8 - r) + val rows = 0x01_01_01_01_01_01_01_01L ushr ((8 - h) shl 3) shl ((8 - b) shl 3) + return rows * scanline +} + +fun main() { + val grid = Grid(1, 1, 5, 5) + println(grid) +}