Kex Integration

README.md

@@ -42,6 +42,15 @@ TestSpark currently supports two test generation strategies:
 <li>Generate tests for Java classes, methods, and single lines.</li>
 </ul>
 
+<h4>Symbolic execution-based test generation</h4>
+<p>For this type of test generation, TestSpark uses <a href="https://github.com/vorpal-research/kex">Kex</a>, supporting symbolic execution for Java Byte Code. </p>
+<ul>
+<li>Supports up to Java 8 and upwards.</li>
+<li>Powered by SMT solvers, it supports really high coverages given larger time frames.</li>
+<li>Generated test cases are however not very readable (there are plans to automatically refactor with the help of LLMs).</li>
+<li>Generates tests for Java classes and methods.</li>
+</ul>
+
 <p>Initially implemented by <a href="https://www.ciselab.nl">CISELab</a> at <a href="https://se.ewi.tudelft.nl">SERG @ TU Delft</a>, TestSpark is currently developed and maintained by <a href="https://lp.jetbrains.com/research/ictl/">ICTL at JetBrains Research</a>.</p>
 
 ## <span style="color:crimson; font-size:150%; font-weight:bold"> DISCLAIMER </span>

build.gradle.kts

@@ -172,6 +172,9 @@ dependencies {
     // https://gitlab.com/mvysny/konsume-xml
     implementation("com.gitlab.mvysny.konsume-xml:konsume-xml:1.0")
 
+    // for merging generated kex tests into a single file
+    implementation("com.github.javaparser:javaparser-core:3.26.1")
+
     // From the jetbrains repository
     testImplementation("com.intellij.remoterobot:remote-robot:0.11.13")
     testImplementation("com.intellij.remoterobot:remote-fixtures:0.11.13")
@@ -236,7 +239,7 @@ intellijPlatform {
         }
         freeArgs = listOf(
             "-mute",
-            "TemplateWordInPluginId,ForbiddenPluginIdPrefix"
+            "TemplateWordInPluginId,ForbiddenPluginIdPrefix",
         )
     }
 }

core/src/main/kotlin/org/jetbrains/research/testspark/core/data/TestGenerationData.kt

@@ -16,6 +16,7 @@ data class TestGenerationData(
     var importsCode: MutableSet<String> = mutableSetOf(),
     var packageName: String = "",
     var runWith: String = "",
+    // Modifications to this code in the tool-window editor are forgotten when apply to test suite
     var otherInfo: String = "",
 
     // changing parameters with a large prompt

gradle.properties

@@ -7,6 +7,7 @@ pluginName = TestSpark
 pluginVersion = 0.3.0
 
 evosuiteVersion = 1.0.5
+kexVersion = 0.0.8
 
 # See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 # for insight into build numbers and IntelliJ Platform versions.

java/src/main/kotlin/org/jetbrains/research/testspark/java/JavaPsiMethodWrapper.kt

@@ -37,6 +37,15 @@ class JavaPsiMethodWrapper(private val psiMethod: PsiMethod) : PsiMethodWrapper
     override val signature: String
         get() = buildSignature(psiMethod)
 
+    override val parameterNames: List<String>
+        get() = psiMethod.parameterList.parameters.map { it.name }
+
+    override val parameterTypes: List<String>
+        get() = psiMethod.parameterList.parameters.map { it.type.canonicalText }
+
+    override val returnType: String
+        get() = psiMethod.returnType?.canonicalText ?: "void"
+
     val parameterList = psiMethod.parameterList
 
     val isConstructor: Boolean = psiMethod.isConstructor

kotlin/src/main/kotlin/org/jetbrains/research/testspark/kotlin/KotlinPsiMethodWrapper.kt

@@ -37,6 +37,15 @@ class KotlinPsiMethodWrapper(val psiFunction: KtFunction) : PsiMethodWrapper {
     override val signature: String
         get() = buildSignature(psiFunction)
 
+    override val parameterNames: List<String>
+        get() = psiFunction.valueParameters.map { it.name ?: "" }
+
+    override val parameterTypes: List<String>
+        get() = psiFunction.valueParameters.map { it.typeReference?.text ?: "Any" }
+
+    override val returnType: String
+        get() = psiFunction.typeReference?.text ?: "Unit"
+
     val parameterList = psiFunction.valueParameterList
 
     val isPrimaryConstructor: Boolean = psiFunction is KtPrimaryConstructor

langwrappers/src/main/kotlin/org/jetbrains/research/testspark/langwrappers/PsiComponents.kt

@@ -25,6 +25,9 @@ interface PsiMethodWrapper {
     val name: String
     val methodDescriptor: String
     val signature: String
+    val parameterNames: List<String>
+    val parameterTypes: List<String>
+    val returnType: String
     val text: String?
     val containingClass: PsiClassWrapper?
     val containingFile: PsiFile?

src/main/kotlin/org/jetbrains/research/testspark/actions/TestSparkAction.kt

@@ -28,21 +28,26 @@ import org.jetbrains.research.testspark.settings.evosuite.EvoSuiteSettingsState
 import org.jetbrains.research.testspark.settings.llm.LLMSettingsState
 import org.jetbrains.research.testspark.tools.TestsExecutionResultManager
 import org.jetbrains.research.testspark.tools.evosuite.EvoSuite
+import org.jetbrains.research.testspark.tools.kex.Kex
 import org.jetbrains.research.testspark.tools.llm.Llm
 import org.jetbrains.research.testspark.tools.template.Tool
 import java.awt.BorderLayout
 import java.awt.CardLayout
+import java.awt.Component
 import java.awt.Dimension
 import java.awt.Font
 import java.awt.Toolkit
 import java.awt.event.WindowAdapter
 import java.awt.event.WindowEvent
+import javax.swing.Box
+import javax.swing.BoxLayout
 import javax.swing.ButtonGroup
 import javax.swing.JButton
 import javax.swing.JFrame
 import javax.swing.JLabel
 import javax.swing.JPanel
 import javax.swing.JRadioButton
+import javax.swing.SwingConstants
 
 /**
  * Represents an action to be performed in the TestSpark plugin.
@@ -110,7 +115,9 @@ class TestSparkAction : AnAction() {
 
         private val llmButton = JRadioButton("<html><b>${Llm().name}</b></html>")
         private val evoSuiteButton = JRadioButton("<html><b>${EvoSuite().name}</b></html>")
+        private val kexButton = JRadioButton("<html><b>${Kex().name}</b></html>")
         private val testGeneratorButtonGroup = ButtonGroup()
+        private val kexForLineCodeTypeErrMsg = JLabel() // The error displayed when if kex and line code type are chosen
 
         private val psiHelper: PsiHelper
             get() {
@@ -203,11 +210,13 @@ class TestSparkAction : AnAction() {
 
             testGeneratorButtonGroup.add(llmButton)
             testGeneratorButtonGroup.add(evoSuiteButton)
+            testGeneratorButtonGroup.add(kexButton)
 
             val testGeneratorPanel = JPanel()
             testGeneratorPanel.add(JLabel("Select the test generator:"))
             testGeneratorPanel.add(llmButton)
             testGeneratorPanel.add(evoSuiteButton)
+            testGeneratorPanel.add(kexButton)
 
             for ((codeType, codeTypeName) in codeTypes) {
                 val button = JRadioButton(codeTypeName)
@@ -236,7 +245,13 @@ class TestSparkAction : AnAction() {
                 .panel
 
             val nextButtonPanel = JPanel()
+            nextButtonPanel.layout = BoxLayout(nextButtonPanel, BoxLayout.Y_AXIS)
             nextButton.isEnabled = false
+            nextButton.alignmentX = Component.CENTER_ALIGNMENT
+            kexForLineCodeTypeErrMsg.alignmentX = Component.CENTER_ALIGNMENT
+            kexForLineCodeTypeErrMsg.horizontalAlignment = SwingConstants.CENTER
+            nextButtonPanel.add(kexForLineCodeTypeErrMsg)
+            nextButtonPanel.add(Box.createVerticalStrut(10)) // Add some space between label and button
             nextButtonPanel.add(nextButton)
 
             val cardPanel = JPanel(BorderLayout())
@@ -267,6 +282,10 @@ class TestSparkAction : AnAction() {
                 updateNextButton()
             }
 
+            kexButton.addActionListener {
+                updateNextButton()
+            }
+
             for ((_, button) in codeTypeButtons) {
                 button.addActionListener {
                     llmSetupPanelFactory.setPromptEditorType(button.text)
@@ -286,6 +305,8 @@ class TestSparkAction : AnAction() {
                     cardLayout.next(panel)
                     cardLayout.next(panel)
                     pack()
+                } else if (kexButton.isSelected) {
+                    startKexGeneration()
                 } else if (evoSuiteButton.isSelected && !evoSuiteSettingsState.evosuiteSetupCheckBoxSelected) {
                     startEvoSuiteGeneration()
                 } else {
@@ -387,6 +408,7 @@ class TestSparkAction : AnAction() {
             dispose()
         }
 
+        private fun startKexGeneration() = startUnitTestGenerationTool(tool = Kex())
         private fun startEvoSuiteGeneration() = startUnitTestGenerationTool(tool = EvoSuite())
         private fun startLLMGeneration() = startUnitTestGenerationTool(tool = Llm())
 
@@ -398,12 +420,23 @@ class TestSparkAction : AnAction() {
          * This method should be called whenever the mentioned above buttons are clicked.
          */
         private fun updateNextButton() {
-            val isTestGeneratorButtonGroupSelected = llmButton.isSelected || evoSuiteButton.isSelected
+            val isTestGeneratorButtonGroupSelected = llmButton.isSelected || evoSuiteButton.isSelected || kexButton.isSelected
             val isCodeTypeButtonGroupSelected = codeTypeButtons.any { it.second.isSelected }
-            nextButton.isEnabled = isTestGeneratorButtonGroupSelected && isCodeTypeButtonGroupSelected
+            val kexForCodeLineType =
+                kexButton.isSelected && codeTypeButtons.any { (codeType, button) -> codeType == CodeType.LINE && button.isSelected }
+            if (kexForCodeLineType) {
+                kexForLineCodeTypeErrMsg.text =
+                    "<html><b><font color='orange'>* Kex cannot generate tests for a single line. Please change your selection</font></b></html>"
+            } else {
+                kexForLineCodeTypeErrMsg.text = ""
+            }
+
+            nextButton.isEnabled =
+                isTestGeneratorButtonGroupSelected && isCodeTypeButtonGroupSelected && !kexForCodeLineType
 
             if ((llmButton.isSelected && !llmSettingsState.llmSetupCheckBoxSelected && !llmSettingsState.provideTestSamplesCheckBoxSelected) ||
-                (evoSuiteButton.isSelected && !evoSuiteSettingsState.evosuiteSetupCheckBoxSelected)
+                (evoSuiteButton.isSelected && !evoSuiteSettingsState.evosuiteSetupCheckBoxSelected) ||
+                kexButton.isSelected
             ) {
                 nextButton.text = PluginLabelsBundle.get("ok")
             } else {

src/main/kotlin/org/jetbrains/research/testspark/bundles/evosuite/EvoSuiteDefaultsBundle.kt

@@ -14,4 +14,8 @@ object EvoSuiteDefaultsBundle : DynamicBundle(EvoSuiteBundlePaths.defaults) {
      */
     @Nls
     fun get(@PropertyKey(resourceBundle = EvoSuiteBundlePaths.defaults) key: String): String = getMessage(key)
+        // In Intellij Platform version 2, the DynamicBundle returns the whole path and the value at the end in plugin verification.
+        // Each is separated by "|" (e.g., "|b|properties.llm.LLMDefaults|k|maxLLMRequest|3")
+        // if we do not split them here, the process will throw java.lang.NumberFormatException
+        .split("|").last()
 }

src/main/kotlin/org/jetbrains/research/testspark/bundles/kex/KexBundlePaths.kt

@@ -0,0 +1,8 @@
+package org.jetbrains.research.testspark.bundles.kex
+
+object KexBundlePaths {
+    const val defaults: String = "properties.kex.KexDefaults"
+    const val messages: String = "properties.kex.KexMessages"
+    const val labels: String = "properties.kex.KexLabels"
+    const val settings: String = "properties.kex.KexSettings"
+}

src/main/kotlin/org/jetbrains/research/testspark/bundles/kex/KexDefaultsBundle.kt

@@ -0,0 +1,21 @@
+package org.jetbrains.research.testspark.bundles.kex
+
+import com.intellij.DynamicBundle
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.PropertyKey
+
+/**
+ * Loads the `resources` directory.
+ */
+object KexDefaultsBundle : DynamicBundle(KexBundlePaths.defaults) {
+
+    /**
+     * Gets the requested default value.
+     */
+    @Nls
+    fun get(@PropertyKey(resourceBundle = KexBundlePaths.defaults) key: String): String = getMessage(key)
+        // In Intellij Platform version 2, the DynamicBundle returns the whole path and the value at the end in plugin verification.
+        // Each is separated by "|" (e.g., "|b|properties.llm.LLMDefaults|k|maxLLMRequest|3")
+        // if we do not split them here, the process will throw java.lang.NumberFormatException
+        .split("|").last()
+}

src/main/kotlin/org/jetbrains/research/testspark/bundles/kex/KexLabelsBundle.kt

@@ -0,0 +1,17 @@
+package org.jetbrains.research.testspark.bundles.kex
+
+import com.intellij.DynamicBundle
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.PropertyKey
+
+/**
+ * Loads the `resources` directory.
+ */
+object KexLabelsBundle : DynamicBundle(KexBundlePaths.labels) {
+
+    /**
+     * Gets the requested default value.
+     */
+    @Nls
+    fun get(@PropertyKey(resourceBundle = KexBundlePaths.labels) key: String): String = getMessage(key)
+}

src/main/kotlin/org/jetbrains/research/testspark/bundles/kex/KexMessagesBundle.kt

@@ -0,0 +1,17 @@
+package org.jetbrains.research.testspark.bundles.kex
+
+import com.intellij.DynamicBundle
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.PropertyKey
+
+/**
+ * Loads the `resources` directory.
+ */
+object KexMessagesBundle : DynamicBundle(KexBundlePaths.messages) {
+
+    /**
+     * Gets the requested default value.
+     */
+    @Nls
+    fun get(@PropertyKey(resourceBundle = KexBundlePaths.messages) key: String): String = getMessage(key)
+}

src/main/kotlin/org/jetbrains/research/testspark/bundles/kex/KexSettingsBundle.kt

@@ -0,0 +1,17 @@
+package org.jetbrains.research.testspark.bundles.kex
+
+import com.intellij.DynamicBundle
+import org.jetbrains.annotations.Nls
+import org.jetbrains.annotations.PropertyKey
+
+/**
+ * Loads the `resources` directory.
+ */
+object KexSettingsBundle : DynamicBundle(KexBundlePaths.settings) {
+
+    /**
+     * Gets the requested default value.
+     */
+    @Nls
+    fun get(@PropertyKey(resourceBundle = KexBundlePaths.settings) key: String): String = getMessage(key)
+}

src/main/kotlin/org/jetbrains/research/testspark/data/kex/KexMode.kt

@@ -0,0 +1,5 @@
+package org.jetbrains.research.testspark.data.kex
+
+enum class KexMode {
+    Symbolic, Concolic,
+}