Merge pull request #32931 from vespa-engine/interns/magnus/xmlintellij

Interns/magnus/xmlintellij

integration/schema-language-server/clients/intellij/build.gradle.kts

@@ -11,6 +11,14 @@ val JAVA_VERSION = "17"
 repositories {
   mavenCentral()
 
+  maven {
+    url = uri("https://repo.eclipse.org/content/repositories/lemminx")
+    metadataSources {
+      mavenPom()
+      artifact()
+    }
+  }
+
   mavenLocal()
   maven {
     url = uri("file://${System.getProperty("user.home")}/.m2/repository")
@@ -35,6 +43,13 @@ dependencies {
   implementation("org.jsoup:jsoup:1.17.2")
   implementation("com.vladsch.flexmark:flexmark-html2md-converter:0.64.8")
 
+  // Note: its quite important we ignore lsp4j, as the classes would collide
+  //       with the lsp4ij plugins classes.
+  implementation("org.eclipse.lemminx:org.eclipse.lemminx:0.28.0") {
+    exclude(group = "org.eclipse.lsp4j")
+    exclude(group = "com.google.code.gson")
+  }
+
   intellijPlatform {
     intellijIdeaCommunity("2024.2")
     instrumentationTools()
@@ -68,14 +83,16 @@ tasks {
   }
 
   prepareSandbox {
-    val fromPath = "../../language-server/target/schema-language-server-jar-with-dependencies.jar"
+    val fromPathSchema = "../../language-server/target/schema-language-server-jar-with-dependencies.jar"
+    val fromPathLemminx = "../../lemminx-vespa/target/lemminx-vespa-jar-with-dependencies.jar"
     val toPath = pluginDirectory.get()
 
     // see: https://docs.gradle.org/8.7/userguide/configuration_cache.html#config_cache:requirements:disallowed_types
     val injected = project.objects.newInstance<InjectFileSystem>()
     doLast {
       injected.fs.copy {
-        from(fromPath)
+        from(fromPathSchema)
+        from(fromPathLemminx)
         into(toPath)
       }
     }
@@ -95,4 +112,4 @@ tasks {
   publishPlugin {
     token.set(System.getenv("PUBLISH_TOKEN"))
   }
-}
+}

integration/schema-language-server/clients/intellij/src/main/java/ai/vespa/schemals/intellij/LemminxVespaClient.java

@@ -0,0 +1,49 @@
+package ai.vespa.schemals.intellij;
+import com.intellij.openapi.project.Project;
+import com.redhat.devtools.lsp4ij.LanguageServerManager;
+import com.redhat.devtools.lsp4ij.client.LanguageClientImpl;
+import com.redhat.devtools.lsp4ij.commands.CommandExecutor;
+import com.redhat.devtools.lsp4ij.commands.LSPCommandContext;
+import org.eclipse.lsp4j.*;
+import org.eclipse.lsp4j.jsonrpc.CompletableFutures;
+import org.eclipse.lsp4j.jsonrpc.services.JsonRequest;
+import org.eclipse.lemminx.customservice.XMLLanguageClientAPI;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+/*
+The XMLLanguageClientAPI from LemMinX declares the custom LSP extensions they made.
+It is two things:
+- JSON Notification: "actionableNotification" (throws UnsupportedOperationException unless implemented here)
+- JSON Request: "executeClientCommand" (this is the reason we implement it).
+ */
+public class LemminxVespaClient extends LanguageClientImpl implements XMLLanguageClientAPI {
+
+    private static final Map<String, String> clientVespaCommands = Map.of(
+            "vespaSchemaLS.commands.schema.findSchemaDefinition", "FIND_SCHEMA_DEFINITION",
+            "vespaSchemaLS.commands.schema.setupWorkspace", "SETUP_WORKSPACE",
+            "vespaSchemaLS.commands.schema.hasSetupWorkspace", "HAS_SETUP_WORKSPACE",
+            "vespaSchemaLS.commands.schema.createSchemaFile", "CREATE_SCHEMA_FILE",
+            "vespaSchemaLS.commands.schema.getDefinedSchemas", "GET_DEFINED_SCHEMAS"
+    );
+
+    public LemminxVespaClient(Project project) { super(project); }
+
+    @Override
+    public CompletableFuture<Object> executeClientCommand(ExecuteCommandParams params) {
+        super.logMessage(new MessageParams(MessageType.Info, "Execute: " + params.toString()));
+        String commandKey = params.getCommand();
+
+        if (!clientVespaCommands.containsKey(commandKey)) {
+            return null;
+        }
+
+        // Forward command to vespa schema LS
+        Command command = new Command("SchemaCommand", clientVespaCommands.get(commandKey));
+        command.setArguments(params.getArguments());
+        LSPCommandContext context = new LSPCommandContext(command, getProject());
+        context.setPreferredLanguageServerId("vespaSchemaLanguageServer");
+        return CommandExecutor.executeCommand(context).response();
+    }
+}

integration/schema-language-server/clients/intellij/src/main/java/ai/vespa/schemals/intellij/LemminxVespaServer.java

@@ -0,0 +1,51 @@
+package ai.vespa.schemals.intellij;
+
+import com.intellij.openapi.project.Project;
+import com.redhat.devtools.lsp4ij.server.JavaProcessCommandBuilder;
+import com.redhat.devtools.lsp4ij.server.ProcessStreamConnectionProvider;
+
+import java.io.File;
+import java.util.List;
+
+import com.intellij.openapi.extensions.PluginId;
+import com.intellij.ide.plugins.PluginManagerCore;
+
+public class LemminxVespaServer extends ProcessStreamConnectionProvider {
+    public LemminxVespaServer(Project project) {
+        PluginId id = PluginId.getId("ai.vespa");
+        var vespaPlugin = PluginManagerCore.getPlugin(id);
+        if (vespaPlugin == null) {
+            throw new IllegalStateException("Plugin " + id + " not found. Cannot start the Vespa Schema Language Support plugin.");
+        }
+        var vespaPluginPath = vespaPlugin.getPluginPath();
+
+        var lsp4ijPlugin = PluginManagerCore.getPlugin(PluginId.getId("com.redhat.devtools.lsp4ij"));
+        if (lsp4ijPlugin == null) {
+            throw new IllegalStateException("LSP4IJ could not be found. Cannot start the Vespa Schema Language Support plugin.");
+        }
+
+        var vespaServerPath = vespaPluginPath
+                .resolve("lemminx-vespa-jar-with-dependencies.jar")
+                .toAbsolutePath()
+                .toString();
+
+        var lemminxPath = vespaPluginPath
+                .resolve("lib")
+                .resolve("*")
+                .toAbsolutePath()
+                .toString();
+
+        var lsp4ijPath = lsp4ijPlugin.getPluginPath()
+                .resolve("lib")
+                .resolve("*")
+                .toAbsolutePath()
+                .toString();
+
+        List<String> commands = new JavaProcessCommandBuilder(project, "vespaLemminxLanguageServer")
+                .setCp(lemminxPath + File.pathSeparator + vespaServerPath + File.pathSeparator + lsp4ijPath)
+                .create();
+        commands.add("org.eclipse.lemminx.XMLServerLauncher");
+
+        super.setCommands(commands);
+    }
+}

integration/schema-language-server/clients/intellij/src/main/java/ai/vespa/schemals/intellij/LemminxVespaServerFactory.java

@@ -0,0 +1,18 @@
+package ai.vespa.schemals.intellij;
+import com.redhat.devtools.lsp4ij.LanguageServerFactory;
+import com.redhat.devtools.lsp4ij.server.StreamConnectionProvider;
+import com.intellij.openapi.project.Project;
+import com.redhat.devtools.lsp4ij.client.LanguageClientImpl;
+import org.eclipse.lsp4j.services.LanguageServer;
+
+public class LemminxVespaServerFactory implements LanguageServerFactory {
+    @Override
+    public StreamConnectionProvider createConnectionProvider(Project project) {
+        return new LemminxVespaServer(project);
+    }
+
+    @Override
+    public LanguageClientImpl createLanguageClient(Project project) {
+        return new LemminxVespaClient(project);
+    }
+}

integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml

@@ -50,5 +50,19 @@ The plugin now supports syntax highlighting of <code>.yql</code> files, in addit
     <fileNamePatternMapping patterns="*.yql"
                             serverId="vespaSchemaLanguageServer"
                             languageId="vespaYQL"/>
+
+      <server id="lemminxVespa"
+              name="Vespa LemMinX Extension"
+              factoryClass="ai.vespa.schemals.intellij.LemminxVespaServerFactory"
+              lastDocumentDisconnectedTimeout="36000">
+        <description><![CDATA[
+                    The Vespa LemMinX Language Server provides support in:
+                    <ul>
+                      <li>The services.xml file of Vespa Applications</li>
+                    </ul>
+                    ]]>
+        </description>
+      </server>
+      <fileNamePatternMapping patterns="services.xml" serverId="lemminxVespa" languageId="xml" />
   </extensions>
 </idea-plugin>

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/documentation/FetchDocumentation.java

@@ -48,23 +48,21 @@ public static void fetchSchemaDocs(Path targetPath) throws IOException {
         Map<String, String> schemaMarkdownContent = new SchemaDocumentationFetcher(SCHEMA_URL).getMarkdownContent();
 
         for (var entry : schemaMarkdownContent.entrySet()) {
-            String fileName = convertToToken(entry.getKey());
+            String tokenName = convertToToken(entry.getKey());
             String content = entry.getValue();
 
-            if (REPLACE_FILENAME_MAP.containsKey(fileName)) {
-                for (String replacedFileName : REPLACE_FILENAME_MAP.get(fileName)) {
-                    Files.write(writePath.resolve(replacedFileName + ".md"), content.getBytes(), StandardOpenOption.CREATE);
-                }
-            } else {
-                Files.write(writePath.resolve(fileName + ".md"), content.getBytes(), StandardOpenOption.CREATE);
+            List<String> fileNamesToWrite = REPLACE_FILENAME_MAP.getOrDefault(tokenName, List.of(tokenName));
+
+            for (String fileName : fileNamesToWrite) {
+                writeMarkdown(writePath.resolve(fileName + ".md"), content);
             }
         }
 
         Map<String, String> rankFeatureMarkdownContent = new RankFeatureDocumentationFetcher(RANK_FEATURE_URL).getMarkdownContent();
 
         writePath = targetPath.resolve("rankExpression");
         for (var entry : rankFeatureMarkdownContent.entrySet()) {
-            Files.write(writePath.resolve(entry.getKey() + ".md"), entry.getValue().getBytes(), StandardOpenOption.CREATE);
+            writeMarkdown(writePath.resolve(entry.getKey() + ".md"), entry.getValue());
         }
     }
 
@@ -81,11 +79,15 @@ public static void fetchServicesDocs(Path targetPath) throws IOException {
             for (var entry : markdownContent.entrySet()) {
                 if (entry.getKey().contains("/")) continue;
                 String fileName = entry.getKey().toLowerCase();
-                Files.write(writePath.resolve(fileName + ".md"), entry.getValue().getBytes(), StandardOpenOption.CREATE);
+                writeMarkdown(writePath.resolve(fileName + ".md"), entry.getValue());
             }
         }
     }
 
+    private static void writeMarkdown(Path writePath, String markdown) throws IOException {
+        Files.write(writePath, markdown.getBytes(), StandardOpenOption.CREATE);
+    }
+
     private static String convertToToken(String h2Id) {
         return h2Id.toUpperCase().replaceAll("-", "_");
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/documentation/SchemaDocumentationFetcher.java

@@ -75,13 +75,10 @@ Map<String, String> getMarkdownContent() throws IOException {
             }
 
             if (element.tag().equals(Tag.valueOf("table"))) {
-                Element tbody = element.selectFirst("tbody");
-                // replace all <th> in tbody with <td>
-                tbody.select("th").tagName("td");
-
-                // some tables have very big texts in td. For our purposes, only keep the first sentence.
-                if (prevH2.id().equals("field"))
-                    manuallyFixFieldTable(tbody);
+                // The tables in the docs are inherently problematic 
+                // so we just replace the first table and everything after with "read more"
+                prevH2 = null;
+                continue;
             }
 
             currentBuilder.append(element.outerHtml());
@@ -107,28 +104,4 @@ Map<String, String> getMarkdownContent() throws IOException {
         }
         return result;
 	}
-
-    private static void manuallyFixFieldTable(Element tbodyElement) {
-        for (Element td : tbodyElement.select("tr td:nth-child(2)")) {
-            String curr = td.html();
-            int level = 0;
-            int i;
-            for (i = 0; i < curr.length(); ++i) {
-                if ((
-                    (curr.charAt(i) == '.' && !curr.substring(i-1, Math.min(curr.length(), i+3)).equals("i.e.") && !curr.substring(i-3,i+1).equals("i.e."))
-                    || curr.substring(i).startsWith("<code>") 
-                    || curr.substring(i).startsWith("<pre>") 
-                    || curr.charAt(i) == ':') && level == 0) {
-                    break;
-                }
-                if (curr.charAt(i) == '(')++level;
-                if (curr.charAt(i) == ')')--level;
-                if (curr.charAt(i) == '<')++level;
-                if (curr.charAt(i) == '>')--level;
-            }
-            String firstSentence = curr.substring(0, i) + ".";
-            td.html(firstSentence);
-        }
-    }
-
 }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/documentation/ServicesDocumentationFetcher.java

@@ -97,6 +97,11 @@ Map<String, String> getMarkdownContent() throws IOException {
                     currentBuilder.append(nodeIterator.toString());
                 continue;
             }
+            if (element.tag().equals(Tag.valueOf("table"))) {
+                // tables are inherently problematic so we just replace everything after the first table with "read more"
+                prevH2 = null;
+                continue;
+            }
 
             currentBuilder.append(getElementHTML(element));
         }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/ExecuteCommand.java

@@ -17,6 +17,11 @@ public static Object executeCommand(EventExecuteCommandContext context) {
         if (command.isEmpty()) {
             context.logger.error("Unknown command " + context.params.getCommand());
             context.logger.error("Arguments:");
+            if (context.params.getArguments() == null) {
+                context.logger.error("null");
+                return null;
+            }
+
             for (Object obj : context.params.getArguments()) {
                 context.logger.info(obj.getClass().toString() + ": " + obj.toString());
             }