Merge pull request #32929 from vespa-engine/theodorkl/improveLSPYQLPa…

…rsing2

Feat: Add support for YQL Queries in LSP

integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml

@@ -21,11 +21,20 @@
     <li>Renaming</li>
 </ul>
 
+<h4>YQL Features</h4>
+<ul>
+  <li>Error highlighting</li>
+  <li>Syntax highlighting</li>
+  <li>Running queries directly from <code>.yql</code> files
+</ul>
+
+<h2>Requirements</h4>
+The plugin requires <a href="https://docs.vespa.ai/en/vespa-cli.html">Vespa CLI</a> to be installed to be able to run Vespa Queries from <code>.yql</code> files.
+
   ]]></description>
   <change-notes><![CDATA[
-<h2>Refactored to use LSP4IJ</h2>
-The plugin will now support better syntax highlighting with semantic tokens and renaming.
-In addition, the plugin will be available for community editions as well.
+<h2>Simple support for YQL</h2>
+The plugin now supports syntax highlighting of <code>.yql</code> files, in addition to run the queries directly from the editor.
 ]]></change-notes>
   <depends>com.intellij.modules.platform</depends>
   <depends>com.redhat.devtools.lsp4ij</depends>
@@ -38,5 +47,8 @@ In addition, the plugin will be available for community editions as well.
     <fileNamePatternMapping patterns="*.sd;*.profile"
                             serverId="vespaSchemaLanguageServer"
                             languageId="vespaSchema"/>
+    <fileNamePatternMapping patterns="*.yql"
+                            serverId="vespaSchemaLanguageServer"
+                            languageId="vespaYQL"/>
   </extensions>
 </idea-plugin>

integration/schema-language-server/clients/vscode/README.md

@@ -14,6 +14,11 @@ Features:
 - Renaming/refactoring
 - List document symbols
 
+YQL Features:
+- Error highlighting
+- Semantic token highlighting
+- Running queries directly from `.yql` files
+
 ## Requirements
 The extension requires Java 17 or greater. Upon activation, the extension will look in the following locations in this order for a Java executable:
 
@@ -23,6 +28,8 @@ The extension requires Java 17 or greater. Upon activation, the extension will l
 - JDK_HOME environment variable
 - JAVA_HOME environment variable
 
+The extension also requires [Vespa CLI](https://docs.vespa.ai/en/vespa-cli.html) to run Vespa Queries from `.yql` files.
+
 ## XML support
 This extension bundles with an extension to the [LemMinX XML Language server](https://github.com/eclipse/lemminx).
 This is to provide additional support when editing the services.xml file in Vespa applications. 

integration/schema-language-server/clients/vscode/package.json

@@ -15,7 +15,8 @@
   ],
   "keywords": [
     "Vespa",
-    "Schema"
+    "Schema",
+    "YQL"
   ],
   "repository": {
     "type": "git",  
@@ -24,7 +25,8 @@
   "icon": "images/icon.png",
   "activationEvents": [
     "onLanguage:xml",
-    "onLanguage:vespaSchema"
+    "onLanguage:vespaSchema",
+    "onLanguage:vespaYQL"
   ],
   "main": "./dist/extension.js",
   "contributes": {
@@ -39,6 +41,16 @@
           ".profile"
         ],
         "configuration": "./language-configuration.json"
+      },
+      {
+        "id": "vespaYQL",
+        "aliases": [
+          "Vespa YQL"
+        ],
+        "extensions": [
+          ".yql"
+        ],
+        "configuration": "./language-configuration.json"
       }
     ],
     "xml.javaExtensions": [

integration/schema-language-server/clients/vscode/src/extension.ts

@@ -72,24 +72,16 @@ function createAndStartClient(serverPath: string): LanguageClient | null {
 
 	let clientOptions: LanguageClientOptions = {
 		// Register the server for plain text documents
-		documentSelector: [{
-			scheme: 'file',
-			language: 'vespaSchema',
-		}],
-		middleware: { 
-			provideCompletionItem: async (document, position, context, token, next) => {
-				const r = await next(document, position, context, token);
-				return r;
-			},
-			provideDocumentHighlights: async (document, position, token, next) => {
-				const r = await next(document, position, token);
-				return r;
-			},
-			provideDocumentSemanticTokens: async (document, token, next) => {
-				const r = await next(document, token);
-				return r;
-			},
-		},
+		documentSelector: [
+            {
+			    scheme: 'file',
+			    language: 'vespaSchema',
+		    },
+            {
+                scheme: 'file',
+                language: 'vespaYQL'
+            }
+        ],
         synchronize: {
             fileEvents: vscode.workspace.createFileSystemWatcher("**/*{.sd,.profile}")
         }

integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc

@@ -40,6 +40,12 @@ INJECT GroupingParser:
 
 }
 
+INJECT GroupingParserLexer:
+{
+    public static EnumSet<TokenType> getRegularTokens() {
+        return EnumSet.copyOf(regularTokens);
+    }
+}
 
 
 TOKEN :

integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc

@@ -12,6 +12,13 @@ INJECT YQLPlusParser:
     protected Deque<Boolean> expression_stack = new ArrayDeque<>();
 }
 
+INJECT YQLPlusLexer:
+{
+    public static EnumSet<TokenType> getRegularTokens() {
+        return EnumSet.copyOf(regularTokens);
+    }
+}
+
 // --------------------------------------------------------------------------------
 //
 // Token declarations.
@@ -466,7 +473,7 @@ argument(boolean in_select):
 expression(boolean select):
     (
         null_operator
-        | annotate_expression
+        | (SCAN annotate_expression => annotate_expression)
         | logical_OR_expression
     )
 ;
@@ -515,7 +522,7 @@ equality_expression:
 
 in_not_in_target:
     ( // TODO: Add expression stack peek
-        ( <LPAREN> select_statement <RPAREN> )
+        SCAN 2 => ( <LPAREN> select_statement <RPAREN> )
         | literal_list
     )
 ;
@@ -609,7 +616,7 @@ primary_expression:
         (<LPAREN> expression(in_select) <RPAREN> )
         | constant_expression
         | (
-            (SCAN 2 => call_expression(in_select))
+            (SCAN namespaced_name <LPAREN> => call_expression(in_select) )
             | fieldref
         )
     )

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java

@@ -56,7 +56,7 @@ public Object execute(EventExecuteCommandContext context) {
 
         runVespaQuery(queryCommand, context.logger).thenAccept(result -> {
             if (!result.success()) {
-                if (result.result().toLowerCase().contains("command not found")) {
+                if (result.result().toLowerCase().contains("cannot run program")) {
                     context.messageHandler.sendMessage(MessageType.Error, "Could not find vespa CLI. Make sure vespa CLI is installed and added to path. Download vespa CLI here: https://docs.vespa.ai/en/vespa-cli.html");
                     return;
                 }
@@ -107,13 +107,10 @@ private CompletableFuture<QueryResult> runVespaQuery(String query, ClientLogger
 
         ProcessBuilder builder = new ProcessBuilder();
 
-        String queryEscaped = query.replace("\"", "\\\"");
-        String vespaCommand = String.format("vespa query \"%s\"", queryEscaped);
-
         if (isWindows) {
-            builder.command("cmd.exe", "/c", vespaCommand); // TODO: Test this on windows
+            builder.command("cmd.exe", "/c", "vespa", "query", query); // TODO: Test this on windows
         } else {
-            builder.command("/bin/sh", "-c", vespaCommand);
+            builder.command("vespa", "query", query);
         }
 
         return CompletableFuture.supplyAsync(() -> {
@@ -146,8 +143,7 @@ private CompletableFuture<QueryResult> runVespaQuery(String query, ClientLogger
             } catch (InterruptedException e) {
                 return new QueryResult(false, "Program interrupted");
             } catch (IOException e) {
-                logger.error(e.getMessage());
-                return new QueryResult(false, "IOException occurred.");
+                return new QueryResult(false, e.getMessage());
             }
         });
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java

@@ -21,7 +21,6 @@
 import ai.vespa.schemals.tree.Node;
 import ai.vespa.schemals.tree.SchemaNode;
 import ai.vespa.schemals.tree.YQLNode;
-import ai.vespa.schemals.tree.YQL.YQLUtils;
 
 public class YQLDocument implements DocumentManager {
 
@@ -107,13 +106,42 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge
 
         int charsRead = parser.getToken(0).getEndOffset();
 
+        if (charsRead == 0) return new YQLPartParseResult(List.of(), Optional.empty(), charsRead);
+
         ai.vespa.schemals.parser.yqlplus.Node node = parser.rootNode();
         YQLNode retNode = new YQLNode(node, offset);
         // YQLUtils.printTree(logger, node);
 
         return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead);
     }
 
+    private static boolean detectContinuation(String inputString) {
+        for (int i = 0; i < inputString.length(); i++) {
+            if (inputString.charAt(i) != ' ') {
+                return inputString.charAt(i) == '{';
+            }
+        }
+        return false;
+    }
+
+    private static YQLPartParseResult parseContinuation(String inputString, Position offset) {
+
+        YQLPlusParser parser = new YQLPlusParser(inputString);
+
+        try {
+            parser.map_expression();
+        } catch (ParseException exception) {
+            // Ignored, marked as dirty node
+        }
+
+        var node = parser.rootNode();
+        YQLNode retNode = new YQLNode(node, offset);
+
+        int charsRead = parser.getToken(0).getEndOffset();
+
+        return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead);
+    }
+
     private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset) {
         YQLNode ret = new YQLNode(new Range(offset, offset));
 
@@ -139,14 +167,36 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que
                 Position groupOffset = CSTUtils.addPositions(groupOffsetWithoutPipe, new Position(0, 1)); // Add pipe char
 
                 ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|"));
-
-                YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset);
-                if (groupingResult.CST.isPresent()) {
-                    ret.addChild(groupingResult.CST.get());
+                charsRead++;
+
+                // Look for continuation
+                boolean continuationDetected = detectContinuation(groupingString);
+                if (continuationDetected) {
+                    YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset);
+
+                    diagnostics.addAll(continuationResults.diagnostics());
+                    if (continuationResults.CST().isPresent()) {
+                        ret.addChild(continuationResults.CST().get());
+                    }
+
+                    charsRead += continuationResults.charsRead();
+                    String continuationString = groupingString.substring(0, continuationResults.charsRead());
+                    Position continuationPosition = StringUtils.getStringPosition(continuationString);
+
+                    groupingString = groupingString.substring(continuationResults.charsRead());
+                    groupOffset = CSTUtils.addPositions(groupOffset, continuationPosition);
+                }
+
+                if (groupingString.length() > 0 && groupingString.strip().length() > 0) {
+                    YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset);
+                    if (groupingResult.CST.isPresent()) {
+                        ret.addChild(groupingResult.CST.get());
+                    }
+
+                    diagnostics.addAll(groupingResult.diagnostics());
+                    charsRead += groupingResult.charsRead(); // Add one for the pipe symbol
                 }
 
-                diagnostics.addAll(groupingResult.diagnostics());
-                charsRead += 1 + groupingResult.charsRead(); // Add one for the pipe symbol
             }
 
         }
@@ -180,6 +230,8 @@ public static ParseResult parseContent(ParseContext context) {
             if (result.CST().isPresent()) {
                 ret.addChild(result.CST().get());
             }
+
+            if (result.charsRead() == 0) result.charsRead++;
 
             int newOffset = content.indexOf('\n', charsRead + result.charsRead());
             if (newOffset == -1) {