From 97957b8c1c1a149ef8cde305c78f9131b33c93ba Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Thu, 22 Aug 2024 15:56:49 +0200 Subject: [PATCH 01/31] feat: setup lemminx extension --- .../clients/intellij/.gitignore | 1 + .../clients/vscode/package.json | 15 +++- .../lemminx-vespa/README.md | 2 + .../lemminx-vespa/pom.xml | 77 +++++++++++++++++++ .../ai/vespa/lemminxvespa/VespaExtension.java | 36 +++++++++ ....lemminx.services.extensions.IXMLExtension | 1 + pom.xml | 1 + 7 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 integration/schema-language-server/lemminx-vespa/README.md create mode 100644 integration/schema-language-server/lemminx-vespa/pom.xml create mode 100644 integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java create mode 100644 integration/schema-language-server/lemminx-vespa/src/main/resources/META-INF/services/org.eclipse.lemminx.services.extensions.IXMLExtension diff --git a/integration/schema-language-server/clients/intellij/.gitignore b/integration/schema-language-server/clients/intellij/.gitignore index f272288afa29..8806173ad559 100644 --- a/integration/schema-language-server/clients/intellij/.gitignore +++ b/integration/schema-language-server/clients/intellij/.gitignore @@ -2,6 +2,7 @@ .intellijPlatform/ .run/ build/ +bin/ gradle/ gradle.properties gradlew diff --git a/integration/schema-language-server/clients/vscode/package.json b/integration/schema-language-server/clients/vscode/package.json index 59db8a7ac099..1887d7ab06de 100644 --- a/integration/schema-language-server/clients/vscode/package.json +++ b/integration/schema-language-server/clients/vscode/package.json @@ -22,8 +22,11 @@ "url": "https://github.com/vespa-engine/vespa" }, "icon": "images/icon.png", - "activationEvents": [], - "main": "./dist/extension.js", + "activationEvents": [ + "onLanguage:xml", + "onLanguage:vespaSchema" + ], + "main": "./out/extension.js", "contributes": { "languages": [ { @@ -38,6 +41,9 @@ "configuration": "./language-configuration.json" } ], + "xml.javaExtensions": [ + "./server/lemminx-vespa-jar-with-dependencies.jar" + ], "configuration": { "title": "Vespa Schema Language Support", "properties": { @@ -65,7 +71,8 @@ "test": "vscode-test", "check-types": "tsc --noEmit", "copy-images": "cp ../../resources/* ./images", - "langserver-install": "mkdir -p server && cp ../../language-server/target/schema-language-server-jar-with-dependencies.jar ./server/", + "langserver-install": "mkdir -p server && cp ../../language-server/target/schema-language-server-jar-with-dependencies.jar ./server/ && npm run lemminx-install", + "lemminx-install": "mkdir -p server && cp ../../lemminx-vespa/target/lemminx-vespa-jar-with-dependencies.jar ./server/", "publish": "npm run compile && node out/publish.js" }, "devDependencies": { @@ -86,4 +93,4 @@ "hasbin": "^1.2.3", "vscode-languageclient": "^9.0.1" } -} \ No newline at end of file +} diff --git a/integration/schema-language-server/lemminx-vespa/README.md b/integration/schema-language-server/lemminx-vespa/README.md new file mode 100644 index 000000000000..ba4ced370bc5 --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/README.md @@ -0,0 +1,2 @@ +# Lemminx Vespa +Providing support when editing Vespa XML files. diff --git a/integration/schema-language-server/lemminx-vespa/pom.xml b/integration/schema-language-server/lemminx-vespa/pom.xml new file mode 100644 index 000000000000..430d7374b81e --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/pom.xml @@ -0,0 +1,77 @@ + + + + 4.0.0 + + com.yahoo.vespa + parent + 8-SNAPSHOT + ../../../parent/pom.xml + + lemminx-vespa + jar + 8-SNAPSHOT + Vespa LemminX Extension + + + org.eclipse.lemminx + org.eclipse.lemminx + 0.12.0 + provided + + + + + lemminx-releases + https://repo.eclipse.org/content/repositories/lemminx-releases/ + + false + + + true + + + + + + + + src/main/resources + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + -Xlint:all + -Xlint:-deprecation + -Xlint:-unchecked + -Xlint:-rawtypes + + + + + org.apache.maven.plugins + maven-install-plugin + true + + + com.yahoo.vespa + bundle-plugin + + + package + assemble-fat-jar + + + + + com.github.os72 + protoc-jar-maven-plugin + + + + diff --git a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java new file mode 100644 index 000000000000..96aad1286157 --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java @@ -0,0 +1,36 @@ +package ai.vespa.lemminxvespa; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PipedReader; +import java.io.PrintStream; +import java.util.logging.Logger; + +import org.eclipse.lemminx.services.extensions.IXMLExtension; +import org.eclipse.lemminx.services.extensions.XMLExtensionsRegistry; +import org.eclipse.lemminx.services.extensions.save.ISaveContext; +import org.eclipse.lsp4j.InitializeParams; + +/** + * ServicesPlugin + */ +public class VespaExtension implements IXMLExtension { + PrintStream logger = System.out; + + @Override + public void doSave(ISaveContext arg0) { + } + + @Override + public void start(InitializeParams params, XMLExtensionsRegistry registry) { + try { + logger = new PrintStream(new FileOutputStream("/Users/magnus/repos/integrationtest7/log.txt", true)); + logger.println("Asserted dominance."); + } catch(Exception ex) { + } + } + + @Override + public void stop(XMLExtensionsRegistry arg0) { + } +} diff --git a/integration/schema-language-server/lemminx-vespa/src/main/resources/META-INF/services/org.eclipse.lemminx.services.extensions.IXMLExtension b/integration/schema-language-server/lemminx-vespa/src/main/resources/META-INF/services/org.eclipse.lemminx.services.extensions.IXMLExtension new file mode 100644 index 000000000000..ce20a4e86b65 --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/src/main/resources/META-INF/services/org.eclipse.lemminx.services.extensions.IXMLExtension @@ -0,0 +1 @@ +ai.vespa.lemminxvespa.VespaExtension diff --git a/pom.xml b/pom.xml index 1f75bb9667a2..e9d09d9373ca 100644 --- a/pom.xml +++ b/pom.xml @@ -143,6 +143,7 @@ zookeeper-common zookeeper-server integration/schema-language-server/language-server + integration/schema-language-server/lemminx-vespa From 1244a7045471dfbd6d7d12c6b5c33e319d491bcb Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Thu, 22 Aug 2024 17:05:10 +0200 Subject: [PATCH 02/31] inprogress: implement validation of xml --- .../ai/vespa/lemminxvespa/VespaExtension.java | 25 ++++++++--- .../participant/DiagnosticsParticipant.java | 33 +++++++++++++++ .../participant/HoverParticipant.java | 42 +++++++++++++++++++ .../vespa/lemminxvespa/util/LoggerUtils.java | 21 ++++++++++ 4 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/DiagnosticsParticipant.java create mode 100644 integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/HoverParticipant.java create mode 100644 integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/util/LoggerUtils.java diff --git a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java index 96aad1286157..488ec7f05c8b 100644 --- a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java +++ b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/VespaExtension.java @@ -1,24 +1,31 @@ package ai.vespa.lemminxvespa; -import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.PipedReader; import java.io.PrintStream; -import java.util.logging.Logger; import org.eclipse.lemminx.services.extensions.IXMLExtension; import org.eclipse.lemminx.services.extensions.XMLExtensionsRegistry; +import org.eclipse.lemminx.services.extensions.diagnostics.IDiagnosticsParticipant; import org.eclipse.lemminx.services.extensions.save.ISaveContext; import org.eclipse.lsp4j.InitializeParams; +import ai.vespa.lemminxvespa.participant.DiagnosticsParticipant; +import ai.vespa.lemminxvespa.participant.HoverParticipant; + /** * ServicesPlugin + * TODO: Figure out class loading of lemminx stuff. + * TODO: Fetch *.xsd files e.g. ./config-model/target/generated-sources/trang/resources/schema/services.xsd + * and use to validate. */ public class VespaExtension implements IXMLExtension { PrintStream logger = System.out; + private IDiagnosticsParticipant diagnosticsParticipant; + //private IHoverParticipant hoverParticipant; + @Override - public void doSave(ISaveContext arg0) { + public void doSave(ISaveContext saveContext) { } @Override @@ -28,9 +35,17 @@ public void start(InitializeParams params, XMLExtensionsRegistry registry) { logger.println("Asserted dominance."); } catch(Exception ex) { } + + diagnosticsParticipant = new DiagnosticsParticipant(this.logger); + registry.registerDiagnosticsParticipant(diagnosticsParticipant); + + //hoverParticipant = new HoverParticipant(this.logger); + //registry.registerHoverParticipant(hoverParticipant); } @Override - public void stop(XMLExtensionsRegistry arg0) { + public void stop(XMLExtensionsRegistry registry) { + registry.unregisterDiagnosticsParticipant(diagnosticsParticipant); + //registry.unregisterHoverParticipant(hoverParticipant); } } diff --git a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/DiagnosticsParticipant.java b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/DiagnosticsParticipant.java new file mode 100644 index 000000000000..06c7b7ecbc40 --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/DiagnosticsParticipant.java @@ -0,0 +1,33 @@ +package ai.vespa.lemminxvespa.participant; + +import java.io.PrintStream; +import java.util.List; + +import org.eclipse.lemminx.dom.DOMDocument; +import org.eclipse.lemminx.services.extensions.diagnostics.IDiagnosticsParticipant; +import org.eclipse.lsp4j.Diagnostic; +import org.eclipse.lsp4j.DiagnosticSeverity; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.Range; +import org.eclipse.lsp4j.jsonrpc.CancelChecker; + +import ai.vespa.lemminxvespa.util.LoggerUtils; + +/** + * DiagnosticsParticipant + */ +public class DiagnosticsParticipant implements IDiagnosticsParticipant { + + private PrintStream logger; + + public DiagnosticsParticipant(PrintStream logger) { + this.logger = logger; + logger.println("Created participant"); + } + + @Override + public void doDiagnostics(DOMDocument xmlDocument, List diagnostics, CancelChecker monitor) { + logger.println("Do diagnostics"); + diagnostics.add(new Diagnostic(new Range(new Position(0, 0), new Position(0, 5)), "Helloo", DiagnosticSeverity.Error, "VESPALEMMINX")); + } +} diff --git a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/HoverParticipant.java b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/HoverParticipant.java new file mode 100644 index 000000000000..ebbd5096183d --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/participant/HoverParticipant.java @@ -0,0 +1,42 @@ +package ai.vespa.lemminxvespa.participant; + +import java.io.PrintStream; + +import org.eclipse.lemminx.services.extensions.IHoverParticipant; +import org.eclipse.lemminx.services.extensions.IHoverRequest; +import org.eclipse.lsp4j.Hover; + +/** + * HoverParticipant + */ +public class HoverParticipant implements IHoverParticipant { + PrintStream logger; + + public HoverParticipant(PrintStream logger) { + this.logger = logger; + } + + @Override + public Hover onTag(IHoverRequest request) throws Exception { + logger.println("On tag"); + return null; + } + + @Override + public Hover onAttributeName(IHoverRequest request) throws Exception { + logger.println("On attr name"); + return null; + } + + @Override + public Hover onAttributeValue(IHoverRequest request) throws Exception { + logger.println("On attr value"); + return null; + } + + @Override + public Hover onText(IHoverRequest request) throws Exception { + logger.println("On text"); + return null; + } +} diff --git a/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/util/LoggerUtils.java b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/util/LoggerUtils.java new file mode 100644 index 000000000000..95d6fc1edbb0 --- /dev/null +++ b/integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminxvespa/util/LoggerUtils.java @@ -0,0 +1,21 @@ +package ai.vespa.lemminxvespa.util; + +import java.io.PrintStream; + +import org.eclipse.lemminx.dom.DOMNode; + +/** + * LoggerUtils + */ +public class LoggerUtils { + public static void printDOM(PrintStream logger, DOMNode node) { + printDOM(logger, node, 0); + } + + private static void printDOM(PrintStream logger, DOMNode node, int indent) { + logger.println(new String(new char[indent]).replaceAll("\0", "\t") + node.getNodeName()); + for (DOMNode child : node.getChildren()) { + printDOM(logger, child, indent + 1); + } + } +} From 12393a2e7565e9726a10429fb8bb0e20738115cf Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 5 Sep 2024 16:09:03 +0200 Subject: [PATCH 03/31] feat: Add simple structure to run yql queries --- .../clients/vscode/package.json | 3 +- .../vespa/schemals/SchemaLanguageServer.java | 2 + .../schemals/SchemaTextDocumentService.java | 13 +++- .../schemals/context/EventContextCreator.java | 5 ++ .../schemals/lsp/codelens/SchemaCodeLens.java | 29 +++++++ .../schemals/lsp/command/CommandRegistry.java | 5 ++ .../command/commandtypes/RunVespaQuery.java | 21 +++++ .../lsp/completion/SchemaCompletion.java | 5 ++ .../schemadocument/DocumentManager.java | 8 ++ .../schemadocument/RankProfileDocument.java | 3 + .../schemadocument/SchemaDocument.java | 3 + .../SchemaDocumentScheduler.java | 38 +++++++-- .../schemals/schemadocument/YQLDocument.java | 78 +++++++++++++++++++ 13 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/codelens/SchemaCodeLens.java create mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/commandtypes/RunVespaQuery.java create mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java diff --git a/integration/schema-language-server/clients/vscode/package.json b/integration/schema-language-server/clients/vscode/package.json index a6d2ec47abbb..fca2260f9186 100644 --- a/integration/schema-language-server/clients/vscode/package.json +++ b/integration/schema-language-server/clients/vscode/package.json @@ -33,7 +33,8 @@ ], "extensions": [ ".sd", - ".profile" + ".profile", + ".yql" ], "configuration": "./language-configuration.json" } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java index d628d2611466..537901c1bce7 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java @@ -5,6 +5,7 @@ import org.eclipse.lsp4j.CodeActionKind; import org.eclipse.lsp4j.CodeActionOptions; +import org.eclipse.lsp4j.CodeLensOptions; import org.eclipse.lsp4j.CompletionOptions; import org.eclipse.lsp4j.ExecuteCommandOptions; import org.eclipse.lsp4j.InitializeParams; @@ -85,6 +86,7 @@ public CompletableFuture initialize(InitializeParams initializ initializeResult.getCapabilities().setSemanticTokensProvider(SchemaSemanticTokens.getSemanticTokensRegistrationOptions()); initializeResult.getCapabilities().setDocumentSymbolProvider(true); initializeResult.getCapabilities().setExecuteCommandProvider(new ExecuteCommandOptions(CommandRegistry.getSupportedCommandList())); + initializeResult.getCapabilities().setCodeLensProvider(new CodeLensOptions()); var options = new CodeActionOptions(List.of( CodeActionKind.QuickFix, diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java index ddc724b526bb..6f54fa36987b 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java @@ -47,6 +47,7 @@ import org.eclipse.lsp4j.TextDocumentItem; import org.eclipse.lsp4j.TextEdit; import org.eclipse.lsp4j.WorkspaceEdit; +import org.eclipse.lsp4j.jsonrpc.CancelChecker; import org.eclipse.lsp4j.jsonrpc.CompletableFutures; import org.eclipse.lsp4j.jsonrpc.messages.Either; import org.eclipse.lsp4j.jsonrpc.messages.Either3; @@ -57,6 +58,7 @@ import ai.vespa.schemals.context.EventDocumentContext; import ai.vespa.schemals.index.SchemaIndex; import ai.vespa.schemals.lsp.codeaction.SchemaCodeAction; +import ai.vespa.schemals.lsp.codelens.SchemaCodeLens; import ai.vespa.schemals.lsp.completion.SchemaCompletion; import ai.vespa.schemals.lsp.definition.SchemaDefinition; import ai.vespa.schemals.lsp.documentsymbols.SchemaDocumentSymbols; @@ -158,7 +160,16 @@ public CompletableFuture>> docume @Override public CompletableFuture> codeLens(CodeLensParams codeLensParams) { - return null; + + return CompletableFutures.computeAsync((cancelChecker) -> { + try { + EventDocumentContext context = eventContextCreator.createContext(codeLensParams); + return SchemaCodeLens.codeLens(context); + } catch (Exception e) { + logger.error("Error during code lens request: " + e.getMessage()); + } + return List.of(); + }); } @Override diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/context/EventContextCreator.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/context/EventContextCreator.java index a7d1a90b4e66..e62bfa39bae3 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/context/EventContextCreator.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/context/EventContextCreator.java @@ -3,6 +3,7 @@ import java.io.PrintStream; import org.eclipse.lsp4j.CodeActionParams; +import org.eclipse.lsp4j.CodeLensParams; import org.eclipse.lsp4j.CompletionParams; import org.eclipse.lsp4j.DocumentSymbolParams; import org.eclipse.lsp4j.ExecuteCommandParams; @@ -77,4 +78,8 @@ public EventExecuteCommandContext createContext(ExecuteCommandParams params) { return new EventExecuteCommandContext(scheduler, schemaIndex, messageHandler, params); } + public EventDocumentContext createContext(CodeLensParams params) { + return new EventDocumentContext(scheduler, schemaIndex, messageHandler, params.getTextDocument()); + } + } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/codelens/SchemaCodeLens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/codelens/SchemaCodeLens.java new file mode 100644 index 000000000000..989574a23a13 --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/codelens/SchemaCodeLens.java @@ -0,0 +1,29 @@ +package ai.vespa.schemals.lsp.codelens; + +import java.util.List; + +import org.eclipse.lsp4j.CodeLens; +import org.eclipse.lsp4j.Position; +import org.eclipse.lsp4j.Range; + +import ai.vespa.schemals.context.EventDocumentContext; +import ai.vespa.schemals.lsp.command.CommandRegistry; +import ai.vespa.schemals.lsp.command.CommandRegistry.CommandType; +import ai.vespa.schemals.schemadocument.DocumentManager.DocumentType; + +public class SchemaCodeLens { + + + static public List codeLens(EventDocumentContext context) { + + if (context.document.getDocumentType() != DocumentType.YQL) { + return List.of(); + } + + return List.of(new CodeLens( + new Range(new Position(1, 1), new Position(1, 4)), + CommandRegistry.createLSPCommand(CommandType.RUN_VESPA_QUERY, List.of()), + null + )); + } +} diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/CommandRegistry.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/CommandRegistry.java index c346febe8e43..9f19e8ab81db 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/CommandRegistry.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/CommandRegistry.java @@ -9,6 +9,7 @@ import ai.vespa.schemals.lsp.command.commandtypes.DocumentOpen; import ai.vespa.schemals.lsp.command.commandtypes.DocumentParse; +import ai.vespa.schemals.lsp.command.commandtypes.RunVespaQuery; import ai.vespa.schemals.lsp.command.commandtypes.CommandList; import ai.vespa.schemals.lsp.command.commandtypes.SchemaCommand; @@ -32,6 +33,10 @@ public enum CommandType implements GenericCommandType { COMMAND_LIST { public String title() { return "Command list"; } public SchemaCommand construct() { return new CommandList(); } + }, + RUN_VESPA_QUERY { + public String title() { return "Run Vespa query"; } + public SchemaCommand construct() { return new RunVespaQuery(); } } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/commandtypes/RunVespaQuery.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/commandtypes/RunVespaQuery.java new file mode 100644 index 000000000000..5f34a498028f --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/command/commandtypes/RunVespaQuery.java @@ -0,0 +1,21 @@ +package ai.vespa.schemals.lsp.command.commandtypes; + +import java.util.List; + +import ai.vespa.schemals.context.EventExecuteCommandContext; + +public class RunVespaQuery implements SchemaCommand { + + public int getArity() { + return -1; + } + + public boolean setArguments(List arguments) { + return true; + } + + public void execute(EventExecuteCommandContext context) { + context.logger.info("Running Vespa query..."); + } + +} diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/completion/SchemaCompletion.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/completion/SchemaCompletion.java index 44c31c92eaa6..f29b90798a8a 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/completion/SchemaCompletion.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/completion/SchemaCompletion.java @@ -18,6 +18,7 @@ import ai.vespa.schemals.lsp.completion.provider.SimpleColonCompletion; import ai.vespa.schemals.lsp.completion.provider.StructFieldCompletion; import ai.vespa.schemals.lsp.completion.provider.TypeCompletion; +import ai.vespa.schemals.schemadocument.DocumentManager.DocumentType; /** * Responsible for LSP textDocument/completion requests. @@ -44,6 +45,10 @@ public static ArrayList getCompletionItems(EventCompletionContex return ret; } + if (context.document.getDocumentType() != DocumentType.SCHEMA && context.document.getDocumentType() != DocumentType.PROFILE) { + return ret; + } + for (CompletionProvider provider : providers) { try { ret.addAll(provider.getCompletionItems(context)); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java index 0f4e5c1979a4..d142560d6734 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java @@ -10,6 +10,12 @@ */ public interface DocumentManager { + public enum DocumentType { + SCHEMA, + PROFILE, + YQL + } + public void updateFileContent(String content); public void updateFileContent(String content, Integer version); @@ -27,4 +33,6 @@ public interface DocumentManager { public String getCurrentContent(); public VersionedTextDocumentIdentifier getVersionedTextDocumentIdentifier(); + + public DocumentType getDocumentType(); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java index 478a92849f0e..cd524ed8f6d2 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java @@ -142,4 +142,7 @@ public VersionedTextDocumentIdentifier getVersionedTextDocumentIdentifier() { public SchemaDocumentLexer lexer() { return this.lexer; } + + @Override + public DocumentType getDocumentType() { return DocumentType.PROFILE; } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java index 0c1004e7a333..7d742fac30be 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java @@ -346,4 +346,7 @@ public String getCurrentContent() { public SchemaDocumentLexer lexer() { return this.lexer; } + + @Override + public DocumentType getDocumentType() { return DocumentType.SCHEMA; } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java index ea082e3a3868..6d9a19d295dc 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java @@ -16,11 +16,12 @@ import ai.vespa.schemals.SchemaMessageHandler; import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.common.FileUtils; -import ai.vespa.schemals.common.StringUtils; import ai.vespa.schemals.index.SchemaIndex; import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolType; +import ai.vespa.schemals.schemadocument.DocumentManager.DocumentType; + /** * Class responsible for maintaining the set of open documents and reparsing them. * When {@link SchemaDocumentScheduler#updateFile} is called, it will call {@link DocumentManager#updateFileContent} on the appropriate file @@ -28,6 +29,12 @@ */ public class SchemaDocumentScheduler { + private Map fileExtensions = new HashMap<>() {{ + put("sd", DocumentType.SCHEMA); + put("profile", DocumentType.PROFILE); + put("yql", DocumentType.YQL); + }}; + private ClientLogger logger; private SchemaDiagnosticsHandler diagnosticsHandler; private SchemaIndex schemaIndex; @@ -47,13 +54,27 @@ public void updateFile(String fileURI, String content) { updateFile(fileURI, content, null); } + private Optional getDocumentTypeFromURI(String fileURI) { + int dotIndex = fileURI.lastIndexOf('.'); + String fileExtension = fileURI.substring(dotIndex + 1).toLowerCase(); + + DocumentType documentType = fileExtensions.get(fileExtension); + if (documentType == null) return Optional.empty(); + return Optional.of(documentType); + } + public void updateFile(String fileURI, String content, Integer version) { - boolean isSchemaFile = fileURI.toLowerCase().endsWith(".sd"); + + Optional documentType = getDocumentTypeFromURI(fileURI); + if (documentType.isEmpty()) return; + if (!workspaceFiles.containsKey(fileURI)) { - if (isSchemaFile) { + if (documentType.get() == DocumentType.SCHEMA) { workspaceFiles.put(fileURI, new SchemaDocument(logger, diagnosticsHandler, schemaIndex, this, fileURI)); - } else { + } else if (documentType.get() == DocumentType.PROFILE) { workspaceFiles.put(fileURI, new RankProfileDocument(logger, diagnosticsHandler, schemaIndex, this, fileURI)); + } else if (documentType.get() == DocumentType.YQL) { + workspaceFiles.put(fileURI, new YQLDocument(logger, diagnosticsHandler, fileURI)); } } @@ -61,7 +82,7 @@ public void updateFile(String fileURI, String content, Integer version) { workspaceFiles.get(fileURI).updateFileContent(content, version); boolean needsReparse = false; - if (isSchemaFile && reparseDescendants) { + if (documentType.get() == DocumentType.SCHEMA && reparseDescendants) { Set parsedURIs = new HashSet<>() {{ add(fileURI); }}; for (String descendantURI : schemaIndex.getDocumentInheritanceGraph().getAllDescendants(fileURI)) { if (descendantURI.equals(fileURI)) continue; @@ -110,7 +131,12 @@ public String getWorkspaceURI() { public void openDocument(TextDocumentItem document) { logger.info("Opening document: " + document.getUri()); - if (workspaceURI == null) { + Optional documentType = getDocumentTypeFromURI(document.getUri()); + + if (workspaceURI == null && documentType.isPresent() && ( + documentType.get() == DocumentType.SCHEMA || + documentType.get() == DocumentType.PROFILE + )) { Optional workspaceURI = FileUtils.findSchemaDirectory(URI.create(document.getUri())); if (workspaceURI.isEmpty()) { messageHandler.sendMessage(MessageType.Warning, diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java new file mode 100644 index 000000000000..8bb191e991f8 --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -0,0 +1,78 @@ +package ai.vespa.schemals.schemadocument; + +import org.eclipse.lsp4j.VersionedTextDocumentIdentifier; + +import ai.vespa.schemals.SchemaDiagnosticsHandler; +import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.tree.SchemaNode; + +public class YQLDocument implements DocumentManager { + + boolean isOpen = false; + String fileURI; + String fileContent = ""; + + ClientLogger logger; + SchemaDiagnosticsHandler diagnosticsHandler; + + YQLDocument(ClientLogger logger, SchemaDiagnosticsHandler diagnosticsHandler, String fileURI) { + this.fileURI = fileURI; + this.logger = logger; + this.diagnosticsHandler = diagnosticsHandler; + } + + @Override + public void updateFileContent(String content) { + fileContent = content; + + reparseContent(); + } + + @Override + public void updateFileContent(String content, Integer version) { + updateFileContent(content); + } + + @Override + public void reparseContent() { + + } + + @Override + public boolean setIsOpen(boolean isOpen) { + this.isOpen = isOpen; + return isOpen; + } + + @Override + public boolean getIsOpen() { return isOpen; } + + @Override + public SchemaNode getRootNode() { + return null; + }; + + @Override + public SchemaDocumentLexer lexer() { + return new SchemaDocumentLexer(); + } + + @Override + public String getFileURI() { + return fileURI; + } + + @Override + public String getCurrentContent() { + return fileContent; + } + + @Override + public VersionedTextDocumentIdentifier getVersionedTextDocumentIdentifier() { + return new VersionedTextDocumentIdentifier(fileURI, 0); + } + + @Override + public DocumentType getDocumentType() { return DocumentType.YQL; } + +} From 9a4804d84fc8d34c3f127806c64142b22c83b7be Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 5 Sep 2024 17:42:51 +0200 Subject: [PATCH 04/31] chore: Add beginning of congoCC YQL parser --- .../language-server/pom.xml | 11 + .../src/main/ccc/yqlplus/YQLPlus.ccc | 279 ++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc diff --git a/integration/schema-language-server/language-server/pom.xml b/integration/schema-language-server/language-server/pom.xml index a1bc0e7483b8..5b63a2c543bf 100644 --- a/integration/schema-language-server/language-server/pom.xml +++ b/integration/schema-language-server/language-server/pom.xml @@ -159,6 +159,17 @@ 17 + + 4 + + ccc-generate + + + ${project.basedir}/src/main/ccc/yqlplus/YQLPlus.ccc + ${project.basedir}/target/generated-sources/ccc + 17 + + diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc new file mode 100644 index 000000000000..604a82fb13ae --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -0,0 +1,279 @@ +PARSER_CLASS=YQLPlusParser; +PARSER_PACKAGE=ai.vespa.schemals.parser.yqlplus; +FAULT_TOLERANT=true; +SMART_NODE_CREATION=false; // Will create a tree node for every rule + +INJECT YQLPlusParser: + + import java.util.Deque; + import java.util.ArrayDeque; + import com.yahoo.search.yql.*; +{ + protected Deque expression_stack = new ArrayDeque<>(); +} + +// -------------------------------------------------------------------------------- +// +// Token declarations. +// +// -------------------------------------------------------------------------------- + +SKIP : + " " | "\t" | "\r" | "\n" +; + +TOKEN : + < SELECT: 'select' > + +| < LIMIT: 'limit' > +| < OFFSET: 'offset' > +| < WHERE: 'where' > +| < ORDER: 'order' > +| < BY: 'by' > +| < ORDERBY: > +| < DESC: 'desc' > +| < FROM: 'from' > +| < SOURCES: 'sources' > +| < AS: 'as' > + +| < COMMA: ',' > +| < OUTPUT: 'output' > +| < COUNT: 'count' > + +| < TRUE: 'true' > +| < FALSE: 'false' > + +// brackets and other tokens in literals +| < LPAREN: '(' > +| < RPAREN: ')' > +| < LBRACKET: '[' > +| < RBRACKET: ']' > +| < LBRACE: '{' > +| < RBRACE: '}' > +| < COLON: ':' > +| < PIPE: '|' > + +// operators +| < AND: 'and' > +| < OR: 'or' > +| < NOT_IN: 'not in' > +| < IN: 'in' > + +| < LT: '<' > +| < GT: '>' > +| < LTEQ: '<=' > +| < GTEQ: '>=' > +| < NEQ: '!=' > +| < STAR: '*' > +| < EQ: '=' > +| < LIKE: 'like' > +| < CONTAINS: 'contains' > +| < NOTLIKE: 'not like' > +| < MATCHES: 'matches' > +| < NOTMATCHES: 'not matches' > + +// effectively unary operators +| < IS_NULL: 'is null' > +| < IS_NOT_NULL: 'is not null' > + +// dereference +| < DOT: '.' > +| < AT: '@' > + +// quotes +| < SQ: '\'' > +| < DQ: '"' > + +// statement delimiter +| < SEMI: ';' > + +| < TIMEOUT: 'timeout' > + +// identifier +| < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > +| < LONG_INT: ("-")?(["0"-"9"])+("L"|"l") > +| < INT: ("-")?(["0"-"9"])+ > +| < FLOAT: ("-")?(((["0"-"9"])+"."(["0"-"9"])* ()?)|("."(["0"-"9"])+ ()?)|((["0"-"9"])+ )) > +| < EXPONENT: ("e"|"E") ("+"|"-")? (["0"-"9"])+ > +| < DIGIT: ["0"-"9"] > +| < LETTER: ["a"-"z","A"-"Z"] > +// TODO: Add string +// .... +; + + +// -------------------------------------------------------------------------------- +// +// Production rules. +// +// -------------------------------------------------------------------------------- + +String identifierStr: + ( + + | + | + | + | + | + | + | + | + | + | + | + ) + { + return lastConsumedToken.toString(); + } +; + +program: + ( + (statement ?)* + ) +; + +statement: + ( + output_statement + ) +; + +output_statement: + ( + source_statement output_spec? + ) +; + +source_statement: + ( + query_statement ( pipeline_step)* + ) +; + +pipeline_step: + ( + namespaced_name (arguments(false))? | vespa_grouping + ) +; + +vespa_grouping_fun: + ( + vespa_grouping | annotation vespa_grouping + ) +; + +output_spec: + ( + ? ident + ) +; + +query_statement: + ( + select_statement + ) +; + +select_statement: + ( + select_field_spec //TODO: missing parsing + + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + | + ) + { + return lastConsumedToken.toString(); + } +; + +vespa_grouping_identifier: (