diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java index 223156398af8..8a26a6eb2304 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java @@ -1,17 +1,24 @@ package ai.vespa.schemals; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.function.Predicate; import org.eclipse.lsp4j.Diagnostic; import org.eclipse.lsp4j.PublishDiagnosticsParams; import org.eclipse.lsp4j.services.LanguageClient; +import ai.vespa.schemals.common.SchemaDiagnostic.DiagnosticCode; + /** * SchemaDiagnosticHandler is a wrapper for publishing diagnostics to the client */ public class SchemaDiagnosticsHandler { private LanguageClient client; + // Represents the previous list of diagnostics sent to the client for a given document + private Map> documentDiagnostics = new HashMap<>(); public void connectClient(LanguageClient client) { this.client = client; @@ -19,14 +26,38 @@ public void connectClient(LanguageClient client) { public void clearDiagnostics(String fileURI) { publishDiagnostics(fileURI, new ArrayList()); + documentDiagnostics.remove(fileURI); } public void publishDiagnostics(String fileURI, List diagnostics) { + insertDocumentIfNotExists(fileURI); + client.publishDiagnostics( new PublishDiagnosticsParams( fileURI, diagnostics ) ); + + documentDiagnostics.get(fileURI).clear(); + documentDiagnostics.get(fileURI).addAll(diagnostics); + } + + public void replaceUndefinedSymbolDiagnostics(String fileURI, List unresolvedSymbolDiagnostics) { + Predicate undefinedSymbolPredicate = + (diagnostic) -> ( + diagnostic.getCode().isRight() + && diagnostic.getCode().getRight().equals(DiagnosticCode.UNDEFINED_SYMBOL.ordinal())); + + insertDocumentIfNotExists(fileURI); + documentDiagnostics.get(fileURI).removeIf(undefinedSymbolPredicate); + documentDiagnostics.get(fileURI).addAll(unresolvedSymbolDiagnostics.stream().filter(undefinedSymbolPredicate).toList()); + + publishDiagnostics(fileURI, List.copyOf(documentDiagnostics.get(fileURI))); + } + + private void insertDocumentIfNotExists(String fileURI) { + if (documentDiagnostics.containsKey(fileURI)) return; + documentDiagnostics.put(fileURI, new ArrayList<>()); } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java index d20644dc59b6..2608a2372479 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java @@ -26,7 +26,8 @@ public static enum DiagnosticCode { DEPRECATED_TOKEN_SUMMARY_TO, DEPRECATED_TOKEN_SEARCH, FEATURES_INHERITS_NON_PARENT, - FIELD_ARGUMENT_MISSING_INDEXING_TYPE + FIELD_ARGUMENT_MISSING_INDEXING_TYPE, + UNDEFINED_SYMBOL }; /** diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/InheritanceGraph.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/InheritanceGraph.java index 71758932b612..bdf9ea0db2df 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/InheritanceGraph.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/InheritanceGraph.java @@ -36,6 +36,10 @@ public SearchResult(NodeType node, ResultType result) { public InheritanceGraph() { } + // Note: does not remove childrenOfNode + // because if this node is added back we + // lazily remember the children, and access valid children through + // getValidChildren public void clearInheritsList(NodeType node) { if (!nodeExists(node)) return; @@ -45,7 +49,7 @@ public void clearInheritsList(NodeType node) { } parentsOfNode.remove(node); - childrenOfNode.remove(node); + } public void createNodeIfNotExists(NodeType node) { diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java index 49d3c7d429e9..5ad5eca50bef 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java @@ -4,8 +4,10 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Optional; import ai.vespa.schemals.common.ClientLogger; @@ -56,6 +58,7 @@ public class SchemaIndex { private Map> symbolDefinitions; private Map> symbolReferences; private Map definitionOfReference; + private Set unresolvedSymbols; // TODO: bad to use string as node type here. private InheritanceGraph documentInheritanceGraph; @@ -77,6 +80,7 @@ public SchemaIndex(ClientLogger logger) { this.symbolDefinitions = new HashMap<>(); this.symbolReferences = new HashMap<>(); this.definitionOfReference = new HashMap<>(); + this.unresolvedSymbols = new HashSet<>(); for (SymbolType type : SymbolType.values()) { this.symbolDefinitions.put(type, new ArrayList()); @@ -154,6 +158,9 @@ public void clearDocument(String fileURI) { } } + // Clear unresolved symbols from this document + this.unresolvedSymbols.removeIf(symbol -> symbol.fileURIEquals(fileURIURI)); + this.fieldIndex.clearFieldsByURI(fileURIURI); if (fileURI.endsWith(".sd")) { @@ -442,6 +449,18 @@ public void insertSymbolReference(Symbol definition, Symbol reference) { definitionOfReference.put(reference, definition); list.add(reference); + + if (reference.getType() == SymbolType.RANK_PROFILE && reference.getScope() != null) { + tryRegisterRankProfileInheritance(reference.getScope(), definition); + } + + if (reference.getType() == SymbolType.STRUCT && reference.getScope() != null) { + tryRegisterStructInheritance(reference.getScope(), definition); + } + + if (reference.getType() == SymbolType.DOCUMENT_SUMMARY && reference.getScope() != null) { + tryRegisterDocumentSummaryInheritance(reference.getScope(), definition); + } } @@ -538,6 +557,15 @@ public List listSymbolsInScope(Symbol scope, EnumSet types) return ret; } + public void addUnresolvedSymbol(Symbol unresolvedSymbol) { + unresolvedSymbols.add(unresolvedSymbol); + } + + public List getUnresolvedSymbols() { + unresolvedSymbols.removeIf(symbol -> getSymbolDefinition(symbol).isPresent()); + return List.copyOf(unresolvedSymbols); + } + /** * Dumps the index to the console. */ diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java index 3b1132e77000..3031f668aa59 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java @@ -2,6 +2,7 @@ import org.eclipse.lsp4j.VersionedTextDocumentIdentifier; +import ai.vespa.schemals.context.ParseContext; import ai.vespa.schemals.tree.SchemaNode; import ai.vespa.schemals.tree.YQLNode; @@ -17,6 +18,8 @@ public enum DocumentType { YQL } + public ParseContext getParseContext(); + public void updateFileContent(String content); public void updateFileContent(String content, Integer version); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java index 2904f5c124eb..73e4d9367dfd 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java @@ -53,9 +53,7 @@ public void updateFileContent(String content) { this.schemaIndex.clearDocument(this.fileURI); - ParseContext context = new ParseContext(content, logger, fileURI, schemaIndex, this.scheduler); - context.useRankProfileIdentifiers(); - var result = parseContent(context); + var result = parseContent(getParseContext()); diagnosticsHandler.publishDiagnostics(this.fileURI, result.diagnostics()); if (result.CST().isPresent()) { @@ -64,6 +62,13 @@ public void updateFileContent(String content) { } } + @Override + public ParseContext getParseContext() { + ParseContext context = new ParseContext(content, logger, fileURI, schemaIndex, this.scheduler); + context.useRankProfileIdentifiers(); + return context; + } + public static SchemaDocument.ParseResult parseContent(ParseContext context) { CharSequence sequence = context.content(); SchemaParser parserFaultTolerant = new SchemaParser(context.fileURI(), sequence); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java index d1cdd91fcf57..bc41a4ef23e0 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java @@ -75,7 +75,8 @@ public SchemaDocument(ClientLogger logger, SchemaDiagnosticsHandler diagnosticsH this.scheduler = scheduler; } - public ParseContext getParseContext(String content) { + @Override + public ParseContext getParseContext() { ParseContext context = new ParseContext(content, this.logger, this.fileURI, this.schemaIndex, this.scheduler); context.useDocumentIdentifiers(); return context; @@ -100,7 +101,7 @@ public void updateFileContent(String content) { schemaIndex.clearDocument(fileURI); logger.info("Parsing: " + fileURI); - ParseContext context = getParseContext(content); + ParseContext context = getParseContext(); var parsingResult = parseContent(context); parsingResult.diagnostics().addAll(verifyFileName()); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java index e38bedd2f15d..1026f4f9a913 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java @@ -3,12 +3,18 @@ import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import org.eclipse.lsp4j.Diagnostic; +import org.eclipse.lsp4j.DiagnosticSeverity; import org.eclipse.lsp4j.MessageType; import org.eclipse.lsp4j.TextDocumentItem; @@ -16,11 +22,15 @@ import ai.vespa.schemals.SchemaMessageHandler; import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.common.FileUtils; +import ai.vespa.schemals.common.SchemaDiagnostic; +import ai.vespa.schemals.common.SchemaDiagnostic.DiagnosticCode; +import ai.vespa.schemals.context.ParseContext; import ai.vespa.schemals.index.SchemaIndex; import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolType; import ai.vespa.schemals.schemadocument.DocumentManager.DocumentType; +import ai.vespa.schemals.schemadocument.resolvers.SymbolReferenceResolver; /** * Class responsible for maintaining the set of open documents and reparsing them. @@ -83,47 +93,96 @@ public void updateFile(String fileURI, String content, Integer version) { // TODO: a lot of parsing going on. It mostly should be reference resolving, not necessarily reparsing of entire contents. workspaceFiles.get(fileURI).updateFileContent(content, version); - boolean needsReparse = false; - if (documentType.get() == DocumentType.SCHEMA && reparseDescendants) { + if (reparseDescendants) { Set parsedURIs = new HashSet<>() {{ add(fileURI); }}; - for (String descendantURI : schemaIndex.getDocumentInheritanceGraph().getAllDescendants(fileURI)) { - if (descendantURI.equals(fileURI)) continue; + if (documentType.get() == DocumentType.SCHEMA) { - if (workspaceFiles.containsKey(descendantURI)) { - workspaceFiles.get(descendantURI).reparseContent(); - parsedURIs.add(descendantURI); + reparseSchemaFileDescendants(fileURI, parsedURIs); + + } else if (documentType.get() == DocumentType.PROFILE) { + // Find the schema this rank profile belongs to and reparse from there + RankProfileDocument document = getRankProfileDocument(fileURI); + Optional schemaSymbol = document.schemaSymbol(); + + if (schemaSymbol.isPresent()) { + reparseSchemaFileDescendants(schemaSymbol.get().getFileURI(), parsedURIs); } } + } + + // Resolve remaining unresolved symbols after everything has parsed + Map> undefinedSymbolDiagnostics = new HashMap<>(); + while (true) { + boolean didResolve = false; + for (var symbol : schemaIndex.getUnresolvedSymbols()) { + String symbolURI = symbol.getFileURI(); + + DocumentManager symbolDocument = getDocument(symbolURI); + if (symbolDocument == null) continue; // bad situation + + if (!undefinedSymbolDiagnostics.containsKey(symbolURI))undefinedSymbolDiagnostics.put(symbolURI, new ArrayList<>()); - // Reparse documents that holds references to this document - String schemaIdentifier = ((SchemaDocument)workspaceFiles.get(fileURI)).getSchemaIdentifier(); - Optional documentDefinition = schemaIndex.findSymbol(null, SymbolType.DOCUMENT, schemaIdentifier); - - if (documentDefinition.isPresent()) { - for (Symbol referencesThisDocument : schemaIndex.getDocumentReferenceGraph().getAllDescendants(documentDefinition.get())) { - String descendantURI = referencesThisDocument.getFileURI(); - if (!parsedURIs.contains(descendantURI) && workspaceFiles.containsKey(descendantURI)) { - workspaceFiles.get(referencesThisDocument.getFileURI()).reparseContent(); - parsedURIs.add(descendantURI); - } + ParseContext context = symbolDocument.getParseContext(); + List diagnostics = new ArrayList<>(); + SymbolReferenceResolver.resolveSymbolReference(symbol.getNode(), context, diagnostics); + if (diagnostics.isEmpty()) { + didResolve = true; + break; } } + if (!didResolve) break; + } - // reparse rank profile files belonging to this document - for (var entry : workspaceFiles.entrySet()) { - if ((entry.getValue() instanceof RankProfileDocument)) { - RankProfileDocument document = (RankProfileDocument)entry.getValue(); - if (document.schemaSymbol().isPresent() && document.schemaSymbol().get().fileURIEquals(fileURI)) { - entry.getValue().reparseContent(); - needsReparse = true; - } - } + + for (var symbol : schemaIndex.getUnresolvedSymbols()) { + undefinedSymbolDiagnostics.get(symbol.getFileURI()).add( + new SchemaDiagnostic.Builder() + .setRange( symbol.getNode().getRange()) + .setMessage( "Undefined symbol " + symbol.getNode().getText()) + .setSeverity(DiagnosticSeverity.Error) + .setCode(DiagnosticCode.UNDEFINED_SYMBOL) + .build() + ); + } + + for (var entry : undefinedSymbolDiagnostics.entrySet()) { + diagnosticsHandler.replaceUndefinedSymbolDiagnostics(entry.getKey(), entry.getValue()); + } + } + + private void reparseSchemaFileDescendants(String fileURI, Set parsedURIs) { + for (String descendantURI : schemaIndex.getDocumentInheritanceGraph().getAllDescendants(fileURI)) { + if (parsedURIs.contains(descendantURI)) continue; + if (workspaceFiles.containsKey(descendantURI)) { + workspaceFiles.get(descendantURI).reparseContent(); + parsedURIs.add(descendantURI); } } - if (needsReparse) { - workspaceFiles.get(fileURI).reparseContent(); + // Reparse documents that holds references to this document + String schemaIdentifier = ((SchemaDocument)workspaceFiles.get(fileURI)).getSchemaIdentifier(); + Optional documentDefinition = schemaIndex.findSymbol(null, SymbolType.DOCUMENT, schemaIdentifier); + + if (documentDefinition.isPresent()) { + for (Symbol referencesThisDocument : schemaIndex.getDocumentReferenceGraph().getAllDescendants(documentDefinition.get())) { + String descendantURI = referencesThisDocument.getFileURI(); + if (!parsedURIs.contains(descendantURI) && workspaceFiles.containsKey(descendantURI)) { + workspaceFiles.get(referencesThisDocument.getFileURI()).reparseContent(); + parsedURIs.add(descendantURI); + } + } + } + // reparse rank profile files belonging to this document + for (var entry : workspaceFiles.entrySet()) { + if (parsedURIs.contains(entry.getKey())) continue; + if ((entry.getValue() instanceof RankProfileDocument)) { + RankProfileDocument document = (RankProfileDocument)entry.getValue(); + if (document.schemaSymbol().isPresent() && document.schemaSymbol().get().fileURIEquals(fileURI)) { + entry.getValue().reparseContent(); + parsedURIs.add(entry.getKey()); + } + } } } @@ -180,6 +239,10 @@ public void closeDocument(String fileURI) { if (!file.exists()) { cleanUpDocument(fileURI); } + + if (!isInWorkspace(fileURI)) { + diagnosticsHandler.clearDiagnostics(fileURI); + } } private void cleanUpDocument(String fileURI) { @@ -187,6 +250,7 @@ private void cleanUpDocument(String fileURI) { schemaIndex.clearDocument(fileURI); workspaceFiles.remove(fileURI); + diagnosticsHandler.clearDiagnostics(fileURI); } public boolean removeDocument(String fileURI) { @@ -253,4 +317,11 @@ public void setupWorkspace(URI workspaceURI) { reparseInInheritanceOrder(); setReparseDescendants(true); } + + private boolean isInWorkspace(String fileURI) { + if (workspaceURI == null) return false; + Path filePath = Paths.get(URI.create(fileURI)); + Path workspacePath = Paths.get(workspaceURI); + return filePath.startsWith(workspacePath); + } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 9f6497fb839c..1d24fe2f418f 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -4,6 +4,7 @@ import ai.vespa.schemals.SchemaDiagnosticsHandler; import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.context.ParseContext; import ai.vespa.schemals.parser.yqlplus.Node; import ai.vespa.schemals.parser.yqlplus.ParseException; import ai.vespa.schemals.parser.yqlplus.YQLPlusParser; @@ -28,6 +29,11 @@ public class YQLDocument implements DocumentManager { this.diagnosticsHandler = diagnosticsHandler; } + @Override + public ParseContext getParseContext() { + return null; + } + @Override public void updateFileContent(String content) { fileContent = content; diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java index f65b4a3aacba..64f77a839a64 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java @@ -181,11 +181,12 @@ public static void resolveSymbolReference(SchemaNode node, ParseContext context, context.schemaIndex().insertSymbolReference(referencedSymbol.get(), node.getSymbol()); } else if (referencedType != SymbolType.QUERY_INPUT) { - + context.schemaIndex().addUnresolvedSymbol(node.getSymbol()); diagnostics.add(new SchemaDiagnostic.Builder() .setRange( node.getRange()) .setMessage( "Undefined symbol " + node.getText()) .setSeverity(DiagnosticSeverity.Error) + .setCode(DiagnosticCode.UNDEFINED_SYMBOL) .build() ); } }