feat: resolve more symbols after each parse

vespa-engine · Nov 1, 2024 · 092ae74 · 092ae74
1 parent cf347a6
commit 092ae74
Show file tree

Hide file tree

Showing 9 changed files with 118 additions and 7 deletions.
diff --git a/...uage-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java b/...uage-server/language-server/src/main/java/ai/vespa/schemals/SchemaDiagnosticsHandler.java
@@ -1,32 +1,63 @@
 package ai.vespa.schemals;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
 
 import org.eclipse.lsp4j.Diagnostic;
 import org.eclipse.lsp4j.PublishDiagnosticsParams;
 import org.eclipse.lsp4j.services.LanguageClient;
 
+import ai.vespa.schemals.common.SchemaDiagnostic.DiagnosticCode;
+
 /**
  * SchemaDiagnosticHandler is a wrapper for publishing diagnostics to the client
  */
 public class SchemaDiagnosticsHandler {
     private LanguageClient client;
+    // Represents the previous list of diagnostics sent to the client for a given document
+    private Map<String, List<Diagnostic>> documentDiagnostics = new HashMap<>();
 
     public void connectClient(LanguageClient client) {
         this.client = client;
     }
 
     public void clearDiagnostics(String fileURI) {
         publishDiagnostics(fileURI, new ArrayList<Diagnostic>());
+        documentDiagnostics.remove(fileURI);
     }
 
     public void publishDiagnostics(String fileURI, List<Diagnostic> diagnostics) {
+        insertDocumentIfNotExists(fileURI);
+
         client.publishDiagnostics(
             new PublishDiagnosticsParams(
                 fileURI,
                 diagnostics
             )
         );
+
+        documentDiagnostics.get(fileURI).clear();
+        documentDiagnostics.get(fileURI).addAll(diagnostics);
+    }
+
+    public void replaceUndefinedSymbolDiagnostics(String fileURI, List<Diagnostic> unresolvedSymbolDiagnostics) {
+        Predicate<Diagnostic> undefinedSymbolPredicate = 
+            (diagnostic) -> (
+                diagnostic.getCode().isRight() 
+                && diagnostic.getCode().getRight().equals(DiagnosticCode.UNDEFINED_SYMBOL.ordinal()));
+
+        insertDocumentIfNotExists(fileURI);
+        documentDiagnostics.get(fileURI).removeIf(undefinedSymbolPredicate);
+        documentDiagnostics.get(fileURI).addAll(unresolvedSymbolDiagnostics.stream().filter(undefinedSymbolPredicate).toList());
+
+        publishDiagnostics(fileURI, List.copyOf(documentDiagnostics.get(fileURI)));
+    }
+
+    private void insertDocumentIfNotExists(String fileURI) {
+        if (documentDiagnostics.containsKey(fileURI)) return;
+        documentDiagnostics.put(fileURI, new ArrayList<>());
     }
 }
diff --git a/...guage-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java b/...guage-server/language-server/src/main/java/ai/vespa/schemals/common/SchemaDiagnostic.java
@@ -26,7 +26,8 @@ public static enum DiagnosticCode {
         DEPRECATED_TOKEN_SUMMARY_TO,
         DEPRECATED_TOKEN_SEARCH,
         FEATURES_INHERITS_NON_PARENT,
-        FIELD_ARGUMENT_MISSING_INDEXING_TYPE
+        FIELD_ARGUMENT_MISSING_INDEXING_TYPE,
+        UNDEFINED_SYMBOL
     };
 
     /**

diff --git a/...ma-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java b/...ma-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java
@@ -4,8 +4,10 @@
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Optional;
 
 import ai.vespa.schemals.common.ClientLogger;
@@ -56,6 +58,7 @@ public class SchemaIndex {
     private Map<SymbolType, List<Symbol>> symbolDefinitions;
     private Map<Symbol, List<Symbol>> symbolReferences;
     private Map<Symbol, Symbol> definitionOfReference;
+    private Set<Symbol> unresolvedSymbols;
 
     // TODO: bad to use string as node type here.
     private InheritanceGraph<String> documentInheritanceGraph;
@@ -77,6 +80,7 @@ public SchemaIndex(ClientLogger logger) {
         this.symbolDefinitions     = new HashMap<>();
         this.symbolReferences      = new HashMap<>();
         this.definitionOfReference = new HashMap<>();
+        this.unresolvedSymbols     = new HashSet<>();
 
         for (SymbolType type : SymbolType.values()) {
             this.symbolDefinitions.put(type, new ArrayList<Symbol>());
@@ -154,6 +158,9 @@ public void clearDocument(String fileURI) {
             }
         }
 
+        // Clear unresolved symbols from this document
+        this.unresolvedSymbols.removeIf(symbol -> symbol.fileURIEquals(fileURIURI));
+
         this.fieldIndex.clearFieldsByURI(fileURIURI);
 
         if (fileURI.endsWith(".sd")) {
@@ -538,6 +545,15 @@ public List<Symbol> listSymbolsInScope(Symbol scope, EnumSet<SymbolType> types)
         return ret;
     }
 
+    public void addUnresolvedSymbol(Symbol unresolvedSymbol) {
+        unresolvedSymbols.add(unresolvedSymbol);
+    }
+
+    public List<Symbol> getUnresolvedSymbols() {
+        unresolvedSymbols.removeIf(symbol -> getSymbolDefinition(symbol).isPresent());
+        return List.copyOf(unresolvedSymbols);
+    }
+
     /**
      * Dumps the index to the console.
      */

diff --git a/...erver/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java b/...erver/language-server/src/main/java/ai/vespa/schemals/schemadocument/DocumentManager.java
@@ -2,6 +2,7 @@
 
 import org.eclipse.lsp4j.VersionedTextDocumentIdentifier;
 
+import ai.vespa.schemals.context.ParseContext;
 import ai.vespa.schemals.tree.SchemaNode;
 import ai.vespa.schemals.tree.YQLNode;
 
@@ -17,6 +18,8 @@ public enum DocumentType {
         YQL
     }
 
+    public ParseContext getParseContext();
+
     public void updateFileContent(String content);
     public void updateFileContent(String content, Integer version);
 

diff --git a/...r/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java b/...r/language-server/src/main/java/ai/vespa/schemals/schemadocument/RankProfileDocument.java
@@ -53,9 +53,7 @@ public void updateFileContent(String content) {
 
         this.schemaIndex.clearDocument(this.fileURI);
 
-        ParseContext context = new ParseContext(content, logger, fileURI, schemaIndex, this.scheduler);
-        context.useRankProfileIdentifiers();
-        var result = parseContent(context);
+        var result = parseContent(getParseContext());
 
         diagnosticsHandler.publishDiagnostics(this.fileURI, result.diagnostics());
         if (result.CST().isPresent()) {
@@ -64,6 +62,13 @@ public void updateFileContent(String content) {
         }
     }
 
+    @Override
+    public ParseContext getParseContext() {
+        ParseContext context = new ParseContext(content, logger, fileURI, schemaIndex, this.scheduler);
+        context.useRankProfileIdentifiers();
+        return context;
+    }
+
     public static SchemaDocument.ParseResult parseContent(ParseContext context) {
         CharSequence sequence = context.content();
         SchemaParser parserFaultTolerant = new SchemaParser(context.fileURI(), sequence);

diff --git a/...server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java b/...server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java
@@ -75,7 +75,8 @@ public SchemaDocument(ClientLogger logger, SchemaDiagnosticsHandler diagnosticsH
         this.scheduler = scheduler;
     }
 
-    public ParseContext getParseContext(String content) {
+    @Override
+    public ParseContext getParseContext() {
         ParseContext context = new ParseContext(content, this.logger, this.fileURI, this.schemaIndex, this.scheduler);
         context.useDocumentIdentifiers();
         return context;
@@ -100,7 +101,7 @@ public void updateFileContent(String content) {
         schemaIndex.clearDocument(fileURI);
 
         logger.info("Parsing: " + fileURI);
-        ParseContext context = getParseContext(content);
+        ParseContext context = getParseContext();
         var parsingResult = parseContent(context);
 
         parsingResult.diagnostics().addAll(verifyFileName());

diff --git a/...nguage-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java b/...nguage-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java
@@ -3,24 +3,32 @@
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 
+import org.eclipse.lsp4j.Diagnostic;
+import org.eclipse.lsp4j.DiagnosticSeverity;
 import org.eclipse.lsp4j.MessageType;
 import org.eclipse.lsp4j.TextDocumentItem;
 
 import ai.vespa.schemals.SchemaDiagnosticsHandler;
 import ai.vespa.schemals.SchemaMessageHandler;
 import ai.vespa.schemals.common.ClientLogger;
 import ai.vespa.schemals.common.FileUtils;
+import ai.vespa.schemals.common.SchemaDiagnostic;
+import ai.vespa.schemals.common.SchemaDiagnostic.DiagnosticCode;
+import ai.vespa.schemals.context.ParseContext;
 import ai.vespa.schemals.index.SchemaIndex;
 import ai.vespa.schemals.index.Symbol;
 import ai.vespa.schemals.index.Symbol.SymbolType;
 
 import ai.vespa.schemals.schemadocument.DocumentManager.DocumentType;
+import ai.vespa.schemals.schemadocument.resolvers.SymbolReferenceResolver;
 
 /**
  * Class responsible for maintaining the set of open documents and reparsing them.
@@ -125,6 +133,45 @@ public void updateFile(String fileURI, String content, Integer version) {
         if (needsReparse) {
             workspaceFiles.get(fileURI).reparseContent();
         }
+
+        // Resolve remaining unresolved symbols after everything has parsed
+        Map<String, List<Diagnostic>> undefinedSymbolDiagnostics = new HashMap<>();
+        while (true) {
+            boolean didResolve = false;
+            for (var symbol : schemaIndex.getUnresolvedSymbols()) {
+                String symbolURI = symbol.getFileURI();
+
+                DocumentManager symbolDocument = getDocument(symbolURI);
+                if (symbolDocument == null) continue; // bad situation
+
+                if (!undefinedSymbolDiagnostics.containsKey(symbolURI))undefinedSymbolDiagnostics.put(symbolURI, new ArrayList<>());
+
+                ParseContext context = symbolDocument.getParseContext();
+                List<Diagnostic> diagnostics = new ArrayList<>();
+                SymbolReferenceResolver.resolveSymbolReference(symbol.getNode(), context, diagnostics);
+                if (diagnostics.isEmpty()) {
+                    didResolve = true;
+                    break;
+                }
+            }
+            if (!didResolve) break;
+        }
+
+
+        for (var symbol : schemaIndex.getUnresolvedSymbols()) {
+            undefinedSymbolDiagnostics.get(symbol.getFileURI()).add(
+            new SchemaDiagnostic.Builder()
+                    .setRange( symbol.getNode().getRange())
+                    .setMessage( "Undefined symbol " + symbol.getNode().getText())
+                    .setSeverity(DiagnosticSeverity.Error)
+                    .setCode(DiagnosticCode.UNDEFINED_SYMBOL)
+                    .build()
+            );
+        }
+
+        for (var entry : undefinedSymbolDiagnostics.entrySet()) {
+            diagnosticsHandler.replaceUndefinedSymbolDiagnostics(entry.getKey(), entry.getValue());
+        }
     }
 
     public String getWorkspaceURI() {

diff --git a/...ge-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/...ge-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java
@@ -4,6 +4,7 @@
 
 import ai.vespa.schemals.SchemaDiagnosticsHandler;
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.context.ParseContext;
 import ai.vespa.schemals.parser.yqlplus.Node;
 import ai.vespa.schemals.parser.yqlplus.ParseException;
 import ai.vespa.schemals.parser.yqlplus.YQLPlusParser;
@@ -28,6 +29,11 @@ public class YQLDocument implements DocumentManager {
         this.diagnosticsHandler = diagnosticsHandler;
     }
 
+    @Override
+    public ParseContext getParseContext() {
+        return null;
+    }
+
     @Override
     public void updateFileContent(String content) {
         fileContent = content;

diff --git a/...ver/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java b/...ver/src/main/java/ai/vespa/schemals/schemadocument/resolvers/SymbolReferenceResolver.java
@@ -181,11 +181,12 @@ public static void resolveSymbolReference(SchemaNode node, ParseContext context,
             context.schemaIndex().insertSymbolReference(referencedSymbol.get(), node.getSymbol());
 
         } else if (referencedType != SymbolType.QUERY_INPUT)  {
-
+            context.schemaIndex().addUnresolvedSymbol(node.getSymbol());
             diagnostics.add(new SchemaDiagnostic.Builder()
                     .setRange( node.getRange())
                     .setMessage( "Undefined symbol " + node.getText())
                     .setSeverity(DiagnosticSeverity.Error)
+                    .setCode(DiagnosticCode.UNDEFINED_SYMBOL)
                     .build() );
         }
     }