Skip to content

Commit

Permalink
Merge pull request #497 from usethesource/semantic-tokenizer-fall2024
Browse files Browse the repository at this point in the history 
Semantic tokenizer improvements: tests, simplification, syntax-in-syntax bugfix
  • Loading branch information
@DavyLandman
DavyLandman authored Nov 8, 2024
2 parents d53805a + 3f4e303 commit a23957c
Show file tree
Hide file tree
Showing 18 changed files with 662 additions and 134 deletions.
2 changes: 2 additions & 0 deletions rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ public interface ILanguageContributions {
public CompletableFuture<Boolean> hasImplementation();
public CompletableFuture<Boolean> hasCodeAction();

public CompletableFuture<Boolean> specialCaseHighlighting();

public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig();
public CompletableFuture<SummaryConfig> getBuilderSummaryConfig();
public CompletableFuture<SummaryConfig> getOndemandSummaryConfig();
Expand Down
25 changes: 21 additions & 4 deletions ...p/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ public class InterpretedLanguageContributions implements ILanguageContributions
private final CompletableFuture<Boolean> hasImplementation;
private final CompletableFuture<Boolean> hasCodeAction;

private final CompletableFuture<Boolean> specialCaseHighlighting;

private final CompletableFuture<SummaryConfig> analyzerSummaryConfig;
private final CompletableFuture<SummaryConfig> builderSummaryConfig;
private final CompletableFuture<SummaryConfig> ondemandSummaryConfig;
Expand Down Expand Up @@ -151,6 +153,10 @@ public InterpretedLanguageContributions(LanguageParameter lang, IBaseTextDocumen
this.hasImplementation = nonNull(this.implementation);
this.hasCodeAction = nonNull(this.codeAction);

this.specialCaseHighlighting = getContributionParameter(contributions,
LanguageContributions.PARSING,
LanguageContributions.Parameters.USES_SPECIAL_CASE_HIGHLIGHTING);

this.analyzerSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.ANALYSIS);
this.builderSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.BUILD);
this.ondemandSummaryConfig = ondemandSummaryConfig(contributions);
Expand All @@ -170,10 +176,10 @@ private static CompletableFuture<SummaryConfig> scheduledSummaryConfig(Completab
var constructor = getContribution(c, summarizer);
if (constructor != null) {
return new SummaryConfig(
isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_HOVERS),
isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_DEFINITIONS),
isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_REFERENCES),
isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_IMPLEMENTATIONS));
isTrue(constructor, LanguageContributions.Parameters.PROVIDES_HOVERS),
isTrue(constructor, LanguageContributions.Parameters.PROVIDES_DEFINITIONS),
isTrue(constructor, LanguageContributions.Parameters.PROVIDES_REFERENCES),
isTrue(constructor, LanguageContributions.Parameters.PROVIDES_IMPLEMENTATIONS));
} else {
return SummaryConfig.FALSY;
}
Expand Down Expand Up @@ -202,6 +208,12 @@ private static boolean hasContribution(ISet contributions, String name) {
return getContribution(contributions, name) != null;
}

private static CompletableFuture<Boolean> getContributionParameter(
CompletableFuture<ISet> contributions, String name, String parameter) {

return contributions.thenApply(c -> isTrue(getContribution(c, name), parameter));
}

private static boolean isTrue(@Nullable IConstructor constructor, String parameter) {
if (constructor == null) {
return false;
Expand Down Expand Up @@ -388,6 +400,11 @@ public CompletableFuture<Boolean> hasBuild() {
return hasBuild;
}

@Override
public CompletableFuture<Boolean> specialCaseHighlighting() {
return specialCaseHighlighting;
}

@Override
public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
return analyzerSummaryConfig;
Expand Down
12 changes: 12 additions & 0 deletions ...p/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ private static final <T> CompletableFuture<T> failedInitialization() {
private volatile CompletableFuture<Boolean> hasImplementation = failedInitialization();
private volatile CompletableFuture<Boolean> hasCodeAction = failedInitialization();

private volatile CompletableFuture<Boolean> specialCaseHighlighting = failedInitialization();

private volatile CompletableFuture<SummaryConfig> analyzerSummaryConfig;
private volatile CompletableFuture<SummaryConfig> builderSummaryConfig;
private volatile CompletableFuture<SummaryConfig> ondemandSummaryConfig;
Expand Down Expand Up @@ -159,6 +161,11 @@ private synchronized void calculateRouting() {
hasReferences = anyTrue(ILanguageContributions::hasReferences);
hasImplementation = anyTrue(ILanguageContributions::hasImplementation);

// Always use the special-case highlighting status of *the first*
// contribution (possibly using the default value in the Rascal ADT if
// it's not explicitly set), just as for `parsing` itself
specialCaseHighlighting = firstOrFail().specialCaseHighlighting();

analyzerSummaryConfig = anyTrue(ILanguageContributions::getAnalyzerSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
builderSummaryConfig = anyTrue(ILanguageContributions::getBuilderSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
ondemandSummaryConfig = anyTrue(ILanguageContributions::getOndemandSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
Expand Down Expand Up @@ -343,6 +350,11 @@ public CompletableFuture<Boolean> hasInlayHint() {
return hasInlayHint;
}

@Override
public CompletableFuture<Boolean> specialCaseHighlighting() {
return specialCaseHighlighting;
}

@Override
public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
return analyzerSummaryConfig;
Expand Down
3 changes: 2 additions & 1 deletion ...-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,9 +462,10 @@ public void shutdown() {
}

private CompletableFuture<SemanticTokens> getSemanticTokens(TextDocumentIdentifier doc) {
var specialCaseHighlighting = contributions(doc).specialCaseHighlighting();
return recoverExceptions(getFile(doc).getCurrentTreeAsync()
.thenApply(Versioned::get)
.thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter)
.thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter)
.whenComplete((r, e) ->
logger.trace("Semantic tokens success, reporting {} tokens back", r == null ? 0 : r.getData().size() / 5)
)
Expand Down
7 changes: 7 additions & 0 deletions rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class ParserOnlyContribution implements ILanguageContributions {
private final String name;
private final @Nullable Exception loadingParserError;
private final @Nullable IFunction parser;
private final CompletableFuture<Boolean> specialCaseHighlighting;

public ParserOnlyContribution(String name, ParserSpecification spec) {
this.name = name;
Expand All @@ -67,6 +68,7 @@ public ParserOnlyContribution(String name, ParserSpecification spec) {
Either<IFunction,Exception> result = loadParser(spec);
this.parser = result.getLeft();
this.loadingParserError = result.getRight();
this.specialCaseHighlighting = CompletableFuture.completedFuture(spec.getSpecialCaseHighlighting());
}

@Override
Expand Down Expand Up @@ -228,6 +230,11 @@ public CompletableFuture<Boolean> hasInlayHint() {
return CompletableFuture.completedFuture(false);
}

@Override
public CompletableFuture<Boolean> specialCaseHighlighting() {
return specialCaseHighlighting;
}

@Override
public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
return CompletableFuture.completedFuture(SummaryConfig.FALSY);
Expand Down
6 changes: 4 additions & 2 deletions rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,10 @@ private LanguageContributions () {}
public static final String IMPLEMENTATION = "implementation";
public static final String CODE_ACTION = "codeAction";

public static class Summarizers {
private Summarizers() {}
public static class Parameters {
private Parameters() {}

public static final String USES_SPECIAL_CASE_HIGHLIGHTING = "usesSpecialCaseHighlighting";

public static final String PROVIDES_HOVERS = "providesHovers";
public static final String PROVIDES_DEFINITIONS = "providesDefinitions";
Expand Down
3 changes: 2 additions & 1 deletion rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,10 @@ public void shutdown() {
}

private CompletableFuture<SemanticTokens> getSemanticTokens(TextDocumentIdentifier doc) {
var specialCaseHighlighting = CompletableFuture.completedFuture(false);
return getFile(doc).getCurrentTreeAsync()
.thenApply(Versioned::get)
.thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter)
.thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter)
.exceptionally(e -> {
logger.error("Tokenization failed", e);
return new SemanticTokens(Collections.emptyList());
Expand Down
10 changes: 9 additions & 1 deletion rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,13 +528,17 @@ public static class ParserSpecification {
private final @Nullable Boolean nonTerminalIsStart;
/** allowAmbiguity (default is false) */
private final @Nullable Boolean allowAmbiguity;
/** apply the special case for highlighting syntax-in-syntax, default: true */
private final @Nullable Boolean specialCaseHighlighting;


public ParserSpecification(String parserLocation, String nonTerminalName, @Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity) {
public ParserSpecification(String parserLocation, String nonTerminalName,
@Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity, @Nullable Boolean specialCaseHighlighting) {
this.parserLocation = parserLocation;
this.nonTerminalName = nonTerminalName;
this.nonTerminalIsStart = nonTerminalIsStart;
this.allowAmbiguity = allowAmbiguity;
this.specialCaseHighlighting = specialCaseHighlighting;
}

public ISourceLocation getParserLocation() throws FactTypeUseException {
Expand All @@ -553,6 +557,10 @@ public boolean getAllowAmbiguity() {
return allowAmbiguity != null && allowAmbiguity;
}

public boolean getSpecialCaseHighlighting() {
return specialCaseHighlighting == null || specialCaseHighlighting;
}

@Override
public String toString() {
return "ParserSpecification [parserLocation=" + parserLocation + ", nonTerminalName=" + nonTerminalName
Expand Down
Loading

0 comments on commit a23957c

Please sign in to comment.