Skip to content

Commit

Permalink
Merge pull request #557 from sourcegraph/olafurpg/scip-gap
Browse files Browse the repository at this point in the history 
Close SCIP/LSIF feature gap
  • Loading branch information
@olafurpg
olafurpg authored Apr 20, 2023
2 parents dbb7d6a + 2bc9b3e commit bf4ca0c
Show file tree
Hide file tree
Showing 19 changed files with 718 additions and 453 deletions.
177 changes: 106 additions & 71 deletions scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipSemanticdb.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ private void runTyped(List<Path> files, PackageTable packages) {
}

private String typedSymbol(String symbol, Package pkg) {
if (symbol.isEmpty()) {
return "";
}
if (symbol.startsWith("local")) {
return "local " + symbol.substring("local".length());
}
Expand All @@ -91,6 +94,9 @@ private void processTypedDocument(Path path, PackageTable packages) {
.collect(Collectors.joining("/"));
Scip.Document.Builder tdoc = Scip.Document.newBuilder().setRelativePath(relativePath);
for (SymbolOccurrence occ : doc.sortedSymbolOccurrences()) {
if (occ.getSymbol().isEmpty()) {
continue;
}
int role = 0;
if (isDefinitionRole(occ.getRole())) {
role |= Scip.SymbolRole.Definition_VALUE;
Expand All @@ -116,12 +122,35 @@ private void processTypedDocument(Path path, PackageTable packages) {
}
Symtab symtab = new Symtab(doc.semanticdb);
for (SymbolInformation info : doc.semanticdb.getSymbolsList()) {
if (info.getSymbol().isEmpty()) {
continue;
}
Package pkg = packages.packageForSymbol(info.getSymbol()).orElse(Package.EMPTY);
Scip.SymbolInformation.Builder tinfo =
Scip.SymbolInformation.newBuilder().setSymbol(typedSymbol(info.getSymbol(), pkg));

for (int i = 0; i < info.getDefinitionRelationshipsCount(); i++) {
String definitionSymbol = info.getDefinitionRelationships(i);
if (definitionSymbol.isEmpty()) {
continue;
}
Package definitionSymbolPkg =
packages.packageForSymbol(definitionSymbol).orElse(Package.EMPTY);
SymbolInformation definitionInfo = symtab.symbols.get(definitionSymbol);
tinfo.addRelationships(
Scip.Relationship.newBuilder()
.setSymbol(typedSymbol(definitionSymbol, definitionSymbolPkg))
.setIsDefinition(true)
.setIsReference(
definitionInfo != null
&& definitionInfo.getDisplayName().equals(info.getDisplayName())
&& supportsReferenceRelationship(info)));
}
for (int i = 0; i < info.getOverriddenSymbolsCount(); i++) {
String overriddenSymbol = info.getOverriddenSymbols(i);
if (overriddenSymbol.isEmpty()) {
continue;
}
if (isIgnoredOverriddenSymbol(overriddenSymbol)) {
continue;
}
Expand Down Expand Up @@ -225,82 +254,85 @@ private Integer processDocumentUnsafe(
Set<Integer> rangeIds = new LinkedHashSet<>();

for (SymbolOccurrence occ : doc.sortedSymbolOccurrences()) {
SymbolInformation symbolInformation =
doc.symbols.getOrDefault(occ.getSymbol(), SymbolInformation.getDefaultInstance());
ResultIds ids = results.getOrInsertResultSet(occ.getSymbol());
int rangeId = writer.emitRange(occ.getRange());
rangeIds.add(rangeId);

// Range
if (occ.getRole() != Role.SYNTHETIC_DEFINITION) {
writer.emitNext(rangeId, ids.resultSet);
}

// Reference
writer.emitItem(ids.referenceResult, rangeId, doc.id);

// Definition
if (isDefinitionRole(occ.getRole())) {
if (ids.isDefinitionDefined()) {
writer.emitItem(ids.definitionResult, rangeId, doc.id);
} else {
options.reporter.error(
new NoSuchElementException(
String.format("no definition ID for symbol '%s'", occ.getSymbol())));
for (String symbol : occ.getSymbol().split(";")) {
SymbolInformation symbolInformation =
doc.symbols.getOrDefault(symbol, SymbolInformation.getDefaultInstance());
ResultIds ids = results.getOrInsertResultSet(symbol);
int rangeId = writer.emitRange(occ.getRange());
rangeIds.add(rangeId);

// Range
if (occ.getRole() != Role.SYNTHETIC_DEFINITION) {
writer.emitNext(rangeId, ids.resultSet);
}

// Hover 1: signature
String documentation = symbolInformation.getDocumentation().getMessage();
StringBuilder markupContent = new StringBuilder(documentation.length());
if (symbolInformation.hasSignature()) {
String language =
doc.semanticdb.getLanguage().toString().toLowerCase(Locale.ROOT).intern();
String signature = new SignatureFormatter(symbolInformation, symtab).formatSymbol();
markupContent
.append("```")
.append(language)
.append('\n')
.append(signature)
.append("\n```");
}
// Reference
writer.emitItem(ids.referenceResult, rangeId, doc.id);

// Hover 2: docstring
if (!documentation.isEmpty()) {
if (markupContent.length() != 0) markupContent.append("\n---\n");
markupContent.append(documentation.replaceAll("\n", "\n\n"));
}
// Definition
if (isDefinitionRole(occ.getRole())) {
if (ids.isDefinitionDefined()) {
writer.emitItem(ids.definitionResult, rangeId, doc.id);
} else {
options.reporter.error(
new NoSuchElementException(
String.format("no definition ID for symbol '%s'", symbol)));
}

if (markupContent.length() == 0) {
// Always emit a non-empty hover message to prevent Sourcegraph from falling back to
// Search-Based hover messages.
markupContent.append(symbolInformation.getDisplayName());
}
// Hover 1: signature
String documentation = symbolInformation.getDocumentation().getMessage();
StringBuilder markupContent = new StringBuilder(documentation.length());
if (symbolInformation.hasSignature()) {
String language =
doc.semanticdb.getLanguage().toString().toLowerCase(Locale.ROOT).intern();
String signature = new SignatureFormatter(symbolInformation, symtab).formatSymbol();
markupContent
.append("```")
.append(language)
.append('\n')
.append(signature)
.append("\n```");
}

int hoverId =
writer.emitHoverResult(
new MarkupContent(MarkupKind.MARKDOWN, markupContent.toString()));
writer.emitHoverEdge(ids.resultSet, hoverId);
}
// Hover 2: docstring
if (!documentation.isEmpty()) {
if (markupContent.length() != 0) markupContent.append("\n---\n");
markupContent.append(documentation.replaceAll("\n", "\n\n"));
}

// Overrides
if (symbolInformation.getOverriddenSymbolsCount() > 0
&& supportsReferenceRelationship(symbolInformation)
&& occ.getRole() == Role.DEFINITION) {
List<Integer> overriddenReferenceResultIds =
new ArrayList<>(symbolInformation.getOverriddenSymbolsCount());
for (int i = 0; i < symbolInformation.getOverriddenSymbolsCount(); i++) {
String overriddenSymbol = symbolInformation.getOverriddenSymbols(i);
if (isIgnoredOverriddenSymbol(overriddenSymbol)) {
continue;
if (markupContent.length() == 0) {
// Always emit a non-empty hover message to prevent Sourcegraph from falling
// back to
// Search-Based hover messages.
markupContent.append(symbolInformation.getDisplayName());
}
ResultIds overriddenIds = results.getOrInsertResultSet(overriddenSymbol);
overriddenReferenceResultIds.add(overriddenIds.referenceResult);
writer.emitReferenceResultsItemEdge(
overriddenIds.referenceResult, Collections.singletonList(rangeId), doc.id);

int hoverId =
writer.emitHoverResult(
new MarkupContent(MarkupKind.MARKDOWN, markupContent.toString()));
writer.emitHoverEdge(ids.resultSet, hoverId);
}
if (overriddenReferenceResultIds.size() > 0) {
writer.emitReferenceResultsItemEdge(
ids.referenceResult, overriddenReferenceResultIds, doc.id);

// Overrides
if (symbolInformation.getOverriddenSymbolsCount() > 0
&& supportsReferenceRelationship(symbolInformation)
&& occ.getRole() == Role.DEFINITION) {
List<Integer> overriddenReferenceResultIds =
new ArrayList<>(symbolInformation.getOverriddenSymbolsCount());
for (int i = 0; i < symbolInformation.getOverriddenSymbolsCount(); i++) {
String overriddenSymbol = symbolInformation.getOverriddenSymbols(i);
if (isIgnoredOverriddenSymbol(overriddenSymbol)) {
continue;
}
ResultIds overriddenIds = results.getOrInsertResultSet(overriddenSymbol);
overriddenReferenceResultIds.add(overriddenIds.referenceResult);
writer.emitReferenceResultsItemEdge(
overriddenIds.referenceResult, Collections.singletonList(rangeId), doc.id);
}
if (overriddenReferenceResultIds.size() > 0) {
writer.emitReferenceResultsItemEdge(
ids.referenceResult, overriddenReferenceResultIds, doc.id);
}
}
}
}
Expand Down Expand Up @@ -363,16 +395,19 @@ private Semanticdb.TextDocuments textDocumentsParseFromBytes(byte[] bytes) throw
in.setRecursionLimit(1000);
return Semanticdb.TextDocuments.parseFrom(in);
} catch (NoSuchMethodError ignored) {
// NOTE(olafur): For some reason, NoSuchMethodError gets thrown when running `snapshots/run`
// in the sbt build. I'm unable to reproduce the error in `snapshots/test` or when running the
// NOTE(olafur): For some reason, NoSuchMethodError gets thrown when running
// `snapshots/run`
// in the sbt build. I'm unable to reproduce the error in `snapshots/test` or
// when running the
// published version
// of `scip-java index`.
return Semanticdb.TextDocuments.parseFrom(bytes);
}
}

private boolean isIgnoredOverriddenSymbol(String symbol) {
// Skip java/lang/Object# and similar symbols from Scala since it's the parent of all classes
// Skip java/lang/Object# and similar symbols from Scala since it's the parent
// of all classes
// making it noisy for "find implementations" results.
return symbol.equals("java/lang/Object#");
}
Expand Down
10 changes: 5 additions & 5 deletions scip-semanticdb/src/main/java/com/sourcegraph/scip_semanticdb/ScipTextDocument.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,29 +87,29 @@ public static Semanticdb.TextDocument manifestOccurrencesForSyntheticSymbols(
return semanticdb;
}
Semanticdb.TextDocument.Builder builder = Semanticdb.TextDocument.newBuilder(semanticdb);
builder.clearSymbols();
HashMap<String, Semanticdb.SymbolOccurrence> definitionOccurrences = new HashMap<>();
for (Semanticdb.SymbolOccurrence occ : semanticdb.getOccurrencesList()) {
if (occ.getRole() == Semanticdb.SymbolOccurrence.Role.DEFINITION) {
definitionOccurrences.put(occ.getSymbol(), occ);
}
}
for (Semanticdb.SymbolInformation info : semanticdb.getSymbolsList()) {
Semanticdb.SymbolInformation.Builder newInfo = Semanticdb.SymbolInformation.newBuilder(info);
Semanticdb.SymbolOccurrence definition = definitionOccurrences.get(info.getSymbol());
if (definition != null) {
builder.addSymbols(newInfo);
continue;
}
for (Semanticdb.SymbolOccurrence alternativeSymbol : alternativeSymbols(info)) {
Semanticdb.SymbolOccurrence alternativeDefinition =
definitionOccurrences.get(alternativeSymbol.getSymbol());
if (alternativeDefinition != null) {
builder.addOccurrences(
Semanticdb.SymbolOccurrence.newBuilder()
.setRange(alternativeDefinition.getRange())
.setRole(alternativeSymbol.getRole())
.setSymbol(info.getSymbol()));
newInfo.addDefinitionRelationships(alternativeDefinition.getSymbol());
break;
}
}
builder.addSymbols(newInfo);
}
return builder.build();
}
Expand Down
28 changes: 17 additions & 11 deletions tests/snapshots/src/main/generated/BaseByteRenderer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,31 +24,35 @@ import upickle.core.{ArrVisitor, ObjVisitor}
class BaseByteRenderer[T <: upickle.core.ByteOps.Output]
// ^^^^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#
// documentation ```scala\nclass BaseByteRenderer[T <: Output]\n```
// ^^^^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer.
// ________________ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer.
// documentation ```scala\nobject BaseByteRenderer\n```
// relationship is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#
// ^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#[T]
// documentation ```scala\nT <: Output\n```
// ^^^^^^^ reference semanticdb maven . . upickle/
// ^^^^ reference semanticdb maven . . upickle/core/
// ^^^^^^^ reference semanticdb maven maven/com.lihaoyi/upickle-core_2.13 1.4.0 upickle/core/ByteOps.
// ^^^^^^ reference semanticdb maven maven/com.lihaoyi/upickle-core_2.13 1.4.0 upickle/core/ByteOps.Output#
(out: T,
// ^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(out)
// documentation ```scala\nout: T \n```
// ^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#out.
// documentation ```scala\nprivate[this] val out: T\n```
// ___ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(out)
// documentation ```scala\nout: T \n```
// relationship is_reference is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#out.
// ^ reference semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#[T]
indent: Int = -1,
// ^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(indent)
// documentation ```scala\ndefault indent: Int \n```
// ^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#indent.
// documentation ```scala\nprivate[this] val indent: Int\n```
// ______ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(indent)
// documentation ```scala\ndefault indent: Int \n```
// relationship is_reference is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#indent.
// ^^^ reference semanticdb maven maven/org.scala-lang/scala-library 2.13.10 scala/Int#
escapeUnicode: Boolean = false) extends JsVisitor[T, T]{
// ^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(escapeUnicode)
// documentation ```scala\ndefault escapeUnicode: Boolean \n```
// ^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#escapeUnicode.
// documentation ```scala\nprivate[this] val escapeUnicode: Boolean\n```
// _____________ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`<init>`().(escapeUnicode)
// documentation ```scala\ndefault escapeUnicode: Boolean \n```
// relationship is_reference is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#escapeUnicode.
// ^^^^^^^ reference semanticdb maven maven/org.scala-lang/scala-library 2.13.10 scala/Boolean#
// ^^^^^^^^^ reference semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/JsVisitor#
// ^ reference semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#[T]
Expand Down Expand Up @@ -80,18 +84,20 @@ class BaseByteRenderer[T <: upickle.core.ByteOps.Output]
}

private[this] var depth: Int = 0
// ^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`depth_=`().
// documentation ```scala\nprivate[this] var depth_=(x$1: Int): Unit\n```
// ^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#depth().
// documentation ```scala\nprivate[this] var depth: Int\n```
// _____ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`depth_=`().
// documentation ```scala\nprivate[this] var depth_=(x$1: Int): Unit\n```
// relationship is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#depth().
// ^^^ reference semanticdb maven maven/org.scala-lang/scala-library 2.13.10 scala/Int#


private[this] var commaBuffered = false
// ^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`commaBuffered_=`().
// documentation ```scala\nprivate[this] var commaBuffered_=(x$1: Boolean): Unit\n```
// ^^^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#commaBuffered().
// documentation ```scala\nprivate[this] var commaBuffered: Boolean\n```
// _____________ synthetic_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#`commaBuffered_=`().
// documentation ```scala\nprivate[this] var commaBuffered_=(x$1: Boolean): Unit\n```
// relationship is_definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#commaBuffered().

def flushBuffer() = {
// ^^^^^^^^^^^ definition semanticdb maven maven/com.lihaoyi/ujson_2.13 1.4.0 ujson/BaseByteRenderer#flushBuffer().
Expand Down
Loading

0 comments on commit bf4ca0c

Please sign in to comment.