Skip to content

Commit

Permalink
Merge pull request #32936 from vespa-engine/theodorkl/moreYQLTests
Browse files Browse the repository at this point in the history 
Theodorkl/more yql tests
  • Loading branch information
@theodorklauritzen
theodorklauritzen authored Nov 22, 2024
2 parents 0780923 + a5b1b3f commit 52da6ce
Show file tree
Hide file tree
Showing 8 changed files with 106 additions and 42 deletions.
20 changes: 7 additions & 13 deletions integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ TOKEN :
| < WHERE: 'where' >
| < ORDER: 'order' >
| < BY: 'by' >
| < ORDERBY: <ORDER> <BY> >
| < ORDER: 'order' >
| < ORDERBY: <ORDER> ' ' <BY> >
| < DESC: 'desc' >
| < ASC: 'asc' >
| < FROM: 'from' >
Expand Down Expand Up @@ -76,9 +77,9 @@ TOKEN :
| < EQ: '=' >
| < LIKE: 'like' >
| < CONTAINS: 'contains' >
| < NOTLIKE: 'not like' >
| < NOTLIKE: 'not ' <LIKE> >
| < MATCHES: 'matches' >
| < NOTMATCHES: 'not matches' >
| < NOTMATCHES: 'not ' <MATCHES> >

| < PLUS: '+' >
| < MINUS: '-' >
Expand All @@ -88,8 +89,8 @@ TOKEN :

// effectively unary operators
| < NULL: 'null' >
| < IS_NULL: 'is' <NULL> >
| < IS_NOT_NULL: 'is not' <NULL> >
| < IS_NULL: 'is ' <NULL> >
| < IS_NOT_NULL: 'is not ' <NULL> >

// dereference
| < DOT: '.' >
Expand Down Expand Up @@ -143,15 +144,12 @@ String identifierStr:
| <DESC>
| <OUTPUT>
| <COUNT>
| <NOT_IN>
| <IN>
| <LIKE>
| <CONTAINS>
| <NOTLIKE>
| <MATCHES>
| <NOTMATCHES>
| <IS_NULL>
| <IS_NOT_NULL>
| <TIMEOUT>
| <IDENTIFIER>
| <LETTER>
Expand All @@ -169,7 +167,6 @@ vespa_grouping_identifier:
| <WHERE>
| <ORDER>
| <BY>
| <ORDERBY>
| <DESC>
| <ASC>
| <FROM>
Expand All @@ -191,8 +188,6 @@ vespa_grouping_identifier:
| <MATCHES>
| <NOTMATCHES>
| <NULL>
| <IS_NULL>
| <IS_NOT_NULL>
| <DOT>
| <AT>
| <TIMEOUT>
Expand Down Expand Up @@ -421,8 +416,7 @@ orderby_fields:

orderby_field:
(
(expression(true) <DESC>)
| (expression(true) (<ASC>)? )
expression(true) (<DESC> | <ASC>)?
)
;

Expand Down
6 changes: 6 additions & 0 deletions ...rc/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokenConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import ai.vespa.schemals.parser.yqlplus.ast.TRUE;
import ai.vespa.schemals.parser.yqlplus.ast.AT;
import ai.vespa.schemals.parser.yqlplus.ast.CONTAINS;
import ai.vespa.schemals.parser.yqlplus.ast.DESC;
import ai.vespa.schemals.parser.yqlplus.ast.FLOAT;
import ai.vespa.schemals.parser.yqlplus.ast.FROM;
import ai.vespa.schemals.parser.yqlplus.ast.INT;
Expand All @@ -26,7 +27,9 @@
import ai.vespa.schemals.parser.yqlplus.ast.relational_op;
import ai.vespa.schemals.parser.yqlplus.ast.unary_op;
import ai.vespa.schemals.parser.yqlplus.ast.OR;
import ai.vespa.schemals.parser.yqlplus.ast.ORDERBY;
import ai.vespa.schemals.parser.yqlplus.ast.AND;
import ai.vespa.schemals.parser.yqlplus.ast.ASC;
import ai.vespa.schemals.parser.yqlplus.ast.NOT_IN;
import ai.vespa.schemals.parser.yqlplus.ast.IN;
import ai.vespa.schemals.parser.yqlplus.ast.LIKE;
Expand All @@ -48,6 +51,9 @@ class YQLPlusSemanticTokenConfig {
add(NOTLIKE.class);
add(MATCHES.class);
add(NOTMATCHES.class);
add(ORDERBY.class);
add(ASC.class);
add(DESC.class);
}};

static final Map<Class<?>, String> tokensMap = new HashMap<Class<?>, String>() {{
Expand Down
4 changes: 2 additions & 2 deletions ...r/language-server/src/main/java/ai/vespa/schemals/schemadocument/VespaGroupingParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

class VespaGroupingParser {

static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger, Position offset) {
static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger, Position offset, int charOffset) {

CharSequence charSequence = input.toLowerCase();
GroupingParser parser = new GroupingParser(charSequence);
Expand All @@ -27,7 +27,7 @@ static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger,
}

Node node = parser.rootNode();
YQLNode CST = new YQLNode(node, offset);
YQLNode CST = new YQLNode(node, offset, charOffset);
// GroupingUtils.printTree(logger, node);

int charsRead = parser.getToken(0).getEndOffset();
Expand Down
32 changes: 17 additions & 15 deletions ...ge-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import ai.vespa.schemals.tree.Node;
import ai.vespa.schemals.tree.SchemaNode;
import ai.vespa.schemals.tree.YQLNode;
import ai.vespa.schemals.tree.YQL.YQLUtils;

public class YQLDocument implements DocumentManager {

Expand Down Expand Up @@ -94,9 +95,8 @@ public void updateFileContent(String content, Integer version) {
updateFileContent(content);
}

private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogger logger, Position offset) {
// CharSequence charSequence = content.toLowerCase();
YQLPlusParser parser = new YQLPlusParser(content);
private static YQLPartParseResult parseYQLPart(String content, ClientLogger logger, Position offset, int charOffset) {
YQLPlusParser parser = new YQLPlusParser(content.toLowerCase());

try {
parser.statement();
Expand All @@ -109,7 +109,7 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge
if (charsRead == 0) return new YQLPartParseResult(List.of(), Optional.empty(), charsRead);

ai.vespa.schemals.parser.yqlplus.Node node = parser.rootNode();
YQLNode retNode = new YQLNode(node, offset);
YQLNode retNode = new YQLNode(node, offset, charOffset, content);
// YQLUtils.printTree(logger, node);

return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead);
Expand All @@ -124,7 +124,7 @@ private static boolean detectContinuation(String inputString) {
return false;
}

private static YQLPartParseResult parseContinuation(String inputString, Position offset) {
private static YQLPartParseResult parseContinuation(String inputString, Position offset, int charOffset) {

YQLPlusParser parser = new YQLPlusParser(inputString);

Expand All @@ -135,19 +135,19 @@ private static YQLPartParseResult parseContinuation(String inputString, Position
}

var node = parser.rootNode();
YQLNode retNode = new YQLNode(node, offset);
YQLNode retNode = new YQLNode(node, offset, charOffset);

int charsRead = parser.getToken(0).getEndOffset();

return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead);
}

private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset) {
private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset, int charOffset) {
YQLNode ret = new YQLNode(new Range(offset, offset));

int pipeIndex = queryString.indexOf('|');
String YQLString = pipeIndex == -1 ? queryString : queryString.substring(0, pipeIndex);
YQLPartParseResult YQLResult = parseYQLPart(YQLString, context.logger(), offset);
YQLPartParseResult YQLResult = parseYQLPart(YQLString, context.logger(), offset, charOffset);

if (YQLResult.CST.isEmpty()) return YQLResult;

Expand All @@ -161,18 +161,18 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que
String charsBeforePipe = queryString.substring(charsRead, pipeIndex);
if (charsBeforePipe.strip().length() == 0) {
String groupingString = queryString.substring(pipeIndex + 1); // Do not include pipe char
charsRead = pipeIndex + 1;
Position YQLStringPosition = StringUtils.getStringPosition(YQLString);
Position groupOffsetWithoutPipe = CSTUtils.addPositions(offset, YQLStringPosition);

Position groupOffset = CSTUtils.addPositions(groupOffsetWithoutPipe, new Position(0, 1)); // Add pipe char

ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|"));
charsRead++;
ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|", charOffset + pipeIndex));

// Look for continuation
boolean continuationDetected = detectContinuation(groupingString);
if (continuationDetected) {
YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset);
YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset, charOffset + charsRead);

diagnostics.addAll(continuationResults.diagnostics());
if (continuationResults.CST().isPresent()) {
Expand All @@ -188,7 +188,7 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que
}

if (groupingString.length() > 0 && groupingString.strip().length() > 0) {
YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset);
YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset, charOffset + charsRead);
if (groupingResult.CST.isPresent()) {
ret.addChild(groupingResult.CST.get());
}
Expand All @@ -214,6 +214,8 @@ public static ParseResult parseContent(ParseContext context) {
YQLNode ret = new YQLNode(StringUtils.getStringRange(content));
ArrayList<Diagnostic> diagnostics = new ArrayList<>();

if (content.trim().length() == 0) return new ParseResult(diagnostics, Optional.of(ret));

int charsRead = 0;
int linesRead = 0;

Expand All @@ -224,15 +226,15 @@ public static ParseResult parseContent(ParseContext context) {
break;
}

YQLPartParseResult result = parseYQLQuery(context, toParser, new Position(linesRead, 0));
YQLPartParseResult result = parseYQLQuery(context, toParser, new Position(linesRead, 0), charsRead);
diagnostics.addAll(result.diagnostics());

if (result.CST().isPresent()) {
ret.addChild(result.CST().get());
}

if (result.charsRead() == 0) result.charsRead++;

int newOffset = content.indexOf('\n', charsRead + result.charsRead());
if (newOffset == -1) {
newOffset = content.length();
Expand Down
1 change: 1 addition & 0 deletions ...ion/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ public Symbol setSymbol(SymbolType type, String fileURI, Optional<Symbol> scope)
}

public abstract int getBeginOffset();
public abstract int getEndOffset();

public abstract String getText();
public abstract Class<?> getASTClass();
Expand Down
17 changes: 17 additions & 0 deletions ...hema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,23 @@ public int getBeginOffset() {
}
}

@Override
public int getEndOffset() {
switch (language) {
case SCHEMA:
if (originalSchemaNode == null) return -1;
return originalSchemaNode.getEndOffset();
case INDEXING:
if (originalIndexingNode == null) return -1;
return originalIndexingNode.getEndOffset();
case RANK_EXPRESSION:
if (originalRankExpressionNode == null) return -1;
return originalRankExpressionNode.getEndOffset();
default:
return -1;
}
}

public String getClassLeafIdentifierString() {
if (language == LanguageType.CUSTOM && getASTClass() != null) {
return getASTClass().getSimpleName();
Expand Down
47 changes: 40 additions & 7 deletions .../schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,32 +14,47 @@ public class YQLNode extends ai.vespa.schemals.tree.Node {
private ai.vespa.schemals.parser.grouping.Node originalGroupingNode;

private String customText;
private int startCharOffset;

public YQLNode(Node node, Position offset) {
public YQLNode(Node node, Position offset, int startCharOffset, String originalString) {
super(LanguageType.YQLPlus, CSTUtils.addPositionToRange(offset, YQLUtils.getNodeRange(node)), node.isDirty());
originalYQLNode = node;
this.startCharOffset = startCharOffset;
this.customText = originalString.substring(node.getBeginOffset(), node.getEndOffset());

for (Node child : node.children()) {
addChild(new YQLNode(child, offset));
addChild(new YQLNode(child, offset, startCharOffset, originalString));
}
}

public YQLNode(ai.vespa.schemals.parser.grouping.Node node, Position rangeOffset) {
public YQLNode(Node node, Position offset, int startCharOffset) {
super(LanguageType.YQLPlus, CSTUtils.addPositionToRange(offset, YQLUtils.getNodeRange(node)), node.isDirty());
originalYQLNode = node;
this.startCharOffset = startCharOffset;

for (Node child : node.children()) {
addChild(new YQLNode(child, offset, startCharOffset));
}
}

public YQLNode(ai.vespa.schemals.parser.grouping.Node node, Position rangeOffset, int startCharOffset) {
super(LanguageType.GROUPING, CSTUtils.addPositionToRange(rangeOffset, GroupingUtils.getNodeRange(node)), node.isDirty());
originalGroupingNode = node;
this.startCharOffset = startCharOffset;

for (ai.vespa.schemals.parser.grouping.Node child : node.children()) {
addChild(new YQLNode(child, rangeOffset));
addChild(new YQLNode(child, rangeOffset, startCharOffset));
}
}

public YQLNode(Range range) {
super(LanguageType.CUSTOM, range, false);
}

public YQLNode(Range range, String customText) {
public YQLNode(Range range, String customText, int startCharOffset) {
this(range);
this.customText = customText;
this.startCharOffset = startCharOffset;
}

public Range setRange(Range range) {
Expand All @@ -49,6 +64,7 @@ public Range setRange(Range range) {

public String getText() {
if (language == LanguageType.YQLPlus) {
if (customText != null) return customText;
return originalYQLNode.getSource();
}

Expand All @@ -72,6 +88,7 @@ public String getText() {
var child = get(i);
ret += " " + child.getText();
}
if (ret.length() == 0) return "";

return ret.substring(1);
}
Expand All @@ -97,12 +114,28 @@ public Class<?> getASTClass() {

@Override
public int getBeginOffset() {
if (language == LanguageType.YQLPlus) return originalYQLNode.getBeginOffset();
if (language == LanguageType.GROUPING) return originalGroupingNode.getBeginOffset();
if (language == LanguageType.YQLPlus) return startCharOffset + originalYQLNode.getBeginOffset();
if (language == LanguageType.GROUPING) return startCharOffset + originalGroupingNode.getBeginOffset();
if (language == LanguageType.CUSTOM) return startCharOffset;

throw new RuntimeException("Could not find the begin offset of YQLNode.");
}

@Override
public int getEndOffset() {
if (language == LanguageType.YQLPlus) return startCharOffset + originalYQLNode.getEndOffset();
if (language == LanguageType.GROUPING) return startCharOffset + originalGroupingNode.getEndOffset();

if (language == LanguageType.CUSTOM && size() > 0) {
return get(size() - 1).getEndOffset();
}
if (language == LanguageType.CUSTOM && customText != null) {
return startCharOffset + customText.length();
}

throw new RuntimeException("Could not find the end offset of YQLNode.");
}

public String toString() {
Range range = getRange();
Position start = range.getStart();
Expand Down
Loading

0 comments on commit 52da6ce

Please sign in to comment.