From 96ccb17d99f215934dc712de712ea1a49252c49b Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 13:04:28 +0100 Subject: [PATCH 1/4] test: Expand testing for YQL --- .../src/main/ccc/yqlplus/YQLPlus.ccc | 2 +- .../schemals/schemadocument/YQLDocument.java | 3 ++- .../main/java/ai/vespa/schemals/tree/Node.java | 1 + .../java/ai/vespa/schemals/tree/SchemaNode.java | 17 +++++++++++++++++ .../java/ai/vespa/schemals/tree/YQLNode.java | 13 +++++++++++++ .../java/ai/vespa/schemals/YQLParserTest.java | 15 +++++++++++++-- 6 files changed, 47 insertions(+), 4 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 9dfee585d334..90aac83c2f17 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -37,7 +37,7 @@ TOKEN : | < WHERE: 'where' > | < ORDER: 'order' > | < BY: 'by' > -| < ORDERBY: > +| < ORDERBY: 'order by' > | < DESC: 'desc' > | < ASC: 'asc' > | < FROM: 'from' > diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 3633a5bd378c..539076595b39 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -21,6 +21,7 @@ import ai.vespa.schemals.tree.Node; import ai.vespa.schemals.tree.SchemaNode; import ai.vespa.schemals.tree.YQLNode; +import ai.vespa.schemals.tree.YQL.YQLUtils; public class YQLDocument implements DocumentManager { @@ -242,7 +243,7 @@ public static ParseResult parseContent(ParseContext context) { charsRead = newOffset; } - // YQLUtils.printTree(context.logger(), ret); + YQLUtils.printTree(context.logger(), ret); return new ParseResult(diagnostics, Optional.of(ret)); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java index a717e577aa3b..9f5b166286c2 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java @@ -239,6 +239,7 @@ public Symbol setSymbol(SymbolType type, String fileURI, Optional scope) } public abstract int getBeginOffset(); + public abstract int getEndOffset(); public abstract String getText(); public abstract Class getASTClass(); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java index d44b1fe1821d..9fd520c5d3c0 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java @@ -307,6 +307,23 @@ public int getBeginOffset() { } } + @Override + public int getEndOffset() { + switch (language) { + case SCHEMA: + if (originalSchemaNode == null) return -1; + return originalSchemaNode.getEndOffset(); + case INDEXING: + if (originalIndexingNode == null) return -1; + return originalIndexingNode.getEndOffset(); + case RANK_EXPRESSION: + if (originalRankExpressionNode == null) return -1; + return originalRankExpressionNode.getEndOffset(); + default: + return -1; + } + } + public String getClassLeafIdentifierString() { if (language == LanguageType.CUSTOM && getASTClass() != null) { return getASTClass().getSimpleName(); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java index f305fd636d44..b9704dfda783 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java @@ -103,6 +103,19 @@ public int getBeginOffset() { throw new RuntimeException("Could not find the begin offset of YQLNode."); } + @Override + public int getEndOffset() { + if (language == LanguageType.YQLPlus) return originalYQLNode.getEndOffset(); + if (language == LanguageType.GROUPING) return originalGroupingNode.getEndOffset(); + + if (language == LanguageType.CUSTOM && size() > 0) { + var child = get(size() - 1); + return child.getEndOffset(); + } + + throw new RuntimeException("Could not find the end offset of YQLNode."); + } + public String toString() { Range range = getRange(); Position start = range.getStart(); diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index d5808b23dd5a..a5fa883db806 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -1,10 +1,12 @@ package ai.vespa.schemals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Arrays; +import java.util.Optional; import java.util.stream.Stream; import org.junit.jupiter.api.DynamicTest; @@ -16,6 +18,7 @@ import ai.vespa.schemals.schemadocument.YQLDocument.ParseResult; import ai.vespa.schemals.testutils.*; +import ai.vespa.schemals.tree.YQLNode; public class YQLParserTest { @@ -34,6 +37,14 @@ void checkQueryParses(int expectedErrors, String input) throws Exception { var parseResult = parseString(input); String testMessage = "For input: " + input + Utils.constructDiagnosticMessage(parseResult.diagnostics(), 1); assertEquals(expectedErrors, Utils.countErrors(parseResult.diagnostics()), testMessage); + + if (expectedErrors == 0) { + assertTrue(parseResult.CST().isPresent(), "Expected that a YQLNode was present in the input: " + input); + + YQLNode node = parseResult.CST().get(); + int charsRead = node.getEndOffset(); + assertEquals(input.length(), charsRead, "Expected the parser to read all the chars in the input, but it read " + charsRead + " of " + input.length() + " for input: " + input); + } } catch (Exception e) { StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter); @@ -65,8 +76,8 @@ Stream generateGoodTests() { "all(group(predefined(customer, bucket(-inf,\"Jones\"), bucket(\"Jones\", inf))) each(each(output(summary()))))", "all(group(predefined(customer, bucket<-inf,\"Jones\">, bucket[\"Jones\"], bucket<\"Jones\", inf>)) each(each(output(summary()))))", "all(group(predefined(tax, bucket[0.0,0.2>, bucket[0.2,0.5>, bucket[0.5,inf>)) each(each(output(summary()))))", - // "{ 'continuations':['BGAAABEBCA'] }all(output(count()))", - // "{ 'continuations':['BGAAABEBCA', 'BGAAABEBEBC'] }all(output(count()))", + "{ 'continuations':['BGAAABEBCA'] }all(output(count()))", + "{ 'continuations':['BGAAABEBCA', 'BGAAABEBEBC'] }all(output(count()))", "all(group(mod(div(date,mul(60,60)),24)) each(output(sum(price))))", "all(group(customer) each(output(sum(mul(price,sub(1,tax))))))", "all( group(a) each(output(count())) )", From 4564912ce8466c02c37b09353e99761954ee4f1d Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 15:14:53 +0100 Subject: [PATCH 2/4] fix: Parsing of ORDER BY token in LSP --- .../src/main/ccc/yqlplus/YQLPlus.ccc | 12 ++------ .../YQLPlusSemanticTokenConfig.java | 6 ++++ .../schemadocument/VespaGroupingParser.java | 4 +-- .../schemals/schemadocument/YQLDocument.java | 24 +++++++-------- .../java/ai/vespa/schemals/tree/YQLNode.java | 29 ++++++++++++------- .../java/ai/vespa/schemals/YQLParserTest.java | 4 +-- 6 files changed, 43 insertions(+), 36 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 90aac83c2f17..8137db743352 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -37,7 +37,8 @@ TOKEN : | < WHERE: 'where' > | < ORDER: 'order' > | < BY: 'by' > -| < ORDERBY: 'order by' > +| < ORDER: 'order' > +| < ORDERBY: ' ' > | < DESC: 'desc' > | < ASC: 'asc' > | < FROM: 'from' > @@ -143,15 +144,12 @@ String identifierStr: | | | - | | | | | | | - | - | | | | @@ -169,7 +167,6 @@ vespa_grouping_identifier: | | | - | | | | @@ -191,8 +188,6 @@ vespa_grouping_identifier: | | | - | - | | | | @@ -421,8 +416,7 @@ orderby_fields: orderby_field: ( - (expression(true) ) - | (expression(true) ()? ) + expression(true) ( | )? ) ; diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokenConfig.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokenConfig.java index 2eff88182461..bedd990f8471 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokenConfig.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokenConfig.java @@ -11,6 +11,7 @@ import ai.vespa.schemals.parser.yqlplus.ast.TRUE; import ai.vespa.schemals.parser.yqlplus.ast.AT; import ai.vespa.schemals.parser.yqlplus.ast.CONTAINS; +import ai.vespa.schemals.parser.yqlplus.ast.DESC; import ai.vespa.schemals.parser.yqlplus.ast.FLOAT; import ai.vespa.schemals.parser.yqlplus.ast.FROM; import ai.vespa.schemals.parser.yqlplus.ast.INT; @@ -26,7 +27,9 @@ import ai.vespa.schemals.parser.yqlplus.ast.relational_op; import ai.vespa.schemals.parser.yqlplus.ast.unary_op; import ai.vespa.schemals.parser.yqlplus.ast.OR; +import ai.vespa.schemals.parser.yqlplus.ast.ORDERBY; import ai.vespa.schemals.parser.yqlplus.ast.AND; +import ai.vespa.schemals.parser.yqlplus.ast.ASC; import ai.vespa.schemals.parser.yqlplus.ast.NOT_IN; import ai.vespa.schemals.parser.yqlplus.ast.IN; import ai.vespa.schemals.parser.yqlplus.ast.LIKE; @@ -48,6 +51,9 @@ class YQLPlusSemanticTokenConfig { add(NOTLIKE.class); add(MATCHES.class); add(NOTMATCHES.class); + add(ORDERBY.class); + add(ASC.class); + add(DESC.class); }}; static final Map, String> tokensMap = new HashMap, String>() {{ diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/VespaGroupingParser.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/VespaGroupingParser.java index baf9d598e619..62a7ffa1cb26 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/VespaGroupingParser.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/VespaGroupingParser.java @@ -15,7 +15,7 @@ class VespaGroupingParser { - static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger, Position offset) { + static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger, Position offset, int charOffset) { CharSequence charSequence = input.toLowerCase(); GroupingParser parser = new GroupingParser(charSequence); @@ -27,7 +27,7 @@ static YQLPartParseResult parseVespaGrouping(String input, ClientLogger logger, } Node node = parser.rootNode(); - YQLNode CST = new YQLNode(node, offset); + YQLNode CST = new YQLNode(node, offset, charOffset); // GroupingUtils.printTree(logger, node); int charsRead = parser.getToken(0).getEndOffset(); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 539076595b39..4293a88eae9f 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -95,7 +95,7 @@ public void updateFileContent(String content, Integer version) { updateFileContent(content); } - private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogger logger, Position offset) { + private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogger logger, Position offset, int charOffset) { // CharSequence charSequence = content.toLowerCase(); YQLPlusParser parser = new YQLPlusParser(content); @@ -110,7 +110,7 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge if (charsRead == 0) return new YQLPartParseResult(List.of(), Optional.empty(), charsRead); ai.vespa.schemals.parser.yqlplus.Node node = parser.rootNode(); - YQLNode retNode = new YQLNode(node, offset); + YQLNode retNode = new YQLNode(node, offset, charOffset); // YQLUtils.printTree(logger, node); return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); @@ -125,7 +125,7 @@ private static boolean detectContinuation(String inputString) { return false; } - private static YQLPartParseResult parseContinuation(String inputString, Position offset) { + private static YQLPartParseResult parseContinuation(String inputString, Position offset, int charOffset) { YQLPlusParser parser = new YQLPlusParser(inputString); @@ -136,19 +136,19 @@ private static YQLPartParseResult parseContinuation(String inputString, Position } var node = parser.rootNode(); - YQLNode retNode = new YQLNode(node, offset); + YQLNode retNode = new YQLNode(node, offset, charOffset); int charsRead = parser.getToken(0).getEndOffset(); return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); } - private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset) { + private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset, int charOffset) { YQLNode ret = new YQLNode(new Range(offset, offset)); int pipeIndex = queryString.indexOf('|'); String YQLString = pipeIndex == -1 ? queryString : queryString.substring(0, pipeIndex); - YQLPartParseResult YQLResult = parseYQLPart(YQLString, context.logger(), offset); + YQLPartParseResult YQLResult = parseYQLPart(YQLString, context.logger(), offset, charOffset); if (YQLResult.CST.isEmpty()) return YQLResult; @@ -167,13 +167,13 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que Position groupOffset = CSTUtils.addPositions(groupOffsetWithoutPipe, new Position(0, 1)); // Add pipe char - ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|")); + ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|", charOffset + pipeIndex)); charsRead++; // Look for continuation boolean continuationDetected = detectContinuation(groupingString); if (continuationDetected) { - YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset); + YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset, charOffset + charsRead); diagnostics.addAll(continuationResults.diagnostics()); if (continuationResults.CST().isPresent()) { @@ -189,7 +189,7 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que } if (groupingString.length() > 0 && groupingString.strip().length() > 0) { - YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset); + YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset, charOffset + charsRead); if (groupingResult.CST.isPresent()) { ret.addChild(groupingResult.CST.get()); } @@ -225,15 +225,15 @@ public static ParseResult parseContent(ParseContext context) { break; } - YQLPartParseResult result = parseYQLQuery(context, toParser, new Position(linesRead, 0)); + YQLPartParseResult result = parseYQLQuery(context, toParser, new Position(linesRead, 0), charsRead); diagnostics.addAll(result.diagnostics()); - + if (result.CST().isPresent()) { ret.addChild(result.CST().get()); } if (result.charsRead() == 0) result.charsRead++; - + int newOffset = content.indexOf('\n', charsRead + result.charsRead()); if (newOffset == -1) { newOffset = content.length(); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java index b9704dfda783..4d4962551888 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java @@ -14,22 +14,25 @@ public class YQLNode extends ai.vespa.schemals.tree.Node { private ai.vespa.schemals.parser.grouping.Node originalGroupingNode; private String customText; + private int startCharOffset; - public YQLNode(Node node, Position offset) { + public YQLNode(Node node, Position offset, int startCharOffset) { super(LanguageType.YQLPlus, CSTUtils.addPositionToRange(offset, YQLUtils.getNodeRange(node)), node.isDirty()); originalYQLNode = node; + this.startCharOffset = startCharOffset; for (Node child : node.children()) { - addChild(new YQLNode(child, offset)); + addChild(new YQLNode(child, offset, startCharOffset)); } } - public YQLNode(ai.vespa.schemals.parser.grouping.Node node, Position rangeOffset) { + public YQLNode(ai.vespa.schemals.parser.grouping.Node node, Position rangeOffset, int startCharOffset) { super(LanguageType.GROUPING, CSTUtils.addPositionToRange(rangeOffset, GroupingUtils.getNodeRange(node)), node.isDirty()); originalGroupingNode = node; + this.startCharOffset = startCharOffset; for (ai.vespa.schemals.parser.grouping.Node child : node.children()) { - addChild(new YQLNode(child, rangeOffset)); + addChild(new YQLNode(child, rangeOffset, startCharOffset)); } } @@ -37,9 +40,10 @@ public YQLNode(Range range) { super(LanguageType.CUSTOM, range, false); } - public YQLNode(Range range, String customText) { + public YQLNode(Range range, String customText, int startCharOffset) { this(range); this.customText = customText; + this.startCharOffset = startCharOffset; } public Range setRange(Range range) { @@ -97,20 +101,23 @@ public Class getASTClass() { @Override public int getBeginOffset() { - if (language == LanguageType.YQLPlus) return originalYQLNode.getBeginOffset(); - if (language == LanguageType.GROUPING) return originalGroupingNode.getBeginOffset(); + if (language == LanguageType.YQLPlus) return startCharOffset + originalYQLNode.getBeginOffset(); + if (language == LanguageType.GROUPING) return startCharOffset + originalGroupingNode.getBeginOffset(); + if (language == LanguageType.CUSTOM) return startCharOffset; throw new RuntimeException("Could not find the begin offset of YQLNode."); } @Override public int getEndOffset() { - if (language == LanguageType.YQLPlus) return originalYQLNode.getEndOffset(); - if (language == LanguageType.GROUPING) return originalGroupingNode.getEndOffset(); + if (language == LanguageType.YQLPlus) return startCharOffset + originalYQLNode.getEndOffset(); + if (language == LanguageType.GROUPING) return startCharOffset + originalGroupingNode.getEndOffset(); if (language == LanguageType.CUSTOM && size() > 0) { - var child = get(size() - 1); - return child.getEndOffset(); + return get(size() - 1).getEndOffset(); + } + if (language == LanguageType.CUSTOM && customText != null) { + return startCharOffset + customText.length(); } throw new RuntimeException("Could not find the end offset of YQLNode."); diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index a5fa883db806..ba7aa5f6faba 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -37,7 +37,7 @@ void checkQueryParses(int expectedErrors, String input) throws Exception { var parseResult = parseString(input); String testMessage = "For input: " + input + Utils.constructDiagnosticMessage(parseResult.diagnostics(), 1); assertEquals(expectedErrors, Utils.countErrors(parseResult.diagnostics()), testMessage); - + if (expectedErrors == 0) { assertTrue(parseResult.CST().isPresent(), "Expected that a YQLNode was present in the input: " + input); @@ -174,7 +174,7 @@ Stream generateGoodTests() { "select * from music where weakAnd(a contains \"A\", b contains \"B\")", "select * from music where ({targetHits: 7}weakAnd(a contains \"A\", b contains \"B\"))", "select * from music where geoLocation(myfieldname, 63.5, 10.5, \"200 km\")", - "select * from music where ({targetHits: 10}nearestNeighbor(doc_vector, query_vector))&input.query(query_vector)=[3,5,7]", + "select * from music where ({targetHits: 10}nearestNeighbor(doc_vector, query_vector))", "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo)", "select * from music where predicate(predicate_field,{\"gender\":\"Female\"},{\"age\":20L})", "select * from music where predicate(predicate_field,0,{\"age\":20L})", From 1b96c6512afdc0768bbe6f7a082c637e5a06e01b Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 15:34:24 +0100 Subject: [PATCH 3/4] fix: Parsing offset borefor pipe symbol --- .../main/java/ai/vespa/schemals/schemadocument/YQLDocument.java | 2 +- .../src/test/java/ai/vespa/schemals/YQLParserTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 4293a88eae9f..ad01a753c0f4 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -162,13 +162,13 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que String charsBeforePipe = queryString.substring(charsRead, pipeIndex); if (charsBeforePipe.strip().length() == 0) { String groupingString = queryString.substring(pipeIndex + 1); // Do not include pipe char + charsRead = pipeIndex + 1; Position YQLStringPosition = StringUtils.getStringPosition(YQLString); Position groupOffsetWithoutPipe = CSTUtils.addPositions(offset, YQLStringPosition); Position groupOffset = CSTUtils.addPositions(groupOffsetWithoutPipe, new Position(0, 1)); // Add pipe char ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|", charOffset + pipeIndex)); - charsRead++; // Look for continuation boolean continuationDetected = detectContinuation(groupingString); diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index ba7aa5f6faba..b86f27c44ad8 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -98,7 +98,7 @@ Stream generateGoodTests() { "all( group(a) max(5) each(output(count()) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))) )))", "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))) )))", "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary(complexsummary)))) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary(simplesummary)))) all(group(c) max(5) each(max(69) output(count()) each(output(summary(fastsummary)))))) )))", - "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))) )))", + "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))) )", "all( group(time.year(a)) each(output(count())) )", "all( group(time.year(a)) each(output(count()) all(group(time.monthofyear(a)) each(output(count())))) )", "all( group(time.year(a)) each(output(count()) all(group(time.monthofyear(a)) each(output(count()) all(group(time.dayofmonth(a)) each(output(count()) all(group(time.hourofday(a)) each(output(count())))))))) )", From a5b1b3f78cd98a5df580406b0f040f09a9e15059 Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 16:31:20 +0100 Subject: [PATCH 4/4] fix: Parsing of YQL queries is not caseinsensitive --- .../src/main/ccc/yqlplus/YQLPlus.ccc | 8 ++++---- .../vespa/schemals/schemadocument/YQLDocument.java | 11 ++++++----- .../main/java/ai/vespa/schemals/tree/YQLNode.java | 13 +++++++++++++ .../test/java/ai/vespa/schemals/YQLParserTest.java | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 8137db743352..36c989a8e989 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -77,9 +77,9 @@ TOKEN : | < EQ: '=' > | < LIKE: 'like' > | < CONTAINS: 'contains' > -| < NOTLIKE: 'not like' > +| < NOTLIKE: 'not ' > | < MATCHES: 'matches' > -| < NOTMATCHES: 'not matches' > +| < NOTMATCHES: 'not ' > | < PLUS: '+' > | < MINUS: '-' > @@ -89,8 +89,8 @@ TOKEN : // effectively unary operators | < NULL: 'null' > -| < IS_NULL: 'is' > -| < IS_NOT_NULL: 'is not' > +| < IS_NULL: 'is ' > +| < IS_NOT_NULL: 'is not ' > // dereference | < DOT: '.' > diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index ad01a753c0f4..e0ee071b8c0b 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -95,9 +95,8 @@ public void updateFileContent(String content, Integer version) { updateFileContent(content); } - private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogger logger, Position offset, int charOffset) { - // CharSequence charSequence = content.toLowerCase(); - YQLPlusParser parser = new YQLPlusParser(content); + private static YQLPartParseResult parseYQLPart(String content, ClientLogger logger, Position offset, int charOffset) { + YQLPlusParser parser = new YQLPlusParser(content.toLowerCase()); try { parser.statement(); @@ -110,7 +109,7 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge if (charsRead == 0) return new YQLPartParseResult(List.of(), Optional.empty(), charsRead); ai.vespa.schemals.parser.yqlplus.Node node = parser.rootNode(); - YQLNode retNode = new YQLNode(node, offset, charOffset); + YQLNode retNode = new YQLNode(node, offset, charOffset, content); // YQLUtils.printTree(logger, node); return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); @@ -215,6 +214,8 @@ public static ParseResult parseContent(ParseContext context) { YQLNode ret = new YQLNode(StringUtils.getStringRange(content)); ArrayList diagnostics = new ArrayList<>(); + if (content.trim().length() == 0) return new ParseResult(diagnostics, Optional.of(ret)); + int charsRead = 0; int linesRead = 0; @@ -243,7 +244,7 @@ public static ParseResult parseContent(ParseContext context) { charsRead = newOffset; } - YQLUtils.printTree(context.logger(), ret); + // YQLUtils.printTree(context.logger(), ret); return new ParseResult(diagnostics, Optional.of(ret)); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java index 4d4962551888..dd21c9435856 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java @@ -16,6 +16,17 @@ public class YQLNode extends ai.vespa.schemals.tree.Node { private String customText; private int startCharOffset; + public YQLNode(Node node, Position offset, int startCharOffset, String originalString) { + super(LanguageType.YQLPlus, CSTUtils.addPositionToRange(offset, YQLUtils.getNodeRange(node)), node.isDirty()); + originalYQLNode = node; + this.startCharOffset = startCharOffset; + this.customText = originalString.substring(node.getBeginOffset(), node.getEndOffset()); + + for (Node child : node.children()) { + addChild(new YQLNode(child, offset, startCharOffset, originalString)); + } + } + public YQLNode(Node node, Position offset, int startCharOffset) { super(LanguageType.YQLPlus, CSTUtils.addPositionToRange(offset, YQLUtils.getNodeRange(node)), node.isDirty()); originalYQLNode = node; @@ -53,6 +64,7 @@ public Range setRange(Range range) { public String getText() { if (language == LanguageType.YQLPlus) { + if (customText != null) return customText; return originalYQLNode.getSource(); } @@ -76,6 +88,7 @@ public String getText() { var child = get(i); ret += " " + child.getText(); } + if (ret.length() == 0) return ""; return ret.substring(1); } diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index b86f27c44ad8..3bb386cbd91d 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -189,7 +189,7 @@ Stream generateGoodTests() { "select * from music where myUrlField.hostname contains uri(\"vespa.ai\")", "select * from music where myUrlField.hostname contains ({startAnchor: true}uri(\"vespa.ai\"))", "select * from music where title contains ({weight:200}\"heads\")", - "select * from sources * where ({stem: false}(foo contains \"a\" and bar contains \"b\")) or foo contains {stem: false}\"c\"", + "select * from sources * where ({stem: false}(foo contains \"a\" and bar contains \"b\")) or foo contains ({stem: false}\"c\")", "select * from sources * where foo contains @animal and foo contains phrase(@animal, @syntaxExample, @animal)", "select * from sources * where sddocname contains 'purchase' | all(group(customer) each(output(sum(price))))", };