From 23228cdbf7808568aee9a08cdaea4bdaadebc3cd Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 14 Nov 2024 09:50:55 +0100 Subject: [PATCH 01/13] fix: struct dot syntax to one level --- .../language-server/src/main/ccc/yqlplus/YQLPlus.ccc | 2 +- .../java/ai/vespa/schemals/schemadocument/YQLDocument.java | 2 +- .../src/test/java/ai/vespa/schemals/YQLParserTest.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 027192fce509..4448c372c87a 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -609,7 +609,7 @@ primary_expression: ( expression(in_select) ) | constant_expression | ( - (SCAN 2 => call_expression(in_select)) + (SCAN 4 => call_expression(in_select)) // WARNING: The scan number could be very large. This will catch all Myfield.MyChild. However nested structs will not parse | fieldref ) ) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 7a9d5bd3ddae..d4a23ec004bb 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -190,7 +190,7 @@ public static ParseResult parseContent(ParseContext context) { charsRead = newOffset; } - // YQLUtils.printTree(context.logger(), ret); + YQLUtils.printTree(context.logger(), ret); return new ParseResult(diagnostics, Optional.of(ret)); } diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index cf4dea61e3e5..a97bcea5d415 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -79,7 +79,8 @@ Stream generateGoodTests() { "select * from music where title contains \"madonna\" and !(title contains \"saint\")", "select * from music where text contains phrase(\"st\", \"louis\", \"blues\")", "select * from music where persons contains sameElement(first_name contains 'Joe', last_name contains 'Smith', year_of_birth < 1940)", - // "select * from music where identities contains sameElement(key contains 'father', value.first_name contains 'Joe', value.last_name contains 'Smith', value.year_of_birth < 1940)", + "select * from music where identities contains sameElement(key contains 'father', value.first_name contains 'Joe', value.last_name contains 'Smith', value.year_of_birth < 1940)", + // "select * from music where gradparentStruct.parentStruct.childField contains 'madonna'", "select * from music where fieldName contains equiv(\"A\",\"B\")", "select * from music where myUrlField contains uri(\"vespa.ai/foo\")", "select * from music where myStringAttribute contains ({prefixLength:1, maxEditDistance:2}fuzzy(\"parantesis\"))", From 1a199100bf6a729b487a610d127ccdfa0a052fd7 Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 14 Nov 2024 11:33:47 +0100 Subject: [PATCH 02/13] fix: All YQL tests now passes --- .../src/main/ccc/yqlplus/YQLPlus.ccc | 6 ++--- .../java/ai/vespa/schemals/YQLParserTest.java | 22 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 4448c372c87a..8b7555013b95 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -466,7 +466,7 @@ argument(boolean in_select): expression(boolean select): ( null_operator - | annotate_expression + | (SCAN annotate_expression => annotate_expression) | logical_OR_expression ) ; @@ -515,7 +515,7 @@ equality_expression: in_not_in_target: ( // TODO: Add expression stack peek - ( select_statement ) + SCAN 2 => ( select_statement ) | literal_list ) ; @@ -609,7 +609,7 @@ primary_expression: ( expression(in_select) ) | constant_expression | ( - (SCAN 4 => call_expression(in_select)) // WARNING: The scan number could be very large. This will catch all Myfield.MyChild. However nested structs will not parse + (SCAN namespaced_name => call_expression(in_select) ) | fieldref ) ) diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index a97bcea5d415..15338f4eb5f9 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -80,7 +80,7 @@ Stream generateGoodTests() { "select * from music where text contains phrase(\"st\", \"louis\", \"blues\")", "select * from music where persons contains sameElement(first_name contains 'Joe', last_name contains 'Smith', year_of_birth < 1940)", "select * from music where identities contains sameElement(key contains 'father', value.first_name contains 'Joe', value.last_name contains 'Smith', value.year_of_birth < 1940)", - // "select * from music where gradparentStruct.parentStruct.childField contains 'madonna'", + "select * from music where gradparentStruct.parentStruct.childField contains 'madonna'", "select * from music where fieldName contains equiv(\"A\",\"B\")", "select * from music where myUrlField contains uri(\"vespa.ai/foo\")", "select * from music where myStringAttribute contains ({prefixLength:1, maxEditDistance:2}fuzzy(\"parantesis\"))", @@ -90,21 +90,21 @@ Stream generateGoodTests() { "select * from sources * where vendor contains \"brick and mortar\" AND price < 50 AND userQuery()", "select * from music where rank(a contains \"A\", b contains \"B\", c contains \"C\")", "select * from music where rank(nearestNeighbor(field, queryVector), a contains \"A\", b contains \"B\", c contains \"C\")", - // "select * from music where integer_field in (10, 20, 30)", - // "select * from music where string_field in ('germany', 'france', 'norway')", - // "select * from music where integer_field in (@integer_values)", - // "select * from music where string_field in (@string_values)", - // "select * from music where dotProduct(description, {\"a\":1, \"b\":2})", - // "select * from music where weightedSet(description, {\"a\":1, \"b\":2})", - // "select * from music where wand(description, [[11,1], [37,2]])", - // "select * from music where ({scoreThreshold: 0.13, targetHits: 7}wand(description, {\"a\":1, \"b\":2}))", + "select * from music where integer_field in (10, 20, 30)", + "select * from music where string_field in ('germany', 'france', 'norway')", + "select * from music where integer_field in (@integer_values)", + "select * from music where string_field in (@string_values)", + "select * from music where dotProduct(description, {\"a\":1, \"b\":2})", + "select * from music where weightedSet(description, {\"a\":1, \"b\":2})", + "select * from music where wand(description, [[11,1], [37,2]])", + "select * from music where ({scoreThreshold: 0.13, targetHits: 7}wand(description, {\"a\":1, \"b\":2}))", "select * from music where weakAnd(a contains \"A\", b contains \"B\")", "select * from music where ({targetHits: 7}weakAnd(a contains \"A\", b contains \"B\"))", "select * from music where geoLocation(myfieldname, 63.5, 10.5, \"200 km\")", "select * from music where ({targetHits: 10}nearestNeighbor(doc_vector, query_vector))&input.query(query_vector)=[3,5,7]", "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo)", - // "select * from music where predicate(predicate_field,{\"gender\":\"Female\"},{\"age\":20L})", - // "select * from music where predicate(predicate_field,0,{\"age\":20L})", + "select * from music where predicate(predicate_field,{\"gender\":\"Female\"},{\"age\":20L})", + "select * from music where predicate(predicate_field,0,{\"age\":20L})", "select * from music where title contains \"madonna\" order by price asc, releasedate desc", "select * from music where title contains \"madonna\" order by {function: \"uca\", locale: \"en_US\", strength: \"IDENTICAL\"}other desc, {function: \"lowercase\"}something", "select * from music where title contains \"madonna\" limit 31 offset 29", From 135ad220ee440a7fd359d0fe7b2713aad00660d3 Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 14 Nov 2024 12:20:09 +0100 Subject: [PATCH 03/13] teat: Add test for new tokens in grouping language --- .../src/main/ccc/grouping/GroupingParser.ccc | 6 +++++ .../ai/vespa/schemals/ParserTokensTest.java | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc index f8f18598eca9..a84f2392585b 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc @@ -40,6 +40,12 @@ INJECT GroupingParser: } +INJECT GroupingParserLexer: +{ + public static EnumSet getRegularTokens() { + return EnumSet.copyOf(regularTokens); + } +} TOKEN : diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java index 39ac8ba9928c..ce524259d26d 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java @@ -4,20 +4,28 @@ import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.DynamicTest; import ai.vespa.schemals.parser.SchemaParserLexer; import ai.vespa.schemals.parser.indexinglanguage.IndexingParserLexer; import ai.vespa.schemals.parser.rankingexpression.RankingExpressionParserLexer; +import ai.vespa.schemals.parser.grouping.GroupingParserLexer; +import com.vladsch.flexmark.parser.Parser; import com.yahoo.schema.parser.SchemaParserConstants; import com.yahoo.vespa.indexinglanguage.parser.IndexingParserConstants; import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; +import com.yahoo.search.grouping.request.parser.GroupingParserConstants; + /** * Tests that the set of tokens declared in JavaCC parsers are also present in CongoCC parsers. @@ -102,4 +110,18 @@ public void testRankingExpressionTokenList() { List missing = findMissingTokens(javaCCFields, congoCCTokenStrings); assertEquals(0, missing.size(), "Missing ranking expression tokens in CongoCC: " + String.join(", ", missing)); } + + @Test + public void testVespaGroupingTokenList() { + Field[] javaCCFields = GroupingParserConstants.class.getDeclaredFields(); + + Set congoCCTokenStrings = new HashSet<>(); + + for (var tokenType : GroupingParserLexer.getRegularTokens()) { + congoCCTokenStrings.add(tokenType.toString()); + } + + List missing = findMissingTokens(javaCCFields, congoCCTokenStrings); + assertEquals(0, missing.size(), "Missing ranking expression tokens in CongoCC: " + String.join(", ", missing)); + } } From a0be0d894e8424ac9a7ed310526753a0f5396dff Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 14 Nov 2024 13:06:29 +0100 Subject: [PATCH 04/13] test: Add test to check for new token in yqlplus antlr parser --- .../src/main/ccc/yqlplus/YQLPlus.ccc | 7 ++++ .../ai/vespa/schemals/ParserTokensTest.java | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 8b7555013b95..9dfee585d334 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -12,6 +12,13 @@ INJECT YQLPlusParser: protected Deque expression_stack = new ArrayDeque<>(); } +INJECT YQLPlusLexer: +{ + public static EnumSet getRegularTokens() { + return EnumSet.copyOf(regularTokens); + } +} + // -------------------------------------------------------------------------------- // // Token declarations. diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java index ce524259d26d..a4d22419e2c3 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/ParserTokensTest.java @@ -13,18 +13,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; +import org.antlr.v4.runtime.Vocabulary; import org.junit.jupiter.api.DynamicTest; import ai.vespa.schemals.parser.SchemaParserLexer; import ai.vespa.schemals.parser.indexinglanguage.IndexingParserLexer; import ai.vespa.schemals.parser.rankingexpression.RankingExpressionParserLexer; import ai.vespa.schemals.parser.grouping.GroupingParserLexer; +import ai.vespa.schemals.parser.yqlplus.YQLPlusLexer; import com.vladsch.flexmark.parser.Parser; import com.yahoo.schema.parser.SchemaParserConstants; import com.yahoo.vespa.indexinglanguage.parser.IndexingParserConstants; import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; import com.yahoo.search.grouping.request.parser.GroupingParserConstants; +import com.yahoo.search.yql.yqlplusLexer; /** @@ -54,6 +57,11 @@ public class ParserTokensTest { "OCTAL" ); + public static Set antlrSpecialTokens = Set.of( + "COMMENT", + "WS" + ); + private List findMissingTokens(Field[] javaCCFields, Set congoCCTokenStrings) { List missing = new ArrayList<>(); @@ -124,4 +132,36 @@ public void testVespaGroupingTokenList() { List missing = findMissingTokens(javaCCFields, congoCCTokenStrings); assertEquals(0, missing.size(), "Missing ranking expression tokens in CongoCC: " + String.join(", ", missing)); } + + @Test + public void testYQLPlusTokenList() { + Vocabulary vocabulary = yqlplusLexer.VOCABULARY; + + Set antlrTokens = new HashSet<>(); + + for (int i = 0; i < vocabulary.getMaxTokenType(); i++) { + String symbolicName = vocabulary.getSymbolicName(i); + if (symbolicName != null) { + antlrTokens.add(symbolicName); + } + } + + Set congoCCTokenStrings = new HashSet<>(); + + for (var tokenType : YQLPlusLexer.getRegularTokens()) { + congoCCTokenStrings.add(tokenType.toString()); + } + + List missing = new ArrayList<>(); + for (var token : antlrTokens) { + if (antlrSpecialTokens.contains(token)) continue; + + if (!congoCCTokenStrings.contains(token)) { + missing.add(token); + } + } + + assertEquals(0, missing.size(), "Missing yqlplus tokens in CongoCC: " + String.join(", ", missing)); + + } } From 4e4b525228a1d2c7806f7571d267728e4dcc8c6d Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Thu, 14 Nov 2024 16:26:06 +0100 Subject: [PATCH 05/13] feat: Add support for continuation in YQL Queries in LSP --- .../schemals/schemadocument/YQLDocument.java | 86 +++++++++++++++++-- .../java/ai/vespa/schemals/YQLParserTest.java | 75 +++++++++++++++- 2 files changed, 152 insertions(+), 9 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index d4a23ec004bb..e6dffbbe2287 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -114,6 +114,57 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); } + private static int findContinuationLength(String inputString) { + + // BUG: This never check if the curly bracket are in a string or something else + + char[] charArr = inputString.toCharArray(); + int continuationStart = -1; + for (int i = 0; i < charArr.length; i++) { + if (!Character.isWhitespace(charArr[i])) { + if (charArr[i] != '{') { + return 0; + } + + continuationStart = i; + break; + + } + } + if (continuationStart == -1) return 0; + + + int level = 0; + int continuationEnd = charArr.length; + for (int i = continuationStart; i < charArr.length; i++) { + if (charArr[i] == '{') level++; + if (charArr[i] == '}') level--; + + if (level == 0) { + continuationEnd = i + 1; + break; + }; + } + + return continuationEnd; + } + + private static ParseResult parseContinuation(String inputString, Position offset) { + + YQLPlusParser parser = new YQLPlusParser(inputString); + + try { + parser.map_expression(); + } catch (ParseException exception) { + // Ignored, marked as dirty node + } + + var node = parser.rootNode(); + YQLNode retNode = new YQLNode(node, offset); + + return new ParseResult(List.of(), Optional.of(retNode)); + } + private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset) { YQLNode ret = new YQLNode(new Range(offset, offset)); @@ -139,14 +190,35 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que Position groupOffset = CSTUtils.addPositions(groupOffsetWithoutPipe, new Position(0, 1)); // Add pipe char ret.addChild(new YQLNode(new Range(groupOffsetWithoutPipe, groupOffset), "|")); - - YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset); - if (groupingResult.CST.isPresent()) { - ret.addChild(groupingResult.CST.get()); + charsRead++; + + // Look for continuation + int continuationLength = findContinuationLength(groupingString); + if (continuationLength != 0) { + String continuationString = groupingString.substring(0, continuationLength); + ParseResult continuationResults = parseContinuation(continuationString, groupOffset); + + diagnostics.addAll(continuationResults.diagnostics()); + if (continuationResults.CST().isPresent()) { + ret.addChild(continuationResults.CST().get()); + } + + charsRead += continuationLength; + groupingString = groupingString.substring(continuationLength); + Position continuationPosition = StringUtils.getStringPosition(continuationString); + groupOffset = CSTUtils.addPositions(groupOffset, continuationPosition); + } + + if (groupingString.length() > 0 && groupingString.strip().length() > 0) { + YQLPartParseResult groupingResult = VespaGroupingParser.parseVespaGrouping(groupingString, context.logger(), groupOffset); + if (groupingResult.CST.isPresent()) { + ret.addChild(groupingResult.CST.get()); + } + + diagnostics.addAll(groupingResult.diagnostics()); + charsRead += groupingResult.charsRead(); // Add one for the pipe symbol } - diagnostics.addAll(groupingResult.diagnostics()); - charsRead += 1 + groupingResult.charsRead(); // Add one for the pipe symbol } } @@ -190,7 +262,7 @@ public static ParseResult parseContent(ParseContext context) { charsRead = newOffset; } - YQLUtils.printTree(context.logger(), ret); + // YQLUtils.printTree(context.logger(), ret); return new ParseResult(diagnostics, Optional.of(ret)); } diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java index 15338f4eb5f9..d5808b23dd5a 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/YQLParserTest.java @@ -44,6 +44,68 @@ void checkQueryParses(int expectedErrors, String input) throws Exception { @TestFactory Stream generateGoodTests() { + String[] groupingQueries = new String[] { + + // From docs: /en/grouping.html + "all( group(customer) each(output(sum(price))) )", + "all(group(customer) max(2) precision(12) order(-count()) each(output(sum(price))))", + "all(group(customer) each(max(3) each(output(summary()))))", + "all(group(a) max(5) each(output(count())))", + "all(group(a) max(5) each(output(count()) max(7) each(output(summary()))))", + "all(all(group(a) max(3) each(output(count()) max(5) each(output(summary())))) all(group(b) max(3) each(output(count()) max(5) each(output(summary())))))", + "all(group(a) max(5) each(output(count()) max(7) each(output(summary()))))", + "all(group(a) each(output(count()) each(output(summary()))))", + "all(group(customer) each(group(time.date(date)) each(output(sum(price)))))", + "all(group(customer) each(max(1) output(sum(price)) each(output(summary()))) each(group(time.date(date)) each(max(10) output(sum(price)) each(output(summary())))))", + "all(group(price) each(each(output(summary()))))", + "all(group(price/1000) each(each(output(summary()))))", + "all(group(fixedwidth(price,1000)) each(each(output(summary()))))", + "all(group(predefined(price, bucket(0,1000), bucket(1000,2000), bucket(2000,5000), bucket(5000,inf))) each(each(output(summary()))))", + "all(group(predefined(price, bucket[0,1000>, bucket[1000,2000>, bucket[2000,5000>, bucket[5000,inf>)) each(each(output(summary()))))", + "all(group(predefined(customer, bucket(-inf,\"Jones\"), bucket(\"Jones\", inf))) each(each(output(summary()))))", + "all(group(predefined(customer, bucket<-inf,\"Jones\">, bucket[\"Jones\"], bucket<\"Jones\", inf>)) each(each(output(summary()))))", + "all(group(predefined(tax, bucket[0.0,0.2>, bucket[0.2,0.5>, bucket[0.5,inf>)) each(each(output(summary()))))", + // "{ 'continuations':['BGAAABEBCA'] }all(output(count()))", + // "{ 'continuations':['BGAAABEBCA', 'BGAAABEBEBC'] }all(output(count()))", + "all(group(mod(div(date,mul(60,60)),24)) each(output(sum(price))))", + "all(group(customer) each(output(sum(mul(price,sub(1,tax))))))", + "all( group(a) each(output(count())) )", + "all( all(group(a) each(output(count()))) all(group(b) each(output(count()))) )", + "all( max(1000) all(group(a) each(output(count()))) )", + "all( group(a % 5) each(output(count())) )", + "all( group(a + b * c) each(output(count())) )", + "all( group(a % 5) order(sum(b)) each(output(count())) )", + "all( group(a + b * c) order(max(d)) each(output(count())) )", + "all( group(a) order(avg(relevance()) * count()) each(output(count())) )", + "all(group(a) order(max(attr) * count()) each(output(count())) )", + "all( group(a) each(max(1) each(output(summary()))) )", + "all( group(a) each(max(1) output(count(), sum(b)) each(output(summary()))) )", + "all(group(a) each(max(1) output(count(), sum(b), xor(md5(cat(a, b, c), 64))) each(output(summary()))))", + "all( group(a) max(5) each(max(69) output(count()) each(output(summary()))) )", + "all( group(a) max(5) each(output(count()) all(group(b) max(5) each(max(69) output(count()) each(output(summary()))))) )", + "all( group(a) max(5) each(output(count()) all(group(b) max(5) each(output(count()) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))) )))", + "all( group(a) max(5) each(output(count()) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))) )))", + "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))) )))", + "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary(complexsummary)))) all(group(b) max(5) each(output(count()) all(max(1) each(output(summary(simplesummary)))) all(group(c) max(5) each(max(69) output(count()) each(output(summary(fastsummary)))))) )))", + "all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))) )))", + "all( group(time.year(a)) each(output(count())) )", + "all( group(time.year(a)) each(output(count()) all(group(time.monthofyear(a)) each(output(count())))) )", + "all( group(time.year(a)) each(output(count()) all(group(time.monthofyear(a)) each(output(count()) all(group(time.dayofmonth(a)) each(output(count()) all(group(time.hourofday(a)) each(output(count())))))))) )", + "all( group(predefined((now() - a) / (60 * 60 * 24), bucket(0,1), bucket(1,2), bucket(3,7), bucket(8,31))) each(output(count()) all(max(2) each(output(summary()))) all(group((now() - a) / (60 * 60 * 24)) each(output(count()) all(max(2) each(output(summary())))))) )", + "all( group(a) output(count()) )", + "all( group(strlen(name)) output(count()) )", + "all( group(a) output(count()) each(output(sum(b))) )", + "all( group(a) max(3) output(count()) each(output(sum(b))) )", + "all( group(a) max(10) output(count()) each(group(b) output(count())) )", + "all(group(1) each(output(avg(rating))))", + "all( group(predefined(rating, bucket[-inf, 0>, bucket[0, inf>)) each(output(count())) )", + "all( group(predefined(rating, bucket[-inf, 0>, bucket[0, inf>)) order(max(rating)) max(1) each( max(100) each(output(summary(name_only)))) )", + }; + + for (int i = 0; i < groupingQueries.length; i++) { + groupingQueries[i] = "select * from sources * where true | " + groupingQueries[i]; + } + String[] queries = new String[] { "select * from music", "select * from sources * where range(title, 0.0, 500.0)", // /container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -111,11 +173,19 @@ Stream generateGoodTests() { "select * from music where title contains \"madonna\" timeout 70", "select * from music where userInput(@userinput)", "select * from music where text contains ({distance: 5}near(\"a\", \"b\")) and text contains ({distance:2}near(\"c\", \"d\"))", + "select * from music where ({bounds:\"rightOpen\"}range(year, 2000, 2018))", + "select * from music where text contains ({distance: 5}near(\"a\", \"b\"))", + "select * from music where myUrlField.hostname contains uri(\"vespa.ai\")", + "select * from music where myUrlField.hostname contains ({startAnchor: true}uri(\"vespa.ai\"))", + "select * from music where title contains ({weight:200}\"heads\")", + "select * from sources * where ({stem: false}(foo contains \"a\" and bar contains \"b\")) or foo contains {stem: false}\"c\"", + "select * from sources * where foo contains @animal and foo contains phrase(@animal, @syntaxExample, @animal)", + "select * from sources * where sddocname contains 'purchase' | all(group(customer) each(output(sum(price))))", }; + Stream queryStream = Stream.concat(Arrays.stream(queries), Arrays.stream(groupingQueries)); - return Arrays.stream(queries) - .map(query -> DynamicTest.dynamicTest(query, () -> checkQueryParses(0, query))); + return queryStream.map(query -> DynamicTest.dynamicTest(query, () -> checkQueryParses(0, query))); } private record TestWithError(int expectedErrors, String query) {} @@ -124,6 +194,7 @@ private record TestWithError(int expectedErrors, String query) {} Stream InvalidQuery() throws Exception { var queries = new TestWithError[] { new TestWithError(1, "seletc *"), + // new TestWithError(1, "select * from sources * where true | all(group(a) order(attr * count()) each(output(count())) )"), }; return Arrays.stream(queries) From 4fc48f322f7e749a805d2a5f29308a6f07818546 Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 10:41:28 +0100 Subject: [PATCH 06/13] chore: more secure way to execute vespa queries --- .../common/command/commandtypes/RunVespaQuery.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java index b66d446da789..b626bb546b24 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java @@ -56,7 +56,7 @@ public Object execute(EventExecuteCommandContext context) { runVespaQuery(queryCommand, context.logger).thenAccept(result -> { if (!result.success()) { - if (result.result().toLowerCase().contains("command not found")) { + if (result.result().toLowerCase().contains("cannot run program")) { context.messageHandler.sendMessage(MessageType.Error, "Could not find vespa CLI. Make sure vespa CLI is installed and added to path. Download vespa CLI here: https://docs.vespa.ai/en/vespa-cli.html"); return; } @@ -107,13 +107,10 @@ private CompletableFuture runVespaQuery(String query, ClientLogger ProcessBuilder builder = new ProcessBuilder(); - String queryEscaped = query.replace("\"", "\\\""); - String vespaCommand = String.format("vespa query \"%s\"", queryEscaped); - if (isWindows) { - builder.command("cmd.exe", "/c", vespaCommand); // TODO: Test this on windows + builder.command("cmd.exe", "/c", "vespa", "query", query); // TODO: Test this on windows } else { - builder.command("/bin/sh", "-c", vespaCommand); + builder.command("vespa", "query", query); } return CompletableFuture.supplyAsync(() -> { @@ -146,8 +143,7 @@ private CompletableFuture runVespaQuery(String query, ClientLogger } catch (InterruptedException e) { return new QueryResult(false, "Program interrupted"); } catch (IOException e) { - logger.error(e.getMessage()); - return new QueryResult(false, "IOException occurred."); + return new QueryResult(false, e.getMessage()); } }); } From 7e05a29df7358f9816fd39edd0866e59c5345fbf Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 10:42:22 +0100 Subject: [PATCH 07/13] feat: Add YQL support to VSCode --- .../src/main/resources/META-INF/plugin.xml | 3 ++ .../clients/vscode/README.md | 7 +++++ .../clients/vscode/package.json | 16 +++++++++-- .../clients/vscode/src/extension.ts | 28 +++++++------------ 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml index 92b5b8c7d9dd..389eded23328 100644 --- a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml +++ b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml @@ -38,5 +38,8 @@ In addition, the plugin will be available for community editions as well. + diff --git a/integration/schema-language-server/clients/vscode/README.md b/integration/schema-language-server/clients/vscode/README.md index 47983f3ea6e6..b2109fc9e8fe 100644 --- a/integration/schema-language-server/clients/vscode/README.md +++ b/integration/schema-language-server/clients/vscode/README.md @@ -14,6 +14,11 @@ Features: - Renaming/refactoring - List document symbols +YQL Features: +- Error highlighting +- Semantic token highlighting +- Running Queries directly from `.yql` files + ## Requirements The extension requires Java 17 or greater. Upon activation, the extension will look in the following locations in this order for a Java executable: @@ -23,6 +28,8 @@ The extension requires Java 17 or greater. Upon activation, the extension will l - JDK_HOME environment variable - JAVA_HOME environment variable +The extension also requires [Vespa CLI](https://docs.vespa.ai/en/vespa-cli.html) to run Vespa Queries from `.yql` files. + ## XML support This extension bundles with an extension to the [LemMinX XML Language server](https://github.com/eclipse/lemminx). This is to provide additional support when editing the services.xml file in Vespa applications. diff --git a/integration/schema-language-server/clients/vscode/package.json b/integration/schema-language-server/clients/vscode/package.json index 14f4265435e0..7e06f3c5d532 100644 --- a/integration/schema-language-server/clients/vscode/package.json +++ b/integration/schema-language-server/clients/vscode/package.json @@ -15,7 +15,8 @@ ], "keywords": [ "Vespa", - "Schema" + "Schema", + "YQL" ], "repository": { "type": "git", @@ -24,7 +25,8 @@ "icon": "images/icon.png", "activationEvents": [ "onLanguage:xml", - "onLanguage:vespaSchema" + "onLanguage:vespaSchema", + "onLanguage:vespaYQL" ], "main": "./dist/extension.js", "contributes": { @@ -39,6 +41,16 @@ ".profile" ], "configuration": "./language-configuration.json" + }, + { + "id": "vespaYQL", + "aliases": [ + "Vespa YQL" + ], + "extensions": [ + ".yql" + ], + "configuration": "./language-configuration.json" } ], "xml.javaExtensions": [ diff --git a/integration/schema-language-server/clients/vscode/src/extension.ts b/integration/schema-language-server/clients/vscode/src/extension.ts index a23b760c89fd..d70bb9dbebf2 100644 --- a/integration/schema-language-server/clients/vscode/src/extension.ts +++ b/integration/schema-language-server/clients/vscode/src/extension.ts @@ -72,24 +72,16 @@ function createAndStartClient(serverPath: string): LanguageClient | null { let clientOptions: LanguageClientOptions = { // Register the server for plain text documents - documentSelector: [{ - scheme: 'file', - language: 'vespaSchema', - }], - middleware: { - provideCompletionItem: async (document, position, context, token, next) => { - const r = await next(document, position, context, token); - return r; - }, - provideDocumentHighlights: async (document, position, token, next) => { - const r = await next(document, position, token); - return r; - }, - provideDocumentSemanticTokens: async (document, token, next) => { - const r = await next(document, token); - return r; - }, - }, + documentSelector: [ + { + scheme: 'file', + language: 'vespaSchema', + }, + { + scheme: 'file', + language: 'vespaYQL' + } + ], synchronize: { fileEvents: vscode.workspace.createFileSystemWatcher("**/*{.sd,.profile}") } From cdfcb040f6efed522498e60d02ac35ac2651f39b Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 11:10:23 +0100 Subject: [PATCH 08/13] chore: update README for the IntelliJ plugin --- .../intellij/src/main/resources/META-INF/plugin.xml | 12 +++++++++--- .../schema-language-server/clients/vscode/README.md | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml index 389eded23328..6f2f61a80515 100644 --- a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml +++ b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml @@ -21,11 +21,17 @@
  • Renaming
    • +

    YQL Features

    +
      +
    • Error highlighting
      • +
    • Syntax highlighting
      • +
    • Running queries directly from .yql files +
    + ]]> Refactored to use LSP4IJ -The plugin will now support better syntax highlighting with semantic tokens and renaming. -In addition, the plugin will be available for community editions as well. +

    Simple support for YQL

    +The plugin now supports syntax highlighting of .yql files, in addition to run the queries directly from the editor. ]]>     com.intellij.modules.platform com.redhat.devtools.lsp4ij diff --git a/integration/schema-language-server/clients/vscode/README.md b/integration/schema-language-server/clients/vscode/README.md index b2109fc9e8fe..bf8b58329137 100644 --- a/integration/schema-language-server/clients/vscode/README.md +++ b/integration/schema-language-server/clients/vscode/README.md @@ -17,7 +17,7 @@ Features: YQL Features: - Error highlighting - Semantic token highlighting -- Running Queries directly from `.yql` files +- Running queries directly from `.yql` files ## Requirements The extension requires Java 17 or greater. Upon activation, the extension will look in the following locations in this order for a Java executable: From ba234f99ef801020096169cf8ed62dd6756d8a9f Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 11:31:54 +0100 Subject: [PATCH 09/13] chore: add VespaCLI requirement note in the IntelliJ plugin description --- .../clients/intellij/src/main/resources/META-INF/plugin.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml index 6f2f61a80515..e0880db8d2d6 100644 --- a/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml +++ b/integration/schema-language-server/clients/intellij/src/main/resources/META-INF/plugin.xml @@ -28,6 +28,9 @@
  • Running queries directly from .yql files +

    Requirements

    +The plugin requires Vespa CLI to be installed to be able to run Vespa Queries from .yql files. + ]]> Simple support for YQL From dbe4d4dadcd99790a12f22c78da0dcae8ed5aed5 Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 11:51:09 +0100 Subject: [PATCH 10/13] fix: Continuation parsing bug --- .../schemals/schemadocument/YQLDocument.java | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index e6dffbbe2287..54d924f201b0 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -21,7 +21,6 @@ import ai.vespa.schemals.tree.Node; import ai.vespa.schemals.tree.SchemaNode; import ai.vespa.schemals.tree.YQLNode; -import ai.vespa.schemals.tree.YQL.YQLUtils; public class YQLDocument implements DocumentManager { @@ -149,7 +148,16 @@ private static int findContinuationLength(String inputString) { return continuationEnd; } - private static ParseResult parseContinuation(String inputString, Position offset) { + private static boolean detectContinuation(String inputString) { + for (int i = 0; i < inputString.length(); i++) { + if (inputString.charAt(i) != ' ') { + return inputString.charAt(i) == '{'; + } + } + return false; + } + + private static YQLPartParseResult parseContinuation(String inputString, Position offset) { YQLPlusParser parser = new YQLPlusParser(inputString); @@ -162,7 +170,9 @@ private static ParseResult parseContinuation(String inputString, Position offset var node = parser.rootNode(); YQLNode retNode = new YQLNode(node, offset); - return new ParseResult(List.of(), Optional.of(retNode)); + int charsRead = parser.getToken(0).getEndOffset(); + + return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); } private static YQLPartParseResult parseYQLQuery(ParseContext context, String queryString, Position offset) { @@ -193,19 +203,20 @@ private static YQLPartParseResult parseYQLQuery(ParseContext context, String que charsRead++; // Look for continuation - int continuationLength = findContinuationLength(groupingString); - if (continuationLength != 0) { - String continuationString = groupingString.substring(0, continuationLength); - ParseResult continuationResults = parseContinuation(continuationString, groupOffset); + boolean continuationDetected = detectContinuation(groupingString); + if (continuationDetected) { + YQLPartParseResult continuationResults = parseContinuation(groupingString, groupOffset); diagnostics.addAll(continuationResults.diagnostics()); if (continuationResults.CST().isPresent()) { ret.addChild(continuationResults.CST().get()); } - charsRead += continuationLength; - groupingString = groupingString.substring(continuationLength); + charsRead += continuationResults.charsRead(); + String continuationString = groupingString.substring(0, continuationResults.charsRead()); Position continuationPosition = StringUtils.getStringPosition(continuationString); + + groupingString = groupingString.substring(continuationResults.charsRead()); groupOffset = CSTUtils.addPositions(groupOffset, continuationPosition); } From 533cb8ec189b6e902ccd724d76533f5f665e0fef Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 11:53:53 +0100 Subject: [PATCH 11/13] chore: remove unsued function --- .../schemals/schemadocument/YQLDocument.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index 54d924f201b0..e2793e5df3d8 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -113,41 +113,6 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge return new YQLPartParseResult(List.of(), Optional.of(retNode), charsRead); } - private static int findContinuationLength(String inputString) { - - // BUG: This never check if the curly bracket are in a string or something else - - char[] charArr = inputString.toCharArray(); - int continuationStart = -1; - for (int i = 0; i < charArr.length; i++) { - if (!Character.isWhitespace(charArr[i])) { - if (charArr[i] != '{') { - return 0; - } - - continuationStart = i; - break; - - } - } - if (continuationStart == -1) return 0; - - - int level = 0; - int continuationEnd = charArr.length; - for (int i = continuationStart; i < charArr.length; i++) { - if (charArr[i] == '{') level++; - if (charArr[i] == '}') level--; - - if (level == 0) { - continuationEnd = i + 1; - break; - }; - } - - return continuationEnd; - } - private static boolean detectContinuation(String inputString) { for (int i = 0; i < inputString.length(); i++) { if (inputString.charAt(i) != ' ') { From bd4f4ed50c0fa1ebb1e4263cf2c2b4c8877eb79d Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 12:29:08 +0100 Subject: [PATCH 12/13] fix: Infinite loop in YQL parser at special chars --- .../java/ai/vespa/schemals/schemadocument/YQLDocument.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index e2793e5df3d8..b38171041783 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -9,6 +9,8 @@ import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.VersionedTextDocumentIdentifier; +import com.google.protobuf.Option; + import ai.vespa.schemals.SchemaDiagnosticsHandler; import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.common.StringUtils; @@ -106,6 +108,8 @@ private static YQLPartParseResult parseYQLPart(CharSequence content, ClientLogge int charsRead = parser.getToken(0).getEndOffset(); + if (charsRead == 0) return new YQLPartParseResult(List.of(), Optional.empty(), charsRead); + ai.vespa.schemals.parser.yqlplus.Node node = parser.rootNode(); YQLNode retNode = new YQLNode(node, offset); // YQLUtils.printTree(logger, node); @@ -228,6 +232,8 @@ public static ParseResult parseContent(ParseContext context) { if (result.CST().isPresent()) { ret.addChild(result.CST().get()); } + + if (result.charsRead() == 0) result.charsRead++; int newOffset = content.indexOf('\n', charsRead + result.charsRead()); if (newOffset == -1) { From 788cb05896437fe637e5b6a4eb52b14874b948df Mon Sep 17 00:00:00 2001 From: Theodor Kvalsvik Lauritzen Date: Fri, 22 Nov 2024 12:30:08 +0100 Subject: [PATCH 13/13] chore: remove unused library --- .../main/java/ai/vespa/schemals/schemadocument/YQLDocument.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java index b38171041783..3633a5bd378c 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/YQLDocument.java @@ -9,8 +9,6 @@ import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.VersionedTextDocumentIdentifier; -import com.google.protobuf.Option; - import ai.vespa.schemals.SchemaDiagnosticsHandler; import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.common.StringUtils;