From b4db18b97dfb16647a28fdaf76187efd801f7f8e Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 30 Sep 2024 22:49:11 +0200 Subject: [PATCH] Revert "Require positive proof of an index field to segment" --- .../yahoo/prelude/query/ExactStringItem.java | 1 + .../prelude/query/PhraseSegmentItem.java | 1 + .../java/com/yahoo/search/yql/YqlParser.java | 6 ++--- .../test/CJKSearcherTestCase.java | 12 +++------- .../yql/MinimalQueryInserterTestCase.java | 4 ++-- .../yahoo/search/yql/YqlParserTestCase.java | 22 +++---------------- 6 files changed, 12 insertions(+), 34 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/query/ExactStringItem.java b/container-search/src/main/java/com/yahoo/prelude/query/ExactStringItem.java index ec440c07ea0a..cb0752e54088 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/ExactStringItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/ExactStringItem.java @@ -6,6 +6,7 @@ * * @author baldersheim */ + public class ExactStringItem extends WordItem { public ExactStringItem(String substring) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java index 6ec75bd52818..b4ed5518f132 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/PhraseSegmentItem.java @@ -8,6 +8,7 @@ import java.util.Objects; import java.util.Optional; + /** * A term which contains a fixed length phrase, a collection of word terms, * resulting from a single segmentation operation. diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 85fb9389c05e..72a6960c2c0f 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -25,7 +25,6 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; -import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; import com.yahoo.prelude.query.AndItem; @@ -742,7 +741,7 @@ private Item instantiatePhraseSegmentItem(String field, OperatorNode 0) { for (String word : words) { phrase.addItem(new WordItem(word, field, true)); } @@ -1516,7 +1515,6 @@ private Item instantiateWordItem(String field, boolean substrMatch = getAnnotation(ast, SUBSTRING, Boolean.class, Boolean.FALSE, "setting for whether to use substring match of input data"); boolean exact = exactMatch != null ? exactMatch : indexFactsSession.getIndex(indexNameExpander.expand(field)).isExact(); - String grammar = getAnnotation(ast, USER_INPUT_GRAMMAR, String.class, Query.Type.WEAKAND.toString(), "grammar for handling word input"); Preconditions.checkArgument((prefixMatch ? 1 : 0) + @@ -1560,7 +1558,7 @@ private Item instantiateWordItem(String field, } private boolean shouldSegment(String field, boolean fromQuery) { - return fromQuery && indexFactsSession.getIndex(indexNameExpander.expand(field)).isIndex(); + return fromQuery && ! indexFactsSession.getIndex(indexNameExpander.expand(field)).isAttribute(); } private TaggableItem segment(String field, OperatorNode ast, String wordData, diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java index 8124b7f0ac09..6c45dd2a1b13 100644 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java @@ -4,7 +4,8 @@ import com.yahoo.component.chain.Chain; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; -import com.yahoo.prelude.*; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexFactsFactory; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.parser.TestLinguistics; @@ -56,15 +57,8 @@ void testCjkQueryWithOverlappingTokens() { @Test public void testEquivAndChinese() { - SearchDefinition schema = new SearchDefinition("music-only"); - Index stringIndex = new Index("default"); - stringIndex.setIndex(true); - stringIndex.setString(true); - schema.addIndex(stringIndex); - var indexFacts = new IndexFacts(new IndexModel(schema)); - Query query = new Query(QueryTestCase.httpEncode("search?yql=select * from music-only where default contains equiv('a', 'b c') or default contains '东'")); - new Execution(new Chain<>(new MinimalQueryInserter(), new CJKSearcher()), Execution.Context.createContextStub(indexFacts)).search(query); + new Execution(new Chain<>(new MinimalQueryInserter(), new CJKSearcher()), Execution.Context.createContextStub()).search(query); assertEquals("OR (EQUIV default:a default:'b c') default:东", query.getModel().getQueryTree().toString()); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index 7148273b92e3..31f736880484 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -185,7 +185,7 @@ void testUserLanguageIsDetectedWithUserQueryEnglishAlsoWithNonEnglishStructuredQ Result result = execution.search(query); assertNull(result.hits().getError()); assertEquals(Language.ENGLISH, query.getModel().getParsingLanguage()); // by UNKNOWN -> ENGLISH - assertEquals("AND attribute_key:我能吞下玻璃而不伤身体。 (WEAKAND(100) executions)", query.getModel().getQueryTree().toString()); + assertEquals("AND attribute_key:我能吞下玻璃而不伤身体 (WEAKAND(100) executions)", query.getModel().getQueryTree().toString()); } @Test @@ -195,7 +195,7 @@ void testUserLanguageIsDetectedWithUserInputEnglishAlsoWithNonEnglishStructuredQ Result result = execution.search(query); assertNull(result.hits().getError()); assertEquals(Language.ENGLISH, query.getModel().getParsingLanguage()); // by UNKNOWN -> ENGLISH - assertEquals("AND attribute_key:我能吞下玻璃而不伤身体。 (WEAKAND(100) default:executions)", query.getModel().getQueryTree().toString()); + assertEquals("AND attribute_key:我能吞下玻璃而不伤身体 (WEAKAND(100) default:executions)", query.getModel().getQueryTree().toString()); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 8d880d6694be..4aa6c65ad8f7 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -198,11 +198,11 @@ void testKeywordAsFieldName() { void testComplexExpression() { String queryTreeYql = "rank((((filter contains ({origin: {original: \"filter:VideoAdsCappingTestCPM\", \"offset\": 7, length: 22}, normalizeCase: false, id: 1}\"videoadscappingtestcpm\") AND hasRankRestriction contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 2}\"0\") AND ((objective contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 3}\"install_app\") AND availableExtendedFields contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 4}\"cpiparams\")) OR (availableExtendedFields contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 5}\"appinstallinfo\") AND availableExtendedFields contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 6}\"appmetroplexinfo\")) OR (dummyField contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 7}\"default\")) AND !(objective contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 8}\"install_app\"))) AND advt_age = ({\"id\": 9}2147483647) AND advt_gender contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 10}\"all\") AND advt_all_segments = ({\"id\": 11}2147483647) AND advt_keywords contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 12}\"all\") AND advMobilePlatform = ({\"id\": 13}2147483647) AND advMobileDeviceType = ({\"id\": 14}2147483647) AND advMobileCon = ({\"id\": 15}2147483647) AND advMobileOSVersions = ({\"id\": 16}2147483647) AND advCarrier = ({\"id\": 17}2147483647) AND ({\"id\": 18}weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 19, \"weight\": 1}\"adv_tuesday\") OR advt_day_parting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 20, \"weight\": 1}\"adv_tuesday_17\") OR advt_day_parting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 21, \"weight\": 1}\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 22}\"all\")) AND isAppReengagementAd = ({\"id\": 23}0) AND dummyField contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 24}\"default\") AND serveWithPromotionOnly = ({\"id\": 26}0) AND budgetAdvertiserThrottleRateFilter = ({\"id\": 27}0) AND budgetResellerThrottleRateFilter = ({\"id\": 28}0) AND (isMystiqueRequired = ({\"id\": 29}0) OR (isMystiqueRequired = ({\"id\": 30}1) AND useBcFactorFilter = ({\"id\": 31}1))) AND (((budgetCampaignThrottleRateBits = ({\"id\": 32}55) AND dummyField contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 33}\"default\"))) AND !(useBcFactorFilter = ({\"id\": 34}1)) OR ((useBcFactorFilter = ({\"id\": 35}1) AND dummyField contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 36}\"default\") AND (bcFactorTiers = ({\"id\": 38}127) OR bcFactorTiers = ({\"id\": 39}0)) AND ((firstPriceEnforced = ({\"id\": 40}0) AND (secondPriceEnforced = ({\"id\": 41}1) OR isPrivateDeal = ({\"id\": 42}0) OR (dummyField contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 43}\"default\")) AND !(bcActiveTier = ({\"id\": 44}0)))) OR mystiqueCampaignThrottleRateBits = ({\"id\": 45}18)))) AND !(isOutOfDailyBudget = ({\"id\": 37}1))) AND testCreative = ({\"id\": 46}0) AND advt_geo = ({\"id\": 47}2147483647) AND ((adType contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 48}\"strm_video\") AND isPortraitVideo = ({\"id\": 49}0)) OR adType contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 50}\"stream_ad\")) AND ((isCPM = ({\"id\": 51}0) AND isOCPC = ({\"id\": 52}0) AND isECPC = ({\"id\": 53}0) AND ((priceType contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 54}\"cpcv\") AND bid >= ({\"id\": 55}0.005)) OR (priceType contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 56}\"cpv\") AND bid >= ({\"id\": 57}0.01)) OR (priceType contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 58}\"cpc\") AND bid >= ({\"id\": 59}0.05)) OR (objective contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 60}\"promote_content\") AND bid >= ({\"id\": 61}0.01)) OR hasFloorPriceUsd = ({\"id\": 62}1))) OR isECPC = ({\"id\": 63}1) OR (isCPM = ({\"id\": 64}1) AND isOCPM = ({\"id\": 65}0) AND (({\"id\": 66}range(bid, 0.25, Infinity)) OR hasFloorPriceUsd = ({\"id\": 67}1)))) AND start_date <= ({\"id\": 68}1572976776299L) AND end_date >= ({\"id\": 69}1572976776299L))) AND !(isHoldoutAd = ({\"id\": 25}1))) AND !((disclaimerExtensionsTypes contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 70}\"pharma\") OR ({\"id\": 71}weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ({\"id\": 72}1) OR blocked_section_ids contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 73}\"223\") OR blocked_publisher_ids contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 74}\"223\") OR blocked_site_ids contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 75}\"223\"))), {\"id\": 76, \"label\": \"ad_ocpc_max_cpc\"}dotProduct(ocpc_max_cpc, {\"0\": 1}), {\"id\": 77, \"label\": \"ad_ocpc_min_cpc\"}dotProduct(ocpc_min_cpc, {\"0\": 1}), {\"id\": 78, \"label\": \"ad_ocpc_max_alpha\"}dotProduct(ocpc_max_alpha, {\"0\": 1}), {\"id\": 79, \"label\": \"ad_ocpc_min_alpha\"}dotProduct(ocpc_min_alpha, {\"0\": 1}), {\"id\": 80, \"label\": \"ad_ocpc_alpha_0\"}dotProduct(ocpc_alpha_0, {\"0\": 1}), {\"id\": 81, \"label\": \"ad_ocpc_alpha_1\"}dotProduct(ocpc_alpha_1, {\"0\": 1}), (bidAdjustmentDayParting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 82, \"weight\": 1}\"adv_tuesday\") OR bidAdjustmentDayParting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 83, \"weight\": 1}\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 84, \"weight\": 1}\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 85, \"weight\": 1}\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 86, \"weight\": 1}\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ({\"normalizeCase\": false, \"implicitTransforms\": false, \"id\": 87, \"weight\": 1}\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ({\"id\": 88, \"weight\": 1}223), {\"id\": 89, \"label\": \"boostingForBackfill\"}dotProduct(boostingForBackfill, {\"priority\": 1000})) limit 0 timeout 3980 | all(group(adTypeForGrouping) each(group(advertiser_id) max(11) output(count() as(groupingCounter)) each(max(1) each(output(summary())))))"; QueryTree parsed = assertParse("select * from sources * where " + queryTreeYql + ";", - "RANK (+(+(AND filter:videoadscappingtestcpm hasRankRestriction:0 (OR (AND objective:install_app availableExtendedFields:cpiparams) (AND availableExtendedFields:appinstallinfo availableExtendedFields:appmetroplexinfo) (+dummyField:default -objective:install_app)) advt_age:2147483647 advt_gender:all advt_all_segments:2147483647 advt_keywords:all advMobilePlatform:2147483647 advMobileDeviceType:2147483647 advMobileCon:2147483647 advMobileOSVersions:2147483647 advCarrier:2147483647 WEIGHTEDSET advt_supply{[1]:\"site223\",[1]:\"pub223\",[1]:\"all\",[1]:\"sec223\"} (OR advt_day_parting:adv_tuesday!1 advt_day_parting:adv_tuesday_17!1 advt_day_parting:adv_tuesday_17_forty_five!1 advt_day_parting:all) isAppReengagementAd:0 dummyField:default serveWithPromotionOnly:0 budgetAdvertiserThrottleRateFilter:0 budgetResellerThrottleRateFilter:0 (OR isMystiqueRequired:0 (AND isMystiqueRequired:1 useBcFactorFilter:1)) (OR (+(AND budgetCampaignThrottleRateBits:55 dummyField:default) -useBcFactorFilter:1) (+(AND useBcFactorFilter:1 dummyField:default (OR bcFactorTiers:127 bcFactorTiers:0) (OR (AND firstPriceEnforced:0 (OR secondPriceEnforced:1 isPrivateDeal:0 (+dummyField:default -bcActiveTier:0))) mystiqueCampaignThrottleRateBits:18)) -isOutOfDailyBudget:1)) testCreative:0 advt_geo:2147483647 (OR (AND adType:strm_video isPortraitVideo:0) adType:stream_ad) (OR (AND isCPM:0 isOCPC:0 isECPC:0 (OR (AND priceType:cpcv bid:[0.005;]) (AND priceType:cpv bid:[0.01;]) (AND priceType:cpc bid:[0.05;]) (AND objective:promote_content bid:[0.01;]) hasFloorPriceUsd:1)) isECPC:1 (AND isCPM:1 isOCPM:0 (OR bid:[0.25;] hasFloorPriceUsd:1))) start_date:[;1572976776299] end_date:[1572976776299;]) -isHoldoutAd:1) -(OR disclaimerExtensionsTypes:pharma WEIGHTEDSET exclusion_advt_supply{[1]:\"extsite223\",[1]:\"site223\",[1]:\"pub223\",[1]:\"sec223\"} isPersonalized:1 blocked_section_ids:223 blocked_publisher_ids:223 blocked_site_ids:223)) DOTPRODUCT ocpc_max_cpc{[1]:\"0\"} DOTPRODUCT ocpc_min_cpc{[1]:\"0\"} DOTPRODUCT ocpc_max_alpha{[1]:\"0\"} DOTPRODUCT ocpc_min_alpha{[1]:\"0\"} DOTPRODUCT ocpc_alpha_0{[1]:\"0\"} DOTPRODUCT ocpc_alpha_1{[1]:\"0\"} (OR bidAdjustmentDayParting:adv_tuesday!1 bidAdjustmentDayParting:adv_tuesday_17!1 bidAdjustmentDayParting:adv_tuesday_17_forty_five!1 bidAdjustmentDayPartingForCostCap:adv_tuesday!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17_forty_five!1) bidAdjustmentForCpi:223!1 DOTPRODUCT boostingForBackfill{[1000]:\"priority\"}"); + "RANK (+(+(AND filter:VideoAdsCappingTestCPM hasRankRestriction:0 (OR (AND objective:install_app availableExtendedFields:cpiparams) (AND availableExtendedFields:appinstallinfo availableExtendedFields:appmetroplexinfo) (+dummyField:default -objective:install_app)) advt_age:2147483647 advt_gender:all advt_all_segments:2147483647 advt_keywords:all advMobilePlatform:2147483647 advMobileDeviceType:2147483647 advMobileCon:2147483647 advMobileOSVersions:2147483647 advCarrier:2147483647 WEIGHTEDSET advt_supply{[1]:\"site223\",[1]:\"pub223\",[1]:\"all\",[1]:\"sec223\"} (OR advt_day_parting:adv_tuesday!1 advt_day_parting:adv_tuesday_17!1 advt_day_parting:adv_tuesday_17_forty_five!1 advt_day_parting:all) isAppReengagementAd:0 dummyField:default serveWithPromotionOnly:0 budgetAdvertiserThrottleRateFilter:0 budgetResellerThrottleRateFilter:0 (OR isMystiqueRequired:0 (AND isMystiqueRequired:1 useBcFactorFilter:1)) (OR (+(AND budgetCampaignThrottleRateBits:55 dummyField:default) -useBcFactorFilter:1) (+(AND useBcFactorFilter:1 dummyField:default (OR bcFactorTiers:127 bcFactorTiers:0) (OR (AND firstPriceEnforced:0 (OR secondPriceEnforced:1 isPrivateDeal:0 (+dummyField:default -bcActiveTier:0))) mystiqueCampaignThrottleRateBits:18)) -isOutOfDailyBudget:1)) testCreative:0 advt_geo:2147483647 (OR (AND adType:strm_video isPortraitVideo:0) adType:stream_ad) (OR (AND isCPM:0 isOCPC:0 isECPC:0 (OR (AND priceType:cpcv bid:[0.005;]) (AND priceType:cpv bid:[0.01;]) (AND priceType:cpc bid:[0.05;]) (AND objective:promote_content bid:[0.01;]) hasFloorPriceUsd:1)) isECPC:1 (AND isCPM:1 isOCPM:0 (OR bid:[0.25;] hasFloorPriceUsd:1))) start_date:[;1572976776299] end_date:[1572976776299;]) -isHoldoutAd:1) -(OR disclaimerExtensionsTypes:pharma WEIGHTEDSET exclusion_advt_supply{[1]:\"extsite223\",[1]:\"site223\",[1]:\"pub223\",[1]:\"sec223\"} isPersonalized:1 blocked_section_ids:223 blocked_publisher_ids:223 blocked_site_ids:223)) DOTPRODUCT ocpc_max_cpc{[1]:\"0\"} DOTPRODUCT ocpc_min_cpc{[1]:\"0\"} DOTPRODUCT ocpc_max_alpha{[1]:\"0\"} DOTPRODUCT ocpc_min_alpha{[1]:\"0\"} DOTPRODUCT ocpc_alpha_0{[1]:\"0\"} DOTPRODUCT ocpc_alpha_1{[1]:\"0\"} (OR bidAdjustmentDayParting:adv_tuesday!1 bidAdjustmentDayParting:adv_tuesday_17!1 bidAdjustmentDayParting:adv_tuesday_17_forty_five!1 bidAdjustmentDayPartingForCostCap:adv_tuesday!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17!1 bidAdjustmentDayPartingForCostCap:adv_tuesday_17_forty_five!1) bidAdjustmentForCpi:223!1 DOTPRODUCT boostingForBackfill{[1000]:\"priority\"}"); String serializedQueryTreeYql = VespaSerializer.serialize(parsed); // Note: All the details here are not verified - assertEquals("rank((((filter contains ({origin: {original: \"filter:VideoAdsCappingTestCPM\", offset: 7, length: 22}, normalizeCase: false, id: 1}\"videoadscappingtestcpm\") AND hasRankRestriction contains ({normalizeCase: false, implicitTransforms: false, id: 2}\"0\") AND ((objective contains ({normalizeCase: false, implicitTransforms: false, id: 3}\"install_app\") AND availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 4}\"cpiparams\")) OR (availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 5}\"appinstallinfo\") AND availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 6}\"appmetroplexinfo\")) OR (dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 7}\"default\")) AND !(objective contains ({normalizeCase: false, implicitTransforms: false, id: 8}\"install_app\"))) AND advt_age = ({id: 9}2147483647) AND advt_gender contains ({normalizeCase: false, implicitTransforms: false, id: 10}\"all\") AND advt_all_segments = ({id: 11}2147483647) AND advt_keywords contains ({normalizeCase: false, implicitTransforms: false, id: 12}\"all\") AND advMobilePlatform = ({id: 13}2147483647) AND advMobileDeviceType = ({id: 14}2147483647) AND advMobileCon = ({id: 15}2147483647) AND advMobileOSVersions = ({id: 16}2147483647) AND advCarrier = ({id: 17}2147483647) AND ({id: 18}weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 19, weight: 1}\"adv_tuesday\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 20, weight: 1}\"adv_tuesday_17\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 21, weight: 1}\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 22}\"all\")) AND isAppReengagementAd = ({id: 23}0) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 24}\"default\") AND serveWithPromotionOnly = ({id: 26}0) AND budgetAdvertiserThrottleRateFilter = ({id: 27}0) AND budgetResellerThrottleRateFilter = ({id: 28}0) AND (isMystiqueRequired = ({id: 29}0) OR (isMystiqueRequired = ({id: 30}1) AND useBcFactorFilter = ({id: 31}1))) AND (((budgetCampaignThrottleRateBits = ({id: 32}55) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 33}\"default\"))) AND !(useBcFactorFilter = ({id: 34}1)) OR ((useBcFactorFilter = ({id: 35}1) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 36}\"default\") AND (bcFactorTiers = ({id: 38}127) OR bcFactorTiers = ({id: 39}0)) AND ((firstPriceEnforced = ({id: 40}0) AND (secondPriceEnforced = ({id: 41}1) OR isPrivateDeal = ({id: 42}0) OR (dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 43}\"default\")) AND !(bcActiveTier = ({id: 44}0)))) OR mystiqueCampaignThrottleRateBits = ({id: 45}18)))) AND !(isOutOfDailyBudget = ({id: 37}1))) AND testCreative = ({id: 46}0) AND advt_geo = ({id: 47}2147483647) AND ((adType contains ({normalizeCase: false, implicitTransforms: false, id: 48}\"strm_video\") AND isPortraitVideo = ({id: 49}0)) OR adType contains ({normalizeCase: false, implicitTransforms: false, id: 50}\"stream_ad\")) AND ((isCPM = ({id: 51}0) AND isOCPC = ({id: 52}0) AND isECPC = ({id: 53}0) AND ((priceType contains ({normalizeCase: false, implicitTransforms: false, id: 54}\"cpcv\") AND bid >= ({id: 55}0.005)) OR (priceType contains ({normalizeCase: false, implicitTransforms: false, id: 56}\"cpv\") AND bid >= ({id: 57}0.01)) OR (priceType contains ({normalizeCase: false, implicitTransforms: false, id: 58}\"cpc\") AND bid >= ({id: 59}0.05)) OR (objective contains ({normalizeCase: false, implicitTransforms: false, id: 60}\"promote_content\") AND bid >= ({id: 61}0.01)) OR hasFloorPriceUsd = ({id: 62}1))) OR isECPC = ({id: 63}1) OR (isCPM = ({id: 64}1) AND isOCPM = ({id: 65}0) AND ({id: 66}range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ({id: 67}1)))) AND start_date <= ({id: 68}1572976776299L) AND end_date >= ({id: 69}1572976776299L))) AND !(isHoldoutAd = ({id: 25}1))) AND !((disclaimerExtensionsTypes contains ({normalizeCase: false, implicitTransforms: false, id: 70}\"pharma\") OR ({id: 71}weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ({id: 72}1) OR blocked_section_ids contains ({normalizeCase: false, implicitTransforms: false, id: 73}\"223\") OR blocked_publisher_ids contains ({normalizeCase: false, implicitTransforms: false, id: 74}\"223\") OR blocked_site_ids contains ({normalizeCase: false, implicitTransforms: false, id: 75}\"223\"))), ({id: 76, label: \"ad_ocpc_max_cpc\"}dotProduct(ocpc_max_cpc, {\"0\": 1})), ({id: 77, label: \"ad_ocpc_min_cpc\"}dotProduct(ocpc_min_cpc, {\"0\": 1})), ({id: 78, label: \"ad_ocpc_max_alpha\"}dotProduct(ocpc_max_alpha, {\"0\": 1})), ({id: 79, label: \"ad_ocpc_min_alpha\"}dotProduct(ocpc_min_alpha, {\"0\": 1})), ({id: 80, label: \"ad_ocpc_alpha_0\"}dotProduct(ocpc_alpha_0, {\"0\": 1})), ({id: 81, label: \"ad_ocpc_alpha_1\"}dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 82, weight: 1}\"adv_tuesday\") OR bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 83, weight: 1}\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 84, weight: 1}\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 85, weight: 1}\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 86, weight: 1}\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 87, weight: 1}\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ({id: 88, weight: 1}223), ({id: 89, label: \"boostingForBackfill\"}dotProduct(boostingForBackfill, {\"priority\": 1000})))", + assertEquals("rank((((filter contains ({normalizeCase: false, id: 1}\"VideoAdsCappingTestCPM\") AND hasRankRestriction contains ({normalizeCase: false, implicitTransforms: false, id: 2}\"0\") AND ((objective contains ({normalizeCase: false, implicitTransforms: false, id: 3}\"install_app\") AND availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 4}\"cpiparams\")) OR (availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 5}\"appinstallinfo\") AND availableExtendedFields contains ({normalizeCase: false, implicitTransforms: false, id: 6}\"appmetroplexinfo\")) OR (dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 7}\"default\")) AND !(objective contains ({normalizeCase: false, implicitTransforms: false, id: 8}\"install_app\"))) AND advt_age = ({id: 9}2147483647) AND advt_gender contains ({normalizeCase: false, implicitTransforms: false, id: 10}\"all\") AND advt_all_segments = ({id: 11}2147483647) AND advt_keywords contains ({normalizeCase: false, implicitTransforms: false, id: 12}\"all\") AND advMobilePlatform = ({id: 13}2147483647) AND advMobileDeviceType = ({id: 14}2147483647) AND advMobileCon = ({id: 15}2147483647) AND advMobileOSVersions = ({id: 16}2147483647) AND advCarrier = ({id: 17}2147483647) AND ({id: 18}weightedSet(advt_supply, {\"all\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) AND (advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 19, weight: 1}\"adv_tuesday\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 20, weight: 1}\"adv_tuesday_17\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 21, weight: 1}\"adv_tuesday_17_forty_five\") OR advt_day_parting contains ({normalizeCase: false, implicitTransforms: false, id: 22}\"all\")) AND isAppReengagementAd = ({id: 23}0) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 24}\"default\") AND serveWithPromotionOnly = ({id: 26}0) AND budgetAdvertiserThrottleRateFilter = ({id: 27}0) AND budgetResellerThrottleRateFilter = ({id: 28}0) AND (isMystiqueRequired = ({id: 29}0) OR (isMystiqueRequired = ({id: 30}1) AND useBcFactorFilter = ({id: 31}1))) AND (((budgetCampaignThrottleRateBits = ({id: 32}55) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 33}\"default\"))) AND !(useBcFactorFilter = ({id: 34}1)) OR ((useBcFactorFilter = ({id: 35}1) AND dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 36}\"default\") AND (bcFactorTiers = ({id: 38}127) OR bcFactorTiers = ({id: 39}0)) AND ((firstPriceEnforced = ({id: 40}0) AND (secondPriceEnforced = ({id: 41}1) OR isPrivateDeal = ({id: 42}0) OR (dummyField contains ({normalizeCase: false, implicitTransforms: false, id: 43}\"default\")) AND !(bcActiveTier = ({id: 44}0)))) OR mystiqueCampaignThrottleRateBits = ({id: 45}18)))) AND !(isOutOfDailyBudget = ({id: 37}1))) AND testCreative = ({id: 46}0) AND advt_geo = ({id: 47}2147483647) AND ((adType contains ({normalizeCase: false, implicitTransforms: false, id: 48}\"strm_video\") AND isPortraitVideo = ({id: 49}0)) OR adType contains ({normalizeCase: false, implicitTransforms: false, id: 50}\"stream_ad\")) AND ((isCPM = ({id: 51}0) AND isOCPC = ({id: 52}0) AND isECPC = ({id: 53}0) AND ((priceType contains ({normalizeCase: false, implicitTransforms: false, id: 54}\"cpcv\") AND bid >= ({id: 55}0.005)) OR (priceType contains ({normalizeCase: false, implicitTransforms: false, id: 56}\"cpv\") AND bid >= ({id: 57}0.01)) OR (priceType contains ({normalizeCase: false, implicitTransforms: false, id: 58}\"cpc\") AND bid >= ({id: 59}0.05)) OR (objective contains ({normalizeCase: false, implicitTransforms: false, id: 60}\"promote_content\") AND bid >= ({id: 61}0.01)) OR hasFloorPriceUsd = ({id: 62}1))) OR isECPC = ({id: 63}1) OR (isCPM = ({id: 64}1) AND isOCPM = ({id: 65}0) AND ({id: 66}range(bid, 0.25, Infinity) OR hasFloorPriceUsd = ({id: 67}1)))) AND start_date <= ({id: 68}1572976776299L) AND end_date >= ({id: 69}1572976776299L))) AND !(isHoldoutAd = ({id: 25}1))) AND !((disclaimerExtensionsTypes contains ({normalizeCase: false, implicitTransforms: false, id: 70}\"pharma\") OR ({id: 71}weightedSet(exclusion_advt_supply, {\"extsite223\": 1, \"pub223\": 1, \"sec223\": 1, \"site223\": 1})) OR isPersonalized = ({id: 72}1) OR blocked_section_ids contains ({normalizeCase: false, implicitTransforms: false, id: 73}\"223\") OR blocked_publisher_ids contains ({normalizeCase: false, implicitTransforms: false, id: 74}\"223\") OR blocked_site_ids contains ({normalizeCase: false, implicitTransforms: false, id: 75}\"223\"))), ({id: 76, label: \"ad_ocpc_max_cpc\"}dotProduct(ocpc_max_cpc, {\"0\": 1})), ({id: 77, label: \"ad_ocpc_min_cpc\"}dotProduct(ocpc_min_cpc, {\"0\": 1})), ({id: 78, label: \"ad_ocpc_max_alpha\"}dotProduct(ocpc_max_alpha, {\"0\": 1})), ({id: 79, label: \"ad_ocpc_min_alpha\"}dotProduct(ocpc_min_alpha, {\"0\": 1})), ({id: 80, label: \"ad_ocpc_alpha_0\"}dotProduct(ocpc_alpha_0, {\"0\": 1})), ({id: 81, label: \"ad_ocpc_alpha_1\"}dotProduct(ocpc_alpha_1, {\"0\": 1})), (bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 82, weight: 1}\"adv_tuesday\") OR bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 83, weight: 1}\"adv_tuesday_17\") OR bidAdjustmentDayParting contains ({normalizeCase: false, implicitTransforms: false, id: 84, weight: 1}\"adv_tuesday_17_forty_five\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 85, weight: 1}\"adv_tuesday\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 86, weight: 1}\"adv_tuesday_17\") OR bidAdjustmentDayPartingForCostCap contains ({normalizeCase: false, implicitTransforms: false, id: 87, weight: 1}\"adv_tuesday_17_forty_five\")), bidAdjustmentForCpi = ({id: 88, weight: 1}223), ({id: 89, label: \"boostingForBackfill\"}dotProduct(boostingForBackfill, {\"priority\": 1000})))", serializedQueryTreeYql); } @@ -479,14 +479,6 @@ void testStemming() { @Test void testRaw() { - SearchDefinition schema = new SearchDefinition("bar"); - Index stringIndex = new Index("baz"); - stringIndex.setIndex(true); - stringIndex.setString(true); - schema.addIndex(stringIndex); - var indexFacts = new IndexFacts(new IndexModel(schema)); - parser = new YqlParser(new ParserEnvironment().setIndexFacts(indexFacts)); - // Default: Not raw, for comparison Item root = parse("select foo from bar where baz contains (\"yoni jo dima\")").getRoot(); assertEquals("baz:'yoni jo dima'", root.toString()); @@ -961,14 +953,6 @@ void testMultipleAnnotatedOrdering() { @Test void testSegmenting() { - SearchDefinition schema = new SearchDefinition("bar"); - Index stringIndex = new Index("title"); - stringIndex.setIndex(true); - stringIndex.setString(true); - schema.addIndex(stringIndex); - var indexFacts = new IndexFacts(new IndexModel(schema)); - parser = new YqlParser(new ParserEnvironment().setIndexFacts(indexFacts)); - assertParse("select * from bar where title contains 'foo.bar'", "title:'foo bar'"); assertParse("select * from bar where title contains 'foo&123'", "title:'foo 123'"); } @@ -1407,7 +1391,7 @@ private void assertSummaryFields(String yqlQuery, Collection expectedSum private WordItem getRootWord(String yqlQuery) { Item root = parse(yqlQuery).getRoot(); - assertInstanceOf(WordItem.class, root); + assertTrue(root instanceof WordItem); return (WordItem)root; }